def reboot_dut(dut, localhost, cmd, wait_time):
logger.info("Reboot dut using cmd='%s'" % cmd)
reboot_task, reboot_res = dut.command(cmd, module_async=True)
logger.info("Wait for DUT to go down")
try:
localhost.wait_for(host=dut.hostname,
port=22,
state="stopped",
delay=10,
timeout=300)
except RunAnsibleModuleFail as e:
logger.error("DUT did not go down, exception: " + repr(e))
if reboot_task.is_alive():
logger.error("Rebooting is not completed")
reboot_task.terminate()
logger.error("reboot result %s" % str(reboot_res.get()))
assert False, "Failed to reboot the DUT"
localhost.wait_for(host=dut.hostname,
port=22,
state="started",
delay=10,
timeout=300)
wait(wait_time,
msg="Wait {} seconds for system to be stable.".format(wait_time))
def test_config_interface_ip(self, setup_config_mode, sample_intf):
"""
Checks whether 'config interface ip add/remove <intf> <ip>'
adds/removes the ip on the test interface when its interface
alias/name is provided as per the configured naming mode
"""
dutHostGuest, mode, ifmode = setup_config_mode
test_intf = sample_intf[mode]
out = dutHostGuest.shell('SONIC_CLI_IFACE_MODE={} sudo config interface ip remove {} 10.0.0.0/31'.format(ifmode, test_intf))
if out['rc'] != 0:
pytest.fail()
wait(3)
show_ip_intf = dutHostGuest.shell('SONIC_CLI_IFACE_MODE={} show ip interface'.format(ifmode))['stdout']
logger.info('show_ip_intf:\n{}'.format(show_ip_intf))
assert re.search(r'{}\s+10.0.0.0/31'.format(test_intf), show_ip_intf) is None
out = dutHostGuest.shell('SONIC_CLI_IFACE_MODE={} sudo config interface ip add {} 10.0.0.0/31'.format(ifmode, test_intf))
if out['rc'] != 0:
pytest.fail()
wait(3)
show_ip_intf = dutHostGuest.shell('SONIC_CLI_IFACE_MODE={} show ip interface'.format(ifmode))['stdout']
logger.info('show_ip_intf:\n{}'.format(show_ip_intf))
assert re.search(r'{}\s+10.0.0.0/31'.format(test_intf), show_ip_intf) is not None
def check_services(dut):
logger.info("Checking services status...")
networking_uptime = dut.get_networking_uptime().seconds
timeout = max((SYSTEM_STABILIZE_MAX_TIME - networking_uptime), 0)
interval = 20
logger.info("networking_uptime=%d seconds, timeout=%d seconds, interval=%d seconds" % \
(networking_uptime, timeout, interval))
check_result = {"failed": True, "check_item": "services"}
if timeout == 0: # Check services status, do not retry.
services_status = dut.critical_services_status()
check_result["failed"] = False if all(
services_status.values()) else True
check_result["services_status"] = services_status
else: # Retry checking service status
start = time.time()
elapsed = 0
while elapsed < timeout:
services_status = dut.critical_services_status()
check_result["failed"] = False if all(
services_status.values()) else True
check_result["services_status"] = services_status
if check_result["failed"]:
wait(interval, msg="Not all services are started, wait %d seconds to retry. Remaining time: %d %s" % \
(interval, int(timeout - elapsed), str(check_result["services_status"])))
elapsed = time.time() - start
else:
break
logger.info("Done checking services status.")
return check_result
def __recover_interfaces(dut, fanouthosts, result, wait_time):
for port in result['down_ports']:
logging.info("Restoring port {}".format(port))
fanout, fanout_port = fanout_switch_port_lookup(fanouthosts, port)
if fanout and fanout_port:
fanout.no_shutdown(fanout_port)
dut.no_shutdown(port)
wait(wait_time,
msg="Wait {} seconds for interface(s) to restore.".format(wait_time))
def recover(dut, localhost, recover_method):
logger.info("Try to recover %s using method %s" %
(dut.hostname, recover_method))
if constants.RECOVER_METHODS[recover_method]["reboot"]:
reboot_dut(dut, localhost,
constants.RECOVER_METHODS[recover_method]["cmd"])
else:
dut.command(constants.RECOVER_METHODS[recover_method]["cmd"])
wait(30, msg="Wait 30 seconds for system to be stable.")
def __recover_interfaces(dut, fanouthosts, result, wait_time):
action = None
for port in result['down_ports']:
logging.info("Restoring port {}".format(port))
pn = str(port).lower()
if 'portchannel' in pn or 'vlan' in pn:
action = 'config_reload'
continue
fanout, fanout_port = fanout_switch_port_lookup(fanouthosts, port)
if fanout and fanout_port:
fanout.no_shutdown(fanout_port)
dut.no_shutdown(port)
wait(wait_time,
msg="Wait {} seconds for interface(s) to restore.".format(wait_time))
return action
def check_interfaces(dut):
logger.info("Checking interfaces status...")
networking_uptime = dut.get_networking_uptime().seconds
timeout = max((SYSTEM_STABILIZE_MAX_TIME - networking_uptime), 0)
interval = 20
logger.info("networking_uptime=%d seconds, timeout=%d seconds, interval=%d seconds" % \
(networking_uptime, timeout, interval))
cfg_facts = dut.config_facts(host=dut.hostname,
source="persistent")['ansible_facts']
interfaces = [
k for k, v in cfg_facts["PORT"].items()
if "admin_status" in v and v["admin_status"] == "up"
]
if "PORTCHANNEL_INTERFACE" in cfg_facts:
interfaces += cfg_facts["PORTCHANNEL_INTERFACE"].keys()
if "VLAN_INTERFACE" in cfg_facts:
interfaces += cfg_facts["VLAN_INTERFACE"].keys()
logger.info(json.dumps(interfaces, indent=4))
check_result = {"failed": True, "check_item": "interfaces"}
if timeout == 0: # Check interfaces status, do not retry.
down_ports = _find_down_ports(dut, interfaces)
check_result["failed"] = True if len(down_ports) > 0 else False
check_result["down_ports"] = down_ports
else: # Retry checking interface status
start = time.time()
elapsed = 0
while elapsed < timeout:
down_ports = _find_down_ports(dut, interfaces)
check_result["failed"] = True if len(down_ports) > 0 else False
check_result["down_ports"] = down_ports
if check_result["failed"]:
wait(interval, msg="Found down ports, wait %d seconds to retry. Remaining time: %d, down_ports=%s" % \
(interval, int(timeout - elapsed), str(check_result["down_ports"])))
elapsed = time.time() - start
else:
break
logger.info("Done checking interfaces status.")
return check_result
def test_config_interface_state(self, setup_config_mode, sample_intf):
"""
Checks whether 'config interface startup/shutdown <intf>'
changes the admin state of the test interface to up/down when
its interface alias/name is provided as per the configured
naming mode
"""
dutHostGuest, mode, ifmode = setup_config_mode
test_intf = sample_intf[mode]
interface = sample_intf['default']
regex_int = re.compile(r'(\S+)\s+[\d,N\/A]+\s+(\w+)\s+(\d+)\s+([\w\/]+)\s+(\w+)\s+(\w+)\s+(\w+)')
out = dutHostGuest.shell('SONIC_CLI_IFACE_MODE={} sudo config interface shutdown {}'.format(ifmode, test_intf))
if out['rc'] != 0:
pytest.fail()
wait(3)
show_intf_status = dutHostGuest.shell('SONIC_CLI_IFACE_MODE={0} show interfaces status {1} | grep -w {1}'.format(ifmode, test_intf))
logger.info('show_intf_status:\n{}'.format(show_intf_status['stdout']))
line = show_intf_status['stdout'].strip()
if regex_int.match(line) and interface == regex_int.match(line).group(1):
admin_state = regex_int.match(line).group(7)
assert admin_state == 'down'
out = dutHostGuest.shell('SONIC_CLI_IFACE_MODE={} sudo config interface startup {}'.format(ifmode, test_intf))
if out['rc'] != 0:
pytest.fail()
wait(3)
show_intf_status = dutHostGuest.shell('SONIC_CLI_IFACE_MODE={0} show interfaces status {1} | grep -w {1}'.format(ifmode, test_intf))
logger.info('show_intf_status:\n{}'.format(show_intf_status['stdout']))
line = show_intf_status['stdout'].strip()
if regex_int.match(line) and interface == regex_int.match(line).group(1):
admin_state = regex_int.match(line).group(7)
assert admin_state == 'up'
def test_portstat_clear(duthost, command):
before_portstat = parse_portstat(
duthost.command('portstat')['stdout_lines'])
pytest_assert(before_portstat, 'No parsed command output')
duthost.command(command)
wait(5, 'Wait for portstat counters to refresh')
after_portstat = parse_portstat(
duthost.command('portstat')['stdout_lines'])
pytest_assert(after_portstat, 'No parsed command output')
for intf in before_portstat:
pytest_assert(
int(before_portstat[intf]['rx_ok']) >= int(
after_portstat[intf]['rx_ok']),
'Value of RX_OK after clear should be lesser')
pytest_assert(
int(before_portstat[intf]['tx_ok']) >= int(
after_portstat[intf]['rx_ok']),
'Value of RX_OK after clear should be lesser')
def __recover_with_command(dut, cmd, wait_time):
dut.command(cmd)
wait(wait_time,
msg="Wait {} seconds for system to be stable.".format(wait_time))
