def check_interfaces_and_services(dut,
interfaces,
xcvr_skip_list,
reboot_type=None):
"""
Perform a further check after reboot-cause, including transceiver status, interface status
@param localhost: The Localhost object.
@param dut: The AnsibleHost object of DUT.
@param interfaces: DUT's interfaces defined by minigraph
"""
logging.info("Wait until all critical services are fully started")
wait_critical_processes(dut)
if reboot_type is not None:
logging.info("Check reboot cause")
assert wait_until(MAX_WAIT_TIME_FOR_REBOOT_CAUSE, 20, check_reboot_cause, dut, reboot_type), \
"got reboot-cause failed after rebooted by %s" % reboot_type
if reboot_ctrl_dict[reboot_type]["test_reboot_cause_only"]:
logging.info(
"Further checking skipped for %s test which intends to verify reboot-cause only"
% reboot_type)
return
if dut.is_supervisor_node():
logging.info("skipping interfaces related check for supervisor")
else:
logging.info(
"Wait {} seconds for all the transceivers to be detected".format(
MAX_WAIT_TIME_FOR_INTERFACES))
result = wait_until(MAX_WAIT_TIME_FOR_INTERFACES, 20,
check_all_interface_information, dut, interfaces,
xcvr_skip_list)
assert result, "Not all transceivers are detected or interfaces are up in {} seconds".format(
MAX_WAIT_TIME_FOR_INTERFACES)
logging.info("Check transceiver status")
for asic_index in dut.get_frontend_asic_ids():
# Get the interfaces pertaining to that asic
interface_list = get_port_map(dut, asic_index)
interfaces_per_asic = {
k: v
for k, v in interface_list.items() if k in interfaces
}
check_transceiver_basic(dut, asic_index, interfaces_per_asic,
xcvr_skip_list)
logging.info("Check pmon daemon status")
assert check_pmon_daemon_status(dut), "Not all pmon daemons running."
if dut.facts["asic_type"] in ["mellanox"]:
from .mellanox.check_hw_mgmt_service import check_hw_management_service
from .mellanox.check_sysfs import check_sysfs
logging.info("Check the hw-management service")
check_hw_management_service(dut)
logging.info("Check sysfs")
check_sysfs(dut)
def test_xcvr_info_in_db(duthosts, enum_rand_one_per_hwsku_frontend_hostname,
enum_frontend_asic_index, conn_graph_facts,
xcvr_skip_list):
"""
@summary: This test case is to verify that xcvrd works as expected by checking transceiver information in DB
"""
duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname]
logging.info("Check transceiver status")
all_interfaces = conn_graph_facts["device_conn"][duthost.hostname]
if enum_frontend_asic_index is not None:
# Get the interface pertaining to that asic
interface_list = get_port_map(duthost, enum_frontend_asic_index)
# Check if the interfaces of this AISC is present in conn_graph_facts
all_interfaces = {
k: v
for k, v in interface_list.items()
if k in conn_graph_facts["device_conn"][duthost.hostname]
}
logging.info("ASIC {} interface_list {}".format(
enum_frontend_asic_index, all_interfaces))
check_transceiver_status(duthost, enum_frontend_asic_index, all_interfaces,
xcvr_skip_list)
def get_dev_conn(duthost, conn_graph_facts, asic_index):
dev_conn = conn_graph_facts["device_conn"][duthost.hostname]
# Get the interface pertaining to that asic
portmap = get_port_map(duthost, asic_index)
logging.info("Got portmap {}".format(portmap))
if asic_index is not None:
# Check if the interfaces of this AISC is present in conn_graph_facts
dev_conn = {
k: v
for k, v in portmap.items()
if k in conn_graph_facts["device_conn"][duthost.hostname]
}
logging.info("ASIC {} interface_list {}".format(asic_index, dev_conn))
return portmap, dev_conn
def test_restart_swss(duthosts, rand_one_dut_hostname, enum_frontend_asic_index, localhost, conn_graph_facts, xcvr_skip_list):
"""
@summary: This test case is to restart the swss service and check platform status
"""
duthost = duthosts[rand_one_dut_hostname]
all_interfaces = conn_graph_facts["device_conn"][duthost.hostname]
if enum_frontend_asic_index is not None:
# Get the interface pertaining to that asic
interface_list = get_port_map(duthost, enum_frontend_asic_index)
# Check if the interfaces of this AISC is present in conn_graph_facts
new_intf_dict = {k:v for k, v in interface_list.items() if k in all_interfaces}
all_interfaces = new_intf_dict
logging.info("ASIC {} interface_list {}".format(enum_frontend_asic_index, all_interfaces))
restart_service_and_check(localhost, duthost, enum_frontend_asic_index, "swss", all_interfaces, xcvr_skip_list)
def test_reload_configuration(duthosts, rand_one_dut_hostname,
conn_graph_facts, xcvr_skip_list):
"""
@summary: This test case is to reload the configuration and check platform status
"""
duthost = duthosts[rand_one_dut_hostname]
interfaces = conn_graph_facts["device_conn"][duthost.hostname]
asic_type = duthost.facts["asic_type"]
if config_force_option_supported(duthost):
assert wait_until(300, 20, 0, config_system_checks_passed, duthost)
logging.info("Reload configuration")
duthost.shell("sudo config reload -y &>/dev/null", executable="/bin/bash")
logging.info("Wait until all critical services are fully started")
wait_critical_processes(duthost)
logging.info("Wait some time for all the transceivers to be detected")
assert wait_until(300, 20, 0, check_all_interface_information, duthost, interfaces, xcvr_skip_list), \
"Not all transceivers are detected in 300 seconds"
logging.info("Check transceiver status")
for asic_index in duthost.get_frontend_asic_ids():
# Get the interfaces pertaining to that asic
interface_list = get_port_map(duthost, asic_index)
interfaces_per_asic = {
k: v
for k, v in interface_list.items() if k in interfaces
}
check_transceiver_basic(duthost, asic_index, interfaces_per_asic,
xcvr_skip_list)
if asic_type in ["mellanox"]:
from .mellanox.check_hw_mgmt_service import check_hw_management_service
from .mellanox.check_sysfs import check_sysfs
logging.info("Check the hw-management service")
check_hw_management_service(duthost)
logging.info("Check sysfs")
check_sysfs(duthost)
def test_sfps(self, duthosts, enum_rand_one_per_hwsku_hostname, localhost,
platform_api_conn):
duthost = duthosts[enum_rand_one_per_hwsku_hostname]
if duthost.is_supervisor_node():
pytest.skip("skipping for supervisor node")
try:
num_sfps = int(chassis.get_num_sfps(platform_api_conn))
except:
pytest.fail("num_sfps is not an integer")
list_sfps = []
if duthost.facts.get("interfaces"):
intfs = duthost.facts.get("interfaces")
for intf in intfs:
index_list = [
int(x) for x in duthost.facts["interfaces"][intf]
['index'].split(",")
]
list_sfps.extend(set(index_list))
else:
int_list = get_port_map(duthost, 'all')
for k, v in int_list.items():
list_sfps.extend(v)
list_sfps.sort()
if duthost.facts.get("chassis"):
expected_num_sfps = len(duthost.facts.get("chassis").get('sfps'))
pytest_assert(
num_sfps == expected_num_sfps,
"Number of sfps ({}) does not match expected number ({})".
format(num_sfps, expected_num_sfps))
sfp_list = chassis.get_all_sfps(platform_api_conn)
pytest_assert(sfp_list is not None, "Failed to retrieve SFPs")
pytest_assert(
isinstance(sfp_list, list) and len(sfp_list) == num_sfps,
"SFPs appear to be incorrect")
for i in list_sfps:
sfp = chassis.get_sfp(platform_api_conn, i)
self.expect(sfp and sfp == sfp_list[i],
"SFP {} is incorrect".format(i))
self.assert_expectations()
def setup(self, request, duthosts, enum_rand_one_per_hwsku_hostname,
platform_api_conn):
duthost = duthosts[enum_rand_one_per_hwsku_hostname]
if duthost.is_supervisor_node():
pytest.skip("skipping for supervisor node")
self.skip_absent_sfp = request.config.getoption("--skip-absent-sfp")
internal_intf = re.compile(r'^Ethernet-BP|^Ethernet-IB')
self.list_sfps = []
# get expected data from platform.json if not present get from port_config.ini
if duthost.facts.get("interfaces"):
intfs = duthost.facts.get("interfaces")
for intf in intfs:
if re.match(internal_intf, intf):
logging.debug(
"skipping internal interface {}".format(intf))
continue
index_list = [
int(x) for x in duthost.facts["interfaces"][intf]
['index'].split(",")
]
self.list_sfps.extend(set(index_list))
else:
int_list = get_port_map(duthost, 'all')
for k, v in int_list.items():
self.list_sfps.extend(v)
self.list_sfps.sort()
self.candidate_sfp = []
if self.skip_absent_sfp:
# Skip absent SFP if option "--skip-absent-sfp" set to True
for i in self.list_sfps:
try:
if sfp.get_presence(platform_api_conn, i):
self.candidate_sfp.append(i)
except Exception:
pytest.fail(
"get_presence API is not supported for index {}, failed to compose present SFP list"
.format(i))
else:
self.candidate_sfp = self.list_sfps
def test_check_sfp_status_and_configure_sfp(duthosts, rand_one_dut_hostname, enum_frontend_asic_index, conn_graph_facts, tbinfo):
"""
@summary: Check SFP status and configure SFP
This case is to use the sfputil tool and show command to check SFP status and configure SFP. Currently the
only configuration is to reset SFP. Commands to be tested:
* sfputil show presence
* show interface transceiver presence
* sfputil show eeprom
* show interface transceiver eeprom
* sfputil reset <interface name>
"""
duthost = duthosts[rand_one_dut_hostname]
if duthost.facts["asic_type"] in ["mellanox"]:
loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix='sfp_cfg')
loganalyzer.load_common_config()
loganalyzer.ignore_regex.append("kernel.*Eeprom query failed*")
marker = loganalyzer.init()
dev_conn = conn_graph_facts["device_conn"][duthost.hostname]
# Get the interface pertaining to that asic
portmap = get_port_map(duthost, enum_frontend_asic_index)
logging.info("Got portmap {}".format(portmap))
if enum_frontend_asic_index is not None:
# Check if the interfaces of this AISC is present in conn_graph_facts
dev_conn = {k:v for k, v in portmap.items() if k in conn_graph_facts["device_conn"][duthost.hostname]}
logging.info("ASIC {} interface_list {}".format(enum_frontend_asic_index, dev_conn))
cmd_sfp_presence = "sudo sfputil show presence"
cmd_sfp_eeprom = "sudo sfputil show eeprom"
cmd_sfp_reset = "sudo sfputil reset"
cmd_xcvr_presence = "show interface transceiver presence"
cmd_xcvr_eeprom = "show interface transceiver eeprom"
global ans_host
ans_host = duthost
logging.info("Check output of '%s'" % cmd_sfp_presence)
sfp_presence = duthost.command(cmd_sfp_presence)
parsed_presence = parse_output(sfp_presence["stdout_lines"][2:])
for intf in dev_conn:
assert intf in parsed_presence, "Interface is not in output of '%s'" % cmd_sfp_presence
assert parsed_presence[intf] == "Present", "Interface presence is not 'Present'"
logging.info("Check output of '%s'" % cmd_xcvr_presence)
xcvr_presence = duthost.command(cmd_xcvr_presence)
parsed_presence = parse_output(xcvr_presence["stdout_lines"][2:])
for intf in dev_conn:
assert intf in parsed_presence, "Interface is not in output of '%s'" % cmd_xcvr_presence
assert parsed_presence[intf] == "Present", "Interface presence is not 'Present'"
logging.info("Check output of '%s'" % cmd_sfp_eeprom)
sfp_eeprom = duthost.command(cmd_sfp_eeprom)
parsed_eeprom = parse_eeprom(sfp_eeprom["stdout_lines"])
for intf in dev_conn:
assert intf in parsed_eeprom, "Interface is not in output of 'sfputil show eeprom'"
assert parsed_eeprom[intf] == "SFP EEPROM detected"
logging.info("Check output of '%s'" % cmd_xcvr_eeprom)
xcvr_eeprom = duthost.command(cmd_xcvr_eeprom)
parsed_eeprom = parse_eeprom(xcvr_eeprom["stdout_lines"])
for intf in dev_conn:
assert intf in parsed_eeprom, "Interface is not in output of '%s'" % cmd_xcvr_eeprom
assert parsed_eeprom[intf] == "SFP EEPROM detected"
logging.info("Test '%s <interface name>'" % cmd_sfp_reset)
tested_physical_ports = set()
for intf in dev_conn:
phy_intf = portmap[intf][0]
if phy_intf in tested_physical_ports:
logging.info("skip tested SFPs {} to avoid repeating operating physical interface {}".format(intf, phy_intf))
continue
tested_physical_ports.add(phy_intf)
logging.info("resetting {} physical interface {}".format(intf, phy_intf))
reset_result = duthost.command("%s %s" % (cmd_sfp_reset, intf))
assert reset_result["rc"] == 0, "'%s %s' failed" % (cmd_sfp_reset, intf)
time.sleep(5)
logging.info("Wait some time for SFP to fully recover after reset")
time.sleep(60)
logging.info("Check sfp presence again after reset")
sfp_presence = duthost.command(cmd_sfp_presence)
parsed_presence = parse_output(sfp_presence["stdout_lines"][2:])
for intf in dev_conn:
assert intf in parsed_presence, "Interface is not in output of '%s'" % cmd_sfp_presence
assert parsed_presence[intf] == "Present", "Interface presence is not 'Present'"
logging.info("Check interface status")
namespace = duthost.get_namespace_from_asic_id(enum_frontend_asic_index)
mg_facts = duthost.get_extended_minigraph_facts(tbinfo)
# TODO Remove this logic when minigraph facts supports namespace in multi_asic
up_ports = mg_facts["minigraph_ports"]
if enum_frontend_asic_index is not None:
# Check if the interfaces of this AISC is present in conn_graph_facts
up_ports = {k:v for k, v in portmap.items() if k in mg_facts["minigraph_ports"]}
intf_facts = duthost.interface_facts(namespace=namespace, up_ports=up_ports)["ansible_facts"]
assert len(intf_facts["ansible_interface_link_down_ports"]) == 0, \
"Some interfaces are down: %s" % str(intf_facts["ansible_interface_link_down_ports"])
if duthost.facts["asic_type"] in ["mellanox"]:
loganalyzer.analyze(marker)
def test_check_sfp_low_power_mode(duthosts, rand_one_dut_hostname, enum_frontend_asic_index, conn_graph_facts, tbinfo):
"""
@summary: Check SFP low power mode
This case is to use the sfputil tool command to check and set SFP low power mode
* sfputil show lpmode
* sfputil lpmode off
* sfputil lpmode on
"""
duthost = duthosts[rand_one_dut_hostname]
asichost = duthost.get_asic(enum_frontend_asic_index)
if duthost.facts["asic_type"] in ["mellanox"]:
loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix='sfp_lpm')
loganalyzer.load_common_config()
loganalyzer.ignore_regex.append("Eeprom query failed")
marker = loganalyzer.init()
dev_conn = conn_graph_facts["device_conn"][duthost.hostname]
# Get the interface pertaining to that asic
portmap = get_port_map(duthost, enum_frontend_asic_index)
logging.info("Got portmap {}".format(portmap))
if enum_frontend_asic_index is not None:
# Check if the interfaces of this AISC is present in conn_graph_facts
dev_conn = {k:v for k, v in portmap.items() if k in conn_graph_facts["device_conn"][duthost.hostname]}
logging.info("ASIC {} interface_list {}".format(enum_frontend_asic_index, dev_conn))
cmd_sfp_presence = "sudo sfputil show presence"
cmd_sfp_show_lpmode = "sudo sfputil show lpmode"
cmd_sfp_set_lpmode = "sudo sfputil lpmode"
global ans_host
ans_host = duthost
logging.info("Check output of '%s'" % cmd_sfp_show_lpmode)
lpmode_show = duthost.command(cmd_sfp_show_lpmode)
parsed_lpmode = parse_output(lpmode_show["stdout_lines"][2:])
original_lpmode = copy.deepcopy(parsed_lpmode)
for intf in dev_conn:
assert intf in parsed_lpmode, "Interface is not in output of '%s'" % cmd_sfp_show_lpmode
assert parsed_lpmode[intf].lower() == "on" or parsed_lpmode[intf].lower() == "off", "Unexpected SFP lpmode"
logging.info("Try to change SFP lpmode")
tested_physical_ports = set()
not_supporting_lpm_physical_ports = set()
for intf in dev_conn:
phy_intf = portmap[intf][0]
if phy_intf in tested_physical_ports:
logging.info("skip tested SFPs {} to avoid repeating operating physical interface {}".format(intf, phy_intf))
continue
sfp_type_cmd = 'redis-cli -n 6 hget "TRANSCEIVER_INFO|{}" type'.format(intf)
sfp_type_docker_cmd = asichost.get_docker_cmd(sfp_type_cmd, "database")
sfp_type = duthost.command(sfp_type_docker_cmd)["stdout"]
power_class_cmd = 'redis-cli -n 6 hget "TRANSCEIVER_INFO|{}" ext_identifier'.format(intf)
power_class_docker_cmd = asichost.get_docker_cmd(power_class_cmd, "database")
power_class = duthost.command(power_class_docker_cmd)["stdout"]
if not "QSFP" in sfp_type or "Power Class 1" in power_class:
logging.info("skip testing port {} which doesn't support LPM".format(intf))
not_supporting_lpm_physical_ports.add(phy_intf)
continue
tested_physical_ports.add(phy_intf)
logging.info("setting {} physical interface {}".format(intf, phy_intf))
new_lpmode = "off" if original_lpmode[intf].lower() == "on" else "on"
lpmode_set_result = duthost.command("%s %s %s" % (cmd_sfp_set_lpmode, new_lpmode, intf))
assert lpmode_set_result["rc"] == 0, "'%s %s %s' failed" % (cmd_sfp_set_lpmode, new_lpmode, intf)
time.sleep(10)
if len(tested_physical_ports) == 0:
pytest.skip("None of the ports supporting LPM, skip the test")
logging.info("Check SFP lower power mode again after changing SFP lpmode")
lpmode_show = duthost.command(cmd_sfp_show_lpmode)
parsed_lpmode = parse_output(lpmode_show["stdout_lines"][2:])
for intf in dev_conn:
assert intf in parsed_lpmode, "Interface is not in output of '%s'" % cmd_sfp_show_lpmode
assert parsed_lpmode[intf].lower() == "on" or parsed_lpmode[intf].lower() == "off", "Unexpected SFP lpmode"
logging.info("Try to change SFP lpmode")
tested_physical_ports = set()
for intf in dev_conn:
phy_intf = portmap[intf][0]
if phy_intf in not_supporting_lpm_physical_ports:
logging.info("skip testing port {} which doesn't support LPM".format(intf))
continue
if phy_intf in tested_physical_ports:
logging.info("skip tested SFPs {} to avoid repeating operating physical interface {}".format(intf, phy_intf))
continue
tested_physical_ports.add(phy_intf)
logging.info("restoring {} physical interface {}".format(intf, phy_intf))
new_lpmode = original_lpmode[intf].lower()
lpmode_set_result = duthost.command("%s %s %s" % (cmd_sfp_set_lpmode, new_lpmode, intf))
assert lpmode_set_result["rc"] == 0, "'%s %s %s' failed" % (cmd_sfp_set_lpmode, new_lpmode, intf)
time.sleep(10)
logging.info("Check SFP lower power mode again after changing SFP lpmode")
lpmode_show = duthost.command(cmd_sfp_show_lpmode)
parsed_lpmode = parse_output(lpmode_show["stdout_lines"][2:])
for intf in dev_conn:
assert intf in parsed_lpmode, "Interface is not in output of '%s'" % cmd_sfp_show_lpmode
assert parsed_lpmode[intf].lower() == "on" or parsed_lpmode[intf].lower() == "off", "Unexpected SFP lpmode"
logging.info("Check sfp presence again after setting lpmode")
sfp_presence = duthost.command(cmd_sfp_presence)
parsed_presence = parse_output(sfp_presence["stdout_lines"][2:])
for intf in dev_conn:
assert intf in parsed_presence, "Interface is not in output of '%s'" % cmd_sfp_presence
assert parsed_presence[intf] == "Present", "Interface presence is not 'Present'"
logging.info("Check interface status")
namespace = duthost.get_namespace_from_asic_id(enum_frontend_asic_index)
mg_facts = duthost.get_extended_minigraph_facts(tbinfo)
# TODO Remove this logic when minigraph facts supports namespace in multi_asic
up_ports = mg_facts["minigraph_ports"]
if enum_frontend_asic_index is not None:
# Check if the interfaces of this AISC is present in conn_graph_facts
up_ports = {k:v for k, v in portmap.items() if k in mg_facts["minigraph_ports"]}
intf_facts = duthost.interface_facts(namespace=namespace, up_ports=up_ports)["ansible_facts"]
assert len(intf_facts["ansible_interface_link_down_ports"]) == 0, \
"Some interfaces are down: %s" % str(intf_facts["ansible_interface_link_down_ports"])
if duthost.facts["asic_type"] in ["mellanox"]:
loganalyzer.analyze(marker)
