def restart_service_and_check(localhost, dut, service, interfaces):
"""
Restart specified service and check platform status
"""
logging.info("Restart the %s service" % service)
dut.command("sudo systemctl restart %s" % service)
logging.info("Wait until all critical services are fully started")
check_critical_services(dut)
logging.info("Wait some time for all the transceivers to be detected")
assert wait_until(300, 20, check_interface_information, dut, interfaces), \
"Not all interface information are detected within 300 seconds"
logging.info("Check transceiver status")
check_transceiver_basic(dut, interfaces)
if dut.facts["asic_type"] in ["mellanox"]:
current_file_dir = os.path.dirname(os.path.realpath(__file__))
sub_folder_dir = os.path.join(current_file_dir, "mellanox")
if sub_folder_dir not in sys.path:
sys.path.append(sub_folder_dir)
from check_hw_mgmt_service import check_hw_management_service
from check_sysfs import check_sysfs
logging.info("Check the hw-management service")
check_hw_management_service(dut)
logging.info("Check sysfs")
check_sysfs(dut)
def test_reload_configuration(testbed_devices, conn_graph_facts):
"""
@summary: This test case is to reload the configuration and check platform status
"""
ans_host = testbed_devices["dut"]
interfaces = conn_graph_facts["device_conn"]
asic_type = ans_host.facts["asic_type"]
logging.info("Reload configuration")
ans_host.command("sudo config reload -y")
logging.info("Wait until all critical services are fully started")
check_critical_services(ans_host)
logging.info("Wait some time for all the transceivers to be detected")
assert wait_until(300, 20, check_interface_information, ans_host, interfaces), \
"Not all transceivers are detected in 300 seconds"
logging.info("Check transceiver status")
check_transceiver_basic(ans_host, interfaces)
if asic_type in ["mellanox"]:
current_file_dir = os.path.dirname(os.path.realpath(__file__))
sub_folder_dir = os.path.join(current_file_dir, "mellanox")
if sub_folder_dir not in sys.path:
sys.path.append(sub_folder_dir)
from check_hw_mgmt_service import check_hw_management_service
from check_sysfs import check_sysfs
logging.info("Check the hw-management service")
check_hw_management_service(ans_host)
logging.info("Check sysfs")
check_sysfs(ans_host)
def test_check_sfp_using_ethtool(duthost, conn_graph_facts):
"""This test case is to check SFP using the ethtool.
"""
ports_config = json.loads(
duthost.command("sudo sonic-cfggen -d --var-json PORT")["stdout"])
logging.info("Use the ethtool to check SFP information")
for intf in conn_graph_facts["device_conn"]:
intf_lanes = ports_config[intf]["lanes"]
sfp_id = int(intf_lanes.split(",")[0]) / 4 + 1
ethtool_sfp_output = duthost.command("sudo ethtool -m sfp%s" %
str(sfp_id))
assert ethtool_sfp_output[
"rc"] == 0, "Failed to read eeprom of sfp%s using ethtool" % str(
sfp_id)
assert len(ethtool_sfp_output["stdout_lines"]) >= 5, \
"Does the ethtool output look normal? " + str(ethtool_sfp_output["stdout_lines"])
for line in ethtool_sfp_output["stdout_lines"]:
assert len(line.split(":")) >= 2, \
"Unexpected line %s in %s" % (line, str(ethtool_sfp_output["stdout_lines"]))
logging.info("Check interface status")
mg_facts = duthost.minigraph_facts(host=duthost.hostname)["ansible_facts"]
intf_facts = duthost.interface_facts(
up_ports=mg_facts["minigraph_ports"])["ansible_facts"]
assert len(intf_facts["ansible_interface_link_down_ports"]) == 0, \
"Some interfaces are down: %s" % str(intf_facts["ansible_interface_link_down_ports"])
check_hw_management_service(duthost)
def reboot_and_check(localhost, dut, interfaces, reboot_type="cold"):
"""
Perform the specified type of reboot and check platform status.
"""
logging.info("Run %s reboot on DUT" % reboot_type)
if reboot_type == "cold":
reboot_cmd = "reboot"
reboot_timeout = 300
elif reboot_type == "fast":
reboot_cmd = "fast-reboot"
reboot_timeout = 180
elif reboot_type == "warm":
reboot_cmd = "warm-reboot"
reboot_timeout = 180
else:
assert False, "Reboot type %s is not supported" % reboot_type
process, queue = dut.command(reboot_cmd, module_async=True)
logging.info("Wait for DUT to go down")
res = localhost.wait_for(host=dut.hostname, port=22, state="stopped", delay=10, timeout=120,
module_ignore_errors=True)
if "failed" in res:
if process.is_alive():
logging.error("Command '%s' is not completed" % reboot_cmd)
process.terminate()
logging.error("reboot result %s" % str(queue.get()))
assert False, "DUT did not go down"
logging.info("Wait for DUT to come back")
localhost.wait_for(host=dut.hostname, port=22, state="started", delay=10, timeout=reboot_timeout)
logging.info("Wait until all critical services are fully started")
check_critical_services(dut)
logging.info("Wait some time for all the transceivers to be detected")
assert wait_until(300, 20, all_transceivers_detected, dut, interfaces), \
"Not all transceivers are detected in 300 seconds"
logging.info("Check interface status")
check_interface_status(dut, interfaces)
logging.info("Check transceiver status")
check_transceiver_basic(dut, interfaces)
if dut.facts["asic_type"] in ["mellanox"]:
current_file_dir = os.path.dirname(os.path.realpath(__file__))
sub_folder_dir = os.path.join(current_file_dir, "mellanox")
if sub_folder_dir not in sys.path:
sys.path.append(sub_folder_dir)
from check_hw_mgmt_service import check_hw_management_service
from check_sysfs import check_sysfs
logging.info("Check the hw-management service")
check_hw_management_service(dut)
logging.info("Check sysfs")
check_sysfs(dut)
def check_interfaces_and_services(dut, interfaces, reboot_type=None):
"""
Perform a further check after reboot-cause, including transceiver status, interface status
@param localhost: The Localhost object.
@param dut: The AnsibleHost object of DUT.
@param interfaces: DUT's interfaces defined by minigraph
"""
logging.info("Wait until all critical services are fully started")
check_critical_services(dut)
if reboot_type is not None:
logging.info("Check reboot cause")
reboot_cause = reboot_ctrl_dict[reboot_type]["cause"]
assert wait_until(MAX_WAIT_TIME_FOR_REBOOT_CAUSE, 20, check_reboot_cause, dut, reboot_cause), \
"got reboot-cause failed after rebooted by %s" % reboot_cause
if reboot_ctrl_dict[reboot_type]["test_reboot_cause_only"]:
logging.info(
"Further checking skipped for %s test which intends to verify reboot-cause only"
% reboot_type)
return
logging.info("Wait %d seconds for all the transceivers to be detected" %
MAX_WAIT_TIME_FOR_INTERFACES)
assert wait_until(MAX_WAIT_TIME_FOR_INTERFACES, 20, check_interface_information, dut, interfaces), \
"Not all transceivers are detected or interfaces are up in %d seconds" % MAX_WAIT_TIME_FOR_INTERFACES
logging.info("Check transceiver status")
check_transceiver_basic(dut, interfaces)
logging.info("Check pmon daemon status")
assert check_pmon_daemon_status(dut), "Not all pmon daemons running."
if dut.facts["asic_type"] in ["mellanox"]:
current_file_dir = os.path.dirname(os.path.realpath(__file__))
sub_folder_dir = os.path.join(current_file_dir, "mellanox")
if sub_folder_dir not in sys.path:
sys.path.append(sub_folder_dir)
from check_hw_mgmt_service import check_hw_management_service
from check_sysfs import check_sysfs
logging.info("Check the hw-management service")
check_hw_management_service(dut)
logging.info("Check sysfs")
check_sysfs(dut)
def test_reload_configuration(localhost, ansible_adhoc, testbed):
"""
@summary: This test case is to reload the configuration and check platform status
"""
hostname = testbed['dut']
ans_host = AnsibleHost(ansible_adhoc, hostname)
ans_host.command("show platform summary")
lab_conn_graph_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), \
"../../ansible/files/lab_connection_graph.xml")
conn_graph_facts = localhost.conn_graph_facts(host=hostname, filename=lab_conn_graph_file).\
contacted['localhost']['ansible_facts']
interfaces = conn_graph_facts["device_conn"]
asic_type = ans_host.shell(
"show platform summary | awk '/ASIC: / {print$2}'")["stdout"].strip()
logging.info("Reload configuration")
ans_host.command("sudo config reload -y")
logging.info("Wait until all critical services are fully started")
check_critical_services(ans_host)
logging.info("Wait some time for all the transceivers to be detected")
assert wait_until(300, 20, all_transceivers_detected, ans_host, interfaces), \
"Not all transceivers are detected in 300 seconds"
logging.info("Check interface status")
time.sleep(60)
check_interface_status(ans_host, interfaces)
logging.info("Check transceiver status")
check_transceiver_basic(ans_host, interfaces)
if asic_type in ["mellanox"]:
current_file_dir = os.path.dirname(os.path.realpath(__file__))
sub_folder_dir = os.path.join(current_file_dir, "mellanox")
if sub_folder_dir not in sys.path:
sys.path.append(sub_folder_dir)
from check_hw_mgmt_service import check_hw_management_service
from check_sysfs import check_sysfs
logging.info("Check the hw-management service")
check_hw_management_service(ans_host)
logging.info("Check sysfs")
check_sysfs(ans_host)
def restart_service_and_check(localhost, dut, service):
"""
Restart specified service and check platform status
"""
dut.command("show platform summary")
lab_conn_graph_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), \
"../../ansible/files/lab_connection_graph.xml")
conn_graph_facts = localhost.conn_graph_facts(host=dut.hostname, filename=lab_conn_graph_file).\
contacted['localhost']['ansible_facts']
interfaces = conn_graph_facts["device_conn"]
asic_type = dut.shell(
"show platform summary | awk '/ASIC: / {print$2}'")["stdout"].strip()
logging.info("Restart the %s service" % service)
dut.command("sudo systemctl restart %s" % service)
logging.info("Wait until all critical services are fully started")
check_critical_services(dut)
logging.info("Wait some time for all the transceivers to be detected")
assert wait_until(300, 20, all_transceivers_detected, dut, interfaces), \
"Not all transceivers are detected in 300 seconds"
logging.info("Check interface status")
time.sleep(60)
check_interface_status(dut, interfaces)
logging.info("Check transceiver status")
check_transceiver_basic(dut, interfaces)
if asic_type in ["mellanox"]:
current_file_dir = os.path.dirname(os.path.realpath(__file__))
sub_folder_dir = os.path.join(current_file_dir, "mellanox")
if sub_folder_dir not in sys.path:
sys.path.append(sub_folder_dir)
from check_hw_mgmt_service import check_hw_management_service
from check_sysfs import check_sysfs
logging.info("Check the hw-management service")
check_hw_management_service(dut)
logging.info("Check sysfs")
check_sysfs(dut)
def test_check_sfp_using_ethtool(duthosts, rand_one_dut_hostname,
conn_graph_facts, tbinfo):
"""This test case is to check SFP using the ethtool.
"""
duthost = duthosts[rand_one_dut_hostname]
ports_config = json.loads(
duthost.command("sudo sonic-cfggen -d --var-json PORT")["stdout"])
logging.info("Use the ethtool to check SFP information")
if duthost.facts["hwsku"] in SPC3_HWSKUS:
lanes_divider = 8
else:
lanes_divider = 4
for intf in conn_graph_facts["device_conn"][duthost.hostname]:
intf_lanes = ports_config[intf]["lanes"]
sfp_id = int(intf_lanes.split(",")[0]) / lanes_divider + 1
ethtool_sfp_output = duthost.command("sudo ethtool -m sfp%s" %
str(sfp_id))
assert ethtool_sfp_output[
"rc"] == 0, "Failed to read eeprom of sfp%s using ethtool" % str(
sfp_id)
# QSFP-DD cable case (currenly ethtool not supporting a full parser)
if len(ethtool_sfp_output["stdout_lines"]) == 1:
assert '0x18' in str(ethtool_sfp_output["stdout_lines"]), \
"Does the ethtool output look normal? " + str(ethtool_sfp_output["stdout_lines"])
else:
assert len(ethtool_sfp_output["stdout_lines"]) >= 5, \
"Does the ethtool output look normal? " + str(ethtool_sfp_output["stdout_lines"])
for line in ethtool_sfp_output["stdout_lines"]:
assert len(line.split(":")) >= 2, \
"Unexpected line %s in %s" % (line, str(ethtool_sfp_output["stdout_lines"]))
logging.info("Check interface status")
mg_facts = duthost.get_extended_minigraph_facts(tbinfo)
intf_facts = duthost.interface_facts(
up_ports=mg_facts["minigraph_ports"])["ansible_facts"]
assert len(intf_facts["ansible_interface_link_down_ports"]) == 0, \
"Some interfaces are down: %s" % str(intf_facts["ansible_interface_link_down_ports"])
check_hw_management_service(duthost)
def test_hw_management_service_status(duthost):
"""This test case is to verify that the hw-management service is running properly
"""
check_hw_management_service(duthost)
def reboot_and_check(localhost,
dut,
interfaces,
reboot_type=REBOOT_TYPE_COLD,
reboot_helper=None,
reboot_kwargs=None):
"""
Perform the specified type of reboot and check platform status.
@param localhost: The Localhost object.
@param dut: The AnsibleHost object of DUT.
@param interfaces: DUT's interfaces defined by minigraph
@param reboot_type: The reboot type, pre-defined const that has name convention of REBOOT_TYPE_XXX.
@param reboot_helper: The helper function used only by power off reboot
@param reboot_kwargs: The argument used by reboot_helper
"""
logging.info("Run %s reboot on DUT" % reboot_type)
assert reboot_type in reboot_ctrl_dict.keys(
), "Unknown reboot type %s" % reboot_type
reboot_timeout = reboot_ctrl_dict[reboot_type]["timeout"]
reboot_cause = reboot_ctrl_dict[reboot_type]["cause"]
dut_datetime = datetime.strptime(
dut.command('date -u +"%Y-%m-%d %H:%M:%S"')["stdout"],
"%Y-%m-%d %H:%M:%S")
if reboot_type == REBOOT_TYPE_POWEROFF:
assert reboot_helper is not None, "A reboot function must be provided for power off reboot"
reboot_helper(reboot_kwargs)
localhost.wait_for(host=dut.hostname,
port=22,
state="stopped",
delay=10,
timeout=120)
else:
reboot_cmd = reboot_ctrl_dict[reboot_type]["command"]
reboot_task, reboot_res = dut.command(reboot_cmd,
module_ignore_errors=True,
module_async=True)
logging.info("Wait for DUT to go down")
res = localhost.wait_for(host=dut.hostname,
port=22,
state="stopped",
timeout=180,
module_ignore_errors=True)
if "failed" in res:
try:
logging.error(
"Wait for switch down failed, try to kill any possible stuck reboot task"
)
pid = dut.command("pgrep -f '%s'" % reboot_cmd)["stdout"]
dut.command("kill -9 %s" % pid)
reboot_task.terminate()
logging.error("Result of command '%s': " +
str(reboot_res.get(timeout=0)))
except Exception as e:
logging.error(
"Exception raised while cleanup reboot task and get result: "
+ repr(e))
logging.info("Wait for DUT to come back")
localhost.wait_for(host=dut.hostname,
port=22,
state="started",
delay=10,
timeout=reboot_timeout)
logging.info("Check the uptime to verify whether reboot was performed")
dut_uptime = datetime.strptime(
dut.command("uptime -s")["stdout"], "%Y-%m-%d %H:%M:%S")
assert float(dut_uptime.strftime("%s")) - float(
dut_datetime.strftime("%s")) > 10, "Device did not reboot"
logging.info("Wait until all critical services are fully started")
check_critical_services(dut)
logging.info("Check reboot cause")
check_reboot_cause(dut, reboot_cause)
if reboot_ctrl_dict[reboot_type]["test_reboot_cause_only"]:
logging.info(
"Further checking skipped for %s test which intends to verify reboot-cause only"
.format(reboot_type))
return
logging.info("Wait some time for all the transceivers to be detected")
assert wait_until(300, 20, check_interface_information, dut, interfaces), \
"Not all transceivers are detected or interfaces are up in 300 seconds"
logging.info("Check transceiver status")
check_transceiver_basic(dut, interfaces)
logging.info("Check pmon daemon status")
assert check_pmon_daemon_status(dut), "Not all pmon daemons running."
if dut.facts["asic_type"] in ["mellanox"]:
current_file_dir = os.path.dirname(os.path.realpath(__file__))
sub_folder_dir = os.path.join(current_file_dir, "mellanox")
if sub_folder_dir not in sys.path:
sys.path.append(sub_folder_dir)
from check_hw_mgmt_service import check_hw_management_service
from check_sysfs import check_sysfs
logging.info("Check the hw-management service")
check_hw_management_service(dut)
logging.info("Check sysfs")
check_sysfs(dut)
def test_hw_management_service_status(duthosts, rand_one_dut_hostname):
"""This test case is to verify that the hw-management service is running properly
"""
duthost = duthosts[rand_one_dut_hostname]
check_hw_management_service(duthost)
def test_hw_management_service_status(testbed_devices):
"""This test case is to verify that the hw-management service is running properly
"""
ans_host = testbed_devices["dut"]
check_hw_management_service(ans_host)
def reboot_and_check(localhost, dut, reboot_type="cold"):
"""
Perform the specified type of reboot and check platform status.
"""
dut.command("show platform summary")
lab_conn_graph_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), \
"../../ansible/files/lab_connection_graph.xml")
conn_graph_facts = localhost.conn_graph_facts(host=dut.hostname, filename=lab_conn_graph_file).\
contacted['localhost']['ansible_facts']
interfaces = conn_graph_facts["device_conn"]
asic_type = dut.shell("show platform summary | awk '/ASIC: / {print$2}'")["stdout"].strip()
logging.info("Run %s reboot on DUT" % reboot_type)
if reboot_type == "cold":
reboot_cmd = "sudo reboot &"
reboot_timeout = 300
elif reboot_type == "fast":
reboot_cmd = "sudo fast-reboot &"
reboot_timeout = 180
elif reboot_type == "warm":
reboot_cmd = "sudo warm-reboot &"
reboot_timeout = 180
else:
assert False, "Reboot type %s is not supported" % reboot_type
dut.shell(reboot_cmd)
logging.info("Wait for DUT to go down")
localhost.wait_for(host=dut.hostname, port=22, state="stopped", delay=10, timeout=120)
logging.info("Wait for DUT to come back")
localhost.wait_for(host=dut.hostname, port=22, state="started", delay=10, timeout=reboot_timeout)
logging.info("Wait until all critical services are fully started")
check_critical_services(dut)
logging.info("Wait some time for all the transceivers to be detected")
assert wait_until(300, 20, all_transceivers_detected, dut, interfaces), \
"Not all transceivers are detected in 300 seconds"
logging.info("Check interface status")
check_interface_status(dut, interfaces)
logging.info("Check transceiver status")
check_transceiver_basic(dut, interfaces)
if asic_type in ["mellanox"]:
current_file_dir = os.path.dirname(os.path.realpath(__file__))
sub_folder_dir = os.path.join(current_file_dir, "mellanox")
if sub_folder_dir not in sys.path:
sys.path.append(sub_folder_dir)
from check_hw_mgmt_service import check_hw_management_service
from check_hw_mgmt_service import wait_until_fan_speed_set_to_default
from check_sysfs import check_sysfs
logging.info("Wait until fan speed is set to default")
wait_until_fan_speed_set_to_default(dut)
logging.info("Check the hw-management service")
check_hw_management_service(dut)
logging.info("Check sysfs")
check_sysfs(dut)
def reboot_and_check(localhost,
dut,
interfaces,
reboot_type=REBOOT_TYPE_COLD,
reboot_helper=None,
reboot_kwargs=None):
"""
Perform the specified type of reboot and check platform status.
@param dut: The AnsibleHost object of DUT.
@param interfaces: DUT's interfaces defined by minigraph
@param reboot_type: The reboot type, pre-defined const that has name convention of REBOOT_TYPE_XXX.
@param reboot_helper: The helper function used only by power off reboot
@param reboot_kwargs: The argument used by reboot_helper
"""
logging.info("Run %s reboot on DUT" % reboot_type)
assert reboot_type in reboot_ctrl_dict.keys(
), "Unknown reboot type %s" % reboot_type
reboot_timeout = reboot_ctrl_dict[reboot_type]["timeout"]
reboot_cause = reboot_ctrl_dict[reboot_type]["cause"]
if reboot_type == REBOOT_TYPE_POWEROFF:
assert reboot_helper is not None, "A reboot function must be provided for power off reboot"
reboot_helper(reboot_kwargs)
localhost.wait_for(host=dut.hostname,
port=22,
state="stopped",
delay=10,
timeout=120)
else:
reboot_cmd = reboot_ctrl_dict[reboot_type]["command"]
process, queue = dut.command(reboot_cmd, module_async=True)
logging.info("Wait for DUT to go down")
res = localhost.wait_for(host=dut.hostname,
port=22,
state="stopped",
delay=10,
timeout=120,
module_ignore_errors=True)
if "failed" in res:
if process.is_alive():
logging.error("Command '%s' is not completed" % reboot_cmd)
process.terminate()
logging.error("reboot result %s" % str(queue.get()))
assert False, "DUT did not go down"
logging.info("Wait for DUT to come back")
localhost.wait_for(host=dut.hostname,
port=22,
state="started",
delay=10,
timeout=reboot_timeout)
logging.info("Wait until all critical services are fully started")
check_critical_services(dut)
logging.info("Check reboot cause")
check_reboot_cause(dut, reboot_cause)
logging.info("Wait some time for all the transceivers to be detected")
assert wait_until(300, 20, check_interface_information, dut, interfaces), \
"Not all transceivers are detected or interfaces are up in 300 seconds"
logging.info("Check transceiver status")
check_transceiver_basic(dut, interfaces)
logging.info("Check pmon daemon status")
assert check_pmon_daemon_status(dut), "Not all pmon daemons running."
if dut.facts["asic_type"] in ["mellanox"]:
current_file_dir = os.path.dirname(os.path.realpath(__file__))
sub_folder_dir = os.path.join(current_file_dir, "mellanox")
if sub_folder_dir not in sys.path:
sys.path.append(sub_folder_dir)
from check_hw_mgmt_service import check_hw_management_service
from check_sysfs import check_sysfs
logging.info("Check the hw-management service")
check_hw_management_service(dut)
logging.info("Check sysfs")
check_sysfs(dut)
