def check_interfaces_and_services(dut, interfaces, xcvr_skip_list, reboot_type=None): """ Perform a further check after reboot-cause, including transceiver status, interface status @param localhost: The Localhost object. @param dut: The AnsibleHost object of DUT. @param interfaces: DUT's interfaces defined by minigraph """ logging.info("Wait until all critical services are fully started") wait_critical_processes(dut) if reboot_type is not None: logging.info("Check reboot cause") assert wait_until(MAX_WAIT_TIME_FOR_REBOOT_CAUSE, 20, check_reboot_cause, dut, reboot_type), \ "got reboot-cause failed after rebooted by %s" % reboot_type if reboot_ctrl_dict[reboot_type]["test_reboot_cause_only"]: logging.info( "Further checking skipped for %s test which intends to verify reboot-cause only" % reboot_type) return if dut.is_supervisor_node(): logging.info("skipping interfaces related check for supervisor") else: logging.info( "Wait {} seconds for all the transceivers to be detected".format( MAX_WAIT_TIME_FOR_INTERFACES)) result = wait_until(MAX_WAIT_TIME_FOR_INTERFACES, 20, check_all_interface_information, dut, interfaces, xcvr_skip_list) assert result, "Not all transceivers are detected or interfaces are up in {} seconds".format( MAX_WAIT_TIME_FOR_INTERFACES) logging.info("Check transceiver status") for asic_index in dut.get_frontend_asic_ids(): # Get the interfaces pertaining to that asic interface_list = get_port_map(dut, asic_index) interfaces_per_asic = { k: v for k, v in interface_list.items() if k in interfaces } check_transceiver_basic(dut, asic_index, interfaces_per_asic, xcvr_skip_list) logging.info("Check pmon daemon status") assert check_pmon_daemon_status(dut), "Not all pmon daemons running." if dut.facts["asic_type"] in ["mellanox"]: from .mellanox.check_hw_mgmt_service import check_hw_management_service from .mellanox.check_sysfs import check_sysfs logging.info("Check the hw-management service") check_hw_management_service(dut) logging.info("Check sysfs") check_sysfs(dut)
def test_config_reload_toggle_reset(duthost, k8scluster): """ Test case to ensure that when DUT is joined to master (disable=false, unsaved) but config is saved with disable=true, DUT resets from master after config reload Saves config with disable=true Joins master, which sets disable=false unsaved Performs config reload Ensures that DUT has reset from the master after config reload, as disable=true was saved Args: duthost: DUT host object k8scluster: shortcut fixture for getting cluster of Kubernetes master hosts """ dut_cmds = ['sudo config kube server disable on', 'sudo config save -y'] duthost.shell_cmds(cmds=dut_cmds) ku.join_master(duthost, k8scluster.vip) config_reload(duthost) wait_critical_processes(duthost) server_connect_exp_status = False server_connect_act_status = ku.check_connected(duthost) server_connect_status_updated = ku.poll_for_status_change(duthost, server_connect_exp_status) pytest_assert(server_connect_status_updated, "Unexpected k8s server connection status after config reload, Expected server connected status: {}, Found server connected status: {}".format(server_connect_exp_status, server_connect_act_status))
def test_power_off_reboot(duthosts, enum_rand_one_per_hwsku_hostname, localhost, conn_graph_facts, xcvr_skip_list, pdu_controller, power_off_delay): """ @summary: This test case is to perform reboot via powercycle and check platform status @param duthost: Fixture for DUT AnsibleHost object @param localhost: Fixture for interacting with localhost through ansible @param conn_graph_facts: Fixture parse and return lab connection graph @param xcvr_skip_list: list of DUT's interfaces for which transeiver checks are skipped @param pdu_controller: The python object of psu controller @param power_off_delay: Pytest parameter. The delay between turning off and on the PSU """ duthost = duthosts[enum_rand_one_per_hwsku_hostname] pdu_ctrl = pdu_controller if pdu_ctrl is None: pytest.skip( "No PSU controller for %s, skip rest of the testing in this case" % duthost.hostname) all_outlets = pdu_ctrl.get_outlet_status() # If PDU supports returning output_watts, making sure that all outlets has power. no_power = [ item for item in all_outlets if int(item.get('output_watts', '1')) == 0 ] pytest_assert(not no_power, "Not all outlets have power output: {}".format(no_power)) # Purpose of this list is to control sequence of turning on PSUs in power off testing. # If there are 2 PSUs, then 3 scenarios would be covered: # 1. Turn off all PSUs, turn on PSU1, then check. # 2. Turn off all PSUs, turn on PSU2, then check. # 3. Turn off all PSUs, turn on one of the PSU, then turn on the other PSU, then check. power_on_seq_list = [] if all_outlets: power_on_seq_list = [[item] for item in all_outlets] power_on_seq_list.append(all_outlets) logging.info("Got all power on sequences {}".format(power_on_seq_list)) poweroff_reboot_kwargs = {"dut": duthost} try: for power_on_seq in power_on_seq_list: poweroff_reboot_kwargs["pdu_ctrl"] = pdu_ctrl poweroff_reboot_kwargs["all_outlets"] = all_outlets poweroff_reboot_kwargs["power_on_seq"] = power_on_seq poweroff_reboot_kwargs["delay_time"] = power_off_delay reboot_and_check(localhost, duthost, conn_graph_facts["device_conn"][duthost.hostname], xcvr_skip_list, REBOOT_TYPE_POWEROFF, _power_off_reboot_helper, poweroff_reboot_kwargs) except Exception as e: logging.debug("Restore power after test failure") for outlet in all_outlets: logging.debug("turning on {}".format(outlet)) pdu_ctrl.turn_on_outlet(outlet) # Sleep 120 for dut to boot up time.sleep(120) wait_critical_processes(duthost) raise e
def test_config_reload_toggle_join(duthost, k8scluster): """ Test case to ensure that when DUT is not joined to the master due to (unsaved) disable=true, but config is saved with disable=false, DUT joins after config reload Saves config with configured VIP and disable=false Sets disable=true without saving config, and ensure that DUT resets from master Performs config reload Ensures that DUT is joined to master after config reload Args: duthost: DUT host object k8scluster: shortcut fixture for getting cluster of Kubernetes master hosts """ dut_cmds = ['sudo config kube server ip {}'.format(k8scluster.vip), 'sudo config kube server disable off', 'sudo config save -y'] duthost.shell_cmds(cmds=dut_cmds) duthost.shell('sudo config kube server disable on') server_connect_exp_status = False server_connect_act_status = ku.check_connected(duthost) server_connect_status_updated = ku.poll_for_status_change(duthost, server_connect_exp_status) pytest_assert(server_connect_status_updated, "Unexpected k8s server connection status after setting disable=true, Expected server connected status: {}, Found server connected status: {}".format(server_connect_exp_status, server_connect_act_status)) config_reload(duthost) wait_critical_processes(duthost) server_connect_exp_status = True server_connect_act_status = ku.check_connected(duthost) server_connect_status_updated = ku.poll_for_status_change(duthost, server_connect_exp_status) pytest_assert(server_connect_status_updated, "Unexpected k8s server connection status after config reload, Expected server connected status: {}, Found server connected status: {}".format(server_connect_exp_status, server_connect_act_status))
def test_reload_configuration(duthost, conn_graph_facts): """ @summary: This test case is to reload the configuration and check platform status """ interfaces = conn_graph_facts["device_conn"] asic_type = duthost.facts["asic_type"] logging.info("Reload configuration") duthost.shell("sudo config reload -y &>/dev/null", executable="/bin/bash") logging.info("Wait until all critical services are fully started") wait_critical_processes(duthost) logging.info("Wait some time for all the transceivers to be detected") assert wait_until(300, 20, check_interface_information, duthost, interfaces), \ "Not all transceivers are detected in 300 seconds" logging.info("Check transceiver status") check_transceiver_basic(duthost, interfaces) if asic_type in ["mellanox"]: from .mellanox.check_hw_mgmt_service import check_hw_management_service from .mellanox.check_sysfs import check_sysfs logging.info("Check the hw-management service") check_hw_management_service(duthost) logging.info("Check sysfs") check_sysfs(duthost)
def restart_service_and_check(localhost, dut, service, interfaces): """ Restart specified service and check platform status """ logging.info("Restart the %s service" % service) dut.command("sudo systemctl restart %s" % service) logging.info("Wait until all critical services are fully started") wait_critical_processes(dut) logging.info("Wait some time for all the transceivers to be detected") pytest_assert( wait_until(300, 20, check_interface_information, dut, interfaces), "Not all interface information are detected within 300 seconds") logging.info("Check transceiver status") check_transceiver_basic(dut, interfaces) if dut.facts["asic_type"] in ["mellanox"]: from .mellanox.check_hw_mgmt_service import check_hw_management_service from .mellanox.check_sysfs import check_sysfs logging.info("Check the hw-management service") check_hw_management_service(dut) logging.info("Check sysfs") check_sysfs(dut) logging.info("Check that critical processes are healthy for 60 seconds") check_critical_processes(dut, 60)
def test_memory_exhaustion(self, duthost, localhost): dut_ip = duthost.mgmt_ip hostname = duthost.hostname dut_datetime = duthost.get_now_time() # Use `tail /dev/zero` to run out of memory completely. Since this command will cause the # DUT reboot, we need to run it in the background (using &) to avoid pytest getting stuck. # We also need to add `nohup` to protect it. cmd = 'nohup tail /dev/zero &' res = duthost.shell(cmd) if not res.is_successful: pytest.fail('DUT {} run command {} failed'.format(hostname, cmd)) # Waiting for SSH connection shutdown pytest_assert( self.check_ssh_state(localhost, dut_ip, SSH_STATE_ABSENT, SSH_SHUTDOWN_TIMEOUT), 'DUT {} did not shutdown'.format(hostname)) # Waiting for SSH connection startup pytest_assert( self.check_ssh_state(localhost, dut_ip, SSH_STATE_STARTED, SSH_STARTUP_TIMEOUT), 'DUT {} did not startup'.format(hostname)) # Wait until all critical processes are healthy. wait_critical_processes(duthost) # Verify DUT uptime is later than the time when the test case started running. dut_uptime = duthost.get_up_time() pytest_assert(dut_uptime > dut_datetime, "Device {} did not reboot".format(hostname))
def test_features_state(duthosts, enum_dut_hostname, localhost): """Checks whether the state of each feature is valid or not. Args: duthosts: Fixture returns a list of Ansible object DuT. enum_dut_hostname: Fixture returns name of DuT. Returns: None. """ duthost = duthosts[enum_dut_hostname] logger.info("Checking the state of each feature in 'CONFIG_DB' ...") if not wait_until(180, FEATURE_STATE_VERIFYING_INTERVAL_SECS, 0, verify_features_state, duthost): logger.warn( "Not all states of features in 'CONFIG_DB' are valid, rebooting DUT {}" .format(duthost.hostname)) reboot(duthost, localhost) # Some services are not ready immeidately after reboot wait_critical_processes(duthost) pytest_assert( wait_until(FEATURE_STATE_VERIFYING_THRESHOLD_SECS, FEATURE_STATE_VERIFYING_INTERVAL_SECS, 0, verify_features_state, duthost), "Not all service states are valid!") logger.info("The states of features in 'CONFIG_DB' are all valid.")
def reload_testbed(duthosts, enum_rand_one_per_hwsku_frontend_hostname): """ Reload dut after test function finished """ duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] yield None logging.info("Reloading config and restarting swss...") config_reload(duthost) wait_critical_processes(duthost)
def test_interface_binding(duthosts, rand_one_dut_hostname, dut_dhcp_relay_data): duthost = duthosts[rand_one_dut_hostname] skip_release(duthost, ["201811", "201911", "202106"]) config_reload(duthost) wait_critical_processes(duthost) wait_until(120, 5, 0, check_interface_status, duthosts, rand_one_dut_hostname) output = duthost.shell("docker exec -it dhcp_relay ss -nlp | grep dhcp6relay")["stdout"].encode("utf-8") logger.info(output) for dhcp_relay in dut_dhcp_relay_data: assert "*:{}".format(dhcp_relay['downlink_vlan_iface']['name']) in output, "{} is not found in {}".format("*:{}".format(dhcp_relay['downlink_vlan_iface']['name']), output)
def reload_testbed_on_failed(request, duthosts, enum_rand_one_per_hwsku_frontend_hostname): """ Reload dut after test function finished """ duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] yield None if request.node.rep_call.failed: # if test case failed, means bgp session down or port channel status not recovered, execute config reload logging.info("Reloading config and restarting swss...") config_reload(duthost) wait_critical_processes(duthost)
def test_interface_binding(duthosts, rand_one_dut_hostname, dut_dhcp_relay_data): duthost = duthosts[rand_one_dut_hostname] skip_release(duthost, ["201811", "201911", "202106"]) if not check_interface_status(duthost): config_reload(duthost) wait_critical_processes(duthost) pytest_assert(wait_until(120, 5, 0, check_interface_status, duthost)) output = duthost.shell("docker exec -it dhcp_relay ss -nlp | grep dhcrelay", module_ignore_errors=True)["stdout"].encode("utf-8") logger.info(output) for dhcp_relay in dut_dhcp_relay_data: assert "{}:67".format(dhcp_relay['downlink_vlan_iface']['name']) in output, "{} is not found in {}".format("{}:67".format(dhcp_relay['downlink_vlan_iface']['name']), output) for iface in dhcp_relay['uplink_interfaces']: assert "{}:67".format(iface) in output, "{} is not found in {}".format("{}:67".format(iface), output)
def test_TSA_B_C_with_no_neighbors(duthost, bgpmon_setup_teardown, nbrhosts, tbinfo): """ Test TSA, TSB, TSC with no neighbors on ASIC0 in case of multi-asic and single-asic. """ bgp_neighbors = {} asic_index = 0 if duthost.is_multi_asic else DEFAULT_ASIC_ID try: routes_4 = parse_rib(duthost, 4) routes_6 = parse_rib(duthost, 6) # Remove the Neighbors for the particular BGP instance bgp_neighbors = remove_bgp_neighbors(duthost, asic_index) # Issue TSA on DUT output = duthost.shell("TSA")['stdout_lines'] # Set the DUT in maintenance state # Verify ASIC0 has no neighbors message. pytest_assert(verify_traffic_shift_per_asic(duthost, output, TS_NO_NEIGHBORS, asic_index), "ASIC is not having no neighbors") # Recover to Normal state duthost.shell("TSB")['stdout_lines'] # Verify DUT is in Normal state, and ASIC0 has no neighbors message. pytest_assert(verify_traffic_shift_per_asic(duthost, output, TS_NO_NEIGHBORS, asic_index), "ASIC is not having no neighbors") # Check the traffic state duthost.shell("TSC")['stdout_lines'] # Verify DUT is in Normal state, and ASIC0 has no neighbors message. pytest_assert(verify_traffic_shift_per_asic(duthost, output, TS_NO_NEIGHBORS, asic_index), "ASIC is not having no neighbors") finally: # Restore BGP neighbors restore_bgp_neighbors(duthost, asic_index, bgp_neighbors) # Recover to Normal state duthost.shell("TSB") wait_critical_processes(duthost) # Wait until bgp sessions are established on DUT pytest_assert(wait_until(100, 10, 0, duthost.check_bgp_session_state, bgp_neighbors.keys()), "Not all BGP sessions are established on DUT") # Wait until all routes are announced to neighbors pytest_assert(wait_until(300, 3, 0, verify_all_routes_announce_to_neighs,duthost, nbrhosts, routes_4, 4), "Not all ipv4 routes are announced to neighbors") pytest_assert(wait_until(300, 3, 0, verify_all_routes_announce_to_neighs,duthost, nbrhosts, routes_6, 6), "Not all ipv6 routes are announced to neighbors")
def restart_service_and_check(localhost, dut, enum_frontend_asic_index, service, interfaces, xcvr_skip_list): """ Restart specified service and check platform status """ logging.info("Restart the %s service on asic %s" % (service, enum_frontend_asic_index)) asichost = dut.asic_instance(enum_frontend_asic_index) service_name = asichost.get_docker_name(service) dut.command("sudo systemctl restart {}".format(service_name)) for container in dut.get_default_critical_services_list(): if is_service_hiting_start_limit(dut, container) is True: logging.info( "{} hits start limit and clear reset-failed flag".format( container)) dut.shell( "sudo systemctl reset-failed {}.service".format(container)) dut.shell("sudo systemctl start {}.service".format(container)) logging.info("Wait until all critical services are fully started") wait_critical_processes(dut) logging.info("Wait some time for all the transceivers to be detected") pytest_assert( wait_until(300, 20, check_interface_information, dut, enum_frontend_asic_index, interfaces, xcvr_skip_list), "Not all interface information are detected within 300 seconds") logging.info("Check transceiver status on asic %s" % enum_frontend_asic_index) check_transceiver_basic(dut, enum_frontend_asic_index, interfaces, xcvr_skip_list) if dut.facts["asic_type"] in ["mellanox"]: from .mellanox.check_hw_mgmt_service import check_hw_management_service from .mellanox.check_sysfs import check_sysfs logging.info("Check the hw-management service") check_hw_management_service(dut) logging.info("Check sysfs") check_sysfs(dut) logging.info("Check that critical processes are healthy for 60 seconds") check_critical_processes(dut, 60)
def test_TSA_B_C_with_no_neighbors(duthost, bgpmon_setup_teardown): """ Test TSA, TSB, TSC with no neighbors on ASIC0 in case of multi-asic and single-asic. """ bgp_neighbors = {} asic_index = 0 if duthost.is_multi_asic else DEFAULT_ASIC_ID try: # Remove the Neighbors for the particular BGP instance bgp_neighbors = remove_bgp_neighbors(duthost, asic_index) # Issue TSA on DUT output = duthost.shell("TSA")['stdout_lines'] # Set the DUT in maintenance state # Verify ASIC0 has no neighbors message. pytest_assert( verify_traffic_shift_per_asic(duthost, output, TS_NO_NEIGHBORS, asic_index), "ASIC is not having no neighbors") # Recover to Normal state duthost.shell("TSB")['stdout_lines'] # Verify DUT is in Normal state, and ASIC0 has no neighbors message. pytest_assert( verify_traffic_shift_per_asic(duthost, output, TS_NO_NEIGHBORS, asic_index), "ASIC is not having no neighbors") # Check the traffic state duthost.shell("TSC")['stdout_lines'] # Verify DUT is in Normal state, and ASIC0 has no neighbors message. pytest_assert( verify_traffic_shift_per_asic(duthost, output, TS_NO_NEIGHBORS, asic_index), "ASIC is not having no neighbors") finally: # Restore BGP neighbors restore_bgp_neighbors(duthost, asic_index, bgp_neighbors) # Recover to Normal state duthost.shell("TSB") wait_critical_processes(duthost)
def test_memory_exhaustion(duthosts, enum_frontend_dut_hostname, localhost): """validate kernel will panic and reboot the DUT when runs out of memory and hits oom event""" duthost = duthosts[enum_frontend_dut_hostname] dut_ip = duthost.mgmt_ip hostname = duthost.hostname dut_datetime = duthost.get_now_time() # Use `tail /dev/zero` to run out of memory completely. Since this command will cause DUT reboot, # we need to run it in the background (using &) to avoid pytest getting stuck. We also need to # add `nohup` to protect it. cmd = 'nohup tail /dev/zero &' res = duthost.shell(cmd) if not res.is_successful: raise Exception('DUT {} run command {} failed'.format(hostname, cmd)) logging.info('waiting for ssh to drop on {}'.format(hostname)) res = localhost.wait_for(host=dut_ip, port=SONIC_SSH_PORT, state='absent', search_regex=SONIC_SSH_REGEX, delay=10, timeout=120, module_ignore_errors=True) pytest_assert(not res.is_failed and 'Timeout' not in res.get('msg', ''), 'DUT {} did not shutdown'.format(hostname)) logging.info('waiting for ssh to startup on {}'.format(hostname)) res = localhost.wait_for(host=dut_ip, port=SONIC_SSH_PORT, state='started', search_regex=SONIC_SSH_REGEX, delay=10, timeout=120, module_ignore_errors=True) pytest_assert(not res.is_failed and 'Timeout' not in res.get('msg', ''), 'DUT {} did not startup'.format(hostname)) # Wait until all critical processes are healthy. wait_critical_processes(duthost) # Verify DUT uptime is later than the time when the test case started running. dut_uptime = duthost.get_up_time() pytest_assert(dut_uptime > dut_datetime, "Device {} did not reboot".format(hostname))
def test_reload_configuration(duthosts, rand_one_dut_hostname, conn_graph_facts, xcvr_skip_list): """ @summary: This test case is to reload the configuration and check platform status """ duthost = duthosts[rand_one_dut_hostname] interfaces = conn_graph_facts["device_conn"][duthost.hostname] asic_type = duthost.facts["asic_type"] if config_force_option_supported(duthost): assert wait_until(300, 20, 0, config_system_checks_passed, duthost) logging.info("Reload configuration") duthost.shell("sudo config reload -y &>/dev/null", executable="/bin/bash") logging.info("Wait until all critical services are fully started") wait_critical_processes(duthost) logging.info("Wait some time for all the transceivers to be detected") assert wait_until(300, 20, 0, check_all_interface_information, duthost, interfaces, xcvr_skip_list), \ "Not all transceivers are detected in 300 seconds" logging.info("Check transceiver status") for asic_index in duthost.get_frontend_asic_ids(): # Get the interfaces pertaining to that asic interface_list = get_port_map(duthost, asic_index) interfaces_per_asic = { k: v for k, v in interface_list.items() if k in interfaces } check_transceiver_basic(duthost, asic_index, interfaces_per_asic, xcvr_skip_list) if asic_type in ["mellanox"]: from .mellanox.check_hw_mgmt_service import check_hw_management_service from .mellanox.check_sysfs import check_sysfs logging.info("Check the hw-management service") check_hw_management_service(duthost) logging.info("Check sysfs") check_sysfs(duthost)
def check_interfaces_and_services(dut, interfaces, reboot_type=None): """ Perform a further check after reboot-cause, including transceiver status, interface status @param localhost: The Localhost object. @param dut: The AnsibleHost object of DUT. @param interfaces: DUT's interfaces defined by minigraph """ logging.info("Wait until all critical services are fully started") wait_critical_processes(dut) if reboot_type is not None: logging.info("Check reboot cause") assert wait_until(MAX_WAIT_TIME_FOR_REBOOT_CAUSE, 20, check_reboot_cause, dut, reboot_type), \ "got reboot-cause failed after rebooted by %s" % reboot_type if reboot_ctrl_dict[reboot_type]["test_reboot_cause_only"]: logging.info( "Further checking skipped for %s test which intends to verify reboot-cause only" % reboot_type) return logging.info("Wait %d seconds for all the transceivers to be detected" % MAX_WAIT_TIME_FOR_INTERFACES) assert wait_until(MAX_WAIT_TIME_FOR_INTERFACES, 20, check_interface_information, dut, interfaces), \ "Not all transceivers are detected or interfaces are up in %d seconds" % MAX_WAIT_TIME_FOR_INTERFACES logging.info("Check transceiver status") check_transceiver_basic(dut, interfaces) logging.info("Check pmon daemon status") assert check_pmon_daemon_status(dut), "Not all pmon daemons running." if dut.facts["asic_type"] in ["mellanox"]: from .mellanox.check_hw_mgmt_service import check_hw_management_service from .mellanox.check_sysfs import check_sysfs logging.info("Check the hw-management service") check_hw_management_service(dut) logging.info("Check sysfs") check_sysfs(dut)
def teardown(self, duthost, localhost, pdu_controller): yield # If the SSH connection is not established, or any critical process is exited, # try to recover the DUT by PDU reboot. dut_ip = duthost.mgmt_ip hostname = duthost.hostname if not self.check_ssh_state(localhost, dut_ip, SSH_STATE_STARTED): if pdu_controller is None: logging.error( "No PDU controller for {}, failed to recover DUT!".format( hostname)) return self.pdu_reboot(pdu_controller) # Waiting for SSH connection startup pytest_assert( self.check_ssh_state(localhost, dut_ip, SSH_STATE_STARTED, SSH_STARTUP_TIMEOUT), 'Recover {} by PDU reboot failed'.format(hostname)) # Wait until all critical processes are healthy. wait_critical_processes(duthost)
def test_config_reload_no_toggle(duthost, k8scluster): """ Test case to ensure that when DUT starts as joined to master, and config is saved with disable=false, DUT is still joined to master after config reload Joins master Performs config reload Ensures that DUT is still joined to master after config reload Args: duthost: DUT host object k8scluster: shortcut fixture for getting cluster of Kubernetes master hosts """ ku.join_master(duthost, k8scluster.vip) # Assertion within to ensure successful join duthost.shell('sudo config save -y') config_reload(duthost) wait_critical_processes(duthost) server_connect_exp_status = True server_connect_act_status = ku.check_connected(duthost) server_connect_status_updated = ku.poll_for_status_change(duthost, server_connect_exp_status) pytest_assert(server_connect_status_updated, "Unexpected k8s server connection status after config reload, Expected server connected status: {}, Found server connected status: {}".format(server_connect_exp_status, server_connect_act_status))