def test_dhcpv6_relay_counter(ptfhost, duthosts, rand_one_dut_hostname, dut_dhcp_relay_data):
""" Test DHCPv6 Counter """
duthost = duthosts[rand_one_dut_hostname]
skip_release(duthost, ["201911", "202106"])
messages = ["Solicit", "Advertise", "Request", "Confirm", "Renew", "Rebind", "Reply", "Release", "Decline", "Relay-Forward", "Relay-Reply"]
for dhcp_relay in dut_dhcp_relay_data:
for message in messages:
cmd = 'sonic-db-cli STATE_DB hmset "DHCPv6_COUNTER_TABLE|{}" {} 0'.format(dhcp_relay['downlink_vlan_iface']['name'], message)
duthost.shell(cmd)
# Send the DHCP relay traffic on the PTF host
ptf_runner(ptfhost,
"ptftests",
"dhcpv6_counter_test.DHCPCounterTest",
platform_dir="ptftests",
params={"hostname": duthost.hostname,
"client_port_index": dhcp_relay['client_iface']['port_idx'],
"leaf_port_indices": repr(dhcp_relay['uplink_port_indices']),
"num_dhcp_servers": len(dhcp_relay['downlink_vlan_iface']['dhcpv6_server_addrs']),
"server_ip": str(dhcp_relay['downlink_vlan_iface']['dhcpv6_server_addrs'][0]),
"relay_iface_ip": str(dhcp_relay['downlink_vlan_iface']['addr']),
"relay_iface_mac": str(dhcp_relay['downlink_vlan_iface']['mac']),
"relay_link_local": str(dhcp_relay['uplink_interface_link_local']),
"vlan_ip": str(dhcp_relay['downlink_vlan_iface']['addr'])},
log_file="/tmp/dhcpv6_relay_test.DHCPCounterTest.log")
for message in messages:
get_message = 'sonic-db-cli STATE_DB hget "DHCPv6_COUNTER_TABLE|{}" {}'.format(dhcp_relay['downlink_vlan_iface']['name'], message)
message_count = duthost.shell(get_message)['stdout']
assert int(message_count) > 0, "Missing {} count".format(message)
def test_check_sfputil_error_status(duthosts,
enum_rand_one_per_hwsku_frontend_hostname,
enum_frontend_asic_index, conn_graph_facts,
cmd_sfp_error_status, xcvr_skip_list):
"""
@summary: Check SFP error status using 'sfputil show error-status' and 'sfputil show error-status --fetch-from-hardware'
This feature is supported on 202106 and above
@param: cmd_sfp_error_status: fixture representing the command used to test
"""
duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname]
skip_release(duthost, ["201811", "201911", "202012"])
portmap, dev_conn = get_dev_conn(duthost, conn_graph_facts,
enum_frontend_asic_index)
logging.info("Check output of '{}'".format(cmd_sfp_error_status))
sfp_error_status = duthost.command(cmd_sfp_error_status)
for line in sfp_error_status["stdout_lines"][2:]:
if "Not implemented" in line:
pytest.skip("Skip test as error status isn't supported")
parsed_presence = parse_output(sfp_error_status["stdout_lines"][2:])
for intf in dev_conn:
if intf not in xcvr_skip_list[duthost.hostname]:
assert intf in parsed_presence, "Interface is not in output of '{}'".format(
cmd_sfp_presence)
assert parsed_presence[
intf] == "OK", "Interface error status is not 'OK'"
def test_pmon_ledd_term_and_start_status(check_daemon_status, duthosts, rand_one_dut_hostname):
"""
@summary: This test case is to check the ledd terminated and restarted status
"""
duthost = duthosts[rand_one_dut_hostname]
skip_release(duthost, ["201811", "201911"])
pre_daemon_status, pre_daemon_pid = duthost.get_pmon_daemon_status(daemon_name)
logger.info("{} daemon is {} with pid {}".format(daemon_name, pre_daemon_status, pre_daemon_pid))
duthost.stop_pmon_daemon(daemon_name, SIG_TERM, pre_daemon_pid)
# Insert 2 seconds delay between termination of the LEDD process by SIG_TERM and status verification,
# to avoid misleading check result which caused by latency of termination process.
time.sleep(2)
# TODO: To arm the wait_until API with a delay parameter, by which to delay specified time
# before invoking the check function.
wait_until(120, 10, check_expected_daemon_status, duthost, expected_running_status)
post_daemon_status, post_daemon_pid = duthost.get_pmon_daemon_status(daemon_name)
pytest_assert(post_daemon_status == expected_running_status,
"{} expected restarted status is {} but is {}".format(daemon_name, expected_running_status, post_daemon_status))
pytest_assert(post_daemon_pid != -1,
"{} expected pid is a positive integer but is {}".format(daemon_name, post_daemon_pid))
pytest_assert(post_daemon_pid > pre_daemon_pid,
"Restarted {} pid should be bigger than {} but it is {}".format(daemon_name, pre_daemon_pid, post_daemon_pid))
def test_get_error_description(self, duthosts,
enum_rand_one_per_hwsku_hostname, localhost,
platform_api_conn):
"""This function tests get_error_description() API (supported on 202106 and above)"""
skip_release(duthosts[enum_rand_one_per_hwsku_hostname],
["201811", "201911", "202012"])
for i in self.sfp_setup["sfp_test_port_indices"]:
error_description = sfp.get_error_description(platform_api_conn, i)
if self.expect(
error_description is not None,
"Unable to retrieve transceiver {} error description".
format(i)):
if "Not implemented" in error_description:
pytest.skip(
"get_error_description isn't implemented. Skip the test"
)
if self.expect(
isinstance(error_description, str)
or isinstance(error_description, unicode),
"Transceiver {} error description appears incorrect".
format(i)):
self.expect(error_description == "OK",
"Transceiver {} is not present".format(i))
self.assert_expectations()
def test_dhcp_relay_random_sport(ptfhost, dut_dhcp_relay_data, validate_dut_routes_exist, testing_config, toggle_all_simulator_ports_to_rand_selected_tor_m):
"""Test DHCP relay functionality on T0 topology with random source port (sport)
If the client is SNAT'd, the source port could be changed to a non-standard port (i.e., not 68).
Verify that DHCP relay works with random high sport.
"""
testing_mode, duthost, testbed_mode = testing_config
if testing_mode == DUAL_TOR_MODE:
skip_release(duthost, ["201811", "201911"])
RANDOM_CLIENT_PORT = random.choice(range(1000, 65535))
for dhcp_relay in dut_dhcp_relay_data:
# Run the DHCP relay test on the PTF host
ptf_runner(ptfhost,
"ptftests",
"dhcp_relay_test.DHCPTest",
platform_dir="ptftests",
params={"hostname": duthost.hostname,
"client_port_index": dhcp_relay['client_iface']['port_idx'],
"client_iface_alias": str(dhcp_relay['client_iface']['alias']),
"leaf_port_indices": repr(dhcp_relay['uplink_port_indices']),
"num_dhcp_servers": len(dhcp_relay['downlink_vlan_iface']['dhcp_server_addrs']),
"server_ip": str(dhcp_relay['downlink_vlan_iface']['dhcp_server_addrs'][0]),
"relay_iface_ip": str(dhcp_relay['downlink_vlan_iface']['addr']),
"relay_iface_mac": str(dhcp_relay['downlink_vlan_iface']['mac']),
"relay_iface_netmask": str(dhcp_relay['downlink_vlan_iface']['mask']),
"dest_mac_address": BROADCAST_MAC,
"client_udp_src_port": RANDOM_CLIENT_PORT,
"switch_loopback_ip": dhcp_relay['switch_loopback_ip'],
"uplink_mac": str(dhcp_relay['uplink_mac']),
"testbed_mode": testbed_mode,
"testing_mode": testing_mode},
log_file="/tmp/dhcp_relay_test.DHCPTest.log")
def test_dhcp_relay_default(ptfhost, dut_dhcp_relay_data, validate_dut_routes_exist, testing_config, toggle_all_simulator_ports_to_rand_selected_tor_m):
"""Test DHCP relay functionality on T0 topology.
For each DHCP relay agent running on the DuT, verify DHCP packets are relayed properly
"""
testing_mode, duthost, testbed_mode = testing_config
if testing_mode == DUAL_TOR_MODE:
skip_release(duthost, ["201811", "201911"])
for dhcp_relay in dut_dhcp_relay_data:
# Run the DHCP relay test on the PTF host
ptf_runner(ptfhost,
"ptftests",
"dhcp_relay_test.DHCPTest",
platform_dir="ptftests",
params={"hostname": duthost.hostname,
"client_port_index": dhcp_relay['client_iface']['port_idx'],
## This port is introduced to test DHCP relay packet received
## on other client port
"other_client_port": repr(dhcp_relay['other_client_ports']),
"client_iface_alias": str(dhcp_relay['client_iface']['alias']),
"leaf_port_indices": repr(dhcp_relay['uplink_port_indices']),
"num_dhcp_servers": len(dhcp_relay['downlink_vlan_iface']['dhcp_server_addrs']),
"server_ip": str(dhcp_relay['downlink_vlan_iface']['dhcp_server_addrs'][0]),
"relay_iface_ip": str(dhcp_relay['downlink_vlan_iface']['addr']),
"relay_iface_mac": str(dhcp_relay['downlink_vlan_iface']['mac']),
"relay_iface_netmask": str(dhcp_relay['downlink_vlan_iface']['mask']),
"dest_mac_address": BROADCAST_MAC,
"client_udp_src_port": DEFAULT_DHCP_CLIENT_PORT,
"switch_loopback_ip": dhcp_relay['switch_loopback_ip'],
"uplink_mac": str(dhcp_relay['uplink_mac']),
"testbed_mode": testbed_mode,
"testing_mode": testing_mode},
log_file="/tmp/dhcp_relay_test.DHCPTest.log")
def test_dhcp_relay_unicast_mac(ptfhost, dut_dhcp_relay_data, validate_dut_routes_exist, testing_config, toggle_all_simulator_ports_to_rand_selected_tor_m):
"""Test DHCP relay functionality on T0 topology with unicast mac
Instead of using broadcast MAC, use unicast MAC of DUT and verify that DHCP relay functionality is entact.
"""
testing_mode, duthost, testbed_mode = testing_config
if testing_mode == DUAL_TOR_MODE:
skip_release(duthost, ["201811", "201911"])
if len(dut_dhcp_relay_data) > 1:
pytest.skip("skip the unicast mac testcase in the multi-Vlan setting")
for dhcp_relay in dut_dhcp_relay_data:
# Run the DHCP relay test on the PTF host
ptf_runner(ptfhost,
"ptftests",
"dhcp_relay_test.DHCPTest",
platform_dir="ptftests",
params={"hostname": duthost.hostname,
"client_port_index": dhcp_relay['client_iface']['port_idx'],
"client_iface_alias": str(dhcp_relay['client_iface']['alias']),
"leaf_port_indices": repr(dhcp_relay['uplink_port_indices']),
"num_dhcp_servers": len(dhcp_relay['downlink_vlan_iface']['dhcp_server_addrs']),
"server_ip": str(dhcp_relay['downlink_vlan_iface']['dhcp_server_addrs'][0]),
"relay_iface_ip": str(dhcp_relay['downlink_vlan_iface']['addr']),
"relay_iface_mac": str(dhcp_relay['downlink_vlan_iface']['mac']),
"relay_iface_netmask": str(dhcp_relay['downlink_vlan_iface']['mask']),
"dest_mac_address": duthost.facts["router_mac"] if testbed_mode != 'dual_testbed' else str(dhcp_relay['downlink_vlan_iface']['mac']),
"client_udp_src_port": DEFAULT_DHCP_CLIENT_PORT,
"switch_loopback_ip": dhcp_relay['switch_loopback_ip'],
"uplink_mac": str(dhcp_relay['uplink_mac']),
"testbed_mode": testbed_mode,
"testing_mode": testing_mode},
log_file="/tmp/dhcp_relay_test.DHCPTest.log")
def test_pmon_pcied_term_and_start_status(check_daemon_status, duthosts, rand_one_dut_hostname, data_before_restart):
"""
@summary: This test case is to check the pcied terminated and restarted status
"""
duthost = duthosts[rand_one_dut_hostname]
skip_release(duthost, ["201811", "201911"])
pre_daemon_status, pre_daemon_pid = duthost.get_pmon_daemon_status(daemon_name)
logger.info("{} daemon is {} with pid {}".format(daemon_name, pre_daemon_status, pre_daemon_pid))
duthost.stop_pmon_daemon(daemon_name, SIG_TERM, pre_daemon_pid)
daemon_status, daemon_pid = duthost.get_pmon_daemon_status(daemon_name)
pytest_assert(daemon_status != expected_running_status and pre_daemon_pid != daemon_pid,
"{} status for SIG_TERM should not be {} with pid:{}!".format(daemon_name, daemon_status, daemon_pid))
time.sleep(10)
post_daemon_status, post_daemon_pid = duthost.get_pmon_daemon_status(daemon_name)
pytest_assert(post_daemon_status == expected_running_status,
"{} expected restarted status is {} but is {}".format(daemon_name, expected_running_status, post_daemon_status))
pytest_assert(post_daemon_pid != -1,
"{} expected pid is -1 but is {}".format(daemon_name, post_daemon_pid))
pytest_assert(post_daemon_pid > pre_daemon_pid,
"Restarted {} pid should be bigger than {} but it is {}".format(daemon_name, pre_daemon_pid, post_daemon_pid))
data_after_restart = wait_data(duthost)
pytest_assert(data_after_restart == data_before_restart, 'DB data present before and after restart does not match')
def check_image_version(duthost):
"""Skips this test if the SONiC image installed on DUT is older than 202112
Args:
duthost: Hostname of DUT.
Returns:
None.
"""
skip_release(duthost, per_command_check_skip_versions)
def check_image_version(duthost):
"""Skips this test if the SONiC image installed on DUT is older than 202112
Args:
duthost: Hostname of DUT.
Returns:
None.
"""
skip_release(duthost, ["201811", "201911", "202012", "202106"])
def get_pcie_devices_tbl_key(duthosts, rand_one_dut_hostname):
duthost = duthosts[rand_one_dut_hostname]
skip_release(duthost, ["201811", "201911"])
command_output = duthost.shell(
"redis-cli -n 6 keys '*' | grep PCIE_DEVICES")
global pcie_devices_status_tbl_key
pcie_devices_status_tbl_key = command_output["stdout"]
def test_dhcp_relay_start_with_uplinks_down(ptfhost, dut_dhcp_relay_data, validate_dut_routes_exist, testing_config):
"""Test DHCP relay functionality on T0 topology when relay agent starts with uplinks down
For each DHCP relay agent running on the DuT, bring the uplinks down, then restart the
relay agent while the uplinks are still down. Then test whether the DHCP relay agent
relays packets properly.
"""
testing_mode, duthost, testbed_mode = testing_config
if testbed_mode == 'dual_testbed':
pytest.skip("skip the uplinks down testcase on dual tor testbeds")
if testing_mode == DUAL_TOR_MODE:
skip_release(duthost, ["201811", "201911"])
for dhcp_relay in dut_dhcp_relay_data:
# Bring all uplink interfaces down
for iface in dhcp_relay['uplink_interfaces']:
duthost.shell('ifconfig {} down'.format(iface))
pytest_assert(wait_until(50, 5, 0, check_link_status, duthost, dhcp_relay['uplink_interfaces'], "down"),
"Not all uplinks go down")
# Restart DHCP relay service on DUT
duthost.shell('systemctl restart dhcp_relay.service')
# Sleep to give the DHCP relay container time to start up and
# allow the relay agent to begin listening on the down interfaces
time.sleep(40)
# Bring all uplink interfaces back up
for iface in dhcp_relay['uplink_interfaces']:
duthost.shell('ifconfig {} up'.format(iface))
# Wait until uplinks are up and routes are recovered
pytest_assert(wait_until(50, 5, 0, check_routes_to_dhcp_server, duthost, dut_dhcp_relay_data),
"Not all DHCP servers are routed")
# Run the DHCP relay test on the PTF host
ptf_runner(ptfhost,
"ptftests",
"dhcp_relay_test.DHCPTest",
platform_dir="ptftests",
params={"hostname": duthost.hostname,
"client_port_index": dhcp_relay['client_iface']['port_idx'],
"client_iface_alias": str(dhcp_relay['client_iface']['alias']),
"leaf_port_indices": repr(dhcp_relay['uplink_port_indices']),
"num_dhcp_servers": len(dhcp_relay['downlink_vlan_iface']['dhcp_server_addrs']),
"server_ip": str(dhcp_relay['downlink_vlan_iface']['dhcp_server_addrs'][0]),
"relay_iface_ip": str(dhcp_relay['downlink_vlan_iface']['addr']),
"relay_iface_mac": str(dhcp_relay['downlink_vlan_iface']['mac']),
"relay_iface_netmask": str(dhcp_relay['downlink_vlan_iface']['mask']),
"dest_mac_address": BROADCAST_MAC,
"client_udp_src_port": DEFAULT_DHCP_CLIENT_PORT,
"switch_loopback_ip": dhcp_relay['switch_loopback_ip'],
"uplink_mac": str(dhcp_relay['uplink_mac']),
"testbed_mode": testbed_mode,
"testing_mode": testing_mode},
log_file="/tmp/dhcp_relay_test.DHCPTest.log")
def test_get_revision(self, duthosts, enum_rand_one_per_hwsku_hostname,
localhost, platform_api_conn):
duthost = duthosts[enum_rand_one_per_hwsku_hostname]
skip_release(duthost, ["201811", "201911", "202012"])
revision = chassis.get_revision(platform_api_conn)
pytest_assert(revision is not None,
"Unable to retrieve chassis revision")
pytest_assert(isinstance(revision, STRING_TYPE),
"Revision appears incorrect")
def check_image_version(duthosts, enum_rand_one_per_hwsku_frontend_hostname):
"""Skips this test because new copp management logic works on 202012 branch and above
Args:
duthost: Hostname of DUT.
Returns:
None.
"""
skip_release(duthosts[enum_rand_one_per_hwsku_frontend_hostname], ["201911"])
def test_interface_binding(duthosts, rand_one_dut_hostname, dut_dhcp_relay_data):
duthost = duthosts[rand_one_dut_hostname]
skip_release(duthost, ["201811", "201911", "202106"])
config_reload(duthost)
wait_critical_processes(duthost)
wait_until(120, 5, 0, check_interface_status, duthosts, rand_one_dut_hostname)
output = duthost.shell("docker exec -it dhcp_relay ss -nlp | grep dhcp6relay")["stdout"].encode("utf-8")
logger.info(output)
for dhcp_relay in dut_dhcp_relay_data:
assert "*:{}".format(dhcp_relay['downlink_vlan_iface']['name']) in output, "{} is not found in {}".format("*:{}".format(dhcp_relay['downlink_vlan_iface']['name']), output)
def testbed_params(duthosts, rand_one_dut_hostname, tbinfo):
duthost = duthosts[rand_one_dut_hostname]
skip_release(duthost, ["201811", "201911"])
mg_facts = duthost.get_extended_minigraph_facts(tbinfo)
vlan_intf_name = mg_facts["minigraph_vlans"].keys()[0]
vlan_member_ports = mg_facts["minigraph_vlans"][vlan_intf_name]["members"]
vlan_member_ports_to_ptf_ports = {_: mg_facts["minigraph_ptf_indices"][_] for _ in vlan_member_ports}
vlan_intf = [_ for _ in mg_facts["minigraph_vlan_interfaces"] if _["attachto"] == vlan_intf_name and is_ipv4_address(_["addr"])][0]
return vlan_intf, vlan_member_ports_to_ptf_ports
def test_dhcp_relay_after_link_flap(ptfhost, duthosts, rand_one_dut_hostname,
dut_dhcp_relay_data,
validate_dut_routes_exist):
"""Test DHCP relay functionality on T0 topology after uplinks flap
For each DHCP relay agent running on the DuT, with relay agent running, flap the uplinks,
then test whether the DHCP relay agent relays packets properly.
"""
duthost = duthosts[rand_one_dut_hostname]
skip_release(duthost, ["201811", "201911", "202106"])
for dhcp_relay in dut_dhcp_relay_data:
# Bring all uplink interfaces down
for iface in dhcp_relay['uplink_interfaces']:
duthost.shell('ifconfig {} down'.format(iface))
# Sleep a bit to ensure uplinks are down
time.sleep(20)
# Bring all uplink interfaces back up
for iface in dhcp_relay['uplink_interfaces']:
duthost.shell('ifconfig {} up'.format(iface))
# Sleep a bit to ensure uplinks are up
time.sleep(20)
# Run the DHCP relay test on the PTF host
ptf_runner(
ptfhost,
"ptftests",
"dhcpv6_relay_test.DHCPTest",
platform_dir="ptftests",
params={
"hostname":
duthost.hostname,
"client_port_index":
dhcp_relay['client_iface']['port_idx'],
"leaf_port_indices":
repr(dhcp_relay['uplink_port_indices']),
"num_dhcp_servers":
len(dhcp_relay['downlink_vlan_iface']['dhcpv6_server_addrs']),
"server_ip":
str(dhcp_relay['downlink_vlan_iface']['dhcpv6_server_addrs']
[0]),
"relay_iface_ip":
str(dhcp_relay['downlink_vlan_iface']['addr']),
"relay_iface_mac":
str(dhcp_relay['downlink_vlan_iface']['mac']),
"relay_link_local":
str(dhcp_relay['uplink_interface_link_local']),
"vlan_ip":
str(dhcp_relay['downlink_vlan_iface']['addr'])
},
log_file="/tmp/dhcpv6_relay_test.DHCPTest.log",
is_python3=True)
def test_get_revision(self, duthosts, enum_rand_one_per_hwsku_hostname,
localhost, platform_api_conn):
duthost = duthosts[enum_rand_one_per_hwsku_hostname]
skip_release(duthost, ["201811", "201911", "202012"])
for i in range(self.num_psus):
revision = psu.get_revision(platform_api_conn, i)
if self.expect(
revision is not None,
"Unable to retrieve PSU {} serial number".format(i)):
self.expect(isinstance(revision, STRING_TYPE),
"PSU {} serial number appears incorrect".format(i))
self.assert_expectations()
def test_interface_binding(duthosts, rand_one_dut_hostname, dut_dhcp_relay_data):
duthost = duthosts[rand_one_dut_hostname]
skip_release(duthost, ["201811", "201911", "202106"])
if not check_interface_status(duthost):
config_reload(duthost)
wait_critical_processes(duthost)
pytest_assert(wait_until(120, 5, 0, check_interface_status, duthost))
output = duthost.shell("docker exec -it dhcp_relay ss -nlp | grep dhcrelay", module_ignore_errors=True)["stdout"].encode("utf-8")
logger.info(output)
for dhcp_relay in dut_dhcp_relay_data:
assert "{}:67".format(dhcp_relay['downlink_vlan_iface']['name']) in output, "{} is not found in {}".format("{}:67".format(dhcp_relay['downlink_vlan_iface']['name']), output)
for iface in dhcp_relay['uplink_interfaces']:
assert "{}:67".format(iface) in output, "{} is not found in {}".format("{}:67".format(iface), output)
def test_ro_disk(localhost, duthosts, enum_rand_one_per_hwsku_hostname,
creds_all_duts, check_tacacs):
"""test tacacs rw user
"""
duthost = duthosts[enum_rand_one_per_hwsku_hostname]
skip_release(duthost, ["201911", "201811"])
dutip = duthost.host.options['inventory_manager'].get_host(
duthost.hostname).vars['ansible_host']
ro_user = creds_all_duts[duthost]['tacacs_ro_user']
ro_pass = creds_all_duts[duthost]['tacacs_ro_user_passwd']
rw_user = creds_all_duts[duthost]['tacacs_rw_user']
rw_pass = creds_all_duts[duthost]['tacacs_rw_user_passwd']
res = duthost.shell("ls -l /home/{}".format(ro_user),
module_ignore_errors=True)
if res["rc"] == 0:
logger.debug("ro user pre-exists; deleting")
try:
duthost.shell("sudo deluser --remove-home {}".format(ro_user),
module_ignore_errors=True)
finally:
# If any failure, it implies user not valid, which is good enough.
logger.info("del user {} done".format(ro_user))
try:
# Ensure rw user can get in, as we need this to be able to reboot
ret = chk_ssh_remote_run(localhost, dutip, rw_user, rw_pass, "ls")
assert ret, "Failed to ssh as rw user"
# Set disk in RO state
simulate_ro(duthost)
logger.debug("user={}".format(ro_user))
assert wait_until(600, 20, 0, chk_ssh_remote_run, localhost, dutip,
ro_user, ro_pass,
"cat /etc/passwd"), "Failed to ssh as ro user"
finally:
logger.debug("START: reboot {} to restore disk RW state".format(
enum_rand_one_per_hwsku_hostname))
do_reboot(duthost, localhost, dutip, rw_user, rw_pass)
assert wait_until(600, 20, 0, duthost.critical_services_fully_started
), "Not all critical services are fully started"
logger.debug(" END: reboot {} to restore disk RW state".format(
enum_rand_one_per_hwsku_hostname))
def restore_counter_poll(duthosts, enum_rand_one_per_hwsku_hostname):
duthost = duthosts[enum_rand_one_per_hwsku_hostname]
skip_release(duthost, ["201811", "201911", "202012"])
counter_poll_show = ConterpollHelper.get_counterpoll_show_output(duthost)
parsed_counterpoll_before = ConterpollHelper.get_parsed_counterpoll_show(
counter_poll_show)
yield
counter_poll_show = ConterpollHelper.get_counterpoll_show_output(duthost)
parsed_counterpoll_after = ConterpollHelper.get_parsed_counterpoll_show(
counter_poll_show)
ConterpollHelper.restore_counterpoll_status(duthost,
parsed_counterpoll_before,
parsed_counterpoll_after)
def test_dhcp_relay_start_with_uplinks_down(ptfhost, duthosts, rand_one_dut_hostname, dut_dhcp_relay_data, validate_dut_routes_exist):
"""Test DHCP relay functionality on T0 topology when relay agent starts with uplinks down
For each DHCP relay agent running on the DuT, bring the uplinks down, then restart the
relay agent while the uplinks are still down. Then test whether the DHCP relay agent
relays packets properly.
"""
duthost = duthosts[rand_one_dut_hostname]
skip_release(duthost, ["201811", "201911", "202106"])
for dhcp_relay in dut_dhcp_relay_data:
# Bring all uplink interfaces down
for iface in dhcp_relay['uplink_interfaces']:
duthost.shell('ifconfig {} down'.format(iface))
# Sleep a bit to ensure uplinks are down
time.sleep(20)
# Restart DHCP relay service on DUT
# dhcp_relay service has 3 times restart limit in 20 mins, for 4 vlans config it will hit the maximum limit
# reset-failed before restart service
cmds = ['systemctl reset-failed dhcp_relay', 'systemctl restart dhcp_relay']
duthost.shell_cmds(cmds=cmds)
# Sleep to give the DHCP relay container time to start up and
# allow the relay agent to begin listening on the down interfaces
time.sleep(40)
# Bring all uplink interfaces back up
for iface in dhcp_relay['uplink_interfaces']:
duthost.shell('ifconfig {} up'.format(iface))
# Sleep a bit to ensure uplinks are up
time.sleep(20)
# Run the DHCP relay test on the PTF host
ptf_runner(ptfhost,
"ptftests",
"dhcpv6_relay_test.DHCPTest",
platform_dir="ptftests",
params={"hostname": duthost.hostname,
"client_port_index": dhcp_relay['client_iface']['port_idx'],
"leaf_port_indices": repr(dhcp_relay['uplink_port_indices']),
"num_dhcp_servers": len(dhcp_relay['downlink_vlan_iface']['dhcpv6_server_addrs']),
"server_ip": str(dhcp_relay['downlink_vlan_iface']['dhcpv6_server_addrs'][0]),
"relay_iface_ip": str(dhcp_relay['downlink_vlan_iface']['addr']),
"relay_iface_mac": str(dhcp_relay['downlink_vlan_iface']['mac']),
"relay_link_local": str(dhcp_relay['uplink_interface_link_local']),
"vlan_ip": str(dhcp_relay['downlink_vlan_iface']['addr'])},
log_file="/tmp/dhcpv6_relay_test.DHCPTest.log")
def test_dhcp_relay_default(ptfhost, duthosts, rand_one_dut_hostname,
dut_dhcp_relay_data, validate_dut_routes_exist):
"""Test DHCP relay functionality on T0 topology.
For each DHCP relay agent running on the DuT, verify DHCP packets are relayed properly
"""
duthost = duthosts[rand_one_dut_hostname]
skip_release(duthost, ["201811", "201911", "202106"])
for dhcp_relay in dut_dhcp_relay_data:
# Run the DHCP relay test on the PTF host
ptf_runner(
ptfhost,
"ptftests",
"dhcpv6_relay_test.DHCPTest",
platform_dir="ptftests",
params={
"hostname":
duthost.hostname,
"client_port_index":
dhcp_relay['client_iface']['port_idx'],
"leaf_port_indices":
repr(dhcp_relay['uplink_port_indices']),
"num_dhcp_servers":
len(dhcp_relay['downlink_vlan_iface']['dhcpv6_server_addrs']),
"server_ip":
str(dhcp_relay['downlink_vlan_iface']['dhcpv6_server_addrs']
[0]),
"relay_iface_ip":
str(dhcp_relay['downlink_vlan_iface']['addr']),
"relay_iface_mac":
str(dhcp_relay['downlink_vlan_iface']['mac']),
"relay_link_local":
str(dhcp_relay['uplink_interface_link_local']),
"vlan_ip":
str(dhcp_relay['downlink_vlan_iface']['addr'])
},
log_file="/tmp/dhcpv6_relay_test.DHCPTest.log",
is_python3=True)
def test_pmon_psud_kill_and_start_status(check_daemon_status, duthosts,
enum_supervisor_dut_hostname,
data_before_restart):
"""
@summary: This test case is to check the psud killed unexpectedly (automatically restarted) status
"""
duthost = duthosts[enum_supervisor_dut_hostname]
skip_release(duthost, ["201811", "201911"])
pre_daemon_status, pre_daemon_pid = duthost.get_pmon_daemon_status(
daemon_name)
logger.info("{} daemon is {} with pid {}".format(daemon_name,
pre_daemon_status,
pre_daemon_pid))
duthost.stop_pmon_daemon(daemon_name, SIG_KILL, pre_daemon_pid)
wait_until(120, 10, 0, check_expected_daemon_status, duthost,
expected_running_status)
post_daemon_status, post_daemon_pid = duthost.get_pmon_daemon_status(
daemon_name)
pytest_assert(
post_daemon_status == expected_running_status,
"{} expected restarted status is {} but is {}".format(
daemon_name, expected_running_status, post_daemon_status))
pytest_assert(
post_daemon_pid != -1,
"{} expected pid is -1 but is {}".format(daemon_name, post_daemon_pid))
pytest_assert(
post_daemon_pid > pre_daemon_pid,
"Restarted {} pid should be bigger than {} but it is {}".format(
daemon_name, pre_daemon_pid, post_daemon_pid))
data_after_restart = wait_data(duthost)
pytest_assert(data_after_restart == data_before_restart,
'DB data present before and after restart does not match')
def test_nhop(request, duthost, tbinfo):
"""
Test next hop group resource count. Steps:
- Add test IP address to an active IP interface
- Add static ARPs
- Create unique next hop groups
- Add IP route and nexthop
- check CRM resource
- clean up
- Verify no erros and crash
"""
skip_release(duthost, ["201811", "201911"])
default_max_nhop_paths = 32
nhop_group_limit = 1024
# program more than the advertised limit
extra_nhops = 10
asic = duthost.asic_instance()
# find out MAX NHOP group count supported on the platform
result = asic.run_redis_cmd(
argv=["redis-cli", "-n", 6, "HGETALL", "SWITCH_CAPABILITY|switch"])
it = iter(result)
switch_capability = dict(zip(it, it))
max_nhop = switch_capability.get("MAX_NEXTHOP_GROUP_COUNT")
max_nhop = nhop_group_limit if max_nhop == None else int(max_nhop)
nhop_group_count = min(max_nhop, nhop_group_limit) + extra_nhops
# find out an active IP port
ip_ifaces = asic.get_active_ip_interfaces(tbinfo).keys()
pytest_assert(len(ip_ifaces), "No IP interfaces found")
eth_if = ip_ifaces[0]
# Generate ARP entries
arp_count = 40
arplist = Arp(duthost, asic, arp_count, eth_if)
arplist.arps_add()
# indices
indices = range(arp_count)
ip_indices = combinations(indices, default_max_nhop_paths)
# intitialize log analyzer
marker = "NHOP TEST PATH COUNT {} {}".format(nhop_group_count, eth_if)
loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix=marker)
marker = loganalyzer.init()
loganalyzer.load_common_config()
loganalyzer.expect_regex = []
loganalyzer.ignore_regex.extend(loganalyzer_ignore_regex_list())
ip_prefix = ipaddr.IPAddress("192.168.0.0")
# list of all IPs available to generate a nexthop group
ip_list = arplist.ip_mac_list
crm_before = get_crm_info(duthost, asic)
# increase CRM polling time
asic.command("crm config polling interval 10")
logging.info("Adding {} next hops on {}".format(nhop_group_count, eth_if))
# create nexthop group
nhop = IPRoutes(duthost, asic)
try:
for i, indx_list in zip(range(nhop_group_count), ip_indices):
# get a list of unique group of next hop IPs
ips = [arplist.ip_mac_list[x].ip for x in indx_list]
ip_route = "{}/31".format(ip_prefix + (2 * i))
# add IP route with the next hop group created
nhop.add_ip_route(ip_route, ips)
nhop.program_routes()
# wait for routes to be synced and programmed
time.sleep(120)
crm_after = get_crm_info(duthost, asic)
finally:
nhop.delete_routes()
arplist.clean_up()
asic.command("crm config polling interval {}".format(
crm_before["polling"]))
# check for any errors or crash
loganalyzer.analyze(marker)
# verify the test used up all the NHOP group resources
# skip this check on Mellanox as ASIC resources are shared
if not is_mellanox_device(duthost):
pytest_assert(
crm_after["available"] == 0,
"Unused NHOP group resource: {}, used:{}".format(
crm_after["available"], crm_after["used"]))
def test_ro_disk(localhost, ptfhost, duthosts,
enum_rand_one_per_hwsku_hostname, tacacs_creds, check_tacacs):
"""test tacacs rw user
"""
duthost = duthosts[enum_rand_one_per_hwsku_hostname]
skip_release(duthost, ["201911", "201811"])
dutip = duthost.mgmt_ip
ro_user = tacacs_creds['tacacs_ro_user']
ro_pass = tacacs_creds['tacacs_ro_user_passwd']
rw_user = tacacs_creds['tacacs_rw_user']
rw_pass = tacacs_creds['tacacs_rw_user_passwd']
if not do_check_clean_state(duthost):
# System has some partial state left behind from last run.
# reboot to clear it
#
logger.info("PRETEST: reboot {} to restore system state".format(
enum_rand_one_per_hwsku_hostname))
do_reboot(duthost, localhost)
assert do_check_clean_state(
duthost), "state not good even after reboot"
do_setup_tacacs(ptfhost, duthost, tacacs_creds)
# just check it out that ro user could indeed login
ret = chk_ssh_remote_run(localhost, dutip, ro_user, ro_pass, "ls")
assert ret, "Failed pre-test ssh login as ro user"
logger.debug("Delete ro user to simulate new login in RO state.")
duthost.shell("sudo deluser --remove-home {}".format(ro_user))
logger.info("del user {} done".format(ro_user))
res = duthost.shell("ls -l /home/{}".format(ro_user),
module_ignore_errors=True)
assert res["rc"] != 0, "Failed to remove ro user dir"
# Ensure rw user can get in, as we need this to be able to reboot
ret = chk_ssh_remote_run(localhost, dutip, rw_user, rw_pass, "ls")
assert ret, "Failed to ssh as rw user"
try:
# Redirect logs to tmpfs
#
duthost.shell("sudo mkdir {}".format(LOG_DIR))
conf_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
"000-ro_disk.conf")
duthost.copy(src=conf_path, dest="/etc/rsyslog.d/000-ro_disk.conf")
# To get file in decent size. Force a rotate
duthost.shell("logrotate --force /etc/logrotate.d/rsyslog")
res = duthost.shell("systemctl restart rsyslog")
assert res["rc"] == 0, "failed to restart rsyslog"
# Pause 2 seconds to ensure the new .conf is read in by rsyslogd
time.sleep(2)
# Remove file, so the reboot at the end of test will revert this logs redirect.
duthost.shell("rm /etc/rsyslog.d/000-ro_disk.conf")
# Set disk in RO state
simulate_ro(duthost)
logger.debug("user={}".format(ro_user))
# Wait for 15 minutes
# Reason:
# Monit does not start upon boot for 5 minutes.
# Note: Monit invokes disk check every 5 cycles/minutes
# We need to wait solid +10mins before concluding.
#
res = wait_until(900, 20, 0, chk_ssh_remote_run, localhost, dutip,
ro_user, ro_pass, "cat /etc/passwd")
logger.info("res={}".format(res))
chk_ssh_remote_run(localhost, dutip, rw_user, rw_pass,
"sudo find {} -ls".format(MOUNT_DIR))
chk_ssh_remote_run(localhost, dutip, rw_user, rw_pass,
"systemctl status monit")
chk_ssh_remote_run(localhost, dutip, rw_user, rw_pass,
"sudo find /home -ls")
if not os.path.exists(DATA_DIR):
os.mkdir(DATA_DIR)
# Fetch files of interest
#
for f in [
"/etc/passwd",
os.path.join(LOG_DIR, "auth.log"),
os.path.join(LOG_DIR, "syslog")
]:
fetch_into_file(localhost, dutip, rw_user, rw_pass, f,
os.path.join(DATA_DIR, os.path.basename(f)))
assert res, "Failed to ssh as ro user"
finally:
logger.debug("START: reboot {} to restore disk RW state".format(
enum_rand_one_per_hwsku_hostname))
do_reboot(duthost, localhost, dutip, rw_user, rw_pass)
assert wait_until(600, 20, 0, duthost.critical_services_fully_started
), "Not all critical services are fully started"
logger.debug(" END: reboot {} to restore disk RW state".format(
enum_rand_one_per_hwsku_hostname))
def skip_unsupported_release(duthost):
""" Span mirror is not supported on release < 202012
"""
skip_release(duthost, ["201811", "201911"])
def test_ro_disk(localhost, ptfhost, duthosts,
enum_rand_one_per_hwsku_hostname, tacacs_creds, check_tacacs):
"""test tacacs rw user
"""
duthost = duthosts[enum_rand_one_per_hwsku_hostname]
skip_release(duthost, ["201911", "201811"])
dutip = duthost.mgmt_ip
ro_user = tacacs_creds['tacacs_ro_user']
ro_pass = tacacs_creds['tacacs_ro_user_passwd']
rw_user = tacacs_creds['tacacs_rw_user']
rw_pass = tacacs_creds['tacacs_rw_user_passwd']
res = duthost.shell("ls -l /run/mount/*", module_ignore_errors=True)
if res["rc"] == 0:
# System has some partial state left behind from last run.
# reboot to clear it
#
logger.info("PRETEST: reboot {} to restore system state".format(
enum_rand_one_per_hwsku_hostname))
do_reboot(duthost, localhost, dutip, rw_user, rw_pass)
do_setup_tacacs(ptfhost, duthost, tacacs_creds)
res = duthost.shell("ls -l /home/{}".format(ro_user),
module_ignore_errors=True)
if res["rc"] == 0:
logger.debug("ro user pre-exists; deleting")
try:
duthost.shell("sudo deluser --remove-home {}".format(ro_user),
module_ignore_errors=True)
finally:
# If any failure, it implies user not valid, which is good enough.
logger.info("del user {} done".format(ro_user))
try:
# Ensure rw user can get in, as we need this to be able to reboot
ret = chk_ssh_remote_run(localhost, dutip, rw_user, rw_pass, "ls")
assert ret, "Failed to ssh as rw user"
# Set disk in RO state
simulate_ro(duthost)
logger.debug("user={}".format(ro_user))
# Wait for 15 minutes
# Reason:
# Monit does not start upon boot for 5 minutes.
# Note: Monit invokes disk check every 5 cycles/minutes
# We need to wait solid +10mins before concluding.
#
assert wait_until(900, 20, 0, chk_ssh_remote_run, localhost, dutip,
ro_user, ro_pass,
"cat /etc/passwd"), "Failed to ssh as ro user"
finally:
chk_ssh_remote_run(localhost, dutip, rw_user, rw_pass,
"ls -alrt /run/mount/")
chk_ssh_remote_run(localhost, dutip, rw_user, rw_pass,
"systemctl status monit")
logger.debug("START: reboot {} to restore disk RW state".format(
enum_rand_one_per_hwsku_hostname))
do_reboot(duthost, localhost, dutip, rw_user, rw_pass)
assert wait_until(600, 20, 0, duthost.critical_services_fully_started
), "Not all critical services are fully started"
logger.debug(" END: reboot {} to restore disk RW state".format(
enum_rand_one_per_hwsku_hostname))
