def post_setup_hook(self, dut, localhost):
"""
toggle ports after rules are applied
:param dut: dut host
:param localhost: localhost object
:return:
"""
port_toggle(dut)
def post_setup_hook(self, dut, localhost, populate_vlan_arp_entries):
"""Toggle ports after rules are applied.
Args:
dut: The DUT having ACLs applied.
localhost: The host from which tests are run.
populate_vlan_arp_entries: A fixture to populate ARP/FDB tables for VLAN interfaces.
"""
port_toggle(dut)
populate_vlan_arp_entries()
def test_port_toggle(self, duthost, bring_up_dut_interfaces):
"""
Validates that port toggle works as expected
Test steps:
1.) Flap all interfaces on DUT one by one.
2.) Verify interfaces are up correctly.
Pass Criteria: All interfaces are up correctly.
"""
port_toggle(duthost)
def test_port_toggle(self, duthosts, enum_rand_one_per_hwsku_frontend_hostname, bring_up_dut_interfaces, tbinfo):
"""
Validates that port toggle works as expected
Test steps:
1.) Flap all interfaces on DUT one by one.
2.) Verify interfaces are up correctly.
Pass Criteria: All interfaces are up correctly.
"""
duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname]
port_toggle(duthost, tbinfo)
def test_cont_link_flap(self, request, duthosts, nbrhosts,
enum_rand_one_per_hwsku_frontend_hostname,
fanouthosts, bring_up_dut_interfaces, tbinfo):
"""
Validates that continuous link flap works as expected
Test steps:
1.) Flap all interfaces one by one in 1-3 iteration
to cause BGP Flaps.
2.) Flap all interfaces on peer (FanOutLeaf) one by one 1-3 iteration
to cause BGP Flaps.
3.) Watch for memory (show system-memory) ,orchagent CPU Utilization
and Redis_memory.
Pass Criteria: All routes must be re-learned with < 5% increase in Redis and
ORCH agent CPU consumption below threshold after 3 mins after stopping flaps.
"""
duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname]
orch_cpu_threshold = request.config.getoption("--orch_cpu_threshold")
# Record memory status at start
memory_output = duthost.shell("show system-memory")["stdout"]
logging.info("Memory Status at start: %s", memory_output)
# Record Redis Memory at start
start_time_redis_memory = duthost.shell(
"redis-cli info memory | grep used_memory_human | sed -e 's/.*:\(.*\)M/\\1/'"
)["stdout"]
logging.info("Redis Memory: %s M", start_time_redis_memory)
# Record ipv4 route counts at start
sumv4, sumv6 = duthost.get_ip_route_summary()
totalsv4 = sumv4.get('Totals', {})
totalsv6 = sumv6.get('Totals', {})
start_time_ipv4_route_counts = totalsv4.get('routes', 0)
start_time_ipv6_route_counts = totalsv6.get('routes', 0)
logging.info("IPv4 routes: start {}, summary {}".format(
start_time_ipv4_route_counts, sumv4))
logging.info("IPv6 routes: start {}, summary {}".format(
start_time_ipv6_route_counts, sumv6))
# Make Sure Orch CPU < orch_cpu_threshold before starting test.
logging.info(
"Make Sure orchagent CPU utilization is less that %d before link flap",
orch_cpu_threshold)
pytest_assert(
wait_until(100, 2, 0, check_orch_cpu_utilization, duthost,
orch_cpu_threshold),
"Orch CPU utilization {} > orch cpu threshold {} before link flap".
format(
duthost.shell(
"show processes cpu | grep orchagent | awk '{print $9}'")
["stdout"], orch_cpu_threshold))
# Flap all interfaces one by one on DUT
for iteration in range(3):
logging.info("%d Iteration flap all interfaces one by one on DUT",
iteration + 1)
port_toggle(duthost, tbinfo, watch=True)
# Flap all interfaces one by one on Peer Device
for iteration in range(3):
logging.info(
"%d Iteration flap all interfaces one by one on Peer Device",
iteration + 1)
candidates = build_test_candidates(duthost, fanouthosts,
'all_ports')
pytest_require(
candidates,
"Didn't find any port that is admin up and present in the connection graph"
)
for dut_port, fanout, fanout_port in candidates:
toggle_one_link(duthost,
dut_port,
fanout,
fanout_port,
watch=True)
config_facts = duthost.get_running_config_facts()
for portchannel in config_facts['PORTCHANNEL'].keys():
pytest_assert(
check_portchannel_status(duthost,
portchannel,
"up",
verbose=True),
"Fail: dut interface {}: link operational down".format(
portchannel))
# Make Sure all ipv4/ipv6 routes are relearned with jitter of ~5
if not wait_until(120, 2, 0, check_bgp_routes, duthost,
start_time_ipv4_route_counts,
start_time_ipv6_route_counts):
endv4, endv6 = duthost.get_ip_route_summary()
failmsg = []
failmsg.append(
"IP routes are not equal after link flap: before ipv4 {} ipv6 {}, after ipv4 {} ipv6 {}"
.format(sumv4, sumv6, endv4, endv6))
nei_meta = config_facts.get('DEVICE_NEIGHBOR_METADATA', {})
for k in nei_meta.keys():
nbrhost = nbrhosts[k]['host']
if isinstance(nbrhost, EosHost):
res = nbrhost.eos_command(commands=['show ip bgp sum'])
elif isinstance(nbrhost, SonicHost):
res = nbrhost.command('vtysh -c "show ip bgp sum"')
else:
res = ""
failmsg.append(res['stdout'])
pytest.fail(str(failmsg))
# Record memory status at end
memory_output = duthost.shell("show system-memory")["stdout"]
logging.info("Memory Status at end: %s", memory_output)
# Record orchagent CPU utilization at end
orch_cpu = duthost.shell(
"show processes cpu | grep orchagent | awk '{print $9}'")["stdout"]
logging.info("Orchagent CPU Util at end: %s", orch_cpu)
# Record Redis Memory at end
end_time_redis_memory = duthost.shell(
"redis-cli info memory | grep used_memory_human | sed -e 's/.*:\(.*\)M/\\1/'"
)["stdout"]
logging.info("Redis Memory at start: %s M", start_time_redis_memory)
logging.info("Redis Memory at end: %s M", end_time_redis_memory)
# Calculate diff in Redis memory
incr_redis_memory = float(end_time_redis_memory) - float(
start_time_redis_memory)
logging.info("Redis absolute difference: %d", incr_redis_memory)
# Check redis memory only if it is increased else default to pass
if incr_redis_memory > 0.0:
percent_incr_redis_memory = (incr_redis_memory /
float(start_time_redis_memory)) * 100
logging.info("Redis Memory percentage Increase: %d",
percent_incr_redis_memory)
pytest_assert(
percent_incr_redis_memory < 5,
"Redis Memory Increase more than expected: {}".format(
percent_incr_redis_memory))
# Orchagent CPU should consume < orch_cpu_threshold at last.
logging.info("watch orchagent CPU utilization when it goes below %d",
orch_cpu_threshold)
pytest_assert(
wait_until(45, 2, 0, check_orch_cpu_utilization, duthost,
orch_cpu_threshold),
"Orch CPU utilization {} > orch cpu threshold {} before link flap".
format(
duthost.shell(
"show processes cpu | grep orchagent | awk '{print $9}'")
["stdout"], orch_cpu_threshold))
def test_cont_link_flap(self, request, duthosts, rand_one_dut_hostname,
fanouthosts, bring_up_dut_interfaces, tbinfo):
"""
Validates that continuous link flap works as expected
Test steps:
1.) Flap all interfaces one by one in 1-3 iteration
to cause BGP Flaps.
2.) Flap all interfaces on peer (FanOutLeaf) one by one 1-3 iteration
to cause BGP Flaps.
3.) Watch for memory (show system-memory) ,orchagent CPU Utilization
and Redis_memory.
Pass Criteria: All routes must be re-learned with < 5% increase in Redis and
ORCH agent CPU consumption below threshold after 3 mins after stopping flaps.
"""
duthost = duthosts[rand_one_dut_hostname]
orch_cpu_threshold = request.config.getoption("--orch_cpu_threshold")
# Record memory status at start
memory_output = duthost.shell("show system-memory")["stdout"]
logging.info("Memory Status at start: %s", memory_output)
# Record Redis Memory at start
start_time_redis_memory = duthost.shell(
"redis-cli info memory | grep used_memory_human | sed -e 's/.*:\(.*\)M/\\1/'"
)["stdout"]
logging.info("Redis Memory: %s M", start_time_redis_memory)
# Record ipv4 route counts at start
start_time_ipv4_route_counts = duthost.shell(
"show ip route summary | grep Total | awk '{print $2}'")["stdout"]
logging.info("IPv4 routes at start: %s", start_time_ipv4_route_counts)
# Record ipv6 route counts at start
start_time_ipv6_route_counts = duthost.shell(
"show ipv6 route summary | grep Total | awk '{print $2}'"
)["stdout"]
logging.info("IPv6 routes at start %s", start_time_ipv6_route_counts)
# Make Sure Orch CPU < orch_cpu_threshold before starting test.
logging.info(
"Make Sure orchagent CPU utilization is less that %d before link flap",
orch_cpu_threshold)
pytest_assert(
wait_until(100, 2, check_orch_cpu_utilization, duthost,
orch_cpu_threshold),
"Orch CPU utilization {} > orch cpu threshold {} before link flap".
format(
duthost.shell(
"show processes cpu | grep orchagent | awk '{print $9}'")
["stdout"], orch_cpu_threshold))
# Flap all interfaces one by one on DUT
for iteration in range(3):
logging.info("%d Iteration flap all interfaces one by one on DUT",
iteration + 1)
port_toggle(duthost, tbinfo, watch=True)
# Flap all interfaces one by one on Peer Device
for iteration in range(3):
logging.info(
"%d Iteration flap all interfaces one by one on Peer Device",
iteration + 1)
candidates = build_test_candidates(duthost, fanouthosts,
'all_ports')
pytest_require(
candidates,
"Didn't find any port that is admin up and present in the connection graph"
)
for dut_port, fanout, fanout_port in candidates:
toggle_one_link(duthost,
dut_port,
fanout,
fanout_port,
watch=True)
# Make Sure all ipv4 routes are relearned with jitter of ~5
logging.info("IPv4 routes at start: %s", start_time_ipv4_route_counts)
pytest_assert(
wait_until(60, 1, check_bgp_routes, duthost,
start_time_ipv4_route_counts, True),
"Ipv4 routes are not equal after link flap")
# Make Sure all ipv6 routes are relearned with jitter of ~5
logging.info("IPv6 routes at start: %s", start_time_ipv6_route_counts)
pytest_assert(
wait_until(60, 1, check_bgp_routes, duthost,
start_time_ipv6_route_counts),
"Ipv6 routes are not equal after link flap")
# Record memory status at end
memory_output = duthost.shell("show system-memory")["stdout"]
logging.info("Memory Status at end: %s", memory_output)
# Record orchagent CPU utilization at end
orch_cpu = duthost.shell(
"show processes cpu | grep orchagent | awk '{print $9}'")["stdout"]
logging.info("Orchagent CPU Util at end: %s", orch_cpu)
# Record Redis Memory at end
end_time_redis_memory = duthost.shell(
"redis-cli info memory | grep used_memory_human | sed -e 's/.*:\(.*\)M/\\1/'"
)["stdout"]
logging.info("Redis Memory at start: %s M", start_time_redis_memory)
logging.info("Redis Memory at end: %s M", end_time_redis_memory)
# Calculate diff in Redis memory
incr_redis_memory = float(end_time_redis_memory) - float(
start_time_redis_memory)
logging.info("Redis absolute difference: %d", incr_redis_memory)
# Check redis memory only if it is increased else default to pass
if incr_redis_memory > 0.0:
percent_incr_redis_memory = (incr_redis_memory /
float(start_time_redis_memory)) * 100
logging.info("Redis Memory percentage Increase: %d",
percent_incr_redis_memory)
pytest_assert(
percent_incr_redis_memory < 5,
"Redis Memory Increase more than expected: {}".format(
percent_incr_redis_memory))
# Orchagent CPU should consume < orch_cpu_threshold at last.
logging.info("watch orchagent CPU utilization when it goes below %d",
orch_cpu_threshold)
pytest_assert(
wait_until(45, 2, check_orch_cpu_utilization, duthost,
orch_cpu_threshold),
"Orch CPU utilization {} > orch cpu threshold {} before link flap".
format(
duthost.shell(
"show processes cpu | grep orchagent | awk '{print $9}'")
["stdout"], orch_cpu_threshold))
def test_pfcwd_port_toggle(self, request, fake_storm, setup_pfc_test, fanout_graph_facts, tbinfo, ptfhost, duthosts, rand_one_dut_hostname, fanouthosts):
"""
Test PfCWD functionality after toggling port
Test verifies the following:
1. Select the port and lossless queue
2. Start PFCWD on selected test port
3. Start PFC storm on selected test port and lossless queue
4. Verify that PFC storm is detected
5. Stop PFC storm on selected test port and lossless queue
6. Verify that PFC storm is restored
7. Toggle test port (put administrativelly down and then up)
8. Verify that PFC storm is not detected
Args:
request(object) : pytest request object
fake_storm(fixture) : Module scoped fixture for enable/disable fake storm
setup_pfc_test(fixture) : Module scoped autouse fixture for PFCWD
fanout_graph_facts(fixture) : Fanout graph info
tbinfo(fixture) : Testbed info
ptfhost(AnsibleHost) : PTF host instance
duthost(AnsibleHost) : DUT instance
fanouthosts(AnsibleHost): Fanout instance
"""
duthost = duthosts[rand_one_dut_hostname]
setup_info = setup_pfc_test
self.fanout_info = fanout_graph_facts
self.ptf = ptfhost
self.dut = duthost
self.fanout = fanouthosts
self.timers = setup_info['pfc_timers']
self.ports = setup_info['selected_test_ports']
self.neighbors = setup_info['neighbors']
dut_facts = self.dut.facts
self.peer_dev_list = dict()
self.fake_storm = fake_storm
self.storm_hndle = None
action = "dontcare"
for idx, port in enumerate(self.ports):
logger.info("")
logger.info("--- Testing port toggling with PFCWD enabled on {} ---".format(port))
self.setup_test_params(port, setup_info['vlan'], init=not idx)
self.traffic_inst = SendVerifyTraffic(self.ptf, dut_facts['router_mac'], self.pfc_wd)
pfc_wd_restore_time_large = request.config.getoption("--restore-time")
# wait time before we check the logs for the 'restore' signature. 'pfc_wd_restore_time_large' is in ms.
self.timers['pfc_wd_wait_for_restore_time'] = int(pfc_wd_restore_time_large / 1000 * 2)
try:
# Verify that PFC storm is detected and restored
self.stats = PfcPktCntrs(self.dut, action)
logger.info("{} on port {}".format(WD_ACTION_MSG_PFX[action], port))
self.run_test(self.dut, port, action)
# Toggle test port and verify that PFC storm is not detected
loganalyzer = LogAnalyzer(ansible_host=self.dut, marker_prefix="pfc_function_storm_detect_{}_port_{}".format(action, port))
marker = loganalyzer.init()
ignore_file = os.path.join(TEMPLATES_DIR, "ignore_pfc_wd_messages")
reg_exp = loganalyzer.parse_regexp_file(src=ignore_file)
loganalyzer.ignore_regex.extend(reg_exp)
loganalyzer.expect_regex = []
loganalyzer.expect_regex.extend([EXPECT_PFC_WD_DETECT_RE])
loganalyzer.match_regex = []
port_toggle(self.dut, tbinfo, ports=[port])
logger.info("Verify that PFC storm is not detected on port {}".format(port))
result = loganalyzer.analyze(marker, fail=False)
if result["total"]["expected_missing_match"] == 0:
pytest.fail(result)
except Exception as e:
pytest.fail(str(e))
finally:
if self.storm_hndle:
logger.info("--- Stop PFC storm on port {}".format(port))
self.storm_hndle.stop_storm()
else:
logger.info("--- Disabling fake storm on port {} queue {}".format(port, self.queue_oid))
PfcCmd.set_storm_status(self.dut, self.queue_oid, "disabled")
logger.info("--- Stop PFCWD ---")
self.dut.command("pfcwd stop")
