def run_link_flap_test(self, dut, fanouthosts, port):
"""
Test runner of link flap test.
Args:
dut: DUT host object
fanouthosts: List of fanout switch instances.
"""
candidates = build_test_candidates(dut, fanouthosts, port, self.completeness_level)
pytest_require(candidates, "Didn't find any port that is admin up and present in the connection graph")
for dut_port, fanout, fanout_port in candidates:
toggle_one_link(dut, dut_port, fanout, fanout_port)
def test_cont_link_flap(self, request, duthosts, nbrhosts,
enum_rand_one_per_hwsku_frontend_hostname,
fanouthosts, bring_up_dut_interfaces, tbinfo):
"""
Validates that continuous link flap works as expected
Test steps:
1.) Flap all interfaces one by one in 1-3 iteration
to cause BGP Flaps.
2.) Flap all interfaces on peer (FanOutLeaf) one by one 1-3 iteration
to cause BGP Flaps.
3.) Watch for memory (show system-memory) ,orchagent CPU Utilization
and Redis_memory.
Pass Criteria: All routes must be re-learned with < 5% increase in Redis and
ORCH agent CPU consumption below threshold after 3 mins after stopping flaps.
"""
duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname]
orch_cpu_threshold = request.config.getoption("--orch_cpu_threshold")
# Record memory status at start
memory_output = duthost.shell("show system-memory")["stdout"]
logging.info("Memory Status at start: %s", memory_output)
# Record Redis Memory at start
start_time_redis_memory = duthost.shell(
"redis-cli info memory | grep used_memory_human | sed -e 's/.*:\(.*\)M/\\1/'"
)["stdout"]
logging.info("Redis Memory: %s M", start_time_redis_memory)
# Record ipv4 route counts at start
sumv4, sumv6 = duthost.get_ip_route_summary()
totalsv4 = sumv4.get('Totals', {})
totalsv6 = sumv6.get('Totals', {})
start_time_ipv4_route_counts = totalsv4.get('routes', 0)
start_time_ipv6_route_counts = totalsv6.get('routes', 0)
logging.info("IPv4 routes: start {}, summary {}".format(
start_time_ipv4_route_counts, sumv4))
logging.info("IPv6 routes: start {}, summary {}".format(
start_time_ipv6_route_counts, sumv6))
# Make Sure Orch CPU < orch_cpu_threshold before starting test.
logging.info(
"Make Sure orchagent CPU utilization is less that %d before link flap",
orch_cpu_threshold)
pytest_assert(
wait_until(100, 2, 0, check_orch_cpu_utilization, duthost,
orch_cpu_threshold),
"Orch CPU utilization {} > orch cpu threshold {} before link flap".
format(
duthost.shell(
"show processes cpu | grep orchagent | awk '{print $9}'")
["stdout"], orch_cpu_threshold))
# Flap all interfaces one by one on DUT
for iteration in range(3):
logging.info("%d Iteration flap all interfaces one by one on DUT",
iteration + 1)
port_toggle(duthost, tbinfo, watch=True)
# Flap all interfaces one by one on Peer Device
for iteration in range(3):
logging.info(
"%d Iteration flap all interfaces one by one on Peer Device",
iteration + 1)
candidates = build_test_candidates(duthost, fanouthosts,
'all_ports')
pytest_require(
candidates,
"Didn't find any port that is admin up and present in the connection graph"
)
for dut_port, fanout, fanout_port in candidates:
toggle_one_link(duthost,
dut_port,
fanout,
fanout_port,
watch=True)
config_facts = duthost.get_running_config_facts()
for portchannel in config_facts['PORTCHANNEL'].keys():
pytest_assert(
check_portchannel_status(duthost,
portchannel,
"up",
verbose=True),
"Fail: dut interface {}: link operational down".format(
portchannel))
# Make Sure all ipv4/ipv6 routes are relearned with jitter of ~5
if not wait_until(120, 2, 0, check_bgp_routes, duthost,
start_time_ipv4_route_counts,
start_time_ipv6_route_counts):
endv4, endv6 = duthost.get_ip_route_summary()
failmsg = []
failmsg.append(
"IP routes are not equal after link flap: before ipv4 {} ipv6 {}, after ipv4 {} ipv6 {}"
.format(sumv4, sumv6, endv4, endv6))
nei_meta = config_facts.get('DEVICE_NEIGHBOR_METADATA', {})
for k in nei_meta.keys():
nbrhost = nbrhosts[k]['host']
if isinstance(nbrhost, EosHost):
res = nbrhost.eos_command(commands=['show ip bgp sum'])
elif isinstance(nbrhost, SonicHost):
res = nbrhost.command('vtysh -c "show ip bgp sum"')
else:
res = ""
failmsg.append(res['stdout'])
pytest.fail(str(failmsg))
# Record memory status at end
memory_output = duthost.shell("show system-memory")["stdout"]
logging.info("Memory Status at end: %s", memory_output)
# Record orchagent CPU utilization at end
orch_cpu = duthost.shell(
"show processes cpu | grep orchagent | awk '{print $9}'")["stdout"]
logging.info("Orchagent CPU Util at end: %s", orch_cpu)
# Record Redis Memory at end
end_time_redis_memory = duthost.shell(
"redis-cli info memory | grep used_memory_human | sed -e 's/.*:\(.*\)M/\\1/'"
)["stdout"]
logging.info("Redis Memory at start: %s M", start_time_redis_memory)
logging.info("Redis Memory at end: %s M", end_time_redis_memory)
# Calculate diff in Redis memory
incr_redis_memory = float(end_time_redis_memory) - float(
start_time_redis_memory)
logging.info("Redis absolute difference: %d", incr_redis_memory)
# Check redis memory only if it is increased else default to pass
if incr_redis_memory > 0.0:
percent_incr_redis_memory = (incr_redis_memory /
float(start_time_redis_memory)) * 100
logging.info("Redis Memory percentage Increase: %d",
percent_incr_redis_memory)
pytest_assert(
percent_incr_redis_memory < 5,
"Redis Memory Increase more than expected: {}".format(
percent_incr_redis_memory))
# Orchagent CPU should consume < orch_cpu_threshold at last.
logging.info("watch orchagent CPU utilization when it goes below %d",
orch_cpu_threshold)
pytest_assert(
wait_until(45, 2, 0, check_orch_cpu_utilization, duthost,
orch_cpu_threshold),
"Orch CPU utilization {} > orch cpu threshold {} before link flap".
format(
duthost.shell(
"show processes cpu | grep orchagent | awk '{print $9}'")
["stdout"], orch_cpu_threshold))
def test_cont_link_flap(self, request, duthosts, rand_one_dut_hostname,
fanouthosts, bring_up_dut_interfaces, tbinfo):
"""
Validates that continuous link flap works as expected
Test steps:
1.) Flap all interfaces one by one in 1-3 iteration
to cause BGP Flaps.
2.) Flap all interfaces on peer (FanOutLeaf) one by one 1-3 iteration
to cause BGP Flaps.
3.) Watch for memory (show system-memory) ,orchagent CPU Utilization
and Redis_memory.
Pass Criteria: All routes must be re-learned with < 5% increase in Redis and
ORCH agent CPU consumption below threshold after 3 mins after stopping flaps.
"""
duthost = duthosts[rand_one_dut_hostname]
orch_cpu_threshold = request.config.getoption("--orch_cpu_threshold")
# Record memory status at start
memory_output = duthost.shell("show system-memory")["stdout"]
logging.info("Memory Status at start: %s", memory_output)
# Record Redis Memory at start
start_time_redis_memory = duthost.shell(
"redis-cli info memory | grep used_memory_human | sed -e 's/.*:\(.*\)M/\\1/'"
)["stdout"]
logging.info("Redis Memory: %s M", start_time_redis_memory)
# Record ipv4 route counts at start
start_time_ipv4_route_counts = duthost.shell(
"show ip route summary | grep Total | awk '{print $2}'")["stdout"]
logging.info("IPv4 routes at start: %s", start_time_ipv4_route_counts)
# Record ipv6 route counts at start
start_time_ipv6_route_counts = duthost.shell(
"show ipv6 route summary | grep Total | awk '{print $2}'"
)["stdout"]
logging.info("IPv6 routes at start %s", start_time_ipv6_route_counts)
# Make Sure Orch CPU < orch_cpu_threshold before starting test.
logging.info(
"Make Sure orchagent CPU utilization is less that %d before link flap",
orch_cpu_threshold)
pytest_assert(
wait_until(100, 2, check_orch_cpu_utilization, duthost,
orch_cpu_threshold),
"Orch CPU utilization {} > orch cpu threshold {} before link flap".
format(
duthost.shell(
"show processes cpu | grep orchagent | awk '{print $9}'")
["stdout"], orch_cpu_threshold))
# Flap all interfaces one by one on DUT
for iteration in range(3):
logging.info("%d Iteration flap all interfaces one by one on DUT",
iteration + 1)
port_toggle(duthost, tbinfo, watch=True)
# Flap all interfaces one by one on Peer Device
for iteration in range(3):
logging.info(
"%d Iteration flap all interfaces one by one on Peer Device",
iteration + 1)
candidates = build_test_candidates(duthost, fanouthosts,
'all_ports')
pytest_require(
candidates,
"Didn't find any port that is admin up and present in the connection graph"
)
for dut_port, fanout, fanout_port in candidates:
toggle_one_link(duthost,
dut_port,
fanout,
fanout_port,
watch=True)
# Make Sure all ipv4 routes are relearned with jitter of ~5
logging.info("IPv4 routes at start: %s", start_time_ipv4_route_counts)
pytest_assert(
wait_until(60, 1, check_bgp_routes, duthost,
start_time_ipv4_route_counts, True),
"Ipv4 routes are not equal after link flap")
# Make Sure all ipv6 routes are relearned with jitter of ~5
logging.info("IPv6 routes at start: %s", start_time_ipv6_route_counts)
pytest_assert(
wait_until(60, 1, check_bgp_routes, duthost,
start_time_ipv6_route_counts),
"Ipv6 routes are not equal after link flap")
# Record memory status at end
memory_output = duthost.shell("show system-memory")["stdout"]
logging.info("Memory Status at end: %s", memory_output)
# Record orchagent CPU utilization at end
orch_cpu = duthost.shell(
"show processes cpu | grep orchagent | awk '{print $9}'")["stdout"]
logging.info("Orchagent CPU Util at end: %s", orch_cpu)
# Record Redis Memory at end
end_time_redis_memory = duthost.shell(
"redis-cli info memory | grep used_memory_human | sed -e 's/.*:\(.*\)M/\\1/'"
)["stdout"]
logging.info("Redis Memory at start: %s M", start_time_redis_memory)
logging.info("Redis Memory at end: %s M", end_time_redis_memory)
# Calculate diff in Redis memory
incr_redis_memory = float(end_time_redis_memory) - float(
start_time_redis_memory)
logging.info("Redis absolute difference: %d", incr_redis_memory)
# Check redis memory only if it is increased else default to pass
if incr_redis_memory > 0.0:
percent_incr_redis_memory = (incr_redis_memory /
float(start_time_redis_memory)) * 100
logging.info("Redis Memory percentage Increase: %d",
percent_incr_redis_memory)
pytest_assert(
percent_incr_redis_memory < 5,
"Redis Memory Increase more than expected: {}".format(
percent_incr_redis_memory))
# Orchagent CPU should consume < orch_cpu_threshold at last.
logging.info("watch orchagent CPU utilization when it goes below %d",
orch_cpu_threshold)
pytest_assert(
wait_until(45, 2, check_orch_cpu_utilization, duthost,
orch_cpu_threshold),
"Orch CPU utilization {} > orch cpu threshold {} before link flap".
format(
duthost.shell(
"show processes cpu | grep orchagent | awk '{print $9}'")
["stdout"], orch_cpu_threshold))
def test_link_flap(request, duthosts, rand_one_dut_hostname, tbinfo,
fanouthosts, get_loop_times):
"""
Validates that link flap works as expected
"""
duthost = duthosts[rand_one_dut_hostname]
orch_cpu_threshold = request.config.getoption("--orch_cpu_threshold")
# Record memory status at start
memory_output = duthost.shell("show system-memory")["stdout"]
logger.info("Memory Status at start: %s", memory_output)
# Record Redis Memory at start
start_time_redis_memory = duthost.shell(
"redis-cli info memory | grep used_memory_human | sed -e 's/.*:\(.*\)M/\\1/'"
)["stdout"]
logger.info("Redis Memory: %s M", start_time_redis_memory)
# Make Sure Orch CPU < orch_cpu_threshold before starting test.
logger.info(
"Make Sure orchagent CPU utilization is less that %d before link flap",
orch_cpu_threshold)
pytest_assert(
wait_until(100, 2, 0, check_orch_cpu_utilization, duthost,
orch_cpu_threshold),
"Orch CPU utilization {} > orch cpu threshold {} before link flap".
format(
duthost.shell(
"show processes cpu | grep orchagent | awk '{print $9}'")
["stdout"], orch_cpu_threshold))
loop_times = get_loop_times
port_lists = get_port_list(duthost, tbinfo)
candidates = []
for port in port_lists:
fanout, fanout_port = fanout_switch_port_lookup(
fanouthosts, duthost.hostname, port)
candidates.append((port, fanout, fanout_port))
for loop_time in range(0, loop_times):
watch = False
check_status = False
if loop_time == 0 or loop_time == loop_times - 1:
watch = True
check_status = True
for dut_port, fanout, fanout_port in candidates:
toggle_one_link(duthost,
dut_port,
fanout,
fanout_port,
watch=watch,
check_status=check_status)
# Record memory status at end
memory_output = duthost.shell("show system-memory")["stdout"]
logger.info("Memory Status at end: %s", memory_output)
# Record orchagent CPU utilization at end
orch_cpu = duthost.shell(
"show processes cpu | grep orchagent | awk '{print $9}'")["stdout"]
logger.info("Orchagent CPU Util at end: %s", orch_cpu)
# Record Redis Memory at end
end_time_redis_memory = duthost.shell(
"redis-cli info memory | grep used_memory_human | sed -e 's/.*:\(.*\)M/\\1/'"
)["stdout"]
logger.info("Redis Memory at start: %s M", start_time_redis_memory)
logger.info("Redis Memory at end: %s M", end_time_redis_memory)
# Calculate diff in Redis memory
incr_redis_memory = float(end_time_redis_memory) - float(
start_time_redis_memory)
logger.info("Redis absolute difference: %d", incr_redis_memory)
# Check redis memory only if it is increased else default to pass
if incr_redis_memory > 0.0:
percent_incr_redis_memory = (incr_redis_memory /
float(start_time_redis_memory)) * 100
logger.info("Redis Memory percentage Increase: %d",
percent_incr_redis_memory)
pytest_assert(
percent_incr_redis_memory < 5,
"Redis Memory Increase more than expected: {}".format(
percent_incr_redis_memory))
# Orchagent CPU should consume < orch_cpu_threshold at last.
logger.info("watch orchagent CPU utilization when it goes below %d",
orch_cpu_threshold)
pytest_assert(
wait_until(45, 2, 0, check_orch_cpu_utilization, duthost,
orch_cpu_threshold),
"Orch CPU utilization {} > orch cpu threshold {} before link flap".
format(
duthost.shell(
"show processes cpu | grep orchagent | awk '{print $9}'")
["stdout"], orch_cpu_threshold))