def mock_server(fanouthosts, testbed_params, arp_responder, ptfadapter, duthost):
"""
Mock the presence of a server beneath a T0.
Returns:
A MockServer which will allow the caller to mock the behavior of
a server within a VLAN under a T0.
"""
server_dst_port = random.choice(testbed_params["vlan_ports"])
server_dst_addr = random.choice(arp_responder[server_dst_port].keys())
server_dst_intf = testbed_params["physical_port_map"][server_dst_port]
logging.info("Creating mock server with IP %s; dut port = %s, dut intf = %s",
server_dst_addr, server_dst_port, server_dst_intf)
logging.info("Clearing ARP and FDB tables for test setup")
duthost.command("sonic-clear fdb all")
duthost.command("sonic-clear arp")
# Populate FDB
logging.info("Populating FDB and ARP entry for mock server under VLAN")
# Issue a ping to populate ARP table on DUT
duthost.command('ping %s -c 3' % server_dst_addr, module_ignore_errors=True)
fanout_neighbor, fanout_intf = fanout_switch_port_lookup(fanouthosts, server_dst_intf)
return {"server_dst_port": server_dst_port,
"server_dst_addr": server_dst_addr,
"server_dst_intf": server_dst_intf,
"fanout_neighbor": fanout_neighbor,
"fanout_intf": fanout_intf}
def rif_port_down(duthosts, rand_one_dut_hostname, setup, fanouthosts, loganalyzer):
"""Shut RIF interface and return neighbor IP address attached to this interface."""
duthost = duthosts[rand_one_dut_hostname]
wait_after_ports_up = 30
if not setup["rif_members"]:
pytest.skip("RIF interface is absent")
rif_member_iface = setup["rif_members"].keys()[0]
vm_name = setup["mg_facts"]["minigraph_neighbors"][rif_member_iface].get("name", None)
pytest_assert(vm_name, 'Neighbor not found for RIF member "{}"'.format(rif_member_iface))
ip_dst = None
for item in setup["mg_facts"]["minigraph_bgp"]:
if item["name"] == vm_name and netaddr.valid_ipv4(item["addr"]):
ip_dst = item["addr"]
break
pytest_assert(ip_dst, 'Unable to find IP address for neighbor "{}"'.format(vm_name))
fanout_neighbor, fanout_intf = fanout_switch_port_lookup(fanouthosts, duthost.hostname, rif_member_iface)
loganalyzer[rand_one_dut_hostname].expect_regex = [LOG_EXPECT_PORT_OPER_DOWN_RE.format(rif_member_iface)]
with loganalyzer[rand_one_dut_hostname] as _:
fanout_neighbor.shutdown(fanout_intf)
time.sleep(1)
yield ip_dst
loganalyzer[rand_one_dut_hostname].expect_regex = [LOG_EXPECT_PORT_OPER_UP_RE.format(rif_member_iface)]
with loganalyzer[rand_one_dut_hostname] as _:
fanout_neighbor.no_shutdown(fanout_intf)
time.sleep(wait_after_ports_up)
def setup(self, test_data):
super(NeighVlanMemberDown, self).setup(test_data)
for port in self.ports:
fanout, fanport = fanout_switch_port_lookup(
self.fanouthosts, self.duthost.hostname, port)
if fanout and fanport:
fanout.shutdown(fanport)
def build_test_candidates(dut, fanouthosts, port, completeness_level=None):
"""
Find test candidates for link flap test.
Args:
dut: DUT host object
fanouthosts: List of fanout switch instances.
port: port, when port == 'unknown' or 'all_ports'
candidate will be all ports. A warning will
be generated if the port == 'unknown'.
caller can use 'all_ports' explicitly to mute
the warning.
completeness_level: Completeness level.
Returns:
A list of tuple with DUT's port, fanout port
and fanout
"""
candidates = []
if port not in ['unknown', 'all_ports']:
status = __get_dut_if_status(dut, port)
fanout, fanout_port = fanout_switch_port_lookup(
fanouthosts, dut.hostname, port)
__build_candidate_list(candidates, fanout, fanout_port, port, status)
else:
# Build the full list
if port == 'unknown':
logger.warning(
"Failed to get ports enumerated as parameter. Fall back to test all ports"
)
status = __get_dut_if_status(dut)
for dut_port in status.keys():
fanout, fanout_port = fanout_switch_port_lookup(
fanouthosts, dut.hostname, dut_port)
__build_candidate_list(candidates, fanout, fanout_port, dut_port,
status)
if completeness_level == 'debug':
candidates = random.sample(candidates, 1)
return candidates
def revert(self):
for port in self.ports:
if self.duthost.facts['platform'] == 'x86_64-kvm_x86_64-r0':
index = int(self.port_info.get(port).get("index")) + 1
self.vmhost.shell("virsh domif-setlink {} {}-{} up".format(
self.duthost.hostname, self.duthost.hostname, index))
fanout, fanport = fanout_switch_port_lookup(
self.fanouthosts, self.duthost.hostname, port)
if fanout and fanport:
fanout.no_shutdown(fanport)
def __build_test_candidates(self, dut, fanouthosts, port):
candidates = []
if port != 'unknown':
status = self.__get_dut_if_status(dut, port)
fanout, fanout_port = fanout_switch_port_lookup(fanouthosts, port)
self.__build_candidate_list(candidates, fanout, fanout_port, port, status)
else:
# Build the full list
logger.warning("Failed to get ports enumerated as parameter. Fall back to test all ports")
status = self.__get_dut_if_status(dut)
for dut_port in status.keys():
fanout, fanout_port = fanout_switch_port_lookup(fanouthosts, dut_port)
self.__build_candidate_list(candidates, fanout, fanout_port, dut_port, status)
if self.completeness_level == 'debug':
candidates = random.sample(candidates, 1)
return candidates
def setup(self, test_data):
super(NeighVlanMemberDown, self).setup(test_data)
for port in self.ports:
if self.duthost.facts['platform'] == 'x86_64-kvm_x86_64-r0':
index = int(self.port_info.get(port).get("index")) + 1
self.vmhost.shell("virsh domif-setlink {} {}-{} down".format(
self.duthost.hostname, self.duthost.hostname, index))
fanout, fanport = fanout_switch_port_lookup(
self.fanouthosts, self.duthost.hostname, port)
if fanout and fanport:
fanout.shutdown(fanport)
def _change_ports_state(self, bring_up):
for port in self.ports:
nbrname = self.dut_port_to_nbr[port]
nbrport = self.dut_port_to_nbr_port[port]
nbrhost = self.nbrhosts[nbrname]["host"]
if bring_up:
nbrhost.no_shutdown(nbrport)
else:
nbrhost.shutdown(nbrport)
fanout, fanport = fanout_switch_port_lookup(self.fanouthosts, self.duthost.hostname, port)
if bring_up:
fanout.no_shutdown(fanport)
else:
fanout.shutdown(fanport)
def __recover_interfaces(dut, fanouthosts, result, wait_time):
action = None
for port in result['down_ports']:
logging.warning("Restoring port: {}".format(port))
pn = str(port).lower()
if 'portchannel' in pn or 'vlan' in pn:
action = 'config_reload'
continue
fanout, fanout_port = fanout_switch_port_lookup(fanouthosts, port)
if fanout and fanout_port:
fanout.no_shutdown(fanout_port)
dut.no_shutdown(port)
wait(wait_time, msg="Wait {} seconds for interface(s) to restore.".format(wait_time))
return action
def __build_test_candidates(self, dut, fanouthosts, completeness_level):
status = self.__get_dut_if_status(dut)
candidates = []
for dut_port in status.keys():
fanout, fanout_port = fanout_switch_port_lookup(
fanouthosts, dut_port)
if not fanout or not fanout_port:
logging.info(
"Skipping port {} that is not found in connection graph".
format(dut_port))
elif status[dut_port]['admin_state'] == 'down':
logging.info(
"Skipping port {} that is admin down".format(dut_port))
else:
candidates.append((dut_port, fanout, fanout_port))
if CompletenessLevel.debug in completeness_level:
# Run the test for one port only - to just test if the test works fine
return candidates
return candidates
def mock_server(fanouthosts, testbed_params, arp_responder, ptfadapter,
duthost):
"""
Mock the presence of a server beneath a T0.
Returns:
A MockServer which will allow the caller to mock the behavior of
a server within a VLAN under a T0.
"""
server_dst_port = random.choice(testbed_params["vlan_ports"])
server_dst_addr = random.choice(arp_responder[server_dst_port].keys())
server_dst_intf = testbed_params["physical_port_map"][server_dst_port]
logging.info(
"Creating mock server with IP %s; dut port = %s, dut intf = %s",
server_dst_addr, server_dst_port, server_dst_intf)
logging.info("Clearing ARP and FDB tables for test setup")
duthost.command("sonic-clear fdb all")
duthost.command("sonic-clear arp")
# Populate FDB
logging.info("Populating FDB entry for mock server under VLAN")
src_mac = _hex_to_mac(arp_responder[server_dst_port][server_dst_addr])
pkt = _get_simple_ip_packet(src_mac,
duthost.get_dut_iface_mac(server_dst_intf),
server_dst_addr, MOCK_DEST_IP)
_send_packets(duthost, ptfadapter, pkt, server_dst_port, count=100)
fanout_neighbor, fanout_intf = fanout_switch_port_lookup(
fanouthosts, server_dst_intf)
return {
"server_dst_port": server_dst_port,
"server_dst_addr": server_dst_addr,
"server_dst_intf": server_dst_intf,
"fanout_neighbor": fanout_neighbor,
"fanout_intf": fanout_intf
}
def test_front_panel_linkflap_port(self, duthosts, enum_rand_one_per_hwsku_frontend_hostname, enum_asic_index,
all_cfg_facts,
fanouthosts, setup, teardown, nbrhosts, established_arp):
"""
Verify tables, databases, and kernel routes are correctly deleted when the front panel port flaps.
Test Steps
* Admin down interface on fanout to cause LOS on DUT.
* On local linecard:
* Verify ARP/NDP entries are removed from CLI for neighbors on down port.
* Verify table entries in ASIC, AppDb are removed for addresses on down port.
* On Supervisor card:
* Verify Chassis App DB entry are removed for only the cleared address. Entries for addresses on other line cards
should still be present.
* On remote linecards:
* Verify table entries in ASICDB, APPDB, and host ARP table are removed for cleared addresses.
* Verify kernel routes for cleared address are deleted.
* Admin interface up, verify recreation after restarting traffic.
Args:
duthosts: duthosts fixture.
enum_rand_one_per_hwsku_frontend_hostname: frontend iteration fixture.
enum_asic_index: asic iteration fixture.
all_cfg_facts: all_cfg_facts fixture from voq/conftest.py
fanouthosts: fanouthosts fixture.
setup: setup fixture for this module.
nbrhosts: nbrhosts fixture.
established_arp: Fixture to establish ARP to all neighbors.
"""
if fanouthosts == {}:
pytest.skip("Fanouthosts fixture did not return anything, this test case can not run.")
per_host = duthosts[enum_rand_one_per_hwsku_frontend_hostname]
asic = per_host.asics[enum_asic_index if enum_asic_index is not None else 0]
cfg_facts = all_cfg_facts[per_host.hostname][asic.asic_index]['ansible_facts']
if 'BGP_NEIGHBOR' in cfg_facts:
neighs = cfg_facts['BGP_NEIGHBOR']
else:
logger.info("No local neighbors for host: %s/%s, skipping", per_host.hostname, asic.asic_index)
return
intfs, intfs_to_test = pick_ports(cfg_facts)
logger.info("Will test interfaces: %s", intfs_to_test)
for intf in intfs_to_test:
local_ips = [i.split("/")[0] for i in intfs[intf].keys()] # [u'2064:100::2/64', u'100.0.0.2/24']
neighbors = [n for n in neighs if neighs[n]['local_addr'] in local_ips]
logger.info("Testing neighbors: %s on intf: %s", neighbors, intf)
if "portchannel" in intf.lower():
pc_cfg = cfg_facts['PORTCHANNEL_MEMBER']
pc_members = pc_cfg[intf]
logger.info("Portchannel members %s: %s", intf, pc_members.keys())
portbounce_list = pc_members.keys()
else:
portbounce_list = [intf]
try:
for lport in portbounce_list:
fanout, fanport = fanout_switch_port_lookup(fanouthosts, per_host.hostname, lport)
logger.info("fanout port: %s %s, host: %s", fanout, fanport, fanout.host)
self.linkflap_down(fanout, fanport, per_host, lport)
for neighbor in neighbors:
check_one_neighbor_present(duthosts, per_host, asic, neighbor, nbrhosts, all_cfg_facts)
finally:
for lport in portbounce_list:
fanout, fanport = fanout_switch_port_lookup(fanouthosts, per_host.hostname, lport)
self.linkflap_up(fanout, fanport, per_host, lport)
for neighbor in neighbors:
check_one_neighbor_present(duthosts, per_host, asic, neighbor, nbrhosts, all_cfg_facts)
def test_link_flap(request, duthosts, rand_one_dut_hostname, tbinfo,
fanouthosts, get_loop_times):
"""
Validates that link flap works as expected
"""
duthost = duthosts[rand_one_dut_hostname]
orch_cpu_threshold = request.config.getoption("--orch_cpu_threshold")
# Record memory status at start
memory_output = duthost.shell("show system-memory")["stdout"]
logger.info("Memory Status at start: %s", memory_output)
# Record Redis Memory at start
start_time_redis_memory = duthost.shell(
"redis-cli info memory | grep used_memory_human | sed -e 's/.*:\(.*\)M/\\1/'"
)["stdout"]
logger.info("Redis Memory: %s M", start_time_redis_memory)
# Make Sure Orch CPU < orch_cpu_threshold before starting test.
logger.info(
"Make Sure orchagent CPU utilization is less that %d before link flap",
orch_cpu_threshold)
pytest_assert(
wait_until(100, 2, 0, check_orch_cpu_utilization, duthost,
orch_cpu_threshold),
"Orch CPU utilization {} > orch cpu threshold {} before link flap".
format(
duthost.shell(
"show processes cpu | grep orchagent | awk '{print $9}'")
["stdout"], orch_cpu_threshold))
loop_times = get_loop_times
port_lists = get_port_list(duthost, tbinfo)
candidates = []
for port in port_lists:
fanout, fanout_port = fanout_switch_port_lookup(
fanouthosts, duthost.hostname, port)
candidates.append((port, fanout, fanout_port))
for loop_time in range(0, loop_times):
watch = False
check_status = False
if loop_time == 0 or loop_time == loop_times - 1:
watch = True
check_status = True
for dut_port, fanout, fanout_port in candidates:
toggle_one_link(duthost,
dut_port,
fanout,
fanout_port,
watch=watch,
check_status=check_status)
# Record memory status at end
memory_output = duthost.shell("show system-memory")["stdout"]
logger.info("Memory Status at end: %s", memory_output)
# Record orchagent CPU utilization at end
orch_cpu = duthost.shell(
"show processes cpu | grep orchagent | awk '{print $9}'")["stdout"]
logger.info("Orchagent CPU Util at end: %s", orch_cpu)
# Record Redis Memory at end
end_time_redis_memory = duthost.shell(
"redis-cli info memory | grep used_memory_human | sed -e 's/.*:\(.*\)M/\\1/'"
)["stdout"]
logger.info("Redis Memory at start: %s M", start_time_redis_memory)
logger.info("Redis Memory at end: %s M", end_time_redis_memory)
# Calculate diff in Redis memory
incr_redis_memory = float(end_time_redis_memory) - float(
start_time_redis_memory)
logger.info("Redis absolute difference: %d", incr_redis_memory)
# Check redis memory only if it is increased else default to pass
if incr_redis_memory > 0.0:
percent_incr_redis_memory = (incr_redis_memory /
float(start_time_redis_memory)) * 100
logger.info("Redis Memory percentage Increase: %d",
percent_incr_redis_memory)
pytest_assert(
percent_incr_redis_memory < 5,
"Redis Memory Increase more than expected: {}".format(
percent_incr_redis_memory))
# Orchagent CPU should consume < orch_cpu_threshold at last.
logger.info("watch orchagent CPU utilization when it goes below %d",
orch_cpu_threshold)
pytest_assert(
wait_until(45, 2, 0, check_orch_cpu_utilization, duthost,
orch_cpu_threshold),
"Orch CPU utilization {} > orch cpu threshold {} before link flap".
format(
duthost.shell(
"show processes cpu | grep orchagent | awk '{print $9}'")
["stdout"], orch_cpu_threshold))
def revert(self):
for port in self.ports:
fanout, fanport = fanout_switch_port_lookup(
self.fanouthosts, self.duthost.hostname, port)
if fanout and fanport:
fanout.no_shutdown(fanport)
