def test_neighbor_clear_all(duthosts, enum_rand_one_per_hwsku_frontend_hostname, enum_asic_index,
setup, teardown, nbrhosts, all_cfg_facts, nbr_macs, established_arp):
"""
Verify tables, databases, and kernel routes are correctly deleted when the entire neighbor table is cleared.
Test Steps
* On local linecard:
* Issue `sonic-clear arp` command. and verify all addresses are removed and kernel routes are deleted on
all hosts and ASICs.
* Verify ARP/NDP entries are removed from CLI.
* Verify table entries in ASIC, AppDb are removed for all cleared addresses.
* On Supervisor card:
* Verify Chassis App DB entry are removed for only the cleared address. Entries for addresses on other line
cards should still be present.
* On remote linecards:
* Verify table entries in ASICDB, APPDB, and host ARP table are removed for cleared addresses.
* Verify kernel routes for cleared address are deleted.
* Send full mesh traffic and verify relearn and DB.
Args:
duthosts: duthosts fixture.
setup: setup fixture for this module.
nbrhosts: nbrhosts fixture.
all_cfg_facts: all_cfg_facts fixture from voq/conftest.py
established_arp: Fixture to establish ARP to all neighbors.
"""
per_host = duthosts[enum_rand_one_per_hwsku_frontend_hostname]
asic = per_host.asics[enum_asic_index if enum_asic_index is not None else 0]
cfg_facts = all_cfg_facts[per_host.hostname][asic.asic_index]['ansible_facts']
if 'BGP_NEIGHBOR' in cfg_facts:
neighs = cfg_facts['BGP_NEIGHBOR']
else:
logger.info("No local neighbors for host: %s/%s, skipping", per_host.hostname, asic.asic_index)
return
asic_cmd(asic, "sonic-clear arp")
asic_cmd(asic, "sonic-clear ndp")
logger.info("Wait for clear.")
poll_neighbor_table_delete(duthosts, neighs)
logger.info("Verify neighbors are gone.")
check_neighbors_are_gone(duthosts, all_cfg_facts, per_host, asic, neighs.keys())
# relearn and check
logger.info("Relearn neighbors on all nodes")
ping_all_dut_local_nbrs(duthosts)
check_all_neighbors_present(duthosts, nbrhosts, all_cfg_facts, nbr_macs)
def test_neighbor_hw_mac_change(duthosts, enum_rand_one_per_hwsku_frontend_hostname, enum_asic_index,
setup, teardown, nbrhosts, all_cfg_facts, nbr_macs, established_arp):
"""
Verify tables, databases, and kernel routes are correctly updated when the MAC address of a neighbor changes
and is updated via request/reply exchange.
Test Steps
* Change the MAC address on a remote host that is already present in the ARP table.
* Without clearing the entry in the DUT, allow the existing entry to time out and the new reply to have the new MAC
address.
* On local linecard:
* Verify table entries in local ASIC, APP, and host ARP table are updated with new MAC.
* On supervisor card:
* Verify Chassis App DB entry is correct for with the updated MAC address.
* On remote linecards:
* Verify table entries in remote hosts/ASICs in APPDB, and host ARP table are still present with inband MAC
address
* Verify ASIC DB is updated with new MAC.
* Verify kernel route in remote hosts are still present to inband port.
* Verify that packets can be sent from local and remote linecards to the learned address.
Args:
duthosts: duthosts fixture.
enum_rand_one_per_hwsku_frontend_hostname: frontend iteration fixture.
enum_asic_index: asic iteration fixture.
setup: setup fixture for this module.
nbrhosts: nbrhosts fixture.
established_arp: Fixture to establish arp on all nodes
all_cfg_facts: all_cfg_facts fixture from voq/conftest.py
"""
per_host = duthosts[enum_rand_one_per_hwsku_frontend_hostname]
asic = per_host.asics[enum_asic_index if enum_asic_index is not None else 0]
cfg_facts = all_cfg_facts[per_host.hostname][asic.asic_index]['ansible_facts']
if 'BGP_NEIGHBOR' in cfg_facts:
neighs = cfg_facts['BGP_NEIGHBOR']
else:
logger.info("No local neighbors for host: %s/%s, skipping", per_host.hostname, asic.asic_index)
return
eth_cfg = cfg_facts['INTERFACE'] if 'INTERFACE' in cfg_facts else {}
if eth_cfg == {}:
pytest.skip("Can't run this test without any IP interfaces on ethernet ports")
eth_ports = [intf for intf in eth_cfg]
local_port = random.choice(eth_ports)
logger.info("We will test port: %s on host %s, asic %s", local_port, per_host.hostname, asic.asic_index)
nbr_to_test = []
for neighbor in neighs:
local_ip = neighs[neighbor]['local_addr']
nbr_port = get_port_by_ip(cfg_facts, local_ip)
if local_port == nbr_port:
nbr_to_test.append(neighbor)
logger.info("We will test neighbors: %s", nbr_to_test)
nbrinfo = get_neighbor_info(nbr_to_test[0], nbrhosts)
original_mac = nbrinfo['mac']
for neighbor in nbr_to_test:
# Check neighbor on local linecard
logger.info("*" * 60)
logger.info("Verify initial neighbor: %s, port %s", neighbor, local_port)
pytest_assert(wait_until(60, 2, check_arptable_mac, per_host, asic, neighbor, original_mac, checkstate=False),
"MAC {} didn't change in ARP table".format(original_mac))
sonic_ping(asic, neighbor, verbose=True)
pytest_assert(wait_until(60, 2, check_arptable_mac, per_host, asic, neighbor, original_mac),
"MAC {} didn't change in ARP table".format(original_mac))
dump_and_verify_neighbors_on_asic(duthosts, per_host, asic, nbr_to_test, nbrhosts, all_cfg_facts, nbr_macs)
try:
logger.info("Changing ethernet mac on port %s, vm %s", nbrinfo['shell_intf'], nbrinfo['vm'])
change_mac(nbrhosts[nbrinfo['vm']], nbrinfo['shell_intf'], NEW_MAC)
for neighbor in nbr_to_test:
if ":" in neighbor:
logger.info("Force neighbor solicitation to workaround long IPV6 timer.")
asic_cmd(asic, "ndisc6 %s %s" % (neighbor, local_port))
pytest_assert(wait_until(60, 2, check_arptable_mac, per_host, asic, neighbor, NEW_MAC, checkstate=False),
"MAC {} didn't change in ARP table".format(NEW_MAC))
sonic_ping(asic, neighbor, verbose=True)
pytest_assert(wait_until(60, 2, check_arptable_mac, per_host, asic, neighbor, NEW_MAC),
"MAC {} didn't change in ARP table".format(NEW_MAC))
logger.info("Verify neighbor after mac change: %s, port %s", neighbor, local_port)
check_one_neighbor_present(duthosts, per_host, asic, neighbor, nbrhosts, all_cfg_facts)
logger.info("Ping neighbors: %s from all line cards", nbr_to_test)
ping_all_neighbors(duthosts, all_cfg_facts, nbr_to_test)
finally:
logger.info("-" * 60)
logger.info("Will Restore ethernet mac on port %s, vm %s", nbrinfo['shell_intf'], nbrinfo['vm'])
change_mac(nbrhosts[nbrinfo['vm']], nbrinfo['shell_intf'], original_mac)
for neighbor in nbr_to_test:
if ":" in neighbor:
logger.info("Force neighbor solicitation to workaround long IPV6 timer.")
asic_cmd(asic, "ndisc6 %s %s" % (neighbor, local_port))
pytest_assert(
wait_until(60, 2, check_arptable_mac, per_host, asic, neighbor, original_mac, checkstate=False),
"MAC {} didn't change in ARP table".format(original_mac))
sonic_ping(asic, neighbor, verbose=True)
pytest_assert(wait_until(60, 2, check_arptable_mac, per_host, asic, neighbor, original_mac),
"MAC {} didn't change in ARP table".format(original_mac))
dump_and_verify_neighbors_on_asic(duthosts, per_host, asic, nbr_to_test, nbrhosts, all_cfg_facts, nbr_macs)
ping_all_neighbors(duthosts, all_cfg_facts, nbr_to_test)
def test_neighbor_clear_one(duthosts, enum_rand_one_per_hwsku_frontend_hostname, enum_asic_index,
setup, teardown, nbrhosts, all_cfg_facts, nbr_macs, established_arp):
"""
Verify tables, databases, and kernel routes are correctly deleted when a single neighbor adjacency is cleared.
Test Steps
* On local linecard:
* Clear single address with command: `ip neigh flush to "addr"`.
* Verify ARP/NDP entry removed from CLI.
* Verify table entries in ASIC, AppDb are removed for only the cleared address.
* On Supervisor card:
* Verify Chassis App DB entry are removed for only the cleared address.
* On remote linecards:
* Verify table entries in ASICDB, APPDB, and host ARP table are removed.
* Verify kernel route for cleared address is deleted.
* Restart traffic, verify relearn.
Args:
duthosts: duthosts fixture.
setup: setup fixture for this module.
nbrhosts: nbrhosts fixture.
all_cfg_facts: all_cfg_facts fixture from voq/conftest.py
established_arp: Fixture to establish ARP to all neighbors.
"""
per_host = duthosts[enum_rand_one_per_hwsku_frontend_hostname]
asic = per_host.asics[enum_asic_index if enum_asic_index is not None else 0]
cfg_facts = all_cfg_facts[per_host.hostname][asic.asic_index]['ansible_facts']
if 'BGP_NEIGHBOR' in cfg_facts:
neighs = cfg_facts['BGP_NEIGHBOR']
else:
logger.info("No local neighbors for host: %s/%s, skipping", per_host.hostname, asic.asic_index)
return
eth_cfg = cfg_facts['INTERFACE'] if 'INTERFACE' in cfg_facts else {}
pos_cfg = cfg_facts['PORTCHANNEL_INTERFACE'] if 'PORTCHANNEL_INTERFACE' in cfg_facts else {}
nbr_to_test = []
if eth_cfg != {}:
nbr_to_test.extend(select_neighbors(eth_cfg, cfg_facts))
if pos_cfg != {}:
nbr_to_test.extend(select_neighbors(pos_cfg, cfg_facts))
untouched_nbrs = [nbr for nbr in neighs if nbr not in nbr_to_test]
logger.info("We will test these neighbors: %s", nbr_to_test)
logger.info("These neighbors should not be affected: %s", untouched_nbrs)
for neighbor in nbr_to_test:
logger.info(
"Flushing neighbor: {} on host {}/{}".format(neighbor, per_host.hostname, asic.asic_index))
asic_cmd(asic, "ip neigh flush to %s" % neighbor)
logger.info("Wait for flush.")
poll_neighbor_table_delete(duthosts, nbr_to_test)
logger.info("Verify neighbors are gone.")
check_neighbors_are_gone(duthosts, all_cfg_facts, per_host, asic, nbr_to_test)
logger.info("Verify other neighbors are not affected.")
dump_and_verify_neighbors_on_asic(duthosts, per_host, asic, untouched_nbrs, nbrhosts, all_cfg_facts, nbr_macs)
# relearn and check
logger.info("Relearn neighbors on all nodes")
ping_all_dut_local_nbrs(duthosts)
logger.info("Check neighbor relearn on all nodes.")
check_all_neighbors_present(duthosts, nbrhosts, all_cfg_facts, nbr_macs)
def setup(duthosts, nbrhosts, all_cfg_facts):
"""
Setup fixture to disable all neighbors on DUT and VMs.
Args:
duthosts: Duthosts fixture
nbrhosts: Nbrhosts fixture
all_cfg_facts: all_cfg_facts fixture from voq/conftest.py
"""
@reset_ansible_local_tmp
def disable_dut_bgp_neighs(cfg_facts, node=None, results=None):
"""Target function do disable BGP neighbors on sonic DUTs.
Args:
cfg_facts: instance of fixture from voq/conftest.py
node (object, optional): A value item of the dict type fixture 'nbrhosts'. Defaults to None.
results (Proxy to shared dict, optional): An instance of multiprocessing.Manager().dict(). Proxy to a dict
shared by all processes for returning execution results. Defaults to None
"""
if node is None or results is None:
logger.error('Missing kwarg "node" or "results"')
return
node_results = []
for asic in node.asics:
asic_cfg_facts = cfg_facts[node.hostname][asic.asic_index]['ansible_facts']
if 'BGP_NEIGHBOR' not in asic_cfg_facts:
continue
for neighbor in asic_cfg_facts['BGP_NEIGHBOR']:
logger.info(
"Shut down neighbor: {} on host {} asic {}".format(neighbor, node.hostname, asic.asic_index))
node_results.append(node.command("sudo config bgp shutdown neighbor {}".format(neighbor)))
results[node.hostname] = node_results
parallel_run(disable_dut_bgp_neighs, [all_cfg_facts], {}, duthosts.frontend_nodes, timeout=120)
# disable bgp neighbors on vms
@reset_ansible_local_tmp
def disable_nbr_bgp_neighs(node=None, results=None):
"""Target function to disable bgp neighbors on VMS.
Args:
node (object, optional): A value item of the dict type fixture 'nbrhosts'. Defaults to None.
results (Proxy to shared dict, optional): An instance of multiprocessing.Manager().dict(). Proxy to a dict
shared by all processes for returning execution results. Defaults to None.
"""
if node is None or results is None:
logger.error('Missing kwarg "node" or "results"')
return
node_results = []
logger.info(
'disable neighbors {} on neighbor host {}'.format(node['conf']['bgp']['peers'], node['host'].hostname))
for peer in node['conf']['bgp']['peers']:
for neighbor in node['conf']['bgp']['peers'][peer]:
node_results.append(node['host'].eos_config(
lines=["neighbor %s shutdown" % neighbor],
parents=['router bgp {}'.format(node['conf']['bgp']['asn'])],
module_ignore_errors=True)
)
if ":" in neighbor:
node_results.append(node['host'].eos_config(
lines=["ipv6 route ::/0 %s " % neighbor],
module_ignore_errors=True)
)
else:
node_results.append(node['host'].eos_config(
lines=["ip route 0.0.0.0/0 %s " % neighbor],
module_ignore_errors=True)
)
results[node['host'].hostname] = node_results
parallel_run(disable_nbr_bgp_neighs, [], {}, nbrhosts.values(), timeout=120)
logger.info("Poll for routes to be gone.")
endtime = time.time() + 120
for dut in duthosts.frontend_nodes:
for asc in dut.asics:
routes = len(asic_cmd(asc, 'redis-cli -n 0 KEYS ROUTE_TABLE*')['stdout_lines'])
logger.info("Found %d routes in appdb on %s/%s", routes, dut.hostname, asc.asic_index)
while routes > 1000:
time.sleep(5)
routes = len(asic_cmd(asc, 'redis-cli -n 0 KEYS ROUTE_TABLE*')['stdout_lines'])
logger.info("Found %d routes in appdb on %s/%s, polling", routes, dut.hostname, asc.asic_index)
if time.time() > endtime:
break
routes = len(asic_cmd(asc, 'redis-cli -n 1 KEYS *ROUTE_ENTRY*')['stdout_lines'])
logger.info("Found %d routes in asicdb on %s/%s", routes, dut.hostname, asc.asic_index)
while routes > 1000:
time.sleep(5)
routes = len(asic_cmd(asc, 'redis-cli -n 1 KEYS *ROUTE_ENTRY*')['stdout_lines'])
logger.info("Found %d routes in asicdb on %s/%s, polling", routes, dut.hostname, asc.asic_index)
if time.time() > endtime:
break
def test_gratarp_macchange(self, duthosts, enum_rand_one_per_hwsku_frontend_hostname, enum_asic_index,
ptfhost, tbinfo, nbrhosts, setup, teardown, all_cfg_facts, established_arp):
"""
Verify tables, databases, and kernel routes are correctly updated when a unsolicited ARP packet changes
the MAC address of learned neighbor.
Test Steps
* Send unsolicited ARP packet into DUT for an IP known by DUT with a different MAC address for the neighbor.
* Change the MAC address of the neighbor VM.
* On local linecard:
* Verify table entries in local ASIC, APP, and host ARP table are updated with new MAC.
* On supervisor card:
* Verify Chassis App DB entry is correct for with the updated MAC address.
* On remote linecards:
* Verify table entries in remote hosts/ASICs in APPDB, and host ARP table are still present with inband MAC
address
* Verify ASIC DB is updated with new MAC.
* Verify kernel route in remote hosts are still present to inband port.
* Verify that packets can be sent from local and remote linecards to learned address.
Args:
duthosts: The duthosts fixture
enum_rand_one_per_hwsku_frontend_hostname: frontend enumeration fixture
enum_asic_index: asic enumeration fixture
ptfhost: The ptfhost fixure.
tbinfo: The tbinfo fixture
nbrhosts: The nbrhosts fixture.
setup: The setup fixture from this module.
established_arp: The established_arp fixture from this module.
all_cfg_facts: The all_cfg_facts fixture from voq/contest.py
"""
self.ptfhost = ptfhost
duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname]
asic = duthost.asics[enum_asic_index if enum_asic_index is not None else 0]
cfg_facts = all_cfg_facts[duthost.hostname][asic.asic_index]['ansible_facts']
if 'BGP_NEIGHBOR' in cfg_facts:
neighs = cfg_facts['BGP_NEIGHBOR']
else:
logger.info("No local neighbors for host: %s/%s, skipping", duthost.hostname, asic.asic_index)
return
eth_cfg = cfg_facts['INTERFACE'] if 'INTERFACE' in cfg_facts else {}
if eth_cfg == {}:
pytest.skip("Can't run this test without any IP interfaces on ethernet ports")
eth_ports = [intf for intf in eth_cfg]
local_port = random.choice(eth_ports)
logger.info("We will test port: %s on host %s, asic %s", local_port, duthost.hostname, asic.asic_index)
nbr_to_test = []
for neighbor in neighs:
local_ip = neighs[neighbor]['local_addr']
nbr_port = get_port_by_ip(cfg_facts, local_ip)
if local_port == nbr_port:
nbr_to_test.append(neighbor)
logger.info("We will test neighbors: %s", nbr_to_test)
for neighbor in nbr_to_test:
nbrinfo = get_neighbor_info(neighbor, nbrhosts)
tb_port = get_ptf_port(duthosts,
all_cfg_facts[duthost.hostname][asic.asic_index]['ansible_facts'],
tbinfo, duthost, local_port)[0]
original_mac = nbrinfo['mac']
logger.info("*" * 60)
logger.info("Verify initial neighbor: %s, port %s", neighbor, local_port)
logger.info("%s port %s is on ptf port: %s", duthost.hostname, local_port, tb_port)
logger.info("-" * 60)
sonic_ping(asic, neighbor)
pytest_assert(wait_until(60, 2, check_arptable_mac, duthost, asic, neighbor, original_mac),
"MAC {} didn't change in ARP table".format(original_mac))
check_one_neighbor_present(duthosts, duthost, asic, neighbor, nbrhosts, all_cfg_facts)
try:
change_mac(nbrhosts[nbrinfo['vm']], nbrinfo['shell_intf'], NEW_MAC)
self.send_grat_pkt(NEW_MAC, neighbor, int(tb_port))
pytest_assert(wait_until(60, 2, check_arptable_mac, duthost, asic, neighbor, NEW_MAC, checkstate=False),
"MAC {} didn't change in ARP table of neighbor {}".format(NEW_MAC, neighbor))
try:
sonic_ping(asic, neighbor)
except AssertionError:
logging.info("No initial response from ping, begin poll to see if ARP table responds.")
pytest_assert(wait_until(60, 2, check_arptable_mac, duthost, asic, neighbor, NEW_MAC, checkstate=True),
"MAC {} didn't change in ARP table of neighbor {}".format(NEW_MAC, neighbor))
check_one_neighbor_present(duthosts, duthost, asic, neighbor, nbrhosts, all_cfg_facts)
ping_all_neighbors(duthosts, all_cfg_facts, [neighbor])
finally:
logger.info("Will Restore ethernet mac on neighbor: %s, port %s, vm %s", neighbor,
nbrinfo['shell_intf'], nbrinfo['vm'])
change_mac(nbrhosts[nbrinfo['vm']], nbrinfo['shell_intf'], original_mac)
if ":" in neighbor:
logger.info("Force neighbor solicitation to workaround long IPV6 timer.")
asic_cmd(asic, "ndisc6 %s %s" % (neighbor, local_port))
pytest_assert(
wait_until(60, 2, check_arptable_mac, duthost, asic, neighbor, original_mac, checkstate=False),
"MAC {} didn't change in ARP table".format(original_mac))
sonic_ping(asic, neighbor, verbose=True)
pytest_assert(wait_until(60, 2, check_arptable_mac, duthost, asic, neighbor, original_mac),
"MAC {} didn't change in ARP table".format(original_mac))
check_one_neighbor_present(duthosts, duthost, asic, neighbor, nbrhosts, all_cfg_facts)
ping_all_neighbors(duthosts, all_cfg_facts, [neighbor])