def check_intfs_and_nbrs(duthosts, all_cfg_facts, nbrhosts, nbr_macs):
"""
Checks interfaces and neighbors are correct on frontend nodes and chassisdb.
Args:
duthosts: duthosts fixture
all_cfg_facts: all_cfg_facts fixture
nbrhosts: nbrhosts fixture
nbr_macs: nbr_macs fixture
"""
for host in duthosts.frontend_nodes:
for asic in host.asics:
cfg_facts = all_cfg_facts[host.hostname][
asic.asic_index]['ansible_facts']
check_voq_interfaces(duthosts, host, asic, cfg_facts)
logger.info("Checking local neighbors on host: %s, asic: %s",
host.hostname, asic.asic_index)
if 'BGP_NEIGHBOR' in cfg_facts:
neighs = cfg_facts['BGP_NEIGHBOR']
else:
logger.info("No local neighbors for host: %s/%s, skipping",
host.hostname, asic.asic_index)
continue
dump_and_verify_neighbors_on_asic(duthosts, host, asic, neighs,
nbrhosts, all_cfg_facts,
nbr_macs)
def test_front_panel_admindown_port(self, duthosts, enum_rand_one_per_hwsku_frontend_hostname, enum_asic_index,
all_cfg_facts, setup, nbrhosts, nbr_macs, established_arp):
"""
Verify tables, databases, and kernel routes are correctly deleted when the DUT port is admin down/up.
Test Steps
* Admin down interface on DUT.
* On local linecard:
* Verify ARP/NDP entries are removed from CLI for neighbors on down port.
* Verify table entries in ASIC, AppDb are removed for addresses on down port.
* On Supervisor card:
* Verify Chassis App DB entry are removed for only the cleared address. Entries for addresses on other
line cards should still be present.
* On remote linecards:
* Verify table entries in ASICDB, APPDB, and host ARP table are removed for cleared addresses.
* Verify kernel routes for cleared address are deleted.
* Admin interface up, verify recreation after restarting traffic.
Args:
duthosts: duthosts fixture.
enum_rand_one_per_hwsku_frontend_hostname: frontend iteration fixture.
enum_asic_index: asic iteration fixture.
all_cfg_facts: all_cfg_facts fixture from voq/conftest.py
setup: setup fixture for this module.
nbrhosts: nbrhosts fixture.
established_arp: Fixture to establish ARP to all neighbors.
"""
per_host = duthosts[enum_rand_one_per_hwsku_frontend_hostname]
asic = per_host.asics[enum_asic_index if enum_asic_index is not None else 0]
cfg_facts = all_cfg_facts[per_host.hostname][asic.asic_index]['ansible_facts']
neighs = cfg_facts['BGP_NEIGHBOR']
intfs, intfs_to_test = pick_ports(cfg_facts)
logger.info("Will test interfaces: %s", intfs_to_test)
for intf in intfs_to_test:
local_ips = [i.split("/")[0] for i in intfs[intf].keys()] # [u'2064:100::2/64', u'100.0.0.2/24']
neighbors = [n for n in neighs if neighs[n]['local_addr'] in local_ips]
logger.info("Testing neighbors: %s on intf: %s", neighbors, intf)
self.localport_admindown(per_host, intf)
try:
check_neighbors_are_gone(duthosts, all_cfg_facts, per_host, asic, neighbors)
finally:
self.localport_adminup(per_host, intf)
for neighbor in neighbors:
sonic_ping(asic, neighbor)
for neighbor in neighbors:
pytest_assert(wait_until(60, 2, check_arptable_state, per_host, neighbor, "REACHABLE"),
"STATE for neighbor {} did not change to reachable".format(neighbor))
dump_and_verify_neighbors_on_asic(duthosts, per_host, asic, neighbors, nbrhosts, all_cfg_facts, nbr_macs)
def test_voq_neighbor_create(duthosts, enum_frontend_dut_hostname,
enum_asic_index, nbrhosts, all_cfg_facts,
nbr_macs):
"""
Verify neighbor entries are created on linecards for local and remote VMS.
For local neighbors:
* ARP/NDP should be resolved when BGP to adjacent VMs is established.
* On local linecard, verify ASIC DB entries.
* MAC address matches MAC of neighbor VM.
* Router interface OID matches back to the correct interface and port the neighbor was learned on.
* On local linecard, verify show arp/ndp, ip neigh commands.
* MAC address matches MAC of neighbor VM.
* On local linecard. verify neighbor table in appDB.
* MAC address matches MAC of neighbor VM.
* On supervisor card, verify SYSTEM_NEIGH table in Chassis AppDB (redis-dump -h <ip> -p 6380 -d 12 on supervisor).
* Verify encap index and MAC address match between ASICDB the Chassis AppDB
* Repeat with IPv4, IPv6, dual-stack.
For remote neighbors:
* When local neighbors are established as in the Local Neighbor testcase, corresponding entries will be established
on all other line cards. On each remote card, verify:
* Verify ASIC DB entries on remote linecards.
* Verify impose index=True in ASIC DB.
* Verify MAC address in ASIC DB is the remote neighbor mac.
* Verify encap index for ASIC DB entry matches Chassis App DB.
* Verify router interface OID matches the interface the neighbor was learned on.
* Verify on linecard CLI, show arp/ndp, ip neigh commands.
* For inband port, MAC should be inband port mac in kernel table and LC appDb.
* For inband vlan mode, MAC will be remote ASIC mac in kernel table and LC appdb.
* Verify neighbor table in linecard appdb.
* Verify static route is installed in kernel routing table with /32 (or /128 for IPv6) for neighbor entry.
* Repeat with IPv4, IPv6, dual-stack.
"""
per_host = duthosts[enum_frontend_dut_hostname]
asic = per_host.asics[
enum_asic_index if enum_asic_index is not None else 0]
cfg_facts = all_cfg_facts[per_host.hostname][
asic.asic_index]['ansible_facts']
logger.info("Checking local neighbors on host: %s, asic: %s",
per_host.hostname, asic.asic_index)
if 'BGP_NEIGHBOR' in cfg_facts:
neighs = cfg_facts['BGP_NEIGHBOR']
else:
logger.info("No local neighbors for host: %s/%s, skipping",
per_host.hostname, asic.asic_index)
return
dump_and_verify_neighbors_on_asic(duthosts, per_host, asic, neighs,
nbrhosts, all_cfg_facts, nbr_macs)
def test_neighbor_hw_mac_change(duthosts, enum_rand_one_per_hwsku_frontend_hostname, enum_asic_index,
setup, teardown, nbrhosts, all_cfg_facts, nbr_macs, established_arp):
"""
Verify tables, databases, and kernel routes are correctly updated when the MAC address of a neighbor changes
and is updated via request/reply exchange.
Test Steps
* Change the MAC address on a remote host that is already present in the ARP table.
* Without clearing the entry in the DUT, allow the existing entry to time out and the new reply to have the new MAC
address.
* On local linecard:
* Verify table entries in local ASIC, APP, and host ARP table are updated with new MAC.
* On supervisor card:
* Verify Chassis App DB entry is correct for with the updated MAC address.
* On remote linecards:
* Verify table entries in remote hosts/ASICs in APPDB, and host ARP table are still present with inband MAC
address
* Verify ASIC DB is updated with new MAC.
* Verify kernel route in remote hosts are still present to inband port.
* Verify that packets can be sent from local and remote linecards to the learned address.
Args:
duthosts: duthosts fixture.
enum_rand_one_per_hwsku_frontend_hostname: frontend iteration fixture.
enum_asic_index: asic iteration fixture.
setup: setup fixture for this module.
nbrhosts: nbrhosts fixture.
established_arp: Fixture to establish arp on all nodes
all_cfg_facts: all_cfg_facts fixture from voq/conftest.py
"""
per_host = duthosts[enum_rand_one_per_hwsku_frontend_hostname]
asic = per_host.asics[enum_asic_index if enum_asic_index is not None else 0]
cfg_facts = all_cfg_facts[per_host.hostname][asic.asic_index]['ansible_facts']
if 'BGP_NEIGHBOR' in cfg_facts:
neighs = cfg_facts['BGP_NEIGHBOR']
else:
logger.info("No local neighbors for host: %s/%s, skipping", per_host.hostname, asic.asic_index)
return
eth_cfg = cfg_facts['INTERFACE'] if 'INTERFACE' in cfg_facts else {}
if eth_cfg == {}:
pytest.skip("Can't run this test without any IP interfaces on ethernet ports")
eth_ports = [intf for intf in eth_cfg]
local_port = random.choice(eth_ports)
logger.info("We will test port: %s on host %s, asic %s", local_port, per_host.hostname, asic.asic_index)
nbr_to_test = []
for neighbor in neighs:
local_ip = neighs[neighbor]['local_addr']
nbr_port = get_port_by_ip(cfg_facts, local_ip)
if local_port == nbr_port:
nbr_to_test.append(neighbor)
logger.info("We will test neighbors: %s", nbr_to_test)
nbrinfo = get_neighbor_info(nbr_to_test[0], nbrhosts)
original_mac = nbrinfo['mac']
for neighbor in nbr_to_test:
# Check neighbor on local linecard
logger.info("*" * 60)
logger.info("Verify initial neighbor: %s, port %s", neighbor, local_port)
pytest_assert(wait_until(60, 2, check_arptable_mac, per_host, asic, neighbor, original_mac, checkstate=False),
"MAC {} didn't change in ARP table".format(original_mac))
sonic_ping(asic, neighbor, verbose=True)
pytest_assert(wait_until(60, 2, check_arptable_mac, per_host, asic, neighbor, original_mac),
"MAC {} didn't change in ARP table".format(original_mac))
dump_and_verify_neighbors_on_asic(duthosts, per_host, asic, nbr_to_test, nbrhosts, all_cfg_facts, nbr_macs)
try:
logger.info("Changing ethernet mac on port %s, vm %s", nbrinfo['shell_intf'], nbrinfo['vm'])
change_mac(nbrhosts[nbrinfo['vm']], nbrinfo['shell_intf'], NEW_MAC)
for neighbor in nbr_to_test:
if ":" in neighbor:
logger.info("Force neighbor solicitation to workaround long IPV6 timer.")
asic_cmd(asic, "ndisc6 %s %s" % (neighbor, local_port))
pytest_assert(wait_until(60, 2, check_arptable_mac, per_host, asic, neighbor, NEW_MAC, checkstate=False),
"MAC {} didn't change in ARP table".format(NEW_MAC))
sonic_ping(asic, neighbor, verbose=True)
pytest_assert(wait_until(60, 2, check_arptable_mac, per_host, asic, neighbor, NEW_MAC),
"MAC {} didn't change in ARP table".format(NEW_MAC))
logger.info("Verify neighbor after mac change: %s, port %s", neighbor, local_port)
check_one_neighbor_present(duthosts, per_host, asic, neighbor, nbrhosts, all_cfg_facts)
logger.info("Ping neighbors: %s from all line cards", nbr_to_test)
ping_all_neighbors(duthosts, all_cfg_facts, nbr_to_test)
finally:
logger.info("-" * 60)
logger.info("Will Restore ethernet mac on port %s, vm %s", nbrinfo['shell_intf'], nbrinfo['vm'])
change_mac(nbrhosts[nbrinfo['vm']], nbrinfo['shell_intf'], original_mac)
for neighbor in nbr_to_test:
if ":" in neighbor:
logger.info("Force neighbor solicitation to workaround long IPV6 timer.")
asic_cmd(asic, "ndisc6 %s %s" % (neighbor, local_port))
pytest_assert(
wait_until(60, 2, check_arptable_mac, per_host, asic, neighbor, original_mac, checkstate=False),
"MAC {} didn't change in ARP table".format(original_mac))
sonic_ping(asic, neighbor, verbose=True)
pytest_assert(wait_until(60, 2, check_arptable_mac, per_host, asic, neighbor, original_mac),
"MAC {} didn't change in ARP table".format(original_mac))
dump_and_verify_neighbors_on_asic(duthosts, per_host, asic, nbr_to_test, nbrhosts, all_cfg_facts, nbr_macs)
ping_all_neighbors(duthosts, all_cfg_facts, nbr_to_test)
def test_neighbor_clear_one(duthosts, enum_rand_one_per_hwsku_frontend_hostname, enum_asic_index,
setup, teardown, nbrhosts, all_cfg_facts, nbr_macs, established_arp):
"""
Verify tables, databases, and kernel routes are correctly deleted when a single neighbor adjacency is cleared.
Test Steps
* On local linecard:
* Clear single address with command: `ip neigh flush to "addr"`.
* Verify ARP/NDP entry removed from CLI.
* Verify table entries in ASIC, AppDb are removed for only the cleared address.
* On Supervisor card:
* Verify Chassis App DB entry are removed for only the cleared address.
* On remote linecards:
* Verify table entries in ASICDB, APPDB, and host ARP table are removed.
* Verify kernel route for cleared address is deleted.
* Restart traffic, verify relearn.
Args:
duthosts: duthosts fixture.
setup: setup fixture for this module.
nbrhosts: nbrhosts fixture.
all_cfg_facts: all_cfg_facts fixture from voq/conftest.py
established_arp: Fixture to establish ARP to all neighbors.
"""
per_host = duthosts[enum_rand_one_per_hwsku_frontend_hostname]
asic = per_host.asics[enum_asic_index if enum_asic_index is not None else 0]
cfg_facts = all_cfg_facts[per_host.hostname][asic.asic_index]['ansible_facts']
if 'BGP_NEIGHBOR' in cfg_facts:
neighs = cfg_facts['BGP_NEIGHBOR']
else:
logger.info("No local neighbors for host: %s/%s, skipping", per_host.hostname, asic.asic_index)
return
eth_cfg = cfg_facts['INTERFACE'] if 'INTERFACE' in cfg_facts else {}
pos_cfg = cfg_facts['PORTCHANNEL_INTERFACE'] if 'PORTCHANNEL_INTERFACE' in cfg_facts else {}
nbr_to_test = []
if eth_cfg != {}:
nbr_to_test.extend(select_neighbors(eth_cfg, cfg_facts))
if pos_cfg != {}:
nbr_to_test.extend(select_neighbors(pos_cfg, cfg_facts))
untouched_nbrs = [nbr for nbr in neighs if nbr not in nbr_to_test]
logger.info("We will test these neighbors: %s", nbr_to_test)
logger.info("These neighbors should not be affected: %s", untouched_nbrs)
for neighbor in nbr_to_test:
logger.info(
"Flushing neighbor: {} on host {}/{}".format(neighbor, per_host.hostname, asic.asic_index))
asic_cmd(asic, "ip neigh flush to %s" % neighbor)
logger.info("Wait for flush.")
poll_neighbor_table_delete(duthosts, nbr_to_test)
logger.info("Verify neighbors are gone.")
check_neighbors_are_gone(duthosts, all_cfg_facts, per_host, asic, nbr_to_test)
logger.info("Verify other neighbors are not affected.")
dump_and_verify_neighbors_on_asic(duthosts, per_host, asic, untouched_nbrs, nbrhosts, all_cfg_facts, nbr_macs)
# relearn and check
logger.info("Relearn neighbors on all nodes")
ping_all_dut_local_nbrs(duthosts)
logger.info("Check neighbor relearn on all nodes.")
check_all_neighbors_present(duthosts, nbrhosts, all_cfg_facts, nbr_macs)
