def test_decap_standby_tor(apply_mock_dual_tor_tables,
apply_mock_dual_tor_kernel_configs,
apply_standby_state_to_orchagent,
build_encapsulated_packet, rand_selected_interface,
ptfadapter, tbinfo, rand_selected_dut,
tunnel_traffic_monitor):
tor = rand_selected_dut
encapsulated_packet = build_encapsulated_packet
iface, _ = rand_selected_interface
exp_ptf_port_index = get_ptf_server_intf_index(tor, tbinfo, iface)
exp_pkt = build_expected_packet_to_server(encapsulated_packet)
ptf_t1_intf = random.choice(get_t1_ptf_ports(tor, tbinfo))
logging.info("send encapsulated packet from ptf t1 interface %s",
ptf_t1_intf)
with tunnel_traffic_monitor(tor, existing=False):
testutils.send(ptfadapter,
int(ptf_t1_intf.strip("eth")),
encapsulated_packet,
count=1)
testutils.verify_no_packet_any(ptfadapter,
exp_pkt,
ports=[exp_ptf_port_index])
def test_dscp_to_queue_during_encap_on_standby(
build_non_encapsulated_ip_packet,
rand_selected_interface, ptfadapter,
tbinfo,
rand_selected_dut,
tunnel_traffic_monitor,
duthosts,
rand_one_dut_hostname
):
"""
Test if DSCP to Q mapping for outer header is matching with inner header during encap on standby
"""
rand_selected_dut.shell("/usr/local/bin/write_standby.py")
tor = rand_selected_dut
non_encapsulated_packet = build_non_encapsulated_ip_packet
iface, _ = rand_selected_interface
exp_ptf_port_index = get_ptf_server_intf_index(tor, tbinfo, iface)
# Clear queue counters
duthost = duthosts[rand_one_dut_hostname]
duthost.shell('sonic-clear queuecounters')
logging.info("Clearing queue counters before starting traffic")
ptfadapter.dataplane.flush()
ptf_t1_intf = random.choice(get_t1_ptf_ports(tor, tbinfo))
logging.info("send IP packet from ptf t1 interface %s", ptf_t1_intf)
with tunnel_traffic_monitor(tor, existing=True):
testutils.send(ptfadapter, int(ptf_t1_intf.strip("eth")), non_encapsulated_packet, count=10)
def verify_traffic(duthost,
connection,
route,
is_duthost_active=True,
is_route_existed=True):
prefix = ipaddress.ip_network(route["prefix"])
dst_host = str(random.choice(list(prefix.hosts())))
pkt, exp_pkt = build_packet_to_server(duthost, ptfadapter, dst_host)
ptf_t1_intf = random.choice(get_t1_ptf_ports(duthost, tbinfo))
ptf_t1_intf_index = int(ptf_t1_intf.strip("eth"))
is_tunnel_traffic_existed = is_route_existed and not is_duthost_active
is_server_traffic_existed = is_route_existed and is_duthost_active
tunnel_monitor = tunnel_traffic_monitor(
duthost, existing=is_tunnel_traffic_existed)
server_traffic_monitor = ServerTrafficMonitor(
duthost,
ptfhost,
vmhost,
tbinfo,
connection["test_intf"],
conn_graph_facts,
exp_pkt,
existing=is_server_traffic_existed)
with tunnel_monitor, server_traffic_monitor:
testutils.send(ptfadapter, ptf_t1_intf_index, pkt, count=10)
def test_decap_standby_tor(apply_standby_state_to_orchagent,
build_encapsulated_packet, rand_selected_interface,
ptfadapter, tbinfo, rand_selected_dut,
tunnel_traffic_monitor):
def verify_downstream_packet_to_server(ptfadapter, port, exp_pkt):
"""Verify packet is passed downstream to server."""
packets = ptfadapter.dataplane.packet_queues[(0, port)]
for packet in packets:
if exp_pkt.pkt_match(packet):
return True
return False
tor = rand_selected_dut
encapsulated_packet = build_encapsulated_packet
iface, _ = rand_selected_interface
exp_ptf_port_index = get_ptf_server_intf_index(tor, tbinfo, iface)
exp_pkt = build_expected_packet_to_server(encapsulated_packet)
ptf_t1_intf = random.choice(get_t1_ptf_ports(tor, tbinfo))
logging.info("send encapsulated packet from ptf t1 interface %s",
ptf_t1_intf)
with tunnel_traffic_monitor(tor, existing=False):
testutils.send(ptfadapter,
int(ptf_t1_intf.strip("eth")),
encapsulated_packet,
count=10)
time.sleep(2)
verify_downstream_packet_to_server(ptfadapter, exp_ptf_port_index,
exp_pkt)
def verify_traffic(duthost,
connection,
route,
is_duthost_active=True,
is_route_existed=True):
prefix = ipaddress.ip_network(route["prefix"])
dst_host = str(next(prefix.hosts()))
pkt, exp_pkt = build_packet_to_server(duthost, ptfadapter, dst_host)
ptf_t1_intf = random.choice(get_t1_ptf_ports(duthost, tbinfo))
ptf_t1_intf_index = int(ptf_t1_intf.strip("eth"))
is_tunnel_traffic_existed = is_route_existed and not is_duthost_active
is_server_traffic_existed = is_route_existed and is_duthost_active
if isinstance(prefix, ipaddress.IPv4Network):
tunnel_innner_pkt = pkt[scapyall.IP].copy()
tunnel_innner_pkt[scapyall.IP].ttl -= 1
else:
tunnel_innner_pkt = pkt[scapyall.IPv6].copy()
tunnel_innner_pkt[scapyall.IPv6].hlim -= 1
tunnel_monitor = tunnel_traffic_monitor(
duthost,
existing=is_tunnel_traffic_existed,
inner_packet=tunnel_innner_pkt)
server_traffic_monitor = ServerTrafficMonitor(
duthost,
ptfhost,
vmhost,
tbinfo,
connection["test_intf"],
conn_graph_facts,
exp_pkt,
existing=is_server_traffic_existed)
with tunnel_monitor, server_traffic_monitor:
testutils.send(ptfadapter, ptf_t1_intf_index, pkt, count=10)
def monitor_tunnel_and_server_traffic(torhost, expect_tunnel_traffic=True, expect_server_traffic=True):
tunnel_monitor = tunnel_traffic_monitor(rand_selected_dut, existing=True)
server_traffic_monitor = ServerTrafficMonitor(
torhost, ptfhost, vmhost, tbinfo, test_params["selected_port"],
conn_graph_facts, exp_pkt, existing=False, is_mocked=is_mocked_dualtor(tbinfo)
)
tunnel_monitor.existing = expect_tunnel_traffic
server_traffic_monitor.existing = expect_server_traffic
with tunnel_monitor, server_traffic_monitor:
testutils.send(ptfadapter, int(ptf_t1_intf.strip("eth")), pkt, count=10)
def test_standby_tor_kill_bgpd_downstream_active(
upper_tor_host, lower_tor_host, send_t1_to_server_with_action,
toggle_all_simulator_ports_to_upper_tor, kill_bgpd,
tunnel_traffic_monitor):
'''
Case: T1 -> Active ToR -> Server (Standby ToR BGP Down)
Action: Shutdown all BGP sessions on the standby ToR
Expectation:
Verify packet flow after the standby ToR (B) loses BGP sessions
T1 switch receives no IP-in-IP packet; server receives packet
'''
with tunnel_traffic_monitor(lower_tor_host, existing=False):
send_t1_to_server_with_action(upper_tor_host,
verify=True,
action=lambda: kill_bgpd(lower_tor_host))
verify_tor_states(expected_active_host=upper_tor_host,
expected_standby_host=lower_tor_host)
def test_standby_tor_remove_neighbor_downstream_standby(
conn_graph_facts, ptfadapter, ptfhost,
rand_selected_dut, rand_unselected_dut, tbinfo,
require_mocked_dualtor, set_crm_polling_interval,
tunnel_traffic_monitor, vmhost
):
"""
@summary: Verify that after removing neighbor entry for a server over standby
ToR, the packets sent to the server will be dropped(neither passed to the server
or redirected to the active ToR).
"""
@contextlib.contextmanager
def stop_garp(ptfhost):
"""Temporarily stop garp service."""
ptfhost.shell("supervisorctl stop garp_service")
yield
ptfhost.shell("supervisorctl start garp_service")
tor = rand_selected_dut
test_params = dualtor_info(ptfhost, rand_selected_dut, rand_unselected_dut, tbinfo)
server_ipv4 = test_params["target_server_ip"]
pkt, exp_pkt = build_packet_to_server(tor, ptfadapter, server_ipv4)
ptf_t1_intf = random.choice(get_t1_ptf_ports(tor, tbinfo))
logging.info("send traffic to server %s from ptf t1 interface %s", server_ipv4, ptf_t1_intf)
tunnel_monitor = tunnel_traffic_monitor(tor, existing=True)
with tunnel_monitor:
testutils.send(ptfadapter, int(ptf_t1_intf.strip("eth")), pkt, count=10)
logging.info("send traffic to server %s after removing neighbor entry", server_ipv4)
tunnel_monitor.existing = False
server_traffic_monitor = ServerTrafficMonitor(
tor, ptfhost, vmhost, tbinfo, test_params["selected_port"],
conn_graph_facts, exp_pkt, existing=False, is_mocked=is_mocked_dualtor(tbinfo)
)
# for real dualtor testbed, leave the neighbor restoration to garp service
flush_neighbor_ct = flush_neighbor(tor, server_ipv4, restore=is_t0_mocked_dualtor)
with crm_neighbor_checker(tor), stop_garp(ptfhost), flush_neighbor_ct, tunnel_monitor, server_traffic_monitor:
testutils.send(ptfadapter, int(ptf_t1_intf.strip("eth")), pkt, count=10)
logging.info("send traffic to server %s after neighbor entry is restored", server_ipv4)
tunnel_monitor.existing = True
with crm_neighbor_checker(tor), tunnel_monitor:
testutils.send(ptfadapter, int(ptf_t1_intf.strip("eth")), pkt, count=10)
def test_active_tor_kill_bgpd_downstream_standby(
upper_tor_host, lower_tor_host, send_t1_to_server_with_action,
toggle_all_simulator_ports_to_upper_tor, kill_bgpd,
tunnel_traffic_monitor):
'''
Case: T1 -> Standby ToR -> Server (Active ToR BGP Down)
Action: Shutdown all BGP sessions on the active ToR
Expectation:
Verify packet flow after the active ToR (A) loses BGP sessions
T1 switch continues to receive IP-in-IP traffic, from lower to upper ToR
No switchover occurs
verify traffic interruption is < 1 second
'''
with tunnel_traffic_monitor(lower_tor_host, existing=True):
send_t1_to_server_with_action(lower_tor_host,
verify=True,
delay=1,
action=lambda: kill_bgpd(upper_tor_host))
verify_tor_states(expected_active_host=upper_tor_host,
expected_standby_host=lower_tor_host)
def test_active_tor_shutdown_bgp_downstream_standby(
upper_tor_host, lower_tor_host, send_t1_to_server_with_action,
toggle_all_simulator_ports_to_upper_tor, shutdown_tor_bgp,
tunnel_traffic_monitor):
'''
Case: T1 -> Standby ToR -> Server (Active ToR BGP Down)
Action: Shutdown all BGP sessions on the active ToR
Expectation:
Verify packet flow after the active ToR (A) loses BGP sessions
T1 switch receives no IP-in-IP packet; server receives packet;
verify traffic interruption is < 1 second
'''
with tunnel_traffic_monitor(lower_tor_host, existing=False):
send_t1_to_server_with_action(
lower_tor_host, verify=True, delay=1,
action=lambda: shutdown_tor_bgp(upper_tor_host)
)
verify_tor_states(
expected_active_host=lower_tor_host,
expected_standby_host=upper_tor_host
)
def test_ecn_during_encap_on_standby(
build_non_encapsulated_ip_packet,
rand_selected_interface,
ptfadapter,
tbinfo,
rand_selected_dut,
tunnel_traffic_monitor
):
"""
Test if the ECN stamping on outer header is matching with inner during encap on standby
"""
rand_selected_dut.shell("/usr/local/bin/write_standby.py")
tor = rand_selected_dut
non_encapsulated_packet = build_non_encapsulated_ip_packet
iface, _ = rand_selected_interface
exp_ptf_port_index = get_ptf_server_intf_index(tor, tbinfo, iface)
ptf_t1_intf = random.choice(get_t1_ptf_ports(tor, tbinfo))
logging.info("send IP packet from ptf t1 interface %s", ptf_t1_intf)
with tunnel_traffic_monitor(tor, existing=True):
testutils.send(ptfadapter, int(ptf_t1_intf.strip("eth")), non_encapsulated_packet, count=10)
def test_mac_move(require_mocked_dualtor, announce_new_neighbor,
apply_active_state_to_orchagent, conn_graph_facts,
ptfadapter, ptfhost, rand_selected_dut,
set_crm_polling_interval, tbinfo, tunnel_traffic_monitor,
vmhost):
tor = rand_selected_dut
ptf_t1_intf = random.choice(get_t1_ptf_ports(tor, tbinfo))
ptf_t1_intf_index = int(ptf_t1_intf.strip("eth"))
# new neighbor learnt on an active port
test_port = next(announce_new_neighbor)
announce_new_neighbor.send(None)
logging.info("let new neighbor learnt on active port %s", test_port)
pkt, exp_pkt = build_packet_to_server(tor, ptfadapter,
NEW_NEIGHBOR_IPV4_ADDR)
tunnel_monitor = tunnel_traffic_monitor(tor, existing=False)
server_traffic_monitor = ServerTrafficMonitor(
tor,
ptfhost,
vmhost,
tbinfo,
test_port,
conn_graph_facts,
exp_pkt,
existing=True,
is_mocked=is_mocked_dualtor(tbinfo))
with crm_neighbor_checker(tor), tunnel_monitor, server_traffic_monitor:
testutils.send(ptfadapter, ptf_t1_intf_index, pkt, count=10)
# mac move to a standby port
test_port = next(announce_new_neighbor)
announce_new_neighbor.send(
lambda iface: set_dual_tor_state_to_orchagent(tor, "standby", [iface]))
logging.info("mac move to a standby port %s", test_port)
pkt, exp_pkt = build_packet_to_server(tor, ptfadapter,
NEW_NEIGHBOR_IPV4_ADDR)
tunnel_monitor = tunnel_traffic_monitor(tor, existing=True)
server_traffic_monitor = ServerTrafficMonitor(
tor,
ptfhost,
vmhost,
tbinfo,
test_port,
conn_graph_facts,
exp_pkt,
existing=False,
is_mocked=is_mocked_dualtor(tbinfo))
with crm_neighbor_checker(tor), tunnel_monitor, server_traffic_monitor:
testutils.send(ptfadapter, ptf_t1_intf_index, pkt, count=10)
# standby forwarding check after fdb ageout/flush
tor.shell("fdbclear")
server_traffic_monitor = ServerTrafficMonitor(
tor,
ptfhost,
vmhost,
tbinfo,
test_port,
conn_graph_facts,
exp_pkt,
existing=False,
is_mocked=is_mocked_dualtor(tbinfo))
with crm_neighbor_checker(tor), tunnel_monitor, server_traffic_monitor:
testutils.send(ptfadapter, ptf_t1_intf_index, pkt, count=10)
# mac move to another active port
test_port = next(announce_new_neighbor)
announce_new_neighbor.send(None)
logging.info("mac move to another active port %s", test_port)
pkt, exp_pkt = build_packet_to_server(tor, ptfadapter,
NEW_NEIGHBOR_IPV4_ADDR)
tunnel_monitor = tunnel_traffic_monitor(tor, existing=False)
server_traffic_monitor = ServerTrafficMonitor(
tor,
ptfhost,
vmhost,
tbinfo,
test_port,
conn_graph_facts,
exp_pkt,
existing=True,
is_mocked=is_mocked_dualtor(tbinfo))
with crm_neighbor_checker(tor), tunnel_monitor, server_traffic_monitor:
testutils.send(ptfadapter, ptf_t1_intf_index, pkt, count=10)
# active forwarding check after fdb ageout/flush
tor.shell("fdbclear")
server_traffic_monitor = ServerTrafficMonitor(
tor,
ptfhost,
vmhost,
tbinfo,
test_port,
conn_graph_facts,
exp_pkt,
existing=False,
is_mocked=is_mocked_dualtor(tbinfo))
with crm_neighbor_checker(tor), tunnel_monitor, server_traffic_monitor:
testutils.send(ptfadapter, ptf_t1_intf_index, pkt, count=10)
def test_active_tor_remove_neighbor_downstream_active(
conn_graph_facts, ptfadapter, ptfhost, rand_selected_dut,
rand_unselected_dut, tbinfo, set_crm_polling_interval,
tunnel_traffic_monitor, vmhost):
"""
@Verify those two scenarios:
If the neighbor entry of a server is present on active ToR,
all traffic to server should be directly forwarded.
If the neighbor entry of a server is removed, all traffic to server
should be dropped and no tunnel traffic.
"""
@contextlib.contextmanager
def stop_garp(ptfhost):
"""Temporarily stop garp service."""
ptfhost.shell("supervisorctl stop garp_service")
yield
ptfhost.shell("supervisorctl start garp_service")
tor = rand_selected_dut
test_params = dualtor_info(ptfhost, rand_selected_dut, rand_unselected_dut,
tbinfo)
server_ipv4 = test_params["target_server_ip"]
pkt, exp_pkt = build_packet_to_server(tor, ptfadapter, server_ipv4)
ptf_t1_intf = random.choice(get_t1_ptf_ports(tor, tbinfo))
logging.info("send traffic to server %s from ptf t1 interface %s",
server_ipv4, ptf_t1_intf)
server_traffic_monitor = ServerTrafficMonitor(
tor,
ptfhost,
vmhost,
tbinfo,
test_params["selected_port"],
conn_graph_facts,
exp_pkt,
existing=True,
is_mocked=is_mocked_dualtor(tbinfo))
tunnel_monitor = tunnel_traffic_monitor(tor, existing=False)
with crm_neighbor_checker(tor), tunnel_monitor, server_traffic_monitor:
testutils.send(ptfadapter,
int(ptf_t1_intf.strip("eth")),
pkt,
count=10)
logging.info("send traffic to server %s after removing neighbor entry",
server_ipv4)
server_traffic_monitor = ServerTrafficMonitor(
tor,
ptfhost,
vmhost,
tbinfo,
test_params["selected_port"],
conn_graph_facts,
exp_pkt,
existing=False,
is_mocked=is_mocked_dualtor(tbinfo)
) # for real dualtor testbed, leave the neighbor restoration to garp service
flush_neighbor_ct = flush_neighbor(tor,
server_ipv4,
restore=is_t0_mocked_dualtor)
with crm_neighbor_checker(tor), stop_garp(
ptfhost
), flush_neighbor_ct, tunnel_monitor, server_traffic_monitor:
testutils.send(ptfadapter,
int(ptf_t1_intf.strip("eth")),
pkt,
count=10)
logging.info("send traffic to server %s after neighbor entry is restored",
server_ipv4)
server_traffic_monitor = ServerTrafficMonitor(
tor,
ptfhost,
vmhost,
tbinfo,
test_params["selected_port"],
conn_graph_facts,
exp_pkt,
existing=True,
is_mocked=is_mocked_dualtor(tbinfo))
with crm_neighbor_checker(tor), tunnel_monitor, server_traffic_monitor:
testutils.send(ptfadapter,
int(ptf_t1_intf.strip("eth")),
pkt,
count=10)
def test_active_tor_remove_neighbor_downstream_active(
conn_graph_facts, ptfadapter, ptfhost, testbed_setup,
rand_selected_dut, tbinfo, require_mocked_dualtor,
set_crm_polling_interval, tunnel_traffic_monitor, vmhost):
"""
@Verify those two scenarios:
If the neighbor entry of a server is present on active ToR,
all traffic to server should be directly forwarded.
If the neighbor entry of a server is removed, all traffic to server
should be dropped and no tunnel traffic.
"""
@contextlib.contextmanager
def remove_neighbor(ptfhost, duthost, server_ip, ip_version):
# restore ipv4 neighbor since it is statically configured
if ip_version == "ipv4":
restore = True
neighbor_advertise_process = "garp_service"
elif ip_version == "ipv6":
restore = False
neighbor_advertise_process = "arp_responder"
else:
raise ValueError("Unknown IP version '%s'" % ip_version)
flush_neighbor_ct = flush_neighbor(duthost, server_ip, restore=restore)
try:
ptfhost.shell("supervisorctl stop %s" % neighbor_advertise_process)
with flush_neighbor_ct:
yield
finally:
ptfhost.shell("supervisorctl start %s" %
neighbor_advertise_process)
tor = rand_selected_dut
test_port, server_ip, ip_version = testbed_setup
pkt, exp_pkt = build_packet_to_server(tor, ptfadapter, server_ip)
ptf_t1_intf = random.choice(get_t1_ptf_ports(tor, tbinfo))
logging.info("send traffic to server %s from ptf t1 interface %s",
server_ip, ptf_t1_intf)
server_traffic_monitor = ServerTrafficMonitor(
tor,
ptfhost,
vmhost,
tbinfo,
test_port,
conn_graph_facts,
exp_pkt,
existing=True,
is_mocked=is_mocked_dualtor(tbinfo))
tunnel_monitor = tunnel_traffic_monitor(tor, existing=False)
with crm_neighbor_checker(tor), tunnel_monitor, server_traffic_monitor:
testutils.send(ptfadapter,
int(ptf_t1_intf.strip("eth")),
pkt,
count=10)
logging.info("send traffic to server %s after removing neighbor entry",
server_ip)
server_traffic_monitor = ServerTrafficMonitor(
tor,
ptfhost,
vmhost,
tbinfo,
test_port,
conn_graph_facts,
exp_pkt,
existing=False,
is_mocked=is_mocked_dualtor(tbinfo))
remove_neighbor_ct = remove_neighbor(ptfhost, tor, server_ip, ip_version)
with crm_neighbor_checker(
tor), remove_neighbor_ct, tunnel_monitor, server_traffic_monitor:
testutils.send(ptfadapter,
int(ptf_t1_intf.strip("eth")),
pkt,
count=10)
logging.info("send traffic to server %s after neighbor entry is restored",
server_ip)
server_traffic_monitor = ServerTrafficMonitor(
tor,
ptfhost,
vmhost,
tbinfo,
test_port,
conn_graph_facts,
exp_pkt,
existing=True,
is_mocked=is_mocked_dualtor(tbinfo))
with crm_neighbor_checker(tor), tunnel_monitor, server_traffic_monitor:
testutils.send(ptfadapter,
int(ptf_t1_intf.strip("eth")),
pkt,
count=10)
