def test_decap_standby_tor(apply_mock_dual_tor_tables, apply_mock_dual_tor_kernel_configs, apply_standby_state_to_orchagent, build_encapsulated_packet, rand_selected_interface, ptfadapter, tbinfo, rand_selected_dut, tunnel_traffic_monitor): tor = rand_selected_dut encapsulated_packet = build_encapsulated_packet iface, _ = rand_selected_interface exp_ptf_port_index = get_ptf_server_intf_index(tor, tbinfo, iface) exp_pkt = build_expected_packet_to_server(encapsulated_packet) ptf_t1_intf = random.choice(get_t1_ptf_ports(tor, tbinfo)) logging.info("send encapsulated packet from ptf t1 interface %s", ptf_t1_intf) with tunnel_traffic_monitor(tor, existing=False): testutils.send(ptfadapter, int(ptf_t1_intf.strip("eth")), encapsulated_packet, count=1) testutils.verify_no_packet_any(ptfadapter, exp_pkt, ports=[exp_ptf_port_index])
def test_dscp_to_queue_during_encap_on_standby( build_non_encapsulated_ip_packet, rand_selected_interface, ptfadapter, tbinfo, rand_selected_dut, tunnel_traffic_monitor, duthosts, rand_one_dut_hostname ): """ Test if DSCP to Q mapping for outer header is matching with inner header during encap on standby """ rand_selected_dut.shell("/usr/local/bin/write_standby.py") tor = rand_selected_dut non_encapsulated_packet = build_non_encapsulated_ip_packet iface, _ = rand_selected_interface exp_ptf_port_index = get_ptf_server_intf_index(tor, tbinfo, iface) # Clear queue counters duthost = duthosts[rand_one_dut_hostname] duthost.shell('sonic-clear queuecounters') logging.info("Clearing queue counters before starting traffic") ptfadapter.dataplane.flush() ptf_t1_intf = random.choice(get_t1_ptf_ports(tor, tbinfo)) logging.info("send IP packet from ptf t1 interface %s", ptf_t1_intf) with tunnel_traffic_monitor(tor, existing=True): testutils.send(ptfadapter, int(ptf_t1_intf.strip("eth")), non_encapsulated_packet, count=10)
def verify_traffic(duthost, connection, route, is_duthost_active=True, is_route_existed=True): prefix = ipaddress.ip_network(route["prefix"]) dst_host = str(random.choice(list(prefix.hosts()))) pkt, exp_pkt = build_packet_to_server(duthost, ptfadapter, dst_host) ptf_t1_intf = random.choice(get_t1_ptf_ports(duthost, tbinfo)) ptf_t1_intf_index = int(ptf_t1_intf.strip("eth")) is_tunnel_traffic_existed = is_route_existed and not is_duthost_active is_server_traffic_existed = is_route_existed and is_duthost_active tunnel_monitor = tunnel_traffic_monitor( duthost, existing=is_tunnel_traffic_existed) server_traffic_monitor = ServerTrafficMonitor( duthost, ptfhost, vmhost, tbinfo, connection["test_intf"], conn_graph_facts, exp_pkt, existing=is_server_traffic_existed) with tunnel_monitor, server_traffic_monitor: testutils.send(ptfadapter, ptf_t1_intf_index, pkt, count=10)
def test_decap_standby_tor(apply_standby_state_to_orchagent, build_encapsulated_packet, rand_selected_interface, ptfadapter, tbinfo, rand_selected_dut, tunnel_traffic_monitor): def verify_downstream_packet_to_server(ptfadapter, port, exp_pkt): """Verify packet is passed downstream to server.""" packets = ptfadapter.dataplane.packet_queues[(0, port)] for packet in packets: if exp_pkt.pkt_match(packet): return True return False tor = rand_selected_dut encapsulated_packet = build_encapsulated_packet iface, _ = rand_selected_interface exp_ptf_port_index = get_ptf_server_intf_index(tor, tbinfo, iface) exp_pkt = build_expected_packet_to_server(encapsulated_packet) ptf_t1_intf = random.choice(get_t1_ptf_ports(tor, tbinfo)) logging.info("send encapsulated packet from ptf t1 interface %s", ptf_t1_intf) with tunnel_traffic_monitor(tor, existing=False): testutils.send(ptfadapter, int(ptf_t1_intf.strip("eth")), encapsulated_packet, count=10) time.sleep(2) verify_downstream_packet_to_server(ptfadapter, exp_ptf_port_index, exp_pkt)
def verify_traffic(duthost, connection, route, is_duthost_active=True, is_route_existed=True): prefix = ipaddress.ip_network(route["prefix"]) dst_host = str(next(prefix.hosts())) pkt, exp_pkt = build_packet_to_server(duthost, ptfadapter, dst_host) ptf_t1_intf = random.choice(get_t1_ptf_ports(duthost, tbinfo)) ptf_t1_intf_index = int(ptf_t1_intf.strip("eth")) is_tunnel_traffic_existed = is_route_existed and not is_duthost_active is_server_traffic_existed = is_route_existed and is_duthost_active if isinstance(prefix, ipaddress.IPv4Network): tunnel_innner_pkt = pkt[scapyall.IP].copy() tunnel_innner_pkt[scapyall.IP].ttl -= 1 else: tunnel_innner_pkt = pkt[scapyall.IPv6].copy() tunnel_innner_pkt[scapyall.IPv6].hlim -= 1 tunnel_monitor = tunnel_traffic_monitor( duthost, existing=is_tunnel_traffic_existed, inner_packet=tunnel_innner_pkt) server_traffic_monitor = ServerTrafficMonitor( duthost, ptfhost, vmhost, tbinfo, connection["test_intf"], conn_graph_facts, exp_pkt, existing=is_server_traffic_existed) with tunnel_monitor, server_traffic_monitor: testutils.send(ptfadapter, ptf_t1_intf_index, pkt, count=10)
def monitor_tunnel_and_server_traffic(torhost, expect_tunnel_traffic=True, expect_server_traffic=True): tunnel_monitor = tunnel_traffic_monitor(rand_selected_dut, existing=True) server_traffic_monitor = ServerTrafficMonitor( torhost, ptfhost, vmhost, tbinfo, test_params["selected_port"], conn_graph_facts, exp_pkt, existing=False, is_mocked=is_mocked_dualtor(tbinfo) ) tunnel_monitor.existing = expect_tunnel_traffic server_traffic_monitor.existing = expect_server_traffic with tunnel_monitor, server_traffic_monitor: testutils.send(ptfadapter, int(ptf_t1_intf.strip("eth")), pkt, count=10)
def test_standby_tor_kill_bgpd_downstream_active( upper_tor_host, lower_tor_host, send_t1_to_server_with_action, toggle_all_simulator_ports_to_upper_tor, kill_bgpd, tunnel_traffic_monitor): ''' Case: T1 -> Active ToR -> Server (Standby ToR BGP Down) Action: Shutdown all BGP sessions on the standby ToR Expectation: Verify packet flow after the standby ToR (B) loses BGP sessions T1 switch receives no IP-in-IP packet; server receives packet ''' with tunnel_traffic_monitor(lower_tor_host, existing=False): send_t1_to_server_with_action(upper_tor_host, verify=True, action=lambda: kill_bgpd(lower_tor_host)) verify_tor_states(expected_active_host=upper_tor_host, expected_standby_host=lower_tor_host)
def test_standby_tor_remove_neighbor_downstream_standby( conn_graph_facts, ptfadapter, ptfhost, rand_selected_dut, rand_unselected_dut, tbinfo, require_mocked_dualtor, set_crm_polling_interval, tunnel_traffic_monitor, vmhost ): """ @summary: Verify that after removing neighbor entry for a server over standby ToR, the packets sent to the server will be dropped(neither passed to the server or redirected to the active ToR). """ @contextlib.contextmanager def stop_garp(ptfhost): """Temporarily stop garp service.""" ptfhost.shell("supervisorctl stop garp_service") yield ptfhost.shell("supervisorctl start garp_service") tor = rand_selected_dut test_params = dualtor_info(ptfhost, rand_selected_dut, rand_unselected_dut, tbinfo) server_ipv4 = test_params["target_server_ip"] pkt, exp_pkt = build_packet_to_server(tor, ptfadapter, server_ipv4) ptf_t1_intf = random.choice(get_t1_ptf_ports(tor, tbinfo)) logging.info("send traffic to server %s from ptf t1 interface %s", server_ipv4, ptf_t1_intf) tunnel_monitor = tunnel_traffic_monitor(tor, existing=True) with tunnel_monitor: testutils.send(ptfadapter, int(ptf_t1_intf.strip("eth")), pkt, count=10) logging.info("send traffic to server %s after removing neighbor entry", server_ipv4) tunnel_monitor.existing = False server_traffic_monitor = ServerTrafficMonitor( tor, ptfhost, vmhost, tbinfo, test_params["selected_port"], conn_graph_facts, exp_pkt, existing=False, is_mocked=is_mocked_dualtor(tbinfo) ) # for real dualtor testbed, leave the neighbor restoration to garp service flush_neighbor_ct = flush_neighbor(tor, server_ipv4, restore=is_t0_mocked_dualtor) with crm_neighbor_checker(tor), stop_garp(ptfhost), flush_neighbor_ct, tunnel_monitor, server_traffic_monitor: testutils.send(ptfadapter, int(ptf_t1_intf.strip("eth")), pkt, count=10) logging.info("send traffic to server %s after neighbor entry is restored", server_ipv4) tunnel_monitor.existing = True with crm_neighbor_checker(tor), tunnel_monitor: testutils.send(ptfadapter, int(ptf_t1_intf.strip("eth")), pkt, count=10)
def test_active_tor_kill_bgpd_downstream_standby( upper_tor_host, lower_tor_host, send_t1_to_server_with_action, toggle_all_simulator_ports_to_upper_tor, kill_bgpd, tunnel_traffic_monitor): ''' Case: T1 -> Standby ToR -> Server (Active ToR BGP Down) Action: Shutdown all BGP sessions on the active ToR Expectation: Verify packet flow after the active ToR (A) loses BGP sessions T1 switch continues to receive IP-in-IP traffic, from lower to upper ToR No switchover occurs verify traffic interruption is < 1 second ''' with tunnel_traffic_monitor(lower_tor_host, existing=True): send_t1_to_server_with_action(lower_tor_host, verify=True, delay=1, action=lambda: kill_bgpd(upper_tor_host)) verify_tor_states(expected_active_host=upper_tor_host, expected_standby_host=lower_tor_host)
def test_active_tor_shutdown_bgp_downstream_standby( upper_tor_host, lower_tor_host, send_t1_to_server_with_action, toggle_all_simulator_ports_to_upper_tor, shutdown_tor_bgp, tunnel_traffic_monitor): ''' Case: T1 -> Standby ToR -> Server (Active ToR BGP Down) Action: Shutdown all BGP sessions on the active ToR Expectation: Verify packet flow after the active ToR (A) loses BGP sessions T1 switch receives no IP-in-IP packet; server receives packet; verify traffic interruption is < 1 second ''' with tunnel_traffic_monitor(lower_tor_host, existing=False): send_t1_to_server_with_action( lower_tor_host, verify=True, delay=1, action=lambda: shutdown_tor_bgp(upper_tor_host) ) verify_tor_states( expected_active_host=lower_tor_host, expected_standby_host=upper_tor_host )
def test_ecn_during_encap_on_standby( build_non_encapsulated_ip_packet, rand_selected_interface, ptfadapter, tbinfo, rand_selected_dut, tunnel_traffic_monitor ): """ Test if the ECN stamping on outer header is matching with inner during encap on standby """ rand_selected_dut.shell("/usr/local/bin/write_standby.py") tor = rand_selected_dut non_encapsulated_packet = build_non_encapsulated_ip_packet iface, _ = rand_selected_interface exp_ptf_port_index = get_ptf_server_intf_index(tor, tbinfo, iface) ptf_t1_intf = random.choice(get_t1_ptf_ports(tor, tbinfo)) logging.info("send IP packet from ptf t1 interface %s", ptf_t1_intf) with tunnel_traffic_monitor(tor, existing=True): testutils.send(ptfadapter, int(ptf_t1_intf.strip("eth")), non_encapsulated_packet, count=10)
def test_mac_move(require_mocked_dualtor, announce_new_neighbor, apply_active_state_to_orchagent, conn_graph_facts, ptfadapter, ptfhost, rand_selected_dut, set_crm_polling_interval, tbinfo, tunnel_traffic_monitor, vmhost): tor = rand_selected_dut ptf_t1_intf = random.choice(get_t1_ptf_ports(tor, tbinfo)) ptf_t1_intf_index = int(ptf_t1_intf.strip("eth")) # new neighbor learnt on an active port test_port = next(announce_new_neighbor) announce_new_neighbor.send(None) logging.info("let new neighbor learnt on active port %s", test_port) pkt, exp_pkt = build_packet_to_server(tor, ptfadapter, NEW_NEIGHBOR_IPV4_ADDR) tunnel_monitor = tunnel_traffic_monitor(tor, existing=False) server_traffic_monitor = ServerTrafficMonitor( tor, ptfhost, vmhost, tbinfo, test_port, conn_graph_facts, exp_pkt, existing=True, is_mocked=is_mocked_dualtor(tbinfo)) with crm_neighbor_checker(tor), tunnel_monitor, server_traffic_monitor: testutils.send(ptfadapter, ptf_t1_intf_index, pkt, count=10) # mac move to a standby port test_port = next(announce_new_neighbor) announce_new_neighbor.send( lambda iface: set_dual_tor_state_to_orchagent(tor, "standby", [iface])) logging.info("mac move to a standby port %s", test_port) pkt, exp_pkt = build_packet_to_server(tor, ptfadapter, NEW_NEIGHBOR_IPV4_ADDR) tunnel_monitor = tunnel_traffic_monitor(tor, existing=True) server_traffic_monitor = ServerTrafficMonitor( tor, ptfhost, vmhost, tbinfo, test_port, conn_graph_facts, exp_pkt, existing=False, is_mocked=is_mocked_dualtor(tbinfo)) with crm_neighbor_checker(tor), tunnel_monitor, server_traffic_monitor: testutils.send(ptfadapter, ptf_t1_intf_index, pkt, count=10) # standby forwarding check after fdb ageout/flush tor.shell("fdbclear") server_traffic_monitor = ServerTrafficMonitor( tor, ptfhost, vmhost, tbinfo, test_port, conn_graph_facts, exp_pkt, existing=False, is_mocked=is_mocked_dualtor(tbinfo)) with crm_neighbor_checker(tor), tunnel_monitor, server_traffic_monitor: testutils.send(ptfadapter, ptf_t1_intf_index, pkt, count=10) # mac move to another active port test_port = next(announce_new_neighbor) announce_new_neighbor.send(None) logging.info("mac move to another active port %s", test_port) pkt, exp_pkt = build_packet_to_server(tor, ptfadapter, NEW_NEIGHBOR_IPV4_ADDR) tunnel_monitor = tunnel_traffic_monitor(tor, existing=False) server_traffic_monitor = ServerTrafficMonitor( tor, ptfhost, vmhost, tbinfo, test_port, conn_graph_facts, exp_pkt, existing=True, is_mocked=is_mocked_dualtor(tbinfo)) with crm_neighbor_checker(tor), tunnel_monitor, server_traffic_monitor: testutils.send(ptfadapter, ptf_t1_intf_index, pkt, count=10) # active forwarding check after fdb ageout/flush tor.shell("fdbclear") server_traffic_monitor = ServerTrafficMonitor( tor, ptfhost, vmhost, tbinfo, test_port, conn_graph_facts, exp_pkt, existing=False, is_mocked=is_mocked_dualtor(tbinfo)) with crm_neighbor_checker(tor), tunnel_monitor, server_traffic_monitor: testutils.send(ptfadapter, ptf_t1_intf_index, pkt, count=10)
def test_active_tor_remove_neighbor_downstream_active( conn_graph_facts, ptfadapter, ptfhost, rand_selected_dut, rand_unselected_dut, tbinfo, set_crm_polling_interval, tunnel_traffic_monitor, vmhost): """ @Verify those two scenarios: If the neighbor entry of a server is present on active ToR, all traffic to server should be directly forwarded. If the neighbor entry of a server is removed, all traffic to server should be dropped and no tunnel traffic. """ @contextlib.contextmanager def stop_garp(ptfhost): """Temporarily stop garp service.""" ptfhost.shell("supervisorctl stop garp_service") yield ptfhost.shell("supervisorctl start garp_service") tor = rand_selected_dut test_params = dualtor_info(ptfhost, rand_selected_dut, rand_unselected_dut, tbinfo) server_ipv4 = test_params["target_server_ip"] pkt, exp_pkt = build_packet_to_server(tor, ptfadapter, server_ipv4) ptf_t1_intf = random.choice(get_t1_ptf_ports(tor, tbinfo)) logging.info("send traffic to server %s from ptf t1 interface %s", server_ipv4, ptf_t1_intf) server_traffic_monitor = ServerTrafficMonitor( tor, ptfhost, vmhost, tbinfo, test_params["selected_port"], conn_graph_facts, exp_pkt, existing=True, is_mocked=is_mocked_dualtor(tbinfo)) tunnel_monitor = tunnel_traffic_monitor(tor, existing=False) with crm_neighbor_checker(tor), tunnel_monitor, server_traffic_monitor: testutils.send(ptfadapter, int(ptf_t1_intf.strip("eth")), pkt, count=10) logging.info("send traffic to server %s after removing neighbor entry", server_ipv4) server_traffic_monitor = ServerTrafficMonitor( tor, ptfhost, vmhost, tbinfo, test_params["selected_port"], conn_graph_facts, exp_pkt, existing=False, is_mocked=is_mocked_dualtor(tbinfo) ) # for real dualtor testbed, leave the neighbor restoration to garp service flush_neighbor_ct = flush_neighbor(tor, server_ipv4, restore=is_t0_mocked_dualtor) with crm_neighbor_checker(tor), stop_garp( ptfhost ), flush_neighbor_ct, tunnel_monitor, server_traffic_monitor: testutils.send(ptfadapter, int(ptf_t1_intf.strip("eth")), pkt, count=10) logging.info("send traffic to server %s after neighbor entry is restored", server_ipv4) server_traffic_monitor = ServerTrafficMonitor( tor, ptfhost, vmhost, tbinfo, test_params["selected_port"], conn_graph_facts, exp_pkt, existing=True, is_mocked=is_mocked_dualtor(tbinfo)) with crm_neighbor_checker(tor), tunnel_monitor, server_traffic_monitor: testutils.send(ptfadapter, int(ptf_t1_intf.strip("eth")), pkt, count=10)
def test_active_tor_remove_neighbor_downstream_active( conn_graph_facts, ptfadapter, ptfhost, testbed_setup, rand_selected_dut, tbinfo, require_mocked_dualtor, set_crm_polling_interval, tunnel_traffic_monitor, vmhost): """ @Verify those two scenarios: If the neighbor entry of a server is present on active ToR, all traffic to server should be directly forwarded. If the neighbor entry of a server is removed, all traffic to server should be dropped and no tunnel traffic. """ @contextlib.contextmanager def remove_neighbor(ptfhost, duthost, server_ip, ip_version): # restore ipv4 neighbor since it is statically configured if ip_version == "ipv4": restore = True neighbor_advertise_process = "garp_service" elif ip_version == "ipv6": restore = False neighbor_advertise_process = "arp_responder" else: raise ValueError("Unknown IP version '%s'" % ip_version) flush_neighbor_ct = flush_neighbor(duthost, server_ip, restore=restore) try: ptfhost.shell("supervisorctl stop %s" % neighbor_advertise_process) with flush_neighbor_ct: yield finally: ptfhost.shell("supervisorctl start %s" % neighbor_advertise_process) tor = rand_selected_dut test_port, server_ip, ip_version = testbed_setup pkt, exp_pkt = build_packet_to_server(tor, ptfadapter, server_ip) ptf_t1_intf = random.choice(get_t1_ptf_ports(tor, tbinfo)) logging.info("send traffic to server %s from ptf t1 interface %s", server_ip, ptf_t1_intf) server_traffic_monitor = ServerTrafficMonitor( tor, ptfhost, vmhost, tbinfo, test_port, conn_graph_facts, exp_pkt, existing=True, is_mocked=is_mocked_dualtor(tbinfo)) tunnel_monitor = tunnel_traffic_monitor(tor, existing=False) with crm_neighbor_checker(tor), tunnel_monitor, server_traffic_monitor: testutils.send(ptfadapter, int(ptf_t1_intf.strip("eth")), pkt, count=10) logging.info("send traffic to server %s after removing neighbor entry", server_ip) server_traffic_monitor = ServerTrafficMonitor( tor, ptfhost, vmhost, tbinfo, test_port, conn_graph_facts, exp_pkt, existing=False, is_mocked=is_mocked_dualtor(tbinfo)) remove_neighbor_ct = remove_neighbor(ptfhost, tor, server_ip, ip_version) with crm_neighbor_checker( tor), remove_neighbor_ct, tunnel_monitor, server_traffic_monitor: testutils.send(ptfadapter, int(ptf_t1_intf.strip("eth")), pkt, count=10) logging.info("send traffic to server %s after neighbor entry is restored", server_ip) server_traffic_monitor = ServerTrafficMonitor( tor, ptfhost, vmhost, tbinfo, test_port, conn_graph_facts, exp_pkt, existing=True, is_mocked=is_mocked_dualtor(tbinfo)) with crm_neighbor_checker(tor), tunnel_monitor, server_traffic_monitor: testutils.send(ptfadapter, int(ptf_t1_intf.strip("eth")), pkt, count=10)