def test_standby_tor_downstream_t1_link_recovered( ptfhost, rand_selected_dut, rand_unselected_dut, verify_crm_nexthop_counter_not_increased, tbinfo): """ Verify traffic is distributed evenly after t1 link is recovered; Verify CRM that no new nexthop created """ PAUSE_TIME = 30 down_link = shutdown_random_one_t1_link(rand_selected_dut) time.sleep(PAUSE_TIME) params = dualtor_info(ptfhost, rand_selected_dut, rand_unselected_dut, tbinfo) try: check_tunnel_balance(**params) except Exception as e: no_shutdown_t1_link(rand_selected_dut, down_link) raise e no_shutdown_t1_link(rand_selected_dut, down_link) time.sleep(PAUSE_TIME) params = dualtor_info(ptfhost, rand_selected_dut, rand_unselected_dut, tbinfo) # For mocked dualtor, we should update static route manually after link recovered if 't0' in tbinfo['topo']['name']: remove_static_routes(rand_selected_dut, params['active_tor_ip']) add_nexthop_routes(rand_selected_dut, params['active_tor_ip']) check_tunnel_balance(**params)
def test_standby_tor_downstream_bgp_recovered( ptfhost, rand_selected_dut, rand_unselected_dut, verify_crm_nexthop_counter_not_increased, tbinfo): """ Verify traffic is shifted to the active links and no traffic drop observed; Verify traffic is distributed evenly after BGP session is recovered; Verify CRM that no new nexthop created """ PAUSE_TIME = 30 down_bgp = shutdown_random_one_bgp_session(rand_selected_dut) time.sleep(PAUSE_TIME) params = dualtor_info(ptfhost, rand_selected_dut, rand_unselected_dut, tbinfo) try: check_tunnel_balance(**params) except Exception as e: startup_bgp_session(rand_selected_dut, down_bgp) raise e startup_bgp_session(rand_selected_dut, down_bgp) time.sleep(PAUSE_TIME) params = dualtor_info(ptfhost, rand_selected_dut, rand_unselected_dut, tbinfo) check_tunnel_balance(**params)
def test_standby_tor_downstream(ptfhost, rand_selected_dut, rand_unselected_dut, tbinfo, require_mocked_dualtor): """ Verify tunnel traffic to active ToR is distributed equally across nexthops, and no traffic is forwarded to server from standby ToR """ params = dualtor_info(ptfhost, rand_selected_dut, rand_unselected_dut, tbinfo) check_tunnel_balance(**params)
def testbed_setup(ip_version, ptfhost, rand_selected_dut, rand_unselected_dut, tbinfo, request): testbed_params = dualtor_info(ptfhost, rand_selected_dut, rand_unselected_dut, tbinfo) test_port = testbed_params["selected_port"] if ip_version == "ipv4": server_ip = testbed_params["target_server_ip"] elif ip_version == "ipv6": server_ip = testbed_params["target_server_ipv6"] # setup arp_responder to answer ipv6 neighbor solicitation messages request.getfixturevalue("run_arp_responder_ipv6") else: raise ValueError("Unknown IP version '%s'" % ip_version) return test_port, server_ip, ip_version
def test_standby_tor_downstream_loopback_route_readded(ptfhost, rand_selected_dut, rand_unselected_dut, tbinfo): """ Verify traffic is equally distributed via loopback route """ pt_require('dualtor' in tbinfo['topo']['name'], "Only run on dualtor testbed") params = dualtor_info(ptfhost, rand_selected_dut, rand_unselected_dut, tbinfo) active_tor_loopback0 = params['active_tor_ip'] # Remove loopback routes and verify traffic is equally distributed remove_static_routes(rand_selected_dut, active_tor_loopback0) check_tunnel_balance(**params) # Readd loopback routes and verify traffic is equally distributed add_nexthop_routes(rand_selected_dut, active_tor_loopback0) check_tunnel_balance(**params)
def test_downstream_standby_mux_toggle_active( conn_graph_facts, ptfadapter, ptfhost, rand_selected_dut, rand_unselected_dut, tbinfo, require_mocked_dualtor, tunnel_traffic_monitor, vmhost, toggle_all_simulator_ports, tor_mux_intfs ): # set rand_selected_dut as standby and rand_unselected_dut to active tor test_params = dualtor_info(ptfhost, rand_selected_dut, rand_unselected_dut, tbinfo) server_ipv4 = test_params["target_server_ip"] random_dst_ip = "1.1.1.2" pkt, exp_pkt = build_packet_to_server(rand_selected_dut, ptfadapter, random_dst_ip) ptf_t1_intf = random.choice(get_t1_ptf_ports(rand_selected_dut, tbinfo)) def monitor_tunnel_and_server_traffic(torhost, expect_tunnel_traffic=True, expect_server_traffic=True): tunnel_monitor = tunnel_traffic_monitor(rand_selected_dut, existing=True) server_traffic_monitor = ServerTrafficMonitor( torhost, ptfhost, vmhost, tbinfo, test_params["selected_port"], conn_graph_facts, exp_pkt, existing=False, is_mocked=is_mocked_dualtor(tbinfo) ) tunnel_monitor.existing = expect_tunnel_traffic server_traffic_monitor.existing = expect_server_traffic with tunnel_monitor, server_traffic_monitor: testutils.send(ptfadapter, int(ptf_t1_intf.strip("eth")), pkt, count=10) logger.info("Stage 1: Verify Standby Forwarding") logger.info("Step 1.1: Add route to a nexthop which is a standby Neighbor") set_mux_state(rand_selected_dut, tbinfo, 'standby', tor_mux_intfs, toggle_all_simulator_ports) add_nexthop_routes(rand_selected_dut, random_dst_ip, nexthops=[server_ipv4]) logger.info("Step 1.2: Verify traffic to this route dst is forwarded to Active ToR and equally distributed") check_tunnel_balance(**test_params) monitor_tunnel_and_server_traffic(rand_selected_dut, expect_server_traffic=False, expect_tunnel_traffic=True) logger.info("Stage 2: Verify Active Forwarding") logger.info("Step 2.1: Simulate Mux state change to active") set_mux_state(rand_selected_dut, tbinfo, 'active', tor_mux_intfs, toggle_all_simulator_ports) logger.info("Step 2.2: Verify traffic to this route dst is forwarded directly to server") monitor_tunnel_and_server_traffic(rand_selected_dut, expect_server_traffic=True, expect_tunnel_traffic=False) logger.info("Stage 3: Verify Standby Forwarding Again") logger.info("Step 3.1: Simulate Mux state change to standby") set_mux_state(rand_selected_dut, tbinfo, 'standby', tor_mux_intfs, toggle_all_simulator_ports) logger.info("Step 3.2: Verify traffic to this route dst is now redirected back to Active ToR and equally distributed") monitor_tunnel_and_server_traffic(rand_selected_dut, expect_server_traffic=False, expect_tunnel_traffic=True) check_tunnel_balance(**test_params) remove_static_routes(rand_selected_dut, random_dst_ip)
def test_standby_tor_remove_neighbor_downstream_standby( conn_graph_facts, ptfadapter, ptfhost, rand_selected_dut, rand_unselected_dut, tbinfo, require_mocked_dualtor, set_crm_polling_interval, tunnel_traffic_monitor, vmhost ): """ @summary: Verify that after removing neighbor entry for a server over standby ToR, the packets sent to the server will be dropped(neither passed to the server or redirected to the active ToR). """ @contextlib.contextmanager def stop_garp(ptfhost): """Temporarily stop garp service.""" ptfhost.shell("supervisorctl stop garp_service") yield ptfhost.shell("supervisorctl start garp_service") tor = rand_selected_dut test_params = dualtor_info(ptfhost, rand_selected_dut, rand_unselected_dut, tbinfo) server_ipv4 = test_params["target_server_ip"] pkt, exp_pkt = build_packet_to_server(tor, ptfadapter, server_ipv4) ptf_t1_intf = random.choice(get_t1_ptf_ports(tor, tbinfo)) logging.info("send traffic to server %s from ptf t1 interface %s", server_ipv4, ptf_t1_intf) tunnel_monitor = tunnel_traffic_monitor(tor, existing=True) with tunnel_monitor: testutils.send(ptfadapter, int(ptf_t1_intf.strip("eth")), pkt, count=10) logging.info("send traffic to server %s after removing neighbor entry", server_ipv4) tunnel_monitor.existing = False server_traffic_monitor = ServerTrafficMonitor( tor, ptfhost, vmhost, tbinfo, test_params["selected_port"], conn_graph_facts, exp_pkt, existing=False, is_mocked=is_mocked_dualtor(tbinfo) ) # for real dualtor testbed, leave the neighbor restoration to garp service flush_neighbor_ct = flush_neighbor(tor, server_ipv4, restore=is_t0_mocked_dualtor) with crm_neighbor_checker(tor), stop_garp(ptfhost), flush_neighbor_ct, tunnel_monitor, server_traffic_monitor: testutils.send(ptfadapter, int(ptf_t1_intf.strip("eth")), pkt, count=10) logging.info("send traffic to server %s after neighbor entry is restored", server_ipv4) tunnel_monitor.existing = True with crm_neighbor_checker(tor), tunnel_monitor: testutils.send(ptfadapter, int(ptf_t1_intf.strip("eth")), pkt, count=10)
def test_active_tor_remove_neighbor_downstream_active( conn_graph_facts, ptfadapter, ptfhost, rand_selected_dut, rand_unselected_dut, tbinfo, set_crm_polling_interval, tunnel_traffic_monitor, vmhost): """ @Verify those two scenarios: If the neighbor entry of a server is present on active ToR, all traffic to server should be directly forwarded. If the neighbor entry of a server is removed, all traffic to server should be dropped and no tunnel traffic. """ @contextlib.contextmanager def stop_garp(ptfhost): """Temporarily stop garp service.""" ptfhost.shell("supervisorctl stop garp_service") yield ptfhost.shell("supervisorctl start garp_service") tor = rand_selected_dut test_params = dualtor_info(ptfhost, rand_selected_dut, rand_unselected_dut, tbinfo) server_ipv4 = test_params["target_server_ip"] pkt, exp_pkt = build_packet_to_server(tor, ptfadapter, server_ipv4) ptf_t1_intf = random.choice(get_t1_ptf_ports(tor, tbinfo)) logging.info("send traffic to server %s from ptf t1 interface %s", server_ipv4, ptf_t1_intf) server_traffic_monitor = ServerTrafficMonitor( tor, ptfhost, vmhost, tbinfo, test_params["selected_port"], conn_graph_facts, exp_pkt, existing=True, is_mocked=is_mocked_dualtor(tbinfo)) tunnel_monitor = tunnel_traffic_monitor(tor, existing=False) with crm_neighbor_checker(tor), tunnel_monitor, server_traffic_monitor: testutils.send(ptfadapter, int(ptf_t1_intf.strip("eth")), pkt, count=10) logging.info("send traffic to server %s after removing neighbor entry", server_ipv4) server_traffic_monitor = ServerTrafficMonitor( tor, ptfhost, vmhost, tbinfo, test_params["selected_port"], conn_graph_facts, exp_pkt, existing=False, is_mocked=is_mocked_dualtor(tbinfo) ) # for real dualtor testbed, leave the neighbor restoration to garp service flush_neighbor_ct = flush_neighbor(tor, server_ipv4, restore=is_t0_mocked_dualtor) with crm_neighbor_checker(tor), stop_garp( ptfhost ), flush_neighbor_ct, tunnel_monitor, server_traffic_monitor: testutils.send(ptfadapter, int(ptf_t1_intf.strip("eth")), pkt, count=10) logging.info("send traffic to server %s after neighbor entry is restored", server_ipv4) server_traffic_monitor = ServerTrafficMonitor( tor, ptfhost, vmhost, tbinfo, test_params["selected_port"], conn_graph_facts, exp_pkt, existing=True, is_mocked=is_mocked_dualtor(tbinfo)) with crm_neighbor_checker(tor), tunnel_monitor, server_traffic_monitor: testutils.send(ptfadapter, int(ptf_t1_intf.strip("eth")), pkt, count=10)
def _get_testbed_params(): params = dualtor_info(ptfhost, rand_selected_dut, rand_unselected_dut, tbinfo) params["check_ipv6"] = (ip_version == "ipv6") return params