def test_normal_op_upstream(upper_tor_host, lower_tor_host, send_server_to_t1_with_action, toggle_all_simulator_ports_to_upper_tor): """Send upstream traffic and confirm no disruption or switchover occurs""" send_server_to_t1_with_action(upper_tor_host, verify=True) verify_tor_states(expected_active_host=upper_tor_host, expected_standby_host=lower_tor_host)
def test_standby_config_reload_upstream( upper_tor_host, lower_tor_host, send_server_to_t1_with_action, toggle_all_simulator_ports_to_upper_tor): """ Send upstream traffic and `config reload` the standby ToR. Confirm no switchover occurs and no disruption """ send_server_to_t1_with_action( upper_tor_host, verify=True, action=lambda: config_reload(lower_tor_host, wait=0)) verify_tor_states(expected_active_host=upper_tor_host, expected_standby_host=lower_tor_host)
def test_standby_tor_heartbeat_failure_upstream( toggle_all_simulator_ports_to_upper_tor, upper_tor_host, lower_tor_host, send_server_to_t1_with_action, shutdown_tor_heartbeat): """ Send upstream traffic and stop the LinkProber module on the standby ToR. Confirm no switchover and no disruption. """ send_server_to_t1_with_action( upper_tor_host, verify=True, action=lambda: shutdown_tor_heartbeat(lower_tor_host)) verify_tor_states(expected_active_host=upper_tor_host, expected_standby_host=lower_tor_host)
def test_active_tor_heartbeat_failure_upstream( toggle_all_simulator_ports_to_upper_tor, upper_tor_host, lower_tor_host, send_server_to_t1_with_action, shutdown_tor_heartbeat): """ Send upstream traffic and stop the LinkProber module on the active ToR. Confirm switchover and disruption lasts < 1 second. """ send_server_to_t1_with_action( upper_tor_host, verify=True, delay=MUX_SIM_ALLOWED_DISRUPTION_SEC, action=lambda: shutdown_tor_heartbeat(upper_tor_host)) verify_tor_states(expected_active_host=lower_tor_host, expected_standby_host=upper_tor_host)
def test_standby_tor_reboot_upstream(upper_tor_host, lower_tor_host, send_server_to_t1_with_action, toggle_all_simulator_ports_to_upper_tor, reboot_tor, wait_for_device_reachable): """ Send upstream traffic and reboot the standby ToR. Confirm no switchover occurred and no disruption """ send_server_to_t1_with_action(upper_tor_host, verify=True, action=lambda: reboot_tor(lower_tor_host)) wait_for_device_reachable(lower_tor_host) verify_tor_states(expected_active_host=upper_tor_host, expected_standby_host=lower_tor_host)
def test_active_config_reload_upstream( upper_tor_host, lower_tor_host, send_server_to_t1_with_action, toggle_all_simulator_ports_to_upper_tor): """ Send upstream traffic and `config reload` the active ToR. Confirm switchover occurs and disruption lasted < 1 second """ send_server_to_t1_with_action( upper_tor_host, verify=True, delay=CONFIG_RELOAD_ALLOWED_DISRUPTION_SEC, action=lambda: config_reload(upper_tor_host, wait=0)) verify_tor_states(expected_active_host=lower_tor_host, expected_standby_host=upper_tor_host)
def test_normal_op_upstream( upper_tor_host, lower_tor_host, send_server_to_t1_with_action, toggle_all_simulator_ports_to_upper_tor, toggle_all_ports_both_tors_admin_forwarding_state_to_active, cable_type): """Send upstream traffic and confirm no disruption or switchover occurs""" send_server_to_t1_with_action(upper_tor_host, verify=True, stop_after=60) if cable_type == CableType.active_standby: verify_tor_states(expected_active_host=upper_tor_host, expected_standby_host=lower_tor_host) else: verify_tor_states( expected_active_host=[upper_tor_host, lower_tor_host], cable_type=cable_type)
def test_active_tor_reboot_upstream(upper_tor_host, lower_tor_host, send_server_to_t1_with_action, toggle_all_simulator_ports_to_upper_tor, reboot_tor, wait_for_device_reachable): """ Send upstream traffic and reboot the active ToR. Confirm switchover occurred and disruption lasts < 1 second """ send_server_to_t1_with_action(upper_tor_host, verify=True, delay=MUX_SIM_ALLOWED_DISRUPTION_SEC, action=lambda: reboot_tor(upper_tor_host)) wait_for_device_reachable(upper_tor_host) verify_tor_states(expected_active_host=lower_tor_host, expected_standby_host=upper_tor_host)
def test_standby_link_down_upstream(upper_tor_host, lower_tor_host, send_server_to_t1_with_action, toggle_all_simulator_ports_to_upper_tor, shutdown_fanout_lower_tor_intfs): """ Send traffic from server to T1 and shutdown the standby ToR link. Verify no switchover and no disruption """ send_server_to_t1_with_action(upper_tor_host, verify=True, delay=MUX_SIM_ALLOWED_DISRUPTION_SEC, action=shutdown_fanout_lower_tor_intfs) verify_tor_states(expected_active_host=upper_tor_host, expected_standby_host=lower_tor_host, expected_standby_health='unhealthy')
def test_active_link_drop_upstream(upper_tor_host, lower_tor_host, send_server_to_t1_with_action, toggle_all_simulator_ports_to_upper_tor, drop_flow_upper_tor): """ Send traffic from servers to T1 and remove the flow between the servers and the active ToR. Verify the switchover and disruption last < 1 second. """ send_server_to_t1_with_action(upper_tor_host, verify=True, delay=1, action=drop_flow_upper_tor) verify_tor_states(expected_active_host=lower_tor_host, expected_standby_host=upper_tor_host, expected_standby_health="unhealthy")
def test_tor_switch_upstream(upper_tor_host, lower_tor_host, send_server_to_t1_with_action, toggle_all_simulator_ports_to_upper_tor, force_active_tor): """ Send upstream traffic and perform switchover via CLI. Confirm switchover occurs and disruption lasts < 1 second """ send_server_to_t1_with_action( upper_tor_host, verify=True, delay=1, action=lambda: force_active_tor(lower_tor_host, 'all')) verify_tor_states(expected_active_host=lower_tor_host, expected_standby_host=upper_tor_host)
def test_standby_tor_downlink_down_upstream( upper_tor_host, lower_tor_host, send_server_to_t1_with_action, toggle_all_simulator_ports_to_upper_tor, shutdown_lower_tor_downlink_intfs): """ Send traffic from server to T1 and shutdown the standby ToR downlink on DUT. Verify no switchover and no disruption """ send_server_to_t1_with_action(upper_tor_host, verify=True, allowed_disruption=0, action=shutdown_lower_tor_downlink_intfs) verify_tor_states(expected_active_host=upper_tor_host, expected_standby_host=lower_tor_host, expected_standby_health='unhealthy')
def test_active_tor_downlink_down_upstream( upper_tor_host, lower_tor_host, send_server_to_t1_with_action, toggle_all_simulator_ports_to_upper_tor, shutdown_upper_tor_downlink_intfs): """ Send traffic from server to T1 and shutdown the active ToR downlink on DUT. Verify switchover and disruption lasts < 1 second """ send_server_to_t1_with_action(upper_tor_host, verify=True, delay=MUX_SIM_ALLOWED_DISRUPTION_SEC, allowed_disruption=1, action=shutdown_upper_tor_downlink_intfs) verify_tor_states(expected_active_host=lower_tor_host, expected_standby_host=upper_tor_host, expected_standby_health='unhealthy')
def test_standby_tor_kill_bgpd_upstream( upper_tor_host, lower_tor_host, send_server_to_t1_with_action, toggle_all_simulator_ports_to_upper_tor, kill_bgpd): ''' Case: Server -> ToR -> T1 (Standby ToR BGP Down) Action: Shutdown all BGP sessions on the standby ToR Expectation: Verify packet flow after the standby ToR (B) loses BGP sessions ToR A DBs indicate active, ToR B DBs indicate standby T1 switch receives packet from the active ToR (A), and not the standby ToR (B) ''' send_server_to_t1_with_action(upper_tor_host, verify=True, action=lambda: kill_bgpd(lower_tor_host)) verify_tor_states(expected_active_host=upper_tor_host, expected_standby_host=lower_tor_host)
def test_standby_link_drop_upstream(upper_tor_host, lower_tor_host, send_server_to_t1_with_action, check_simulator_flap_counter, drop_flow_lower_tor): """ Send traffic from servers to T1 and remove the flow between the servers and the standby ToR. Verify that no switchover and disruption occur. """ send_server_to_t1_with_action(upper_tor_host, verify=True, delay=0, action=drop_flow_lower_tor) verify_tor_states(expected_active_host=upper_tor_host, expected_standby_host=lower_tor_host, expected_standby_health="unhealthy") check_simulator_flap_counter(0)
def test_active_link_down_upstream( upper_tor_host, lower_tor_host, send_server_to_t1_with_action, toggle_all_simulator_ports_to_upper_tor, shutdown_fanout_upper_tor_intfs ): """ Send traffic from server to T1 and shutdown the active ToR link. Verify switchover and disruption lasts < 1 second """ send_server_to_t1_with_action( upper_tor_host, verify=True, delay=1, action=shutdown_fanout_upper_tor_intfs ) verify_tor_states( expected_active_host=lower_tor_host, expected_standby_host=upper_tor_host, expected_standby_health='unhealthy' )
def test_active_tor_kill_bgpd_upstream(upper_tor_host, lower_tor_host, send_server_to_t1_with_action, toggle_all_simulator_ports_to_upper_tor, kill_bgpd): ''' Case: Server -> ToR -> T1 (Active ToR BGP Down) Action: Shutdown all BGP sessions on the active ToR Expectation: Verify packet flow after the active ToR (A) loses BGP sessions ToR A DBs indicate standby, ToR B DBs indicate active T1 switch receives packet from the initial standby ToR (B) and not the active ToR (A) Verify traffic interruption < threshold ''' send_server_to_t1_with_action(upper_tor_host, verify=True, delay=MUX_SIM_ALLOWED_DISRUPTION_SEC, action=lambda: kill_bgpd(upper_tor_host)) verify_tor_states(expected_active_host=lower_tor_host, expected_standby_host=upper_tor_host)
def test_active_tor_shutdown_bgp_upstream( upper_tor_host, lower_tor_host, send_server_to_t1_with_action, toggle_all_simulator_ports_to_upper_tor, shutdown_tor_bgp): ''' Case: Server -> ToR -> T1 (Active ToR BGP Down) Action: Shutdown all BGP sessions on the active ToR Expectation: Verify packet flow after the active ToR (A) loses BGP sessions ToR A DBs indicate standby, ToR B DBs indicate active T1 switch receives packet from the new active ToR (B) and not the new standby ToR (A) Verify traffic interruption < 1 second ''' send_server_to_t1_with_action( upper_tor_host, verify=True, delay=1, action=lambda: shutdown_tor_bgp(upper_tor_host) ) verify_tor_states( expected_active_host=lower_tor_host, expected_standby_host=upper_tor_host )