def test_normal_op_upstream(upper_tor_host, lower_tor_host,
send_server_to_t1_with_action,
toggle_all_simulator_ports_to_upper_tor):
"""Send upstream traffic and confirm no disruption or switchover occurs"""
send_server_to_t1_with_action(upper_tor_host, verify=True)
verify_tor_states(expected_active_host=upper_tor_host,
expected_standby_host=lower_tor_host)
def test_standby_config_reload_upstream(
upper_tor_host, lower_tor_host, send_server_to_t1_with_action,
toggle_all_simulator_ports_to_upper_tor):
"""
Send upstream traffic and `config reload` the standby ToR. Confirm no
switchover occurs and no disruption
"""
send_server_to_t1_with_action(
upper_tor_host,
verify=True,
action=lambda: config_reload(lower_tor_host, wait=0))
verify_tor_states(expected_active_host=upper_tor_host,
expected_standby_host=lower_tor_host)
def test_standby_tor_heartbeat_failure_upstream(
toggle_all_simulator_ports_to_upper_tor, upper_tor_host,
lower_tor_host, send_server_to_t1_with_action, shutdown_tor_heartbeat):
"""
Send upstream traffic and stop the LinkProber module on the standby ToR.
Confirm no switchover and no disruption.
"""
send_server_to_t1_with_action(
upper_tor_host,
verify=True,
action=lambda: shutdown_tor_heartbeat(lower_tor_host))
verify_tor_states(expected_active_host=upper_tor_host,
expected_standby_host=lower_tor_host)
def test_active_tor_heartbeat_failure_upstream(
toggle_all_simulator_ports_to_upper_tor, upper_tor_host,
lower_tor_host, send_server_to_t1_with_action, shutdown_tor_heartbeat):
"""
Send upstream traffic and stop the LinkProber module on the active ToR.
Confirm switchover and disruption lasts < 1 second.
"""
send_server_to_t1_with_action(
upper_tor_host,
verify=True,
delay=MUX_SIM_ALLOWED_DISRUPTION_SEC,
action=lambda: shutdown_tor_heartbeat(upper_tor_host))
verify_tor_states(expected_active_host=lower_tor_host,
expected_standby_host=upper_tor_host)
def test_standby_tor_reboot_upstream(upper_tor_host, lower_tor_host,
send_server_to_t1_with_action,
toggle_all_simulator_ports_to_upper_tor,
reboot_tor, wait_for_device_reachable):
"""
Send upstream traffic and reboot the standby ToR. Confirm no switchover
occurred and no disruption
"""
send_server_to_t1_with_action(upper_tor_host,
verify=True,
action=lambda: reboot_tor(lower_tor_host))
wait_for_device_reachable(lower_tor_host)
verify_tor_states(expected_active_host=upper_tor_host,
expected_standby_host=lower_tor_host)
def test_active_config_reload_upstream(
upper_tor_host, lower_tor_host, send_server_to_t1_with_action,
toggle_all_simulator_ports_to_upper_tor):
"""
Send upstream traffic and `config reload` the active ToR. Confirm
switchover occurs and disruption lasted < 1 second
"""
send_server_to_t1_with_action(
upper_tor_host,
verify=True,
delay=CONFIG_RELOAD_ALLOWED_DISRUPTION_SEC,
action=lambda: config_reload(upper_tor_host, wait=0))
verify_tor_states(expected_active_host=lower_tor_host,
expected_standby_host=upper_tor_host)
def test_normal_op_upstream(
upper_tor_host, lower_tor_host, send_server_to_t1_with_action,
toggle_all_simulator_ports_to_upper_tor,
toggle_all_ports_both_tors_admin_forwarding_state_to_active,
cable_type):
"""Send upstream traffic and confirm no disruption or switchover occurs"""
send_server_to_t1_with_action(upper_tor_host, verify=True, stop_after=60)
if cable_type == CableType.active_standby:
verify_tor_states(expected_active_host=upper_tor_host,
expected_standby_host=lower_tor_host)
else:
verify_tor_states(
expected_active_host=[upper_tor_host, lower_tor_host],
cable_type=cable_type)
def test_active_tor_reboot_upstream(upper_tor_host, lower_tor_host,
send_server_to_t1_with_action,
toggle_all_simulator_ports_to_upper_tor,
reboot_tor, wait_for_device_reachable):
"""
Send upstream traffic and reboot the active ToR. Confirm switchover
occurred and disruption lasts < 1 second
"""
send_server_to_t1_with_action(upper_tor_host,
verify=True,
delay=MUX_SIM_ALLOWED_DISRUPTION_SEC,
action=lambda: reboot_tor(upper_tor_host))
wait_for_device_reachable(upper_tor_host)
verify_tor_states(expected_active_host=lower_tor_host,
expected_standby_host=upper_tor_host)
def test_standby_link_down_upstream(upper_tor_host, lower_tor_host,
send_server_to_t1_with_action,
toggle_all_simulator_ports_to_upper_tor,
shutdown_fanout_lower_tor_intfs):
"""
Send traffic from server to T1 and shutdown the standby ToR link.
Verify no switchover and no disruption
"""
send_server_to_t1_with_action(upper_tor_host,
verify=True,
delay=MUX_SIM_ALLOWED_DISRUPTION_SEC,
action=shutdown_fanout_lower_tor_intfs)
verify_tor_states(expected_active_host=upper_tor_host,
expected_standby_host=lower_tor_host,
expected_standby_health='unhealthy')
def test_active_link_drop_upstream(upper_tor_host, lower_tor_host,
send_server_to_t1_with_action,
toggle_all_simulator_ports_to_upper_tor,
drop_flow_upper_tor):
"""
Send traffic from servers to T1 and remove the flow between the servers and the active ToR.
Verify the switchover and disruption last < 1 second.
"""
send_server_to_t1_with_action(upper_tor_host,
verify=True,
delay=1,
action=drop_flow_upper_tor)
verify_tor_states(expected_active_host=lower_tor_host,
expected_standby_host=upper_tor_host,
expected_standby_health="unhealthy")
def test_tor_switch_upstream(upper_tor_host, lower_tor_host,
send_server_to_t1_with_action,
toggle_all_simulator_ports_to_upper_tor,
force_active_tor):
"""
Send upstream traffic and perform switchover via CLI. Confirm switchover
occurs and disruption lasts < 1 second
"""
send_server_to_t1_with_action(
upper_tor_host,
verify=True,
delay=1,
action=lambda: force_active_tor(lower_tor_host, 'all'))
verify_tor_states(expected_active_host=lower_tor_host,
expected_standby_host=upper_tor_host)
def test_standby_tor_downlink_down_upstream(
upper_tor_host, lower_tor_host, send_server_to_t1_with_action,
toggle_all_simulator_ports_to_upper_tor,
shutdown_lower_tor_downlink_intfs):
"""
Send traffic from server to T1 and shutdown the standby ToR downlink on DUT.
Verify no switchover and no disruption
"""
send_server_to_t1_with_action(upper_tor_host,
verify=True,
allowed_disruption=0,
action=shutdown_lower_tor_downlink_intfs)
verify_tor_states(expected_active_host=upper_tor_host,
expected_standby_host=lower_tor_host,
expected_standby_health='unhealthy')
def test_active_tor_downlink_down_upstream(
upper_tor_host, lower_tor_host, send_server_to_t1_with_action,
toggle_all_simulator_ports_to_upper_tor,
shutdown_upper_tor_downlink_intfs):
"""
Send traffic from server to T1 and shutdown the active ToR downlink on DUT.
Verify switchover and disruption lasts < 1 second
"""
send_server_to_t1_with_action(upper_tor_host,
verify=True,
delay=MUX_SIM_ALLOWED_DISRUPTION_SEC,
allowed_disruption=1,
action=shutdown_upper_tor_downlink_intfs)
verify_tor_states(expected_active_host=lower_tor_host,
expected_standby_host=upper_tor_host,
expected_standby_health='unhealthy')
def test_standby_tor_kill_bgpd_upstream(
upper_tor_host, lower_tor_host, send_server_to_t1_with_action,
toggle_all_simulator_ports_to_upper_tor, kill_bgpd):
'''
Case: Server -> ToR -> T1 (Standby ToR BGP Down)
Action: Shutdown all BGP sessions on the standby ToR
Expectation:
Verify packet flow after the standby ToR (B) loses BGP sessions
ToR A DBs indicate active, ToR B DBs indicate standby
T1 switch receives packet from the active ToR (A), and not the standby ToR (B)
'''
send_server_to_t1_with_action(upper_tor_host,
verify=True,
action=lambda: kill_bgpd(lower_tor_host))
verify_tor_states(expected_active_host=upper_tor_host,
expected_standby_host=lower_tor_host)
def test_standby_link_drop_upstream(upper_tor_host, lower_tor_host,
send_server_to_t1_with_action,
check_simulator_flap_counter,
drop_flow_lower_tor):
"""
Send traffic from servers to T1 and remove the flow between the servers and the standby ToR.
Verify that no switchover and disruption occur.
"""
send_server_to_t1_with_action(upper_tor_host,
verify=True,
delay=0,
action=drop_flow_lower_tor)
verify_tor_states(expected_active_host=upper_tor_host,
expected_standby_host=lower_tor_host,
expected_standby_health="unhealthy")
check_simulator_flap_counter(0)
def test_active_link_down_upstream(
upper_tor_host, lower_tor_host, send_server_to_t1_with_action,
toggle_all_simulator_ports_to_upper_tor,
shutdown_fanout_upper_tor_intfs
):
"""
Send traffic from server to T1 and shutdown the active ToR link.
Verify switchover and disruption lasts < 1 second
"""
send_server_to_t1_with_action(
upper_tor_host, verify=True, delay=1,
action=shutdown_fanout_upper_tor_intfs
)
verify_tor_states(
expected_active_host=lower_tor_host,
expected_standby_host=upper_tor_host,
expected_standby_health='unhealthy'
)
def test_active_tor_kill_bgpd_upstream(upper_tor_host, lower_tor_host,
send_server_to_t1_with_action,
toggle_all_simulator_ports_to_upper_tor,
kill_bgpd):
'''
Case: Server -> ToR -> T1 (Active ToR BGP Down)
Action: Shutdown all BGP sessions on the active ToR
Expectation:
Verify packet flow after the active ToR (A) loses BGP sessions
ToR A DBs indicate standby, ToR B DBs indicate active
T1 switch receives packet from the initial standby ToR (B) and not the active ToR (A)
Verify traffic interruption < threshold
'''
send_server_to_t1_with_action(upper_tor_host,
verify=True,
delay=MUX_SIM_ALLOWED_DISRUPTION_SEC,
action=lambda: kill_bgpd(upper_tor_host))
verify_tor_states(expected_active_host=lower_tor_host,
expected_standby_host=upper_tor_host)
def test_active_tor_shutdown_bgp_upstream(
upper_tor_host, lower_tor_host, send_server_to_t1_with_action,
toggle_all_simulator_ports_to_upper_tor, shutdown_tor_bgp):
'''
Case: Server -> ToR -> T1 (Active ToR BGP Down)
Action: Shutdown all BGP sessions on the active ToR
Expectation:
Verify packet flow after the active ToR (A) loses BGP sessions
ToR A DBs indicate standby, ToR B DBs indicate active
T1 switch receives packet from the new active ToR (B) and not the new standby ToR (A)
Verify traffic interruption < 1 second
'''
send_server_to_t1_with_action(
upper_tor_host, verify=True, delay=1,
action=lambda: shutdown_tor_bgp(upper_tor_host)
)
verify_tor_states(
expected_active_host=lower_tor_host,
expected_standby_host=upper_tor_host
)
