def test_normal_op_downstream_standby(upper_tor_host, lower_tor_host,
send_t1_to_server_with_action,
toggle_all_simulator_ports_to_upper_tor):
"""
Send downstream traffic to the standby ToR and confirm no disruption or
switchover occurs
"""
send_t1_to_server_with_action(lower_tor_host, verify=True)
verify_tor_states(expected_active_host=upper_tor_host,
expected_standby_host=lower_tor_host)
def test_standby_config_reload_downstream_active(
upper_tor_host, lower_tor_host, send_t1_to_server_with_action,
toggle_all_simulator_ports_to_upper_tor):
"""
Send downstream traffic to the active ToR and `config reload` the
standby ToR. Confirm no switchover occurs and no disruption
"""
send_t1_to_server_with_action(
upper_tor_host,
verify=True,
action=lambda: config_reload(lower_tor_host, wait=0))
verify_tor_states(expected_active_host=upper_tor_host,
expected_standby_host=lower_tor_host)
def test_standby_tor_heartbeat_failure_downstream_standby(
toggle_all_simulator_ports_to_upper_tor, upper_tor_host,
lower_tor_host, send_t1_to_server_with_action, shutdown_tor_heartbeat):
"""
Send downstream traffic from T1 to the standby ToR and stop the LinkProber module on the standby ToR.
Confirm no switchover and no disruption.
"""
send_t1_to_server_with_action(
lower_tor_host,
verify=True,
action=lambda: shutdown_tor_heartbeat(lower_tor_host))
verify_tor_states(expected_active_host=upper_tor_host,
expected_standby_host=lower_tor_host)
def test_active_config_reload_downstream_standby(
upper_tor_host, lower_tor_host, send_t1_to_server_with_action,
toggle_all_simulator_ports_to_upper_tor):
"""
Send downstream traffic to the standby ToR and `config reload` the
active ToR. Confirm switchover occurs and disruption lasts < 1 second
"""
send_t1_to_server_with_action(
lower_tor_host,
verify=True,
delay=CONFIG_RELOAD_ALLOWED_DISRUPTION_SEC,
action=lambda: config_reload(upper_tor_host, wait=0))
verify_tor_states(expected_active_host=lower_tor_host,
expected_standby_host=upper_tor_host)
def test_active_tor_heartbeat_failure_downstream_standby(
toggle_all_simulator_ports_to_upper_tor, upper_tor_host,
lower_tor_host, send_t1_to_server_with_action, shutdown_tor_heartbeat):
"""
Send downstream traffic from T1 to the standby ToR and stop the LinkProber module on the active ToR.
Confirm switchover and disruption lasts < 1 second.
"""
send_t1_to_server_with_action(
lower_tor_host,
verify=True,
delay=MUX_SIM_ALLOWED_DISRUPTION_SEC,
action=lambda: shutdown_tor_heartbeat(upper_tor_host))
verify_tor_states(expected_active_host=lower_tor_host,
expected_standby_host=upper_tor_host)
def test_standby_tor_reboot_downstream_active(
upper_tor_host, lower_tor_host, send_t1_to_server_with_action,
toggle_all_simulator_ports_to_upper_tor, reboot_tor,
wait_for_device_reachable):
"""
Send downstream traffic to the active ToR and reboot the standby ToR.
Confirm no switchover occurred and no disruption
"""
send_t1_to_server_with_action(upper_tor_host,
verify=True,
action=lambda: reboot_tor(lower_tor_host))
wait_for_device_reachable(lower_tor_host)
verify_tor_states(expected_active_host=upper_tor_host,
expected_standby_host=lower_tor_host)
def test_standby_tor_downlink_down_downstream_standby(
upper_tor_host, lower_tor_host, send_t1_to_server_with_action,
toggle_all_simulator_ports_to_upper_tor,
shutdown_lower_tor_downlink_intfs):
"""
Send traffic from T1 to standby ToR and shutdwon the standby ToR downlink on DUT.
Confirm no switchover and no disruption
"""
send_t1_to_server_with_action(lower_tor_host,
verify=True,
allowed_disruption=0,
action=shutdown_lower_tor_downlink_intfs)
verify_tor_states(expected_active_host=upper_tor_host,
expected_standby_host=lower_tor_host,
expected_standby_health='unhealthy')
def test_active_link_drop_downstream_standby(
upper_tor_host, lower_tor_host, send_t1_to_server_with_action,
toggle_all_simulator_ports_to_upper_tor, drop_flow_upper_tor):
"""
Send traffic from the T1s to the servers via the standby Tor and remove the flow between the
servers and the active ToR.
Verify the switchover and disruption last < 1 second.
"""
send_t1_to_server_with_action(lower_tor_host,
verify=True,
delay=1,
action=drop_flow_upper_tor)
verify_tor_states(expected_active_host=lower_tor_host,
expected_standby_host=upper_tor_host,
expected_standby_health="unhealthy")
def test_active_link_down_downstream_active(
upper_tor_host, lower_tor_host, send_t1_to_server_with_action,
toggle_all_simulator_ports_to_upper_tor,
shutdown_fanout_upper_tor_intfs):
"""
Send traffic from T1 to active ToR and shutdown the active ToR link.
Verify switchover and disruption lasts < 1 second
"""
send_t1_to_server_with_action(upper_tor_host,
verify=True,
delay=MUX_SIM_ALLOWED_DISRUPTION_SEC,
action=shutdown_fanout_upper_tor_intfs)
verify_tor_states(expected_active_host=lower_tor_host,
expected_standby_host=upper_tor_host,
expected_standby_health='unhealthy')
def test_active_tor_reboot_downstream_standby(
upper_tor_host, lower_tor_host, send_t1_to_server_with_action,
toggle_all_simulator_ports_to_upper_tor, reboot_tor,
wait_for_device_reachable):
"""
Send downstream traffic to the standby ToR and reboot the active ToR.
Confirm switchover occurred and disruption lasts < 1 second
"""
send_t1_to_server_with_action(lower_tor_host,
verify=True,
delay=MUX_SIM_ALLOWED_DISRUPTION_SEC,
action=lambda: reboot_tor(upper_tor_host))
wait_for_device_reachable(upper_tor_host)
verify_tor_states(expected_active_host=lower_tor_host,
expected_standby_host=upper_tor_host)
def test_standby_link_down_downstream_standby(
upper_tor_host, lower_tor_host, send_t1_to_server_with_action,
toggle_all_simulator_ports_to_upper_tor,
shutdown_fanout_lower_tor_intfs):
"""
Send traffic from T1 to standby ToR and shutdwon the standby ToR link.
Confirm no switchover and no disruption
"""
send_t1_to_server_with_action(lower_tor_host,
verify=True,
delay=MUX_SIM_ALLOWED_DISRUPTION_SEC,
action=shutdown_fanout_lower_tor_intfs)
verify_tor_states(expected_active_host=upper_tor_host,
expected_standby_host=lower_tor_host,
expected_standby_health='unhealthy')
def test_tor_switch_downstream_standby(upper_tor_host, lower_tor_host,
send_t1_to_server_with_action,
toggle_all_simulator_ports_to_upper_tor,
force_active_tor):
"""
Send downstream traffic to the standby ToR and perform switchover via CLI.
Confirm switchover occurs and disruption lasts < 1 second
"""
send_t1_to_server_with_action(
lower_tor_host,
verify=True,
delay=1,
action=lambda: force_active_tor(lower_tor_host, 'all'))
verify_tor_states(expected_active_host=lower_tor_host,
expected_standby_host=upper_tor_host)
def test_standby_link_drop_downstream_standby(upper_tor_host, lower_tor_host,
send_t1_to_server_with_action,
check_simulator_flap_counter,
drop_flow_lower_tor):
"""
Send traffic from the T1s to the servers via the standby Tor and remove the flow between the
servers and the standby ToR.
Verify that no switchover and disruption occur.
"""
send_t1_to_server_with_action(lower_tor_host,
verify=True,
delay=0,
action=drop_flow_lower_tor)
verify_tor_states(expected_active_host=upper_tor_host,
expected_standby_host=lower_tor_host,
expected_standby_health="unhealthy")
check_simulator_flap_counter(0)
def test_standby_tor_kill_bgpd_downstream_active(
upper_tor_host, lower_tor_host, send_t1_to_server_with_action,
toggle_all_simulator_ports_to_upper_tor, kill_bgpd,
tunnel_traffic_monitor):
'''
Case: T1 -> Active ToR -> Server (Standby ToR BGP Down)
Action: Shutdown all BGP sessions on the standby ToR
Expectation:
Verify packet flow after the standby ToR (B) loses BGP sessions
T1 switch receives no IP-in-IP packet; server receives packet
'''
with tunnel_traffic_monitor(lower_tor_host, existing=False):
send_t1_to_server_with_action(upper_tor_host,
verify=True,
action=lambda: kill_bgpd(lower_tor_host))
verify_tor_states(expected_active_host=upper_tor_host,
expected_standby_host=lower_tor_host)
def test_active_tor_kill_bgpd_downstream_standby(
upper_tor_host, lower_tor_host, send_t1_to_server_with_action,
toggle_all_simulator_ports_to_upper_tor, kill_bgpd,
tunnel_traffic_monitor):
'''
Case: T1 -> Standby ToR -> Server (Active ToR BGP Down)
Action: Shutdown all BGP sessions on the active ToR
Expectation:
Verify packet flow after the active ToR (A) loses BGP sessions
Verify ToR A standby, ToR B active
Verify traffic interruption is < 1 second
'''
send_t1_to_server_with_action(lower_tor_host,
verify=True,
delay=MUX_SIM_ALLOWED_DISRUPTION_SEC,
action=lambda: kill_bgpd(upper_tor_host))
verify_tor_states(expected_active_host=lower_tor_host,
expected_standby_host=upper_tor_host)
def test_active_tor_kill_bgpd_downstream_standby(
upper_tor_host, lower_tor_host, send_t1_to_server_with_action,
toggle_all_simulator_ports_to_upper_tor, kill_bgpd,
tunnel_traffic_monitor):
'''
Case: T1 -> Standby ToR -> Server (Active ToR BGP Down)
Action: Shutdown all BGP sessions on the active ToR
Expectation:
Verify packet flow after the active ToR (A) loses BGP sessions
T1 switch continues to receive IP-in-IP traffic, from lower to upper ToR
No switchover occurs
verify traffic interruption is < 1 second
'''
with tunnel_traffic_monitor(lower_tor_host, existing=True):
send_t1_to_server_with_action(lower_tor_host,
verify=True,
delay=1,
action=lambda: kill_bgpd(upper_tor_host))
verify_tor_states(expected_active_host=upper_tor_host,
expected_standby_host=lower_tor_host)
def test_active_tor_shutdown_bgp_downstream_standby(
upper_tor_host, lower_tor_host, send_t1_to_server_with_action,
toggle_all_simulator_ports_to_upper_tor, shutdown_tor_bgp,
tunnel_traffic_monitor):
'''
Case: T1 -> Standby ToR -> Server (Active ToR BGP Down)
Action: Shutdown all BGP sessions on the active ToR
Expectation:
Verify packet flow after the active ToR (A) loses BGP sessions
T1 switch receives no IP-in-IP packet; server receives packet;
verify traffic interruption is < 1 second
'''
with tunnel_traffic_monitor(lower_tor_host, existing=False):
send_t1_to_server_with_action(
lower_tor_host, verify=True, delay=1,
action=lambda: shutdown_tor_bgp(upper_tor_host)
)
verify_tor_states(
expected_active_host=lower_tor_host,
expected_standby_host=upper_tor_host
)
