def test_server_down(duthosts, tbinfo, rand_selected_interface,
simulator_flap_counter, simulator_server_down,
toggle_simulator_port_to_upper_tor, loganalyzer):
"""
Verify that mux cable is not toggled excessively.
"""
pytest_require('dualtor' in tbinfo['topo']['name'],
"Only run on dualtor testbed")
for analyzer in list(loganalyzer.values()):
analyzer.ignore_regex.append(
r".*ERR swss#orchagent: :- setState: State transition from active to active is not-handled"
)
upper_tor = duthosts[tbinfo['duts'][0]]
lower_tor = duthosts[tbinfo['duts'][1]]
PAUSE_TIME = 5
itfs, _ = rand_selected_interface
# Set upper_tor as active
toggle_simulator_port_to_upper_tor(itfs)
time.sleep(PAUSE_TIME)
mux_flap_counter_0 = simulator_flap_counter(itfs)
# Server down
simulator_server_down(itfs)
time.sleep(PAUSE_TIME)
# Verify mux_cable state on upper_tor is active
mux_state_upper_tor = show_muxcable_status(upper_tor)
pytest_assert(
mux_state_upper_tor[itfs]['status'] == 'active'
and mux_state_upper_tor[itfs]['health'] == 'unhealthy',
"mux_cable status is unexpected. Should be (active, unhealthy)")
# Verify mux_cable state on lower_tor is standby
mux_state_lower_tor = show_muxcable_status(lower_tor)
pytest_assert(
mux_state_lower_tor[itfs]['status'] == 'standby'
and mux_state_lower_tor[itfs]['health'] == 'unhealthy',
"mux_cable status is unexpected. Should be (standby, unhealthy)")
# Verify that mux_cable flap_counter should be no larger than 3
# lower_tor(standby) -> active -> standby
# upper_tor(active) -> active
# The toggle from both tor may be overlapped and invisible
mux_flap_counter_1 = simulator_flap_counter(itfs)
pytest_assert(
mux_flap_counter_1 - mux_flap_counter_0 <= 3,
"The mux_cable flap count should be no larger than 3 ({})".format(
mux_flap_counter_1 - mux_flap_counter_0))
def test_server_down(duthosts, tbinfo, rand_selected_interface, simulator_flap_counter, simulator_server_down, toggle_simulator_port_to_upper_tor, loganalyzer):
"""
Verify that mux cable is not toggled excessively.
"""
for analyzer in list(loganalyzer.values()):
analyzer.ignore_regex.append(r".*ERR swss#orchagent: :- setState: State transition from active to active is not-handled")
upper_tor = duthosts[tbinfo['duts'][0]]
lower_tor = duthosts[tbinfo['duts'][1]]
def upper_tor_mux_state_verification(state, health):
mux_state_upper_tor = show_muxcable_status(upper_tor)
return mux_state_upper_tor[itfs]['status'] == state and mux_state_upper_tor[itfs]['health'] == health
def lower_tor_mux_state_verfication(state, health):
mux_state_lower_tor = show_muxcable_status(lower_tor)
return mux_state_lower_tor[itfs]['status'] == state and mux_state_lower_tor[itfs]['health'] == health
itfs, _ = rand_selected_interface
# Set upper_tor as active
toggle_simulator_port_to_upper_tor(itfs)
pytest_assert(wait_until(30, 1, 0, upper_tor_mux_state_verification, 'active', 'healthy'),
"mux_cable status is unexpected. Should be (active, healthy). Test can't proceed. ")
mux_flap_counter_0 = simulator_flap_counter(itfs)
# Server down
simulator_server_down(itfs)
# Verify mux_cable state on upper_tor is active
pytest_assert(wait_until(20, 1, 0, upper_tor_mux_state_verification, 'active', 'unhealthy'),
"mux_cable status is unexpected. Should be (active, unhealthy)")
# Verify mux_cable state on lower_tor is standby
pytest_assert(wait_until(20, 1, 0, lower_tor_mux_state_verfication, 'standby', 'unhealthy'),
"mux_cable status is unexpected. Should be (standby, unhealthy)")
# Verify that mux_cable flap_counter should be no larger than 3
# lower_tor(standby) -> active -> standby
# upper_tor(active) -> active
# The toggle from both tor may be overlapped and invisible
mux_flap_counter_1 = simulator_flap_counter(itfs)
pytest_assert(mux_flap_counter_1 - mux_flap_counter_0 <= 3,
"The mux_cable flap count should be no larger than 3 ({})".format(mux_flap_counter_1 - mux_flap_counter_0))
def check_simulator_flap_counter(simulator_flap_counter,
toggle_all_simulator_ports_to_upper_tor,
tor_mux_intfs):
"""Check the flap count for each server-facing interfaces."""
def set_expected_counter_diff(diff):
"""Set expected counter difference."""
expected_diff.append(diff)
expected_diff = []
tor_mux_intfs = [str(_) for _ in tor_mux_intfs]
counters_before = {
intf: simulator_flap_counter(intf)
for intf in tor_mux_intfs
}
yield set_expected_counter_diff
counters_after = {
intf: simulator_flap_counter(intf)
for intf in tor_mux_intfs
}
logging.info(
"\n%s\n",
tabulate.tabulate(
[[intf, counters_before[intf], counters_after[intf]]
for intf in tor_mux_intfs],
headers=["port", "flap counter before", "flap counter after"]))
counter_diffs = {
intf: counters_after[intf] - counters_before[intf]
for intf in tor_mux_intfs
}
if expected_diff:
not_expected_counter_diffs = [
intf for intf, counter_diff in counter_diffs.items()
if counter_diff != expected_diff[-1]
]
error_str = json.dumps(not_expected_counter_diffs, indent=4)
if not_expected_counter_diffs:
logging.error(error_str)
raise ValueError(error_str)