def test_link_flap(request, duthosts, all_ports, fanouthosts):
tlf = TestLinkFlap(request)
dutname, portname = decode_dut_port_name(all_ports)
for dut in duthosts:
if dutname == 'unknown' or dutname == dut.hostname:
tlf.run_link_flap_test(dut, fanouthosts, portname)
def enum_dut_portname_module_fixture(request):
param = request.param
dutname, portname = decode_dut_port_name(param)
if dutname not in cadidate_test_ports.keys(
) or portname not in cadidate_test_ports[dutname].keys():
pytest.skip('%s port was not selected for test' % param)
return dutname, portname
def test_lag(common_setup_teardown, duthosts, tbinfo, nbrhosts, fanouthosts,
conn_graph_facts, enum_dut_portchannel, testcase):
ptfhost = common_setup_teardown
dut_name, dut_lag = decode_dut_port_name(enum_dut_portchannel)
some_test_ran = False
for duthost in duthosts:
if dut_name in ['unknown', duthost.hostname]:
lag_facts = duthost.lag_facts(
host=duthost.hostname)['ansible_facts']['lag_facts']
test_instance = LagTest(duthost, tbinfo, ptfhost, nbrhosts,
fanouthosts, conn_graph_facts)
# Test for each lag
if dut_lag == "unknown":
test_lags = lag_facts['names']
else:
pytest_require(
dut_lag in lag_facts['names'],
"No lag {} configuration found in {}".format(
dut_lag, duthost.hostname))
test_lags = [dut_lag]
for lag_name in test_lags:
if testcase in ["single_lag", "lacp_rate"]:
try:
lag_facts['lags'][lag_name]['po_config']['runner'][
'min_ports']
except KeyError:
msg = "Skip {} for lag {} due to min_ports not exists".format(
testcase, lag_name)
pytest_require(lag_name == "unknown", msg)
logger.info(msg)
continue
else:
some_test_ran = True
if testcase == "single_lag":
test_instance.run_single_lag_test(
lag_name, lag_facts)
else:
test_instance.run_single_lag_lacp_rate_test(
lag_name, lag_facts)
else: # fallback testcase
try:
lag_facts['lags'][lag_name]['po_config']['runner'][
'fallback']
except KeyError:
msg = "Skip {} for lag {} due to fallback was not set for it".format(
testcase, lag_name)
pytest_require(lag_name == "unknown", msg)
continue
else:
some_test_ran = True
test_instance.run_lag_fallback_test(
lag_name, lag_facts)
pytest_assert(some_test_ran, "Didn't run any test.")
def test_force_speed(enum_dut_portname_module_fixture):
"""Test all candidate ports to force to all supported speeds one by one and verify
that the port operational status is up after auto negotiation
"""
dutname, portname = decode_dut_port_name(enum_dut_portname_module_fixture)
if dutname not in cadidate_test_ports.keys(
) or portname not in cadidate_test_ports[dutname].keys():
return
duthost, dut_port, fanout, fanout_port = cadidate_test_ports[dutname][
portname]
logger.info('Start test for DUT port {} and fanout port {}'.format(
dut_port, fanout_port))
# Disable auto negotiation on fanout port
success = fanout.set_auto_negotiation_mode(fanout_port, False)
if not success:
# Fanout does not support set auto negotiation mode for this port
logger.info(
'Ignore port {} due to fanout port {} does not support setting auto-neg mode'
.format(dut_port, fanout_port))
return
logger.info(
'Trying to get a common supported speeds set among dut port, fanout port and cable'
)
supported_speeds = get_supported_speeds_for_port(duthost, dut_port, fanout,
fanout_port)
if not supported_speeds:
logger.warn(
'Ignore test for port {} due to cannot get supported speed for it'.
format(dut_port))
return
logger.info(
'Run test based on supported speeds: {}'.format(supported_speeds))
duthost.shell('config interface autoneg {} disabled'.format(dut_port))
for speed in supported_speeds:
success = fanout.set_speed(fanout_port, speed)
if not success:
logger.info(
'Skip speed {} because fanout does not support it'.format(
speed))
continue
duthost.shell('config interface speed {} {}'.format(dut_port, speed))
logger.info(
'Wait until the port status is up, expected speed: {}'.format(
speed))
wait_result = wait_until(SINGLE_PORT_WAIT_TIME,
PORT_STATUS_CHECK_INTERVAL, 0, check_ports_up,
duthost, [dut_port], speed)
pytest_assert(wait_result, '{} are still down'.format(dut_port))
fanout_actual_speed = fanout.get_speed(fanout_port)
pytest_assert(
fanout_actual_speed == speed,
'expect fanout speed: {}, but got {}'.format(
speed, fanout_actual_speed))
def test_link_flap(request, duthosts, enum_dut_portname, fanouthosts):
"""
Validates that link flap works as expected
"""
tlf = TestLinkFlap(request)
dutname, portname = decode_dut_port_name(enum_dut_portname)
for dut in duthosts:
if dutname == 'unknown' or dutname == dut.hostname:
tlf.run_link_flap_test(dut, fanouthosts, portname)
def test_containers_autorestart(duthosts, enum_dut_feature_container,
enum_rand_one_per_hwsku_frontend_hostname, tbinfo):
"""
@summary: Test the auto-restart feature of each container against two scenarios: killing
a non-critical process to verify the container is still running; killing each
critical process to verify the container will be stopped and restarted
"""
dut_name, container_name = decode_dut_port_name(enum_dut_feature_container)
pytest_require(dut_name == enum_rand_one_per_hwsku_frontend_hostname and container_name != "unknown",
"Skips testing auto-restart of container '{}' on DuT '{}' since another DuT '{}' was chosen."
.format(container_name, dut_name, enum_rand_one_per_hwsku_frontend_hostname))
duthost = duthosts[dut_name]
run_test_on_single_container(duthost, container_name, tbinfo)
def test_containers_autorestart(duthosts, enum_dut_feature,
rand_one_dut_hostname, tbinfo):
"""
@summary: Test the auto-restart feature of each container against two scenarios: killing
a non-critical process to verify the container is still running; killing each
critical process to verify the container will be stopped and restarted
"""
dut_name, feature = decode_dut_port_name(enum_dut_feature)
pytest_require(
dut_name == rand_one_dut_hostname and feature != "unknown",
"Skip test on dut host {} (chosen {}) feature {}".format(
dut_name, rand_one_dut_hostname, feature))
duthost = duthosts[dut_name]
run_test_on_single_container(duthost, feature, tbinfo)
def test_auto_negotiation_fanout_advertises_each_speed(enum_dut_portname_module_fixture, dut_all_supported_speeds):
"""
Test the case when DUT advertises all supported speeds while fanout advertises one speed at a time.
Verify that the port operational status is up after auto negotiation
"""
dutname, portname = decode_dut_port_name(enum_dut_portname_module_fixture)
if dutname not in cadidate_test_ports.keys() or portname not in cadidate_test_ports[dutname].keys():
pytest.skip('%s port was not selected for test' % enum_dut_portname_module_fixture)
duthost, dut_port, fanout, fanout_port = cadidate_test_ports[dutname][portname]
logger.info('Start test for DUT port {} and fanout port {}'.format(dut_port, fanout_port))
dut_advertised_speeds = dut_all_supported_speeds[dutname][dut_port]
duthost.shell('config interface autoneg {} enabled'.format(dut_port))
duthost.shell('config interface advertised-speeds {} {}'.format(dut_port, dut_advertised_speeds))
logger.info('Trying to get a common supported speed set among dut port, fanout port and cable')
supported_speeds = get_supported_speeds_for_port(duthost, dut_port, fanout, fanout_port)
if not supported_speeds:
logger.warn('Ignore test for port {} due to cannot get supported speed for it'.format(dut_port))
return
logger.info('Run test based on supported speeds: {}'.format(supported_speeds))
success = fanout.set_auto_negotiation_mode(fanout_port, True)
if not success:
# Fanout does not support set auto negotiation mode for this port
logger.info('Ignore port {} due to fanout port {} does not support setting auto-neg mode'.format(dut_port, fanout_port))
return
for speed in supported_speeds:
success = fanout.set_speed(fanout_port, speed)
if not success:
# Fanout does not support set advertise speeds for this port
logger.info('Ignore port {} due to fanout port {} does not support setting advertised speeds'.format(dut_port, fanout_port))
continue
logger.info('Wait until the port status is up, expected speed: {}'.format(speed))
wait_result = wait_until(SINGLE_PORT_WAIT_TIME,
PORT_STATUS_CHECK_INTERVAL,
check_ports_up,
duthost,
[dut_port],
speed)
pytest_assert(wait_result, '{} are still down. Advertised speeds: DUT = {}, fanout = {}'
.format(dut_port, dut_advertised_speeds, speed))
fanout_actual_speed = fanout.get_speed(fanout_port)
pytest_assert(fanout_actual_speed == speed, 'expected fanout speed: {}, but got {}'.format(speed, fanout_actual_speed))
def recover_ports(duthosts, enum_dut_portname_module_fixture, fanouthosts):
"""Module level fixture that automatically do following job:
1. Build global candidate test ports
2. Save fanout port state before the test
3. Restor fanout and DUT after test
Args:
duthosts: DUT object
enum_dut_portname_module_fixture (str): DUT port name
fanouthosts: Fanout objects
"""
global cadidate_test_ports
fanout_original_port_states = {}
dutname, portname = decode_dut_port_name(enum_dut_portname_module_fixture)
logger.info('Collecting existing port configuration for DUT and fanout...')
for duthost in duthosts:
if dutname == 'unknown' or dutname == duthost.hostname:
all_ports = build_test_candidates(duthost, fanouthosts, portname)
# Test all ports takes too much time (sometimes more than an hour),
# so we choose 3 ports randomly as the cadidates ports
cadidate_test_ports[duthost] = random.sample(all_ports, 3)
for _, fanout, fanout_port in cadidate_test_ports[duthost]:
auto_neg_mode = fanout.get_auto_negotiation_mode(fanout_port)
speed = fanout.get_speed(fanout_port)
if not fanout in fanout_original_port_states:
fanout_original_port_states[fanout] = {}
fanout_original_port_states[fanout][fanout_port] = (
auto_neg_mode, speed)
yield
logger.info('Recovering port configuration for fanout...')
for fanout, port_data in fanout_original_port_states.items():
for port, state in port_data.items():
fanout.set_auto_negotiation_mode(port, state[0])
fanout.set_speed(port, state[1])
logger.info('Recovering port configuration for DUT...')
for duthost in duthosts:
config_reload(duthost)
def ignore_expected_loganalyzer_exception(loganalyzer, enum_dut_feature):
"""
Ignore expected failure/error messages during testing the autorestart feature.
First, since we killed a critical process in a specific container to test the feature of
autorestart, we expect to see error messages which were fired by Monit such as
"ERR monit[563]: 'lldp|lldpd' status failed (1) -- 'lldpd' is not running."
Second, if teammgrd process was killed for testing, orchagent process in
swss container would write the error messages into syslog such as
"ERR swss#orchagent: :- removeLag: Failed to remove ref count 1 LAG PortChannel10008."
When teamd was restarted, there was an error message in the syslog: "ERR teamd#teamsyncd:
:- readData: netlink reports an error=-33 on reading a netlink socket."
Third, during pmon container was restarted due to ledd process was killed for testing,
xcvrd process would write an error message into syslog such as "ERR pmon#xcvrd[29]: :-initialize
GlobalConfig: Sonic database config global file doesn't exist at /var/run/redis/sonic-db/database_global.json."
thermalctld process would write an error message into syslog such as "ERR pmon#thermalctld[33]:
Caught exception while initializing thermal manager."
Fourth, if orchagent process was killed and swss container was restarted, then syncd process
would write error messages such as "ERR syncd#syncd: [none] driverEgressMemoryUpdate:1395
Error getting cosq for port 1.". At the same time, syncd process also wrote two WARNING messages
into syslog such as "WARNING syncd#syncd:- saiDiscover: skipping since it causes crash:
SAI_STP_ATTR_BRIDGE_ID". Since there was a keyword "crash" in these warning message, logAnalyzer
would fail.
Fifth, systemd would fire an error message:"ERR systemd[1]: Failed to start SNMP/TEAMD container." since
SNMP/TEAMD container hits the limitation of restart. route_check.py also wrote an error message into syslog.
"""
swss_syncd_teamd_regex = [
".*ERR swss#orchagent.*removeLag.*",
".*ERR syncd#syncd.*driverEgressMemoryUpdate.*",
".*ERR syncd#syncd.*brcm_sai*",
".*ERR syncd#syncd.*SAI_API_UNSPECIFIED:sai_api_query.*",
".*ERR syncd#syncd.*SAI_API_SWITCH:sai_query_attribute_enum_values_capability.*",
".*ERR syncd#syncd.*SAI_API_SWITCH:sai_object_type_get_availability.*",
".*ERR syncd#syncd.*sendApiResponse: api SAI_COMMON_API_SET failed in syncd mode.*",
".*ERR syncd#syncd.*processQuadEvent.*",
".*WARNING syncd#syncd.*skipping since it causes crash.*",
".*ERR swss#portsyncd.*readData.*netlink reports an error=-33 on reading a netlink socket.*",
".*ERR swss#orchagent.*set status: SAI_STATUS_ATTR_NOT_IMPLEMENTED_0.*",
".*ERR swss#orchagent.*setIntfVlanFloodType.*",
]
ignore_regex_dict = {
'common': [
".*ERR monit.*",
".*ERR systemd.*Failed to start .* container*",
".*ERR kernel.*PortChannel.*",
".*ERR route_check.*",
],
'pmon': [
".*ERR pmon#xcvrd.*initializeGlobalConfig.*",
".*ERR pmon#thermalctld.*Caught exception while initializing thermal manager.*",
],
'swss':
swss_syncd_teamd_regex,
'syncd':
swss_syncd_teamd_regex,
'teamd':
swss_syncd_teamd_regex,
}
_, feature = decode_dut_port_name(enum_dut_feature)
if loganalyzer:
loganalyzer.ignore_regex.extend(ignore_regex_dict['common'])
if feature in ignore_regex_dict:
loganalyzer.ignore_regex.extend(ignore_regex_dict[feature])
