def test_link_flap(request, duthosts, all_ports, fanouthosts): tlf = TestLinkFlap(request) dutname, portname = decode_dut_port_name(all_ports) for dut in duthosts: if dutname == 'unknown' or dutname == dut.hostname: tlf.run_link_flap_test(dut, fanouthosts, portname)
def enum_dut_portname_module_fixture(request): param = request.param dutname, portname = decode_dut_port_name(param) if dutname not in cadidate_test_ports.keys( ) or portname not in cadidate_test_ports[dutname].keys(): pytest.skip('%s port was not selected for test' % param) return dutname, portname
def test_lag(common_setup_teardown, duthosts, tbinfo, nbrhosts, fanouthosts, conn_graph_facts, enum_dut_portchannel, testcase): ptfhost = common_setup_teardown dut_name, dut_lag = decode_dut_port_name(enum_dut_portchannel) some_test_ran = False for duthost in duthosts: if dut_name in ['unknown', duthost.hostname]: lag_facts = duthost.lag_facts( host=duthost.hostname)['ansible_facts']['lag_facts'] test_instance = LagTest(duthost, tbinfo, ptfhost, nbrhosts, fanouthosts, conn_graph_facts) # Test for each lag if dut_lag == "unknown": test_lags = lag_facts['names'] else: pytest_require( dut_lag in lag_facts['names'], "No lag {} configuration found in {}".format( dut_lag, duthost.hostname)) test_lags = [dut_lag] for lag_name in test_lags: if testcase in ["single_lag", "lacp_rate"]: try: lag_facts['lags'][lag_name]['po_config']['runner'][ 'min_ports'] except KeyError: msg = "Skip {} for lag {} due to min_ports not exists".format( testcase, lag_name) pytest_require(lag_name == "unknown", msg) logger.info(msg) continue else: some_test_ran = True if testcase == "single_lag": test_instance.run_single_lag_test( lag_name, lag_facts) else: test_instance.run_single_lag_lacp_rate_test( lag_name, lag_facts) else: # fallback testcase try: lag_facts['lags'][lag_name]['po_config']['runner'][ 'fallback'] except KeyError: msg = "Skip {} for lag {} due to fallback was not set for it".format( testcase, lag_name) pytest_require(lag_name == "unknown", msg) continue else: some_test_ran = True test_instance.run_lag_fallback_test( lag_name, lag_facts) pytest_assert(some_test_ran, "Didn't run any test.")
def test_force_speed(enum_dut_portname_module_fixture): """Test all candidate ports to force to all supported speeds one by one and verify that the port operational status is up after auto negotiation """ dutname, portname = decode_dut_port_name(enum_dut_portname_module_fixture) if dutname not in cadidate_test_ports.keys( ) or portname not in cadidate_test_ports[dutname].keys(): return duthost, dut_port, fanout, fanout_port = cadidate_test_ports[dutname][ portname] logger.info('Start test for DUT port {} and fanout port {}'.format( dut_port, fanout_port)) # Disable auto negotiation on fanout port success = fanout.set_auto_negotiation_mode(fanout_port, False) if not success: # Fanout does not support set auto negotiation mode for this port logger.info( 'Ignore port {} due to fanout port {} does not support setting auto-neg mode' .format(dut_port, fanout_port)) return logger.info( 'Trying to get a common supported speeds set among dut port, fanout port and cable' ) supported_speeds = get_supported_speeds_for_port(duthost, dut_port, fanout, fanout_port) if not supported_speeds: logger.warn( 'Ignore test for port {} due to cannot get supported speed for it'. format(dut_port)) return logger.info( 'Run test based on supported speeds: {}'.format(supported_speeds)) duthost.shell('config interface autoneg {} disabled'.format(dut_port)) for speed in supported_speeds: success = fanout.set_speed(fanout_port, speed) if not success: logger.info( 'Skip speed {} because fanout does not support it'.format( speed)) continue duthost.shell('config interface speed {} {}'.format(dut_port, speed)) logger.info( 'Wait until the port status is up, expected speed: {}'.format( speed)) wait_result = wait_until(SINGLE_PORT_WAIT_TIME, PORT_STATUS_CHECK_INTERVAL, 0, check_ports_up, duthost, [dut_port], speed) pytest_assert(wait_result, '{} are still down'.format(dut_port)) fanout_actual_speed = fanout.get_speed(fanout_port) pytest_assert( fanout_actual_speed == speed, 'expect fanout speed: {}, but got {}'.format( speed, fanout_actual_speed))
def test_link_flap(request, duthosts, enum_dut_portname, fanouthosts): """ Validates that link flap works as expected """ tlf = TestLinkFlap(request) dutname, portname = decode_dut_port_name(enum_dut_portname) for dut in duthosts: if dutname == 'unknown' or dutname == dut.hostname: tlf.run_link_flap_test(dut, fanouthosts, portname)
def test_containers_autorestart(duthosts, enum_dut_feature_container, enum_rand_one_per_hwsku_frontend_hostname, tbinfo): """ @summary: Test the auto-restart feature of each container against two scenarios: killing a non-critical process to verify the container is still running; killing each critical process to verify the container will be stopped and restarted """ dut_name, container_name = decode_dut_port_name(enum_dut_feature_container) pytest_require(dut_name == enum_rand_one_per_hwsku_frontend_hostname and container_name != "unknown", "Skips testing auto-restart of container '{}' on DuT '{}' since another DuT '{}' was chosen." .format(container_name, dut_name, enum_rand_one_per_hwsku_frontend_hostname)) duthost = duthosts[dut_name] run_test_on_single_container(duthost, container_name, tbinfo)
def test_containers_autorestart(duthosts, enum_dut_feature, rand_one_dut_hostname, tbinfo): """ @summary: Test the auto-restart feature of each container against two scenarios: killing a non-critical process to verify the container is still running; killing each critical process to verify the container will be stopped and restarted """ dut_name, feature = decode_dut_port_name(enum_dut_feature) pytest_require( dut_name == rand_one_dut_hostname and feature != "unknown", "Skip test on dut host {} (chosen {}) feature {}".format( dut_name, rand_one_dut_hostname, feature)) duthost = duthosts[dut_name] run_test_on_single_container(duthost, feature, tbinfo)
def test_auto_negotiation_fanout_advertises_each_speed(enum_dut_portname_module_fixture, dut_all_supported_speeds): """ Test the case when DUT advertises all supported speeds while fanout advertises one speed at a time. Verify that the port operational status is up after auto negotiation """ dutname, portname = decode_dut_port_name(enum_dut_portname_module_fixture) if dutname not in cadidate_test_ports.keys() or portname not in cadidate_test_ports[dutname].keys(): pytest.skip('%s port was not selected for test' % enum_dut_portname_module_fixture) duthost, dut_port, fanout, fanout_port = cadidate_test_ports[dutname][portname] logger.info('Start test for DUT port {} and fanout port {}'.format(dut_port, fanout_port)) dut_advertised_speeds = dut_all_supported_speeds[dutname][dut_port] duthost.shell('config interface autoneg {} enabled'.format(dut_port)) duthost.shell('config interface advertised-speeds {} {}'.format(dut_port, dut_advertised_speeds)) logger.info('Trying to get a common supported speed set among dut port, fanout port and cable') supported_speeds = get_supported_speeds_for_port(duthost, dut_port, fanout, fanout_port) if not supported_speeds: logger.warn('Ignore test for port {} due to cannot get supported speed for it'.format(dut_port)) return logger.info('Run test based on supported speeds: {}'.format(supported_speeds)) success = fanout.set_auto_negotiation_mode(fanout_port, True) if not success: # Fanout does not support set auto negotiation mode for this port logger.info('Ignore port {} due to fanout port {} does not support setting auto-neg mode'.format(dut_port, fanout_port)) return for speed in supported_speeds: success = fanout.set_speed(fanout_port, speed) if not success: # Fanout does not support set advertise speeds for this port logger.info('Ignore port {} due to fanout port {} does not support setting advertised speeds'.format(dut_port, fanout_port)) continue logger.info('Wait until the port status is up, expected speed: {}'.format(speed)) wait_result = wait_until(SINGLE_PORT_WAIT_TIME, PORT_STATUS_CHECK_INTERVAL, check_ports_up, duthost, [dut_port], speed) pytest_assert(wait_result, '{} are still down. Advertised speeds: DUT = {}, fanout = {}' .format(dut_port, dut_advertised_speeds, speed)) fanout_actual_speed = fanout.get_speed(fanout_port) pytest_assert(fanout_actual_speed == speed, 'expected fanout speed: {}, but got {}'.format(speed, fanout_actual_speed))
def recover_ports(duthosts, enum_dut_portname_module_fixture, fanouthosts): """Module level fixture that automatically do following job: 1. Build global candidate test ports 2. Save fanout port state before the test 3. Restor fanout and DUT after test Args: duthosts: DUT object enum_dut_portname_module_fixture (str): DUT port name fanouthosts: Fanout objects """ global cadidate_test_ports fanout_original_port_states = {} dutname, portname = decode_dut_port_name(enum_dut_portname_module_fixture) logger.info('Collecting existing port configuration for DUT and fanout...') for duthost in duthosts: if dutname == 'unknown' or dutname == duthost.hostname: all_ports = build_test_candidates(duthost, fanouthosts, portname) # Test all ports takes too much time (sometimes more than an hour), # so we choose 3 ports randomly as the cadidates ports cadidate_test_ports[duthost] = random.sample(all_ports, 3) for _, fanout, fanout_port in cadidate_test_ports[duthost]: auto_neg_mode = fanout.get_auto_negotiation_mode(fanout_port) speed = fanout.get_speed(fanout_port) if not fanout in fanout_original_port_states: fanout_original_port_states[fanout] = {} fanout_original_port_states[fanout][fanout_port] = ( auto_neg_mode, speed) yield logger.info('Recovering port configuration for fanout...') for fanout, port_data in fanout_original_port_states.items(): for port, state in port_data.items(): fanout.set_auto_negotiation_mode(port, state[0]) fanout.set_speed(port, state[1]) logger.info('Recovering port configuration for DUT...') for duthost in duthosts: config_reload(duthost)
def ignore_expected_loganalyzer_exception(loganalyzer, enum_dut_feature): """ Ignore expected failure/error messages during testing the autorestart feature. First, since we killed a critical process in a specific container to test the feature of autorestart, we expect to see error messages which were fired by Monit such as "ERR monit[563]: 'lldp|lldpd' status failed (1) -- 'lldpd' is not running." Second, if teammgrd process was killed for testing, orchagent process in swss container would write the error messages into syslog such as "ERR swss#orchagent: :- removeLag: Failed to remove ref count 1 LAG PortChannel10008." When teamd was restarted, there was an error message in the syslog: "ERR teamd#teamsyncd: :- readData: netlink reports an error=-33 on reading a netlink socket." Third, during pmon container was restarted due to ledd process was killed for testing, xcvrd process would write an error message into syslog such as "ERR pmon#xcvrd[29]: :-initialize GlobalConfig: Sonic database config global file doesn't exist at /var/run/redis/sonic-db/database_global.json." thermalctld process would write an error message into syslog such as "ERR pmon#thermalctld[33]: Caught exception while initializing thermal manager." Fourth, if orchagent process was killed and swss container was restarted, then syncd process would write error messages such as "ERR syncd#syncd: [none] driverEgressMemoryUpdate:1395 Error getting cosq for port 1.". At the same time, syncd process also wrote two WARNING messages into syslog such as "WARNING syncd#syncd:- saiDiscover: skipping since it causes crash: SAI_STP_ATTR_BRIDGE_ID". Since there was a keyword "crash" in these warning message, logAnalyzer would fail. Fifth, systemd would fire an error message:"ERR systemd[1]: Failed to start SNMP/TEAMD container." since SNMP/TEAMD container hits the limitation of restart. route_check.py also wrote an error message into syslog. """ swss_syncd_teamd_regex = [ ".*ERR swss#orchagent.*removeLag.*", ".*ERR syncd#syncd.*driverEgressMemoryUpdate.*", ".*ERR syncd#syncd.*brcm_sai*", ".*ERR syncd#syncd.*SAI_API_UNSPECIFIED:sai_api_query.*", ".*ERR syncd#syncd.*SAI_API_SWITCH:sai_query_attribute_enum_values_capability.*", ".*ERR syncd#syncd.*SAI_API_SWITCH:sai_object_type_get_availability.*", ".*ERR syncd#syncd.*sendApiResponse: api SAI_COMMON_API_SET failed in syncd mode.*", ".*ERR syncd#syncd.*processQuadEvent.*", ".*WARNING syncd#syncd.*skipping since it causes crash.*", ".*ERR swss#portsyncd.*readData.*netlink reports an error=-33 on reading a netlink socket.*", ".*ERR swss#orchagent.*set status: SAI_STATUS_ATTR_NOT_IMPLEMENTED_0.*", ".*ERR swss#orchagent.*setIntfVlanFloodType.*", ] ignore_regex_dict = { 'common': [ ".*ERR monit.*", ".*ERR systemd.*Failed to start .* container*", ".*ERR kernel.*PortChannel.*", ".*ERR route_check.*", ], 'pmon': [ ".*ERR pmon#xcvrd.*initializeGlobalConfig.*", ".*ERR pmon#thermalctld.*Caught exception while initializing thermal manager.*", ], 'swss': swss_syncd_teamd_regex, 'syncd': swss_syncd_teamd_regex, 'teamd': swss_syncd_teamd_regex, } _, feature = decode_dut_port_name(enum_dut_feature) if loganalyzer: loganalyzer.ignore_regex.extend(ignore_regex_dict['common']) if feature in ignore_regex_dict: loganalyzer.ignore_regex.extend(ignore_regex_dict[feature])