def acl_rules(self, duthosts, rand_one_dut_hostname, localhost, setup, acl_table, populate_vlan_arp_entries): """Setup/teardown ACL rules for the current set of tests. Args: duthosts: All DUTs belong to the testbed. rand_one_dut_hostname: hostname of a random chosen dut to run test. localhost: The host from which tests are run. setup: Parameters for the ACL tests. acl_table: Configuration info for the ACL table. populate_vlan_arp_entries: A function to populate ARP/FDB tables for VLAN interfaces. """ duthost = duthosts[rand_one_dut_hostname] loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix="acl_rules") loganalyzer.load_common_config() try: loganalyzer.expect_regex = [LOG_EXPECT_ACL_RULE_CREATE_RE] with loganalyzer: self.setup_rules(duthost, acl_table) self.post_setup_hook(duthost, localhost, populate_vlan_arp_entries) except LogAnalyzerError as err: # Cleanup Config DB if rule creation failed logger.error("ACL table creation failed, attempting to clean-up...") self.teardown_rules(duthost) raise err try: yield finally: loganalyzer.expect_regex = [LOG_EXPECT_ACL_RULE_REMOVE_RE] with loganalyzer: logger.info("Removing ACL rules") self.teardown_rules(duthost)
def test_dynamic_minimum_table(duthost, mocker_factory): air_flow_dirs = ['p2c', 'c2p', 'unk'] max_temperature = 45000 # 45 C cooling_cur_state = get_cooling_cur_state(duthost) if cooling_cur_state >= COOLING_CUR_STATE_THRESHOLD: pytest.skip('The cooling level {} is higher than threshold {}.'.format(cooling_cur_state, COOLING_CUR_STATE_THRESHOLD)) mocker = mocker_factory(duthost, 'MinTableMocker') loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix='thermal_control') loganalyzer.load_common_config() for index in range(len(air_flow_dirs)): air_flow_index = random.randint(0, len(air_flow_dirs) - 1) air_flow_dir = air_flow_dirs[air_flow_index] air_flow_dirs.remove(air_flow_dir) temperature = random.randint(0, max_temperature) trust_state = True if random.randint(0, 1) else False logging.info('Testing with air_flow_dir={}, temperature={}, trust_state={}'.format(air_flow_dir, temperature, trust_state)) expect_minimum_cooling_level = mocker.get_expect_cooling_level(air_flow_dir, temperature, trust_state) loganalyzer.expect_regex = [LOG_EXPECT_CHANGE_MIN_COOLING_LEVEL_RE.format(expect_minimum_cooling_level)] with loganalyzer: mocker.mock_min_table(air_flow_dir, temperature, trust_state) time.sleep(THERMAL_CONTROL_TEST_WAIT_TIME) temperature = random.randint(0, max_temperature) logging.info('Testing with air_flow_dir={}, temperature={}, trust_state={}'.format(air_flow_dir, temperature, not trust_state)) expect_minimum_cooling_level = mocker.get_expect_cooling_level(air_flow_dir, temperature, not trust_state) loganalyzer.expect_regex = [LOG_EXPECT_CHANGE_MIN_COOLING_LEVEL_RE.format(expect_minimum_cooling_level)] with loganalyzer: mocker.mock_min_table(air_flow_dir, temperature, not trust_state) time.sleep(THERMAL_CONTROL_TEST_WAIT_TIME)
def acl_rules(self, duthost, localhost, setup, acl_table): """ setup/teardown ACL rules based on test class requirements :param duthost: DUT host object :param localhost: localhost object :param setup: setup information :param acl_table: table creating fixture :return: """ loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix='acl_rules') loganalyzer.load_common_config() try: loganalyzer.expect_regex = [LOG_EXPECT_ACL_RULE_CREATE_RE] with loganalyzer: self.setup_rules(duthost, setup, acl_table) self.post_setup_hook(duthost, localhost) except LogAnalyzerError as err: # cleanup config DB in case of log analysis error self.teardown_rules(duthost, setup) raise err try: yield finally: loganalyzer.expect_regex = [LOG_EXPECT_ACL_RULE_REMOVE_RE] with loganalyzer: self.teardown_rules(duthost, setup)
def acl(duthost, acl_setup): """ setup/teardown ACL rules based on test class requirements :param duthost: DUT host object :param acl_setup: setup information :return: """ loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix='acl') loganalyzer.load_common_config() try: loganalyzer.expect_regex = [LOG_EXPECT_ACL_RULE_CREATE_RE] with loganalyzer: setup_acl_rules(duthost, acl_setup) except LogAnalyzerError as err: # cleanup config DB in case of log analysis error teardown_acl(duthost, acl_setup) raise err try: yield finally: loganalyzer.expect_regex = [LOG_EXPECT_ACL_RULE_REMOVE_RE] with loganalyzer: teardown_acl(duthost, acl_setup)
def acl(duthosts, enum_rand_one_per_hwsku_frontend_hostname, acl_setup): """ setup/teardown ACL rules based on test class requirements :param duthost: DUT host object :param acl_setup: setup information :return: """ duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] acl_facts = duthost.acl_facts()["ansible_facts"]["ansible_acl_facts"] pytest_require(ACL_TABLE_NAME in acl_facts, "{} acl table not exists") loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix='acl') loganalyzer.load_common_config() try: loganalyzer.expect_regex = [LOG_EXPECT_ACL_RULE_CREATE_RE] with loganalyzer: setup_acl_rules(duthost, acl_setup) except LogAnalyzerError as err: # cleanup config DB in case of log analysis error teardown_acl(duthost, acl_setup) raise err try: yield finally: loganalyzer.expect_regex = [LOG_EXPECT_ACL_RULE_REMOVE_RE] with loganalyzer: teardown_acl(duthost, acl_setup)
def acl_table(duthost, acl_table_config): """ fixture to apply ACL table configuration and remove after tests :param duthost: DUT object :param acl_table_config: ACL table configuration dictionary :return: forwards acl_table_config """ name = acl_table_config['name'] conf = acl_table_config['config_file'] loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix='acl') loganalyzer.load_common_config() try: loganalyzer.expect_regex = [LOG_EXPECT_ACL_TABLE_CREATE_RE] with loganalyzer: logger.info('creating ACL table: applying {}'.format(conf)) # TODO: use sonic config CLI duthost.command('sonic-cfggen -j {} --write-to-db'.format(conf)) except LogAnalyzerError as err: # cleanup config DB if create failed duthost.command('config acl remove table {}'.format(name)) raise err try: yield acl_table_config finally: loganalyzer.expect_regex = [LOG_EXPECT_ACL_TABLE_REMOVE_RE] with loganalyzer: logger.info('removing ACL table {}'.format(name)) duthost.command('config acl remove table {}'.format(name)) # save cleaned configuration duthost.command('config save -y')
def acl_table(duthosts, rand_one_dut_hostname, setup, stage, ip_version, backup_and_restore_config_db_module): """Apply ACL table configuration and remove after tests. Args: duthosts: All DUTs belong to the testbed. rand_one_dut_hostname: hostname of a random chosen dut to run test. setup: Parameters for the ACL tests. stage: The ACL stage under test. ip_version: The IP version under test. backup_and_restore_config_db_module: A fixture that handles restoring Config DB after the tests are over. Yields: The ACL table configuration. """ table_name = "DATA_{}_{}_TEST".format(stage.upper(), ip_version.upper()) acl_table_config = { "table_name": table_name, "table_ports": ",".join(setup["acl_table_ports"]['']), "table_stage": stage, "table_type": "L3" if ip_version == "ipv4" else "L3V6" } logger.info("Generated ACL table configuration:\n{}".format( pprint.pformat(acl_table_config))) dut_to_analyzer_map = {} for duthost in duthosts: loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix="acl") loganalyzer.load_common_config() dut_to_analyzer_map[duthost] = loganalyzer try: loganalyzer.expect_regex = [LOG_EXPECT_ACL_TABLE_CREATE_RE] with loganalyzer: create_or_remove_acl_table(duthost, acl_table_config, setup, "add") except LogAnalyzerError as err: # Cleanup Config DB if table creation failed logger.error( "ACL table creation failed, attempting to clean-up...") create_or_remove_acl_table(duthost, acl_table_config, setup, "remove") raise err try: yield acl_table_config finally: for duthost, loganalyzer in dut_to_analyzer_map.items(): loganalyzer.expect_regex = [LOG_EXPECT_ACL_TABLE_REMOVE_RE] with loganalyzer: create_or_remove_acl_table(duthost, acl_table_config, setup, "remove")
def disable_analyzer_for_mellanox(duthost): if duthost.facts["asic_type"] in ["mellanox"]: loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix='sfp_cfg') loganalyzer.load_common_config() loganalyzer.ignore_regex.append("kernel.*Eeprom query failed*") marker = loganalyzer.init() yield if duthost.facts["asic_type"] in ["mellanox"]: loganalyzer.analyze(marker)
def ignore_particular_error_log(request, duthosts, rand_one_dut_hostname): duthost = duthosts[rand_one_dut_hostname] loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix='turn_on_off_psu_and_check_psustatus') loganalyzer.load_common_config() ignore_list = request.param loganalyzer.ignore_regex.extend(ignore_list) marker = loganalyzer.init() yield loganalyzer.analyze(marker)
def ignore_log_analyzer_by_vendor(request, duthosts, enum_rand_one_per_hwsku_hostname): duthost = duthosts[enum_rand_one_per_hwsku_hostname] asic_type = duthost.facts["asic_type"] ignore_asic_list = request.param if asic_type not in ignore_asic_list: loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix=request.node.name) loganalyzer.load_common_config() marker = loganalyzer.init() yield loganalyzer.analyze(marker) else: yield
def disable_analyzer_for_mellanox(duthost): if duthost.facts["asic_type"] in ["mellanox"]: loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix='sfp_cfg') loganalyzer.load_common_config() loganalyzer.ignore_regex.append("kernel.*Eeprom query failed*") # Ignore PMPE error https://github.com/Azure/sonic-buildimage/issues/7163 loganalyzer.ignore_regex.append( r".*ERR pmon#xcvrd: Receive PMPE error event on module.*") marker = loganalyzer.init() yield if duthost.facts["asic_type"] in ["mellanox"]: loganalyzer.analyze(marker)
def acl_rules(self, duthosts, localhost, setup, acl_table, populate_vlan_arp_entries, tbinfo, ip_version): """Setup/teardown ACL rules for the current set of tests. Args: duthosts: All DUTs belong to the testbed. rand_one_dut_hostname: hostname of a random chosen dut to run test. localhost: The host from which tests are run. setup: Parameters for the ACL tests. acl_table: Configuration info for the ACL table. populate_vlan_arp_entries: A function to populate ARP/FDB tables for VLAN interfaces. """ dut_to_analyzer_map = {} for duthost in duthosts: loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix="acl_rules") loganalyzer.load_common_config() dut_to_analyzer_map[duthost] = loganalyzer try: loganalyzer.expect_regex = [LOG_EXPECT_ACL_RULE_CREATE_RE] # Ignore any other errors to reduce noise loganalyzer.ignore_regex = [r".*"] with loganalyzer: self.setup_rules(duthost, acl_table, ip_version) self.post_setup_hook(duthost, localhost, populate_vlan_arp_entries, tbinfo) assert self.check_rule_counters( duthost), "Rule counters should be ready!" except LogAnalyzerError as err: # Cleanup Config DB if rule creation failed logger.error( "ACL rule application failed, attempting to clean-up...") self.teardown_rules(duthost) raise err try: yield finally: for duthost, loganalyzer in dut_to_analyzer_map.items(): loganalyzer.expect_regex = [LOG_EXPECT_ACL_RULE_REMOVE_RE] with loganalyzer: logger.info("Removing ACL rules") self.teardown_rules(duthost)
def mirroring(duthosts, enum_rand_one_per_hwsku_frontend_hostname, neighbor_ip, mirror_setup, gre_version): """ fixture gathers all configuration fixtures :param duthost: DUT host :param mirror_setup: mirror_setup fixture :param mirror_config: mirror_config fixture """ duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] logger.info("Adding mirror_session to DUT") acl_rule_file = os.path.join(mirror_setup['dut_tmp_dir'], ACL_RULE_PERSISTENT_FILE) extra_vars = { 'acl_table_name': EVERFLOW_TABLE_NAME, } logger.info('Extra variables for MIRROR table:\n{}'.format( pprint.pformat(extra_vars))) duthost.host.options['variable_manager'].extra_vars.update(extra_vars) duthost.template(src=os.path.join(TEMPLATE_DIR, ACL_RULE_PERSISTENT_TEMPLATE), dest=acl_rule_file) duthost.command('config mirror_session add {} {} {} {} {} {} {}'.format( SESSION_INFO['name'], SESSION_INFO['src_ip'], neighbor_ip, SESSION_INFO['dscp'], SESSION_INFO['ttl'], SESSION_INFO['gre'], SESSION_INFO['queue'])) logger.info('Loading acl mirror rules ...') load_rule_cmd = "acl-loader update full {} --session_name={}".format( acl_rule_file, SESSION_INFO['name']) duthost.command('{}'.format(load_rule_cmd)) try: yield finally: loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix='acl') loganalyzer.load_common_config() try: loganalyzer.expect_regex = [LOG_EXCEPT_MIRROR_SESSION_REMOVE] with loganalyzer: teardown_mirroring(duthost, mirror_setup['dut_tmp_dir']) except LogAnalyzerError as err: raise err
def acl_table(duthosts, rand_one_dut_hostname, acl_table_config, backup_and_restore_config_db_module): """Apply ACL table configuration and remove after tests. Args: duthosts: All DUTs belong to the testbed. rand_one_dut_hostname: hostname of a random chosen dut to run test. acl_table_config: A dictionary describing the ACL table configuration to apply. backup_and_restore_config_db_module: A fixture that handles restoring Config DB after the tests are over. Yields: The ACL table configuration. """ duthost = duthosts[rand_one_dut_hostname] table_name = acl_table_config["table_name"] config_file = acl_table_config["config_file"] loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix="acl") loganalyzer.load_common_config() try: loganalyzer.expect_regex = [LOG_EXPECT_ACL_TABLE_CREATE_RE] with loganalyzer: logger.info("Creating ACL table from config file: \"{}\"".format( config_file)) # TODO: Use `config` CLI to create ACL table duthost.command( "sonic-cfggen -j {} --write-to-db".format(config_file)) except LogAnalyzerError as err: # Cleanup Config DB if table creation failed logger.error("ACL table creation failed, attempting to clean-up...") duthost.command("config acl remove table {}".format(table_name)) raise err try: yield acl_table_config finally: loganalyzer.expect_regex = [LOG_EXPECT_ACL_TABLE_REMOVE_RE] with loganalyzer: logger.info("Removing ACL table \"{}\"".format(table_name)) duthost.command("config acl remove table {}".format(table_name))
def advanceboot_loganalyzer(duthosts, rand_one_dut_hostname, request): """ Advance reboot log analysis. This fixture starts log analysis at the beginning of the test. At the end, the collected expect messages are verified and timing of start/stop is calculated. Args: duthosts : List of DUT hosts rand_one_dut_hostname: hostname of a randomly selected DUT """ duthost = duthosts[rand_one_dut_hostname] test_name = request.node.name if "warm" in test_name: reboot_type = "warm" elif "fast" in test_name: reboot_type = "fast" else: reboot_type = "unknown" # Currently, advanced reboot test would skip for kvm platform if the test has no device_type marker for vs. # Doing the same skip logic in this fixture to avoid running loganalyzer without the test executed if duthost.facts['platform'] == 'x86_64-kvm_x86_64-r0': device_marks = [ arg for mark in request.node.iter_markers(name='device_type') for arg in mark.args ] if 'vs' not in device_marks: pytest.skip('Testcase not supported for kvm') loganalyzer = LogAnalyzer( ansible_host=duthost, marker_prefix="test_advanced_reboot_{}".format(test_name), additional_files={ '/var/log/swss/sairedis.rec': 'recording on: /var/log/swss/sairedis.rec', '/var/log/frr/bgpd.log': '' }) marker = loganalyzer.init() loganalyzer.load_common_config() ignore_file = os.path.join(TEMPLATES_DIR, "ignore_boot_messages") expect_file = os.path.join(TEMPLATES_DIR, "expect_boot_messages") ignore_reg_exp = loganalyzer.parse_regexp_file(src=ignore_file) expect_reg_exp = loganalyzer.parse_regexp_file(src=expect_file) loganalyzer.ignore_regex.extend(ignore_reg_exp) loganalyzer.expect_regex = [] loganalyzer.expect_regex.extend(expect_reg_exp) loganalyzer.match_regex = [] yield result = loganalyzer.analyze(marker, fail=False) analyze_result = {"time_span": dict(), "offset_from_kexec": dict()} offset_from_kexec = dict() for key, messages in result["expect_messages"].items(): if "syslog" in key or "bgpd.log" in key: analyze_log_file(duthost, messages, analyze_result, offset_from_kexec) elif "sairedis.rec" in key: analyze_sairedis_rec(messages, analyze_result, offset_from_kexec) for marker, time_data in analyze_result["offset_from_kexec"].items(): marker_start_time = time_data.get("timestamp", {}).get("Start") reboot_start_time = analyze_result.get("reboot_time", {}).get("timestamp", {}).get("Start") if reboot_start_time and reboot_start_time != "N/A" and marker_start_time: time_data["time_taken"] = (datetime.strptime(marker_start_time, FMT) -\ datetime.strptime(reboot_start_time, FMT)).total_seconds() else: time_data["time_taken"] = "N/A" get_data_plane_report(analyze_result, reboot_type) result_summary = get_report_summary(analyze_result, reboot_type) logging.info(json.dumps(analyze_result, indent=4)) logging.info(json.dumps(result_summary, indent=4)) report_file_name = request.node.name + "_report.json" summary_file_name = request.node.name + "_summary.json" report_file_dir = os.path.realpath((os.path.join(os.path.dirname(__file__),\ "../logs/platform_tests/"))) report_file_path = report_file_dir + "/" + report_file_name summary_file_path = report_file_dir + "/" + summary_file_name if not os.path.exists(report_file_dir): os.makedirs(report_file_dir) with open(report_file_path, 'w') as fp: json.dump(analyze_result, fp, indent=4) with open(summary_file_path, 'w') as fp: json.dump(result_summary, fp, indent=4)
def advanceboot_loganalyzer(duthosts, rand_one_dut_hostname): """ Advance reboot log analysis. This fixture starts log analysis at the beginning of the test. At the end, the collected expect messages are verified and timing of start/stop is calculated. Args: duthosts : List of DUT hosts rand_one_dut_hostname: hostname of a randomly selected DUT """ duthost = duthosts[rand_one_dut_hostname] loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix="test_advanced_reboot") marker = loganalyzer.init() loganalyzer.load_common_config() ignore_file = os.path.join(TEMPLATES_DIR, "ignore_boot_messages") expect_file = os.path.join(TEMPLATES_DIR, "expect_boot_messages") ignore_reg_exp = loganalyzer.parse_regexp_file(src=ignore_file) expect_reg_exp = loganalyzer.parse_regexp_file(src=expect_file) loganalyzer.ignore_regex.extend(ignore_reg_exp) loganalyzer.expect_regex = [] loganalyzer.expect_regex.extend(expect_reg_exp) loganalyzer.match_regex = [] yield result = loganalyzer.analyze(marker, fail=False) messages = result["expect_messages"].values() if not messages: logging.error("Expected messages not found in syslog") return messages = messages[0] service_restart_times = dict() service_patterns = { "Stopping": re.compile(r'.*Stopping.*service.*'), "Stopped": re.compile(r'.*Stopped.*service.*'), "Starting": re.compile(r'.*Starting.*service.*'), "Started": re.compile(r'.*Started.*service.*') } def service_time_check(message, status): time = message.split(duthost.hostname)[0].strip() service_name = message.split(status + " ")[1].split()[0] service_dict = service_restart_times.get(service_name, {"timestamp": {}}) timestamps = service_dict.get("timestamp") if status in timestamps: service_dict[status + " count"] = service_dict.get(status + " count", 1) + 1 timestamps[status] = time service_restart_times.update({service_name: service_dict}) for message in messages: for status, pattern in service_patterns.items(): if re.search(pattern, message): service_time_check(message, status) loganalyzer.save_extracted_log(dest="/tmp/log/syslog") logging.info(json.dumps(service_restart_times, indent=4)) FMT = "%b %d %H:%M:%S.%f" for _, timings in service_restart_times.items(): timestamps = timings["timestamp"] timings["stop_time"] = (datetime.strptime(timestamps["Stopped"], FMT) -\ datetime.strptime(timestamps["Stopping"], FMT)).total_seconds() \ if "Stopped" in timestamps and "Stopping" in timestamps else None timings["start_time"] = (datetime.strptime(timestamps["Started"], FMT) -\ datetime.strptime(timestamps["Starting"], FMT)).total_seconds() \ if "Started" in timestamps and "Starting" in timestamps else None timings["reboot_time"] = (datetime.strptime(timestamps["Started"], FMT) -\ datetime.strptime(timestamps["Stopped"], FMT)).total_seconds() \ if "Started" in timestamps and "Stopped" in timestamps else None files = glob.glob('/tmp/*-report.json') if files: filepath = files[0] with open(filepath) as json_file: report = json.load(json_file) service_restart_times.update(report) result = service_restart_times logging.info(json.dumps(result, indent=4))
def test_turn_on_off_psu_and_check_psustatus(duthost, psu_controller): """ @summary: Turn off/on PSU and check PSU status using 'show platform psustatus' """ loganalyzer = LogAnalyzer( ansible_host=duthost, marker_prefix='turn_on_off_psu_and_check_psustatus') loganalyzer.load_common_config() loganalyzer.ignore_regex.append( "Error getting sensor data: dps460.*Kernel interface error") marker = loganalyzer.init() psu_line_pattern = re.compile(r"PSU\s+\d+\s+(OK|NOT OK|NOT PRESENT)") psu_num = get_psu_num(duthost) if psu_num < 2: pytest.skip( "At least 2 PSUs required for rest of the testing in this case") logging.info("Create PSU controller for testing") psu_ctrl = psu_controller if psu_ctrl is None: pytest.skip( "No PSU controller for %s, skip rest of the testing in this case" % duthost.hostname) logging.info( "To avoid DUT being shutdown, need to turn on PSUs that are not powered" ) turn_all_psu_on(psu_ctrl) logging.info("Initialize test results") psu_test_results = {} if not check_all_psu_on(duthost, psu_test_results): pytest.skip( "Some PSU are still down, skip rest of the testing in this case") assert len(psu_test_results.keys()) == psu_num, \ "In consistent PSU number output by '%s' and '%s'" % (CMD_PLATFORM_PSUSTATUS, "sudo psuutil numpsus") logging.info("Start testing turn off/on PSUs") all_psu_status = psu_ctrl.get_psu_status() for psu in all_psu_status: psu_under_test = None logging.info("Turn off PSU %s" % str(psu["psu_id"])) psu_ctrl.turn_off_psu(psu["psu_id"]) time.sleep(5) cli_psu_status = duthost.command(CMD_PLATFORM_PSUSTATUS) for line in cli_psu_status["stdout_lines"][2:]: assert psu_line_pattern.match(line), "Unexpected PSU status output" fields = line.split() if fields[2] != "OK": psu_under_test = fields[1] check_vendor_specific_psustatus(duthost, line) assert psu_under_test is not None, "No PSU is turned off" logging.info("Turn on PSU %s" % str(psu["psu_id"])) psu_ctrl.turn_on_psu(psu["psu_id"]) time.sleep(5) cli_psu_status = duthost.command(CMD_PLATFORM_PSUSTATUS) for line in cli_psu_status["stdout_lines"][2:]: assert psu_line_pattern.match(line), "Unexpected PSU status output" fields = line.split() if fields[1] == psu_under_test: assert fields[ 2] == "OK", "Unexpected PSU status after turned it on" check_vendor_specific_psustatus(duthost, line) psu_test_results[psu_under_test] = True for psu in psu_test_results: assert psu_test_results[psu], "Test psu status of PSU %s failed" % psu loganalyzer.analyze(marker)
def test_check_sfp_status_and_configure_sfp(duthosts, rand_one_dut_hostname, enum_frontend_asic_index, conn_graph_facts, tbinfo): """ @summary: Check SFP status and configure SFP This case is to use the sfputil tool and show command to check SFP status and configure SFP. Currently the only configuration is to reset SFP. Commands to be tested: * sfputil show presence * show interface transceiver presence * sfputil show eeprom * show interface transceiver eeprom * sfputil reset <interface name> """ duthost = duthosts[rand_one_dut_hostname] if duthost.facts["asic_type"] in ["mellanox"]: loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix='sfp_cfg') loganalyzer.load_common_config() loganalyzer.ignore_regex.append("kernel.*Eeprom query failed*") marker = loganalyzer.init() dev_conn = conn_graph_facts["device_conn"][duthost.hostname] # Get the interface pertaining to that asic portmap = get_port_map(duthost, enum_frontend_asic_index) logging.info("Got portmap {}".format(portmap)) if enum_frontend_asic_index is not None: # Check if the interfaces of this AISC is present in conn_graph_facts dev_conn = {k:v for k, v in portmap.items() if k in conn_graph_facts["device_conn"][duthost.hostname]} logging.info("ASIC {} interface_list {}".format(enum_frontend_asic_index, dev_conn)) cmd_sfp_presence = "sudo sfputil show presence" cmd_sfp_eeprom = "sudo sfputil show eeprom" cmd_sfp_reset = "sudo sfputil reset" cmd_xcvr_presence = "show interface transceiver presence" cmd_xcvr_eeprom = "show interface transceiver eeprom" global ans_host ans_host = duthost logging.info("Check output of '%s'" % cmd_sfp_presence) sfp_presence = duthost.command(cmd_sfp_presence) parsed_presence = parse_output(sfp_presence["stdout_lines"][2:]) for intf in dev_conn: assert intf in parsed_presence, "Interface is not in output of '%s'" % cmd_sfp_presence assert parsed_presence[intf] == "Present", "Interface presence is not 'Present'" logging.info("Check output of '%s'" % cmd_xcvr_presence) xcvr_presence = duthost.command(cmd_xcvr_presence) parsed_presence = parse_output(xcvr_presence["stdout_lines"][2:]) for intf in dev_conn: assert intf in parsed_presence, "Interface is not in output of '%s'" % cmd_xcvr_presence assert parsed_presence[intf] == "Present", "Interface presence is not 'Present'" logging.info("Check output of '%s'" % cmd_sfp_eeprom) sfp_eeprom = duthost.command(cmd_sfp_eeprom) parsed_eeprom = parse_eeprom(sfp_eeprom["stdout_lines"]) for intf in dev_conn: assert intf in parsed_eeprom, "Interface is not in output of 'sfputil show eeprom'" assert parsed_eeprom[intf] == "SFP EEPROM detected" logging.info("Check output of '%s'" % cmd_xcvr_eeprom) xcvr_eeprom = duthost.command(cmd_xcvr_eeprom) parsed_eeprom = parse_eeprom(xcvr_eeprom["stdout_lines"]) for intf in dev_conn: assert intf in parsed_eeprom, "Interface is not in output of '%s'" % cmd_xcvr_eeprom assert parsed_eeprom[intf] == "SFP EEPROM detected" logging.info("Test '%s <interface name>'" % cmd_sfp_reset) tested_physical_ports = set() for intf in dev_conn: phy_intf = portmap[intf][0] if phy_intf in tested_physical_ports: logging.info("skip tested SFPs {} to avoid repeating operating physical interface {}".format(intf, phy_intf)) continue tested_physical_ports.add(phy_intf) logging.info("resetting {} physical interface {}".format(intf, phy_intf)) reset_result = duthost.command("%s %s" % (cmd_sfp_reset, intf)) assert reset_result["rc"] == 0, "'%s %s' failed" % (cmd_sfp_reset, intf) time.sleep(5) logging.info("Wait some time for SFP to fully recover after reset") time.sleep(60) logging.info("Check sfp presence again after reset") sfp_presence = duthost.command(cmd_sfp_presence) parsed_presence = parse_output(sfp_presence["stdout_lines"][2:]) for intf in dev_conn: assert intf in parsed_presence, "Interface is not in output of '%s'" % cmd_sfp_presence assert parsed_presence[intf] == "Present", "Interface presence is not 'Present'" logging.info("Check interface status") namespace = duthost.get_namespace_from_asic_id(enum_frontend_asic_index) mg_facts = duthost.get_extended_minigraph_facts(tbinfo) # TODO Remove this logic when minigraph facts supports namespace in multi_asic up_ports = mg_facts["minigraph_ports"] if enum_frontend_asic_index is not None: # Check if the interfaces of this AISC is present in conn_graph_facts up_ports = {k:v for k, v in portmap.items() if k in mg_facts["minigraph_ports"]} intf_facts = duthost.interface_facts(namespace=namespace, up_ports=up_ports)["ansible_facts"] assert len(intf_facts["ansible_interface_link_down_ports"]) == 0, \ "Some interfaces are down: %s" % str(intf_facts["ansible_interface_link_down_ports"]) if duthost.facts["asic_type"] in ["mellanox"]: loganalyzer.analyze(marker)
def test_check_sfp_low_power_mode(duthosts, rand_one_dut_hostname, enum_frontend_asic_index, conn_graph_facts, tbinfo): """ @summary: Check SFP low power mode This case is to use the sfputil tool command to check and set SFP low power mode * sfputil show lpmode * sfputil lpmode off * sfputil lpmode on """ duthost = duthosts[rand_one_dut_hostname] asichost = duthost.get_asic(enum_frontend_asic_index) if duthost.facts["asic_type"] in ["mellanox"]: loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix='sfp_lpm') loganalyzer.load_common_config() loganalyzer.ignore_regex.append("Eeprom query failed") marker = loganalyzer.init() dev_conn = conn_graph_facts["device_conn"][duthost.hostname] # Get the interface pertaining to that asic portmap = get_port_map(duthost, enum_frontend_asic_index) logging.info("Got portmap {}".format(portmap)) if enum_frontend_asic_index is not None: # Check if the interfaces of this AISC is present in conn_graph_facts dev_conn = {k:v for k, v in portmap.items() if k in conn_graph_facts["device_conn"][duthost.hostname]} logging.info("ASIC {} interface_list {}".format(enum_frontend_asic_index, dev_conn)) cmd_sfp_presence = "sudo sfputil show presence" cmd_sfp_show_lpmode = "sudo sfputil show lpmode" cmd_sfp_set_lpmode = "sudo sfputil lpmode" global ans_host ans_host = duthost logging.info("Check output of '%s'" % cmd_sfp_show_lpmode) lpmode_show = duthost.command(cmd_sfp_show_lpmode) parsed_lpmode = parse_output(lpmode_show["stdout_lines"][2:]) original_lpmode = copy.deepcopy(parsed_lpmode) for intf in dev_conn: assert intf in parsed_lpmode, "Interface is not in output of '%s'" % cmd_sfp_show_lpmode assert parsed_lpmode[intf].lower() == "on" or parsed_lpmode[intf].lower() == "off", "Unexpected SFP lpmode" logging.info("Try to change SFP lpmode") tested_physical_ports = set() not_supporting_lpm_physical_ports = set() for intf in dev_conn: phy_intf = portmap[intf][0] if phy_intf in tested_physical_ports: logging.info("skip tested SFPs {} to avoid repeating operating physical interface {}".format(intf, phy_intf)) continue sfp_type_cmd = 'redis-cli -n 6 hget "TRANSCEIVER_INFO|{}" type'.format(intf) sfp_type_docker_cmd = asichost.get_docker_cmd(sfp_type_cmd, "database") sfp_type = duthost.command(sfp_type_docker_cmd)["stdout"] power_class_cmd = 'redis-cli -n 6 hget "TRANSCEIVER_INFO|{}" ext_identifier'.format(intf) power_class_docker_cmd = asichost.get_docker_cmd(power_class_cmd, "database") power_class = duthost.command(power_class_docker_cmd)["stdout"] if not "QSFP" in sfp_type or "Power Class 1" in power_class: logging.info("skip testing port {} which doesn't support LPM".format(intf)) not_supporting_lpm_physical_ports.add(phy_intf) continue tested_physical_ports.add(phy_intf) logging.info("setting {} physical interface {}".format(intf, phy_intf)) new_lpmode = "off" if original_lpmode[intf].lower() == "on" else "on" lpmode_set_result = duthost.command("%s %s %s" % (cmd_sfp_set_lpmode, new_lpmode, intf)) assert lpmode_set_result["rc"] == 0, "'%s %s %s' failed" % (cmd_sfp_set_lpmode, new_lpmode, intf) time.sleep(10) if len(tested_physical_ports) == 0: pytest.skip("None of the ports supporting LPM, skip the test") logging.info("Check SFP lower power mode again after changing SFP lpmode") lpmode_show = duthost.command(cmd_sfp_show_lpmode) parsed_lpmode = parse_output(lpmode_show["stdout_lines"][2:]) for intf in dev_conn: assert intf in parsed_lpmode, "Interface is not in output of '%s'" % cmd_sfp_show_lpmode assert parsed_lpmode[intf].lower() == "on" or parsed_lpmode[intf].lower() == "off", "Unexpected SFP lpmode" logging.info("Try to change SFP lpmode") tested_physical_ports = set() for intf in dev_conn: phy_intf = portmap[intf][0] if phy_intf in not_supporting_lpm_physical_ports: logging.info("skip testing port {} which doesn't support LPM".format(intf)) continue if phy_intf in tested_physical_ports: logging.info("skip tested SFPs {} to avoid repeating operating physical interface {}".format(intf, phy_intf)) continue tested_physical_ports.add(phy_intf) logging.info("restoring {} physical interface {}".format(intf, phy_intf)) new_lpmode = original_lpmode[intf].lower() lpmode_set_result = duthost.command("%s %s %s" % (cmd_sfp_set_lpmode, new_lpmode, intf)) assert lpmode_set_result["rc"] == 0, "'%s %s %s' failed" % (cmd_sfp_set_lpmode, new_lpmode, intf) time.sleep(10) logging.info("Check SFP lower power mode again after changing SFP lpmode") lpmode_show = duthost.command(cmd_sfp_show_lpmode) parsed_lpmode = parse_output(lpmode_show["stdout_lines"][2:]) for intf in dev_conn: assert intf in parsed_lpmode, "Interface is not in output of '%s'" % cmd_sfp_show_lpmode assert parsed_lpmode[intf].lower() == "on" or parsed_lpmode[intf].lower() == "off", "Unexpected SFP lpmode" logging.info("Check sfp presence again after setting lpmode") sfp_presence = duthost.command(cmd_sfp_presence) parsed_presence = parse_output(sfp_presence["stdout_lines"][2:]) for intf in dev_conn: assert intf in parsed_presence, "Interface is not in output of '%s'" % cmd_sfp_presence assert parsed_presence[intf] == "Present", "Interface presence is not 'Present'" logging.info("Check interface status") namespace = duthost.get_namespace_from_asic_id(enum_frontend_asic_index) mg_facts = duthost.get_extended_minigraph_facts(tbinfo) # TODO Remove this logic when minigraph facts supports namespace in multi_asic up_ports = mg_facts["minigraph_ports"] if enum_frontend_asic_index is not None: # Check if the interfaces of this AISC is present in conn_graph_facts up_ports = {k:v for k, v in portmap.items() if k in mg_facts["minigraph_ports"]} intf_facts = duthost.interface_facts(namespace=namespace, up_ports=up_ports)["ansible_facts"] assert len(intf_facts["ansible_interface_link_down_ports"]) == 0, \ "Some interfaces are down: %s" % str(intf_facts["ansible_interface_link_down_ports"]) if duthost.facts["asic_type"] in ["mellanox"]: loganalyzer.analyze(marker)
def test_thermal_control_fan_status(duthosts, rand_one_dut_hostname, mocker_factory): """ @summary: Make FAN absence, over speed and under speed, check logs and LED color. """ duthost = duthosts[rand_one_dut_hostname] loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix='thermal_control') loganalyzer.load_common_config() with ThermalPolicyFileContext(duthost, THERMAL_POLICY_VALID_FILE): fan_mocker = mocker_factory(duthost, 'FanStatusMocker') if fan_mocker is None: pytest.skip("No FanStatusMocker for %s, skip rest of the testing in this case" % duthost.facts['asic_type']) logging.info('Mock FAN status data...') fan_mocker.mock_data() # make data random restart_thermal_control_daemon(duthost) wait_until(THERMAL_CONTROL_TEST_WAIT_TIME, THERMAL_CONTROL_TEST_CHECK_INTERVAL, fan_mocker.check_all_fan_speed, 60) check_thermal_algorithm_status(duthost, mocker_factory, False) single_fan_mocker = mocker_factory(duthost, 'SingleFanMocker') time.sleep(THERMAL_CONTROL_TEST_WAIT_TIME) _fan_log_supported = duthost.command('docker exec pmon grep -E "{}" /usr/bin/thermalctld'\ .format(LOG_EXPECT_INSUFFICIENT_FAN_NUM_RE), module_ignore_errors=True) if single_fan_mocker.is_fan_removable(): loganalyzer.expect_regex = [LOG_EXPECT_FAN_REMOVE_RE, LOG_EXPECT_INSUFFICIENT_FAN_NUM_RE] if _fan_log_supported.is_failed: loganalyzer.expect_regex.remove(LOG_EXPECT_INSUFFICIENT_FAN_NUM_RE) with loganalyzer: logging.info('Mocking an absence FAN...') single_fan_mocker.mock_absence() check_cli_output_with_mocker(duthost, single_fan_mocker, CMD_PLATFORM_FANSTATUS, THERMAL_CONTROL_TEST_WAIT_TIME, 2) loganalyzer.expect_regex = [LOG_EXPECT_FAN_REMOVE_CLEAR_RE, LOG_EXPECT_INSUFFICIENT_FAN_NUM_CLEAR_RE] if _fan_log_supported.is_failed: loganalyzer.expect_regex.remove(LOG_EXPECT_INSUFFICIENT_FAN_NUM_CLEAR_RE) with loganalyzer: logging.info('Make the absence FAN back to presence...') single_fan_mocker.mock_presence() check_cli_output_with_mocker(duthost, single_fan_mocker, CMD_PLATFORM_FANSTATUS, THERMAL_CONTROL_TEST_WAIT_TIME, 2) if not _fan_log_supported.is_failed: loganalyzer.expect_regex = [LOG_EXPECT_FAN_FAULT_RE, LOG_EXPECT_INSUFFICIENT_FAN_NUM_RE] with loganalyzer: logging.info('Mocking a fault FAN...') single_fan_mocker.mock_status(False) check_cli_output_with_mocker(duthost, single_fan_mocker, CMD_PLATFORM_FANSTATUS, THERMAL_CONTROL_TEST_WAIT_TIME, 2) loganalyzer.expect_regex = [LOG_EXPECT_FAN_FAULT_CLEAR_RE, LOG_EXPECT_INSUFFICIENT_FAN_NUM_CLEAR_RE] with loganalyzer: logging.info('Mocking the fault FAN back to normal...') single_fan_mocker.mock_status(True) check_cli_output_with_mocker(duthost, single_fan_mocker, CMD_PLATFORM_FANSTATUS, THERMAL_CONTROL_TEST_WAIT_TIME, 2) loganalyzer.expect_regex = [LOG_EXPECT_FAN_OVER_SPEED_RE] with loganalyzer: logging.info('Mocking an over speed FAN...') single_fan_mocker.mock_over_speed() check_cli_output_with_mocker(duthost, single_fan_mocker, CMD_PLATFORM_FANSTATUS, THERMAL_CONTROL_TEST_WAIT_TIME, 2) loganalyzer.expect_regex = [LOG_EXPECT_FAN_OVER_SPEED_CLEAR_RE] with loganalyzer: logging.info('Make the over speed FAN back to normal...') single_fan_mocker.mock_normal_speed() check_cli_output_with_mocker(duthost, single_fan_mocker, CMD_PLATFORM_FANSTATUS, THERMAL_CONTROL_TEST_WAIT_TIME, 2) loganalyzer.expect_regex = [LOG_EXPECT_FAN_UNDER_SPEED_RE] with loganalyzer: logging.info('Mocking an under speed FAN...') single_fan_mocker.mock_under_speed() check_cli_output_with_mocker(duthost, single_fan_mocker, CMD_PLATFORM_FANSTATUS, THERMAL_CONTROL_TEST_WAIT_TIME, 2) loganalyzer.expect_regex = [LOG_EXPECT_FAN_UNDER_SPEED_CLEAR_RE] with loganalyzer: logging.info('Make the under speed FAN back to normal...') single_fan_mocker.mock_normal_speed() check_cli_output_with_mocker(duthost, single_fan_mocker, CMD_PLATFORM_FANSTATUS, THERMAL_CONTROL_TEST_WAIT_TIME, 2)
def test_check_sfp_status_and_configure_sfp(duthost, conn_graph_facts): """ @summary: Check SFP status and configure SFP This case is to use the sfputil tool and show command to check SFP status and configure SFP. Currently the only configuration is to reset SFP. Commands to be tested: * sfputil show presence * show interface transceiver presence * sfputil show eeprom * show interface transceiver eeprom * sfputil reset <interface name> """ if duthost.facts["asic_type"] in ["mellanox"]: loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix='sfp_cfg') loganalyzer.load_common_config() loganalyzer.ignore_regex.append("kernel.*Eeprom query failed*") marker = loganalyzer.init() cmd_sfp_presence = "sudo sfputil show presence" cmd_sfp_eeprom = "sudo sfputil show eeprom" cmd_sfp_reset = "sudo sfputil reset" cmd_xcvr_presence = "show interface transceiver presence" cmd_xcvr_eeprom = "show interface transceiver eeprom" portmap = get_port_map(duthost) logging.info("Got portmap {}".format(portmap)) logging.info("Check output of '%s'" % cmd_sfp_presence) sfp_presence = duthost.command(cmd_sfp_presence) parsed_presence = parse_output(sfp_presence["stdout_lines"][2:]) for intf in conn_graph_facts["device_conn"]: assert intf in parsed_presence, "Interface is not in output of '%s'" % cmd_sfp_presence assert parsed_presence[intf] == "Present", "Interface presence is not 'Present'" logging.info("Check output of '%s'" % cmd_xcvr_presence) xcvr_presence = duthost.command(cmd_xcvr_presence) parsed_presence = parse_output(xcvr_presence["stdout_lines"][2:]) for intf in conn_graph_facts["device_conn"]: assert intf in parsed_presence, "Interface is not in output of '%s'" % cmd_xcvr_presence assert parsed_presence[intf] == "Present", "Interface presence is not 'Present'" logging.info("Check output of '%s'" % cmd_sfp_eeprom) sfp_eeprom = duthost.command(cmd_sfp_eeprom) parsed_eeprom = parse_eeprom(sfp_eeprom["stdout_lines"]) for intf in conn_graph_facts["device_conn"]: assert intf in parsed_eeprom, "Interface is not in output of 'sfputil show eeprom'" assert parsed_eeprom[intf] == "SFP EEPROM detected" logging.info("Check output of '%s'" % cmd_xcvr_eeprom) xcvr_eeprom = duthost.command(cmd_xcvr_eeprom) parsed_eeprom = parse_eeprom(xcvr_eeprom["stdout_lines"]) for intf in conn_graph_facts["device_conn"]: assert intf in parsed_eeprom, "Interface is not in output of '%s'" % cmd_xcvr_eeprom assert parsed_eeprom[intf] == "SFP EEPROM detected" logging.info("Test '%s <interface name>'" % cmd_sfp_reset) tested_physical_ports = set() for intf in conn_graph_facts["device_conn"]: phy_intf = portmap[intf][0] if phy_intf in tested_physical_ports: logging.info("skip tested SFPs {} to avoid repeating operating physical interface {}".format(intf, phy_intf)) continue tested_physical_ports.add(phy_intf) logging.info("resetting {} physical interface {}".format(intf, phy_intf)) reset_result = duthost.command("%s %s" % (cmd_sfp_reset, intf)) assert reset_result["rc"] == 0, "'%s %s' failed" % (cmd_sfp_reset, intf) time.sleep(5) logging.info("Wait some time for SFP to fully recover after reset") time.sleep(60) logging.info("Check sfp presence again after reset") sfp_presence = duthost.command(cmd_sfp_presence) parsed_presence = parse_output(sfp_presence["stdout_lines"][2:]) for intf in conn_graph_facts["device_conn"]: assert intf in parsed_presence, "Interface is not in output of '%s'" % cmd_sfp_presence assert parsed_presence[intf] == "Present", "Interface presence is not 'Present'" logging.info("Check interface status") mg_facts = duthost.minigraph_facts(host=duthost.hostname)["ansible_facts"] intf_facts = duthost.interface_facts(up_ports=mg_facts["minigraph_ports"])["ansible_facts"] assert len(intf_facts["ansible_interface_link_down_ports"]) == 0, \ "Some interfaces are down: %s" % str(intf_facts["ansible_interface_link_down_ports"]) if duthost.facts["asic_type"] in ["mellanox"]: loganalyzer.analyze(marker)
def test_check_sfp_low_power_mode(duthost, conn_graph_facts): """ @summary: Check SFP low power mode This case is to use the sfputil tool command to check and set SFP low power mode * sfputil show lpmode * sfputil lpmode off * sfputil lpmode on """ if duthost.facts["asic_type"] in ["mellanox"]: loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix='sfp_lpm') loganalyzer.load_common_config() loganalyzer.ignore_regex.append("Eeprom query failed") marker = loganalyzer.init() cmd_sfp_presence = "sudo sfputil show presence" cmd_sfp_show_lpmode = "sudo sfputil show lpmode" cmd_sfp_set_lpmode = "sudo sfputil lpmode" portmap = get_port_map(duthost) logging.info("Got portmap {}".format(portmap)) logging.info("Check output of '%s'" % cmd_sfp_show_lpmode) lpmode_show = duthost.command(cmd_sfp_show_lpmode) parsed_lpmode = parse_output(lpmode_show["stdout_lines"][2:]) original_lpmode = copy.deepcopy(parsed_lpmode) for intf in conn_graph_facts["device_conn"]: assert intf in parsed_lpmode, "Interface is not in output of '%s'" % cmd_sfp_show_lpmode assert parsed_lpmode[intf].lower() == "on" or parsed_lpmode[intf].lower() == "off", "Unexpected SFP lpmode" logging.info("Try to change SFP lpmode") tested_physical_ports = set() for intf in conn_graph_facts["device_conn"]: phy_intf = portmap[intf][0] if phy_intf in tested_physical_ports: logging.info("skip tested SFPs {} to avoid repeating operating physical interface {}".format(intf, phy_intf)) continue tested_physical_ports.add(phy_intf) logging.info("setting {} physical interface {}".format(intf, phy_intf)) new_lpmode = "off" if original_lpmode[intf].lower() == "on" else "on" lpmode_set_result = duthost.command("%s %s %s" % (cmd_sfp_set_lpmode, new_lpmode, intf)) assert lpmode_set_result["rc"] == 0, "'%s %s %s' failed" % (cmd_sfp_set_lpmode, new_lpmode, intf) time.sleep(10) logging.info("Check SFP lower power mode again after changing SFP lpmode") lpmode_show = duthost.command(cmd_sfp_show_lpmode) parsed_lpmode = parse_output(lpmode_show["stdout_lines"][2:]) for intf in conn_graph_facts["device_conn"]: assert intf in parsed_lpmode, "Interface is not in output of '%s'" % cmd_sfp_show_lpmode assert parsed_lpmode[intf].lower() == "on" or parsed_lpmode[intf].lower() == "off", "Unexpected SFP lpmode" logging.info("Try to change SFP lpmode") tested_physical_ports = set() for intf in conn_graph_facts["device_conn"]: phy_intf = portmap[intf][0] if phy_intf in tested_physical_ports: logging.info("skip tested SFPs {} to avoid repeating operating physical interface {}".format(intf, phy_intf)) continue tested_physical_ports.add(phy_intf) logging.info("restoring {} physical interface {}".format(intf, phy_intf)) new_lpmode = original_lpmode[intf].lower() lpmode_set_result = duthost.command("%s %s %s" % (cmd_sfp_set_lpmode, new_lpmode, intf)) assert lpmode_set_result["rc"] == 0, "'%s %s %s' failed" % (cmd_sfp_set_lpmode, new_lpmode, intf) time.sleep(10) logging.info("Check SFP lower power mode again after changing SFP lpmode") lpmode_show = duthost.command(cmd_sfp_show_lpmode) parsed_lpmode = parse_output(lpmode_show["stdout_lines"][2:]) for intf in conn_graph_facts["device_conn"]: assert intf in parsed_lpmode, "Interface is not in output of '%s'" % cmd_sfp_show_lpmode assert parsed_lpmode[intf].lower() == "on" or parsed_lpmode[intf].lower() == "off", "Unexpected SFP lpmode" logging.info("Check sfp presence again after setting lpmode") sfp_presence = duthost.command(cmd_sfp_presence) parsed_presence = parse_output(sfp_presence["stdout_lines"][2:]) for intf in conn_graph_facts["device_conn"]: assert intf in parsed_presence, "Interface is not in output of '%s'" % cmd_sfp_presence assert parsed_presence[intf] == "Present", "Interface presence is not 'Present'" logging.info("Check interface status") mg_facts = duthost.minigraph_facts(host=duthost.hostname)["ansible_facts"] intf_facts = duthost.interface_facts(up_ports=mg_facts["minigraph_ports"])["ansible_facts"] assert len(intf_facts["ansible_interface_link_down_ports"]) == 0, \ "Some interfaces are down: %s" % str(intf_facts["ansible_interface_link_down_ports"]) if duthost.facts["asic_type"] in ["mellanox"]: loganalyzer.analyze(marker)
def test_nhop(request, duthost, tbinfo): """ Test next hop group resource count. Steps: - Add test IP address to an active IP interface - Add static ARPs - Create unique next hop groups - Add IP route and nexthop - check CRM resource - clean up - Verify no erros and crash """ skip_release(duthost, ["201811", "201911"]) default_max_nhop_paths = 32 nhop_group_limit = 1024 # program more than the advertised limit extra_nhops = 10 asic = duthost.asic_instance() # find out MAX NHOP group count supported on the platform result = asic.run_redis_cmd( argv=["redis-cli", "-n", 6, "HGETALL", "SWITCH_CAPABILITY|switch"]) it = iter(result) switch_capability = dict(zip(it, it)) max_nhop = switch_capability.get("MAX_NEXTHOP_GROUP_COUNT") max_nhop = nhop_group_limit if max_nhop == None else int(max_nhop) nhop_group_count = min(max_nhop, nhop_group_limit) + extra_nhops # find out an active IP port ip_ifaces = asic.get_active_ip_interfaces(tbinfo).keys() pytest_assert(len(ip_ifaces), "No IP interfaces found") eth_if = ip_ifaces[0] # Generate ARP entries arp_count = 40 arplist = Arp(duthost, asic, arp_count, eth_if) arplist.arps_add() # indices indices = range(arp_count) ip_indices = combinations(indices, default_max_nhop_paths) # intitialize log analyzer marker = "NHOP TEST PATH COUNT {} {}".format(nhop_group_count, eth_if) loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix=marker) marker = loganalyzer.init() loganalyzer.load_common_config() loganalyzer.expect_regex = [] loganalyzer.ignore_regex.extend(loganalyzer_ignore_regex_list()) ip_prefix = ipaddr.IPAddress("192.168.0.0") # list of all IPs available to generate a nexthop group ip_list = arplist.ip_mac_list crm_before = get_crm_info(duthost, asic) # increase CRM polling time asic.command("crm config polling interval 10") logging.info("Adding {} next hops on {}".format(nhop_group_count, eth_if)) # create nexthop group nhop = IPRoutes(duthost, asic) try: for i, indx_list in zip(range(nhop_group_count), ip_indices): # get a list of unique group of next hop IPs ips = [arplist.ip_mac_list[x].ip for x in indx_list] ip_route = "{}/31".format(ip_prefix + (2 * i)) # add IP route with the next hop group created nhop.add_ip_route(ip_route, ips) nhop.program_routes() # wait for routes to be synced and programmed time.sleep(120) crm_after = get_crm_info(duthost, asic) finally: nhop.delete_routes() arplist.clean_up() asic.command("crm config polling interval {}".format( crm_before["polling"])) # check for any errors or crash loganalyzer.analyze(marker) # verify the test used up all the NHOP group resources # skip this check on Mellanox as ASIC resources are shared if not is_mellanox_device(duthost): pytest_assert( crm_after["available"] == 0, "Unused NHOP group resource: {}, used:{}".format( crm_after["available"], crm_after["used"]))