def acl_rules(self, duthosts, rand_one_dut_hostname, localhost, setup, acl_table, populate_vlan_arp_entries):
"""Setup/teardown ACL rules for the current set of tests.
Args:
duthosts: All DUTs belong to the testbed.
rand_one_dut_hostname: hostname of a random chosen dut to run test.
localhost: The host from which tests are run.
setup: Parameters for the ACL tests.
acl_table: Configuration info for the ACL table.
populate_vlan_arp_entries: A function to populate ARP/FDB tables for VLAN interfaces.
"""
duthost = duthosts[rand_one_dut_hostname]
loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix="acl_rules")
loganalyzer.load_common_config()
try:
loganalyzer.expect_regex = [LOG_EXPECT_ACL_RULE_CREATE_RE]
with loganalyzer:
self.setup_rules(duthost, acl_table)
self.post_setup_hook(duthost, localhost, populate_vlan_arp_entries)
except LogAnalyzerError as err:
# Cleanup Config DB if rule creation failed
logger.error("ACL table creation failed, attempting to clean-up...")
self.teardown_rules(duthost)
raise err
try:
yield
finally:
loganalyzer.expect_regex = [LOG_EXPECT_ACL_RULE_REMOVE_RE]
with loganalyzer:
logger.info("Removing ACL rules")
self.teardown_rules(duthost)
def test_dynamic_minimum_table(duthost, mocker_factory):
air_flow_dirs = ['p2c', 'c2p', 'unk']
max_temperature = 45000 # 45 C
cooling_cur_state = get_cooling_cur_state(duthost)
if cooling_cur_state >= COOLING_CUR_STATE_THRESHOLD:
pytest.skip('The cooling level {} is higher than threshold {}.'.format(cooling_cur_state, COOLING_CUR_STATE_THRESHOLD))
mocker = mocker_factory(duthost, 'MinTableMocker')
loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix='thermal_control')
loganalyzer.load_common_config()
for index in range(len(air_flow_dirs)):
air_flow_index = random.randint(0, len(air_flow_dirs) - 1)
air_flow_dir = air_flow_dirs[air_flow_index]
air_flow_dirs.remove(air_flow_dir)
temperature = random.randint(0, max_temperature)
trust_state = True if random.randint(0, 1) else False
logging.info('Testing with air_flow_dir={}, temperature={}, trust_state={}'.format(air_flow_dir, temperature, trust_state))
expect_minimum_cooling_level = mocker.get_expect_cooling_level(air_flow_dir, temperature, trust_state)
loganalyzer.expect_regex = [LOG_EXPECT_CHANGE_MIN_COOLING_LEVEL_RE.format(expect_minimum_cooling_level)]
with loganalyzer:
mocker.mock_min_table(air_flow_dir, temperature, trust_state)
time.sleep(THERMAL_CONTROL_TEST_WAIT_TIME)
temperature = random.randint(0, max_temperature)
logging.info('Testing with air_flow_dir={}, temperature={}, trust_state={}'.format(air_flow_dir, temperature, not trust_state))
expect_minimum_cooling_level = mocker.get_expect_cooling_level(air_flow_dir, temperature, not trust_state)
loganalyzer.expect_regex = [LOG_EXPECT_CHANGE_MIN_COOLING_LEVEL_RE.format(expect_minimum_cooling_level)]
with loganalyzer:
mocker.mock_min_table(air_flow_dir, temperature, not trust_state)
time.sleep(THERMAL_CONTROL_TEST_WAIT_TIME)
def acl_rules(self, duthost, localhost, setup, acl_table):
"""
setup/teardown ACL rules based on test class requirements
:param duthost: DUT host object
:param localhost: localhost object
:param setup: setup information
:param acl_table: table creating fixture
:return:
"""
loganalyzer = LogAnalyzer(ansible_host=duthost,
marker_prefix='acl_rules')
loganalyzer.load_common_config()
try:
loganalyzer.expect_regex = [LOG_EXPECT_ACL_RULE_CREATE_RE]
with loganalyzer:
self.setup_rules(duthost, setup, acl_table)
self.post_setup_hook(duthost, localhost)
except LogAnalyzerError as err:
# cleanup config DB in case of log analysis error
self.teardown_rules(duthost, setup)
raise err
try:
yield
finally:
loganalyzer.expect_regex = [LOG_EXPECT_ACL_RULE_REMOVE_RE]
with loganalyzer:
self.teardown_rules(duthost, setup)
def acl(duthost, acl_setup):
"""
setup/teardown ACL rules based on test class requirements
:param duthost: DUT host object
:param acl_setup: setup information
:return:
"""
loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix='acl')
loganalyzer.load_common_config()
try:
loganalyzer.expect_regex = [LOG_EXPECT_ACL_RULE_CREATE_RE]
with loganalyzer:
setup_acl_rules(duthost, acl_setup)
except LogAnalyzerError as err:
# cleanup config DB in case of log analysis error
teardown_acl(duthost, acl_setup)
raise err
try:
yield
finally:
loganalyzer.expect_regex = [LOG_EXPECT_ACL_RULE_REMOVE_RE]
with loganalyzer:
teardown_acl(duthost, acl_setup)
def acl(duthosts, enum_rand_one_per_hwsku_frontend_hostname, acl_setup):
"""
setup/teardown ACL rules based on test class requirements
:param duthost: DUT host object
:param acl_setup: setup information
:return:
"""
duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname]
acl_facts = duthost.acl_facts()["ansible_facts"]["ansible_acl_facts"]
pytest_require(ACL_TABLE_NAME in acl_facts, "{} acl table not exists")
loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix='acl')
loganalyzer.load_common_config()
try:
loganalyzer.expect_regex = [LOG_EXPECT_ACL_RULE_CREATE_RE]
with loganalyzer:
setup_acl_rules(duthost, acl_setup)
except LogAnalyzerError as err:
# cleanup config DB in case of log analysis error
teardown_acl(duthost, acl_setup)
raise err
try:
yield
finally:
loganalyzer.expect_regex = [LOG_EXPECT_ACL_RULE_REMOVE_RE]
with loganalyzer:
teardown_acl(duthost, acl_setup)
def acl_table(duthost, acl_table_config):
"""
fixture to apply ACL table configuration and remove after tests
:param duthost: DUT object
:param acl_table_config: ACL table configuration dictionary
:return: forwards acl_table_config
"""
name = acl_table_config['name']
conf = acl_table_config['config_file']
loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix='acl')
loganalyzer.load_common_config()
try:
loganalyzer.expect_regex = [LOG_EXPECT_ACL_TABLE_CREATE_RE]
with loganalyzer:
logger.info('creating ACL table: applying {}'.format(conf))
# TODO: use sonic config CLI
duthost.command('sonic-cfggen -j {} --write-to-db'.format(conf))
except LogAnalyzerError as err:
# cleanup config DB if create failed
duthost.command('config acl remove table {}'.format(name))
raise err
try:
yield acl_table_config
finally:
loganalyzer.expect_regex = [LOG_EXPECT_ACL_TABLE_REMOVE_RE]
with loganalyzer:
logger.info('removing ACL table {}'.format(name))
duthost.command('config acl remove table {}'.format(name))
# save cleaned configuration
duthost.command('config save -y')
def acl_table(duthosts, rand_one_dut_hostname, setup, stage, ip_version,
backup_and_restore_config_db_module):
"""Apply ACL table configuration and remove after tests.
Args:
duthosts: All DUTs belong to the testbed.
rand_one_dut_hostname: hostname of a random chosen dut to run test.
setup: Parameters for the ACL tests.
stage: The ACL stage under test.
ip_version: The IP version under test.
backup_and_restore_config_db_module: A fixture that handles restoring Config DB
after the tests are over.
Yields:
The ACL table configuration.
"""
table_name = "DATA_{}_{}_TEST".format(stage.upper(), ip_version.upper())
acl_table_config = {
"table_name": table_name,
"table_ports": ",".join(setup["acl_table_ports"]['']),
"table_stage": stage,
"table_type": "L3" if ip_version == "ipv4" else "L3V6"
}
logger.info("Generated ACL table configuration:\n{}".format(
pprint.pformat(acl_table_config)))
dut_to_analyzer_map = {}
for duthost in duthosts:
loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix="acl")
loganalyzer.load_common_config()
dut_to_analyzer_map[duthost] = loganalyzer
try:
loganalyzer.expect_regex = [LOG_EXPECT_ACL_TABLE_CREATE_RE]
with loganalyzer:
create_or_remove_acl_table(duthost, acl_table_config, setup,
"add")
except LogAnalyzerError as err:
# Cleanup Config DB if table creation failed
logger.error(
"ACL table creation failed, attempting to clean-up...")
create_or_remove_acl_table(duthost, acl_table_config, setup,
"remove")
raise err
try:
yield acl_table_config
finally:
for duthost, loganalyzer in dut_to_analyzer_map.items():
loganalyzer.expect_regex = [LOG_EXPECT_ACL_TABLE_REMOVE_RE]
with loganalyzer:
create_or_remove_acl_table(duthost, acl_table_config, setup,
"remove")
def disable_analyzer_for_mellanox(duthost):
if duthost.facts["asic_type"] in ["mellanox"]:
loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix='sfp_cfg')
loganalyzer.load_common_config()
loganalyzer.ignore_regex.append("kernel.*Eeprom query failed*")
marker = loganalyzer.init()
yield
if duthost.facts["asic_type"] in ["mellanox"]:
loganalyzer.analyze(marker)
def ignore_particular_error_log(request, duthosts, rand_one_dut_hostname):
duthost = duthosts[rand_one_dut_hostname]
loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix='turn_on_off_psu_and_check_psustatus')
loganalyzer.load_common_config()
ignore_list = request.param
loganalyzer.ignore_regex.extend(ignore_list)
marker = loganalyzer.init()
yield
loganalyzer.analyze(marker)
def ignore_log_analyzer_by_vendor(request, duthosts, enum_rand_one_per_hwsku_hostname):
duthost = duthosts[enum_rand_one_per_hwsku_hostname]
asic_type = duthost.facts["asic_type"]
ignore_asic_list = request.param
if asic_type not in ignore_asic_list:
loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix=request.node.name)
loganalyzer.load_common_config()
marker = loganalyzer.init()
yield
loganalyzer.analyze(marker)
else:
yield
def disable_analyzer_for_mellanox(duthost):
if duthost.facts["asic_type"] in ["mellanox"]:
loganalyzer = LogAnalyzer(ansible_host=duthost,
marker_prefix='sfp_cfg')
loganalyzer.load_common_config()
loganalyzer.ignore_regex.append("kernel.*Eeprom query failed*")
# Ignore PMPE error https://github.com/Azure/sonic-buildimage/issues/7163
loganalyzer.ignore_regex.append(
r".*ERR pmon#xcvrd: Receive PMPE error event on module.*")
marker = loganalyzer.init()
yield
if duthost.facts["asic_type"] in ["mellanox"]:
loganalyzer.analyze(marker)
def acl_rules(self, duthosts, localhost, setup, acl_table,
populate_vlan_arp_entries, tbinfo, ip_version):
"""Setup/teardown ACL rules for the current set of tests.
Args:
duthosts: All DUTs belong to the testbed.
rand_one_dut_hostname: hostname of a random chosen dut to run test.
localhost: The host from which tests are run.
setup: Parameters for the ACL tests.
acl_table: Configuration info for the ACL table.
populate_vlan_arp_entries: A function to populate ARP/FDB tables for VLAN interfaces.
"""
dut_to_analyzer_map = {}
for duthost in duthosts:
loganalyzer = LogAnalyzer(ansible_host=duthost,
marker_prefix="acl_rules")
loganalyzer.load_common_config()
dut_to_analyzer_map[duthost] = loganalyzer
try:
loganalyzer.expect_regex = [LOG_EXPECT_ACL_RULE_CREATE_RE]
# Ignore any other errors to reduce noise
loganalyzer.ignore_regex = [r".*"]
with loganalyzer:
self.setup_rules(duthost, acl_table, ip_version)
self.post_setup_hook(duthost, localhost,
populate_vlan_arp_entries, tbinfo)
assert self.check_rule_counters(
duthost), "Rule counters should be ready!"
except LogAnalyzerError as err:
# Cleanup Config DB if rule creation failed
logger.error(
"ACL rule application failed, attempting to clean-up...")
self.teardown_rules(duthost)
raise err
try:
yield
finally:
for duthost, loganalyzer in dut_to_analyzer_map.items():
loganalyzer.expect_regex = [LOG_EXPECT_ACL_RULE_REMOVE_RE]
with loganalyzer:
logger.info("Removing ACL rules")
self.teardown_rules(duthost)
def mirroring(duthosts, enum_rand_one_per_hwsku_frontend_hostname, neighbor_ip,
mirror_setup, gre_version):
"""
fixture gathers all configuration fixtures
:param duthost: DUT host
:param mirror_setup: mirror_setup fixture
:param mirror_config: mirror_config fixture
"""
duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname]
logger.info("Adding mirror_session to DUT")
acl_rule_file = os.path.join(mirror_setup['dut_tmp_dir'],
ACL_RULE_PERSISTENT_FILE)
extra_vars = {
'acl_table_name': EVERFLOW_TABLE_NAME,
}
logger.info('Extra variables for MIRROR table:\n{}'.format(
pprint.pformat(extra_vars)))
duthost.host.options['variable_manager'].extra_vars.update(extra_vars)
duthost.template(src=os.path.join(TEMPLATE_DIR,
ACL_RULE_PERSISTENT_TEMPLATE),
dest=acl_rule_file)
duthost.command('config mirror_session add {} {} {} {} {} {} {}'.format(
SESSION_INFO['name'], SESSION_INFO['src_ip'], neighbor_ip,
SESSION_INFO['dscp'], SESSION_INFO['ttl'], SESSION_INFO['gre'],
SESSION_INFO['queue']))
logger.info('Loading acl mirror rules ...')
load_rule_cmd = "acl-loader update full {} --session_name={}".format(
acl_rule_file, SESSION_INFO['name'])
duthost.command('{}'.format(load_rule_cmd))
try:
yield
finally:
loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix='acl')
loganalyzer.load_common_config()
try:
loganalyzer.expect_regex = [LOG_EXCEPT_MIRROR_SESSION_REMOVE]
with loganalyzer:
teardown_mirroring(duthost, mirror_setup['dut_tmp_dir'])
except LogAnalyzerError as err:
raise err
def acl_table(duthosts, rand_one_dut_hostname, acl_table_config,
backup_and_restore_config_db_module):
"""Apply ACL table configuration and remove after tests.
Args:
duthosts: All DUTs belong to the testbed.
rand_one_dut_hostname: hostname of a random chosen dut to run test.
acl_table_config: A dictionary describing the ACL table configuration to apply.
backup_and_restore_config_db_module: A fixture that handles restoring Config DB
after the tests are over.
Yields:
The ACL table configuration.
"""
duthost = duthosts[rand_one_dut_hostname]
table_name = acl_table_config["table_name"]
config_file = acl_table_config["config_file"]
loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix="acl")
loganalyzer.load_common_config()
try:
loganalyzer.expect_regex = [LOG_EXPECT_ACL_TABLE_CREATE_RE]
with loganalyzer:
logger.info("Creating ACL table from config file: \"{}\"".format(
config_file))
# TODO: Use `config` CLI to create ACL table
duthost.command(
"sonic-cfggen -j {} --write-to-db".format(config_file))
except LogAnalyzerError as err:
# Cleanup Config DB if table creation failed
logger.error("ACL table creation failed, attempting to clean-up...")
duthost.command("config acl remove table {}".format(table_name))
raise err
try:
yield acl_table_config
finally:
loganalyzer.expect_regex = [LOG_EXPECT_ACL_TABLE_REMOVE_RE]
with loganalyzer:
logger.info("Removing ACL table \"{}\"".format(table_name))
duthost.command("config acl remove table {}".format(table_name))
def advanceboot_loganalyzer(duthosts, rand_one_dut_hostname, request):
"""
Advance reboot log analysis.
This fixture starts log analysis at the beginning of the test. At the end,
the collected expect messages are verified and timing of start/stop is calculated.
Args:
duthosts : List of DUT hosts
rand_one_dut_hostname: hostname of a randomly selected DUT
"""
duthost = duthosts[rand_one_dut_hostname]
test_name = request.node.name
if "warm" in test_name:
reboot_type = "warm"
elif "fast" in test_name:
reboot_type = "fast"
else:
reboot_type = "unknown"
# Currently, advanced reboot test would skip for kvm platform if the test has no device_type marker for vs.
# Doing the same skip logic in this fixture to avoid running loganalyzer without the test executed
if duthost.facts['platform'] == 'x86_64-kvm_x86_64-r0':
device_marks = [
arg for mark in request.node.iter_markers(name='device_type')
for arg in mark.args
]
if 'vs' not in device_marks:
pytest.skip('Testcase not supported for kvm')
loganalyzer = LogAnalyzer(
ansible_host=duthost,
marker_prefix="test_advanced_reboot_{}".format(test_name),
additional_files={
'/var/log/swss/sairedis.rec':
'recording on: /var/log/swss/sairedis.rec',
'/var/log/frr/bgpd.log': ''
})
marker = loganalyzer.init()
loganalyzer.load_common_config()
ignore_file = os.path.join(TEMPLATES_DIR, "ignore_boot_messages")
expect_file = os.path.join(TEMPLATES_DIR, "expect_boot_messages")
ignore_reg_exp = loganalyzer.parse_regexp_file(src=ignore_file)
expect_reg_exp = loganalyzer.parse_regexp_file(src=expect_file)
loganalyzer.ignore_regex.extend(ignore_reg_exp)
loganalyzer.expect_regex = []
loganalyzer.expect_regex.extend(expect_reg_exp)
loganalyzer.match_regex = []
yield
result = loganalyzer.analyze(marker, fail=False)
analyze_result = {"time_span": dict(), "offset_from_kexec": dict()}
offset_from_kexec = dict()
for key, messages in result["expect_messages"].items():
if "syslog" in key or "bgpd.log" in key:
analyze_log_file(duthost, messages, analyze_result,
offset_from_kexec)
elif "sairedis.rec" in key:
analyze_sairedis_rec(messages, analyze_result, offset_from_kexec)
for marker, time_data in analyze_result["offset_from_kexec"].items():
marker_start_time = time_data.get("timestamp", {}).get("Start")
reboot_start_time = analyze_result.get("reboot_time",
{}).get("timestamp",
{}).get("Start")
if reboot_start_time and reboot_start_time != "N/A" and marker_start_time:
time_data["time_taken"] = (datetime.strptime(marker_start_time, FMT) -\
datetime.strptime(reboot_start_time, FMT)).total_seconds()
else:
time_data["time_taken"] = "N/A"
get_data_plane_report(analyze_result, reboot_type)
result_summary = get_report_summary(analyze_result, reboot_type)
logging.info(json.dumps(analyze_result, indent=4))
logging.info(json.dumps(result_summary, indent=4))
report_file_name = request.node.name + "_report.json"
summary_file_name = request.node.name + "_summary.json"
report_file_dir = os.path.realpath((os.path.join(os.path.dirname(__file__),\
"../logs/platform_tests/")))
report_file_path = report_file_dir + "/" + report_file_name
summary_file_path = report_file_dir + "/" + summary_file_name
if not os.path.exists(report_file_dir):
os.makedirs(report_file_dir)
with open(report_file_path, 'w') as fp:
json.dump(analyze_result, fp, indent=4)
with open(summary_file_path, 'w') as fp:
json.dump(result_summary, fp, indent=4)
def advanceboot_loganalyzer(duthosts, rand_one_dut_hostname):
"""
Advance reboot log analysis.
This fixture starts log analysis at the beginning of the test. At the end,
the collected expect messages are verified and timing of start/stop is calculated.
Args:
duthosts : List of DUT hosts
rand_one_dut_hostname: hostname of a randomly selected DUT
"""
duthost = duthosts[rand_one_dut_hostname]
loganalyzer = LogAnalyzer(ansible_host=duthost,
marker_prefix="test_advanced_reboot")
marker = loganalyzer.init()
loganalyzer.load_common_config()
ignore_file = os.path.join(TEMPLATES_DIR, "ignore_boot_messages")
expect_file = os.path.join(TEMPLATES_DIR, "expect_boot_messages")
ignore_reg_exp = loganalyzer.parse_regexp_file(src=ignore_file)
expect_reg_exp = loganalyzer.parse_regexp_file(src=expect_file)
loganalyzer.ignore_regex.extend(ignore_reg_exp)
loganalyzer.expect_regex = []
loganalyzer.expect_regex.extend(expect_reg_exp)
loganalyzer.match_regex = []
yield
result = loganalyzer.analyze(marker, fail=False)
messages = result["expect_messages"].values()
if not messages:
logging.error("Expected messages not found in syslog")
return
messages = messages[0]
service_restart_times = dict()
service_patterns = {
"Stopping": re.compile(r'.*Stopping.*service.*'),
"Stopped": re.compile(r'.*Stopped.*service.*'),
"Starting": re.compile(r'.*Starting.*service.*'),
"Started": re.compile(r'.*Started.*service.*')
}
def service_time_check(message, status):
time = message.split(duthost.hostname)[0].strip()
service_name = message.split(status + " ")[1].split()[0]
service_dict = service_restart_times.get(service_name,
{"timestamp": {}})
timestamps = service_dict.get("timestamp")
if status in timestamps:
service_dict[status +
" count"] = service_dict.get(status + " count", 1) + 1
timestamps[status] = time
service_restart_times.update({service_name: service_dict})
for message in messages:
for status, pattern in service_patterns.items():
if re.search(pattern, message):
service_time_check(message, status)
loganalyzer.save_extracted_log(dest="/tmp/log/syslog")
logging.info(json.dumps(service_restart_times, indent=4))
FMT = "%b %d %H:%M:%S.%f"
for _, timings in service_restart_times.items():
timestamps = timings["timestamp"]
timings["stop_time"] = (datetime.strptime(timestamps["Stopped"], FMT) -\
datetime.strptime(timestamps["Stopping"], FMT)).total_seconds() \
if "Stopped" in timestamps and "Stopping" in timestamps else None
timings["start_time"] = (datetime.strptime(timestamps["Started"], FMT) -\
datetime.strptime(timestamps["Starting"], FMT)).total_seconds() \
if "Started" in timestamps and "Starting" in timestamps else None
timings["reboot_time"] = (datetime.strptime(timestamps["Started"], FMT) -\
datetime.strptime(timestamps["Stopped"], FMT)).total_seconds() \
if "Started" in timestamps and "Stopped" in timestamps else None
files = glob.glob('/tmp/*-report.json')
if files:
filepath = files[0]
with open(filepath) as json_file:
report = json.load(json_file)
service_restart_times.update(report)
result = service_restart_times
logging.info(json.dumps(result, indent=4))
def test_turn_on_off_psu_and_check_psustatus(duthost, psu_controller):
"""
@summary: Turn off/on PSU and check PSU status using 'show platform psustatus'
"""
loganalyzer = LogAnalyzer(
ansible_host=duthost,
marker_prefix='turn_on_off_psu_and_check_psustatus')
loganalyzer.load_common_config()
loganalyzer.ignore_regex.append(
"Error getting sensor data: dps460.*Kernel interface error")
marker = loganalyzer.init()
psu_line_pattern = re.compile(r"PSU\s+\d+\s+(OK|NOT OK|NOT PRESENT)")
psu_num = get_psu_num(duthost)
if psu_num < 2:
pytest.skip(
"At least 2 PSUs required for rest of the testing in this case")
logging.info("Create PSU controller for testing")
psu_ctrl = psu_controller
if psu_ctrl is None:
pytest.skip(
"No PSU controller for %s, skip rest of the testing in this case" %
duthost.hostname)
logging.info(
"To avoid DUT being shutdown, need to turn on PSUs that are not powered"
)
turn_all_psu_on(psu_ctrl)
logging.info("Initialize test results")
psu_test_results = {}
if not check_all_psu_on(duthost, psu_test_results):
pytest.skip(
"Some PSU are still down, skip rest of the testing in this case")
assert len(psu_test_results.keys()) == psu_num, \
"In consistent PSU number output by '%s' and '%s'" % (CMD_PLATFORM_PSUSTATUS, "sudo psuutil numpsus")
logging.info("Start testing turn off/on PSUs")
all_psu_status = psu_ctrl.get_psu_status()
for psu in all_psu_status:
psu_under_test = None
logging.info("Turn off PSU %s" % str(psu["psu_id"]))
psu_ctrl.turn_off_psu(psu["psu_id"])
time.sleep(5)
cli_psu_status = duthost.command(CMD_PLATFORM_PSUSTATUS)
for line in cli_psu_status["stdout_lines"][2:]:
assert psu_line_pattern.match(line), "Unexpected PSU status output"
fields = line.split()
if fields[2] != "OK":
psu_under_test = fields[1]
check_vendor_specific_psustatus(duthost, line)
assert psu_under_test is not None, "No PSU is turned off"
logging.info("Turn on PSU %s" % str(psu["psu_id"]))
psu_ctrl.turn_on_psu(psu["psu_id"])
time.sleep(5)
cli_psu_status = duthost.command(CMD_PLATFORM_PSUSTATUS)
for line in cli_psu_status["stdout_lines"][2:]:
assert psu_line_pattern.match(line), "Unexpected PSU status output"
fields = line.split()
if fields[1] == psu_under_test:
assert fields[
2] == "OK", "Unexpected PSU status after turned it on"
check_vendor_specific_psustatus(duthost, line)
psu_test_results[psu_under_test] = True
for psu in psu_test_results:
assert psu_test_results[psu], "Test psu status of PSU %s failed" % psu
loganalyzer.analyze(marker)
def test_check_sfp_status_and_configure_sfp(duthosts, rand_one_dut_hostname, enum_frontend_asic_index, conn_graph_facts, tbinfo):
"""
@summary: Check SFP status and configure SFP
This case is to use the sfputil tool and show command to check SFP status and configure SFP. Currently the
only configuration is to reset SFP. Commands to be tested:
* sfputil show presence
* show interface transceiver presence
* sfputil show eeprom
* show interface transceiver eeprom
* sfputil reset <interface name>
"""
duthost = duthosts[rand_one_dut_hostname]
if duthost.facts["asic_type"] in ["mellanox"]:
loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix='sfp_cfg')
loganalyzer.load_common_config()
loganalyzer.ignore_regex.append("kernel.*Eeprom query failed*")
marker = loganalyzer.init()
dev_conn = conn_graph_facts["device_conn"][duthost.hostname]
# Get the interface pertaining to that asic
portmap = get_port_map(duthost, enum_frontend_asic_index)
logging.info("Got portmap {}".format(portmap))
if enum_frontend_asic_index is not None:
# Check if the interfaces of this AISC is present in conn_graph_facts
dev_conn = {k:v for k, v in portmap.items() if k in conn_graph_facts["device_conn"][duthost.hostname]}
logging.info("ASIC {} interface_list {}".format(enum_frontend_asic_index, dev_conn))
cmd_sfp_presence = "sudo sfputil show presence"
cmd_sfp_eeprom = "sudo sfputil show eeprom"
cmd_sfp_reset = "sudo sfputil reset"
cmd_xcvr_presence = "show interface transceiver presence"
cmd_xcvr_eeprom = "show interface transceiver eeprom"
global ans_host
ans_host = duthost
logging.info("Check output of '%s'" % cmd_sfp_presence)
sfp_presence = duthost.command(cmd_sfp_presence)
parsed_presence = parse_output(sfp_presence["stdout_lines"][2:])
for intf in dev_conn:
assert intf in parsed_presence, "Interface is not in output of '%s'" % cmd_sfp_presence
assert parsed_presence[intf] == "Present", "Interface presence is not 'Present'"
logging.info("Check output of '%s'" % cmd_xcvr_presence)
xcvr_presence = duthost.command(cmd_xcvr_presence)
parsed_presence = parse_output(xcvr_presence["stdout_lines"][2:])
for intf in dev_conn:
assert intf in parsed_presence, "Interface is not in output of '%s'" % cmd_xcvr_presence
assert parsed_presence[intf] == "Present", "Interface presence is not 'Present'"
logging.info("Check output of '%s'" % cmd_sfp_eeprom)
sfp_eeprom = duthost.command(cmd_sfp_eeprom)
parsed_eeprom = parse_eeprom(sfp_eeprom["stdout_lines"])
for intf in dev_conn:
assert intf in parsed_eeprom, "Interface is not in output of 'sfputil show eeprom'"
assert parsed_eeprom[intf] == "SFP EEPROM detected"
logging.info("Check output of '%s'" % cmd_xcvr_eeprom)
xcvr_eeprom = duthost.command(cmd_xcvr_eeprom)
parsed_eeprom = parse_eeprom(xcvr_eeprom["stdout_lines"])
for intf in dev_conn:
assert intf in parsed_eeprom, "Interface is not in output of '%s'" % cmd_xcvr_eeprom
assert parsed_eeprom[intf] == "SFP EEPROM detected"
logging.info("Test '%s <interface name>'" % cmd_sfp_reset)
tested_physical_ports = set()
for intf in dev_conn:
phy_intf = portmap[intf][0]
if phy_intf in tested_physical_ports:
logging.info("skip tested SFPs {} to avoid repeating operating physical interface {}".format(intf, phy_intf))
continue
tested_physical_ports.add(phy_intf)
logging.info("resetting {} physical interface {}".format(intf, phy_intf))
reset_result = duthost.command("%s %s" % (cmd_sfp_reset, intf))
assert reset_result["rc"] == 0, "'%s %s' failed" % (cmd_sfp_reset, intf)
time.sleep(5)
logging.info("Wait some time for SFP to fully recover after reset")
time.sleep(60)
logging.info("Check sfp presence again after reset")
sfp_presence = duthost.command(cmd_sfp_presence)
parsed_presence = parse_output(sfp_presence["stdout_lines"][2:])
for intf in dev_conn:
assert intf in parsed_presence, "Interface is not in output of '%s'" % cmd_sfp_presence
assert parsed_presence[intf] == "Present", "Interface presence is not 'Present'"
logging.info("Check interface status")
namespace = duthost.get_namespace_from_asic_id(enum_frontend_asic_index)
mg_facts = duthost.get_extended_minigraph_facts(tbinfo)
# TODO Remove this logic when minigraph facts supports namespace in multi_asic
up_ports = mg_facts["minigraph_ports"]
if enum_frontend_asic_index is not None:
# Check if the interfaces of this AISC is present in conn_graph_facts
up_ports = {k:v for k, v in portmap.items() if k in mg_facts["minigraph_ports"]}
intf_facts = duthost.interface_facts(namespace=namespace, up_ports=up_ports)["ansible_facts"]
assert len(intf_facts["ansible_interface_link_down_ports"]) == 0, \
"Some interfaces are down: %s" % str(intf_facts["ansible_interface_link_down_ports"])
if duthost.facts["asic_type"] in ["mellanox"]:
loganalyzer.analyze(marker)
def test_check_sfp_low_power_mode(duthosts, rand_one_dut_hostname, enum_frontend_asic_index, conn_graph_facts, tbinfo):
"""
@summary: Check SFP low power mode
This case is to use the sfputil tool command to check and set SFP low power mode
* sfputil show lpmode
* sfputil lpmode off
* sfputil lpmode on
"""
duthost = duthosts[rand_one_dut_hostname]
asichost = duthost.get_asic(enum_frontend_asic_index)
if duthost.facts["asic_type"] in ["mellanox"]:
loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix='sfp_lpm')
loganalyzer.load_common_config()
loganalyzer.ignore_regex.append("Eeprom query failed")
marker = loganalyzer.init()
dev_conn = conn_graph_facts["device_conn"][duthost.hostname]
# Get the interface pertaining to that asic
portmap = get_port_map(duthost, enum_frontend_asic_index)
logging.info("Got portmap {}".format(portmap))
if enum_frontend_asic_index is not None:
# Check if the interfaces of this AISC is present in conn_graph_facts
dev_conn = {k:v for k, v in portmap.items() if k in conn_graph_facts["device_conn"][duthost.hostname]}
logging.info("ASIC {} interface_list {}".format(enum_frontend_asic_index, dev_conn))
cmd_sfp_presence = "sudo sfputil show presence"
cmd_sfp_show_lpmode = "sudo sfputil show lpmode"
cmd_sfp_set_lpmode = "sudo sfputil lpmode"
global ans_host
ans_host = duthost
logging.info("Check output of '%s'" % cmd_sfp_show_lpmode)
lpmode_show = duthost.command(cmd_sfp_show_lpmode)
parsed_lpmode = parse_output(lpmode_show["stdout_lines"][2:])
original_lpmode = copy.deepcopy(parsed_lpmode)
for intf in dev_conn:
assert intf in parsed_lpmode, "Interface is not in output of '%s'" % cmd_sfp_show_lpmode
assert parsed_lpmode[intf].lower() == "on" or parsed_lpmode[intf].lower() == "off", "Unexpected SFP lpmode"
logging.info("Try to change SFP lpmode")
tested_physical_ports = set()
not_supporting_lpm_physical_ports = set()
for intf in dev_conn:
phy_intf = portmap[intf][0]
if phy_intf in tested_physical_ports:
logging.info("skip tested SFPs {} to avoid repeating operating physical interface {}".format(intf, phy_intf))
continue
sfp_type_cmd = 'redis-cli -n 6 hget "TRANSCEIVER_INFO|{}" type'.format(intf)
sfp_type_docker_cmd = asichost.get_docker_cmd(sfp_type_cmd, "database")
sfp_type = duthost.command(sfp_type_docker_cmd)["stdout"]
power_class_cmd = 'redis-cli -n 6 hget "TRANSCEIVER_INFO|{}" ext_identifier'.format(intf)
power_class_docker_cmd = asichost.get_docker_cmd(power_class_cmd, "database")
power_class = duthost.command(power_class_docker_cmd)["stdout"]
if not "QSFP" in sfp_type or "Power Class 1" in power_class:
logging.info("skip testing port {} which doesn't support LPM".format(intf))
not_supporting_lpm_physical_ports.add(phy_intf)
continue
tested_physical_ports.add(phy_intf)
logging.info("setting {} physical interface {}".format(intf, phy_intf))
new_lpmode = "off" if original_lpmode[intf].lower() == "on" else "on"
lpmode_set_result = duthost.command("%s %s %s" % (cmd_sfp_set_lpmode, new_lpmode, intf))
assert lpmode_set_result["rc"] == 0, "'%s %s %s' failed" % (cmd_sfp_set_lpmode, new_lpmode, intf)
time.sleep(10)
if len(tested_physical_ports) == 0:
pytest.skip("None of the ports supporting LPM, skip the test")
logging.info("Check SFP lower power mode again after changing SFP lpmode")
lpmode_show = duthost.command(cmd_sfp_show_lpmode)
parsed_lpmode = parse_output(lpmode_show["stdout_lines"][2:])
for intf in dev_conn:
assert intf in parsed_lpmode, "Interface is not in output of '%s'" % cmd_sfp_show_lpmode
assert parsed_lpmode[intf].lower() == "on" or parsed_lpmode[intf].lower() == "off", "Unexpected SFP lpmode"
logging.info("Try to change SFP lpmode")
tested_physical_ports = set()
for intf in dev_conn:
phy_intf = portmap[intf][0]
if phy_intf in not_supporting_lpm_physical_ports:
logging.info("skip testing port {} which doesn't support LPM".format(intf))
continue
if phy_intf in tested_physical_ports:
logging.info("skip tested SFPs {} to avoid repeating operating physical interface {}".format(intf, phy_intf))
continue
tested_physical_ports.add(phy_intf)
logging.info("restoring {} physical interface {}".format(intf, phy_intf))
new_lpmode = original_lpmode[intf].lower()
lpmode_set_result = duthost.command("%s %s %s" % (cmd_sfp_set_lpmode, new_lpmode, intf))
assert lpmode_set_result["rc"] == 0, "'%s %s %s' failed" % (cmd_sfp_set_lpmode, new_lpmode, intf)
time.sleep(10)
logging.info("Check SFP lower power mode again after changing SFP lpmode")
lpmode_show = duthost.command(cmd_sfp_show_lpmode)
parsed_lpmode = parse_output(lpmode_show["stdout_lines"][2:])
for intf in dev_conn:
assert intf in parsed_lpmode, "Interface is not in output of '%s'" % cmd_sfp_show_lpmode
assert parsed_lpmode[intf].lower() == "on" or parsed_lpmode[intf].lower() == "off", "Unexpected SFP lpmode"
logging.info("Check sfp presence again after setting lpmode")
sfp_presence = duthost.command(cmd_sfp_presence)
parsed_presence = parse_output(sfp_presence["stdout_lines"][2:])
for intf in dev_conn:
assert intf in parsed_presence, "Interface is not in output of '%s'" % cmd_sfp_presence
assert parsed_presence[intf] == "Present", "Interface presence is not 'Present'"
logging.info("Check interface status")
namespace = duthost.get_namespace_from_asic_id(enum_frontend_asic_index)
mg_facts = duthost.get_extended_minigraph_facts(tbinfo)
# TODO Remove this logic when minigraph facts supports namespace in multi_asic
up_ports = mg_facts["minigraph_ports"]
if enum_frontend_asic_index is not None:
# Check if the interfaces of this AISC is present in conn_graph_facts
up_ports = {k:v for k, v in portmap.items() if k in mg_facts["minigraph_ports"]}
intf_facts = duthost.interface_facts(namespace=namespace, up_ports=up_ports)["ansible_facts"]
assert len(intf_facts["ansible_interface_link_down_ports"]) == 0, \
"Some interfaces are down: %s" % str(intf_facts["ansible_interface_link_down_ports"])
if duthost.facts["asic_type"] in ["mellanox"]:
loganalyzer.analyze(marker)
def test_thermal_control_fan_status(duthosts, rand_one_dut_hostname, mocker_factory):
"""
@summary: Make FAN absence, over speed and under speed, check logs and LED color.
"""
duthost = duthosts[rand_one_dut_hostname]
loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix='thermal_control')
loganalyzer.load_common_config()
with ThermalPolicyFileContext(duthost, THERMAL_POLICY_VALID_FILE):
fan_mocker = mocker_factory(duthost, 'FanStatusMocker')
if fan_mocker is None:
pytest.skip("No FanStatusMocker for %s, skip rest of the testing in this case" % duthost.facts['asic_type'])
logging.info('Mock FAN status data...')
fan_mocker.mock_data() # make data random
restart_thermal_control_daemon(duthost)
wait_until(THERMAL_CONTROL_TEST_WAIT_TIME, THERMAL_CONTROL_TEST_CHECK_INTERVAL, fan_mocker.check_all_fan_speed,
60)
check_thermal_algorithm_status(duthost, mocker_factory, False)
single_fan_mocker = mocker_factory(duthost, 'SingleFanMocker')
time.sleep(THERMAL_CONTROL_TEST_WAIT_TIME)
_fan_log_supported = duthost.command('docker exec pmon grep -E "{}" /usr/bin/thermalctld'\
.format(LOG_EXPECT_INSUFFICIENT_FAN_NUM_RE), module_ignore_errors=True)
if single_fan_mocker.is_fan_removable():
loganalyzer.expect_regex = [LOG_EXPECT_FAN_REMOVE_RE, LOG_EXPECT_INSUFFICIENT_FAN_NUM_RE]
if _fan_log_supported.is_failed:
loganalyzer.expect_regex.remove(LOG_EXPECT_INSUFFICIENT_FAN_NUM_RE)
with loganalyzer:
logging.info('Mocking an absence FAN...')
single_fan_mocker.mock_absence()
check_cli_output_with_mocker(duthost, single_fan_mocker, CMD_PLATFORM_FANSTATUS, THERMAL_CONTROL_TEST_WAIT_TIME, 2)
loganalyzer.expect_regex = [LOG_EXPECT_FAN_REMOVE_CLEAR_RE, LOG_EXPECT_INSUFFICIENT_FAN_NUM_CLEAR_RE]
if _fan_log_supported.is_failed:
loganalyzer.expect_regex.remove(LOG_EXPECT_INSUFFICIENT_FAN_NUM_CLEAR_RE)
with loganalyzer:
logging.info('Make the absence FAN back to presence...')
single_fan_mocker.mock_presence()
check_cli_output_with_mocker(duthost, single_fan_mocker, CMD_PLATFORM_FANSTATUS, THERMAL_CONTROL_TEST_WAIT_TIME, 2)
if not _fan_log_supported.is_failed:
loganalyzer.expect_regex = [LOG_EXPECT_FAN_FAULT_RE, LOG_EXPECT_INSUFFICIENT_FAN_NUM_RE]
with loganalyzer:
logging.info('Mocking a fault FAN...')
single_fan_mocker.mock_status(False)
check_cli_output_with_mocker(duthost, single_fan_mocker, CMD_PLATFORM_FANSTATUS, THERMAL_CONTROL_TEST_WAIT_TIME, 2)
loganalyzer.expect_regex = [LOG_EXPECT_FAN_FAULT_CLEAR_RE, LOG_EXPECT_INSUFFICIENT_FAN_NUM_CLEAR_RE]
with loganalyzer:
logging.info('Mocking the fault FAN back to normal...')
single_fan_mocker.mock_status(True)
check_cli_output_with_mocker(duthost, single_fan_mocker, CMD_PLATFORM_FANSTATUS, THERMAL_CONTROL_TEST_WAIT_TIME, 2)
loganalyzer.expect_regex = [LOG_EXPECT_FAN_OVER_SPEED_RE]
with loganalyzer:
logging.info('Mocking an over speed FAN...')
single_fan_mocker.mock_over_speed()
check_cli_output_with_mocker(duthost, single_fan_mocker, CMD_PLATFORM_FANSTATUS, THERMAL_CONTROL_TEST_WAIT_TIME, 2)
loganalyzer.expect_regex = [LOG_EXPECT_FAN_OVER_SPEED_CLEAR_RE]
with loganalyzer:
logging.info('Make the over speed FAN back to normal...')
single_fan_mocker.mock_normal_speed()
check_cli_output_with_mocker(duthost, single_fan_mocker, CMD_PLATFORM_FANSTATUS, THERMAL_CONTROL_TEST_WAIT_TIME, 2)
loganalyzer.expect_regex = [LOG_EXPECT_FAN_UNDER_SPEED_RE]
with loganalyzer:
logging.info('Mocking an under speed FAN...')
single_fan_mocker.mock_under_speed()
check_cli_output_with_mocker(duthost, single_fan_mocker, CMD_PLATFORM_FANSTATUS, THERMAL_CONTROL_TEST_WAIT_TIME, 2)
loganalyzer.expect_regex = [LOG_EXPECT_FAN_UNDER_SPEED_CLEAR_RE]
with loganalyzer:
logging.info('Make the under speed FAN back to normal...')
single_fan_mocker.mock_normal_speed()
check_cli_output_with_mocker(duthost, single_fan_mocker, CMD_PLATFORM_FANSTATUS, THERMAL_CONTROL_TEST_WAIT_TIME, 2)
def test_check_sfp_status_and_configure_sfp(duthost, conn_graph_facts):
"""
@summary: Check SFP status and configure SFP
This case is to use the sfputil tool and show command to check SFP status and configure SFP. Currently the
only configuration is to reset SFP. Commands to be tested:
* sfputil show presence
* show interface transceiver presence
* sfputil show eeprom
* show interface transceiver eeprom
* sfputil reset <interface name>
"""
if duthost.facts["asic_type"] in ["mellanox"]:
loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix='sfp_cfg')
loganalyzer.load_common_config()
loganalyzer.ignore_regex.append("kernel.*Eeprom query failed*")
marker = loganalyzer.init()
cmd_sfp_presence = "sudo sfputil show presence"
cmd_sfp_eeprom = "sudo sfputil show eeprom"
cmd_sfp_reset = "sudo sfputil reset"
cmd_xcvr_presence = "show interface transceiver presence"
cmd_xcvr_eeprom = "show interface transceiver eeprom"
portmap = get_port_map(duthost)
logging.info("Got portmap {}".format(portmap))
logging.info("Check output of '%s'" % cmd_sfp_presence)
sfp_presence = duthost.command(cmd_sfp_presence)
parsed_presence = parse_output(sfp_presence["stdout_lines"][2:])
for intf in conn_graph_facts["device_conn"]:
assert intf in parsed_presence, "Interface is not in output of '%s'" % cmd_sfp_presence
assert parsed_presence[intf] == "Present", "Interface presence is not 'Present'"
logging.info("Check output of '%s'" % cmd_xcvr_presence)
xcvr_presence = duthost.command(cmd_xcvr_presence)
parsed_presence = parse_output(xcvr_presence["stdout_lines"][2:])
for intf in conn_graph_facts["device_conn"]:
assert intf in parsed_presence, "Interface is not in output of '%s'" % cmd_xcvr_presence
assert parsed_presence[intf] == "Present", "Interface presence is not 'Present'"
logging.info("Check output of '%s'" % cmd_sfp_eeprom)
sfp_eeprom = duthost.command(cmd_sfp_eeprom)
parsed_eeprom = parse_eeprom(sfp_eeprom["stdout_lines"])
for intf in conn_graph_facts["device_conn"]:
assert intf in parsed_eeprom, "Interface is not in output of 'sfputil show eeprom'"
assert parsed_eeprom[intf] == "SFP EEPROM detected"
logging.info("Check output of '%s'" % cmd_xcvr_eeprom)
xcvr_eeprom = duthost.command(cmd_xcvr_eeprom)
parsed_eeprom = parse_eeprom(xcvr_eeprom["stdout_lines"])
for intf in conn_graph_facts["device_conn"]:
assert intf in parsed_eeprom, "Interface is not in output of '%s'" % cmd_xcvr_eeprom
assert parsed_eeprom[intf] == "SFP EEPROM detected"
logging.info("Test '%s <interface name>'" % cmd_sfp_reset)
tested_physical_ports = set()
for intf in conn_graph_facts["device_conn"]:
phy_intf = portmap[intf][0]
if phy_intf in tested_physical_ports:
logging.info("skip tested SFPs {} to avoid repeating operating physical interface {}".format(intf, phy_intf))
continue
tested_physical_ports.add(phy_intf)
logging.info("resetting {} physical interface {}".format(intf, phy_intf))
reset_result = duthost.command("%s %s" % (cmd_sfp_reset, intf))
assert reset_result["rc"] == 0, "'%s %s' failed" % (cmd_sfp_reset, intf)
time.sleep(5)
logging.info("Wait some time for SFP to fully recover after reset")
time.sleep(60)
logging.info("Check sfp presence again after reset")
sfp_presence = duthost.command(cmd_sfp_presence)
parsed_presence = parse_output(sfp_presence["stdout_lines"][2:])
for intf in conn_graph_facts["device_conn"]:
assert intf in parsed_presence, "Interface is not in output of '%s'" % cmd_sfp_presence
assert parsed_presence[intf] == "Present", "Interface presence is not 'Present'"
logging.info("Check interface status")
mg_facts = duthost.minigraph_facts(host=duthost.hostname)["ansible_facts"]
intf_facts = duthost.interface_facts(up_ports=mg_facts["minigraph_ports"])["ansible_facts"]
assert len(intf_facts["ansible_interface_link_down_ports"]) == 0, \
"Some interfaces are down: %s" % str(intf_facts["ansible_interface_link_down_ports"])
if duthost.facts["asic_type"] in ["mellanox"]:
loganalyzer.analyze(marker)
def test_check_sfp_low_power_mode(duthost, conn_graph_facts):
"""
@summary: Check SFP low power mode
This case is to use the sfputil tool command to check and set SFP low power mode
* sfputil show lpmode
* sfputil lpmode off
* sfputil lpmode on
"""
if duthost.facts["asic_type"] in ["mellanox"]:
loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix='sfp_lpm')
loganalyzer.load_common_config()
loganalyzer.ignore_regex.append("Eeprom query failed")
marker = loganalyzer.init()
cmd_sfp_presence = "sudo sfputil show presence"
cmd_sfp_show_lpmode = "sudo sfputil show lpmode"
cmd_sfp_set_lpmode = "sudo sfputil lpmode"
portmap = get_port_map(duthost)
logging.info("Got portmap {}".format(portmap))
logging.info("Check output of '%s'" % cmd_sfp_show_lpmode)
lpmode_show = duthost.command(cmd_sfp_show_lpmode)
parsed_lpmode = parse_output(lpmode_show["stdout_lines"][2:])
original_lpmode = copy.deepcopy(parsed_lpmode)
for intf in conn_graph_facts["device_conn"]:
assert intf in parsed_lpmode, "Interface is not in output of '%s'" % cmd_sfp_show_lpmode
assert parsed_lpmode[intf].lower() == "on" or parsed_lpmode[intf].lower() == "off", "Unexpected SFP lpmode"
logging.info("Try to change SFP lpmode")
tested_physical_ports = set()
for intf in conn_graph_facts["device_conn"]:
phy_intf = portmap[intf][0]
if phy_intf in tested_physical_ports:
logging.info("skip tested SFPs {} to avoid repeating operating physical interface {}".format(intf, phy_intf))
continue
tested_physical_ports.add(phy_intf)
logging.info("setting {} physical interface {}".format(intf, phy_intf))
new_lpmode = "off" if original_lpmode[intf].lower() == "on" else "on"
lpmode_set_result = duthost.command("%s %s %s" % (cmd_sfp_set_lpmode, new_lpmode, intf))
assert lpmode_set_result["rc"] == 0, "'%s %s %s' failed" % (cmd_sfp_set_lpmode, new_lpmode, intf)
time.sleep(10)
logging.info("Check SFP lower power mode again after changing SFP lpmode")
lpmode_show = duthost.command(cmd_sfp_show_lpmode)
parsed_lpmode = parse_output(lpmode_show["stdout_lines"][2:])
for intf in conn_graph_facts["device_conn"]:
assert intf in parsed_lpmode, "Interface is not in output of '%s'" % cmd_sfp_show_lpmode
assert parsed_lpmode[intf].lower() == "on" or parsed_lpmode[intf].lower() == "off", "Unexpected SFP lpmode"
logging.info("Try to change SFP lpmode")
tested_physical_ports = set()
for intf in conn_graph_facts["device_conn"]:
phy_intf = portmap[intf][0]
if phy_intf in tested_physical_ports:
logging.info("skip tested SFPs {} to avoid repeating operating physical interface {}".format(intf, phy_intf))
continue
tested_physical_ports.add(phy_intf)
logging.info("restoring {} physical interface {}".format(intf, phy_intf))
new_lpmode = original_lpmode[intf].lower()
lpmode_set_result = duthost.command("%s %s %s" % (cmd_sfp_set_lpmode, new_lpmode, intf))
assert lpmode_set_result["rc"] == 0, "'%s %s %s' failed" % (cmd_sfp_set_lpmode, new_lpmode, intf)
time.sleep(10)
logging.info("Check SFP lower power mode again after changing SFP lpmode")
lpmode_show = duthost.command(cmd_sfp_show_lpmode)
parsed_lpmode = parse_output(lpmode_show["stdout_lines"][2:])
for intf in conn_graph_facts["device_conn"]:
assert intf in parsed_lpmode, "Interface is not in output of '%s'" % cmd_sfp_show_lpmode
assert parsed_lpmode[intf].lower() == "on" or parsed_lpmode[intf].lower() == "off", "Unexpected SFP lpmode"
logging.info("Check sfp presence again after setting lpmode")
sfp_presence = duthost.command(cmd_sfp_presence)
parsed_presence = parse_output(sfp_presence["stdout_lines"][2:])
for intf in conn_graph_facts["device_conn"]:
assert intf in parsed_presence, "Interface is not in output of '%s'" % cmd_sfp_presence
assert parsed_presence[intf] == "Present", "Interface presence is not 'Present'"
logging.info("Check interface status")
mg_facts = duthost.minigraph_facts(host=duthost.hostname)["ansible_facts"]
intf_facts = duthost.interface_facts(up_ports=mg_facts["minigraph_ports"])["ansible_facts"]
assert len(intf_facts["ansible_interface_link_down_ports"]) == 0, \
"Some interfaces are down: %s" % str(intf_facts["ansible_interface_link_down_ports"])
if duthost.facts["asic_type"] in ["mellanox"]:
loganalyzer.analyze(marker)
def test_nhop(request, duthost, tbinfo):
"""
Test next hop group resource count. Steps:
- Add test IP address to an active IP interface
- Add static ARPs
- Create unique next hop groups
- Add IP route and nexthop
- check CRM resource
- clean up
- Verify no erros and crash
"""
skip_release(duthost, ["201811", "201911"])
default_max_nhop_paths = 32
nhop_group_limit = 1024
# program more than the advertised limit
extra_nhops = 10
asic = duthost.asic_instance()
# find out MAX NHOP group count supported on the platform
result = asic.run_redis_cmd(
argv=["redis-cli", "-n", 6, "HGETALL", "SWITCH_CAPABILITY|switch"])
it = iter(result)
switch_capability = dict(zip(it, it))
max_nhop = switch_capability.get("MAX_NEXTHOP_GROUP_COUNT")
max_nhop = nhop_group_limit if max_nhop == None else int(max_nhop)
nhop_group_count = min(max_nhop, nhop_group_limit) + extra_nhops
# find out an active IP port
ip_ifaces = asic.get_active_ip_interfaces(tbinfo).keys()
pytest_assert(len(ip_ifaces), "No IP interfaces found")
eth_if = ip_ifaces[0]
# Generate ARP entries
arp_count = 40
arplist = Arp(duthost, asic, arp_count, eth_if)
arplist.arps_add()
# indices
indices = range(arp_count)
ip_indices = combinations(indices, default_max_nhop_paths)
# intitialize log analyzer
marker = "NHOP TEST PATH COUNT {} {}".format(nhop_group_count, eth_if)
loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix=marker)
marker = loganalyzer.init()
loganalyzer.load_common_config()
loganalyzer.expect_regex = []
loganalyzer.ignore_regex.extend(loganalyzer_ignore_regex_list())
ip_prefix = ipaddr.IPAddress("192.168.0.0")
# list of all IPs available to generate a nexthop group
ip_list = arplist.ip_mac_list
crm_before = get_crm_info(duthost, asic)
# increase CRM polling time
asic.command("crm config polling interval 10")
logging.info("Adding {} next hops on {}".format(nhop_group_count, eth_if))
# create nexthop group
nhop = IPRoutes(duthost, asic)
try:
for i, indx_list in zip(range(nhop_group_count), ip_indices):
# get a list of unique group of next hop IPs
ips = [arplist.ip_mac_list[x].ip for x in indx_list]
ip_route = "{}/31".format(ip_prefix + (2 * i))
# add IP route with the next hop group created
nhop.add_ip_route(ip_route, ips)
nhop.program_routes()
# wait for routes to be synced and programmed
time.sleep(120)
crm_after = get_crm_info(duthost, asic)
finally:
nhop.delete_routes()
arplist.clean_up()
asic.command("crm config polling interval {}".format(
crm_before["polling"]))
# check for any errors or crash
loganalyzer.analyze(marker)
# verify the test used up all the NHOP group resources
# skip this check on Mellanox as ASIC resources are shared
if not is_mellanox_device(duthost):
pytest_assert(
crm_after["available"] == 0,
"Unused NHOP group resource: {}, used:{}".format(
crm_after["available"], crm_after["used"]))