def check_thermal_control_load_invalid_file(testbed_devices, file_name):
"""
@summary: Load an invalid thermal policy file check thermal
control daemon is up and there is an error log printed
"""
dut = testbed_devices["dut"]
loganalyzer = LogAnalyzer(ansible_host=dut,
marker_prefix='thermal_control')
with ThermalPolicyFileContext(dut, file_name):
loganalyzer.expect_regex = [LOG_EXPECT_POLICY_FILE_INVALID]
with loganalyzer:
restart_thermal_control_daemon(dut)
def loganalyzer(duthost, request):
if request.config.getoption("--disable_loganalyzer"
) or "disable_loganalyzer" in request.keywords:
logging.info("Log analyzer is disabled")
yield
return
# Force rotate logs
try:
duthost.shell(
"/usr/sbin/logrotate -f /etc/logrotate.conf > /dev/null 2>&1")
except RunAnsibleModuleFail as e:
logging.warning("logrotate is failed. Command returned:\n"
"Stdout: {}\n"
"Stderr: {}\n"
"Return code: {}".format(e.results["stdout"],
e.results["stderr"],
e.results["rc"]))
loganalyzer = LogAnalyzer(ansible_host=duthost,
marker_prefix=request.node.name)
logging.info("Add start marker into DUT syslog")
marker = loganalyzer.init()
logging.info("Load config and analyze log")
# Read existed common regular expressions located with legacy loganalyzer module
loganalyzer.load_common_config()
yield loganalyzer
# Skip LogAnalyzer if case is skipped
if "rep_call" in request.node.__dict__ and request.node.rep_call.skipped:
return
loganalyzer.analyze(marker)
def loganalyzer(duthost, request):
loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix=request.node.name)
# Add start marker into DUT syslog
marker = loganalyzer.init()
yield loganalyzer
if not request.config.getoption("--disable_loganalyzer") and "disable_loganalyzer" not in request.keywords:
# Read existed common regular expressions located with legacy loganalyzer module
loganalyzer.load_common_config()
# Parse syslog and process result. Raise "LogAnalyzerError" exception if: total match or expected missing
# match is not equal to zero
loganalyzer.analyze(marker)
else:
# Add end marker into DUT syslog
loganalyzer._add_end_marker(marker)
def acl_rules(self, duthost, localhost, setup, acl_table):
"""
setup/teardown ACL rules based on test class requirements
:param duthost: DUT host object
:param localhost: localhost object
:param setup: setup information
:param acl_table: table creating fixture
:return:
"""
loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix='acl_rules')
loganalyzer.load_common_config()
try:
loganalyzer.expect_regex = [LOG_EXPECT_ACL_RULE_CREATE_RE]
with loganalyzer:
self.setup_rules(duthost, setup, acl_table)
self.post_setup_hook(duthost, localhost)
except LogAnalyzerError as err:
# cleanup config DB in case of log analysis error
self.teardown_rules(duthost, setup)
raise err
try:
yield
finally:
loganalyzer.expect_regex = [LOG_EXPECT_ACL_RULE_REMOVE_RE]
with loganalyzer:
self.teardown_rules(duthost, setup)
def acl_table(duthost, acl_table_config):
"""
fixture to apply ACL table configuration and remove after tests
:param duthost: DUT object
:param acl_table_config: ACL table configuration dictionary
:return: forwards acl_table_config
"""
name = acl_table_config['name']
conf = acl_table_config['config_file']
loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix='acl')
loganalyzer.load_common_config()
try:
loganalyzer.expect_regex = [LOG_EXPECT_ACL_TABLE_CREATE_RE]
with loganalyzer:
logger.info('creating ACL table: applying {}'.format(conf))
# TODO: use sonic config CLI
duthost.command('sonic-cfggen -j {} --write-to-db'.format(conf))
except LogAnalyzerError as err:
# cleanup config DB if create failed
duthost.command('config acl remove table {}'.format(name))
raise err
try:
yield acl_table_config
finally:
loganalyzer.expect_regex = [LOG_EXPECT_ACL_TABLE_REMOVE_RE]
with loganalyzer:
logger.info('removing ACL table {}'.format(name))
duthost.command('config acl remove table {}'.format(name))
# save cleaned configuration
duthost.command('config save -y')
def loganalyzer(duthost, request):
if request.config.getoption("--disable_loganalyzer"
) or "disable_loganalyzer" in request.keywords:
logging.info("Log analyzer is disabled")
yield
return
loganalyzer = LogAnalyzer(ansible_host=duthost,
marker_prefix=request.node.name)
logging.info("Add start marker into DUT syslog")
marker = loganalyzer.init()
logging.info("Load config and analyze log")
# Read existed common regular expressions located with legacy loganalyzer module
loganalyzer.load_common_config()
yield loganalyzer
loganalyzer.analyze(marker)
def loganalyzer(duthosts, request):
if request.config.getoption("--disable_loganalyzer"
) or "disable_loganalyzer" in request.keywords:
logging.info("Log analyzer is disabled")
yield
return
# Analyze all the duts
analyzers = {}
parallel_run(analyzer_logrotate, [], {}, duthosts, timeout=120)
for duthost in duthosts:
analyzers[duthost.hostname] = LogAnalyzer(
ansible_host=duthost, marker_prefix=request.node.name)
markers = parallel_run(analyzer_add_marker, [analyzers], {},
duthosts,
timeout=120)
yield analyzers
# Skip LogAnalyzer if case is skipped
if "rep_call" in request.node.__dict__ and request.node.rep_call.skipped:
return
logging.info("Starting to analyse on all DUTs")
parallel_run(analyze_logs, [analyzers, markers], {}, duthosts, timeout=120)
def test_check_sfp_status_and_configure_sfp(testbed_devices, conn_graph_facts):
"""
@summary: Check SFP status and configure SFP
This case is to use the sfputil tool and show command to check SFP status and configure SFP. Currently the
only configuration is to reset SFP. Commands to be tested:
* sfputil show presence
* show interface transceiver presence
* sfputil show eeprom
* show interface transceiver eeprom
* sfputil reset <interface name>
"""
ans_host = testbed_devices["dut"]
if ans_host.facts["asic_type"] in ["mellanox"]:
loganalyzer = LogAnalyzer(ansible_host=ans_host,
marker_prefix='sfp_cfg')
loganalyzer.load_common_config()
loganalyzer.ignore_regex.append("kernel.*Eeprom query failed*")
marker = loganalyzer.init()
cmd_sfp_presence = "sudo sfputil show presence"
cmd_sfp_eeprom = "sudo sfputil show eeprom"
cmd_sfp_reset = "sudo sfputil reset"
cmd_xcvr_presence = "show interface transceiver presence"
cmd_xcvr_eeprom = "show interface transceiver eeprom"
portmap = get_port_map(testbed_devices)
logging.info("Got portmap {}".format(portmap))
logging.info("Check output of '%s'" % cmd_sfp_presence)
sfp_presence = ans_host.command(cmd_sfp_presence)
parsed_presence = parse_output(sfp_presence["stdout_lines"][2:])
for intf in conn_graph_facts["device_conn"]:
assert intf in parsed_presence, "Interface is not in output of '%s'" % cmd_sfp_presence
assert parsed_presence[
intf] == "Present", "Interface presence is not 'Present'"
logging.info("Check output of '%s'" % cmd_xcvr_presence)
xcvr_presence = ans_host.command(cmd_xcvr_presence)
parsed_presence = parse_output(xcvr_presence["stdout_lines"][2:])
for intf in conn_graph_facts["device_conn"]:
assert intf in parsed_presence, "Interface is not in output of '%s'" % cmd_xcvr_presence
assert parsed_presence[
intf] == "Present", "Interface presence is not 'Present'"
logging.info("Check output of '%s'" % cmd_sfp_eeprom)
sfp_eeprom = ans_host.command(cmd_sfp_eeprom)
parsed_eeprom = parse_eeprom(sfp_eeprom["stdout_lines"])
for intf in conn_graph_facts["device_conn"]:
assert intf in parsed_eeprom, "Interface is not in output of 'sfputil show eeprom'"
assert parsed_eeprom[intf] == "SFP EEPROM detected"
logging.info("Check output of '%s'" % cmd_xcvr_eeprom)
xcvr_eeprom = ans_host.command(cmd_xcvr_eeprom)
parsed_eeprom = parse_eeprom(xcvr_eeprom["stdout_lines"])
for intf in conn_graph_facts["device_conn"]:
assert intf in parsed_eeprom, "Interface is not in output of '%s'" % cmd_xcvr_eeprom
assert parsed_eeprom[intf] == "SFP EEPROM detected"
logging.info("Test '%s <interface name>'" % cmd_sfp_reset)
tested_physical_ports = set()
for intf in conn_graph_facts["device_conn"]:
phy_intf = portmap[intf][0]
if phy_intf in tested_physical_ports:
logging.info(
"skip tested SFPs {} to avoid repeating operating physical interface {}"
.format(intf, phy_intf))
continue
tested_physical_ports.add(phy_intf)
logging.info("resetting {} physical interface {}".format(
intf, phy_intf))
reset_result = ans_host.command("%s %s" % (cmd_sfp_reset, intf))
assert reset_result["rc"] == 0, "'%s %s' failed" % (cmd_sfp_reset,
intf)
time.sleep(5)
logging.info("Wait some time for SFP to fully recover after reset")
time.sleep(60)
logging.info("Check sfp presence again after reset")
sfp_presence = ans_host.command(cmd_sfp_presence)
parsed_presence = parse_output(sfp_presence["stdout_lines"][2:])
for intf in conn_graph_facts["device_conn"]:
assert intf in parsed_presence, "Interface is not in output of '%s'" % cmd_sfp_presence
assert parsed_presence[
intf] == "Present", "Interface presence is not 'Present'"
logging.info("Check interface status")
mg_facts = ans_host.minigraph_facts(
host=ans_host.hostname)["ansible_facts"]
intf_facts = ans_host.interface_facts(
up_ports=mg_facts["minigraph_ports"])["ansible_facts"]
assert len(intf_facts["ansible_interface_link_down_ports"]) == 0, \
"Some interfaces are down: %s" % str(intf_facts["ansible_interface_link_down_ports"])
if ans_host.facts["asic_type"] in ["mellanox"]:
loganalyzer.analyze(marker)
def test_check_sfp_low_power_mode(testbed_devices, conn_graph_facts):
"""
@summary: Check SFP low power mode
This case is to use the sfputil tool command to check and set SFP low power mode
* sfputil show lpmode
* sfputil lpmode off
* sfputil lpmode on
"""
ans_host = testbed_devices["dut"]
if ans_host.facts["asic_type"] in ["mellanox"]:
loganalyzer = LogAnalyzer(ansible_host=ans_host,
marker_prefix='sfp_lpm')
loganalyzer.load_common_config()
loganalyzer.ignore_regex.append("Eeprom query failed")
marker = loganalyzer.init()
cmd_sfp_presence = "sudo sfputil show presence"
cmd_sfp_show_lpmode = "sudo sfputil show lpmode"
cmd_sfp_set_lpmode = "sudo sfputil lpmode"
portmap = get_port_map(testbed_devices)
logging.info("Got portmap {}".format(portmap))
logging.info("Check output of '%s'" % cmd_sfp_show_lpmode)
lpmode_show = ans_host.command(cmd_sfp_show_lpmode)
parsed_lpmode = parse_output(lpmode_show["stdout_lines"][2:])
original_lpmode = copy.deepcopy(parsed_lpmode)
for intf in conn_graph_facts["device_conn"]:
assert intf in parsed_lpmode, "Interface is not in output of '%s'" % cmd_sfp_show_lpmode
assert parsed_lpmode[intf].lower() == "on" or parsed_lpmode[
intf].lower() == "off", "Unexpected SFP lpmode"
logging.info("Try to change SFP lpmode")
tested_physical_ports = set()
for intf in conn_graph_facts["device_conn"]:
phy_intf = portmap[intf][0]
if phy_intf in tested_physical_ports:
logging.info(
"skip tested SFPs {} to avoid repeating operating physical interface {}"
.format(intf, phy_intf))
continue
tested_physical_ports.add(phy_intf)
logging.info("setting {} physical interface {}".format(intf, phy_intf))
new_lpmode = "off" if original_lpmode[intf].lower() == "on" else "on"
lpmode_set_result = ans_host.command(
"%s %s %s" % (cmd_sfp_set_lpmode, new_lpmode, intf))
assert lpmode_set_result["rc"] == 0, "'%s %s %s' failed" % (
cmd_sfp_set_lpmode, new_lpmode, intf)
time.sleep(10)
logging.info("Check SFP lower power mode again after changing SFP lpmode")
lpmode_show = ans_host.command(cmd_sfp_show_lpmode)
parsed_lpmode = parse_output(lpmode_show["stdout_lines"][2:])
for intf in conn_graph_facts["device_conn"]:
assert intf in parsed_lpmode, "Interface is not in output of '%s'" % cmd_sfp_show_lpmode
assert parsed_lpmode[intf].lower() == "on" or parsed_lpmode[
intf].lower() == "off", "Unexpected SFP lpmode"
logging.info("Try to change SFP lpmode")
tested_physical_ports = set()
for intf in conn_graph_facts["device_conn"]:
phy_intf = portmap[intf][0]
if phy_intf in tested_physical_ports:
logging.info(
"skip tested SFPs {} to avoid repeating operating physical interface {}"
.format(intf, phy_intf))
continue
tested_physical_ports.add(phy_intf)
logging.info("restoring {} physical interface {}".format(
intf, phy_intf))
new_lpmode = original_lpmode[intf].lower()
lpmode_set_result = ans_host.command(
"%s %s %s" % (cmd_sfp_set_lpmode, new_lpmode, intf))
assert lpmode_set_result["rc"] == 0, "'%s %s %s' failed" % (
cmd_sfp_set_lpmode, new_lpmode, intf)
time.sleep(10)
logging.info("Check SFP lower power mode again after changing SFP lpmode")
lpmode_show = ans_host.command(cmd_sfp_show_lpmode)
parsed_lpmode = parse_output(lpmode_show["stdout_lines"][2:])
for intf in conn_graph_facts["device_conn"]:
assert intf in parsed_lpmode, "Interface is not in output of '%s'" % cmd_sfp_show_lpmode
assert parsed_lpmode[intf].lower() == "on" or parsed_lpmode[
intf].lower() == "off", "Unexpected SFP lpmode"
logging.info("Check sfp presence again after setting lpmode")
sfp_presence = ans_host.command(cmd_sfp_presence)
parsed_presence = parse_output(sfp_presence["stdout_lines"][2:])
for intf in conn_graph_facts["device_conn"]:
assert intf in parsed_presence, "Interface is not in output of '%s'" % cmd_sfp_presence
assert parsed_presence[
intf] == "Present", "Interface presence is not 'Present'"
logging.info("Check interface status")
mg_facts = ans_host.minigraph_facts(
host=ans_host.hostname)["ansible_facts"]
intf_facts = ans_host.interface_facts(
up_ports=mg_facts["minigraph_ports"])["ansible_facts"]
assert len(intf_facts["ansible_interface_link_down_ports"]) == 0, \
"Some interfaces are down: %s" % str(intf_facts["ansible_interface_link_down_ports"])
if ans_host.facts["asic_type"] in ["mellanox"]:
loganalyzer.analyze(marker)
def test_thermal_control_fan_status(testbed_devices, mocker_factory):
"""
@summary: Make FAN absence, over speed and under speed, check logs and LED color.
"""
dut = testbed_devices["dut"]
loganalyzer = LogAnalyzer(ansible_host=dut,
marker_prefix='thermal_control')
loganalyzer.load_common_config()
with ThermalPolicyFileContext(dut, THERMAL_POLICY_VALID_FILE):
fan_mocker = mocker_factory(dut, 'FanStatusMocker')
if fan_mocker is None:
pytest.skip(
"No FanStatusMocker for %s, skip rest of the testing in this case"
% dut.facts['asic_type'])
logging.info('Mock FAN status data...')
fan_mocker.mock_data() # make data random
restart_thermal_control_daemon(dut)
wait_until(THERMAL_CONTROL_TEST_WAIT_TIME,
THERMAL_CONTROL_TEST_CHECK_INTERVAL,
fan_mocker.check_all_fan_speed, 60)
check_thermal_algorithm_status(dut, mocker_factory, False)
single_fan_mocker = mocker_factory(dut, 'SingleFanMocker')
time.sleep(THERMAL_CONTROL_TEST_WAIT_TIME)
loganalyzer.expect_regex = [LOG_EXPECT_FAN_REMOVE_RE]
with loganalyzer:
logging.info('Mocking an absence FAN...')
single_fan_mocker.mock_absence()
check_cli_output_with_mocker(dut, single_fan_mocker,
CMD_PLATFORM_FANSTATUS,
THERMAL_CONTROL_TEST_WAIT_TIME)
loganalyzer.expect_regex = [LOG_EXPECT_FAN_REMOVE_CLEAR_RE]
with loganalyzer:
logging.info('Make the absence FAN back to presence...')
single_fan_mocker.mock_presence()
check_cli_output_with_mocker(dut, single_fan_mocker,
CMD_PLATFORM_FANSTATUS,
THERMAL_CONTROL_TEST_WAIT_TIME)
loganalyzer.expect_regex = [LOG_EXPECT_FAN_OVER_SPEED_RE]
with loganalyzer:
logging.info('Mocking an over speed FAN...')
single_fan_mocker.mock_over_speed()
check_cli_output_with_mocker(dut, single_fan_mocker,
CMD_PLATFORM_FANSTATUS,
THERMAL_CONTROL_TEST_WAIT_TIME)
loganalyzer.expect_regex = [LOG_EXPECT_FAN_OVER_SPEED_CLEAR_RE]
with loganalyzer:
logging.info('Make the over speed FAN back to normal...')
single_fan_mocker.mock_normal_speed()
check_cli_output_with_mocker(dut, single_fan_mocker,
CMD_PLATFORM_FANSTATUS,
THERMAL_CONTROL_TEST_WAIT_TIME)
loganalyzer.expect_regex = [LOG_EXPECT_FAN_UNDER_SPEED_RE]
with loganalyzer:
logging.info('Mocking an under speed FAN...')
single_fan_mocker.mock_under_speed()
check_cli_output_with_mocker(dut, single_fan_mocker,
CMD_PLATFORM_FANSTATUS,
THERMAL_CONTROL_TEST_WAIT_TIME)
loganalyzer.expect_regex = [LOG_EXPECT_FAN_UNDER_SPEED_CLEAR_RE]
with loganalyzer:
logging.info('Make the under speed FAN back to normal...')
single_fan_mocker.mock_normal_speed()
check_cli_output_with_mocker(dut, single_fan_mocker,
CMD_PLATFORM_FANSTATUS,
THERMAL_CONTROL_TEST_WAIT_TIME)