def create_acl_table(rand_selected_dut, tbinfo):
"""
Create two ACL tables on DUT for testing.
"""
mg_facts = rand_selected_dut.get_extended_minigraph_facts(tbinfo)
# Get the list of LAGs
port_channels = ",".join(mg_facts["minigraph_portchannels"].keys())
cmds = [
"config acl add table {} L3 -p {}".format(ACL_TABLE_NAME_V4,
port_channels),
"config acl add table {} L3V6 -p {}".format(ACL_TABLE_NAME_V6,
port_channels)
]
logger.info("Creating ACL table for testing")
loganalyzer = LogAnalyzer(ansible_host=rand_selected_dut,
marker_prefix="null_route_helper")
loganalyzer.match_regex = [LOG_ERROR_INSUFFICIENT_RESOURCES]
# Skip test case if ACL table created failed due to insufficient resources
try:
with loganalyzer:
rand_selected_dut.shell_cmds(cmds=cmds)
except LogAnalyzerError:
skip_msg = "ACL table creation failed due to insufficient resources, test case will be skipped"
logger.error(skip_msg)
remove_acl_table(rand_selected_dut)
pytest.skip(skip_msg)
yield
remove_acl_table(rand_selected_dut)
def run_test(self, duthost, storm_hndle, expect_regex, syslog_marker,
action):
"""
Storm generation/restoration on all ports and verification
Args:
duthost (AnsibleHost): DUT instance
storm_hndle (PFCMultiStorm): class PFCMultiStorm intance
expect_regex (list): list of expect regexs to be matched in the syslog
syslog_marker (string): marker prefix written to the syslog
action (string): storm/restore action
"""
loganalyzer = LogAnalyzer(ansible_host=duthost,
marker_prefix=syslog_marker)
ignore_file = os.path.join(TEMPLATES_DIR, "ignore_pfc_wd_messages")
reg_exp = loganalyzer.parse_regexp_file(src=ignore_file)
loganalyzer.ignore_regex.extend(reg_exp)
loganalyzer.expect_regex = []
loganalyzer.expect_regex.extend(expect_regex)
loganalyzer.match_regex = []
with loganalyzer:
if action == "storm":
storm_hndle.start_pfc_storm()
elif action == "restore":
storm_hndle.stop_pfc_storm()
time.sleep(5)
def check_not_implemented_warnings(duthosts, enum_rand_one_per_hwsku_hostname):
duthost = duthosts[enum_rand_one_per_hwsku_hostname]
loganalyzer = LogAnalyzer(ansible_host=duthost,
marker_prefix="platformapi_test")
marker = loganalyzer.init()
yield
loganalyzer.match_regex.extend(['WARNING pmon#platform_api_server.py: API.+not implemented'])
loganalyzer.analyze(marker)
def check_thermal_control_load_invalid_file(duthost, file_name):
"""
@summary: Load an invalid thermal policy file check thermal
control daemon is up and there is an error log printed
"""
loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix='thermal_control')
loganalyzer.expect_regex = [LOG_EXPECT_POLICY_FILE_INVALID]
with loganalyzer:
with ThermalPolicyFileContext(duthost, file_name):
restart_thermal_control_daemon(duthost)
def test_container_checker(duthosts, enum_rand_one_per_hwsku_hostname,
enum_rand_one_asic_index, enum_dut_feature, tbinfo):
"""Tests the feature of container checker.
This function will check whether the container names will appear in the Monit
alerting message if they are stopped explicitly or they hit start limitation.
Args:
duthosts: list of DUTs.
enum_rand_one_per_hwsku_hostname: Fixture returning list of hostname selected per hwsku.
enum_rand_one_asic_index: Fixture returning list of asics for selected duts.
enum_dut_feature: A list contains features.
tbinfo: Testbed information.
Returns:
None.
"""
service_name = enum_dut_feature
duthost = duthosts[enum_rand_one_per_hwsku_hostname]
asic = duthost.asic_instance(enum_rand_one_asic_index)
container_name = asic.get_docker_name(service_name)
loganalyzer = LogAnalyzer(
ansible_host=duthost,
marker_prefix="container_checker_{}".format(container_name))
disabled_containers = get_disabled_container_list(duthost)
skip_containers = disabled_containers[:]
# Skip 'radv' container on devices whose role is not T0.
if tbinfo["topo"]["type"] != "t0":
skip_containers.append("radv")
pytest_require(
service_name not in skip_containers,
"Container '{}' is skipped for testing.".format(container_name))
asic.stop_service(service_name)
logger.info(
"Waiting until container '{}' is stopped...".format(container_name))
stopped = wait_until(CONTAINER_STOP_THRESHOLD_SECS,
CONTAINER_CHECK_INTERVAL_SECS, 0,
check_container_state, duthost, container_name, False)
pytest_assert(stopped,
"Failed to stop container '{}'".format(container_name))
logger.info("Container '{}' on DuT '{}' was stopped".format(
container_name, duthost.hostname))
loganalyzer.expect_regex = get_expected_alerting_message(container_name)
with loganalyzer:
# Wait for 1 minutes such that Monit has a chance to write alerting message into syslog.
logger.info("Sleep 1 minutes to wait for the alerting message...")
time.sleep(70)
def test_container_checker(duthosts, enum_dut_feature_container,
rand_selected_dut, tbinfo):
"""Tests the feature of container checker.
This function will check whether the container names will appear in the Monit
alerting message if they are stopped explicitly or they hit start limitation.
Args:
duthosts: list of DUTs.
enum_dut_feature_container: A list contains strings ("<dut_name>|<container_name>").
rand_selected_dut: The fixture returns a randomly selected DuT.
tbinfo: Testbed information.
Returns:
None.
"""
dut_name, container_name = decode_dut_and_container_name(
enum_dut_feature_container)
pytest_require(
dut_name == rand_selected_dut.hostname and container_name != "unknown",
"Skips testing container_checker of container '{}' on the DuT '{}' since another DuT '{}' was chosen."
.format(container_name, dut_name, rand_selected_dut.hostname))
duthost = duthosts[dut_name]
loganalyzer = LogAnalyzer(
ansible_host=duthost,
marker_prefix="container_checker_{}".format(container_name))
disabled_containers = get_disabled_container_list(duthost)
skip_containers = disabled_containers[:]
skip_containers.append("gbsyncd")
skip_containers.append("database")
skip_containers.append("database-chassis")
# Skip 'radv' container on devices whose role is not T0.
if tbinfo["topo"]["type"] != "t0":
skip_containers.append("radv")
pytest_require(
container_name not in skip_containers,
"Container '{}' is skipped for testing.".format(container_name))
stop_container(duthost, container_name)
loganalyzer.expect_regex = get_expected_alerting_message(container_name)
with loganalyzer:
# Wait for 1 minutes such that Monit has a chance to write alerting message into syslog.
logger.info("Sleep 1 minutes to wait for the alerting message...")
time.sleep(70)
def _remove_acl_table(self, duthost, stage, ip_ver):
table_name = ACL_TABLE_NAME_TEMPLATE.format(stage, ip_ver)
cmd = "config acl remove table {}".format(table_name)
logger.info("Removing ACL table {}".format(table_name))
loganalyzer = LogAnalyzer(ansible_host=duthost,
marker_prefix="TestAclVlanOuter")
loganalyzer.expect_regex = [LOG_EXPECT_ACL_TABLE_REMOVE_RE]
try:
with loganalyzer:
duthost.shell(cmd)
except LogAnalyzerError:
#Todo: cleanup
pytest.fail("Failed to remove ACL table {}".format(table_name))
def storm_detect_path(self, port, queue, first_detect_after_wb=False):
"""
Storm detection action and associated verifications
Args:
port(string) : DUT port
queue(int): queue on the port that will be stormed
first_detect_after_wb(bool): first detect iteration after warm reboot (default: False)
"""
# for the first iteration after wb, do not write a marker to the log but specify the start msg from
# where to search the logs
start_marker = None
if first_detect_after_wb:
start_marker = "NOTICE swss#orchagent: :- setWarmStartState: orchagent warm start state changed to initialized"
self.loganalyzer = LogAnalyzer(
ansible_host=self.dut,
marker_prefix="pfcwd_wb_storm_detect_port_{}_queue_{}".format(
port, queue),
start_marker=start_marker)
marker = self.loganalyzer.init()
time.sleep(5)
ignore_file = os.path.join(TEMPLATES_DIR, "ignore_pfc_wd_messages")
reg_exp = self.loganalyzer.parse_regexp_file(src=ignore_file)
self.loganalyzer.ignore_regex.extend(reg_exp)
self.loganalyzer.expect_regex = []
self.loganalyzer.expect_regex.extend([EXPECT_PFC_WD_DETECT_RE])
self.loganalyzer.match_regex = []
# ongoing storm. no need to start a new one
if not first_detect_after_wb:
if not self.pfc_wd['fake_storm']:
self.storm_handle[port][queue].start_storm()
time.sleep(15 * len(self.pfc_wd['queue_indices']))
else:
PfcCmd.set_storm_status(self.dut, self.oid_map[(port, queue)],
"enabled")
time.sleep(5)
else:
# for the first iteration after wb, check the log for detect msgs for the ongoing storms
self.loganalyzer.expected_matches_target = len(self.ports) * len(
self.pfc_wd['queue_indices'])
time.sleep(20)
# storm detect check
logger.info(
"Verify if PFC storm is detected on port {} queue {}".format(
port, queue))
self.loganalyzer.analyze(marker)
def _setup_acl_table(self, duthost, stage, ip_ver, bind_ports):
table_name = ACL_TABLE_NAME_TEMPLATE.format(stage, ip_ver)
table_type = "L3" if ip_ver == IPV4 else "L3V6"
cmd = "config acl add table {} {} -s {} -p {}".format(
table_name, table_type, stage, ",".join(bind_ports.keys()))
logger.info("Creating ACL table {} for testing".format(table_name))
loganalyzer = LogAnalyzer(ansible_host=duthost,
marker_prefix="TestAclVlanOuter")
loganalyzer.expect_regex = [LOG_EXPECT_ACL_TABLE_CREATE_RE]
try:
with loganalyzer:
duthost.shell(cmd)
except LogAnalyzerError:
#Todo: cleanup
pytest.fail("Failed to create ACL table {}".format(table_name))
def test_memory_checker(duthosts, creds,
enum_rand_one_per_hwsku_frontend_hostname):
"""Checks whether the telemetry container can be restarted or not if the memory
usage of it is beyond the threshold. The `stress` utility is leveraged as
the memory stressing tool.
Args:
duthosts: The fixture returns list of DuTs.
enum_rand_one_per_hwsku_frontend_hostname: The fixture randomly pick up
a frontend DuT from testbed.
Returns:
None.
"""
duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname]
# TODO: Currently we only test 'telemetry' container which has the memory threshold 400MB
# and number of vm_workers is hard coded. We will extend this testing on all containers after
# the feature 'memory_checker' is fully implemented.
container_name = "telemetry"
vm_workers = 4
pytest_require(
("20191130" in duthost.os_version
and parse_version(duthost.os_version) > parse_version("20191130.72"))
or parse_version(duthost.kernel_version) > parse_version("4.9.0"),
"Test is not supported for 20191130.72 and older image versions!")
expected_alerting_messages = []
loganalyzer = LogAnalyzer(ansible_host=duthost,
marker_prefix="container_restart_due_to_memory")
loganalyzer.expect_regex = []
expected_alerting_messages.append(
".*restart_service.*Restarting service 'telemetry'.*")
expected_alerting_messages.append(".*Stopping Telemetry container.*")
expected_alerting_messages.append(".*Stopped Telemetry container.*")
loganalyzer.expect_regex.extend(expected_alerting_messages)
marker = loganalyzer.init()
install_stress_utility(duthost, creds, container_name)
consume_memory_and_restart_container(duthost, container_name, vm_workers,
loganalyzer, marker)
remove_stress_utility(duthost, container_name)
postcheck_critical_processes(duthost, container_name)
def mirroring(duthosts, enum_rand_one_per_hwsku_frontend_hostname, neighbor_ip,
mirror_setup, gre_version):
"""
fixture gathers all configuration fixtures
:param duthost: DUT host
:param mirror_setup: mirror_setup fixture
:param mirror_config: mirror_config fixture
"""
duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname]
logger.info("Adding mirror_session to DUT")
acl_rule_file = os.path.join(mirror_setup['dut_tmp_dir'],
ACL_RULE_PERSISTENT_FILE)
extra_vars = {
'acl_table_name': EVERFLOW_TABLE_NAME,
}
logger.info('Extra variables for MIRROR table:\n{}'.format(
pprint.pformat(extra_vars)))
duthost.host.options['variable_manager'].extra_vars.update(extra_vars)
duthost.template(src=os.path.join(TEMPLATE_DIR,
ACL_RULE_PERSISTENT_TEMPLATE),
dest=acl_rule_file)
duthost.command('config mirror_session add {} {} {} {} {} {} {}'.format(
SESSION_INFO['name'], SESSION_INFO['src_ip'], neighbor_ip,
SESSION_INFO['dscp'], SESSION_INFO['ttl'], SESSION_INFO['gre'],
SESSION_INFO['queue']))
logger.info('Loading acl mirror rules ...')
load_rule_cmd = "acl-loader update full {} --session_name={}".format(
acl_rule_file, SESSION_INFO['name'])
duthost.command('{}'.format(load_rule_cmd))
try:
yield
finally:
loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix='acl')
loganalyzer.load_common_config()
try:
loganalyzer.expect_regex = [LOG_EXCEPT_MIRROR_SESSION_REMOVE]
with loganalyzer:
teardown_mirroring(duthost, mirror_setup['dut_tmp_dir'])
except LogAnalyzerError as err:
raise err
def test_po_cleanup_after_reload(duthosts,
enum_rand_one_per_hwsku_frontend_hostname,
tbinfo):
"""
test port channel are cleaned up correctly after config reload, with system under stress.
"""
duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname]
host_facts = duthost.setup()['ansible_facts']
# Get the cpu information.
if host_facts.has_key("ansible_processor_vcpus"):
host_vcpus = int(host_facts['ansible_processor_vcpus'])
else:
res = duthost.shell("nproc")
host_vcpus = int(res['stdout'])
logging.info("found {} cpu on the dut".format(host_vcpus))
# Get portchannel facts and interfaces.
lag_facts = duthost.lag_facts(
host=duthost.hostname)['ansible_facts']['lag_facts']
port_channel_intfs = lag_facts['names'].keys()
# Add start marker to the DUT syslog
loganalyzer = LogAnalyzer(ansible_host=duthost,
marker_prefix='port_channel_cleanup')
loganalyzer.expect_regex = []
for pc in port_channel_intfs:
loganalyzer.expect_regex.append(LOG_EXPECT_PO_CLEANUP_RE.format(pc))
try:
# Make CPU high
for i in range(host_vcpus):
duthost.shell("nohup yes > /dev/null 2>&1 & sleep 1")
with loganalyzer:
logging.info("Reloading config..")
config_reload(duthost)
duthost.shell("killall yes")
except:
duthost.shell("killall yes")
raise
def storm_detect_path(self, dut, port, action):
"""
Storm detection action and associated verifications
Args:
dut(AnsibleHost) : DUT instance
port(string) : DUT port
action(string) : PTF test action
Returns:
loganalyzer(Loganalyzer) : instance
"""
restore_time = self.timers['pfc_wd_restore_time_large']
detect_time = self.timers['pfc_wd_detect_time']
loganalyzer = LogAnalyzer(
ansible_host=self.dut,
marker_prefix="pfc_function_storm_detect_{}_port_{}".format(
action, port))
marker = loganalyzer.init()
ignore_file = os.path.join(TEMPLATES_DIR, "ignore_pfc_wd_messages")
reg_exp = loganalyzer.parse_regexp_file(src=ignore_file)
loganalyzer.ignore_regex.extend(reg_exp)
loganalyzer.expect_regex = []
loganalyzer.expect_regex.extend([EXPECT_PFC_WD_DETECT_RE])
loganalyzer.match_regex = []
if action != "dontcare":
start_wd_on_ports(dut, port, restore_time, detect_time, action)
if not self.pfc_wd['fake_storm']:
self.storm_hndle.start_storm()
if action == "dontcare":
self.traffic_inst.fill_buffer()
start_wd_on_ports(dut, port, restore_time, detect_time, "drop")
# placing this here to cover all action types. for 'dontcare' action, wd is started much later after the pfc storm is started
if self.pfc_wd['fake_storm']:
PfcCmd.set_storm_status(dut, self.queue_oid, "enabled")
time.sleep(5)
# storm detect
logger.info("Verify if PFC storm is detected on port {}".format(port))
loganalyzer.analyze(marker)
self.stats.get_pkt_cnts(self.queue_oid, begin=True)
# test pfcwd functionality on a storm
self.traffic_inst.verify_wd_func(
action if action != "dontcare" else "drop")
return loganalyzer
def acl_table(duthosts, rand_one_dut_hostname, setup, stage, ip_version):
"""Apply ACL table configuration and remove after tests.
Args:
duthosts: All DUTs belong to the testbed.
rand_one_dut_hostname: hostname of a random chosen dut to run test.
setup: Parameters for the ACL tests.
stage: The ACL stage under test.
ip_version: The IP version under test
Yields:
The ACL table configuration.
"""
table_name = "DATA_{}_{}_TEST".format(stage.upper(), ip_version.upper())
acl_table_config = {
"table_name": table_name,
"table_ports": ",".join(setup["acl_table_ports"]['']),
"table_stage": stage,
"table_type": "L3" if ip_version == "ipv4" else "L3V6"
}
logger.info("Generated ACL table configuration:\n{}".format(
pprint.pformat(acl_table_config)))
dut_to_analyzer_map = {}
for duthost in duthosts:
loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix="acl")
loganalyzer.load_common_config()
dut_to_analyzer_map[duthost] = loganalyzer
try:
loganalyzer.expect_regex = [LOG_EXPECT_ACL_TABLE_CREATE_RE]
# Ignore any other errors to reduce noise
loganalyzer.ignore_regex = [r".*"]
with loganalyzer:
create_or_remove_acl_table(duthost, acl_table_config, setup,
"add")
except LogAnalyzerError as err:
# Cleanup Config DB if table creation failed
logger.error(
"ACL table creation failed, attempting to clean-up...")
create_or_remove_acl_table(duthost, acl_table_config, setup,
"remove")
raise err
try:
yield acl_table_config
finally:
for duthost, loganalyzer in dut_to_analyzer_map.items():
loganalyzer.expect_regex = [LOG_EXPECT_ACL_TABLE_REMOVE_RE]
with loganalyzer:
create_or_remove_acl_table(duthost, acl_table_config, setup,
"remove")
def acl_rules(self, duthosts, rand_one_dut_hostname, localhost, setup, acl_table, populate_vlan_arp_entries):
"""Setup/teardown ACL rules for the current set of tests.
Args:
duthosts: All DUTs belong to the testbed.
rand_one_dut_hostname: hostname of a random chosen dut to run test.
localhost: The host from which tests are run.
setup: Parameters for the ACL tests.
acl_table: Configuration info for the ACL table.
populate_vlan_arp_entries: A function to populate ARP/FDB tables for VLAN interfaces.
"""
duthost = duthosts[rand_one_dut_hostname]
loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix="acl_rules")
loganalyzer.load_common_config()
try:
loganalyzer.expect_regex = [LOG_EXPECT_ACL_RULE_CREATE_RE]
with loganalyzer:
self.setup_rules(duthost, acl_table)
self.post_setup_hook(duthost, localhost, populate_vlan_arp_entries)
except LogAnalyzerError as err:
# Cleanup Config DB if rule creation failed
logger.error("ACL table creation failed, attempting to clean-up...")
self.teardown_rules(duthost)
raise err
try:
yield
finally:
loganalyzer.expect_regex = [LOG_EXPECT_ACL_RULE_REMOVE_RE]
with loganalyzer:
logger.info("Removing ACL rules")
self.teardown_rules(duthost)
def test_dynamic_minimum_table(duthost, mocker_factory):
air_flow_dirs = ['p2c', 'c2p', 'unk']
max_temperature = 45000 # 45 C
cooling_cur_state = get_cooling_cur_state(duthost)
if cooling_cur_state >= COOLING_CUR_STATE_THRESHOLD:
pytest.skip('The cooling level {} is higher than threshold {}.'.format(cooling_cur_state, COOLING_CUR_STATE_THRESHOLD))
mocker = mocker_factory(duthost, 'MinTableMocker')
loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix='thermal_control')
loganalyzer.load_common_config()
for index in range(len(air_flow_dirs)):
air_flow_index = random.randint(0, len(air_flow_dirs) - 1)
air_flow_dir = air_flow_dirs[air_flow_index]
air_flow_dirs.remove(air_flow_dir)
temperature = random.randint(0, max_temperature)
trust_state = True if random.randint(0, 1) else False
logging.info('Testing with air_flow_dir={}, temperature={}, trust_state={}'.format(air_flow_dir, temperature, trust_state))
expect_minimum_cooling_level = mocker.get_expect_cooling_level(air_flow_dir, temperature, trust_state)
loganalyzer.expect_regex = [LOG_EXPECT_CHANGE_MIN_COOLING_LEVEL_RE.format(expect_minimum_cooling_level)]
with loganalyzer:
mocker.mock_min_table(air_flow_dir, temperature, trust_state)
time.sleep(THERMAL_CONTROL_TEST_WAIT_TIME)
temperature = random.randint(0, max_temperature)
logging.info('Testing with air_flow_dir={}, temperature={}, trust_state={}'.format(air_flow_dir, temperature, not trust_state))
expect_minimum_cooling_level = mocker.get_expect_cooling_level(air_flow_dir, temperature, not trust_state)
loganalyzer.expect_regex = [LOG_EXPECT_CHANGE_MIN_COOLING_LEVEL_RE.format(expect_minimum_cooling_level)]
with loganalyzer:
mocker.mock_min_table(air_flow_dir, temperature, not trust_state)
time.sleep(THERMAL_CONTROL_TEST_WAIT_TIME)
def acl_rules(self, duthost, localhost, setup, acl_table):
"""
setup/teardown ACL rules based on test class requirements
:param duthost: DUT host object
:param localhost: localhost object
:param setup: setup information
:param acl_table: table creating fixture
:return:
"""
loganalyzer = LogAnalyzer(ansible_host=duthost,
marker_prefix='acl_rules')
loganalyzer.load_common_config()
try:
loganalyzer.expect_regex = [LOG_EXPECT_ACL_RULE_CREATE_RE]
with loganalyzer:
self.setup_rules(duthost, setup, acl_table)
self.post_setup_hook(duthost, localhost)
except LogAnalyzerError as err:
# cleanup config DB in case of log analysis error
self.teardown_rules(duthost, setup)
raise err
try:
yield
finally:
loganalyzer.expect_regex = [LOG_EXPECT_ACL_RULE_REMOVE_RE]
with loganalyzer:
self.teardown_rules(duthost, setup)
def acl_table(duthost, acl_table_config):
"""
fixture to apply ACL table configuration and remove after tests
:param duthost: DUT object
:param acl_table_config: ACL table configuration dictionary
:return: forwards acl_table_config
"""
name = acl_table_config['name']
conf = acl_table_config['config_file']
loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix='acl')
loganalyzer.load_common_config()
try:
loganalyzer.expect_regex = [LOG_EXPECT_ACL_TABLE_CREATE_RE]
with loganalyzer:
logger.info('creating ACL table: applying {}'.format(conf))
# TODO: use sonic config CLI
duthost.command('sonic-cfggen -j {} --write-to-db'.format(conf))
except LogAnalyzerError as err:
# cleanup config DB if create failed
duthost.command('config acl remove table {}'.format(name))
raise err
try:
yield acl_table_config
finally:
loganalyzer.expect_regex = [LOG_EXPECT_ACL_TABLE_REMOVE_RE]
with loganalyzer:
logger.info('removing ACL table {}'.format(name))
duthost.command('config acl remove table {}'.format(name))
# save cleaned configuration
duthost.command('config save -y')
def acl(duthosts, enum_rand_one_per_hwsku_frontend_hostname, acl_setup):
"""
setup/teardown ACL rules based on test class requirements
:param duthost: DUT host object
:param acl_setup: setup information
:return:
"""
duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname]
acl_facts = duthost.acl_facts()["ansible_facts"]["ansible_acl_facts"]
pytest_require(ACL_TABLE_NAME in acl_facts, "{} acl table not exists")
loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix='acl')
loganalyzer.load_common_config()
try:
loganalyzer.expect_regex = [LOG_EXPECT_ACL_RULE_CREATE_RE]
with loganalyzer:
setup_acl_rules(duthost, acl_setup)
except LogAnalyzerError as err:
# cleanup config DB in case of log analysis error
teardown_acl(duthost, acl_setup)
raise err
try:
yield
finally:
loganalyzer.expect_regex = [LOG_EXPECT_ACL_RULE_REMOVE_RE]
with loganalyzer:
teardown_acl(duthost, acl_setup)
def acl(duthost, acl_setup):
"""
setup/teardown ACL rules based on test class requirements
:param duthost: DUT host object
:param acl_setup: setup information
:return:
"""
loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix='acl')
loganalyzer.load_common_config()
try:
loganalyzer.expect_regex = [LOG_EXPECT_ACL_RULE_CREATE_RE]
with loganalyzer:
setup_acl_rules(duthost, acl_setup)
except LogAnalyzerError as err:
# cleanup config DB in case of log analysis error
teardown_acl(duthost, acl_setup)
raise err
try:
yield
finally:
loganalyzer.expect_regex = [LOG_EXPECT_ACL_RULE_REMOVE_RE]
with loganalyzer:
teardown_acl(duthost, acl_setup)
def execute_test(self,
duthost,
syslog_marker,
ignore_regex=None,
expect_regex=None,
expect_errors=False):
"""
Helper function that loads each template on the DUT and verifies the expected behavior
Args:
duthost (AnsibleHost): instance
syslog_marker (string): marker prefix name to be inserted in the syslog
ignore_regex (string): file containing regexs to be ignored by loganalyzer
expect_regex (string): regex pattern that is expected to be present in the syslog
expect_erros (bool): if the test expects an error msg in the syslog or not. Default: False
Returns:
None
"""
loganalyzer = LogAnalyzer(ansible_host=duthost,
marker_prefix=syslog_marker)
if ignore_regex:
ignore_file = os.path.join(TEMPLATES_DIR, ignore_regex)
reg_exp = loganalyzer.parse_regexp_file(src=ignore_file)
loganalyzer.ignore_regex.extend(reg_exp)
if expect_regex:
loganalyzer.expect_regex = []
loganalyzer.expect_regex.extend(expect_regex)
loganalyzer.match_regex = []
with loganalyzer(fail=not expect_errors):
cmd = "sonic-cfggen -j {}/{}.json --write-to-db".format(
DUT_RUN_DIR, syslog_marker)
out = duthost.command(cmd)
pytest_assert(
out["rc"] == 0, "Failed to execute cmd {}: Error: {}".format(
cmd, out["stderr"]))
def disable_analyzer_for_mellanox(duthost):
if duthost.facts["asic_type"] in ["mellanox"]:
loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix='sfp_cfg')
loganalyzer.load_common_config()
loganalyzer.ignore_regex.append("kernel.*Eeprom query failed*")
marker = loganalyzer.init()
yield
if duthost.facts["asic_type"] in ["mellanox"]:
loganalyzer.analyze(marker)
def run_test_in_reinstall_loop(self):
logger.info("Verify MAC in image reinstall loop")
duthost = self.request.getfixturevalue('duthost')
localhost = self.request.getfixturevalue('localhost')
for counter in range(1, self.iteration + 1):
current_minigraph = self.minigraph1 if counter % 2 == 1 else self.minigraph2
logger.info("Iteration #{}".format(counter))
if current_minigraph:
logger.info(
"Copy specified minigraph {} to the /etc/sonic folder".
format(current_minigraph))
duthost.copy(src=current_minigraph,
dest="/etc/sonic/minigraph.xml")
loganalyzer = LogAnalyzer(ansible_host=duthost,
marker_prefix="read_mac_metadata")
loganalyzer.match_regex = [".*can't parse mac address 'None'*"]
with loganalyzer:
self.deploy_image_to_duthost(duthost, counter)
reboot(duthost, localhost, wait=120)
logger.info("Wait until system is stable")
pytest_assert(
wait_until(300, 20,
duthost.critical_services_fully_started),
"Not all critical services are fully started")
if current_minigraph:
logger.info(
"Execute cli 'config load_minigraph -y' to apply new minigraph"
)
duthost.shell("config load_minigraph -y")
duthost.shell("config save -y")
logger.info("Remove old (not current) sonic image")
duthost.reduce_and_add_sonic_images(disk_used_pcent=1)
self.check_mtu_and_interfaces(duthost)
def ignore_log_analyzer_by_vendor(request, duthosts, enum_rand_one_per_hwsku_hostname):
duthost = duthosts[enum_rand_one_per_hwsku_hostname]
asic_type = duthost.facts["asic_type"]
ignore_asic_list = request.param
if asic_type not in ignore_asic_list:
loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix=request.node.name)
loganalyzer.load_common_config()
marker = loganalyzer.init()
yield
loganalyzer.analyze(marker)
else:
yield
def ignore_particular_error_log(request, duthosts, rand_one_dut_hostname):
duthost = duthosts[rand_one_dut_hostname]
loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix='turn_on_off_psu_and_check_psustatus')
loganalyzer.load_common_config()
ignore_list = request.param
loganalyzer.ignore_regex.extend(ignore_list)
marker = loganalyzer.init()
yield
loganalyzer.analyze(marker)
def acl_table(duthosts, rand_one_dut_hostname, setup, stage, ip_version,
backup_and_restore_config_db_module):
"""Apply ACL table configuration and remove after tests.
Args:
duthosts: All DUTs belong to the testbed.
rand_one_dut_hostname: hostname of a random chosen dut to run test.
setup: Parameters for the ACL tests.
stage: The ACL stage under test.
ip_version: The IP version under test.
backup_and_restore_config_db_module: A fixture that handles restoring Config DB
after the tests are over.
Yields:
The ACL table configuration.
"""
duthost = duthosts[rand_one_dut_hostname]
table_name = "DATA_{}_{}_TEST".format(stage.upper(), ip_version.upper())
acl_table_config = {
"table_name": table_name,
"table_ports": ",".join(setup["acl_table_ports"]),
"table_stage": stage,
"table_type": "L3" if ip_version == "ipv4" else "L3V6"
}
logger.info("Generated ACL table configuration:\n{}".format(
pprint.pformat(acl_table_config)))
loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix="acl")
loganalyzer.load_common_config()
try:
loganalyzer.expect_regex = [LOG_EXPECT_ACL_TABLE_CREATE_RE]
with loganalyzer:
logger.info("Creating ACL table: \"{}\"".format(table_name))
duthost.command("config acl add table {} {} -s {} -p {}".format(
table_name, acl_table_config["table_type"],
acl_table_config["table_stage"],
acl_table_config["table_ports"]))
except LogAnalyzerError as err:
# Cleanup Config DB if table creation failed
logger.error("ACL table creation failed, attempting to clean-up...")
duthost.command("config acl remove table {}".format(table_name))
raise err
try:
yield acl_table_config
finally:
loganalyzer.expect_regex = [LOG_EXPECT_ACL_TABLE_REMOVE_RE]
with loganalyzer:
logger.info("Removing ACL table \"{}\"".format(table_name))
duthost.command("config acl remove table {}".format(table_name))
def disable_analyzer_for_mellanox(duthost):
if duthost.facts["asic_type"] in ["mellanox"]:
loganalyzer = LogAnalyzer(ansible_host=duthost,
marker_prefix='sfp_cfg')
loganalyzer.load_common_config()
loganalyzer.ignore_regex.append("kernel.*Eeprom query failed*")
# Ignore PMPE error https://github.com/Azure/sonic-buildimage/issues/7163
loganalyzer.ignore_regex.append(
r".*ERR pmon#xcvrd: Receive PMPE error event on module.*")
marker = loganalyzer.init()
yield
if duthost.facts["asic_type"] in ["mellanox"]:
loganalyzer.analyze(marker)
def test_monitoring_critical_processes(duthosts, rand_one_dut_hostname, tbinfo):
"""Tests the feature of monitoring critical processes with Supervisord.
This function will check whether names of critical processes will appear
in the syslog if the autorestart were disabled and these critical processes
were stopped.
Args:
duthosts: list of DUTs.
rand_one_dut_hostname: hostname of DUT.
tbinfo: Testbed information.
Returns:
None.
"""
duthost = duthosts[rand_one_dut_hostname]
loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix="monitoring_critical_processes")
loganalyzer.expect_regex = []
bgp_neighbors = duthost.get_bgp_neighbors()
up_bgp_neighbors = [ k.lower() for k, v in bgp_neighbors.items() if v["state"] == "established" ]
skip_containers = []
skip_containers.append("database")
skip_containers.append("gbsyncd")
# Skip 'radv' container on devices whose role is not T0.
if tbinfo["topo"]["type"] != "t0":
skip_containers.append("radv")
containers_in_namespaces = get_containers_namespace_ids(duthost, skip_containers)
expected_alerting_messages = get_expected_alerting_messages(duthost, containers_in_namespaces)
loganalyzer.expect_regex.extend(expected_alerting_messages)
marker = loganalyzer.init()
stop_critical_processes(duthost, containers_in_namespaces)
# Wait for 70 seconds such that Supervisord has a chance to write alerting message into syslog.
logger.info("Sleep 70 seconds to wait for the alerting message...")
time.sleep(70)
logger.info("Checking the alerting messages from syslog...")
loganalyzer.analyze(marker)
logger.info("Found all the expected alerting messages from syslog!")
logger.info("Executing the config reload...")
config_reload(duthost)
logger.info("Executing the config reload was done!")
ensure_all_critical_processes_running(duthost, containers_in_namespaces)
if not postcheck_critical_processes_status(duthost, up_bgp_neighbors):
pytest.fail("Post-check failed after testing the container checker!")
logger.info("Post-checking status of critical processes and BGP sessions was done!")
def acl_rules(self, duthosts, localhost, setup, acl_table,
populate_vlan_arp_entries, tbinfo, ip_version):
"""Setup/teardown ACL rules for the current set of tests.
Args:
duthosts: All DUTs belong to the testbed.
rand_one_dut_hostname: hostname of a random chosen dut to run test.
localhost: The host from which tests are run.
setup: Parameters for the ACL tests.
acl_table: Configuration info for the ACL table.
populate_vlan_arp_entries: A function to populate ARP/FDB tables for VLAN interfaces.
"""
dut_to_analyzer_map = {}
for duthost in duthosts:
loganalyzer = LogAnalyzer(ansible_host=duthost,
marker_prefix="acl_rules")
loganalyzer.load_common_config()
dut_to_analyzer_map[duthost] = loganalyzer
try:
loganalyzer.expect_regex = [LOG_EXPECT_ACL_RULE_CREATE_RE]
# Ignore any other errors to reduce noise
loganalyzer.ignore_regex = [r".*"]
with loganalyzer:
self.setup_rules(duthost, acl_table, ip_version)
self.post_setup_hook(duthost, localhost,
populate_vlan_arp_entries, tbinfo)
assert self.check_rule_counters(
duthost), "Rule counters should be ready!"
except LogAnalyzerError as err:
# Cleanup Config DB if rule creation failed
logger.error(
"ACL rule application failed, attempting to clean-up...")
self.teardown_rules(duthost)
raise err
try:
yield
finally:
for duthost, loganalyzer in dut_to_analyzer_map.items():
loganalyzer.expect_regex = [LOG_EXPECT_ACL_RULE_REMOVE_RE]
with loganalyzer:
logger.info("Removing ACL rules")
self.teardown_rules(duthost)
def verify_thresholds(duthost, **kwargs):
"""
Verifies that WARNING message logged if there are any resources that exceeds a pre-defined threshold value.
Verifies the following threshold parameters: percentage, actual used, actual free
"""
loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix='crm_test')
for key, value in THR_VERIFY_CMDS.items():
logger.info("Verifying CRM threshold '{}'".format(key))
template = Template(value)
if "exceeded" in key:
loganalyzer.expect_regex = [EXPECT_EXCEEDED]
elif "clear" in key:
loganalyzer.expect_regex = [EXPECT_CLEAR]
if "percentage" in key:
if "nexthop_group" in kwargs[
"crm_cli_res"] and "mellanox" in duthost.facts[
"asic_type"].lower():
# TODO: Fix this. Temporal skip percentage verification for 'test_crm_nexthop_group' test case
# Max supported ECMP group values is less then number of entries we need to configure
# in order to test percentage threshold (Can't even reach 1 percent)
# For test case used 'nexthop_group' need to be configured at least 1 percent from available
continue
used_percent = get_used_percent(kwargs["crm_used"],
kwargs["crm_avail"])
if key == "exceeded_percentage":
if used_percent < 1:
logger.warning("The used percentage for {} is {} and verification for exceeded_percentage is skipped" \
.format(kwargs["crm_cli_res"], used_percent))
continue
kwargs["th_lo"] = used_percent - 1
kwargs["th_hi"] = used_percent
loganalyzer.expect_regex = [EXPECT_EXCEEDED]
elif key == "clear_percentage":
if used_percent >= 100:
logger.warning("The used percentage for {} is {} and verification for clear_percentage is skipped" \
.format(kwargs["crm_cli_res"], used_percent))
continue
kwargs["th_lo"] = used_percent
kwargs["th_hi"] = used_percent + 1
loganalyzer.expect_regex = [EXPECT_CLEAR]
cmd = template.render(**kwargs)
with loganalyzer:
duthost.command(cmd)
# Make sure CRM counters updated
time.sleep(CRM_UPDATE_TIME)
