def acl(duthosts, enum_rand_one_per_hwsku_frontend_hostname, acl_setup):
"""
setup/teardown ACL rules based on test class requirements
:param duthost: DUT host object
:param acl_setup: setup information
:return:
"""
duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname]
acl_facts = duthost.acl_facts()["ansible_facts"]["ansible_acl_facts"]
pytest_require(ACL_TABLE_NAME in acl_facts, "{} acl table not exists")
loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix='acl')
loganalyzer.load_common_config()
try:
loganalyzer.expect_regex = [LOG_EXPECT_ACL_RULE_CREATE_RE]
with loganalyzer:
setup_acl_rules(duthost, acl_setup)
except LogAnalyzerError as err:
# cleanup config DB in case of log analysis error
teardown_acl(duthost, acl_setup)
raise err
try:
yield
finally:
loganalyzer.expect_regex = [LOG_EXPECT_ACL_RULE_REMOVE_RE]
with loganalyzer:
teardown_acl(duthost, acl_setup)
def acl(duthost, acl_setup):
"""
setup/teardown ACL rules based on test class requirements
:param duthost: DUT host object
:param acl_setup: setup information
:return:
"""
loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix='acl')
loganalyzer.load_common_config()
try:
loganalyzer.expect_regex = [LOG_EXPECT_ACL_RULE_CREATE_RE]
with loganalyzer:
setup_acl_rules(duthost, acl_setup)
except LogAnalyzerError as err:
# cleanup config DB in case of log analysis error
teardown_acl(duthost, acl_setup)
raise err
try:
yield
finally:
loganalyzer.expect_regex = [LOG_EXPECT_ACL_RULE_REMOVE_RE]
with loganalyzer:
teardown_acl(duthost, acl_setup)
def acl_rules(self, duthosts, rand_one_dut_hostname, localhost, setup, acl_table, populate_vlan_arp_entries):
"""Setup/teardown ACL rules for the current set of tests.
Args:
duthosts: All DUTs belong to the testbed.
rand_one_dut_hostname: hostname of a random chosen dut to run test.
localhost: The host from which tests are run.
setup: Parameters for the ACL tests.
acl_table: Configuration info for the ACL table.
populate_vlan_arp_entries: A function to populate ARP/FDB tables for VLAN interfaces.
"""
duthost = duthosts[rand_one_dut_hostname]
loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix="acl_rules")
loganalyzer.load_common_config()
try:
loganalyzer.expect_regex = [LOG_EXPECT_ACL_RULE_CREATE_RE]
with loganalyzer:
self.setup_rules(duthost, acl_table)
self.post_setup_hook(duthost, localhost, populate_vlan_arp_entries)
except LogAnalyzerError as err:
# Cleanup Config DB if rule creation failed
logger.error("ACL table creation failed, attempting to clean-up...")
self.teardown_rules(duthost)
raise err
try:
yield
finally:
loganalyzer.expect_regex = [LOG_EXPECT_ACL_RULE_REMOVE_RE]
with loganalyzer:
logger.info("Removing ACL rules")
self.teardown_rules(duthost)
def acl_table(duthost, acl_table_config):
"""
fixture to apply ACL table configuration and remove after tests
:param duthost: DUT object
:param acl_table_config: ACL table configuration dictionary
:return: forwards acl_table_config
"""
name = acl_table_config['name']
conf = acl_table_config['config_file']
loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix='acl')
loganalyzer.load_common_config()
try:
loganalyzer.expect_regex = [LOG_EXPECT_ACL_TABLE_CREATE_RE]
with loganalyzer:
logger.info('creating ACL table: applying {}'.format(conf))
# TODO: use sonic config CLI
duthost.command('sonic-cfggen -j {} --write-to-db'.format(conf))
except LogAnalyzerError as err:
# cleanup config DB if create failed
duthost.command('config acl remove table {}'.format(name))
raise err
try:
yield acl_table_config
finally:
loganalyzer.expect_regex = [LOG_EXPECT_ACL_TABLE_REMOVE_RE]
with loganalyzer:
logger.info('removing ACL table {}'.format(name))
duthost.command('config acl remove table {}'.format(name))
# save cleaned configuration
duthost.command('config save -y')
def test_dynamic_minimum_table(duthost, mocker_factory):
air_flow_dirs = ['p2c', 'c2p', 'unk']
max_temperature = 45000 # 45 C
cooling_cur_state = get_cooling_cur_state(duthost)
if cooling_cur_state >= COOLING_CUR_STATE_THRESHOLD:
pytest.skip('The cooling level {} is higher than threshold {}.'.format(cooling_cur_state, COOLING_CUR_STATE_THRESHOLD))
mocker = mocker_factory(duthost, 'MinTableMocker')
loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix='thermal_control')
loganalyzer.load_common_config()
for index in range(len(air_flow_dirs)):
air_flow_index = random.randint(0, len(air_flow_dirs) - 1)
air_flow_dir = air_flow_dirs[air_flow_index]
air_flow_dirs.remove(air_flow_dir)
temperature = random.randint(0, max_temperature)
trust_state = True if random.randint(0, 1) else False
logging.info('Testing with air_flow_dir={}, temperature={}, trust_state={}'.format(air_flow_dir, temperature, trust_state))
expect_minimum_cooling_level = mocker.get_expect_cooling_level(air_flow_dir, temperature, trust_state)
loganalyzer.expect_regex = [LOG_EXPECT_CHANGE_MIN_COOLING_LEVEL_RE.format(expect_minimum_cooling_level)]
with loganalyzer:
mocker.mock_min_table(air_flow_dir, temperature, trust_state)
time.sleep(THERMAL_CONTROL_TEST_WAIT_TIME)
temperature = random.randint(0, max_temperature)
logging.info('Testing with air_flow_dir={}, temperature={}, trust_state={}'.format(air_flow_dir, temperature, not trust_state))
expect_minimum_cooling_level = mocker.get_expect_cooling_level(air_flow_dir, temperature, not trust_state)
loganalyzer.expect_regex = [LOG_EXPECT_CHANGE_MIN_COOLING_LEVEL_RE.format(expect_minimum_cooling_level)]
with loganalyzer:
mocker.mock_min_table(air_flow_dir, temperature, not trust_state)
time.sleep(THERMAL_CONTROL_TEST_WAIT_TIME)
def acl_rules(self, duthost, localhost, setup, acl_table):
"""
setup/teardown ACL rules based on test class requirements
:param duthost: DUT host object
:param localhost: localhost object
:param setup: setup information
:param acl_table: table creating fixture
:return:
"""
loganalyzer = LogAnalyzer(ansible_host=duthost,
marker_prefix='acl_rules')
loganalyzer.load_common_config()
try:
loganalyzer.expect_regex = [LOG_EXPECT_ACL_RULE_CREATE_RE]
with loganalyzer:
self.setup_rules(duthost, setup, acl_table)
self.post_setup_hook(duthost, localhost)
except LogAnalyzerError as err:
# cleanup config DB in case of log analysis error
self.teardown_rules(duthost, setup)
raise err
try:
yield
finally:
loganalyzer.expect_regex = [LOG_EXPECT_ACL_RULE_REMOVE_RE]
with loganalyzer:
self.teardown_rules(duthost, setup)
def acl_table(duthosts, rand_one_dut_hostname, setup, stage, ip_version,
backup_and_restore_config_db_module):
"""Apply ACL table configuration and remove after tests.
Args:
duthosts: All DUTs belong to the testbed.
rand_one_dut_hostname: hostname of a random chosen dut to run test.
setup: Parameters for the ACL tests.
stage: The ACL stage under test.
ip_version: The IP version under test.
backup_and_restore_config_db_module: A fixture that handles restoring Config DB
after the tests are over.
Yields:
The ACL table configuration.
"""
table_name = "DATA_{}_{}_TEST".format(stage.upper(), ip_version.upper())
acl_table_config = {
"table_name": table_name,
"table_ports": ",".join(setup["acl_table_ports"]['']),
"table_stage": stage,
"table_type": "L3" if ip_version == "ipv4" else "L3V6"
}
logger.info("Generated ACL table configuration:\n{}".format(
pprint.pformat(acl_table_config)))
dut_to_analyzer_map = {}
for duthost in duthosts:
loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix="acl")
loganalyzer.load_common_config()
dut_to_analyzer_map[duthost] = loganalyzer
try:
loganalyzer.expect_regex = [LOG_EXPECT_ACL_TABLE_CREATE_RE]
with loganalyzer:
create_or_remove_acl_table(duthost, acl_table_config, setup,
"add")
except LogAnalyzerError as err:
# Cleanup Config DB if table creation failed
logger.error(
"ACL table creation failed, attempting to clean-up...")
create_or_remove_acl_table(duthost, acl_table_config, setup,
"remove")
raise err
try:
yield acl_table_config
finally:
for duthost, loganalyzer in dut_to_analyzer_map.items():
loganalyzer.expect_regex = [LOG_EXPECT_ACL_TABLE_REMOVE_RE]
with loganalyzer:
create_or_remove_acl_table(duthost, acl_table_config, setup,
"remove")
def acl_rules(self, duthosts, localhost, setup, acl_table,
populate_vlan_arp_entries, tbinfo, ip_version):
"""Setup/teardown ACL rules for the current set of tests.
Args:
duthosts: All DUTs belong to the testbed.
rand_one_dut_hostname: hostname of a random chosen dut to run test.
localhost: The host from which tests are run.
setup: Parameters for the ACL tests.
acl_table: Configuration info for the ACL table.
populate_vlan_arp_entries: A function to populate ARP/FDB tables for VLAN interfaces.
"""
dut_to_analyzer_map = {}
for duthost in duthosts:
loganalyzer = LogAnalyzer(ansible_host=duthost,
marker_prefix="acl_rules")
loganalyzer.load_common_config()
dut_to_analyzer_map[duthost] = loganalyzer
try:
loganalyzer.expect_regex = [LOG_EXPECT_ACL_RULE_CREATE_RE]
# Ignore any other errors to reduce noise
loganalyzer.ignore_regex = [r".*"]
with loganalyzer:
self.setup_rules(duthost, acl_table, ip_version)
self.post_setup_hook(duthost, localhost,
populate_vlan_arp_entries, tbinfo)
assert self.check_rule_counters(
duthost), "Rule counters should be ready!"
except LogAnalyzerError as err:
# Cleanup Config DB if rule creation failed
logger.error(
"ACL rule application failed, attempting to clean-up...")
self.teardown_rules(duthost)
raise err
try:
yield
finally:
for duthost, loganalyzer in dut_to_analyzer_map.items():
loganalyzer.expect_regex = [LOG_EXPECT_ACL_RULE_REMOVE_RE]
with loganalyzer:
logger.info("Removing ACL rules")
self.teardown_rules(duthost)
def run_test(self, duthost, storm_hndle, expect_regex, syslog_marker,
action):
"""
Storm generation/restoration on all ports and verification
Args:
duthost (AnsibleHost): DUT instance
storm_hndle (PFCMultiStorm): class PFCMultiStorm intance
expect_regex (list): list of expect regexs to be matched in the syslog
syslog_marker (string): marker prefix written to the syslog
action (string): storm/restore action
"""
loganalyzer = LogAnalyzer(ansible_host=duthost,
marker_prefix=syslog_marker)
ignore_file = os.path.join(TEMPLATES_DIR, "ignore_pfc_wd_messages")
reg_exp = loganalyzer.parse_regexp_file(src=ignore_file)
loganalyzer.ignore_regex.extend(reg_exp)
loganalyzer.expect_regex = []
loganalyzer.expect_regex.extend(expect_regex)
loganalyzer.match_regex = []
with loganalyzer:
if action == "storm":
storm_hndle.start_pfc_storm()
elif action == "restore":
storm_hndle.stop_pfc_storm()
time.sleep(5)
def verify_thresholds(duthost, **kwargs):
"""
Verifies that WARNING message logged if there are any resources that exceeds a pre-defined threshold value.
Verifies the following threshold parameters: percentage, actual used, actual free
"""
loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix='crm_test')
for key, value in THR_VERIFY_CMDS.items():
logger.info("Verifying CRM threshold '{}'".format(key))
template = Template(value)
if "exceeded" in key:
loganalyzer.expect_regex = [EXPECT_EXCEEDED]
elif "clear" in key:
loganalyzer.expect_regex = [EXPECT_CLEAR]
if "percentage" in key:
if "nexthop_group" in kwargs[
"crm_cli_res"] and "mellanox" in duthost.facts[
"asic_type"].lower():
# TODO: Fix this. Temporal skip percentage verification for 'test_crm_nexthop_group' test case
# Max supported ECMP group values is less then number of entries we need to configure
# in order to test percentage threshold (Can't even reach 1 percent)
# For test case used 'nexthop_group' need to be configured at least 1 percent from available
continue
used_percent = get_used_percent(kwargs["crm_used"],
kwargs["crm_avail"])
if key == "exceeded_percentage":
if used_percent < 1:
logger.warning("The used percentage for {} is {} and verification for exceeded_percentage is skipped" \
.format(kwargs["crm_cli_res"], used_percent))
continue
kwargs["th_lo"] = used_percent - 1
kwargs["th_hi"] = used_percent
loganalyzer.expect_regex = [EXPECT_EXCEEDED]
elif key == "clear_percentage":
if used_percent >= 100:
logger.warning("The used percentage for {} is {} and verification for clear_percentage is skipped" \
.format(kwargs["crm_cli_res"], used_percent))
continue
kwargs["th_lo"] = used_percent
kwargs["th_hi"] = used_percent + 1
loganalyzer.expect_regex = [EXPECT_CLEAR]
cmd = template.render(**kwargs)
with loganalyzer:
duthost.command(cmd)
# Make sure CRM counters updated
time.sleep(CRM_UPDATE_TIME)
def acl_table(duthosts, rand_one_dut_hostname, acl_table_config,
backup_and_restore_config_db_module):
"""Apply ACL table configuration and remove after tests.
Args:
duthosts: All DUTs belong to the testbed.
rand_one_dut_hostname: hostname of a random chosen dut to run test.
acl_table_config: A dictionary describing the ACL table configuration to apply.
backup_and_restore_config_db_module: A fixture that handles restoring Config DB
after the tests are over.
Yields:
The ACL table configuration.
"""
duthost = duthosts[rand_one_dut_hostname]
table_name = acl_table_config["table_name"]
config_file = acl_table_config["config_file"]
loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix="acl")
loganalyzer.load_common_config()
try:
loganalyzer.expect_regex = [LOG_EXPECT_ACL_TABLE_CREATE_RE]
with loganalyzer:
logger.info("Creating ACL table from config file: \"{}\"".format(
config_file))
# TODO: Use `config` CLI to create ACL table
duthost.command(
"sonic-cfggen -j {} --write-to-db".format(config_file))
except LogAnalyzerError as err:
# Cleanup Config DB if table creation failed
logger.error("ACL table creation failed, attempting to clean-up...")
duthost.command("config acl remove table {}".format(table_name))
raise err
try:
yield acl_table_config
finally:
loganalyzer.expect_regex = [LOG_EXPECT_ACL_TABLE_REMOVE_RE]
with loganalyzer:
logger.info("Removing ACL table \"{}\"".format(table_name))
duthost.command("config acl remove table {}".format(table_name))
def check_thermal_control_load_invalid_file(duthost, file_name):
"""
@summary: Load an invalid thermal policy file check thermal
control daemon is up and there is an error log printed
"""
loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix='thermal_control')
loganalyzer.expect_regex = [LOG_EXPECT_POLICY_FILE_INVALID]
with loganalyzer:
with ThermalPolicyFileContext(duthost, file_name):
restart_thermal_control_daemon(duthost)
def test_container_checker(duthosts, enum_rand_one_per_hwsku_hostname,
enum_rand_one_asic_index, enum_dut_feature, tbinfo):
"""Tests the feature of container checker.
This function will check whether the container names will appear in the Monit
alerting message if they are stopped explicitly or they hit start limitation.
Args:
duthosts: list of DUTs.
enum_rand_one_per_hwsku_hostname: Fixture returning list of hostname selected per hwsku.
enum_rand_one_asic_index: Fixture returning list of asics for selected duts.
enum_dut_feature: A list contains features.
tbinfo: Testbed information.
Returns:
None.
"""
service_name = enum_dut_feature
duthost = duthosts[enum_rand_one_per_hwsku_hostname]
asic = duthost.asic_instance(enum_rand_one_asic_index)
container_name = asic.get_docker_name(service_name)
loganalyzer = LogAnalyzer(
ansible_host=duthost,
marker_prefix="container_checker_{}".format(container_name))
disabled_containers = get_disabled_container_list(duthost)
skip_containers = disabled_containers[:]
# Skip 'radv' container on devices whose role is not T0.
if tbinfo["topo"]["type"] != "t0":
skip_containers.append("radv")
pytest_require(
service_name not in skip_containers,
"Container '{}' is skipped for testing.".format(container_name))
asic.stop_service(service_name)
logger.info(
"Waiting until container '{}' is stopped...".format(container_name))
stopped = wait_until(CONTAINER_STOP_THRESHOLD_SECS,
CONTAINER_CHECK_INTERVAL_SECS, 0,
check_container_state, duthost, container_name, False)
pytest_assert(stopped,
"Failed to stop container '{}'".format(container_name))
logger.info("Container '{}' on DuT '{}' was stopped".format(
container_name, duthost.hostname))
loganalyzer.expect_regex = get_expected_alerting_message(container_name)
with loganalyzer:
# Wait for 1 minutes such that Monit has a chance to write alerting message into syslog.
logger.info("Sleep 1 minutes to wait for the alerting message...")
time.sleep(70)
def test_monitoring_critical_processes(duthosts, rand_one_dut_hostname, tbinfo):
"""Tests the feature of monitoring critical processes with Supervisord.
This function will check whether names of critical processes will appear
in the syslog if the autorestart were disabled and these critical processes
were stopped.
Args:
duthosts: list of DUTs.
rand_one_dut_hostname: hostname of DUT.
tbinfo: Testbed information.
Returns:
None.
"""
duthost = duthosts[rand_one_dut_hostname]
loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix="monitoring_critical_processes")
loganalyzer.expect_regex = []
bgp_neighbors = duthost.get_bgp_neighbors()
up_bgp_neighbors = [ k.lower() for k, v in bgp_neighbors.items() if v["state"] == "established" ]
skip_containers = []
skip_containers.append("database")
skip_containers.append("gbsyncd")
# Skip 'radv' container on devices whose role is not T0.
if tbinfo["topo"]["type"] != "t0":
skip_containers.append("radv")
containers_in_namespaces = get_containers_namespace_ids(duthost, skip_containers)
expected_alerting_messages = get_expected_alerting_messages(duthost, containers_in_namespaces)
loganalyzer.expect_regex.extend(expected_alerting_messages)
marker = loganalyzer.init()
stop_critical_processes(duthost, containers_in_namespaces)
# Wait for 70 seconds such that Supervisord has a chance to write alerting message into syslog.
logger.info("Sleep 70 seconds to wait for the alerting message...")
time.sleep(70)
logger.info("Checking the alerting messages from syslog...")
loganalyzer.analyze(marker)
logger.info("Found all the expected alerting messages from syslog!")
logger.info("Executing the config reload...")
config_reload(duthost)
logger.info("Executing the config reload was done!")
ensure_all_critical_processes_running(duthost, containers_in_namespaces)
if not postcheck_critical_processes_status(duthost, up_bgp_neighbors):
pytest.fail("Post-check failed after testing the container checker!")
logger.info("Post-checking status of critical processes and BGP sessions was done!")
def storm_detect_path(self, dut, port, action):
"""
Storm detection action and associated verifications
Args:
dut(AnsibleHost) : DUT instance
port(string) : DUT port
action(string) : PTF test action
Returns:
loganalyzer(Loganalyzer) : instance
"""
restore_time = self.timers['pfc_wd_restore_time_large']
detect_time = self.timers['pfc_wd_detect_time']
loganalyzer = LogAnalyzer(
ansible_host=self.dut,
marker_prefix="pfc_function_storm_detect_{}_port_{}".format(
action, port))
marker = loganalyzer.init()
ignore_file = os.path.join(TEMPLATES_DIR, "ignore_pfc_wd_messages")
reg_exp = loganalyzer.parse_regexp_file(src=ignore_file)
loganalyzer.ignore_regex.extend(reg_exp)
loganalyzer.expect_regex = []
loganalyzer.expect_regex.extend([EXPECT_PFC_WD_DETECT_RE])
loganalyzer.match_regex = []
if action != "dontcare":
start_wd_on_ports(dut, port, restore_time, detect_time, action)
if not self.pfc_wd['fake_storm']:
self.storm_hndle.start_storm()
if action == "dontcare":
self.traffic_inst.fill_buffer()
start_wd_on_ports(dut, port, restore_time, detect_time, "drop")
# placing this here to cover all action types. for 'dontcare' action, wd is started much later after the pfc storm is started
if self.pfc_wd['fake_storm']:
PfcCmd.set_storm_status(dut, self.queue_oid, "enabled")
time.sleep(5)
# storm detect
logger.info("Verify if PFC storm is detected on port {}".format(port))
loganalyzer.analyze(marker)
self.stats.get_pkt_cnts(self.queue_oid, begin=True)
# test pfcwd functionality on a storm
self.traffic_inst.verify_wd_func(
action if action != "dontcare" else "drop")
return loganalyzer
def test_container_checker(duthosts, enum_dut_feature_container,
rand_selected_dut, tbinfo):
"""Tests the feature of container checker.
This function will check whether the container names will appear in the Monit
alerting message if they are stopped explicitly or they hit start limitation.
Args:
duthosts: list of DUTs.
enum_dut_feature_container: A list contains strings ("<dut_name>|<container_name>").
rand_selected_dut: The fixture returns a randomly selected DuT.
tbinfo: Testbed information.
Returns:
None.
"""
dut_name, container_name = decode_dut_and_container_name(
enum_dut_feature_container)
pytest_require(
dut_name == rand_selected_dut.hostname and container_name != "unknown",
"Skips testing container_checker of container '{}' on the DuT '{}' since another DuT '{}' was chosen."
.format(container_name, dut_name, rand_selected_dut.hostname))
duthost = duthosts[dut_name]
loganalyzer = LogAnalyzer(
ansible_host=duthost,
marker_prefix="container_checker_{}".format(container_name))
disabled_containers = get_disabled_container_list(duthost)
skip_containers = disabled_containers[:]
skip_containers.append("gbsyncd")
skip_containers.append("database")
skip_containers.append("database-chassis")
# Skip 'radv' container on devices whose role is not T0.
if tbinfo["topo"]["type"] != "t0":
skip_containers.append("radv")
pytest_require(
container_name not in skip_containers,
"Container '{}' is skipped for testing.".format(container_name))
stop_container(duthost, container_name)
loganalyzer.expect_regex = get_expected_alerting_message(container_name)
with loganalyzer:
# Wait for 1 minutes such that Monit has a chance to write alerting message into syslog.
logger.info("Sleep 1 minutes to wait for the alerting message...")
time.sleep(70)
def _remove_acl_table(self, duthost, stage, ip_ver):
table_name = ACL_TABLE_NAME_TEMPLATE.format(stage, ip_ver)
cmd = "config acl remove table {}".format(table_name)
logger.info("Removing ACL table {}".format(table_name))
loganalyzer = LogAnalyzer(ansible_host=duthost,
marker_prefix="TestAclVlanOuter")
loganalyzer.expect_regex = [LOG_EXPECT_ACL_TABLE_REMOVE_RE]
try:
with loganalyzer:
duthost.shell(cmd)
except LogAnalyzerError:
#Todo: cleanup
pytest.fail("Failed to remove ACL table {}".format(table_name))
def _setup_acl_table(self, duthost, stage, ip_ver, bind_ports):
table_name = ACL_TABLE_NAME_TEMPLATE.format(stage, ip_ver)
table_type = "L3" if ip_ver == IPV4 else "L3V6"
cmd = "config acl add table {} {} -s {} -p {}".format(
table_name, table_type, stage, ",".join(bind_ports.keys()))
logger.info("Creating ACL table {} for testing".format(table_name))
loganalyzer = LogAnalyzer(ansible_host=duthost,
marker_prefix="TestAclVlanOuter")
loganalyzer.expect_regex = [LOG_EXPECT_ACL_TABLE_CREATE_RE]
try:
with loganalyzer:
duthost.shell(cmd)
except LogAnalyzerError:
#Todo: cleanup
pytest.fail("Failed to create ACL table {}".format(table_name))
def test_container_checker(duthosts, rand_one_dut_hostname, tbinfo):
"""Tests the feature of container checker.
This function will check whether the container names will appear in the Monit
alerting message if they are stopped explicitly or they hit start limitation.
Args:
duthosts: list of DUTs.
rand_one_dut_hostname: hostname of DUT.
tbinfo: Testbed information.
Returns:
None.
"""
duthost = duthosts[rand_one_dut_hostname]
loganalyzer = LogAnalyzer(ansible_host=duthost,
marker_prefix="container_checker")
loganalyzer.expect_regex = []
container_autorestart_states = duthost.get_container_autorestart_states()
disabled_containers = get_disabled_container_list(duthost)
skip_containers = disabled_containers[:]
skip_containers.append("gbsyncd")
# Skip 'radv' container on devices whose role is not T0.
if tbinfo["topo"]["type"] != "t0":
skip_containers.append("radv")
stopped_container_list = stop_containers(duthost,
container_autorestart_states,
skip_containers)
pytest_assert(
len(stopped_container_list) > 0, "None of containers was stopped!")
expected_alerting_messages = get_expected_alerting_messages(
stopped_container_list)
loganalyzer.expect_regex.extend(expected_alerting_messages)
marker = loganalyzer.init()
# Wait for 2 minutes such that Monit has a chance to write alerting message into syslog.
logger.info("Sleep 2 minutes to wait for the alerting message...")
time.sleep(130)
logger.info("Checking the alerting messages from syslog...")
loganalyzer.analyze(marker)
logger.info("Found all the expected alerting messages from syslog!")
def check_syslog(duthost, prefix, trigger_action, expected_log,
restore_action):
loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix=prefix)
loganalyzer.expect_regex = [expected_log]
try:
marker = loganalyzer.init()
duthost.command(trigger_action)
logger.info("Check for expected log {} in syslog".format(expected_log))
loganalyzer.analyze(marker)
except LogAnalyzerError as err:
logger.error("Unable to find expected log in syslog")
raise err
finally:
duthost.command(restore_action)
def test_memory_checker(duthosts, creds,
enum_rand_one_per_hwsku_frontend_hostname):
"""Checks whether the telemetry container can be restarted or not if the memory
usage of it is beyond the threshold. The `stress` utility is leveraged as
the memory stressing tool.
Args:
duthosts: The fixture returns list of DuTs.
enum_rand_one_per_hwsku_frontend_hostname: The fixture randomly pick up
a frontend DuT from testbed.
Returns:
None.
"""
duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname]
# TODO: Currently we only test 'telemetry' container which has the memory threshold 400MB
# and number of vm_workers is hard coded. We will extend this testing on all containers after
# the feature 'memory_checker' is fully implemented.
container_name = "telemetry"
vm_workers = 4
pytest_require(
("20191130" in duthost.os_version
and parse_version(duthost.os_version) > parse_version("20191130.72"))
or parse_version(duthost.kernel_version) > parse_version("4.9.0"),
"Test is not supported for 20191130.72 and older image versions!")
expected_alerting_messages = []
loganalyzer = LogAnalyzer(ansible_host=duthost,
marker_prefix="container_restart_due_to_memory")
loganalyzer.expect_regex = []
expected_alerting_messages.append(
".*restart_service.*Restarting service 'telemetry'.*")
expected_alerting_messages.append(".*Stopping Telemetry container.*")
expected_alerting_messages.append(".*Stopped Telemetry container.*")
loganalyzer.expect_regex.extend(expected_alerting_messages)
marker = loganalyzer.init()
install_stress_utility(duthost, creds, container_name)
consume_memory_and_restart_container(duthost, container_name, vm_workers,
loganalyzer, marker)
remove_stress_utility(duthost, container_name)
postcheck_critical_processes(duthost, container_name)
def mirroring(duthosts, enum_rand_one_per_hwsku_frontend_hostname, neighbor_ip,
mirror_setup, gre_version):
"""
fixture gathers all configuration fixtures
:param duthost: DUT host
:param mirror_setup: mirror_setup fixture
:param mirror_config: mirror_config fixture
"""
duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname]
logger.info("Adding mirror_session to DUT")
acl_rule_file = os.path.join(mirror_setup['dut_tmp_dir'],
ACL_RULE_PERSISTENT_FILE)
extra_vars = {
'acl_table_name': EVERFLOW_TABLE_NAME,
}
logger.info('Extra variables for MIRROR table:\n{}'.format(
pprint.pformat(extra_vars)))
duthost.host.options['variable_manager'].extra_vars.update(extra_vars)
duthost.template(src=os.path.join(TEMPLATE_DIR,
ACL_RULE_PERSISTENT_TEMPLATE),
dest=acl_rule_file)
duthost.command('config mirror_session add {} {} {} {} {} {} {}'.format(
SESSION_INFO['name'], SESSION_INFO['src_ip'], neighbor_ip,
SESSION_INFO['dscp'], SESSION_INFO['ttl'], SESSION_INFO['gre'],
SESSION_INFO['queue']))
logger.info('Loading acl mirror rules ...')
load_rule_cmd = "acl-loader update full {} --session_name={}".format(
acl_rule_file, SESSION_INFO['name'])
duthost.command('{}'.format(load_rule_cmd))
try:
yield
finally:
loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix='acl')
loganalyzer.load_common_config()
try:
loganalyzer.expect_regex = [LOG_EXCEPT_MIRROR_SESSION_REMOVE]
with loganalyzer:
teardown_mirroring(duthost, mirror_setup['dut_tmp_dir'])
except LogAnalyzerError as err:
raise err
def test_po_cleanup_after_reload(duthosts,
enum_rand_one_per_hwsku_frontend_hostname,
tbinfo):
"""
test port channel are cleaned up correctly after config reload, with system under stress.
"""
duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname]
host_facts = duthost.setup()['ansible_facts']
# Get the cpu information.
if host_facts.has_key("ansible_processor_vcpus"):
host_vcpus = int(host_facts['ansible_processor_vcpus'])
else:
res = duthost.shell("nproc")
host_vcpus = int(res['stdout'])
logging.info("found {} cpu on the dut".format(host_vcpus))
# Get portchannel facts and interfaces.
lag_facts = duthost.lag_facts(
host=duthost.hostname)['ansible_facts']['lag_facts']
port_channel_intfs = lag_facts['names'].keys()
# Add start marker to the DUT syslog
loganalyzer = LogAnalyzer(ansible_host=duthost,
marker_prefix='port_channel_cleanup')
loganalyzer.expect_regex = []
for pc in port_channel_intfs:
loganalyzer.expect_regex.append(LOG_EXPECT_PO_CLEANUP_RE.format(pc))
try:
# Make CPU high
for i in range(host_vcpus):
duthost.shell("nohup yes > /dev/null 2>&1 & sleep 1")
with loganalyzer:
logging.info("Reloading config..")
config_reload(duthost)
duthost.shell("killall yes")
except:
duthost.shell("killall yes")
raise
def consumes_memory_and_checks_container_restart(duthost, container_name, vm_workers):
"""Invokes the 'stress' utility to consume memory more than the threshold asynchronously
and checks whether the container can be stopped and restarted. Loganalyzer is leveraged
to check whether the log messages related to container stopped were generated.
Args:
duthost: The AnsibleHost object of DuT.
container_name: A string represents the name of container.
vm_workers: Number of workers which does the spinning on malloc()/free()
to consume memory.
Returns:
None.
"""
expected_alerting_messages = []
expected_alerting_messages.append(".*restart_service.*Restarting service 'telemetry'.*")
expected_alerting_messages.append(".*Stopping Telemetry container.*")
expected_alerting_messages.append(".*Stopped Telemetry container.*")
expected_alerting_messages.append(".*Starting Telemetry container.*")
expected_alerting_messages.append(".*Started Telemetry container.*")
loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix="container_restart_due_to_memory")
loganalyzer.expect_regex = []
loganalyzer.expect_regex.extend(expected_alerting_messages)
marker = loganalyzer.init()
thread_pool = ThreadPool()
thread_pool.apply_async(consume_memory, (duthost, container_name, vm_workers))
logger.info("Sleep '{}' seconds to wait for the alerting messages from syslog ...".format(WAITING_SYSLOG_MSG_SECS))
time.sleep(WAITING_SYSLOG_MSG_SECS)
logger.info("Checking the alerting messages related to container stopped ...")
loganalyzer.analyze(marker)
logger.info("Found all the expected alerting messages from syslog!")
logger.info("Waiting for '{}' container to be restarted ...".format(container_name))
restarted = wait_until(CONTAINER_RESTART_THRESHOLD_SECS,
CONTAINER_CHECK_INTERVAL_SECS,
0,
check_container_state, duthost, container_name, True)
pytest_assert(restarted, "Failed to restart '{}' container!".format(container_name))
logger.info("'{}' container is restarted.".format(container_name))
def execute_test(self,
duthost,
syslog_marker,
ignore_regex=None,
expect_regex=None,
expect_errors=False):
"""
Helper function that loads each template on the DUT and verifies the expected behavior
Args:
duthost (AnsibleHost): instance
syslog_marker (string): marker prefix name to be inserted in the syslog
ignore_regex (string): file containing regexs to be ignored by loganalyzer
expect_regex (string): regex pattern that is expected to be present in the syslog
expect_erros (bool): if the test expects an error msg in the syslog or not. Default: False
Returns:
None
"""
loganalyzer = LogAnalyzer(ansible_host=duthost,
marker_prefix=syslog_marker)
if ignore_regex:
ignore_file = os.path.join(TEMPLATES_DIR, ignore_regex)
reg_exp = loganalyzer.parse_regexp_file(src=ignore_file)
loganalyzer.ignore_regex.extend(reg_exp)
if expect_regex:
loganalyzer.expect_regex = []
loganalyzer.expect_regex.extend(expect_regex)
loganalyzer.match_regex = []
with loganalyzer(fail=not expect_errors):
cmd = "sonic-cfggen -j {}/{}.json --write-to-db".format(
DUT_RUN_DIR, syslog_marker)
out = duthost.command(cmd)
pytest_assert(
out["rc"] == 0, "Failed to execute cmd {}: Error: {}".format(
cmd, out["stderr"]))
def test_thermal_control_fan_status(duthosts, rand_one_dut_hostname, mocker_factory):
"""
@summary: Make FAN absence, over speed and under speed, check logs and LED color.
"""
duthost = duthosts[rand_one_dut_hostname]
loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix='thermal_control')
loganalyzer.load_common_config()
with ThermalPolicyFileContext(duthost, THERMAL_POLICY_VALID_FILE):
fan_mocker = mocker_factory(duthost, 'FanStatusMocker')
if fan_mocker is None:
pytest.skip("No FanStatusMocker for %s, skip rest of the testing in this case" % duthost.facts['asic_type'])
logging.info('Mock FAN status data...')
fan_mocker.mock_data() # make data random
restart_thermal_control_daemon(duthost)
wait_until(THERMAL_CONTROL_TEST_WAIT_TIME, THERMAL_CONTROL_TEST_CHECK_INTERVAL, fan_mocker.check_all_fan_speed,
60)
check_thermal_algorithm_status(duthost, mocker_factory, False)
single_fan_mocker = mocker_factory(duthost, 'SingleFanMocker')
time.sleep(THERMAL_CONTROL_TEST_WAIT_TIME)
_fan_log_supported = duthost.command('docker exec pmon grep -E "{}" /usr/bin/thermalctld'\
.format(LOG_EXPECT_INSUFFICIENT_FAN_NUM_RE), module_ignore_errors=True)
if single_fan_mocker.is_fan_removable():
loganalyzer.expect_regex = [LOG_EXPECT_FAN_REMOVE_RE, LOG_EXPECT_INSUFFICIENT_FAN_NUM_RE]
if _fan_log_supported.is_failed:
loganalyzer.expect_regex.remove(LOG_EXPECT_INSUFFICIENT_FAN_NUM_RE)
with loganalyzer:
logging.info('Mocking an absence FAN...')
single_fan_mocker.mock_absence()
check_cli_output_with_mocker(duthost, single_fan_mocker, CMD_PLATFORM_FANSTATUS, THERMAL_CONTROL_TEST_WAIT_TIME, 2)
loganalyzer.expect_regex = [LOG_EXPECT_FAN_REMOVE_CLEAR_RE, LOG_EXPECT_INSUFFICIENT_FAN_NUM_CLEAR_RE]
if _fan_log_supported.is_failed:
loganalyzer.expect_regex.remove(LOG_EXPECT_INSUFFICIENT_FAN_NUM_CLEAR_RE)
with loganalyzer:
logging.info('Make the absence FAN back to presence...')
single_fan_mocker.mock_presence()
check_cli_output_with_mocker(duthost, single_fan_mocker, CMD_PLATFORM_FANSTATUS, THERMAL_CONTROL_TEST_WAIT_TIME, 2)
if not _fan_log_supported.is_failed:
loganalyzer.expect_regex = [LOG_EXPECT_FAN_FAULT_RE, LOG_EXPECT_INSUFFICIENT_FAN_NUM_RE]
with loganalyzer:
logging.info('Mocking a fault FAN...')
single_fan_mocker.mock_status(False)
check_cli_output_with_mocker(duthost, single_fan_mocker, CMD_PLATFORM_FANSTATUS, THERMAL_CONTROL_TEST_WAIT_TIME, 2)
loganalyzer.expect_regex = [LOG_EXPECT_FAN_FAULT_CLEAR_RE, LOG_EXPECT_INSUFFICIENT_FAN_NUM_CLEAR_RE]
with loganalyzer:
logging.info('Mocking the fault FAN back to normal...')
single_fan_mocker.mock_status(True)
check_cli_output_with_mocker(duthost, single_fan_mocker, CMD_PLATFORM_FANSTATUS, THERMAL_CONTROL_TEST_WAIT_TIME, 2)
loganalyzer.expect_regex = [LOG_EXPECT_FAN_OVER_SPEED_RE]
with loganalyzer:
logging.info('Mocking an over speed FAN...')
single_fan_mocker.mock_over_speed()
check_cli_output_with_mocker(duthost, single_fan_mocker, CMD_PLATFORM_FANSTATUS, THERMAL_CONTROL_TEST_WAIT_TIME, 2)
loganalyzer.expect_regex = [LOG_EXPECT_FAN_OVER_SPEED_CLEAR_RE]
with loganalyzer:
logging.info('Make the over speed FAN back to normal...')
single_fan_mocker.mock_normal_speed()
check_cli_output_with_mocker(duthost, single_fan_mocker, CMD_PLATFORM_FANSTATUS, THERMAL_CONTROL_TEST_WAIT_TIME, 2)
loganalyzer.expect_regex = [LOG_EXPECT_FAN_UNDER_SPEED_RE]
with loganalyzer:
logging.info('Mocking an under speed FAN...')
single_fan_mocker.mock_under_speed()
check_cli_output_with_mocker(duthost, single_fan_mocker, CMD_PLATFORM_FANSTATUS, THERMAL_CONTROL_TEST_WAIT_TIME, 2)
loganalyzer.expect_regex = [LOG_EXPECT_FAN_UNDER_SPEED_CLEAR_RE]
with loganalyzer:
logging.info('Make the under speed FAN back to normal...')
single_fan_mocker.mock_normal_speed()
check_cli_output_with_mocker(duthost, single_fan_mocker, CMD_PLATFORM_FANSTATUS, THERMAL_CONTROL_TEST_WAIT_TIME, 2)
def test_monitoring_critical_processes(duthosts, rand_one_dut_hostname, tbinfo,
skip_vendor_specific_container):
"""Tests the feature of monitoring critical processes by Monit and Supervisord.
This function will check whether names of critical processes will appear
in the syslog if the autorestart were disabled and these critical processes
were stopped.
Args:
duthosts: list of DUTs.
rand_one_dut_hostname: hostname of DUT.
tbinfo: Testbed information.
Returns:
None.
"""
duthost = duthosts[rand_one_dut_hostname]
loganalyzer = LogAnalyzer(ansible_host=duthost,
marker_prefix="monitoring_critical_processes")
loganalyzer.expect_regex = []
bgp_neighbors = duthost.get_bgp_neighbors()
up_bgp_neighbors = [
k.lower() for k, v in bgp_neighbors.items()
if v["state"] == "established"
]
skip_containers = []
skip_containers.append("database")
skip_containers.append("gbsyncd")
# Skip 'restapi' container since 'restapi' service will be restarted immediately after exited, which will not trigger alarm message.
skip_containers.append("restapi")
# Skip 'acms' container since 'acms' process is not running on lab devices and
# another process `cert_converter.py' is set to auto-restart if exited.
skip_containers.append("acms")
# Skip 'radv' container on devices whose role is not T0.
if tbinfo["topo"]["type"] != "t0":
skip_containers.append("radv")
skip_containers = skip_containers + skip_vendor_specific_container
containers_in_namespaces = get_containers_namespace_ids(
duthost, skip_containers)
if "20191130" in duthost.os_version:
expected_alerting_messages = get_expected_alerting_messages_monit(
duthost, containers_in_namespaces)
else:
expected_alerting_messages = get_expected_alerting_messages_supervisor(
duthost, containers_in_namespaces)
loganalyzer.expect_regex.extend(expected_alerting_messages)
marker = loganalyzer.init()
stop_critical_processes(duthost, containers_in_namespaces)
# Wait for 70 seconds such that Supervisord/Monit has a chance to write alerting message into syslog.
logger.info(
"Sleep 70 seconds to wait for the alerting messages in syslog...")
time.sleep(70)
logger.info("Checking the alerting messages from syslog...")
loganalyzer.analyze(marker)
logger.info("Found all the expected alerting messages from syslog!")
logger.info("Executing the config reload...")
config_reload(duthost)
logger.info("Executing the config reload was done!")
ensure_all_critical_processes_running(duthost, containers_in_namespaces)
if not postcheck_critical_processes_status(duthost, up_bgp_neighbors):
pytest.fail("Post-check failed after testing the process monitoring!")
logger.info(
"Post-checking status of critical processes and BGP sessions was done!"
)
def test_pfcwd_port_toggle(self, request, fake_storm, setup_pfc_test, fanout_graph_facts, tbinfo, ptfhost, duthosts, rand_one_dut_hostname, fanouthosts):
"""
Test PfCWD functionality after toggling port
Test verifies the following:
1. Select the port and lossless queue
2. Start PFCWD on selected test port
3. Start PFC storm on selected test port and lossless queue
4. Verify that PFC storm is detected
5. Stop PFC storm on selected test port and lossless queue
6. Verify that PFC storm is restored
7. Toggle test port (put administrativelly down and then up)
8. Verify that PFC storm is not detected
Args:
request(object) : pytest request object
fake_storm(fixture) : Module scoped fixture for enable/disable fake storm
setup_pfc_test(fixture) : Module scoped autouse fixture for PFCWD
fanout_graph_facts(fixture) : Fanout graph info
tbinfo(fixture) : Testbed info
ptfhost(AnsibleHost) : PTF host instance
duthost(AnsibleHost) : DUT instance
fanouthosts(AnsibleHost): Fanout instance
"""
duthost = duthosts[rand_one_dut_hostname]
setup_info = setup_pfc_test
self.fanout_info = fanout_graph_facts
self.ptf = ptfhost
self.dut = duthost
self.fanout = fanouthosts
self.timers = setup_info['pfc_timers']
self.ports = setup_info['selected_test_ports']
self.neighbors = setup_info['neighbors']
dut_facts = self.dut.facts
self.peer_dev_list = dict()
self.fake_storm = fake_storm
self.storm_hndle = None
action = "dontcare"
for idx, port in enumerate(self.ports):
logger.info("")
logger.info("--- Testing port toggling with PFCWD enabled on {} ---".format(port))
self.setup_test_params(port, setup_info['vlan'], init=not idx)
self.traffic_inst = SendVerifyTraffic(self.ptf, dut_facts['router_mac'], self.pfc_wd)
pfc_wd_restore_time_large = request.config.getoption("--restore-time")
# wait time before we check the logs for the 'restore' signature. 'pfc_wd_restore_time_large' is in ms.
self.timers['pfc_wd_wait_for_restore_time'] = int(pfc_wd_restore_time_large / 1000 * 2)
try:
# Verify that PFC storm is detected and restored
self.stats = PfcPktCntrs(self.dut, action)
logger.info("{} on port {}".format(WD_ACTION_MSG_PFX[action], port))
self.run_test(self.dut, port, action)
# Toggle test port and verify that PFC storm is not detected
loganalyzer = LogAnalyzer(ansible_host=self.dut, marker_prefix="pfc_function_storm_detect_{}_port_{}".format(action, port))
marker = loganalyzer.init()
ignore_file = os.path.join(TEMPLATES_DIR, "ignore_pfc_wd_messages")
reg_exp = loganalyzer.parse_regexp_file(src=ignore_file)
loganalyzer.ignore_regex.extend(reg_exp)
loganalyzer.expect_regex = []
loganalyzer.expect_regex.extend([EXPECT_PFC_WD_DETECT_RE])
loganalyzer.match_regex = []
port_toggle(self.dut, tbinfo, ports=[port])
logger.info("Verify that PFC storm is not detected on port {}".format(port))
result = loganalyzer.analyze(marker, fail=False)
if result["total"]["expected_missing_match"] == 0:
pytest.fail(result)
except Exception as e:
pytest.fail(str(e))
finally:
if self.storm_hndle:
logger.info("--- Stop PFC storm on port {}".format(port))
self.storm_hndle.stop_storm()
else:
logger.info("--- Disabling fake storm on port {} queue {}".format(port, self.queue_oid))
PfcCmd.set_storm_status(self.dut, self.queue_oid, "disabled")
logger.info("--- Stop PFCWD ---")
self.dut.command("pfcwd stop")
def advanceboot_loganalyzer(duthosts, rand_one_dut_hostname, request):
"""
Advance reboot log analysis.
This fixture starts log analysis at the beginning of the test. At the end,
the collected expect messages are verified and timing of start/stop is calculated.
Args:
duthosts : List of DUT hosts
rand_one_dut_hostname: hostname of a randomly selected DUT
"""
duthost = duthosts[rand_one_dut_hostname]
test_name = request.node.name
if "warm" in test_name:
reboot_type = "warm"
elif "fast" in test_name:
reboot_type = "fast"
else:
reboot_type = "unknown"
# Currently, advanced reboot test would skip for kvm platform if the test has no device_type marker for vs.
# Doing the same skip logic in this fixture to avoid running loganalyzer without the test executed
if duthost.facts['platform'] == 'x86_64-kvm_x86_64-r0':
device_marks = [
arg for mark in request.node.iter_markers(name='device_type')
for arg in mark.args
]
if 'vs' not in device_marks:
pytest.skip('Testcase not supported for kvm')
loganalyzer = LogAnalyzer(
ansible_host=duthost,
marker_prefix="test_advanced_reboot_{}".format(test_name),
additional_files={
'/var/log/swss/sairedis.rec':
'recording on: /var/log/swss/sairedis.rec',
'/var/log/frr/bgpd.log': ''
})
marker = loganalyzer.init()
loganalyzer.load_common_config()
ignore_file = os.path.join(TEMPLATES_DIR, "ignore_boot_messages")
expect_file = os.path.join(TEMPLATES_DIR, "expect_boot_messages")
ignore_reg_exp = loganalyzer.parse_regexp_file(src=ignore_file)
expect_reg_exp = loganalyzer.parse_regexp_file(src=expect_file)
loganalyzer.ignore_regex.extend(ignore_reg_exp)
loganalyzer.expect_regex = []
loganalyzer.expect_regex.extend(expect_reg_exp)
loganalyzer.match_regex = []
yield
result = loganalyzer.analyze(marker, fail=False)
analyze_result = {"time_span": dict(), "offset_from_kexec": dict()}
offset_from_kexec = dict()
for key, messages in result["expect_messages"].items():
if "syslog" in key or "bgpd.log" in key:
analyze_log_file(duthost, messages, analyze_result,
offset_from_kexec)
elif "sairedis.rec" in key:
analyze_sairedis_rec(messages, analyze_result, offset_from_kexec)
for marker, time_data in analyze_result["offset_from_kexec"].items():
marker_start_time = time_data.get("timestamp", {}).get("Start")
reboot_start_time = analyze_result.get("reboot_time",
{}).get("timestamp",
{}).get("Start")
if reboot_start_time and reboot_start_time != "N/A" and marker_start_time:
time_data["time_taken"] = (datetime.strptime(marker_start_time, FMT) -\
datetime.strptime(reboot_start_time, FMT)).total_seconds()
else:
time_data["time_taken"] = "N/A"
get_data_plane_report(analyze_result, reboot_type)
result_summary = get_report_summary(analyze_result, reboot_type)
logging.info(json.dumps(analyze_result, indent=4))
logging.info(json.dumps(result_summary, indent=4))
report_file_name = request.node.name + "_report.json"
summary_file_name = request.node.name + "_summary.json"
report_file_dir = os.path.realpath((os.path.join(os.path.dirname(__file__),\
"../logs/platform_tests/")))
report_file_path = report_file_dir + "/" + report_file_name
summary_file_path = report_file_dir + "/" + summary_file_name
if not os.path.exists(report_file_dir):
os.makedirs(report_file_dir)
with open(report_file_path, 'w') as fp:
json.dump(analyze_result, fp, indent=4)
with open(summary_file_path, 'w') as fp:
json.dump(result_summary, fp, indent=4)
def advanceboot_loganalyzer(duthosts, rand_one_dut_hostname):
"""
Advance reboot log analysis.
This fixture starts log analysis at the beginning of the test. At the end,
the collected expect messages are verified and timing of start/stop is calculated.
Args:
duthosts : List of DUT hosts
rand_one_dut_hostname: hostname of a randomly selected DUT
"""
duthost = duthosts[rand_one_dut_hostname]
loganalyzer = LogAnalyzer(ansible_host=duthost,
marker_prefix="test_advanced_reboot")
marker = loganalyzer.init()
loganalyzer.load_common_config()
ignore_file = os.path.join(TEMPLATES_DIR, "ignore_boot_messages")
expect_file = os.path.join(TEMPLATES_DIR, "expect_boot_messages")
ignore_reg_exp = loganalyzer.parse_regexp_file(src=ignore_file)
expect_reg_exp = loganalyzer.parse_regexp_file(src=expect_file)
loganalyzer.ignore_regex.extend(ignore_reg_exp)
loganalyzer.expect_regex = []
loganalyzer.expect_regex.extend(expect_reg_exp)
loganalyzer.match_regex = []
yield
result = loganalyzer.analyze(marker, fail=False)
messages = result["expect_messages"].values()
if not messages:
logging.error("Expected messages not found in syslog")
return
messages = messages[0]
service_restart_times = dict()
service_patterns = {
"Stopping": re.compile(r'.*Stopping.*service.*'),
"Stopped": re.compile(r'.*Stopped.*service.*'),
"Starting": re.compile(r'.*Starting.*service.*'),
"Started": re.compile(r'.*Started.*service.*')
}
def service_time_check(message, status):
time = message.split(duthost.hostname)[0].strip()
service_name = message.split(status + " ")[1].split()[0]
service_dict = service_restart_times.get(service_name,
{"timestamp": {}})
timestamps = service_dict.get("timestamp")
if status in timestamps:
service_dict[status +
" count"] = service_dict.get(status + " count", 1) + 1
timestamps[status] = time
service_restart_times.update({service_name: service_dict})
for message in messages:
for status, pattern in service_patterns.items():
if re.search(pattern, message):
service_time_check(message, status)
loganalyzer.save_extracted_log(dest="/tmp/log/syslog")
logging.info(json.dumps(service_restart_times, indent=4))
FMT = "%b %d %H:%M:%S.%f"
for _, timings in service_restart_times.items():
timestamps = timings["timestamp"]
timings["stop_time"] = (datetime.strptime(timestamps["Stopped"], FMT) -\
datetime.strptime(timestamps["Stopping"], FMT)).total_seconds() \
if "Stopped" in timestamps and "Stopping" in timestamps else None
timings["start_time"] = (datetime.strptime(timestamps["Started"], FMT) -\
datetime.strptime(timestamps["Starting"], FMT)).total_seconds() \
if "Started" in timestamps and "Starting" in timestamps else None
timings["reboot_time"] = (datetime.strptime(timestamps["Started"], FMT) -\
datetime.strptime(timestamps["Stopped"], FMT)).total_seconds() \
if "Started" in timestamps and "Stopped" in timestamps else None
files = glob.glob('/tmp/*-report.json')
if files:
filepath = files[0]
with open(filepath) as json_file:
report = json.load(json_file)
service_restart_times.update(report)
result = service_restart_times
logging.info(json.dumps(result, indent=4))