def _get_vendor_id(duthost):
if is_broadcom_device(duthost):
vendor_id = "brcm"
elif is_mellanox_device(duthost):
vendor_id = "mlnx"
else:
error_message = '"{}" does not currently support swap_syncd'.format(duthost.facts["asic_type"])
logger.error(error_message)
raise ValueError(error_message)
return vendor_id
def swap_syncd(dut):
"""
Replaces the running syncd container with the RPC version of it.
This will download a new Docker image to the DUT and restart the swss service.
Args:
dut (SonicHost): The target device.
"""
if is_broadcom_device(dut):
vendor_id = "brcm"
elif is_mellanox_device(dut):
vendor_id = "mlnx"
else:
error_message = "\"{}\" is not currently supported".format(
dut.facts["asic_type"])
_LOGGER.error(error_message)
raise ValueError(error_message)
docker_syncd_name = "docker-syncd-{}".format(vendor_id)
docker_rpc_image = docker_syncd_name + "-rpc"
dut.command("systemctl stop swss")
delete_container(dut, "syncd")
# Set sysctl RCVBUF parameter for tests
dut.command("sysctl -w net.core.rmem_max=609430500")
# Set sysctl SENDBUF parameter for tests
dut.command("sysctl -w net.core.wmem_max=609430500")
# TODO: Getting the base image version should be a common utility
output = dut.command(
"sonic-cfggen -y /etc/sonic/sonic_version.yml -v build_version")
sonic_version = output["stdout_lines"][0].strip()
registry = load_docker_registry_info(dut)
download_image(dut, registry, docker_rpc_image, sonic_version)
tag_image(dut, "{}:latest".format(docker_syncd_name),
"{}/{}".format(registry.host, docker_rpc_image), sonic_version)
dut.command("systemctl reset-failed swss")
dut.command("systemctl start swss")
_LOGGER.info(
"swss has been restarted, waiting 60 seconds to initialize...")
time.sleep(60)
def skip_warm_reboot(duthost, reboot_type):
"""
Skip warm reboot tests for TD2 asics
Args:
duthost (pytest fixture): device under test
reboot_type (string): type of reboot (can be warm, cold, fast)
Returns:
None
"""
SKIP_LIST = ["td2"]
asic_type = duthost.get_asic_name()
pytest_require(
not (is_broadcom_device(duthost) and asic_type in SKIP_LIST
and "warm" in reboot_type),
"Warm reboot is not supported on {}".format(asic_type))
def skip_pfcwd_wb_tests(duthosts, rand_one_dut_hostname):
"""
Skip Pfcwd warm reboot tests on certain asics
Args:
duthosts (pytest fixture): list of Duts
rand_one_dut_hostname (str): hostname of DUT
Returns:
None
"""
duthost = duthosts[rand_one_dut_hostname]
SKIP_LIST = ["td2"]
asic_type = duthost.get_asic_name()
pytest_require(
not (is_broadcom_device(duthost) and asic_type in SKIP_LIST),
"Warm reboot is not supported on {}".format(asic_type))
def restore_default_syncd(dut):
"""
Replaces the running syncd with the default syncd that comes with the image.
This will restart the swss service.
Args:
dut (SonicHost): The target device.
"""
if is_broadcom_device(dut):
vendor_id = "brcm"
elif is_mellanox_device(dut):
vendor_id = "mlnx"
else:
error_message = "\"{}\" is not currently supported".format(
dut.facts["asic_type"])
_LOGGER.error(error_message)
raise ValueError(error_message)
docker_syncd_name = "docker-syncd-{}".format(vendor_id)
dut.command("systemctl stop swss")
delete_container(dut, "syncd")
# TODO: Getting the base image version should be a common utility
output = dut.command(
"sonic-cfggen -y /etc/sonic/sonic_version.yml -v build_version")
sonic_version = output["stdout_lines"][0].strip()
tag_image(dut, "{}:latest".format(docker_syncd_name), docker_syncd_name,
sonic_version)
dut.command("systemctl reset-failed swss")
dut.command("systemctl start swss")
_LOGGER.info(
"swss has been restarted, waiting 60 seconds to initialize...")
time.sleep(60)
# Remove the RPC image from the DUT
docker_rpc_image = docker_syncd_name + "-rpc"
registry = load_docker_registry_info(dut)
dut.command("docker rmi {}/{}:{}".format(registry.host, docker_rpc_image,
sonic_version))
def skip_pfcwd_test(duthost, trigger_pfcwd):
"""
Skip PFC watchdog tests that may cause fake alerts
PFC watchdog on Broadcom devices use some approximation techniques to detect
PFC storms, which may cause some fake alerts. Therefore, we skip test cases
whose trigger_pfcwd is False for Broadcom devices.
Args:
duthost (obj): device to test
trigger_pfcwd (bool): if PFC watchdog is supposed to trigger
Returns:
N/A
"""
pytest_require(
trigger_pfcwd is True or is_broadcom_device(duthost) is False,
'Skip trigger_pfcwd=False test cases for Broadcom devices')
def _get_sai_running_vendor_id(duthost):
"""
Get the vendor id.
Args:
duthost (SonicHost): The target device.
"""
if is_broadcom_device(duthost):
vendor_id = "brcm"
elif is_mellanox_device(duthost):
vendor_id = "mlnx"
elif is_barefoot_device(duthost):
vendor_id = "bfn"
else:
error_message = '"{}" does not currently support saitest'.format(duthost.facts["asic_type"])
logger.error(error_message)
raise ValueError(error_message)
return vendor_id
def analyze_log_file(duthost, messages, result, offset_from_kexec):
service_restart_times = dict()
derived_patterns = OTHER_PATTERNS.get("COMMON")
service_patterns = dict()
# get platform specific regexes
if is_broadcom_device(duthost):
derived_patterns.update(OTHER_PATTERNS.get("BRCM"))
elif is_mellanox_device(duthost):
derived_patterns.update(OTHER_PATTERNS.get("MLNX"))
# get image specific regexes
if "20191130" in duthost.os_version:
derived_patterns.update(OTHER_PATTERNS.get("201911"))
service_patterns.update(SERVICE_PATTERNS.get("201911"))
else:
derived_patterns.update(OTHER_PATTERNS.get("LATEST"))
service_patterns.update(SERVICE_PATTERNS.get("LATEST"))
if not messages:
logging.error("Expected messages not found in syslog")
return None
def service_time_check(message, status):
time = datetime.strptime(
message.split(duthost.hostname)[0].strip(), FMT)
time = time.strftime(FMT)
service_name = message.split(status + " ")[1].split()[0]
service_name = service_name.upper()
if service_name == "ROUTER":
service_name = "RADV"
service_dict = service_restart_times.get(service_name,
{"timestamp": {}})
timestamps = service_dict.get("timestamp")
if status in timestamps:
service_dict[status +
" count"] = service_dict.get(status + " count", 1) + 1
timestamps[status] = time
service_restart_times.update({service_name: service_dict})
reboot_time = "N/A"
for message in messages:
# Get stopping to started timestamps for services (swss, bgp, etc)
for status, pattern in service_patterns.items():
if re.search(pattern, message):
service_time_check(message, status)
break
# Get timestamps of all other entities
for state, pattern in derived_patterns.items():
if re.search(pattern, message):
timestamp = datetime.strptime(
message.split(duthost.hostname)[0].strip(), FMT)
state_name = state.split("|")[0].strip()
if state_name + "|End" not in derived_patterns.keys():
state_times = get_state_times(timestamp, state,
offset_from_kexec)
offset_from_kexec.update(state_times)
else:
state_times = get_state_times(timestamp, state,
service_restart_times)
service_restart_times.update(state_times)
break
# Calculate time that services took to stop/start
for _, timings in service_restart_times.items():
timestamps = timings["timestamp"]
timings["stop_time"] = (datetime.strptime(timestamps["Stopped"], FMT) -\
datetime.strptime(timestamps["Stopping"], FMT)).total_seconds() \
if "Stopped" in timestamps and "Stopping" in timestamps else None
timings["start_time"] = (datetime.strptime(timestamps["Started"], FMT) -\
datetime.strptime(timestamps["Starting"], FMT)).total_seconds() \
if "Started" in timestamps and "Starting" in timestamps else None
if "Started" in timestamps and "Stopped" in timestamps:
timings["time_span"] = (datetime.strptime(timestamps["Started"], FMT) -\
datetime.strptime(timestamps["Stopped"], FMT)).total_seconds()
elif "Start" in timestamps and "End" in timestamps:
if "last_occurence" in timings:
timings["time_span"] = (datetime.strptime(timings["last_occurence"], FMT) -\
datetime.strptime(timestamps["Start"], FMT)).total_seconds()
else:
timings["time_span"] = (datetime.strptime(timestamps["End"], FMT) -\
datetime.strptime(timestamps["Start"], FMT)).total_seconds()
result["time_span"].update(service_restart_times)
result["offset_from_kexec"] = offset_from_kexec
return result
def swap_syncd(dut, creds):
"""
Replaces the running syncd container with the RPC version of it.
This will download a new Docker image to the DUT and restart the swss service.
Args:
dut (SonicHost): The target device.
"""
if is_broadcom_device(dut):
vendor_id = "brcm"
elif is_mellanox_device(dut):
vendor_id = "mlnx"
else:
error_message = "\"{}\" is not currently supported".format(
dut.facts["asic_type"])
_LOGGER.error(error_message)
raise ValueError(error_message)
docker_syncd_name = "docker-syncd-{}".format(vendor_id)
docker_rpc_image = docker_syncd_name + "-rpc"
dut.command("config bgp shutdown all"
) # Force image download to go through mgmt network
dut.command("systemctl stop swss")
delete_container(dut, "syncd")
# Set sysctl RCVBUF parameter for tests
dut.command("sysctl -w net.core.rmem_max=609430500")
# Set sysctl SENDBUF parameter for tests
dut.command("sysctl -w net.core.wmem_max=609430500")
# TODO: Getting the base image version should be a common utility
output = dut.command(
"sonic-cfggen -y /etc/sonic/sonic_version.yml -v build_version")
sonic_version = output["stdout_lines"][0].strip()
def ready_for_swap():
syncd_status = dut.command("docker ps -f name=syncd")["stdout_lines"]
if len(syncd_status) > 1:
return False
swss_status = dut.command("docker ps -f name=swss")["stdout_lines"]
if len(swss_status) > 1:
return False
bgp_summary = dut.command("show ip bgp summary")["stdout_lines"]
idle_count = 0
expected_idle_count = 0
for line in bgp_summary:
if "Idle (Admin)" in line:
idle_count += 1
if "Total number of neighbors" in line:
tokens = line.split()
expected_idle_count = int(tokens[-1])
return idle_count == expected_idle_count
pytest_assert(wait_until(30, 3, ready_for_swap),
"Docker and/or BGP failed to shut down")
registry = load_docker_registry_info(dut, creds)
download_image(dut, registry, docker_rpc_image, sonic_version)
tag_image(dut, "{}:latest".format(docker_syncd_name),
"{}/{}".format(registry.host, docker_rpc_image), sonic_version)
_LOGGER.info("Reloading config and restarting swss...")
config_reload(dut)
