def testFastreboot(self, sflowbase_config, duthost, localhost, partial_ptf_runner, ptfhost):
config_sflow(duthost,sflow_status='enable')
verify_show_sflow(duthost,status='up',collector=['collector0','collector1'])
duthost.command('sudo config save -y')
reboot(duthost, localhost,reboot_type='fast')
assert wait_until(300, 20, duthost.critical_services_fully_started), "Not all critical services are fully started"
verify_show_sflow(duthost,status='up',collector=['collector0','collector1'])
for intf in var['sflow_ports']:
var['sflow_ports'][intf]['ifindex'] = get_ifindex(duthost,intf)
verify_sflow_interfaces(duthost,intf,'up',512)
var['portmap'] = json.dumps(var['sflow_ports'])
ptfhost.copy(content=var['portmap'],dest="/tmp/sflow_ports.json")
partial_ptf_runner(
enabled_sflow_interfaces=var['sflow_ports'].keys(),
active_collectors="['collector0','collector1']" )
def test_upgrade_path(localhost, duthosts, rand_one_dut_hostname, ptfhost,
upgrade_path_lists, ptf_params, setup, tbinfo):
duthost = duthosts[rand_one_dut_hostname]
from_list_images, to_list_images, _ = upgrade_path_lists
from_list = from_list_images.split(',')
to_list = to_list_images.split(',')
assert (from_list and to_list)
for from_image in from_list:
for to_image in to_list:
logger.info("Test upgrade path from {} to {}".format(
from_image, to_image))
# Install base image
logger.info("Installing {}".format(from_image))
target_version = install_sonic(duthost, from_image, tbinfo)
# Perform a cold reboot
logger.info(
"Cold reboot the DUT to make the base image as current")
reboot(duthost, localhost)
check_sonic_version(duthost, target_version)
# Install target image
logger.info("Upgrading to {}".format(to_image))
target_version = install_sonic(duthost, to_image, tbinfo)
test_params = ptf_params
test_params['target_version'] = target_version
test_params['reboot_type'] = get_reboot_type(duthost)
prepare_testbed_ssh_keys(duthost, ptfhost,
test_params['dut_username'])
log_file = "/tmp/advanced-reboot.ReloadTest.{}.log".format(
datetime.now().strftime('%Y-%m-%d-%H:%M:%S'))
ptf_runner(ptfhost,
"ptftests",
"advanced-reboot.ReloadTest",
platform_dir="ptftests",
params=test_params,
platform="remote",
qlen=10000,
log_file=log_file)
reboot_cause = get_reboot_cause(duthost)
logger.info("Check reboot cause. Expected cause {}".format(
REBOOT_TYPE_WARM))
pytest_assert(
reboot_cause == REBOOT_TYPE_WARM,
"Reboot cause {} did not match the trigger - {}".format(
reboot_cause, REBOOT_TYPE_WARM))
check_services(duthost)
def cleanup(localhost, ptfhost, duthost, upgrade_path_lists):
_, _, restore_to_image = upgrade_path_lists
if restore_to_image:
logger.info(
"Preparing to cleanup and restore to {}".format(restore_to_image))
# restore orignial image
install_sonic(duthost, restore_to_image)
# Perform a cold reboot
reboot(duthost, localhost)
# cleanup
ptfhost.shell("rm -f {} {} {}".format(TMP_VLAN_FILE,
TMP_VLAN_PORTCHANNEL_FILE,
TMP_PORTS_FILE),
module_ignore_errors=True)
os.remove(TMP_VLAN_FILE)
os.remove(TMP_VLAN_PORTCHANNEL_FILE)
os.remove(TMP_PORTS_FILE)
def reboot_and_check(localhost, dut, interfaces, xcvr_skip_list, reboot_type=REBOOT_TYPE_COLD, reboot_helper=None, reboot_kwargs=None):
"""
Perform the specified type of reboot and check platform status.
@param localhost: The Localhost object.
@param dut: The AnsibleHost object of DUT.
@param interfaces: DUT's interfaces defined by minigraph
@param xcvr_skip_list: list of DUT's interfaces for which transeiver checks are skipped
@param reboot_type: The reboot type, pre-defined const that has name convention of REBOOT_TYPE_XXX.
@param reboot_helper: The helper function used only by power off reboot
@param reboot_kwargs: The argument used by reboot_helper
"""
logging.info("Run %s reboot on DUT" % reboot_type)
reboot(dut, localhost, reboot_type=reboot_type, reboot_helper=reboot_helper, reboot_kwargs=reboot_kwargs)
REBOOT_TYPE_HISTOYR_QUEUE.append(reboot_type)
check_interfaces_and_services(dut, interfaces, xcvr_skip_list, reboot_type)
def test_warm_upgrade_sad_path(localhost, duthosts, ptfhost,
rand_one_dut_hostname, nbrhosts, fanouthosts,
tbinfo, restore_image, get_advanced_reboot,
verify_dut_health, advanceboot_loganalyzer,
upgrade_path_lists,
backup_and_restore_config_db,
advanceboot_neighbor_restore, sad_case_type):
duthost = duthosts[rand_one_dut_hostname]
upgrade_type, from_list_images, to_list_images, _ = upgrade_path_lists
from_list = from_list_images.split(',')
to_list = to_list_images.split(',')
assert (from_list and to_list)
for from_image in from_list:
for to_image in to_list:
logger.info("Test upgrade path from {} to {}".format(
from_image, to_image))
# Install base image
logger.info("Installing {}".format(from_image))
target_version = install_sonic(duthost, from_image, tbinfo)
# Perform a cold reboot
logger.info(
"Cold reboot the DUT to make the base image as current")
reboot(duthost, localhost)
check_sonic_version(duthost, target_version)
# Install target image
logger.info("Upgrading to {}".format(to_image))
install_sonic(duthost, to_image, tbinfo)
advancedReboot = get_advanced_reboot(rebootType=get_reboot_command(duthost, "warm"),\
advanceboot_loganalyzer=advanceboot_loganalyzer)
sad_preboot_list, sad_inboot_list = get_sad_case_list(
duthost, nbrhosts, fanouthosts, tbinfo, sad_case_type)
advancedReboot.runRebootTestcase(prebootList=sad_preboot_list,
inbootList=sad_inboot_list)
reboot_cause = get_reboot_cause(duthost)
logger.info(
"Check reboot cause. Expected cause {}".format(upgrade_type))
pytest_assert(
reboot_cause == upgrade_type,
"Reboot cause {} did not match the trigger - {}".format(
reboot_cause, upgrade_type))
check_services(duthost)
def test_reboot_supervisor(duthosts, localhost, all_cfg_facts, nbrhosts,
nbr_macs):
"""
Tests the system after supervisor reset, all cards should reboot and interfaces/neighbors should be in sync across
the system.
Args:
duthosts: duthosts fixture
localhost: localhost fixture
all_cfg_facts: all_cfg_facts fixture
nbrhosts: nbrhosts fixture
nbr_macs: nbr_macs fixture
"""
logger.info("=" * 80)
logger.info("Precheck")
logger.info("-" * 80)
check_intfs_and_nbrs(duthosts, all_cfg_facts, nbrhosts, nbr_macs)
check_ip_fwd(duthosts, all_cfg_facts, nbrhosts)
logger.info("=" * 80)
logger.info("Coldboot on node: %s", duthosts.supervisor_nodes[0].hostname)
logger.info("-" * 80)
reboot(duthosts.supervisor_nodes[0], localhost, wait=600)
assert wait_until(
300, 20, duthosts.supervisor_nodes[0].critical_services_fully_started
), "Not all critical services are fully started"
reboot(duthosts.supervisor_nodes[0], localhost, wait=240)
assert wait_until(
300, 20, 2,
duthosts.supervisor_nodes[0].critical_services_fully_started
), "Not all critical services are fully started"
poll_bgp_restored(duthosts)
logger.info("=" * 80)
logger.info("Postcheck")
logger.info("-" * 80)
check_intfs_and_nbrs(duthosts, all_cfg_facts, nbrhosts, nbr_macs)
check_ip_fwd(duthosts, all_cfg_facts, nbrhosts)
def test_upgrade_path(localhost, duthosts, ptfhost, rand_one_dut_hostname,
nbrhosts, fanouthosts, tbinfo, restore_image,
get_advanced_reboot, verify_dut_health,
advanceboot_loganalyzer, upgrade_path_lists):
duthost = duthosts[rand_one_dut_hostname]
upgrade_type, from_list_images, to_list_images, _ = upgrade_path_lists
from_list = from_list_images.split(',')
to_list = to_list_images.split(',')
assert (from_list and to_list)
for from_image in from_list:
for to_image in to_list:
logger.info("Test upgrade path from {} to {}".format(
from_image, to_image))
# Install base image
logger.info("Installing {}".format(from_image))
target_version = install_sonic(duthost, from_image, tbinfo)
# Perform a cold reboot
logger.info(
"Cold reboot the DUT to make the base image as current")
reboot(duthost, localhost)
check_sonic_version(duthost, target_version)
# Install target image
logger.info("Upgrading to {}".format(to_image))
install_sonic(duthost, to_image, tbinfo)
if upgrade_type == REBOOT_TYPE_COLD:
# advance-reboot test (on ptf) does not support cold reboot yet
reboot(duthost, localhost)
else:
advancedReboot = get_advanced_reboot(rebootType=get_reboot_command(duthost, upgrade_type),\
advanceboot_loganalyzer=advanceboot_loganalyzer)
advancedReboot.runRebootTestcase()
reboot_cause = get_reboot_cause(duthost)
logger.info(
"Check reboot cause. Expected cause {}".format(upgrade_type))
pytest_assert(
reboot_cause == upgrade_type,
"Reboot cause {} did not match the trigger - {}".format(
reboot_cause, upgrade_type))
check_services(duthost)
def test_bgp_slb_neighbor_persistence_across_advanced_reboot(
duthosts, rand_one_dut_hostname, bgp_slb_neighbor,
toggle_all_simulator_ports_to_rand_selected_tor, reboot_type,
localhost):
def verify_bgp_session(duthost, bgp_neighbor):
"""Verify the bgp session to the DUT is established."""
bgp_facts = duthost.bgp_facts()["ansible_facts"]
return bgp_neighbor.ip in bgp_facts["bgp_neighbors"] and bgp_facts[
"bgp_neighbors"][bgp_neighbor.ip]["state"] == "established"
duthost = duthosts[rand_one_dut_hostname]
neighbor = bgp_slb_neighbor
try:
neighbor.start_session()
if not wait_until(40, 5, 10, verify_bgp_session, duthost, neighbor):
pytest.fail("dynamic BGP session is not established")
reboot(duthost, localhost, reboot_type=reboot_type)
if not wait_until(40, 5, 10, verify_bgp_session, duthost, neighbor):
pytest.fail("dynamic BGP session is not established after %s" %
reboot_type)
finally:
neighbor.stop_session()
def test_upgrade_path(localhost, duthosts, rand_one_dut_hostname, ptfhost,
upgrade_path_lists, ptf_params, setup):
duthost = duthosts[rand_one_dut_hostname]
from_list_images, to_list_images, _ = upgrade_path_lists
from_list = from_list_images.split(',')
to_list = to_list_images.split(',')
assert (from_list and to_list)
for from_image in from_list:
for to_image in to_list:
logger.info("Test upgrade path from {} to {}".format(
from_image, to_image))
# Install base image
logger.info("Installing {}".format(from_image))
target_version = install_sonic(duthost, from_image)
# Perform a cold reboot
reboot(duthost, localhost)
check_sonic_version(duthost, target_version)
# Install target image
logger.info("Upgrading to {}".format(to_image))
target_version = install_sonic(duthost, to_image)
test_params = ptf_params
test_params['target_version'] = target_version
test_params['reboot_type'] = get_reboot_type(duthost)
prepare_testbed_ssh_keys(duthost, ptfhost,
test_params['dut_username'])
log_file = "/tmp/advanced-reboot.ReloadTest.{}.log".format(
datetime.now().strftime('%Y-%m-%d-%H:%M:%S'))
ptf_runner(ptfhost,
"ptftests",
"advanced-reboot.ReloadTest",
platform_dir="ptftests",
params=test_params,
platform="remote",
qlen=10000,
log_file=log_file)
def testRebootSflowEnable(self, sflowbase_config, config_sflow_agent,
duthost, localhost, partial_ptf_runner, ptfhost):
duthost.command("config sflow polling-interval 80")
verify_show_sflow(duthost, status='up', polling_int=80)
duthost.command('sudo config save -y')
reboot(duthost, localhost)
assert wait_until(300, 20, 0, duthost.critical_services_fully_started
), "Not all critical services are fully started"
verify_show_sflow(duthost,
status='up',
collector=['collector0', 'collector1'],
polling_int=80)
for intf in var['sflow_ports']:
var['sflow_ports'][intf]['ifindex'] = get_ifindex(duthost, intf)
var['sflow_ports'][intf]['port_index'] = get_port_index(
duthost, intf)
verify_sflow_interfaces(duthost, intf, 'up', 512)
var['portmap'] = json.dumps(var['sflow_ports'])
ptfhost.copy(content=var['portmap'], dest="/tmp/sflow_ports.json")
partial_ptf_runner(enabled_sflow_interfaces=var['sflow_ports'].keys(),
active_collectors="['collector0','collector1']")
# Test Polling
partial_ptf_runner(polling_int=80,
active_collectors="['collector0','collector1']")
def test_fastboot(self, duthost, localhost, tbinfo):
duthost.command("sudo config save -y") # This will override config_db.json with mgmt vrf config
reboot(duthost, localhost, reboot_type="fast")
pytest_assert(wait_until(300, 20, duthost.critical_services_fully_started), "Not all critical services are fully started")
self.basic_check_after_reboot(duthost, localhost, tbinfo)
def reboot_node(lh, node=None, results=None):
node_results = []
node_results.append(reboot(node, lh, wait=600))
results[node.hostname] = node_results
def test_reboot(self, duthosts, rand_one_dut_hostname, localhost, ptfhost, creds):
duthost = duthosts[rand_one_dut_hostname]
duthost.command("sudo config save -y") # This will override config_db.json with mgmt vrf config
reboot(duthost, localhost)
pytest_assert(wait_until(300, 20, 0, duthost.critical_services_fully_started), "Not all critical services are fully started")
self.basic_check_after_reboot(duthost, localhost, ptfhost, creds)
def reboot_tor(duthost, reboot_type=REBOOT_TYPE_COLD):
torhost.append(duthost)
logger.info("Issuing reboot of type {} on {}".format(reboot_type, duthost.hostname))
reboot(duthost, localhost, reboot_type=reboot_type, wait_for_ssh=False)
