def loganalyzer(duthosts, request):
if request.config.getoption("--disable_loganalyzer"
) or "disable_loganalyzer" in request.keywords:
logging.info("Log analyzer is disabled")
yield
return
# Analyze all the duts
analyzers = {}
parallel_run(analyzer_logrotate, [], {}, duthosts, timeout=120)
for duthost in duthosts:
analyzers[duthost.hostname] = LogAnalyzer(
ansible_host=duthost, marker_prefix=request.node.name)
markers = parallel_run(analyzer_add_marker, [analyzers], {},
duthosts,
timeout=120)
yield analyzers
# Skip LogAnalyzer if case is skipped
if "rep_call" in request.node.__dict__ and request.node.rep_call.skipped or \
"rep_setup" in request.node.__dict__ and request.node.rep_setup.skipped:
return
logging.info("Starting to analyse on all DUTs")
parallel_run(analyze_logs, [analyzers, markers], {}, duthosts, timeout=120)
def ping_all_dut_local_nbrs(duthosts):
"""
Pings all neighbors locally attached to each frontend node on the DUT.
Args:
duthosts: An instance of the duthosts fixture.
"""
@reset_ansible_local_tmp
def _ping_all_local_nbrs(node=None, results=None):
if node is None or results is None:
logger.error('Missing kwarg "node" or "results"')
return
node_results = []
logger.info("Pinging neighbors to establish ARP on node: %s", node.hostname)
for asic in node.asics:
cfg_facts = asic.config_facts(source="persistent")['ansible_facts']
if 'BGP_NEIGHBOR' in cfg_facts:
neighs = cfg_facts['BGP_NEIGHBOR']
else:
logger.info("No local neighbors for host: %s/%s, skipping", node.hostname, asic.asic_index)
continue
for neighbor in neighs:
node_results.append(sonic_ping(asic, neighbor, verbose=True))
results[node.hostname] = node_results
parallel_run(_ping_all_local_nbrs, [], {}, duthosts.frontend_nodes, timeout=120)
def neighbor_vm_restore(duthost, nbrhosts, tbinfo):
logger.info("Restoring neighbor VMs for {}".format(duthost))
mg_facts = duthost.get_extended_minigraph_facts(tbinfo)
vm_neighbors = mg_facts['minigraph_neighbors']
if vm_neighbors:
parallel_run(neighbor_vm_recover_bgpd, (), {},
nbrhosts.values(),
timeout=300)
def test_reboot_system(duthosts, localhost, all_cfg_facts, nbrhosts, nbr_macs):
"""
Tests the system after all cards are explicitly reset, interfaces/neighbors should be in sync across the system.
Args:
duthosts: duthosts fixture
localhost: localhost fixture
all_cfg_facts: all_cfg_facts fixture
nbrhosts: nbrhosts fixture
nbr_macs: nbr_macs fixture
"""
@reset_ansible_local_tmp
def reboot_node(lh, node=None, results=None):
node_results = []
node_results.append(reboot(node, lh, wait=600))
results[node.hostname] = node_results
logger.info("=" * 80)
logger.info("Precheck")
logger.info("-" * 80)
check_intfs_and_nbrs(duthosts, all_cfg_facts, nbrhosts, nbr_macs)
check_ip_fwd(duthosts, all_cfg_facts, nbrhosts)
logger.info("=" * 80)
logger.info("Coldboot on all nodes")
logger.info("-" * 80)
t0 = time.time()
parallel_run(reboot_node, [localhost], {}, duthosts.nodes, timeout=1000)
for node in duthosts.nodes:
assert wait_until(300, 20, node.critical_services_fully_started
), "Not all critical services are fully started"
poll_bgp_restored(duthosts)
t1 = time.time()
elapsed = t1 - t0
logger.info("-" * 80)
logger.info("Time to reboot and recover: %s seconds.", str(elapsed))
logger.info("-" * 80)
logger.info("=" * 80)
logger.info("Postcheck")
logger.info("-" * 80)
check_intfs_and_nbrs(duthosts, all_cfg_facts, nbrhosts, nbr_macs)
check_ip_fwd(duthosts, all_cfg_facts, nbrhosts)
def neighbor_vm_restore(duthost, nbrhosts, tbinfo):
logger.info("Restoring neighbor VMs for {}".format(duthost))
mg_facts = duthost.get_extended_minigraph_facts(tbinfo)
vm_neighbors = mg_facts['minigraph_neighbors']
if vm_neighbors:
results = parallel_run(neighbor_vm_recover_bgpd, (), {}, nbrhosts.values(), timeout=300)
logger.debug('Results of restoring neighbor VMs: {}'.format(json.dumps(dict(results))))
def _check(*args, **kwargs):
result = parallel_run(_check_processes_on_dut,
args,
kwargs,
duthosts,
timeout=600)
return result.values()
def _check(*args, **kwargs):
result = parallel_run(_check_bgp_on_dut,
args,
kwargs,
duthosts.frontend_nodes,
timeout=600)
return result.values()
def _check(*args, **kwargs):
timeout = 600
# Increase the timeout for multi-asic virtual switch DUT.
for node in duthosts.nodes:
if 'kvm' in node.sonichost.facts['platform'] and node.sonichost.is_multi_asic:
timeout = 1000
break
result = parallel_run(_check_processes_on_dut, args, kwargs, duthosts, timeout=timeout)
return result.values()
def setup(duthosts, tbinfo, localhost):
dut_topo_info = tbinfo['topo']['properties']['topology']['DUT']
if 'vs_chassis' not in dut_topo_info:
return
for dut_index, duthost in enumerate(duthosts):
midplane_ip = dut_topo_info['vs_chassis']['midplane_address'][dut_index]
midplane_port = "eth{}".format(dut_topo_info['vs_chassis']['midplane_port'][dut_index] + 1)
# Check if we already have midplane port configuration in rc.local
try:
t1 = duthost.shell("sudo grep 'sudo ip addr add {}' /etc/rc.local".format(midplane_ip), executable="/bin/bash")
logger.info("On {}, midplane IP configuration is already in /etc/rc.local".format(duthost.hostname))
except:
# edit rc.local to have midplane ip configured on the midplane port as specified in the topology file
# Replace the last line which should start with 'exit 0' with lines to configure and bring up midplane interface followed by exit 0
logger.info("midplane IP configuration is already in /etc/rc.local not in {}, adding it".format(duthost.hostname))
duthost.command("sudo sed -i 's/^exit 0/\\nsudo ip addr add {}\/24 dev {}\\nsudo ifconfig {} up\\n\\nexit 0/' /etc/rc.local".format(
midplane_ip, midplane_port, midplane_port))
# Add chassis_db
logger.info("Adding chassisdb.conf on {}".format(duthost.hostname))
chassis_db_ip = dut_topo_info['vs_chassis']['chassis_db_ip']
duthost.shell("sudo echo 'chassis_db_address={}' > /tmp/chassisdb.conf".format(chassis_db_ip), executable="/bin/bash")
if duthost.is_supervisor_node():
logger.info ("{} is supervisor card, adding config to start chassisdb in chassisdb.conf".format(duthost.hostname))
duthost.shell("echo 'start_chassis_db=1' >> /tmp/chassisdb.conf", executable="/bin/bash")
duthost.shell("echo 'lag_id_start=1' >> /tmp/chassisdb.conf", executable="/bin/bash")
duthost.shell("echo 'lag_id_end=512' >> /tmp/chassisdb.conf", executable="/bin/bash")
duthost.command("sudo cp /tmp/chassisdb.conf /etc/sonic/")
duthost.command("sudo cp /tmp/chassisdb.conf /usr/share/sonic/device/x86_64-kvm_x86_64-r0/")
# scp the config_db's for each card
logger.info("")
BASE_DIR = os.path.dirname(os.path.realpath(__file__))
src_cfg_path = os.path.join(BASE_DIR, "vs_voq_cfgs", "{}_config_db.json".format(duthost.hostname))
dst_cfg_path = os.path.join(os.sep, "tmp", "config_db.json")
logger.info("Copying {} to /etc/sonic/config_db.json on {}".format(src_cfg_path, duthost.hostname))
duthost.copy(src=src_cfg_path, dest=dst_cfg_path)
duthost.command("sudo cp {} {}".format(dst_cfg_path, "/etc/sonic/config_db.json"))
logger.info ("Rebooting all the DUTs in parallel")
parallel_run(reboot_duts, [localhost], {}, duthosts, timeout=240)
def nbr_macs(nbrhosts):
"""
Fixture to get all the neighbor mac addresses in parallel.
Args:
nbrhosts:
Returns:
Dictionary of MAC addresses of neighbor VMS, dict[vm_name][interface_name] = "mac address"
"""
results = {}
logger.debug("Get MACS for all neighbor hosts.")
parallel_run(_get_nbr_macs, [nbrhosts], results, nbrhosts.keys(), timeout=120)
for res in results['results']:
logger.info("parallel_results %s = %s", res, results['results'][res])
return results['results']
def check_routes_on_neighbors_empty_allow_list(self, nbrhosts, setup, permit=True):
other_neighbors = setup['other_neighbors']
@reset_ansible_local_tmp
def check_other_neigh(nbrhosts, permit, node=None, results=None):
logger.info('Checking routes on {}'.format(node))
prefix_results = []
for list_name, prefixes in PREFIX_LISTS.items():
for prefix in prefixes:
prefix_result = {'failed': False, 'prefix': prefix, 'reasons': []}
neigh_route = nbrhosts[node]['host'].get_route(prefix)['vrfs']['default']['bgpRouteEntries']
if permit: # default_action=='permit'
# All routes should be forwarded
if prefix not in neigh_route:
prefix_result['failed'] = True
prefix_result['reasons'].append('Route {} not found on {}'.format(prefix, node))
else:
communityList = neigh_route[prefix]['bgpRoutePaths'][0]['routeDetail']['communityList']
# Should add drop_community to all routes
if DROP_COMMUNITY not in communityList:
prefix_result['failed'] = True
prefix_result['reasons']\
.append('When default_action="permit" and allow list is empty, should add '
'drop_community to all routes. route={}, node={}'.format(prefix, node))
# Should keep original route community
if 'COMMUNITY' in list_name:
if TEST_COMMUNITY not in communityList:
prefix_result['failed'] = True
prefix_result['reasons']\
.append('When default_action="permit" and allow list is empty, should keep '
'the original community {}, route={}, node={}'
.format(TEST_COMMUNITY, prefix, node))
else: # default_action=='deny'
# All routes should be dropped
if prefix in neigh_route:
prefix_result['failed'] = True
prefix_result['reasons'].append('When default_action="deny" and allow list is empty, '
'all routes should be dropped. route={}, node={}'.format(prefix, node))
prefix_results.append(prefix_result)
results[node] = prefix_results
results = parallel_run(check_other_neigh, (nbrhosts, permit), {}, other_neighbors, timeout=180)
self.check_results(results)
def nbr_macs(nbrhosts):
"""
Fixture to get all the neighbor mac addresses in parallel.
Args:
nbrhosts:
Returns:
Dictionary of MAC addresses of neighbor VMS, dict[vm_name][interface_name] = "mac address"
"""
logger.debug("Get MACS for all neighbor hosts.")
results = parallel_run(_get_nbr_macs, [nbrhosts], {},
nbrhosts.keys(),
timeout=120)
# result is DictProxy. Iterate it by using keys().
for res in results.keys():
logger.info("parallel_results %s = %s", res, results[res])
return results
def teardown(duthosts, nbrhosts, all_cfg_facts):
"""
Teardown fixture for the module, restore bgp neighbors.
"""
yield
# cleanup
# restore neighbors on duts
@reset_ansible_local_tmp
def enable_dut_bgp_neighs(cfg_facts, node=None, results=None):
"""Target function to enable BGP neighbors on the sonic hosts..
Args:
cfg_facts: Instance of all_cfg_facts fixture from voq/conftest.py
node (object, optional): A value item of the dict type fixture 'nbrhosts'. Defaults to None.
results (Proxy to shared dict, optional): An instance of multiprocessing.Manager().dict(). Proxy to a dict
shared by all processes for returning execution results. Defaults to None.
"""
if node is None or results is None:
logger.error('Missing kwarg "node" or "results"')
return
node_results = []
for asic in node.asics:
asic_cfg_facts = cfg_facts[node.hostname][asic.asic_index]['ansible_facts']
if 'BGP_NEIGHBOR' not in asic_cfg_facts:
continue
logger.info('enable neighbors {} on dut host {}'.format(asic_cfg_facts['BGP_NEIGHBOR'], node.hostname))
asnum = asic_cfg_facts['DEVICE_METADATA']['localhost']['bgp_asn']
for neighbor in asic_cfg_facts['BGP_NEIGHBOR']:
logger.info(
"Startup neighbor: {} on host {} asic {}".format(neighbor, node.hostname, asic.asic_index))
node_results.append(node.command("sudo config bgp startup neighbor {}".format(neighbor)))
if node.is_multi_asic:
node_results.append(node.command(
"docker exec bgp{} vtysh -c \"config t\" -c \"router bgp {}\" -c \"no neighbor {} shutdown\"".format(
asic.asic_index, asnum, neighbor)))
else:
node_results.append(node.command(
"docker exec bgp vtysh -c \"config t\" -c \"router bgp {}\" -c \"no neighbor {} shutdown\"".format(
asnum, neighbor)))
results[node.hostname] = node_results
# restore neighbors on vms
@reset_ansible_local_tmp
def enable_nbr_bgp_neighs(node=None, results=None):
"""Target function to enable BGP neighbors on VMs.
Args:
node (object, optional): A value item of the dict type fixture 'nbrhosts'. Defaults to None.
results (Proxy to shared dict, optional): An instance of multiprocessing.Manager().dict(). Proxy to a dict
shared by all processes for returning execution results. Defaults to None.
"""
if node is None or results is None:
logger.error('Missing kwarg "node" or "results"')
return
node_results = []
logger.info(
'enable neighbors {} on neighbor host {}'.format(node['conf']['bgp']['peers'], node['host'].hostname))
for peer in node['conf']['bgp']['peers']:
for neighbor in node['conf']['bgp']['peers'][peer]:
try:
node_results.append(node['host'].eos_config(
lines=["no neighbor %s shutdown" % neighbor],
parents=['router bgp {}'.format(node['conf']['bgp']['asn'])])
)
if ":" in neighbor:
node_results.append(node['host'].eos_config(
lines=["no ipv6 route ::/0 %s " % neighbor])
)
else:
node_results.append(node['host'].eos_config(
lines=["no ip route 0.0.0.0/0 %s " % neighbor],
)
)
except Exception:
logger.warning("Enable of neighbor on VM: %s failed, retrying", node['host'].hostname)
time.sleep(10)
node_results.append(node['host'].eos_config(
lines=["no neighbor %s shutdown" % neighbor],
parents=['router bgp {}'.format(node['conf']['bgp']['asn'])])
)
if ":" in neighbor:
node_results.append(node['host'].eos_config(
lines=["no ipv6 route ::/0 %s " % neighbor],
)
)
else:
node_results.append(node['host'].eos_config(
lines=["no ip route 0.0.0.0/0 %s " % neighbor],
)
)
results[node['host'].hostname] = node_results
try:
parallel_run(enable_dut_bgp_neighs, [all_cfg_facts], {}, duthosts.frontend_nodes, timeout=300)
finally:
parallel_run(enable_nbr_bgp_neighs, [], {}, nbrhosts.values(), timeout=120)
def setup(duthosts, nbrhosts, all_cfg_facts):
"""
Setup fixture to disable all neighbors on DUT and VMs.
Args:
duthosts: Duthosts fixture
nbrhosts: Nbrhosts fixture
all_cfg_facts: all_cfg_facts fixture from voq/conftest.py
"""
@reset_ansible_local_tmp
def disable_dut_bgp_neighs(cfg_facts, node=None, results=None):
"""Target function do disable BGP neighbors on sonic DUTs.
Args:
cfg_facts: instance of fixture from voq/conftest.py
node (object, optional): A value item of the dict type fixture 'nbrhosts'. Defaults to None.
results (Proxy to shared dict, optional): An instance of multiprocessing.Manager().dict(). Proxy to a dict
shared by all processes for returning execution results. Defaults to None
"""
if node is None or results is None:
logger.error('Missing kwarg "node" or "results"')
return
node_results = []
for asic in node.asics:
asic_cfg_facts = cfg_facts[node.hostname][asic.asic_index]['ansible_facts']
if 'BGP_NEIGHBOR' not in asic_cfg_facts:
continue
for neighbor in asic_cfg_facts['BGP_NEIGHBOR']:
logger.info(
"Shut down neighbor: {} on host {} asic {}".format(neighbor, node.hostname, asic.asic_index))
node_results.append(node.command("sudo config bgp shutdown neighbor {}".format(neighbor)))
results[node.hostname] = node_results
parallel_run(disable_dut_bgp_neighs, [all_cfg_facts], {}, duthosts.frontend_nodes, timeout=120)
# disable bgp neighbors on vms
@reset_ansible_local_tmp
def disable_nbr_bgp_neighs(node=None, results=None):
"""Target function to disable bgp neighbors on VMS.
Args:
node (object, optional): A value item of the dict type fixture 'nbrhosts'. Defaults to None.
results (Proxy to shared dict, optional): An instance of multiprocessing.Manager().dict(). Proxy to a dict
shared by all processes for returning execution results. Defaults to None.
"""
if node is None or results is None:
logger.error('Missing kwarg "node" or "results"')
return
node_results = []
logger.info(
'disable neighbors {} on neighbor host {}'.format(node['conf']['bgp']['peers'], node['host'].hostname))
for peer in node['conf']['bgp']['peers']:
for neighbor in node['conf']['bgp']['peers'][peer]:
node_results.append(node['host'].eos_config(
lines=["neighbor %s shutdown" % neighbor],
parents=['router bgp {}'.format(node['conf']['bgp']['asn'])],
module_ignore_errors=True)
)
if ":" in neighbor:
node_results.append(node['host'].eos_config(
lines=["ipv6 route ::/0 %s " % neighbor],
module_ignore_errors=True)
)
else:
node_results.append(node['host'].eos_config(
lines=["ip route 0.0.0.0/0 %s " % neighbor],
module_ignore_errors=True)
)
results[node['host'].hostname] = node_results
parallel_run(disable_nbr_bgp_neighs, [], {}, nbrhosts.values(), timeout=120)
logger.info("Poll for routes to be gone.")
endtime = time.time() + 120
for dut in duthosts.frontend_nodes:
for asc in dut.asics:
routes = len(asic_cmd(asc, 'redis-cli -n 0 KEYS ROUTE_TABLE*')['stdout_lines'])
logger.info("Found %d routes in appdb on %s/%s", routes, dut.hostname, asc.asic_index)
while routes > 1000:
time.sleep(5)
routes = len(asic_cmd(asc, 'redis-cli -n 0 KEYS ROUTE_TABLE*')['stdout_lines'])
logger.info("Found %d routes in appdb on %s/%s, polling", routes, dut.hostname, asc.asic_index)
if time.time() > endtime:
break
routes = len(asic_cmd(asc, 'redis-cli -n 1 KEYS *ROUTE_ENTRY*')['stdout_lines'])
logger.info("Found %d routes in asicdb on %s/%s", routes, dut.hostname, asc.asic_index)
while routes > 1000:
time.sleep(5)
routes = len(asic_cmd(asc, 'redis-cli -n 1 KEYS *ROUTE_ENTRY*')['stdout_lines'])
logger.info("Found %d routes in asicdb on %s/%s, polling", routes, dut.hostname, asc.asic_index)
if time.time() > endtime:
break
def _check(*args, **kwargs):
result = parallel_run(_check_services_on_dut, (args),
kwargs,
duthosts,
timeout=SYSTEM_STABILIZE_MAX_TIME)
return result.values()
def check_bbr_route_propagation(duthost,
nbrhosts,
setup,
route,
accepted=True):
def check_tor1(nbrhosts, setup, route):
tor1 = setup['tor1']
# Check route on tor1
logger.info('Check route for prefix {} on {}'.format(
route.prefix, tor1))
tor1_route = nbrhosts[tor1]['host'].get_route(route.prefix)
if not route.prefix in tor1_route['vrfs']['default'][
'bgpRouteEntries'].keys():
logging.warn('No route for {} found on {}'.format(
route.prefix, tor1))
return False
tor1_route_aspath = tor1_route['vrfs']['default']['bgpRouteEntries'][route.prefix]['bgpRoutePaths'][0]\
['asPathEntry']['asPath']
if not tor1_route_aspath == route.aspath:
logging.warn('On {} expected aspath: {}, actual aspath: {}'.format(
tor1, route.aspath, tor1_route_aspath))
return False
return True
def check_dut(duthost,
other_vms,
bgp_neighbors,
setup,
route,
accepted=True):
tor1_asn = setup['tor1_asn']
# Check route on DUT
logger.info('Check route on DUT')
dut_route = duthost.get_route(route.prefix, setup['tor1_namespace'])
if accepted:
if not dut_route:
logging.warn('No route for {} found on DUT'.format(
route.prefix))
return False
dut_route_aspath = dut_route['paths'][0]['aspath']['string']
# Route path from DUT: -> TOR1 -> aspath(other T1 -> DUMMY_ASN1)
dut_route_aspath_expected = '{} {}'.format(tor1_asn, route.aspath)
if not dut_route_aspath == dut_route_aspath_expected:
logging.warn(
'On DUT expected aspath: {}, actual aspath: {}'.format(
dut_route_aspath_expected, dut_route_aspath))
return False
if 'advertisedTo' not in dut_route:
logging.warn("DUT didn't advertise the route")
return False
advertised_to = set(
[bgp_neighbors[_]['name'] for _ in dut_route['advertisedTo']])
for vm in other_vms:
if vm not in advertised_to:
logging.warn("DUT didn't advertise route to neighbor %s" %
vm)
return False
else:
if dut_route:
logging.warn('Prefix {} should not be accepted by DUT'.format(
route.prefix))
return True
# Check route on other VMs
@reset_ansible_local_tmp
def check_other_vms(nbrhosts,
setup,
route,
accepted=True,
node=None,
results=None):
logger.info('Check route {} on {}'.format(str(route), node))
dut_asn = setup['dut_asn']
tor1_asn = setup['tor1_asn']
vm_route = nbrhosts[node]['host'].get_route(route.prefix)
vm_route['failed'] = False
vm_route['message'] = 'Checking route {} on {} passed'.format(
str(route), node)
if accepted:
if route.prefix not in vm_route['vrfs']['default'][
'bgpRouteEntries'].keys():
vm_route['failed'] = True
vm_route['message'] = 'No route for {} found on {}'.format(
route.prefix, node)
else:
tor_route_aspath = vm_route['vrfs']['default']['bgpRouteEntries'][route.prefix]['bgpRoutePaths'][0]\
['asPathEntry']['asPath']
# Route path from other VMs: -> DUT(T1) -> TOR1 -> aspath(other T1 -> DUMMY_ASN1)
tor_route_aspath_expected = '{} {} {}'.format(
dut_asn, tor1_asn, route.aspath)
if tor_route_aspath != tor_route_aspath_expected:
vm_route['failed'] = True
vm_route['message'] = 'On {} expected aspath: {}, actual aspath: {}'\
.format(node, tor_route_aspath_expected, tor_route_aspath)
else:
if route.prefix in vm_route['vrfs']['default'][
'bgpRouteEntries'].keys():
vm_route['failed'] = True
vm_route['message'] = 'No route {} expected on {}'.format(
route.prefix, node)
results[node] = vm_route
other_vms = setup['other_vms']
bgp_neighbors = json.loads(
duthost.shell("sonic-cfggen -d --var-json 'BGP_NEIGHBOR'")['stdout'])
# check tor1
pytest_assert(wait_until(5, 1, check_tor1, nbrhosts, setup, route),
'tor1 check failed')
# check DUT
pytest_assert(
wait_until(5,
1,
check_dut,
duthost,
other_vms,
bgp_neighbors,
setup,
route,
accepted=accepted), 'DUT check failed')
results = parallel_run(check_other_vms, (nbrhosts, setup, route),
{'accepted': accepted},
other_vms,
timeout=120)
failed_results = {}
for node, result in results.items():
if result['failed']:
failed_results[node] = result
pytest_assert(not failed_results, 'Checking route {} failed, failed_results: {}'\
.format(str(route), json.dumps(failed_results, indent=2)))
def setup_bgp_graceful_restart(duthosts, rand_one_dut_hostname, nbrhosts, tbinfo):
duthost = duthosts[rand_one_dut_hostname]
config_facts = duthost.config_facts(host=duthost.hostname, source="running")['ansible_facts']
bgp_neighbors = config_facts.get('BGP_NEIGHBOR', {})
@reset_ansible_local_tmp
def configure_nbr_gr(node=None, results=None):
"""Target function will be used by multiprocessing for configuring VM hosts.
Args:
node (object, optional): A value item of the dict type fixture 'nbrhosts'. Defaults to None.
results (Proxy to shared dict, optional): An instance of multiprocessing.Manager().dict(). Proxy to a dict
shared by all processes for returning execution results. Defaults to None.
"""
if node is None or results is None:
logger.error('Missing kwarg "node" or "results"')
return
node_results = []
logger.info('enable graceful restart on neighbor host {}'.format(node['host'].hostname))
logger.info('bgp asn {}'.format(node['conf']['bgp']['asn']))
node_results.append(node['host'].eos_config(
lines=['graceful-restart restart-time 300'], \
parents=['router bgp {}'.format(node['conf']['bgp']['asn'])], \
module_ignore_errors=True)
)
node_results.append(node['host'].eos_config(
lines=['graceful-restart'], \
parents=['router bgp {}'.format(node['conf']['bgp']['asn']), 'address-family ipv4'], \
module_ignore_errors=True)
)
node_results.append(node['host'].eos_config(
lines=['graceful-restart'], \
parents=['router bgp {}'.format(node['conf']['bgp']['asn']), 'address-family ipv6'], \
module_ignore_errors=True)
)
results[node['host'].hostname] = node_results
@reset_ansible_local_tmp
def restore_nbr_gr(node=None, results=None):
"""Target function will be used by multiprocessing for restoring configuration for the VM hosts.
Args:
node (object, optional): A value item of the dict type fixture 'nbrhosts'. Defaults to None.
results (Proxy to shared dict, optional): An instance of multiprocessing.Manager().dict(). Proxy to a dict
shared by all processes for returning execution results. Defaults to None.
"""
if node is None or results is None:
logger.error('Missing kwarg "node" or "results"')
return
# start bgpd if not started
node_results = []
node['host'].start_bgpd()
logger.info('disable graceful restart on neighbor {}'.format(node))
node_results.append(node['host'].eos_config(
lines=['no graceful-restart'], \
parents=['router bgp {}'.format(node['conf']['bgp']['asn']), 'address-family ipv4'], \
module_ignore_errors=True)
)
node_results.append(node['host'].eos_config(
lines=['no graceful-restart'], \
parents=['router bgp {}'.format(node['conf']['bgp']['asn']), 'address-family ipv6'], \
module_ignore_errors=True)
)
results[node['host'].hostname] = node_results
results = parallel_run(configure_nbr_gr, (), {}, nbrhosts.values(), timeout=120)
check_results(results)
logger.info("bgp neighbors: {}".format(bgp_neighbors.keys()))
res = True
err_msg = ""
if not wait_until(300, 10, 0, duthost.check_bgp_session_state, bgp_neighbors.keys()):
res = False
err_msg = "not all bgp sessions are up after enable graceful restart"
is_backend_topo = "backend" in tbinfo["topo"]["name"]
if not is_backend_topo and res and not wait_until(100, 5, 0, duthost.check_bgp_default_route):
res = False
err_msg = "ipv4 or ipv6 bgp default route not available"
if not res:
# Disable graceful restart in case of failure
parallel_run(restore_nbr_gr, (), {}, nbrhosts.values(), timeout=120)
pytest.fail(err_msg)
yield
results = parallel_run(restore_nbr_gr, (), {}, nbrhosts.values(), timeout=120)
check_results(results)
if not wait_until(300, 10, 0, duthost.check_bgp_session_state, bgp_neighbors.keys()):
pytest.fail("not all bgp sessions are up after disable graceful restart")
def check_bbr_route_propagation(duthost, nbrhosts, setup, route, accepted=True):
tor1 = setup['tor1']
tor1_asn = setup['tor1_asn']
other_vms = setup['other_vms']
# Check route on tor1
logger.info('Check route for prefix {} on {}'.format(route.prefix, tor1))
tor1_route = nbrhosts[tor1]['host'].get_route(route.prefix)
pytest_assert(route.prefix in tor1_route['vrfs']['default']['bgpRouteEntries'].keys(),
'No route for {} found on {}'.format(route.prefix, tor1))
tor1_route_aspath = tor1_route['vrfs']['default']['bgpRouteEntries'][route.prefix]['bgpRoutePaths'][0]\
['asPathEntry']['asPath']
pytest_assert(tor1_route_aspath==route.aspath,
'On {} expected aspath: {}, actual aspath: {}'.format(tor1, route.aspath, tor1_route_aspath))
# Check route on DUT
logger.info('Check route on DUT')
dut_route = duthost.get_route(route.prefix)
if accepted:
pytest_assert(dut_route, 'No route for {} found on DUT'.format(route.prefix))
dut_route_aspath = dut_route['paths'][0]['aspath']['string']
# Route path from DUT: -> TOR1 -> aspath(other T1 -> DUMMY_ASN1)
dut_route_aspath_expected = '{} {}'.format(tor1_asn, route.aspath)
pytest_assert(dut_route_aspath==dut_route_aspath_expected,
'On DUT expected aspath: {}, actual aspath: {}'.format(dut_route_aspath_expected, dut_route_aspath))
else:
pytest_assert(not dut_route, 'Prefix {} should not be accepted by DUT'.format(route.prefix))
# Check route on other VMs
@reset_ansible_local_tmp
def check_other_vms(nbrhosts, setup, route, accepted=True, node=None, results=None):
logger.info('Check route {} on {}'.format(str(route), node))
dut_asn = setup['dut_asn']
tor1_asn = setup['tor1_asn']
vm_route = nbrhosts[node]['host'].get_route(route.prefix)
vm_route = {'failed': False}
vm_route['tor_route'] = vm_route
if accepted:
if route.prefix not in vm_route['vrfs']['default']['bgpRouteEntries'].keys():
vm_route['failed'] = True
vm_route['message'] = 'No route for {} found on {}'.format(route.prefix, node)
else:
tor_route_aspath = vm_route['vrfs']['default']['bgpRouteEntries'][route.prefix]['bgpRoutePaths'][0]\
['asPathEntry']['asPath']
# Route path from other VMs: -> DUT(T1) -> TOR1 -> aspath(other T1 -> DUMMY_ASN1)
tor_route_aspath_expected = '{} {} {}'.format(dut_asn, tor1_asn, route.aspath)
if tor_route_aspath != tor_route_aspath_expected:
vm_route['failed'] = True
vm_route['message'] = 'On {} expected aspath: {}, actual aspath: {}'\
.format(node, tor_route_aspath_expected, tor_route_aspath)
else:
if route.prefix in vm_route['vrfs']['default']['bgpRouteEntries'].keys():
vm_route['failed'] = True
vm_route['message'] = 'No route {} expected on {}'.format(route.prefix, node)
vm_route['message'] = 'Checking route {} on {} passed'.format(str(route), node)
results[node] = vm_route
results = parallel_run(check_other_vms, (nbrhosts, setup, route), {'accepted': accepted}, other_vms, timeout=120)
failed_results = {}
for node, result in results.items():
if result['failed']:
failed_results[node] = result
pytest_assert(not failed_results, 'Checking route {} failed, failed_results: {}'\
.format(str(route), json.dumps(failed_results, indent=2)))
