def loganalyzer(duthosts, request): if request.config.getoption("--disable_loganalyzer" ) or "disable_loganalyzer" in request.keywords: logging.info("Log analyzer is disabled") yield return # Analyze all the duts analyzers = {} parallel_run(analyzer_logrotate, [], {}, duthosts, timeout=120) for duthost in duthosts: analyzers[duthost.hostname] = LogAnalyzer( ansible_host=duthost, marker_prefix=request.node.name) markers = parallel_run(analyzer_add_marker, [analyzers], {}, duthosts, timeout=120) yield analyzers # Skip LogAnalyzer if case is skipped if "rep_call" in request.node.__dict__ and request.node.rep_call.skipped or \ "rep_setup" in request.node.__dict__ and request.node.rep_setup.skipped: return logging.info("Starting to analyse on all DUTs") parallel_run(analyze_logs, [analyzers, markers], {}, duthosts, timeout=120)
def ping_all_dut_local_nbrs(duthosts): """ Pings all neighbors locally attached to each frontend node on the DUT. Args: duthosts: An instance of the duthosts fixture. """ @reset_ansible_local_tmp def _ping_all_local_nbrs(node=None, results=None): if node is None or results is None: logger.error('Missing kwarg "node" or "results"') return node_results = [] logger.info("Pinging neighbors to establish ARP on node: %s", node.hostname) for asic in node.asics: cfg_facts = asic.config_facts(source="persistent")['ansible_facts'] if 'BGP_NEIGHBOR' in cfg_facts: neighs = cfg_facts['BGP_NEIGHBOR'] else: logger.info("No local neighbors for host: %s/%s, skipping", node.hostname, asic.asic_index) continue for neighbor in neighs: node_results.append(sonic_ping(asic, neighbor, verbose=True)) results[node.hostname] = node_results parallel_run(_ping_all_local_nbrs, [], {}, duthosts.frontend_nodes, timeout=120)
def neighbor_vm_restore(duthost, nbrhosts, tbinfo): logger.info("Restoring neighbor VMs for {}".format(duthost)) mg_facts = duthost.get_extended_minigraph_facts(tbinfo) vm_neighbors = mg_facts['minigraph_neighbors'] if vm_neighbors: parallel_run(neighbor_vm_recover_bgpd, (), {}, nbrhosts.values(), timeout=300)
def test_reboot_system(duthosts, localhost, all_cfg_facts, nbrhosts, nbr_macs): """ Tests the system after all cards are explicitly reset, interfaces/neighbors should be in sync across the system. Args: duthosts: duthosts fixture localhost: localhost fixture all_cfg_facts: all_cfg_facts fixture nbrhosts: nbrhosts fixture nbr_macs: nbr_macs fixture """ @reset_ansible_local_tmp def reboot_node(lh, node=None, results=None): node_results = [] node_results.append(reboot(node, lh, wait=600)) results[node.hostname] = node_results logger.info("=" * 80) logger.info("Precheck") logger.info("-" * 80) check_intfs_and_nbrs(duthosts, all_cfg_facts, nbrhosts, nbr_macs) check_ip_fwd(duthosts, all_cfg_facts, nbrhosts) logger.info("=" * 80) logger.info("Coldboot on all nodes") logger.info("-" * 80) t0 = time.time() parallel_run(reboot_node, [localhost], {}, duthosts.nodes, timeout=1000) for node in duthosts.nodes: assert wait_until(300, 20, node.critical_services_fully_started ), "Not all critical services are fully started" poll_bgp_restored(duthosts) t1 = time.time() elapsed = t1 - t0 logger.info("-" * 80) logger.info("Time to reboot and recover: %s seconds.", str(elapsed)) logger.info("-" * 80) logger.info("=" * 80) logger.info("Postcheck") logger.info("-" * 80) check_intfs_and_nbrs(duthosts, all_cfg_facts, nbrhosts, nbr_macs) check_ip_fwd(duthosts, all_cfg_facts, nbrhosts)
def neighbor_vm_restore(duthost, nbrhosts, tbinfo): logger.info("Restoring neighbor VMs for {}".format(duthost)) mg_facts = duthost.get_extended_minigraph_facts(tbinfo) vm_neighbors = mg_facts['minigraph_neighbors'] if vm_neighbors: results = parallel_run(neighbor_vm_recover_bgpd, (), {}, nbrhosts.values(), timeout=300) logger.debug('Results of restoring neighbor VMs: {}'.format(json.dumps(dict(results))))
def _check(*args, **kwargs): result = parallel_run(_check_processes_on_dut, args, kwargs, duthosts, timeout=600) return result.values()
def _check(*args, **kwargs): result = parallel_run(_check_bgp_on_dut, args, kwargs, duthosts.frontend_nodes, timeout=600) return result.values()
def _check(*args, **kwargs): timeout = 600 # Increase the timeout for multi-asic virtual switch DUT. for node in duthosts.nodes: if 'kvm' in node.sonichost.facts['platform'] and node.sonichost.is_multi_asic: timeout = 1000 break result = parallel_run(_check_processes_on_dut, args, kwargs, duthosts, timeout=timeout) return result.values()
def setup(duthosts, tbinfo, localhost): dut_topo_info = tbinfo['topo']['properties']['topology']['DUT'] if 'vs_chassis' not in dut_topo_info: return for dut_index, duthost in enumerate(duthosts): midplane_ip = dut_topo_info['vs_chassis']['midplane_address'][dut_index] midplane_port = "eth{}".format(dut_topo_info['vs_chassis']['midplane_port'][dut_index] + 1) # Check if we already have midplane port configuration in rc.local try: t1 = duthost.shell("sudo grep 'sudo ip addr add {}' /etc/rc.local".format(midplane_ip), executable="/bin/bash") logger.info("On {}, midplane IP configuration is already in /etc/rc.local".format(duthost.hostname)) except: # edit rc.local to have midplane ip configured on the midplane port as specified in the topology file # Replace the last line which should start with 'exit 0' with lines to configure and bring up midplane interface followed by exit 0 logger.info("midplane IP configuration is already in /etc/rc.local not in {}, adding it".format(duthost.hostname)) duthost.command("sudo sed -i 's/^exit 0/\\nsudo ip addr add {}\/24 dev {}\\nsudo ifconfig {} up\\n\\nexit 0/' /etc/rc.local".format( midplane_ip, midplane_port, midplane_port)) # Add chassis_db logger.info("Adding chassisdb.conf on {}".format(duthost.hostname)) chassis_db_ip = dut_topo_info['vs_chassis']['chassis_db_ip'] duthost.shell("sudo echo 'chassis_db_address={}' > /tmp/chassisdb.conf".format(chassis_db_ip), executable="/bin/bash") if duthost.is_supervisor_node(): logger.info ("{} is supervisor card, adding config to start chassisdb in chassisdb.conf".format(duthost.hostname)) duthost.shell("echo 'start_chassis_db=1' >> /tmp/chassisdb.conf", executable="/bin/bash") duthost.shell("echo 'lag_id_start=1' >> /tmp/chassisdb.conf", executable="/bin/bash") duthost.shell("echo 'lag_id_end=512' >> /tmp/chassisdb.conf", executable="/bin/bash") duthost.command("sudo cp /tmp/chassisdb.conf /etc/sonic/") duthost.command("sudo cp /tmp/chassisdb.conf /usr/share/sonic/device/x86_64-kvm_x86_64-r0/") # scp the config_db's for each card logger.info("") BASE_DIR = os.path.dirname(os.path.realpath(__file__)) src_cfg_path = os.path.join(BASE_DIR, "vs_voq_cfgs", "{}_config_db.json".format(duthost.hostname)) dst_cfg_path = os.path.join(os.sep, "tmp", "config_db.json") logger.info("Copying {} to /etc/sonic/config_db.json on {}".format(src_cfg_path, duthost.hostname)) duthost.copy(src=src_cfg_path, dest=dst_cfg_path) duthost.command("sudo cp {} {}".format(dst_cfg_path, "/etc/sonic/config_db.json")) logger.info ("Rebooting all the DUTs in parallel") parallel_run(reboot_duts, [localhost], {}, duthosts, timeout=240)
def nbr_macs(nbrhosts): """ Fixture to get all the neighbor mac addresses in parallel. Args: nbrhosts: Returns: Dictionary of MAC addresses of neighbor VMS, dict[vm_name][interface_name] = "mac address" """ results = {} logger.debug("Get MACS for all neighbor hosts.") parallel_run(_get_nbr_macs, [nbrhosts], results, nbrhosts.keys(), timeout=120) for res in results['results']: logger.info("parallel_results %s = %s", res, results['results'][res]) return results['results']
def check_routes_on_neighbors_empty_allow_list(self, nbrhosts, setup, permit=True): other_neighbors = setup['other_neighbors'] @reset_ansible_local_tmp def check_other_neigh(nbrhosts, permit, node=None, results=None): logger.info('Checking routes on {}'.format(node)) prefix_results = [] for list_name, prefixes in PREFIX_LISTS.items(): for prefix in prefixes: prefix_result = {'failed': False, 'prefix': prefix, 'reasons': []} neigh_route = nbrhosts[node]['host'].get_route(prefix)['vrfs']['default']['bgpRouteEntries'] if permit: # default_action=='permit' # All routes should be forwarded if prefix not in neigh_route: prefix_result['failed'] = True prefix_result['reasons'].append('Route {} not found on {}'.format(prefix, node)) else: communityList = neigh_route[prefix]['bgpRoutePaths'][0]['routeDetail']['communityList'] # Should add drop_community to all routes if DROP_COMMUNITY not in communityList: prefix_result['failed'] = True prefix_result['reasons']\ .append('When default_action="permit" and allow list is empty, should add ' 'drop_community to all routes. route={}, node={}'.format(prefix, node)) # Should keep original route community if 'COMMUNITY' in list_name: if TEST_COMMUNITY not in communityList: prefix_result['failed'] = True prefix_result['reasons']\ .append('When default_action="permit" and allow list is empty, should keep ' 'the original community {}, route={}, node={}' .format(TEST_COMMUNITY, prefix, node)) else: # default_action=='deny' # All routes should be dropped if prefix in neigh_route: prefix_result['failed'] = True prefix_result['reasons'].append('When default_action="deny" and allow list is empty, ' 'all routes should be dropped. route={}, node={}'.format(prefix, node)) prefix_results.append(prefix_result) results[node] = prefix_results results = parallel_run(check_other_neigh, (nbrhosts, permit), {}, other_neighbors, timeout=180) self.check_results(results)
def nbr_macs(nbrhosts): """ Fixture to get all the neighbor mac addresses in parallel. Args: nbrhosts: Returns: Dictionary of MAC addresses of neighbor VMS, dict[vm_name][interface_name] = "mac address" """ logger.debug("Get MACS for all neighbor hosts.") results = parallel_run(_get_nbr_macs, [nbrhosts], {}, nbrhosts.keys(), timeout=120) # result is DictProxy. Iterate it by using keys(). for res in results.keys(): logger.info("parallel_results %s = %s", res, results[res]) return results
def teardown(duthosts, nbrhosts, all_cfg_facts): """ Teardown fixture for the module, restore bgp neighbors. """ yield # cleanup # restore neighbors on duts @reset_ansible_local_tmp def enable_dut_bgp_neighs(cfg_facts, node=None, results=None): """Target function to enable BGP neighbors on the sonic hosts.. Args: cfg_facts: Instance of all_cfg_facts fixture from voq/conftest.py node (object, optional): A value item of the dict type fixture 'nbrhosts'. Defaults to None. results (Proxy to shared dict, optional): An instance of multiprocessing.Manager().dict(). Proxy to a dict shared by all processes for returning execution results. Defaults to None. """ if node is None or results is None: logger.error('Missing kwarg "node" or "results"') return node_results = [] for asic in node.asics: asic_cfg_facts = cfg_facts[node.hostname][asic.asic_index]['ansible_facts'] if 'BGP_NEIGHBOR' not in asic_cfg_facts: continue logger.info('enable neighbors {} on dut host {}'.format(asic_cfg_facts['BGP_NEIGHBOR'], node.hostname)) asnum = asic_cfg_facts['DEVICE_METADATA']['localhost']['bgp_asn'] for neighbor in asic_cfg_facts['BGP_NEIGHBOR']: logger.info( "Startup neighbor: {} on host {} asic {}".format(neighbor, node.hostname, asic.asic_index)) node_results.append(node.command("sudo config bgp startup neighbor {}".format(neighbor))) if node.is_multi_asic: node_results.append(node.command( "docker exec bgp{} vtysh -c \"config t\" -c \"router bgp {}\" -c \"no neighbor {} shutdown\"".format( asic.asic_index, asnum, neighbor))) else: node_results.append(node.command( "docker exec bgp vtysh -c \"config t\" -c \"router bgp {}\" -c \"no neighbor {} shutdown\"".format( asnum, neighbor))) results[node.hostname] = node_results # restore neighbors on vms @reset_ansible_local_tmp def enable_nbr_bgp_neighs(node=None, results=None): """Target function to enable BGP neighbors on VMs. Args: node (object, optional): A value item of the dict type fixture 'nbrhosts'. Defaults to None. results (Proxy to shared dict, optional): An instance of multiprocessing.Manager().dict(). Proxy to a dict shared by all processes for returning execution results. Defaults to None. """ if node is None or results is None: logger.error('Missing kwarg "node" or "results"') return node_results = [] logger.info( 'enable neighbors {} on neighbor host {}'.format(node['conf']['bgp']['peers'], node['host'].hostname)) for peer in node['conf']['bgp']['peers']: for neighbor in node['conf']['bgp']['peers'][peer]: try: node_results.append(node['host'].eos_config( lines=["no neighbor %s shutdown" % neighbor], parents=['router bgp {}'.format(node['conf']['bgp']['asn'])]) ) if ":" in neighbor: node_results.append(node['host'].eos_config( lines=["no ipv6 route ::/0 %s " % neighbor]) ) else: node_results.append(node['host'].eos_config( lines=["no ip route 0.0.0.0/0 %s " % neighbor], ) ) except Exception: logger.warning("Enable of neighbor on VM: %s failed, retrying", node['host'].hostname) time.sleep(10) node_results.append(node['host'].eos_config( lines=["no neighbor %s shutdown" % neighbor], parents=['router bgp {}'.format(node['conf']['bgp']['asn'])]) ) if ":" in neighbor: node_results.append(node['host'].eos_config( lines=["no ipv6 route ::/0 %s " % neighbor], ) ) else: node_results.append(node['host'].eos_config( lines=["no ip route 0.0.0.0/0 %s " % neighbor], ) ) results[node['host'].hostname] = node_results try: parallel_run(enable_dut_bgp_neighs, [all_cfg_facts], {}, duthosts.frontend_nodes, timeout=300) finally: parallel_run(enable_nbr_bgp_neighs, [], {}, nbrhosts.values(), timeout=120)
def setup(duthosts, nbrhosts, all_cfg_facts): """ Setup fixture to disable all neighbors on DUT and VMs. Args: duthosts: Duthosts fixture nbrhosts: Nbrhosts fixture all_cfg_facts: all_cfg_facts fixture from voq/conftest.py """ @reset_ansible_local_tmp def disable_dut_bgp_neighs(cfg_facts, node=None, results=None): """Target function do disable BGP neighbors on sonic DUTs. Args: cfg_facts: instance of fixture from voq/conftest.py node (object, optional): A value item of the dict type fixture 'nbrhosts'. Defaults to None. results (Proxy to shared dict, optional): An instance of multiprocessing.Manager().dict(). Proxy to a dict shared by all processes for returning execution results. Defaults to None """ if node is None or results is None: logger.error('Missing kwarg "node" or "results"') return node_results = [] for asic in node.asics: asic_cfg_facts = cfg_facts[node.hostname][asic.asic_index]['ansible_facts'] if 'BGP_NEIGHBOR' not in asic_cfg_facts: continue for neighbor in asic_cfg_facts['BGP_NEIGHBOR']: logger.info( "Shut down neighbor: {} on host {} asic {}".format(neighbor, node.hostname, asic.asic_index)) node_results.append(node.command("sudo config bgp shutdown neighbor {}".format(neighbor))) results[node.hostname] = node_results parallel_run(disable_dut_bgp_neighs, [all_cfg_facts], {}, duthosts.frontend_nodes, timeout=120) # disable bgp neighbors on vms @reset_ansible_local_tmp def disable_nbr_bgp_neighs(node=None, results=None): """Target function to disable bgp neighbors on VMS. Args: node (object, optional): A value item of the dict type fixture 'nbrhosts'. Defaults to None. results (Proxy to shared dict, optional): An instance of multiprocessing.Manager().dict(). Proxy to a dict shared by all processes for returning execution results. Defaults to None. """ if node is None or results is None: logger.error('Missing kwarg "node" or "results"') return node_results = [] logger.info( 'disable neighbors {} on neighbor host {}'.format(node['conf']['bgp']['peers'], node['host'].hostname)) for peer in node['conf']['bgp']['peers']: for neighbor in node['conf']['bgp']['peers'][peer]: node_results.append(node['host'].eos_config( lines=["neighbor %s shutdown" % neighbor], parents=['router bgp {}'.format(node['conf']['bgp']['asn'])], module_ignore_errors=True) ) if ":" in neighbor: node_results.append(node['host'].eos_config( lines=["ipv6 route ::/0 %s " % neighbor], module_ignore_errors=True) ) else: node_results.append(node['host'].eos_config( lines=["ip route 0.0.0.0/0 %s " % neighbor], module_ignore_errors=True) ) results[node['host'].hostname] = node_results parallel_run(disable_nbr_bgp_neighs, [], {}, nbrhosts.values(), timeout=120) logger.info("Poll for routes to be gone.") endtime = time.time() + 120 for dut in duthosts.frontend_nodes: for asc in dut.asics: routes = len(asic_cmd(asc, 'redis-cli -n 0 KEYS ROUTE_TABLE*')['stdout_lines']) logger.info("Found %d routes in appdb on %s/%s", routes, dut.hostname, asc.asic_index) while routes > 1000: time.sleep(5) routes = len(asic_cmd(asc, 'redis-cli -n 0 KEYS ROUTE_TABLE*')['stdout_lines']) logger.info("Found %d routes in appdb on %s/%s, polling", routes, dut.hostname, asc.asic_index) if time.time() > endtime: break routes = len(asic_cmd(asc, 'redis-cli -n 1 KEYS *ROUTE_ENTRY*')['stdout_lines']) logger.info("Found %d routes in asicdb on %s/%s", routes, dut.hostname, asc.asic_index) while routes > 1000: time.sleep(5) routes = len(asic_cmd(asc, 'redis-cli -n 1 KEYS *ROUTE_ENTRY*')['stdout_lines']) logger.info("Found %d routes in asicdb on %s/%s, polling", routes, dut.hostname, asc.asic_index) if time.time() > endtime: break
def _check(*args, **kwargs): result = parallel_run(_check_services_on_dut, (args), kwargs, duthosts, timeout=SYSTEM_STABILIZE_MAX_TIME) return result.values()
def check_bbr_route_propagation(duthost, nbrhosts, setup, route, accepted=True): def check_tor1(nbrhosts, setup, route): tor1 = setup['tor1'] # Check route on tor1 logger.info('Check route for prefix {} on {}'.format( route.prefix, tor1)) tor1_route = nbrhosts[tor1]['host'].get_route(route.prefix) if not route.prefix in tor1_route['vrfs']['default'][ 'bgpRouteEntries'].keys(): logging.warn('No route for {} found on {}'.format( route.prefix, tor1)) return False tor1_route_aspath = tor1_route['vrfs']['default']['bgpRouteEntries'][route.prefix]['bgpRoutePaths'][0]\ ['asPathEntry']['asPath'] if not tor1_route_aspath == route.aspath: logging.warn('On {} expected aspath: {}, actual aspath: {}'.format( tor1, route.aspath, tor1_route_aspath)) return False return True def check_dut(duthost, other_vms, bgp_neighbors, setup, route, accepted=True): tor1_asn = setup['tor1_asn'] # Check route on DUT logger.info('Check route on DUT') dut_route = duthost.get_route(route.prefix, setup['tor1_namespace']) if accepted: if not dut_route: logging.warn('No route for {} found on DUT'.format( route.prefix)) return False dut_route_aspath = dut_route['paths'][0]['aspath']['string'] # Route path from DUT: -> TOR1 -> aspath(other T1 -> DUMMY_ASN1) dut_route_aspath_expected = '{} {}'.format(tor1_asn, route.aspath) if not dut_route_aspath == dut_route_aspath_expected: logging.warn( 'On DUT expected aspath: {}, actual aspath: {}'.format( dut_route_aspath_expected, dut_route_aspath)) return False if 'advertisedTo' not in dut_route: logging.warn("DUT didn't advertise the route") return False advertised_to = set( [bgp_neighbors[_]['name'] for _ in dut_route['advertisedTo']]) for vm in other_vms: if vm not in advertised_to: logging.warn("DUT didn't advertise route to neighbor %s" % vm) return False else: if dut_route: logging.warn('Prefix {} should not be accepted by DUT'.format( route.prefix)) return True # Check route on other VMs @reset_ansible_local_tmp def check_other_vms(nbrhosts, setup, route, accepted=True, node=None, results=None): logger.info('Check route {} on {}'.format(str(route), node)) dut_asn = setup['dut_asn'] tor1_asn = setup['tor1_asn'] vm_route = nbrhosts[node]['host'].get_route(route.prefix) vm_route['failed'] = False vm_route['message'] = 'Checking route {} on {} passed'.format( str(route), node) if accepted: if route.prefix not in vm_route['vrfs']['default'][ 'bgpRouteEntries'].keys(): vm_route['failed'] = True vm_route['message'] = 'No route for {} found on {}'.format( route.prefix, node) else: tor_route_aspath = vm_route['vrfs']['default']['bgpRouteEntries'][route.prefix]['bgpRoutePaths'][0]\ ['asPathEntry']['asPath'] # Route path from other VMs: -> DUT(T1) -> TOR1 -> aspath(other T1 -> DUMMY_ASN1) tor_route_aspath_expected = '{} {} {}'.format( dut_asn, tor1_asn, route.aspath) if tor_route_aspath != tor_route_aspath_expected: vm_route['failed'] = True vm_route['message'] = 'On {} expected aspath: {}, actual aspath: {}'\ .format(node, tor_route_aspath_expected, tor_route_aspath) else: if route.prefix in vm_route['vrfs']['default'][ 'bgpRouteEntries'].keys(): vm_route['failed'] = True vm_route['message'] = 'No route {} expected on {}'.format( route.prefix, node) results[node] = vm_route other_vms = setup['other_vms'] bgp_neighbors = json.loads( duthost.shell("sonic-cfggen -d --var-json 'BGP_NEIGHBOR'")['stdout']) # check tor1 pytest_assert(wait_until(5, 1, check_tor1, nbrhosts, setup, route), 'tor1 check failed') # check DUT pytest_assert( wait_until(5, 1, check_dut, duthost, other_vms, bgp_neighbors, setup, route, accepted=accepted), 'DUT check failed') results = parallel_run(check_other_vms, (nbrhosts, setup, route), {'accepted': accepted}, other_vms, timeout=120) failed_results = {} for node, result in results.items(): if result['failed']: failed_results[node] = result pytest_assert(not failed_results, 'Checking route {} failed, failed_results: {}'\ .format(str(route), json.dumps(failed_results, indent=2)))
def setup_bgp_graceful_restart(duthosts, rand_one_dut_hostname, nbrhosts, tbinfo): duthost = duthosts[rand_one_dut_hostname] config_facts = duthost.config_facts(host=duthost.hostname, source="running")['ansible_facts'] bgp_neighbors = config_facts.get('BGP_NEIGHBOR', {}) @reset_ansible_local_tmp def configure_nbr_gr(node=None, results=None): """Target function will be used by multiprocessing for configuring VM hosts. Args: node (object, optional): A value item of the dict type fixture 'nbrhosts'. Defaults to None. results (Proxy to shared dict, optional): An instance of multiprocessing.Manager().dict(). Proxy to a dict shared by all processes for returning execution results. Defaults to None. """ if node is None or results is None: logger.error('Missing kwarg "node" or "results"') return node_results = [] logger.info('enable graceful restart on neighbor host {}'.format(node['host'].hostname)) logger.info('bgp asn {}'.format(node['conf']['bgp']['asn'])) node_results.append(node['host'].eos_config( lines=['graceful-restart restart-time 300'], \ parents=['router bgp {}'.format(node['conf']['bgp']['asn'])], \ module_ignore_errors=True) ) node_results.append(node['host'].eos_config( lines=['graceful-restart'], \ parents=['router bgp {}'.format(node['conf']['bgp']['asn']), 'address-family ipv4'], \ module_ignore_errors=True) ) node_results.append(node['host'].eos_config( lines=['graceful-restart'], \ parents=['router bgp {}'.format(node['conf']['bgp']['asn']), 'address-family ipv6'], \ module_ignore_errors=True) ) results[node['host'].hostname] = node_results @reset_ansible_local_tmp def restore_nbr_gr(node=None, results=None): """Target function will be used by multiprocessing for restoring configuration for the VM hosts. Args: node (object, optional): A value item of the dict type fixture 'nbrhosts'. Defaults to None. results (Proxy to shared dict, optional): An instance of multiprocessing.Manager().dict(). Proxy to a dict shared by all processes for returning execution results. Defaults to None. """ if node is None or results is None: logger.error('Missing kwarg "node" or "results"') return # start bgpd if not started node_results = [] node['host'].start_bgpd() logger.info('disable graceful restart on neighbor {}'.format(node)) node_results.append(node['host'].eos_config( lines=['no graceful-restart'], \ parents=['router bgp {}'.format(node['conf']['bgp']['asn']), 'address-family ipv4'], \ module_ignore_errors=True) ) node_results.append(node['host'].eos_config( lines=['no graceful-restart'], \ parents=['router bgp {}'.format(node['conf']['bgp']['asn']), 'address-family ipv6'], \ module_ignore_errors=True) ) results[node['host'].hostname] = node_results results = parallel_run(configure_nbr_gr, (), {}, nbrhosts.values(), timeout=120) check_results(results) logger.info("bgp neighbors: {}".format(bgp_neighbors.keys())) res = True err_msg = "" if not wait_until(300, 10, 0, duthost.check_bgp_session_state, bgp_neighbors.keys()): res = False err_msg = "not all bgp sessions are up after enable graceful restart" is_backend_topo = "backend" in tbinfo["topo"]["name"] if not is_backend_topo and res and not wait_until(100, 5, 0, duthost.check_bgp_default_route): res = False err_msg = "ipv4 or ipv6 bgp default route not available" if not res: # Disable graceful restart in case of failure parallel_run(restore_nbr_gr, (), {}, nbrhosts.values(), timeout=120) pytest.fail(err_msg) yield results = parallel_run(restore_nbr_gr, (), {}, nbrhosts.values(), timeout=120) check_results(results) if not wait_until(300, 10, 0, duthost.check_bgp_session_state, bgp_neighbors.keys()): pytest.fail("not all bgp sessions are up after disable graceful restart")
def check_bbr_route_propagation(duthost, nbrhosts, setup, route, accepted=True): tor1 = setup['tor1'] tor1_asn = setup['tor1_asn'] other_vms = setup['other_vms'] # Check route on tor1 logger.info('Check route for prefix {} on {}'.format(route.prefix, tor1)) tor1_route = nbrhosts[tor1]['host'].get_route(route.prefix) pytest_assert(route.prefix in tor1_route['vrfs']['default']['bgpRouteEntries'].keys(), 'No route for {} found on {}'.format(route.prefix, tor1)) tor1_route_aspath = tor1_route['vrfs']['default']['bgpRouteEntries'][route.prefix]['bgpRoutePaths'][0]\ ['asPathEntry']['asPath'] pytest_assert(tor1_route_aspath==route.aspath, 'On {} expected aspath: {}, actual aspath: {}'.format(tor1, route.aspath, tor1_route_aspath)) # Check route on DUT logger.info('Check route on DUT') dut_route = duthost.get_route(route.prefix) if accepted: pytest_assert(dut_route, 'No route for {} found on DUT'.format(route.prefix)) dut_route_aspath = dut_route['paths'][0]['aspath']['string'] # Route path from DUT: -> TOR1 -> aspath(other T1 -> DUMMY_ASN1) dut_route_aspath_expected = '{} {}'.format(tor1_asn, route.aspath) pytest_assert(dut_route_aspath==dut_route_aspath_expected, 'On DUT expected aspath: {}, actual aspath: {}'.format(dut_route_aspath_expected, dut_route_aspath)) else: pytest_assert(not dut_route, 'Prefix {} should not be accepted by DUT'.format(route.prefix)) # Check route on other VMs @reset_ansible_local_tmp def check_other_vms(nbrhosts, setup, route, accepted=True, node=None, results=None): logger.info('Check route {} on {}'.format(str(route), node)) dut_asn = setup['dut_asn'] tor1_asn = setup['tor1_asn'] vm_route = nbrhosts[node]['host'].get_route(route.prefix) vm_route = {'failed': False} vm_route['tor_route'] = vm_route if accepted: if route.prefix not in vm_route['vrfs']['default']['bgpRouteEntries'].keys(): vm_route['failed'] = True vm_route['message'] = 'No route for {} found on {}'.format(route.prefix, node) else: tor_route_aspath = vm_route['vrfs']['default']['bgpRouteEntries'][route.prefix]['bgpRoutePaths'][0]\ ['asPathEntry']['asPath'] # Route path from other VMs: -> DUT(T1) -> TOR1 -> aspath(other T1 -> DUMMY_ASN1) tor_route_aspath_expected = '{} {} {}'.format(dut_asn, tor1_asn, route.aspath) if tor_route_aspath != tor_route_aspath_expected: vm_route['failed'] = True vm_route['message'] = 'On {} expected aspath: {}, actual aspath: {}'\ .format(node, tor_route_aspath_expected, tor_route_aspath) else: if route.prefix in vm_route['vrfs']['default']['bgpRouteEntries'].keys(): vm_route['failed'] = True vm_route['message'] = 'No route {} expected on {}'.format(route.prefix, node) vm_route['message'] = 'Checking route {} on {} passed'.format(str(route), node) results[node] = vm_route results = parallel_run(check_other_vms, (nbrhosts, setup, route), {'accepted': accepted}, other_vms, timeout=120) failed_results = {} for node, result in results.items(): if result['failed']: failed_results[node] = result pytest_assert(not failed_results, 'Checking route {} failed, failed_results: {}'\ .format(str(route), json.dumps(failed_results, indent=2)))