def check_bbr_route_propagation(duthost, nbrhosts, setup, route, accepted=True): tor1 = setup['tor1'] tor1_asn = setup['tor1_asn'] other_vms = setup['other_vms'] # Check route on tor1 logger.info('Check route for prefix {} on {}'.format(route.prefix, tor1)) tor1_route = nbrhosts[tor1]['host'].get_route(route.prefix) pytest_assert(route.prefix in tor1_route['vrfs']['default']['bgpRouteEntries'].keys(), 'No route for {} found on {}'.format(route.prefix, tor1)) tor1_route_aspath = tor1_route['vrfs']['default']['bgpRouteEntries'][route.prefix]['bgpRoutePaths'][0]\ ['asPathEntry']['asPath'] pytest_assert(tor1_route_aspath==route.aspath, 'On {} expected aspath: {}, actual aspath: {}'.format(tor1, route.aspath, tor1_route_aspath)) # Check route on DUT logger.info('Check route on DUT') dut_route = duthost.get_route(route.prefix) if accepted: pytest_assert(dut_route, 'No route for {} found on DUT'.format(route.prefix)) dut_route_aspath = dut_route['paths'][0]['aspath']['string'] # Route path from DUT: -> TOR1 -> aspath(other T1 -> DUMMY_ASN1) dut_route_aspath_expected = '{} {}'.format(tor1_asn, route.aspath) pytest_assert(dut_route_aspath==dut_route_aspath_expected, 'On DUT expected aspath: {}, actual aspath: {}'.format(dut_route_aspath_expected, dut_route_aspath)) else: pytest_assert(not dut_route, 'Prefix {} should not be accepted by DUT'.format(route.prefix)) # Check route on other VMs @reset_ansible_local_tmp def check_other_vms(nbrhosts, setup, route, accepted=True, node=None, results=None): logger.info('Check route {} on {}'.format(str(route), node)) dut_asn = setup['dut_asn'] tor1_asn = setup['tor1_asn'] vm_route = nbrhosts[node]['host'].get_route(route.prefix) vm_route = {'failed': False} vm_route['tor_route'] = vm_route if accepted: if route.prefix not in vm_route['vrfs']['default']['bgpRouteEntries'].keys(): vm_route['failed'] = True vm_route['message'] = 'No route for {} found on {}'.format(route.prefix, node) else: tor_route_aspath = vm_route['vrfs']['default']['bgpRouteEntries'][route.prefix]['bgpRoutePaths'][0]\ ['asPathEntry']['asPath'] # Route path from other VMs: -> DUT(T1) -> TOR1 -> aspath(other T1 -> DUMMY_ASN1) tor_route_aspath_expected = '{} {} {}'.format(dut_asn, tor1_asn, route.aspath) if tor_route_aspath != tor_route_aspath_expected: vm_route['failed'] = True vm_route['message'] = 'On {} expected aspath: {}, actual aspath: {}'\ .format(node, tor_route_aspath_expected, tor_route_aspath) else: if route.prefix in vm_route['vrfs']['default']['bgpRouteEntries'].keys(): vm_route['failed'] = True vm_route['message'] = 'No route {} expected on {}'.format(route.prefix, node) vm_route['message'] = 'Checking route {} on {} passed'.format(str(route), node) results[node] = vm_route results = parallel_run(check_other_vms, (nbrhosts, setup, route), {'accepted': accepted}, other_vms, timeout=120) failed_results = {} for node, result in results.items(): if result['failed']: failed_results[node] = result pytest_assert(not failed_results, 'Checking route {} failed, failed_results: {}'\ .format(str(route), json.dumps(failed_results, indent=2)))
def check_routes_on_dut(self, duthost, namespace): for prefixes in PREFIX_LISTS.values(): for prefix in prefixes: dut_route = duthost.get_route(prefix, namespace) pytest_assert(dut_route, 'Route {} is not found on DUT'.format(prefix))
def _simulator_clear_flap_counter(interface_name): server_url = url(action=CLEAR_FLAP_COUNTER) mg_facts = duthost.get_extended_minigraph_facts(tbinfo) mbr_index = mg_facts['minigraph_ptf_indices'][interface_name] data = {"port_to_clear": str(mbr_index)} pytest_assert(_post(server_url, data), "Failed to clear flap counter for all ports")
def run_pfc_test(api, testbed_config, port_config_list, conn_data, fanout_data, duthost, dut_port, global_pause, pause_prio_list, test_prio_list, bg_prio_list, prio_dscp_map, test_traffic_pause): """ Run a PFC test Args: api (obj): IXIA session testbed_config (obj): testbed L1/L2/L3 configuration port_config_list (list): list of port configuration conn_data (dict): the dictionary returned by conn_graph_fact. fanout_data (dict): the dictionary returned by fanout_graph_fact. duthost (Ansible host instance): device under test dut_port (str): DUT port to test global_pause (bool): if pause frame is IEEE 802.3X pause pause_prio_list (list): priorities to pause for pause frames test_prio_list (list): priorities of test flows bg_prio_list (list): priorities of background flows prio_dscp_map (dict): Priority vs. DSCP map (key = priority). test_traffic_pause (bool): if test flows are expected to be paused Returns: N/A """ pytest_assert(testbed_config is not None, 'Fail to get L2/3 testbed config') stop_pfcwd(duthost) disable_packet_aging(duthost) """ Get the ID of the port to test """ port_id = get_dut_port_id(dut_hostname=duthost.hostname, dut_port=dut_port, conn_data=conn_data, fanout_data=fanout_data) pytest_assert(port_id is not None, 'Fail to get ID for port {}'.format(dut_port)) """ Rate percent must be an integer """ test_flow_rate_percent = int(TEST_FLOW_AGGR_RATE_PERCENT / len(test_prio_list)) bg_flow_rate_percent = int(BG_FLOW_AGGR_RATE_PERCENT / len(bg_prio_list)) """ Generate traffic config """ flows = __gen_traffic(testbed_config=testbed_config, port_config_list=port_config_list, port_id=port_id, duthost=duthost, pause_flow_name=PAUSE_FLOW_NAME, global_pause=global_pause, pause_prio_list=pause_prio_list, test_flow_name=TEST_FLOW_NAME, test_flow_prio_list=test_prio_list, test_flow_rate_percent=test_flow_rate_percent, bg_flow_name=BG_FLOW_NAME, bg_flow_prio_list=bg_prio_list, bg_flow_rate_percent=bg_flow_rate_percent, data_flow_dur_sec=DATA_FLOW_DURATION_SEC, data_flow_delay_sec=DATA_FLOW_DELAY_SEC, data_pkt_size=DATA_PKT_SIZE, prio_dscp_map=prio_dscp_map) """ Tgen config = testbed config + flow config """ config = testbed_config config.flows = flows """ import json config_json = json.dumps(config, indent=2, default=lambda x: x.__dict__) print(config_json) """ all_flow_names = [flow.name for flow in flows] data_flow_names = [flow.name for flow in flows if PAUSE_FLOW_NAME not in flow.name] """ Run traffic """ flow_stats = __run_traffic(api=api, config=config, data_flow_names=data_flow_names, all_flow_names=all_flow_names, exp_dur_sec=DATA_FLOW_DURATION_SEC+DATA_FLOW_DELAY_SEC) speed_str = config.layer1[0].speed speed_gbps = int(speed_str.split('_')[1]) """ Verify experiment results """ __verify_results(rows=flow_stats, duthost=duthost, pause_flow_name=PAUSE_FLOW_NAME, test_flow_name=TEST_FLOW_NAME, bg_flow_name=BG_FLOW_NAME, data_flow_dur_sec=DATA_FLOW_DURATION_SEC, test_flow_rate_percent=test_flow_rate_percent, bg_flow_rate_percent=bg_flow_rate_percent, data_pkt_size=DATA_PKT_SIZE, speed_gbps=speed_gbps, test_flow_pause=test_traffic_pause, tolerance=TOLERANCE_THRESHOLD)
def verify_port(host_facts, ports): for port in ports: pytest_assert(host_facts[port]['active'], "interface {} is not active".format(port))
def test_pfc_pause_single_lossy_prio_reboot( snappi_api, snappi_testbed_config, conn_graph_facts, fanout_graph_facts, localhost, duthosts, rand_one_dut_hostname, rand_one_dut_portname_oper_up, rand_lossy_prio, all_prio_list, prio_dscp_map, reboot_type): """ Test if PFC will impact a single lossy priority after various kinds of reboots Args: snappi_api (pytest fixture): SNAPPI session snappi_testbed_config (pytest fixture): testbed configuration information conn_graph_facts (pytest fixture): connection graph fanout_graph_facts (pytest fixture): fanout graph localhost (pytest fixture): localhost handle duthosts (pytest fixture): list of DUTs rand_one_dut_hostname (str): hostname of DUT rand_one_dut_portname_oper_up (str): port to test, e.g., 's6100-1|Ethernet0' rand_lossy_prio (str): lossy priority to test, e.g., 's6100-1|2' all_prio_list (pytest fixture): list of all the priorities prio_dscp_map (pytest fixture): priority vs. DSCP map (key = priority). reboot_type (str): reboot type to be issued on the DUT Returns: N/A """ dut_hostname, dut_port = rand_one_dut_portname_oper_up.split('|') dut_hostname2, lossy_prio = rand_lossy_prio.split('|') pytest_require(rand_one_dut_hostname == dut_hostname == dut_hostname2, "Priority and port are not mapped to the expected DUT") testbed_config, port_config_list = snappi_testbed_config duthost = duthosts[rand_one_dut_hostname] lossy_prio = int(lossy_prio) pause_prio_list = [lossy_prio] test_prio_list = [lossy_prio] bg_prio_list = [p for p in all_prio_list] bg_prio_list.remove(lossy_prio) logger.info("Issuing a {} reboot on the dut {}".format( reboot_type, duthost.hostname)) reboot(duthost, localhost, reboot_type=reboot_type) logger.info("Wait until the system is stable") pytest_assert( wait_until(300, 20, 0, duthost.critical_services_fully_started), "Not all critical services are fully started") run_pfc_test(api=snappi_api, testbed_config=testbed_config, port_config_list=port_config_list, conn_data=conn_graph_facts, fanout_data=fanout_graph_facts, duthost=duthost, dut_port=dut_port, global_pause=False, pause_prio_list=pause_prio_list, test_prio_list=test_prio_list, bg_prio_list=bg_prio_list, prio_dscp_map=prio_dscp_map, test_traffic_pause=False)
def testbed_config(conn_graph_facts, fanout_graph_facts, duthost): ixia_fanout = get_peer_ixia_chassis(conn_data=conn_graph_facts, dut_hostname=duthost.hostname) pytest_require(ixia_fanout is not None, skip_message="Cannot find the peer IXIA chassis") ixia_fanout_id = list(fanout_graph_facts.keys()).index(ixia_fanout) ixia_fanout_list = IxiaFanoutManager(fanout_graph_facts) ixia_fanout_list.get_fanout_device_details(device_number=ixia_fanout_id) ixia_ports = ixia_fanout_list.get_ports(peer_device=duthost.hostname) pytest_require(len(ixia_ports) >= 2, skip_message="The test requires at least two ports") rx_id = 0 tx_id = 1 rx_port_location = get_tgen_location(ixia_ports[rx_id]) tx_port_location = get_tgen_location(ixia_ports[tx_id]) rx_port_speed = int(ixia_ports[rx_id]['speed']) tx_port_speed = int(ixia_ports[tx_id]['speed']) pytest_require(rx_port_speed == tx_port_speed, skip_message="Two ports should have the same speed") """ L1 configuration """ rx_port = Port(name='Rx Port', location=rx_port_location) tx_port = Port(name='Tx Port', location=tx_port_location) pfc = Ieee8021qbb(pfc_delay=1, pfc_class_0=0, pfc_class_1=1, pfc_class_2=2, pfc_class_3=3, pfc_class_4=4, pfc_class_5=5, pfc_class_6=6, pfc_class_7=7) flow_ctl = FlowControl(choice=pfc) auto_negotiation = AutoNegotiation(link_training=True, rs_fec=True) l1_config = Layer1(name='L1 config', speed='speed_%d_gbps' % (rx_port_speed / 1000), auto_negotiate=False, auto_negotiation=auto_negotiation, flow_control=flow_ctl, port_names=[tx_port.name, rx_port.name]) config = Config(ports=[tx_port, rx_port], layer1=[l1_config], options=Options(PortOptions(location_preemption=True))) vlan_subnet = get_vlan_subnet(duthost) pytest_assert(vlan_subnet is not None, "Fail to get Vlan subnet information") vlan_ip_addrs = get_addrs_in_subnet(vlan_subnet, 2) gw_addr = vlan_subnet.split('/')[0] prefix = vlan_subnet.split('/')[1] tx_port_ip = vlan_ip_addrs[0] rx_port_ip = vlan_ip_addrs[1] tx_gateway_ip = gw_addr rx_gateway_ip = gw_addr """ L2/L3 configuration """ tx_ipv4 = Ipv4(name='Tx Ipv4', address=Pattern(tx_port_ip), prefix=Pattern(prefix), gateway=Pattern(tx_gateway_ip), ethernet=Ethernet(name='Tx Ethernet')) config.devices.append( Device(name='Tx Device', device_count=1, container_name=tx_port.name, choice=tx_ipv4)) rx_ipv4 = Ipv4(name='Rx Ipv4', address=Pattern(rx_port_ip), prefix=Pattern(prefix), gateway=Pattern(rx_gateway_ip), ethernet=Ethernet(name='Rx Ethernet')) config.devices.append( Device(name='Rx Device', device_count=1, container_name=rx_port.name, choice=rx_ipv4)) return config
def dutQosConfig(self, duthosts, rand_one_dut_hostname, dutConfig, ingressLosslessProfile, ingressLossyProfile, egressLosslessProfile, egressLossyProfile, sharedHeadroomPoolSize, tbinfo): """ Prepares DUT host QoS configuration Args: duthost (AnsibleHost): Device Under Test (DUT) ingressLosslessProfile (Fxiture): ingressLosslessProfile fixture is required to run prior to collecting QoS configuration Returns: QoSConfig (dict): Map containing DUT host QoS configuration """ duthost = duthosts[rand_one_dut_hostname] mgFacts = duthost.get_extended_minigraph_facts(tbinfo) pytest_assert("minigraph_hwsku" in mgFacts, "Could not find DUT SKU") profileName = ingressLosslessProfile["profileName"] if self.isBufferInApplDb(duthost): profile_pattern = "^BUFFER_PROFILE_TABLE\:pg_lossless_(.*)_profile$" else: profile_pattern = "^BUFFER_PROFILE\|pg_lossless_(.*)_profile" m = re.search(profile_pattern, profileName) pytest_assert(m.group(1), "Cannot find port speed/cable length") portSpeedCableLength = m.group(1) qosConfigs = {} with open(r"qos/files/qos.yml") as file: qosConfigs = yaml.load(file, Loader=yaml.FullLoader) vendor = duthost.facts["asic_type"] hostvars = duthost.host.options['variable_manager']._hostvars[ duthost.hostname] dutAsic = None for asic in self.SUPPORTED_ASIC_LIST: vendorAsic = "{0}_{1}_hwskus".format(vendor, asic) if vendorAsic in hostvars.keys( ) and mgFacts["minigraph_hwsku"] in hostvars[vendorAsic]: dutAsic = asic break pytest_assert(dutAsic, "Cannot identify DUT ASIC type") if isMellanoxDevice(duthost): current_file_dir = os.path.dirname(os.path.realpath(__file__)) sub_folder_dir = os.path.join(current_file_dir, "files/mellanox/") if sub_folder_dir not in sys.path: sys.path.append(sub_folder_dir) import qos_param_generator qpm = qos_param_generator.QosParamMellanox( qosConfigs['qos_params']['mellanox'], dutAsic, portSpeedCableLength, dutConfig, ingressLosslessProfile, ingressLossyProfile, egressLosslessProfile, egressLossyProfile, sharedHeadroomPoolSize) qosParams = qpm.run() else: qosParams = qosConfigs['qos_params'][dutAsic] yield { "param": qosParams, "portSpeedCableLength": portSpeedCableLength, }
def send_and_check_mirror_packets(self, setup, mirror_session, ptfadapter, duthost, mirror_packet, src_port=None, dest_ports=None, expect_recv=True, valid_across_namespace=True): if not src_port: src_port = self._get_random_src_port(setup) if not dest_ports: dest_ports = [self._get_monitor_port(setup, mirror_session, duthost)] # In Below logic idea is to send traffic in such a way so that mirror traffic # will need to go across namespaces and within namespace. If source and mirror destination # namespace are different then traffic mirror will go across namespace via (backend asic) # else via same namespace(asic) src_port_namespace = self._get_port_namespace(setup, int(src_port)) dest_ports_namespace = self._get_port_namespace(setup,int (dest_ports[0])) src_port_set = set() # Some of test scenario are not valid across namespaces so test will explicltly pass # valid_across_namespace as False (default is True) if valid_across_namespace == True or src_port_namespace == dest_ports_namespace: src_port_set.add(src_port) # To verify same namespace mirroring we will add destination port also to the Source Port Set if src_port_namespace != dest_ports_namespace: src_port_set.add(dest_ports[0]) expected_mirror_packet_with_ttl = BaseEverflowTest.get_expected_mirror_packet(mirror_session, setup, duthost, mirror_packet, True) expected_mirror_packet_without_ttl = BaseEverflowTest.get_expected_mirror_packet(mirror_session, setup, duthost, mirror_packet, False) # Loop through Source Port Set and send traffic on each source port of the set for src_port in src_port_set: expected_mirror_packet = expected_mirror_packet_with_ttl \ if self._get_port_namespace(setup, int(src_port)) == dest_ports_namespace else expected_mirror_packet_without_ttl ptfadapter.dataplane.flush() testutils.send(ptfadapter, src_port, mirror_packet) if expect_recv: time.sleep(STABILITY_BUFFER) _, received_packet = testutils.verify_packet_any_port( ptfadapter, expected_mirror_packet, ports=dest_ports ) logging.info("Received packet: %s", packet.Ether(received_packet).summary()) inner_packet = self._extract_mirror_payload(received_packet, len(mirror_packet)) logging.info("Received inner packet: %s", inner_packet.summary()) inner_packet = Mask(inner_packet) # For egress mirroring, we expect the DUT to have modified the packet # before forwarding it. Specifically: # # - In L2 the SMAC and DMAC will change. # - In L3 the TTL and checksum will change. # # We know what the TTL and SMAC should be after going through the pipeline, # but DMAC and checksum are trickier. For now, update the TTL and SMAC, and # mask off the DMAC and IP Checksum to verify the packet contents. if self.mirror_type() == "egress": mirror_packet[packet.IP].ttl -= 1 mirror_packet[packet.Ether].src = setup["router_mac"] inner_packet.set_do_not_care_scapy(packet.Ether, "dst") inner_packet.set_do_not_care_scapy(packet.IP, "chksum") logging.info("Expected inner packet: %s", mirror_packet.summary()) pytest_assert(inner_packet.pkt_match(mirror_packet), "Mirror payload does not match received packet") else: testutils.verify_no_packet_any(ptfadapter, expected_mirror_packet, dest_ports)
def test_power_off_reboot(duthosts, enum_rand_one_per_hwsku_hostname, localhost, conn_graph_facts, xcvr_skip_list, pdu_controller, power_off_delay): """ @summary: This test case is to perform reboot via powercycle and check platform status @param duthost: Fixture for DUT AnsibleHost object @param localhost: Fixture for interacting with localhost through ansible @param conn_graph_facts: Fixture parse and return lab connection graph @param xcvr_skip_list: list of DUT's interfaces for which transeiver checks are skipped @param pdu_controller: The python object of psu controller @param power_off_delay: Pytest parameter. The delay between turning off and on the PSU """ duthost = duthosts[enum_rand_one_per_hwsku_hostname] UNSUPPORTED_ASIC_TYPE = ["cisco-8000"] if duthost.facts["asic_type"] in UNSUPPORTED_ASIC_TYPE: pytest.skip( "Skipping test_power_off_reboot. Test unsupported on {} platform". format(duthost.facts["asic_type"])) pdu_ctrl = pdu_controller if pdu_ctrl is None: pytest.skip( "No PSU controller for %s, skip rest of the testing in this case" % duthost.hostname) all_outlets = pdu_ctrl.get_outlet_status() # If PDU supports returning output_watts, making sure that all outlets has power. no_power = [ item for item in all_outlets if int(item.get('output_watts', '1')) == 0 ] pytest_assert(not no_power, "Not all outlets have power output: {}".format(no_power)) # Purpose of this list is to control sequence of turning on PSUs in power off testing. # If there are 2 PSUs, then 3 scenarios would be covered: # 1. Turn off all PSUs, turn on PSU1, then check. # 2. Turn off all PSUs, turn on PSU2, then check. # 3. Turn off all PSUs, turn on one of the PSU, then turn on the other PSU, then check. power_on_seq_list = [] if all_outlets: power_on_seq_list = [[item] for item in all_outlets] power_on_seq_list.append(all_outlets) logging.info("Got all power on sequences {}".format(power_on_seq_list)) poweroff_reboot_kwargs = {"dut": duthost} try: for power_on_seq in power_on_seq_list: poweroff_reboot_kwargs["pdu_ctrl"] = pdu_ctrl poweroff_reboot_kwargs["all_outlets"] = all_outlets poweroff_reboot_kwargs["power_on_seq"] = power_on_seq poweroff_reboot_kwargs["delay_time"] = power_off_delay reboot_and_check(localhost, duthost, conn_graph_facts["device_conn"][duthost.hostname], xcvr_skip_list, REBOOT_TYPE_POWEROFF, _power_off_reboot_helper, poweroff_reboot_kwargs) except Exception as e: logging.debug("Restore power after test failure") for outlet in all_outlets: logging.debug("turning on {}".format(outlet)) pdu_ctrl.turn_on_outlet(outlet) # Sleep 120 for dut to boot up time.sleep(120) wait_critical_processes(duthost) raise e
def test_bgp_multipath_relax(tbinfo, duthosts, rand_one_dut_hostname): duthost = duthosts[rand_one_dut_hostname] logger.info("Starting test_bgp_multipath_relax on topology {}".format( tbinfo['topo']['name'])) topo_config = tbinfo['topo']['properties']['configuration'] bgp_v4nei = get_bgp_v4_neighbors_from_minigraph(duthost, tbinfo) logger.info("bgp_v4nei {}".format(bgp_v4nei)) # get all t0 routers name which has vips defined dut_t0_neigh = get_t0_neigh(tbinfo, topo_config) # get t2 neighbors dut_t2_neigh = get_t2_neigh(tbinfo) if not dut_t0_neigh: pytest.fail("Didn't find multipath t0's") vips_prefix = get_vips_prefix(dut_t0_neigh, topo_config) logger.info("vips_prefix = {}".format(vips_prefix)) # find all paths of the prefix for test vips_t0, vips_asn = get_vips_prefix_paths(dut_t0_neigh, vips_prefix, topo_config) logger.info("vips_t0: {}, vips_asn: {}".format(vips_t0, vips_asn)) pytest_assert( (vips_t0 > 1), "Did not find preconfigured multipath for the vips prefix under test") # Get the route from the DUT for the prefix bgp_route = duthost.bgp_route( prefix=vips_prefix)['ansible_facts']['bgp_route'] logger.info("Bgp route from DUT for prefix {} is {}".format( vips_prefix, bgp_route)) # Verify found vips prefix entry in Sonic bgp routes pytest_assert(bgp_route[vips_prefix]['found'] == True, "BGP route for {} not found".format(vips_prefix)) # Verify total multipath match number of t0 with vips that has prefix for test pytest_assert( int(bgp_route[vips_prefix]['path_num']) == len(vips_t0), "Path number doesnt match the T0s with VIPS") # verify vips asn in each path of installed BGP vips prefix for asn in vips_asn: for aspath in bgp_route[vips_prefix]['aspath']: pytest_assert((str(asn) in aspath)) # gather one t2 neighbor advertised routes to validate routes advertised to t2 are correct with relaxed multipath bgp_route_neiadv = duthost.bgp_route( neighbor=bgp_v4nei[dut_t2_neigh[0]], direction="adv")['ansible_facts']['bgp_route_neiadv'] logger.info( "Bgp neighbor adv from DUT for neigh {} and prefix {} is {}".format( bgp_v4nei[dut_t2_neigh[0]], vips_prefix, bgp_route_neiadv[vips_prefix])) # Verify vips prefix in advertised routes to t2 neighbor pytest_assert(bgp_route_neiadv.has_key(vips_prefix), "{} is not present in bgp neighbor adv".format(vips_prefix)) # vips prefix path has only 2 hops pytest_assert((len(bgp_route_neiadv[vips_prefix]['aspath']) == 2), "vips prefix path doesn't have 2 hops") pytest_assert((int(bgp_route_neiadv[vips_prefix]['aspath'][0]) in vips_t0 and \ int(bgp_route_neiadv[vips_prefix]['aspath'][1]) in vips_asn), "vips_prefix asn doesnt match with bgp route adv")
def test_ro_user_allowed_command(localhost, duthosts, enum_rand_one_per_hwsku_hostname, tacacs_creds, check_tacacs): duthost = duthosts[enum_rand_one_per_hwsku_hostname] dutip = duthost.mgmt_ip # Run as RO and use the commands allowed by the sudoers file commands = { "cat": [ "sudo cat /var/log/syslog", "sudo cat /var/log/syslog.1", "sudo cat /var/log/syslog.2.gz" ], "brctl": ["sudo brctl show"], "docker": [ "sudo docker exec snmp cat /etc/snmp/snmpd.conf", 'sudo docker images --format "table {% raw %}{{.Repository}}\\t{{.Tag}}\\t{{.ID}}\\t{{.Size}}{% endraw %}"', "sudo docker ps", "sudo docker ps -a", ], "lldpctl": ["sudo lldpctl"], "vtysh": [ 'sudo vtysh -c "show version"', 'sudo vtysh -c "show bgp ipv4 summary json"', 'sudo vtysh -c "show bgp ipv6 summary json"' ], "rvtysh": [ 'sudo rvtysh -c "show ip bgp su"', 'sudo rvtysh -n 0 -c "show ip bgp su"' ], "decode-syseeprom": ["sudo decode-syseeprom"], "generate_dump": ['sudo generate_dump -s "5 secs ago"'], "lldpshow": ["sudo lldpshow"], "pcieutil": ["sudo pcieutil check"], "ip": ["sudo ip netns identify 1"], "ipintutil": [ "sudo ipintutil", "sudo ipintutil -a ipv6", "sudo ipintutil -n asic0 -d all", "sudo ipintutil -n asic0 -d all -a ipv6", ], "show": [ "show version", "show interface status", "show interface portchannel", "show ip bgp summary", "show ip bgp neighbors", "show ip bgp network", "show ip interface", "show ipv6 interface", "show ipv6 bgp neighbors", "show ipv6 bgp network", "show lldp table", ], } # NOTE: `sudo tail -F /var/log/syslog` will not exit, not posssible to test here # NOTE: `sudo docker exec bgp cat /etc/quagga/bgpd.conf` can only run on image with quagga # TODO: some commands need further preparation, will enable when runable directly: # sudo sensors # sudo psuutil * # sudo sfputil show for command in commands: if does_command_exist(localhost, dutip, tacacs_creds['tacacs_ro_user'], tacacs_creds['tacacs_ro_user_passwd'], command): for subcommand in commands[command]: allowed = ssh_remote_allow_run( localhost, dutip, tacacs_creds['tacacs_ro_user'], tacacs_creds['tacacs_ro_user_passwd'], subcommand) pytest_assert(allowed, "command '{}' not authorized".format(subcommand)) else: logger.info('"{}" not found on DUT, skipping...'.format(command)) dash_allowed = ssh_remote_allow_run(localhost, dutip, tacacs_creds['tacacs_ro_user'], tacacs_creds['tacacs_ro_user_passwd'], 'sudo sonic-installer list') if not dash_allowed: dash_banned = ssh_remote_ban_run(localhost, dutip, tacacs_creds['tacacs_ro_user'], tacacs_creds['tacacs_ro_user_passwd'], 'sudo sonic-installer list') pytest_assert( dash_banned, "command 'sudo sonic-installer list' should be either allowed or banned" ) underscore_allowed = ssh_remote_allow_run( localhost, dutip, tacacs_creds['tacacs_ro_user'], tacacs_creds['tacacs_ro_user_passwd'], 'sudo sonic_installer list') pytest_assert( underscore_allowed, "command 'sudo sonic_installer list' should be allowed if" " 'sudo sonic-installer list' is banned")
def base_configs(testbed, conn_graph_facts, duthost, lossless_prio_dscp_map, one_hundred_gbe, start_delay, traffic_duration, pause_line_rate, traffic_line_rate, pause_frame_type, frame_size, serializer): for config in one_hundred_gbe: start_delay = start_delay * 1000000000.0 bg_dscp_list = [str(prio) for prio in lossless_prio_dscp_map] test_dscp_list = [str(x) for x in range(64) if x not in bg_dscp_list] tx = config.ports[0] rx = config.ports[1] vlan_subnet = get_vlan_subnet(duthost) pytest_assert(vlan_subnet is not None, "Fail to get Vlan subnet information") vlan_ip_addrs = get_addrs_in_subnet(vlan_subnet, 2) gw_addr = vlan_subnet.split('/')[0] interface_ip_addr = vlan_ip_addrs[0] tx_port_ip = vlan_ip_addrs[1] rx_port_ip = vlan_ip_addrs[0] tx_gateway_ip = gw_addr rx_gateway_ip = gw_addr test_flow_name = 'Test Data' background_flow_name = 'Background Data' test_line_rate = traffic_line_rate background_line_rate = traffic_line_rate pause_line_rate = pause_line_rate pytest_assert( test_line_rate + background_line_rate <= 100, "test_line_rate + background_line_rate should be less than 100") ###################################################################### # Create TX stack configuration ###################################################################### tx_ipv4 = Ipv4(name='Tx Ipv4', address=Pattern(tx_port_ip), prefix=Pattern('24'), gateway=Pattern(tx_gateway_ip), ethernet=Ethernet(name='Tx Ethernet')) tx.devices.append( Device(name='Tx Device', device_count=1, choice=tx_ipv4)) ###################################################################### # Create RX stack configuration ###################################################################### rx_ipv4 = Ipv4(name='Rx Ipv4', address=Pattern(rx_port_ip), prefix=Pattern('24'), gateway=Pattern(rx_gateway_ip), ethernet=Ethernet(name='Rx Ethernet')) rx.devices.append( Device(name='Rx Device', device_count=1, choice=rx_ipv4)) ###################################################################### # Traffic configuration Test data ###################################################################### data_endpoint = DeviceTxRx( tx_device_names=[tx.devices[0].name], rx_device_names=[rx.devices[0].name], ) test_dscp = Priority(Dscp(phb=FieldPattern(choice=test_dscp_list))) test_flow = Flow(name=test_flow_name, tx_rx=TxRx(data_endpoint), packet=[ Header(choice=EthernetHeader()), Header(choice=Ipv4Header(priority=test_dscp)) ], size=Size(frame_size), rate=Rate('line', test_line_rate), duration=Duration( FixedSeconds(seconds=traffic_duration, delay=start_delay, delay_unit='nanoseconds'))) config.flows.append(test_flow) ####################################################################### # Traffic configuration Background data ####################################################################### background_dscp = Priority(Dscp(phb=FieldPattern(choice=bg_dscp_list))) background_flow = Flow( name=background_flow_name, tx_rx=TxRx(data_endpoint), packet=[ Header(choice=EthernetHeader()), Header(choice=Ipv4Header(priority=background_dscp)) ], size=Size(frame_size), rate=Rate('line', background_line_rate), duration=Duration( FixedSeconds(seconds=traffic_duration, delay=start_delay, delay_unit='nanoseconds'))) config.flows.append(background_flow) ####################################################################### # Traffic configuration Pause ####################################################################### pause_endpoint = PortTxRx(tx_port_name='Rx', rx_port_names=['Rx']) if (pause_frame_type == 'priority'): pause = Header( PfcPause( dst=FieldPattern(choice='01:80:C2:00:00:01'), src=FieldPattern(choice='00:00:fa:ce:fa:ce'), class_enable_vector=FieldPattern(choice='E7'), pause_class_0=FieldPattern(choice='ffff'), pause_class_1=FieldPattern(choice='ffff'), pause_class_2=FieldPattern(choice='ffff'), pause_class_3=FieldPattern(choice='0'), pause_class_4=FieldPattern(choice='0'), pause_class_5=FieldPattern(choice='ffff'), pause_class_6=FieldPattern(choice='ffff'), pause_class_7=FieldPattern(choice='ffff'), )) pause_flow = Flow(name='Pause Storm', tx_rx=TxRx(pause_endpoint), packet=[pause], size=Size(64), rate=Rate('line', value=100), duration=Duration( FixedPackets(packets=0, delay=0, delay_unit='nanoseconds'))) elif (pause_frame_type == 'global'): pause = Header( EthernetPause(dst=FieldPattern(choice='01:80:C2:00:00:01'), src=FieldPattern(choice='00:00:fa:ce:fa:ce'))) pause_flow = Flow(name='Pause Storm', tx_rx=TxRx(pause_endpoint), packet=[pause], size=Size(64), rate=Rate('line', value=pause_line_rate), duration=Duration( FixedPackets(packets=0, delay=0, delay_unit='nanoseconds'))) else: pass config.flows.append(pause_flow) return one_hundred_gbe
def test_config_db_parameters(duthost): """Verifies required telemetry parameters from config_db. """ docker_present = verify_telemetry_dockerimage(duthost) if not docker_present: pytest.skip("docker-sonic-telemetry is not part of the image") gnmi = duthost.shell('sonic-db-cli CONFIG_DB HGETALL "TELEMETRY|gnmi"', module_ignore_errors=False)['stdout_lines'] pytest_assert(gnmi is not None, "TELEMETRY|gnmi does not exist in config_db") certs = duthost.shell('sonic-db-cli CONFIG_DB HGETALL "TELEMETRY|certs"', module_ignore_errors=False)['stdout_lines'] pytest_assert(certs is not None, "TELEMETRY|certs does not exist in config_db") d = get_dict_stdout(gnmi, certs) for key, value in d.items(): if str(key) == "port": port_expected = str(TELEMETRY_PORT) pytest_assert( str(value) == port_expected, "'port' value is not '{}'".format(port_expected)) if str(key) == "ca_crt": ca_crt_value_expected = "/etc/sonic/telemetry/dsmsroot.cer" pytest_assert( str(value) == ca_crt_value_expected, "'ca_crt' value is not '{}'".format(ca_crt_value_expected)) if str(key) == "server_key": server_key_expected = "/etc/sonic/telemetry/streamingtelemetryserver.key" pytest_assert( str(value) == server_key_expected, "'server_key' value is not '{}'".format(server_key_expected)) if str(key) == "server_crt": server_crt_expected = "/etc/sonic/telemetry/streamingtelemetryserver.cer" pytest_assert( str(value) == server_crt_expected, "'server_crt' value is not '{}'".format(server_crt_expected))
def test_front_panel_admindown_port(self, duthosts, enum_rand_one_per_hwsku_frontend_hostname, enum_asic_index, all_cfg_facts, setup, teardown, nbrhosts, nbr_macs, established_arp): """ Verify tables, databases, and kernel routes are correctly deleted when the DUT port is admin down/up. Test Steps * Admin down interface on DUT. * On local linecard: * Verify ARP/NDP entries are removed from CLI for neighbors on down port. * Verify table entries in ASIC, AppDb are removed for addresses on down port. * On Supervisor card: * Verify Chassis App DB entry are removed for only the cleared address. Entries for addresses on other line cards should still be present. * On remote linecards: * Verify table entries in ASICDB, APPDB, and host ARP table are removed for cleared addresses. * Verify kernel routes for cleared address are deleted. * Admin interface up, verify recreation after restarting traffic. Args: duthosts: duthosts fixture. enum_rand_one_per_hwsku_frontend_hostname: frontend iteration fixture. enum_asic_index: asic iteration fixture. all_cfg_facts: all_cfg_facts fixture from voq/conftest.py setup: setup fixture for this module. nbrhosts: nbrhosts fixture. established_arp: Fixture to establish ARP to all neighbors. """ per_host = duthosts[enum_rand_one_per_hwsku_frontend_hostname] asic = per_host.asics[enum_asic_index if enum_asic_index is not None else 0] cfg_facts = all_cfg_facts[per_host.hostname][asic.asic_index]['ansible_facts'] if 'BGP_NEIGHBOR' in cfg_facts: neighs = cfg_facts['BGP_NEIGHBOR'] else: logger.info("No local neighbors for host: %s/%s, skipping", per_host.hostname, asic.asic_index) return intfs, intfs_to_test = pick_ports(cfg_facts) logger.info("Will test interfaces: %s", intfs_to_test) for intf in intfs_to_test: local_ips = [i.split("/")[0] for i in intfs[intf].keys()] # [u'2064:100::2/64', u'100.0.0.2/24'] neighbors = [n for n in neighs if neighs[n]['local_addr'] in local_ips] logger.info("Testing neighbors: %s on intf: %s", neighbors, intf) self.localport_admindown(per_host, asic, intf) try: check_neighbors_are_gone(duthosts, all_cfg_facts, per_host, asic, neighbors) finally: self.localport_adminup(per_host, asic, intf) for neighbor in neighbors: sonic_ping(asic, neighbor) for neighbor in neighbors: pytest_assert(wait_until(60, 2, check_arptable_state, per_host, asic, neighbor, "REACHABLE"), "STATE for neighbor {} did not change to reachable".format(neighbor)) dump_and_verify_neighbors_on_asic(duthosts, per_host, asic, neighbors, nbrhosts, all_cfg_facts, nbr_macs)
def test_fdb_mac_move(ptfadapter, duthosts, rand_one_dut_hostname, ptfhost, get_function_conpleteness_level): # Perform FDB clean up before each test fdb_cleanup(duthosts, rand_one_dut_hostname) normalized_level = get_function_conpleteness_level if normalized_level is None: normalized_level = "basic" loop_times = LOOP_TIMES_LEVEL_MAP[normalized_level] duthost = duthosts[rand_one_dut_hostname] conf_facts = duthost.config_facts(host=duthost.hostname, source="persistent")['ansible_facts'] # reinitialize data plane due to above changes on PTF interfaces ptfadapter.reinit() router_mac = duthost.facts['router_mac'] port_index_to_name = { v: k for k, v in conf_facts['port_index_map'].items() } # Only take interfaces that are in ptf topology ptf_ports_available_in_topo = ptfhost.host.options[ 'variable_manager'].extra_vars.get("ifaces_map") available_ports_idx = [] for idx, name in ptf_ports_available_in_topo.items(): if idx in port_index_to_name and conf_facts['PORT'][ port_index_to_name[idx]].get('admin_status', 'down') == 'up': available_ports_idx.append(idx) vlan_table = {} interface_table = defaultdict(set) config_portchannels = conf_facts.get('PORTCHANNEL', {}) # if DUT has more than one VLANs, use the first vlan name = conf_facts['VLAN'].keys()[0] vlan = conf_facts['VLAN'][name] vlan_id = int(vlan['vlanid']) vlan_table[vlan_id] = [] for ifname in conf_facts['VLAN_MEMBER'][name].keys(): if 'tagging_mode' not in conf_facts['VLAN_MEMBER'][name][ifname]: continue tagging_mode = conf_facts['VLAN_MEMBER'][name][ifname]['tagging_mode'] port_index = [] if ifname in config_portchannels: for member in config_portchannels[ifname]['members']: if conf_facts['port_index_map'][member] in available_ports_idx: port_index.append(conf_facts['port_index_map'][member]) if port_index: interface_table[ifname].add(vlan_id) elif conf_facts['port_index_map'][ifname] in available_ports_idx: port_index.append(conf_facts['port_index_map'][ifname]) interface_table[ifname].add(vlan_id) if port_index: vlan_table[vlan_id].append({ 'port_index': port_index, 'tagging_mode': tagging_mode }) vlan = vlan_table.keys()[0] vlan_member_count = len(vlan_table[vlan]) total_fdb_entries = min( TOTAL_FDB_ENTRIES, (get_crm_resources(duthost, "fdb_entry", "available") - get_crm_resources(duthost, "fdb_entry", "used"))) dummay_mac_count = int(math.floor(total_fdb_entries / vlan_member_count)) fdb = get_fdb_dict(ptfadapter, vlan_table, dummay_mac_count) port_list = fdb.keys() dummy_mac_list = fdb.values() for loop_time in range(0, loop_times): port_index_start = (0 + loop_time) % len(port_list) for (port, dummy_mac_set) in zip(range(len(port_list)), dummy_mac_list): port_index = (port_index_start + port) % len(port_list) for dummy_mac in dummy_mac_set: send_arp_request(ptfadapter, port_index, dummy_mac, router_mac, vlan_id) time.sleep(FDB_POPULATE_SLEEP_TIMEOUT) pytest_assert( wait_until( 20, 1, 0, lambda: get_fdb_dynamic_mac_count(duthost) > vlan_member_count), "FDB Table Add failed") # Flush dataplane ptfadapter.dataplane.flush() fdb_cleanup(duthosts, rand_one_dut_hostname) # Wait for 10 seconds before starting next loop time.sleep(10)
def test_verify_status(duthost): """Verify procdockerstatsd is active and running """ status = duthost.get_service_props('procdockerstatsd') pytest_assert(status["ActiveState"] == "active" and status["SubState"] == "running", "Procdockerstatsd either not active or not running")
def test_show_platform_syseeprom(duthosts, enum_rand_one_per_hwsku_hostname, dut_vars): """ @summary: Verify output of `show platform syseeprom` """ duthost = duthosts[enum_rand_one_per_hwsku_hostname] cmd = " ".join([CMD_SHOW_PLATFORM, "syseeprom"]) logging.info("Verifying output of '{}' on '{}' ...".format(cmd, duthost.hostname)) syseeprom_cmd = duthost.command(cmd) syseeprom_output = syseeprom_cmd["stdout"] syseeprom_output_lines = syseeprom_cmd["stdout_lines"] """ Gather expected data from a inventory file instead if 'syseeprom_info' is defined in the inventory # Sample inventory with syseeprom: str-msn2700-01: ansible_host: 10.251.0.188 model: MSN2700-CS2FO serial: MT1234X56789 base_mac: 24:8a:07:12:34:56 syseeprom_info: "0x21": "MSN2700" "0x22": "MSN2700-CS2FO" "0x23": "MT1234X56789" "0x24": "24:8a:07:12:34:56" "0x25": "12/07/2016" "0x26": "0" "0x28": "x86_64-mlnx_x86-r0" "0x29": "2016.11-5.1.0008-9600" "0x2A": "128" "0x2B": "Mellanox" "0xFE": "0xFBA1E964" """ if 'syseeprom_info' in dut_vars: expected_syseeprom_info_dict = dut_vars['syseeprom_info'] parsed_syseeprom = {} # Can't use util.get_fields as the values go beyond the last set of '---' in the hearder line. regex_int = re.compile(r'([\S\s]+)(0x[A-F0-9]+)\s+([\d]+)\s+([\S\s]*)') for line in syseeprom_output_lines[6:]: t1 = regex_int.match(line) if t1: parsed_syseeprom[t1.group(2).strip()] = t1.group(4).strip() for field in expected_syseeprom_info_dict: pytest_assert(field in parsed_syseeprom, "Expected field '{}' not present in syseeprom on '{}'".format(field, duthost.hostname)) pytest_assert(parsed_syseeprom[field] == expected_syseeprom_info_dict[field], "System EEPROM info is incorrect - for '{}', rcvd '{}', expected '{}' on '{}'". format(field, parsed_syseeprom[field], expected_syseeprom_info_dict[field], duthost.hostname)) if duthost.facts["asic_type"] in ["mellanox"]: expected_fields = [ "Product Name", "Part Number", "Serial Number", "Base MAC Address", "Manufacture Date", "Device Version", "MAC Addresses", "Manufacturer", "Vendor Extension", "ONIE Version", "CRC-32"] utility_cmd = "sudo python -c \"import imp; \ m = imp.load_source('eeprom', '/usr/share/sonic/device/{}/plugins/eeprom.py'); \ t = m.board('board', '', '', ''); e = t.read_eeprom(); t.decode_eeprom(e)\"".format(duthost.facts["platform"]) utility_cmd_output = duthost.command(utility_cmd) for field in expected_fields: pytest_assert(syseeprom_output.find(field) >= 0, "Expected field '{}' was not found on '{}'".format(field, duthost.hostname)) pytest_assert(utility_cmd_output["stdout"].find(field) >= 0, "Expected field '{}' was not found on '{}'".format(field, duthost.hostname)) for line in utility_cmd_output["stdout_lines"]: pytest_assert(line in syseeprom_output, "Line '{}' was not found in output on '{}'".format(line, duthost.hostname))
def test_tgen(traffic_config, ixia_api): config = traffic_config flow_name = config.flows[0].name pkt_size = config.flows[0].size.fixed rate_percent = config.flows[0].rate.value duration_sec = config.flows[0].duration.seconds.seconds port_speed = config.layer1[0].speed words = port_speed.split('_') pytest_assert( len(words) == 3 and words[1].isdigit(), 'Fail to get port speed from {}'.format(port_speed)) port_speed_gbps = int(words[1]) """ Apply configuration """ ixia_api.set_state(State(ConfigState(config=config, state='set'))) """ Start traffic """ ixia_api.set_state(State(FlowTransmitState(state='start'))) """ Wait for traffic to finish """ time.sleep(duration_sec) while True: rows = ixia_api.get_flow_results(FlowRequest(flow_names=[flow_name])) if len(rows) == 1 and \ rows[0]['name'] == flow_name and \ rows[0]['transmit'] == 'stopped': """ Wait for counters to fully propagate """ time.sleep(2) break else: time.sleep(1) """ Dump per-flow statistics """ rows = ixia_api.get_flow_results(FlowRequest(flow_names=[flow_name])) """ Stop traffic """ ixia_api.set_state(State(FlowTransmitState(state='stop'))) """ Analyze traffic results """ pytest_assert( len(rows) == 1 and rows[0]['name'] == flow_name, 'Fail to get results of flow {}'.format(flow_name)) row = rows[0] rx_frames = row['frames_rx'] tx_frames = row['frames_tx'] pytest_assert( rx_frames == tx_frames, 'Unexpected packet losses (Tx: {}, Rx: {})'.format( tx_frames, rx_frames)) tput_bps = port_speed_gbps * 1e9 * rate_percent / 100.0 exp_rx_frames = tput_bps * duration_sec / 8 / pkt_size deviation_thresh = 0.05 ratio = float(exp_rx_frames) / rx_frames deviation = abs(ratio - 1) pytest_assert( deviation <= deviation_thresh, 'Expected / Actual # of pkts: {} / {}'.format(exp_rx_frames, rx_frames))
def test_link_flap(request, duthosts, rand_one_dut_hostname, tbinfo, fanouthosts, get_loop_times): """ Validates that link flap works as expected """ duthost = duthosts[rand_one_dut_hostname] orch_cpu_threshold = request.config.getoption("--orch_cpu_threshold") # Record memory status at start memory_output = duthost.shell("show system-memory")["stdout"] logger.info("Memory Status at start: %s", memory_output) # Record Redis Memory at start start_time_redis_memory = duthost.shell( "redis-cli info memory | grep used_memory_human | sed -e 's/.*:\(.*\)M/\\1/'" )["stdout"] logger.info("Redis Memory: %s M", start_time_redis_memory) # Make Sure Orch CPU < orch_cpu_threshold before starting test. logger.info( "Make Sure orchagent CPU utilization is less that %d before link flap", orch_cpu_threshold) pytest_assert( wait_until(100, 2, 0, check_orch_cpu_utilization, duthost, orch_cpu_threshold), "Orch CPU utilization {} > orch cpu threshold {} before link flap". format( duthost.shell( "show processes cpu | grep orchagent | awk '{print $9}'") ["stdout"], orch_cpu_threshold)) loop_times = get_loop_times port_lists = get_port_list(duthost, tbinfo) candidates = [] for port in port_lists: fanout, fanout_port = fanout_switch_port_lookup( fanouthosts, duthost.hostname, port) candidates.append((port, fanout, fanout_port)) for loop_time in range(0, loop_times): watch = False check_status = False if loop_time == 0 or loop_time == loop_times - 1: watch = True check_status = True for dut_port, fanout, fanout_port in candidates: toggle_one_link(duthost, dut_port, fanout, fanout_port, watch=watch, check_status=check_status) # Record memory status at end memory_output = duthost.shell("show system-memory")["stdout"] logger.info("Memory Status at end: %s", memory_output) # Record orchagent CPU utilization at end orch_cpu = duthost.shell( "show processes cpu | grep orchagent | awk '{print $9}'")["stdout"] logger.info("Orchagent CPU Util at end: %s", orch_cpu) # Record Redis Memory at end end_time_redis_memory = duthost.shell( "redis-cli info memory | grep used_memory_human | sed -e 's/.*:\(.*\)M/\\1/'" )["stdout"] logger.info("Redis Memory at start: %s M", start_time_redis_memory) logger.info("Redis Memory at end: %s M", end_time_redis_memory) # Calculate diff in Redis memory incr_redis_memory = float(end_time_redis_memory) - float( start_time_redis_memory) logger.info("Redis absolute difference: %d", incr_redis_memory) # Check redis memory only if it is increased else default to pass if incr_redis_memory > 0.0: percent_incr_redis_memory = (incr_redis_memory / float(start_time_redis_memory)) * 100 logger.info("Redis Memory percentage Increase: %d", percent_incr_redis_memory) pytest_assert( percent_incr_redis_memory < 5, "Redis Memory Increase more than expected: {}".format( percent_incr_redis_memory)) # Orchagent CPU should consume < orch_cpu_threshold at last. logger.info("watch orchagent CPU utilization when it goes below %d", orch_cpu_threshold) pytest_assert( wait_until(45, 2, 0, check_orch_cpu_utilization, duthost, orch_cpu_threshold), "Orch CPU utilization {} > orch cpu threshold {} before link flap". format( duthost.shell( "show processes cpu | grep orchagent | awk '{print $9}'") ["stdout"], orch_cpu_threshold))
def __gen_traffic(testbed_config, port_config_list, port_id, duthost, pause_flow_name, global_pause, pause_prio_list, test_flow_name, test_flow_prio_list, test_flow_rate_percent, bg_flow_name, bg_flow_prio_list, bg_flow_rate_percent, data_flow_dur_sec, data_flow_delay_sec, data_pkt_size, prio_dscp_map): """ Generate configurations of flows, including test flows, background flows and pause storm. Test flows and background flows are also known as data flows. Args: testbed_config (obj): testbed L1/L2/L3 configuration port_config_list (list): list of port configuration port_id (int): ID of DUT port to test duthost (Ansible host instance): device under test pause_flow_name (str): name of pause storm global_pause (bool): if pause frame is IEEE 802.3X pause pause_prio_list (list): priorities to pause for pause frames test_flow_name (str): name of test flows test_prio_list (list): priorities of test flows test_flow_rate_percent (int): rate percentage for each test flow bg_flow_name (str): name of background flows bg_prio_list (list): priorities of background flows bg_flow_rate_percent (int): rate percentage for each background flow data_flow_dur_sec (int): duration of data flows in second data_flow_delay_sec (int): start delay of data flows in second data_pkt_size (int): packet size of data flows in byte prio_dscp_map (dict): Priority vs. DSCP map (key = priority). Returns: flows configurations (list): the list should have configurations of len(test_flow_prio_list) test flow, len(bg_flow_prio_list) background flows and a pause storm. """ result = list() rx_port_id = port_id tx_port_id_list, rx_port_id_list = select_ports(port_config_list=port_config_list, duthost=duthost, pattern="many to one", rx_port_id=rx_port_id) pytest_assert(len(tx_port_id_list) > 0, "Cannot find any TX ports") tx_port_id = select_tx_port(tx_port_id_list=tx_port_id_list, rx_port_id=rx_port_id) pytest_assert(tx_port_id is not None, "Cannot find a suitable TX port") tx_port_config = next((x for x in port_config_list if x.id == tx_port_id), None) rx_port_config = next((x for x in port_config_list if x.id == rx_port_id), None) tx_mac = tx_port_config.mac if tx_port_config.gateway == rx_port_config.gateway and \ tx_port_config.prefix_len == rx_port_config.prefix_len: """ If soruce and destination port are in the same subnet """ rx_mac = rx_port_config.mac else: rx_mac = tx_port_config.gateway_mac data_endpoint = PortTxRx(tx_port_name=testbed_config.ports[tx_port_id].name, rx_port_name=testbed_config.ports[rx_port_id].name) data_flow_delay_nanosec = sec_to_nanosec(data_flow_delay_sec) """ Test flows """ for prio in test_flow_prio_list: eth_hdr = EthernetHeader(src=FieldPattern(tx_mac), dst=FieldPattern(rx_mac), pfc_queue=FieldPattern([prio])) ip_prio = Priority(Dscp(phb=FieldPattern(choice=prio_dscp_map[prio]), ecn=FieldPattern(choice=Dscp.ECN_CAPABLE_TRANSPORT_1))) ipv4_hdr = Ipv4Header(src=FieldPattern(tx_port_config.ip), dst=FieldPattern(rx_port_config.ip), priority=ip_prio) test_flow = Flow( name='{} Prio {}'.format(test_flow_name, prio), tx_rx=TxRx(data_endpoint), packet=[Header(choice=eth_hdr), Header(choice=ipv4_hdr)], size=Size(data_pkt_size), rate=Rate('line', test_flow_rate_percent), duration=Duration(FixedSeconds(seconds=data_flow_dur_sec, delay=data_flow_delay_nanosec, delay_unit='nanoseconds')) ) result.append(test_flow) """ Background flows """ for prio in bg_flow_prio_list: eth_hdr = EthernetHeader(src=FieldPattern(tx_mac), dst=FieldPattern(rx_mac), pfc_queue=FieldPattern([prio])) ip_prio = Priority(Dscp(phb=FieldPattern(choice=prio_dscp_map[prio]), ecn=FieldPattern(choice=Dscp.ECN_CAPABLE_TRANSPORT_1))) ipv4_hdr = Ipv4Header(src=FieldPattern(tx_port_config.ip), dst=FieldPattern(rx_port_config.ip), priority=ip_prio) bg_flow = Flow( name='{} Prio {}'.format(bg_flow_name, prio), tx_rx=TxRx(data_endpoint), packet=[Header(choice=eth_hdr), Header(choice=ipv4_hdr)], size=Size(data_pkt_size), rate=Rate('line', bg_flow_rate_percent), duration=Duration(FixedSeconds(seconds=data_flow_dur_sec, delay=data_flow_delay_nanosec, delay_unit='nanoseconds')) ) result.append(bg_flow) """ Pause storm """ if global_pause: pause_pkt = Header(EthernetPause( dst=FieldPattern(choice='01:80:C2:00:00:01'), src=FieldPattern(choice='00:00:fa:ce:fa:ce') )) else: pause_time = [] for x in range(8): if x in pause_prio_list: pause_time.append('ffff') else: pause_time.append('0000') vector = pfc_class_enable_vector(pause_prio_list) pause_pkt = Header(PfcPause( dst=FieldPattern(choice='01:80:C2:00:00:01'), src=FieldPattern(choice='00:00:fa:ce:fa:ce'), class_enable_vector=FieldPattern(choice=vector), pause_class_0=FieldPattern(choice=pause_time[0]), pause_class_1=FieldPattern(choice=pause_time[1]), pause_class_2=FieldPattern(choice=pause_time[2]), pause_class_3=FieldPattern(choice=pause_time[3]), pause_class_4=FieldPattern(choice=pause_time[4]), pause_class_5=FieldPattern(choice=pause_time[5]), pause_class_6=FieldPattern(choice=pause_time[6]), pause_class_7=FieldPattern(choice=pause_time[7]), )) """ Pause frames are sent from the RX port """ pause_endpoint = PortTxRx(tx_port_name=testbed_config.ports[rx_port_id].name, rx_port_name=testbed_config.ports[tx_port_id].name) speed_str = testbed_config.layer1[0].speed speed_gbps = int(speed_str.split('_')[1]) pause_dur = 65535 * 64 * 8.0 / (speed_gbps * 1e9) pps = int(2 / pause_dur) pause_flow = Flow( name=pause_flow_name, tx_rx=TxRx(pause_endpoint), packet=[pause_pkt], size=Size(64), rate=Rate('pps', value=pps), duration=Duration(Continuous(delay=0, delay_unit='nanoseconds')) ) result.append(pause_flow) return result
def test_tunnel_memory_leak( toggle_all_simulator_ports_to_upper_tor, upper_tor_host, lower_tor_host, ptfhost, ptfadapter, conn_graph_facts, tbinfo, vmhost, run_arp_responder ): """ Test if there is memory leak for service tunnel_packet_handler. Send ip packets from standby TOR T1 to Server, standby TOR will forward the packets to active TOR with tunnel, active TOR will decapsulate the IPinIP packets, but there is no neighbor for destination as we remove neighbor before test, tunnel_packet_handler will be triggered and neighbor will be added. Server will receive the packets. Check if memory usage is increased after tunnel_packet_handler's operation. Since tunnel_packet_handler is only triggered by the first packet, loop the process for all severs to trigger it as much as possible. """ @contextlib.contextmanager def prepare_services(ptfhost): """ Temporarily start arp and icmp service. Make sure to stop garp service, otherwise, it will add neighbor entry back automatically. It has to stop garp_service for triggering tunnel_packet_handler. It has to start arp and icmp service for receiving packets at server side. """ ptfhost.shell("supervisorctl stop garp_service") ptfhost.shell("supervisorctl start arp_responder") ptfhost.shell("supervisorctl start icmp_responder") yield ptfhost.shell("supervisorctl stop arp_responder") ptfhost.shell("supervisorctl stop icmp_responder") @contextlib.contextmanager def remove_neighbor(duthost, server_ip): """ Remove ip neighbor before test for triggering tunnel_packet_handler, restore it after test """ flush_neighbor_ct = flush_neighbor(duthost, server_ip, True) with flush_neighbor_ct: command = "ip neighbor show %s" % server_ip output = [_.strip() for _ in duthost.shell(command)["stdout_lines"]] pytest_assert(not output, "server ip {} isn't flushed in neighbor table.".format(server_ip)) yield pytest_assert(is_tunnel_packet_handler_running(upper_tor_host), "tunnel_packet_handler is not running in SWSS conainter.") ptf_t1_intf = random.choice(get_t1_ptf_ports(lower_tor_host, tbinfo)) all_servers_ips = mux_cable_server_ip(upper_tor_host) with prepare_services(ptfhost): # Get the original memeory percent before test check_memory_leak(upper_tor_host) for iface, server_ips in all_servers_ips.items(): server_ipv4 = server_ips["server_ipv4"].split("/")[0] logging.info("Select DUT interface {} and server IP {} to test.".format(iface, server_ipv4)) pkt, exp_pkt = build_packet_to_server(lower_tor_host, ptfadapter, server_ipv4) rm_neighbor = remove_neighbor(upper_tor_host, server_ipv4) server_traffic_monitor = ServerTrafficMonitor( upper_tor_host, ptfhost, vmhost, tbinfo, iface, conn_graph_facts, exp_pkt, existing=True, is_mocked=False ) with rm_neighbor, server_traffic_monitor: testutils.send(ptfadapter, int(ptf_t1_intf.strip("eth")), pkt, count=PACKET_COUNT) logging.info("Sent {} packets from ptf t1 interface {} on standby TOR {}" .format(PACKET_COUNT, ptf_t1_intf, lower_tor_host.hostname)) # Check memory usage for every operation, used for debugging if test failed check_memory_leak(upper_tor_host) pytest_assert(validate_neighbor_entry_exist(upper_tor_host, server_ipv4), "The server ip {} doesn't exist in neighbor table on dut {}. \ tunnel_packet_handler isn't triggered.".format(server_ipv4, upper_tor_host.hostname)) pytest_assert(len(server_traffic_monitor.matched_packets) > PACKET_COUNT /2, "Received {} expected packets for server {}, drop more than 50%." .format(len(server_traffic_monitor.matched_packets), server_ipv4)) # sleep 10s to wait memory usage stable, check if there is memory leak time.sleep(10) check_result = check_memory_leak(upper_tor_host) pytest_assert(check_result == False, "Test failed because there is memory leak on {}" .format(upper_tor_host.hostname))
def __verify_results(rows, duthost, pause_flow_name, test_flow_name, bg_flow_name, data_flow_dur_sec, test_flow_rate_percent, bg_flow_rate_percent, data_pkt_size, speed_gbps, test_flow_pause, tolerance): """ Verify if we get expected experiment results Args: rows (list): per-flow statistics duthost (Ansible host instance): device under test pause_flow_name (str): name of pause storm test_flow_name (str): name of test flows bg_flow_name (str): name of background flows test_flow_rate_percent (int): rate percentage for each test flow bg_flow_rate_percent (int): rate percentage for each background flow data_pkt_size (int): packet size of data flows in byte speed_gbps (int): link speed in Gbps test_flow_pause (bool): if test flows are expected to be paused tolerance (float): maximum allowable deviation Returns: N/A """ """ All the pause frames should be dropped """ pause_flow_row = next(row for row in rows if row["name"] == pause_flow_name) tx_frames = pause_flow_row['frames_tx'] rx_frames = pause_flow_row['frames_rx'] pytest_assert(tx_frames > 0 and rx_frames == 0, 'All the pause frames should be dropped') """ Check background flows """ for row in rows: if bg_flow_name not in row['name']: continue tx_frames = row['frames_tx'] rx_frames = row['frames_rx'] pytest_assert(tx_frames == rx_frames, '{} should not have any dropped packet'.format(row['name'])) exp_bg_flow_rx_pkts = bg_flow_rate_percent / 100.0 * speed_gbps \ * 1e9 * data_flow_dur_sec / 8.0 / data_pkt_size deviation = (rx_frames - exp_bg_flow_rx_pkts) / float(exp_bg_flow_rx_pkts) pytest_assert(abs(deviation) < tolerance, '{} should receive {} packets (actual {})'.\ format(row['name'], exp_bg_flow_rx_pkts, rx_frames)) """ Check test flows """ for row in rows: if test_flow_name not in row['name']: continue tx_frames = row['frames_tx'] rx_frames = row['frames_rx'] if test_flow_pause: pytest_assert(tx_frames > 0 and rx_frames == 0, '{} should be paused'.format(row['name'])) else: pytest_assert(tx_frames == rx_frames, '{} should not have any dropped packet'.format(row['name'])) exp_test_flow_rx_pkts = test_flow_rate_percent / 100.0 * speed_gbps \ * 1e9 * data_flow_dur_sec / 8.0 / data_pkt_size deviation = (rx_frames - exp_test_flow_rx_pkts) / float(exp_test_flow_rx_pkts) pytest_assert(abs(deviation) < tolerance, '{} should receive {} packets (actual {})'.\ format(test_flow_name, exp_test_flow_rx_pkts, rx_frames)) if test_flow_pause: """ In-flight TX bytes of test flows should be held by switch buffer """ tx_frames_total = sum(row['frames_tx'] for row in rows if test_flow_name in row['name']) tx_bytes_total = tx_frames_total * data_pkt_size dut_buffer_size = get_egress_lossless_buffer_size(host_ans=duthost) pytest_assert(tx_bytes_total < dut_buffer_size, 'Total TX bytes {} should be smaller than DUT buffer size {}'.\ format(tx_bytes_total, dut_buffer_size))
def _extract_mirror_payload(self, encapsulated_packet, payload_size): pytest_assert(len(encapsulated_packet) >= self.OUTER_HEADER_SIZE, "Incomplete packet, expected at least {} header bytes".format(self.OUTER_HEADER_SIZE)) inner_frame = encapsulated_packet[-payload_size:] return packet.Ether(inner_frame)
def test_pmon_pcied_stop_and_start_status(check_daemon_status, duthosts, rand_one_dut_hostname, data_before_restart): """ @summary: This test case is to check the pcied stopped and restarted status """ duthost = duthosts[rand_one_dut_hostname] pre_daemon_status, pre_daemon_pid = duthost.get_pmon_daemon_status( daemon_name) logger.info("{} daemon is {} with pid {}".format(daemon_name, pre_daemon_status, pre_daemon_pid)) duthost.stop_pmon_daemon(daemon_name, SIG_STOP_SERVICE) time.sleep(2) daemon_status, daemon_pid = duthost.get_pmon_daemon_status(daemon_name) pytest_assert( daemon_status == expected_stopped_status, "{} expected stopped status is {} but is {}".format( daemon_name, expected_stopped_status, daemon_status)) pytest_assert( daemon_pid == -1, "{} expected pid is -1 but is {}".format(daemon_name, daemon_pid)) data = collect_data(duthost) pytest_assert(not data['status'], "DB data is not cleared on daemon stop") pytest_assert(not data['devices'], "DB data is not cleared on daemon stop") duthost.start_pmon_daemon(daemon_name) time.sleep(10) post_daemon_status, post_daemon_pid = duthost.get_pmon_daemon_status( daemon_name) pytest_assert( post_daemon_status == expected_running_status, "{} expected restarted status is {} but is {}".format( daemon_name, expected_running_status, post_daemon_status)) pytest_assert( post_daemon_pid != -1, "{} expected pid is -1 but is {}".format(daemon_name, post_daemon_pid)) pytest_assert( post_daemon_pid > pre_daemon_pid, "Restarted {} pid should be bigger than {} but it is {}".format( daemon_name, pre_daemon_pid, post_daemon_pid)) data_after_restart = wait_data(duthost) pytest_assert(data_after_restart == data_before_restart, 'DB data present before and after restart does not match')
def test_gratarp_macchange(self, duthosts, enum_rand_one_per_hwsku_frontend_hostname, enum_asic_index, ptfhost, tbinfo, nbrhosts, setup, teardown, all_cfg_facts, established_arp): """ Verify tables, databases, and kernel routes are correctly updated when a unsolicited ARP packet changes the MAC address of learned neighbor. Test Steps * Send unsolicited ARP packet into DUT for an IP known by DUT with a different MAC address for the neighbor. * Change the MAC address of the neighbor VM. * On local linecard: * Verify table entries in local ASIC, APP, and host ARP table are updated with new MAC. * On supervisor card: * Verify Chassis App DB entry is correct for with the updated MAC address. * On remote linecards: * Verify table entries in remote hosts/ASICs in APPDB, and host ARP table are still present with inband MAC address * Verify ASIC DB is updated with new MAC. * Verify kernel route in remote hosts are still present to inband port. * Verify that packets can be sent from local and remote linecards to learned address. Args: duthosts: The duthosts fixture enum_rand_one_per_hwsku_frontend_hostname: frontend enumeration fixture enum_asic_index: asic enumeration fixture ptfhost: The ptfhost fixure. tbinfo: The tbinfo fixture nbrhosts: The nbrhosts fixture. setup: The setup fixture from this module. established_arp: The established_arp fixture from this module. all_cfg_facts: The all_cfg_facts fixture from voq/contest.py """ self.ptfhost = ptfhost duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] asic = duthost.asics[enum_asic_index if enum_asic_index is not None else 0] cfg_facts = all_cfg_facts[duthost.hostname][asic.asic_index]['ansible_facts'] if 'BGP_NEIGHBOR' in cfg_facts: neighs = cfg_facts['BGP_NEIGHBOR'] else: logger.info("No local neighbors for host: %s/%s, skipping", duthost.hostname, asic.asic_index) return eth_cfg = cfg_facts['INTERFACE'] if 'INTERFACE' in cfg_facts else {} if eth_cfg == {}: pytest.skip("Can't run this test without any IP interfaces on ethernet ports") eth_ports = [intf for intf in eth_cfg] local_port = random.choice(eth_ports) logger.info("We will test port: %s on host %s, asic %s", local_port, duthost.hostname, asic.asic_index) nbr_to_test = [] for neighbor in neighs: local_ip = neighs[neighbor]['local_addr'] nbr_port = get_port_by_ip(cfg_facts, local_ip) if local_port == nbr_port: nbr_to_test.append(neighbor) logger.info("We will test neighbors: %s", nbr_to_test) for neighbor in nbr_to_test: nbrinfo = get_neighbor_info(neighbor, nbrhosts) tb_port = get_ptf_port(duthosts, all_cfg_facts[duthost.hostname][asic.asic_index]['ansible_facts'], tbinfo, duthost, local_port)[0] original_mac = nbrinfo['mac'] logger.info("*" * 60) logger.info("Verify initial neighbor: %s, port %s", neighbor, local_port) logger.info("%s port %s is on ptf port: %s", duthost.hostname, local_port, tb_port) logger.info("-" * 60) sonic_ping(asic, neighbor) pytest_assert(wait_until(60, 2, check_arptable_mac, duthost, asic, neighbor, original_mac), "MAC {} didn't change in ARP table".format(original_mac)) check_one_neighbor_present(duthosts, duthost, asic, neighbor, nbrhosts, all_cfg_facts) try: change_mac(nbrhosts[nbrinfo['vm']], nbrinfo['shell_intf'], NEW_MAC) self.send_grat_pkt(NEW_MAC, neighbor, int(tb_port)) pytest_assert(wait_until(60, 2, check_arptable_mac, duthost, asic, neighbor, NEW_MAC, checkstate=False), "MAC {} didn't change in ARP table of neighbor {}".format(NEW_MAC, neighbor)) try: sonic_ping(asic, neighbor) except AssertionError: logging.info("No initial response from ping, begin poll to see if ARP table responds.") pytest_assert(wait_until(60, 2, check_arptable_mac, duthost, asic, neighbor, NEW_MAC, checkstate=True), "MAC {} didn't change in ARP table of neighbor {}".format(NEW_MAC, neighbor)) check_one_neighbor_present(duthosts, duthost, asic, neighbor, nbrhosts, all_cfg_facts) ping_all_neighbors(duthosts, all_cfg_facts, [neighbor]) finally: logger.info("Will Restore ethernet mac on neighbor: %s, port %s, vm %s", neighbor, nbrinfo['shell_intf'], nbrinfo['vm']) change_mac(nbrhosts[nbrinfo['vm']], nbrinfo['shell_intf'], original_mac) if ":" in neighbor: logger.info("Force neighbor solicitation to workaround long IPV6 timer.") asic_cmd(asic, "ndisc6 %s %s" % (neighbor, local_port)) pytest_assert( wait_until(60, 2, check_arptable_mac, duthost, asic, neighbor, original_mac, checkstate=False), "MAC {} didn't change in ARP table".format(original_mac)) sonic_ping(asic, neighbor, verbose=True) pytest_assert(wait_until(60, 2, check_arptable_mac, duthost, asic, neighbor, original_mac), "MAC {} didn't change in ARP table".format(original_mac)) check_one_neighbor_present(duthosts, duthost, asic, neighbor, nbrhosts, all_cfg_facts) ping_all_neighbors(duthosts, all_cfg_facts, [neighbor])
def toggle_all_simulator_ports_to_random_side(duthosts, mux_server_url, tbinfo): """ A function level fixture to toggle all ports to a random side. """ def _get_mux_status(duthost): cmd = 'show mux status --json' return json.loads(duthost.shell(cmd)['stdout']) def _check_mux_status_consistency(): """Ensure mux status is consistent between the ToRs and mux simulator.""" upper_tor_mux_status = _get_mux_status(upper_tor_host) lower_tor_mux_status = _get_mux_status(lower_tor_host) simulator_mux_status = _get(mux_server_url) if not upper_tor_mux_status: logging.warn("Failed to retrieve mux status from the upper tor") return False if not lower_tor_mux_status: logging.warn("Failed to retrieve mux status from the lower tor") return False if not simulator_mux_status: logging.warn("Failed to retrieve mux status from the mux simulator") return False if not set(upper_tor_mux_status.keys()) == set(lower_tor_mux_status.keys()): logging.warn("Ports mismatch between the upper tor and lower tor") return False # get mapping from port indices to mux status simulator_port_mux_status = {int(k.split('-')[-1]):v for k,v in simulator_mux_status.items()} for intf in upper_tor_mux_status['MUX_CABLE']: intf_index = port_indices[intf] if intf_index not in simulator_port_mux_status: logging.warn("No mux status for interface %s from mux simulator", intf) return False simulator_status = simulator_port_mux_status[intf_index] upper_tor_status = upper_tor_mux_status['MUX_CABLE'][intf]['STATUS'] lower_tor_status = lower_tor_mux_status['MUX_CABLE'][intf]['STATUS'] if upper_tor_status == 'active' and lower_tor_status == 'standby' and simulator_status['active_side'] == 'upper_tor': continue if upper_tor_status == 'standby' and lower_tor_status == 'active' and simulator_status['active_side'] == 'lower_tor': continue logging.warn( "For interface %s, upper tor mux status: %s, lower tor mux status: %s, simulator status: %s", intf, upper_tor_status, lower_tor_status, simulator_status ) logging.warn("Inconsistent mux status for interface %s", intf) return False return True if 'dualtor' not in tbinfo['topo']['name']: return _toggle_all_simulator_ports(mux_server_url, RANDOM, tbinfo) upper_tor_host, lower_tor_host = duthosts[0], duthosts[1] mg_facts = upper_tor_host.get_extended_minigraph_facts(tbinfo) port_indices = mg_facts['minigraph_port_indices'] pytest_assert( utilities.wait_until(30, 5, 0, _check_mux_status_consistency), "Mux status is inconsistent between the DUTs and mux simulator after toggle" )
def test_neighbor_hw_mac_change(duthosts, enum_rand_one_per_hwsku_frontend_hostname, enum_asic_index, setup, teardown, nbrhosts, all_cfg_facts, nbr_macs, established_arp): """ Verify tables, databases, and kernel routes are correctly updated when the MAC address of a neighbor changes and is updated via request/reply exchange. Test Steps * Change the MAC address on a remote host that is already present in the ARP table. * Without clearing the entry in the DUT, allow the existing entry to time out and the new reply to have the new MAC address. * On local linecard: * Verify table entries in local ASIC, APP, and host ARP table are updated with new MAC. * On supervisor card: * Verify Chassis App DB entry is correct for with the updated MAC address. * On remote linecards: * Verify table entries in remote hosts/ASICs in APPDB, and host ARP table are still present with inband MAC address * Verify ASIC DB is updated with new MAC. * Verify kernel route in remote hosts are still present to inband port. * Verify that packets can be sent from local and remote linecards to the learned address. Args: duthosts: duthosts fixture. enum_rand_one_per_hwsku_frontend_hostname: frontend iteration fixture. enum_asic_index: asic iteration fixture. setup: setup fixture for this module. nbrhosts: nbrhosts fixture. established_arp: Fixture to establish arp on all nodes all_cfg_facts: all_cfg_facts fixture from voq/conftest.py """ per_host = duthosts[enum_rand_one_per_hwsku_frontend_hostname] asic = per_host.asics[enum_asic_index if enum_asic_index is not None else 0] cfg_facts = all_cfg_facts[per_host.hostname][asic.asic_index]['ansible_facts'] if 'BGP_NEIGHBOR' in cfg_facts: neighs = cfg_facts['BGP_NEIGHBOR'] else: logger.info("No local neighbors for host: %s/%s, skipping", per_host.hostname, asic.asic_index) return eth_cfg = cfg_facts['INTERFACE'] if 'INTERFACE' in cfg_facts else {} if eth_cfg == {}: pytest.skip("Can't run this test without any IP interfaces on ethernet ports") eth_ports = [intf for intf in eth_cfg] local_port = random.choice(eth_ports) logger.info("We will test port: %s on host %s, asic %s", local_port, per_host.hostname, asic.asic_index) nbr_to_test = [] for neighbor in neighs: local_ip = neighs[neighbor]['local_addr'] nbr_port = get_port_by_ip(cfg_facts, local_ip) if local_port == nbr_port: nbr_to_test.append(neighbor) logger.info("We will test neighbors: %s", nbr_to_test) nbrinfo = get_neighbor_info(nbr_to_test[0], nbrhosts) original_mac = nbrinfo['mac'] for neighbor in nbr_to_test: # Check neighbor on local linecard logger.info("*" * 60) logger.info("Verify initial neighbor: %s, port %s", neighbor, local_port) pytest_assert(wait_until(60, 2, check_arptable_mac, per_host, asic, neighbor, original_mac, checkstate=False), "MAC {} didn't change in ARP table".format(original_mac)) sonic_ping(asic, neighbor, verbose=True) pytest_assert(wait_until(60, 2, check_arptable_mac, per_host, asic, neighbor, original_mac), "MAC {} didn't change in ARP table".format(original_mac)) dump_and_verify_neighbors_on_asic(duthosts, per_host, asic, nbr_to_test, nbrhosts, all_cfg_facts, nbr_macs) try: logger.info("Changing ethernet mac on port %s, vm %s", nbrinfo['shell_intf'], nbrinfo['vm']) change_mac(nbrhosts[nbrinfo['vm']], nbrinfo['shell_intf'], NEW_MAC) for neighbor in nbr_to_test: if ":" in neighbor: logger.info("Force neighbor solicitation to workaround long IPV6 timer.") asic_cmd(asic, "ndisc6 %s %s" % (neighbor, local_port)) pytest_assert(wait_until(60, 2, check_arptable_mac, per_host, asic, neighbor, NEW_MAC, checkstate=False), "MAC {} didn't change in ARP table".format(NEW_MAC)) sonic_ping(asic, neighbor, verbose=True) pytest_assert(wait_until(60, 2, check_arptable_mac, per_host, asic, neighbor, NEW_MAC), "MAC {} didn't change in ARP table".format(NEW_MAC)) logger.info("Verify neighbor after mac change: %s, port %s", neighbor, local_port) check_one_neighbor_present(duthosts, per_host, asic, neighbor, nbrhosts, all_cfg_facts) logger.info("Ping neighbors: %s from all line cards", nbr_to_test) ping_all_neighbors(duthosts, all_cfg_facts, nbr_to_test) finally: logger.info("-" * 60) logger.info("Will Restore ethernet mac on port %s, vm %s", nbrinfo['shell_intf'], nbrinfo['vm']) change_mac(nbrhosts[nbrinfo['vm']], nbrinfo['shell_intf'], original_mac) for neighbor in nbr_to_test: if ":" in neighbor: logger.info("Force neighbor solicitation to workaround long IPV6 timer.") asic_cmd(asic, "ndisc6 %s %s" % (neighbor, local_port)) pytest_assert( wait_until(60, 2, check_arptable_mac, per_host, asic, neighbor, original_mac, checkstate=False), "MAC {} didn't change in ARP table".format(original_mac)) sonic_ping(asic, neighbor, verbose=True) pytest_assert(wait_until(60, 2, check_arptable_mac, per_host, asic, neighbor, original_mac), "MAC {} didn't change in ARP table".format(original_mac)) dump_and_verify_neighbors_on_asic(duthosts, per_host, asic, nbr_to_test, nbrhosts, all_cfg_facts, nbr_macs) ping_all_neighbors(duthosts, all_cfg_facts, nbr_to_test)
def _reset_simulator_port(interface_name=None): logger.warn("Resetting simulator ports {}".format('all' if interface_name is None else interface_name)) server_url = url(interface_name=interface_name, action=RESET) pytest_assert(_post(server_url, {}))
def generate_and_verify_decap_traffic(duthost, ptfadapter, src_port, dst_port, ip_src, ip_dst, tr_type, ip_tunnel=None): """ Send encapsulated packet from PTF to DUT and verify that DUT sends/doesn't send TCP/UDP packet to PTF. Args: duthost: DUT host object ptfadapter: PTF adapter src_port: Source port of PTF dst_port: Destination port of PTF ip_src: Source IP address of PTF ip_dst: Destination IP address of PTF tr_type: Type of traffic (TCP or UDP) ip_tunnel: Tunnel IP address of DUT """ router_mac = duthost.facts['router_mac'] src_port_number = int(get_port_number(src_port)) dst_port_number = int(get_port_number(dst_port)) ip_src = ip_src.split('/')[0] ip_dst = ip_dst.split('/')[0] ip_tunnel = ip_tunnel.split('/')[0] # Define encapsulated packet pkt = create_packet(eth_dst=router_mac, eth_src=ptfadapter.dataplane.get_mac( 0, src_port_number), ip_src=ip_src, ip_dst=ip_dst, ip_tunnel=ip_tunnel, vlan_vid=int(src_port.split('.')[1]), dl_vlan_enable=True, tr_type=tr_type, ttl=64) # Build expected packet inner_packet = pkt[packet.IP].payload[packet.IP].copy() exp_pkt = Ether(src=router_mac, dst=ptfadapter.dataplane.get_mac( 0, dst_port_number)) / Dot1Q( vlan=int(dst_port.split('.')[1])) / inner_packet exp_pkt['IP'].ttl -= 1 update_dut_arp_table(duthost, ip_dst) ptfadapter.dataplane.flush() testutils.send_packet(ptfadapter, src_port_number, pkt) pkt_filter = FilterPktBuffer(ptfadapter=ptfadapter, exp_pkt=exp_pkt, dst_port_number=dst_port_number, match_fields=[("802.1Q", "vlan"), ("Ethernet", "src"), ("Ethernet", "dst"), ("IP", "src"), ("IP", "dst")], ignore_fields=[]) pkt_in_buffer = pkt_filter.filter_pkt_in_buffer() pytest_assert(pkt_in_buffer is True, "Expected packet not available:\n{}".format(pkt_in_buffer))