def mock_server(fanouthosts, testbed_params, arp_responder, ptfadapter, duthost): """ Mock the presence of a server beneath a T0. Returns: A MockServer which will allow the caller to mock the behavior of a server within a VLAN under a T0. """ server_dst_port = random.choice(testbed_params["vlan_ports"]) server_dst_addr = random.choice(arp_responder[server_dst_port].keys()) server_dst_intf = testbed_params["physical_port_map"][server_dst_port] logging.info("Creating mock server with IP %s; dut port = %s, dut intf = %s", server_dst_addr, server_dst_port, server_dst_intf) logging.info("Clearing ARP and FDB tables for test setup") duthost.command("sonic-clear fdb all") duthost.command("sonic-clear arp") # Populate FDB logging.info("Populating FDB and ARP entry for mock server under VLAN") # Issue a ping to populate ARP table on DUT duthost.command('ping %s -c 3' % server_dst_addr, module_ignore_errors=True) fanout_neighbor, fanout_intf = fanout_switch_port_lookup(fanouthosts, server_dst_intf) return {"server_dst_port": server_dst_port, "server_dst_addr": server_dst_addr, "server_dst_intf": server_dst_intf, "fanout_neighbor": fanout_neighbor, "fanout_intf": fanout_intf}
def rif_port_down(duthosts, rand_one_dut_hostname, setup, fanouthosts, loganalyzer): """Shut RIF interface and return neighbor IP address attached to this interface.""" duthost = duthosts[rand_one_dut_hostname] wait_after_ports_up = 30 if not setup["rif_members"]: pytest.skip("RIF interface is absent") rif_member_iface = setup["rif_members"].keys()[0] vm_name = setup["mg_facts"]["minigraph_neighbors"][rif_member_iface].get("name", None) pytest_assert(vm_name, 'Neighbor not found for RIF member "{}"'.format(rif_member_iface)) ip_dst = None for item in setup["mg_facts"]["minigraph_bgp"]: if item["name"] == vm_name and netaddr.valid_ipv4(item["addr"]): ip_dst = item["addr"] break pytest_assert(ip_dst, 'Unable to find IP address for neighbor "{}"'.format(vm_name)) fanout_neighbor, fanout_intf = fanout_switch_port_lookup(fanouthosts, duthost.hostname, rif_member_iface) loganalyzer[rand_one_dut_hostname].expect_regex = [LOG_EXPECT_PORT_OPER_DOWN_RE.format(rif_member_iface)] with loganalyzer[rand_one_dut_hostname] as _: fanout_neighbor.shutdown(fanout_intf) time.sleep(1) yield ip_dst loganalyzer[rand_one_dut_hostname].expect_regex = [LOG_EXPECT_PORT_OPER_UP_RE.format(rif_member_iface)] with loganalyzer[rand_one_dut_hostname] as _: fanout_neighbor.no_shutdown(fanout_intf) time.sleep(wait_after_ports_up)
def setup(self, test_data): super(NeighVlanMemberDown, self).setup(test_data) for port in self.ports: fanout, fanport = fanout_switch_port_lookup( self.fanouthosts, self.duthost.hostname, port) if fanout and fanport: fanout.shutdown(fanport)
def build_test_candidates(dut, fanouthosts, port, completeness_level=None): """ Find test candidates for link flap test. Args: dut: DUT host object fanouthosts: List of fanout switch instances. port: port, when port == 'unknown' or 'all_ports' candidate will be all ports. A warning will be generated if the port == 'unknown'. caller can use 'all_ports' explicitly to mute the warning. completeness_level: Completeness level. Returns: A list of tuple with DUT's port, fanout port and fanout """ candidates = [] if port not in ['unknown', 'all_ports']: status = __get_dut_if_status(dut, port) fanout, fanout_port = fanout_switch_port_lookup( fanouthosts, dut.hostname, port) __build_candidate_list(candidates, fanout, fanout_port, port, status) else: # Build the full list if port == 'unknown': logger.warning( "Failed to get ports enumerated as parameter. Fall back to test all ports" ) status = __get_dut_if_status(dut) for dut_port in status.keys(): fanout, fanout_port = fanout_switch_port_lookup( fanouthosts, dut.hostname, dut_port) __build_candidate_list(candidates, fanout, fanout_port, dut_port, status) if completeness_level == 'debug': candidates = random.sample(candidates, 1) return candidates
def revert(self): for port in self.ports: if self.duthost.facts['platform'] == 'x86_64-kvm_x86_64-r0': index = int(self.port_info.get(port).get("index")) + 1 self.vmhost.shell("virsh domif-setlink {} {}-{} up".format( self.duthost.hostname, self.duthost.hostname, index)) fanout, fanport = fanout_switch_port_lookup( self.fanouthosts, self.duthost.hostname, port) if fanout and fanport: fanout.no_shutdown(fanport)
def __build_test_candidates(self, dut, fanouthosts, port): candidates = [] if port != 'unknown': status = self.__get_dut_if_status(dut, port) fanout, fanout_port = fanout_switch_port_lookup(fanouthosts, port) self.__build_candidate_list(candidates, fanout, fanout_port, port, status) else: # Build the full list logger.warning("Failed to get ports enumerated as parameter. Fall back to test all ports") status = self.__get_dut_if_status(dut) for dut_port in status.keys(): fanout, fanout_port = fanout_switch_port_lookup(fanouthosts, dut_port) self.__build_candidate_list(candidates, fanout, fanout_port, dut_port, status) if self.completeness_level == 'debug': candidates = random.sample(candidates, 1) return candidates
def setup(self, test_data): super(NeighVlanMemberDown, self).setup(test_data) for port in self.ports: if self.duthost.facts['platform'] == 'x86_64-kvm_x86_64-r0': index = int(self.port_info.get(port).get("index")) + 1 self.vmhost.shell("virsh domif-setlink {} {}-{} down".format( self.duthost.hostname, self.duthost.hostname, index)) fanout, fanport = fanout_switch_port_lookup( self.fanouthosts, self.duthost.hostname, port) if fanout and fanport: fanout.shutdown(fanport)
def _change_ports_state(self, bring_up): for port in self.ports: nbrname = self.dut_port_to_nbr[port] nbrport = self.dut_port_to_nbr_port[port] nbrhost = self.nbrhosts[nbrname]["host"] if bring_up: nbrhost.no_shutdown(nbrport) else: nbrhost.shutdown(nbrport) fanout, fanport = fanout_switch_port_lookup(self.fanouthosts, self.duthost.hostname, port) if bring_up: fanout.no_shutdown(fanport) else: fanout.shutdown(fanport)
def __recover_interfaces(dut, fanouthosts, result, wait_time): action = None for port in result['down_ports']: logging.warning("Restoring port: {}".format(port)) pn = str(port).lower() if 'portchannel' in pn or 'vlan' in pn: action = 'config_reload' continue fanout, fanout_port = fanout_switch_port_lookup(fanouthosts, port) if fanout and fanout_port: fanout.no_shutdown(fanout_port) dut.no_shutdown(port) wait(wait_time, msg="Wait {} seconds for interface(s) to restore.".format(wait_time)) return action
def __build_test_candidates(self, dut, fanouthosts, completeness_level): status = self.__get_dut_if_status(dut) candidates = [] for dut_port in status.keys(): fanout, fanout_port = fanout_switch_port_lookup( fanouthosts, dut_port) if not fanout or not fanout_port: logging.info( "Skipping port {} that is not found in connection graph". format(dut_port)) elif status[dut_port]['admin_state'] == 'down': logging.info( "Skipping port {} that is admin down".format(dut_port)) else: candidates.append((dut_port, fanout, fanout_port)) if CompletenessLevel.debug in completeness_level: # Run the test for one port only - to just test if the test works fine return candidates return candidates
def mock_server(fanouthosts, testbed_params, arp_responder, ptfadapter, duthost): """ Mock the presence of a server beneath a T0. Returns: A MockServer which will allow the caller to mock the behavior of a server within a VLAN under a T0. """ server_dst_port = random.choice(testbed_params["vlan_ports"]) server_dst_addr = random.choice(arp_responder[server_dst_port].keys()) server_dst_intf = testbed_params["physical_port_map"][server_dst_port] logging.info( "Creating mock server with IP %s; dut port = %s, dut intf = %s", server_dst_addr, server_dst_port, server_dst_intf) logging.info("Clearing ARP and FDB tables for test setup") duthost.command("sonic-clear fdb all") duthost.command("sonic-clear arp") # Populate FDB logging.info("Populating FDB entry for mock server under VLAN") src_mac = _hex_to_mac(arp_responder[server_dst_port][server_dst_addr]) pkt = _get_simple_ip_packet(src_mac, duthost.get_dut_iface_mac(server_dst_intf), server_dst_addr, MOCK_DEST_IP) _send_packets(duthost, ptfadapter, pkt, server_dst_port, count=100) fanout_neighbor, fanout_intf = fanout_switch_port_lookup( fanouthosts, server_dst_intf) return { "server_dst_port": server_dst_port, "server_dst_addr": server_dst_addr, "server_dst_intf": server_dst_intf, "fanout_neighbor": fanout_neighbor, "fanout_intf": fanout_intf }
def test_front_panel_linkflap_port(self, duthosts, enum_rand_one_per_hwsku_frontend_hostname, enum_asic_index, all_cfg_facts, fanouthosts, setup, teardown, nbrhosts, established_arp): """ Verify tables, databases, and kernel routes are correctly deleted when the front panel port flaps. Test Steps * Admin down interface on fanout to cause LOS on DUT. * On local linecard: * Verify ARP/NDP entries are removed from CLI for neighbors on down port. * Verify table entries in ASIC, AppDb are removed for addresses on down port. * On Supervisor card: * Verify Chassis App DB entry are removed for only the cleared address. Entries for addresses on other line cards should still be present. * On remote linecards: * Verify table entries in ASICDB, APPDB, and host ARP table are removed for cleared addresses. * Verify kernel routes for cleared address are deleted. * Admin interface up, verify recreation after restarting traffic. Args: duthosts: duthosts fixture. enum_rand_one_per_hwsku_frontend_hostname: frontend iteration fixture. enum_asic_index: asic iteration fixture. all_cfg_facts: all_cfg_facts fixture from voq/conftest.py fanouthosts: fanouthosts fixture. setup: setup fixture for this module. nbrhosts: nbrhosts fixture. established_arp: Fixture to establish ARP to all neighbors. """ if fanouthosts == {}: pytest.skip("Fanouthosts fixture did not return anything, this test case can not run.") per_host = duthosts[enum_rand_one_per_hwsku_frontend_hostname] asic = per_host.asics[enum_asic_index if enum_asic_index is not None else 0] cfg_facts = all_cfg_facts[per_host.hostname][asic.asic_index]['ansible_facts'] if 'BGP_NEIGHBOR' in cfg_facts: neighs = cfg_facts['BGP_NEIGHBOR'] else: logger.info("No local neighbors for host: %s/%s, skipping", per_host.hostname, asic.asic_index) return intfs, intfs_to_test = pick_ports(cfg_facts) logger.info("Will test interfaces: %s", intfs_to_test) for intf in intfs_to_test: local_ips = [i.split("/")[0] for i in intfs[intf].keys()] # [u'2064:100::2/64', u'100.0.0.2/24'] neighbors = [n for n in neighs if neighs[n]['local_addr'] in local_ips] logger.info("Testing neighbors: %s on intf: %s", neighbors, intf) if "portchannel" in intf.lower(): pc_cfg = cfg_facts['PORTCHANNEL_MEMBER'] pc_members = pc_cfg[intf] logger.info("Portchannel members %s: %s", intf, pc_members.keys()) portbounce_list = pc_members.keys() else: portbounce_list = [intf] try: for lport in portbounce_list: fanout, fanport = fanout_switch_port_lookup(fanouthosts, per_host.hostname, lport) logger.info("fanout port: %s %s, host: %s", fanout, fanport, fanout.host) self.linkflap_down(fanout, fanport, per_host, lport) for neighbor in neighbors: check_one_neighbor_present(duthosts, per_host, asic, neighbor, nbrhosts, all_cfg_facts) finally: for lport in portbounce_list: fanout, fanport = fanout_switch_port_lookup(fanouthosts, per_host.hostname, lport) self.linkflap_up(fanout, fanport, per_host, lport) for neighbor in neighbors: check_one_neighbor_present(duthosts, per_host, asic, neighbor, nbrhosts, all_cfg_facts)
def test_link_flap(request, duthosts, rand_one_dut_hostname, tbinfo, fanouthosts, get_loop_times): """ Validates that link flap works as expected """ duthost = duthosts[rand_one_dut_hostname] orch_cpu_threshold = request.config.getoption("--orch_cpu_threshold") # Record memory status at start memory_output = duthost.shell("show system-memory")["stdout"] logger.info("Memory Status at start: %s", memory_output) # Record Redis Memory at start start_time_redis_memory = duthost.shell( "redis-cli info memory | grep used_memory_human | sed -e 's/.*:\(.*\)M/\\1/'" )["stdout"] logger.info("Redis Memory: %s M", start_time_redis_memory) # Make Sure Orch CPU < orch_cpu_threshold before starting test. logger.info( "Make Sure orchagent CPU utilization is less that %d before link flap", orch_cpu_threshold) pytest_assert( wait_until(100, 2, 0, check_orch_cpu_utilization, duthost, orch_cpu_threshold), "Orch CPU utilization {} > orch cpu threshold {} before link flap". format( duthost.shell( "show processes cpu | grep orchagent | awk '{print $9}'") ["stdout"], orch_cpu_threshold)) loop_times = get_loop_times port_lists = get_port_list(duthost, tbinfo) candidates = [] for port in port_lists: fanout, fanout_port = fanout_switch_port_lookup( fanouthosts, duthost.hostname, port) candidates.append((port, fanout, fanout_port)) for loop_time in range(0, loop_times): watch = False check_status = False if loop_time == 0 or loop_time == loop_times - 1: watch = True check_status = True for dut_port, fanout, fanout_port in candidates: toggle_one_link(duthost, dut_port, fanout, fanout_port, watch=watch, check_status=check_status) # Record memory status at end memory_output = duthost.shell("show system-memory")["stdout"] logger.info("Memory Status at end: %s", memory_output) # Record orchagent CPU utilization at end orch_cpu = duthost.shell( "show processes cpu | grep orchagent | awk '{print $9}'")["stdout"] logger.info("Orchagent CPU Util at end: %s", orch_cpu) # Record Redis Memory at end end_time_redis_memory = duthost.shell( "redis-cli info memory | grep used_memory_human | sed -e 's/.*:\(.*\)M/\\1/'" )["stdout"] logger.info("Redis Memory at start: %s M", start_time_redis_memory) logger.info("Redis Memory at end: %s M", end_time_redis_memory) # Calculate diff in Redis memory incr_redis_memory = float(end_time_redis_memory) - float( start_time_redis_memory) logger.info("Redis absolute difference: %d", incr_redis_memory) # Check redis memory only if it is increased else default to pass if incr_redis_memory > 0.0: percent_incr_redis_memory = (incr_redis_memory / float(start_time_redis_memory)) * 100 logger.info("Redis Memory percentage Increase: %d", percent_incr_redis_memory) pytest_assert( percent_incr_redis_memory < 5, "Redis Memory Increase more than expected: {}".format( percent_incr_redis_memory)) # Orchagent CPU should consume < orch_cpu_threshold at last. logger.info("watch orchagent CPU utilization when it goes below %d", orch_cpu_threshold) pytest_assert( wait_until(45, 2, 0, check_orch_cpu_utilization, duthost, orch_cpu_threshold), "Orch CPU utilization {} > orch cpu threshold {} before link flap". format( duthost.shell( "show processes cpu | grep orchagent | awk '{print $9}'") ["stdout"], orch_cpu_threshold))
def revert(self): for port in self.ports: fanout, fanport = fanout_switch_port_lookup( self.fanouthosts, self.duthost.hostname, port) if fanout and fanport: fanout.no_shutdown(fanport)