def Trigger(tc): naples_nodes = [] #for every node in the setup for node in tc.nodes: if api.IsNaplesNode(node): naples_nodes.append(node) api.Logger.info(f"Found Naples Node: [{node}]") if len(naples_nodes) == 0: api.Logger.error("Failed to find a Naples Node!") return api.types.status.FAILURE for reboot in range(tc.args.reboots): # Reboot Node. # Reboot method (APC, IPMI, OS Reboot) is passed as a testcase parameter for node in naples_nodes: api.Logger.info( f"==== Reboot Loop # {reboot} on {node}. Reboot method: {tc.iterators.reboot_method} ====" ) if api.RestartNodes( [node], tc.iterators.reboot_method) != api.types.status.SUCCESS: return api.types.status.FAILURE #raise OfflineTestbedException # Enable SSH, some checks require Naples access if enable_ssh.Main(None) != api.types.status.SUCCESS: api.Logger.error("Enabling SSH failed") return api.types.status.FAILURE # there is not a real "PCI" in IOTA for Windows. if tc.os != host.OS_TYPE_WINDOWS: api.Logger.info(f"Verifying PCI on [{node}]: ") if verify_pci.verify_errors_lspci( node, tc.os) != api.types.status.SUCCESS: api.Logger.error(f"PCIe Failure detected on {node}") return api.types.status.FAILURE #raise OfflineTestbedException if checkLinks(tc, node) is api.types.status.FAILURE: api.Logger.error("Error verifying uplink interfaces") return api.types.status.FAILURE #raise OfflineTestbedException # don't run this while we figure out how to do this in ESX if tc.os != host.OS_TYPE_ESX: # Load the ionic driver except windows if tc.os != host.OS_TYPE_WINDOWS: if host.LoadDriver(tc.os, node) is api.types.status.FAILURE: api.Logger.info("ionic already loaded") # Make sure ionic driver attached to Uplink ports. if checkDrv(node) is api.types.status.FAILURE: api.Logger.error("No ionic uplink interfaces detected") return api.types.status.FAILURE #raise OfflineTestbedException return api.types.status.SUCCESS
def Verify(tc): for node in tc.nodes: # this is required to bring the testbed into operation state # after driver unload interfaces need to be initialized host.LoadDriver(api.GetNodeOs(node), node) wl_api.ReAddWorkloads(node) return api.types.status.SUCCESS
def do_lif_reset_test(node, os): for i in range(3): api.Logger.info("LIF reset and driver reload test loop %d" % i) if host.UnloadDriver(os, node, "all") is api.types.status.FAILURE: api.Logger.error("ionic unload failed loop %d" % i) return api.types.status.FAILURE if host.LoadDriver(os, node) is api.types.status.FAILURE: api.Logger.error("ionic load failed loop %d" % i) return api.types.status.FAILURE wl_api.ReAddWorkloads(node) if api.GetNaplesHostInterfaces(node) is None: api.Logger.error("No ionic interface after loop %d" % i) return api.types.status.FAILURE for intf in api.GetNaplesHostInterfaces(node): req = api.Trigger_CreateExecuteCommandsRequest(serial=True) vlan_list = getVlanList(node, intf) filter_list = getFilterList(node, intf) # Single LIF reset api.Trigger_AddHostCommand( req, node, "sysctl dev.%s.reset=1" % (host.GetNaplesSysctl(intf))) resp = api.Trigger(req) time.sleep(5) vlan_list1 = getVlanList(node, intf) filter_list1 = getFilterList(node, intf) if vlan_list != vlan_list1: api.Logger.error( "VLAN list doesn't match for %s, before: %s after: %s" % (intf, str(vlan_list), str(vlan_list1))) return api.types.status.FAILURE if filter_list != filter_list1: api.Logger.error( "Filter list doesn't match for %s, before: %s after: %s" % (intf, str(filter_list), str(filter_list1))) return api.types.status.FAILURE api.Logger.info( "Success running LIF reset test on %s VLAN: %s, Filters; %s" % (intf, str(vlan_list), str(filter_list))) # Now stress test LIF reset for intf in api.GetNaplesHostInterfaces(node): req = api.Trigger_CreateExecuteCommandsRequest(serial=True) api.Trigger_AddHostCommand( req, node, "for ((i=0;i<10;i++)); do sysctl dev.%s.reset=1; done &" % (host.GetNaplesSysctl(intf))) # Some of LIF reset will fill fail since it will be running in background # with reload of driver. resp = api.Trigger(req) return api.types.status.SUCCESS
def Teardown(tc): hostname = tc.host os_name = api.GetNodeOs(hostname) # restore the workloads on the one host we tested if host.UnloadDriver(os_name, hostname, "eth") is api.types.status.FAILURE: return api.types.status.FAILURE if host.LoadDriver(os_name, hostname) is api.types.status.FAILURE: return api.types.status.FAILURE hw_config.ReAddWorkloads(hostname) return api.types.status.SUCCESS
def Verify(tc): hostname = tc.nodes[0] # restore the workloads on the one host we tested if host.UnloadDriver(tc.os, hostname, "eth") is api.types.status.FAILURE: return api.types.status.FAILURE if host.LoadDriver(tc.os, hostname) is api.types.status.FAILURE: return api.types.status.FAILURE bringup.ReAddWorkloads(hostname) return api.types.status.SUCCESS
def Teardown(tc): if tc.os != host.OS_TYPE_BSD: api.Logger.info("Not implemented") return api.types.status.IGNORED # for every node, cycle through unload/load sequence for node in tc.nodes: if host.UnloadDriver(tc.os, node, "eth") is api.types.status.FAILURE: return api.types.status.FAILURE if host.LoadDriver(tc.os, node) is api.types.status.FAILURE: return api.types.status.FAILURE # this is required to bring the testbed into operation state; # after driver unload interfaces need to be initialized wl_api.ReAddWorkloads(node) return api.types.status.SUCCESS
def Trigger(tc): if tc.os != host.OS_TYPE_BSD: api.Logger.info("Not implemented") return api.types.status.IGNORED # Unload ionic and ionic_fw for node in tc.nodes: host.UnloadDriver(tc.os, node, "all") host.UnloadDriver(tc.os, node, "ionic_fw") for node in tc.nodes: req = api.Trigger_CreateExecuteCommandsRequest(serial=True) # XXX: Find the Naples_fw.tar version. api.Trigger_AddHostCommand( req, node, "kenv hw.ionic.fw_update_ver=FILL_FW_VERSION") resp = api.Trigger(req) if resp is None: api.Logger.info("Failed kenv hw.ionic.fw_update_ver=X") return api.types.status.FAILURE if host.LoadDriver(tc.os, node) is api.types.status.FAILURE: api.Logger.info("ionic already loaded") return api.types.status.FAILURE if LoadFwDriver(tc.os, node) is api.types.status.FAILURE: return api.types.status.FAILURE for i in api.GetNaplesHostInterfaces(node): # # In local testing, this step completes in 35-40s, but the default # timeout is 30s. Therefore, increase the timeout to 60s. # # The iota logs may contain messages such as "CHECK_ERR: Nicmgr # crashed for host: node2?" Please note, this is due to finding # the string "fw heartbeat stuck" in the host dmesg. This is # currently the expected behavior when doing fw update. If nicmgr # does crash, than expect subsequent tests to fail, otherwise the # CHECK_ERR message in the iota test logs may be ignored. # api.Trigger_AddHostCommand(req, node, "sysctl dev.%s.fw_update=1" % host.GetNaplesSysctl(i), timeout=60) tc.resp = api.Trigger(req) return api.types.status.SUCCESS
def Teardown(tc): # Clear the ip addr on the remote interface ClearIps(tc.host2, tc.pf_2) # Delte created VFs DeleteVFs(tc.host1, tc.pf_1) for hostname in [tc.host1, tc.host2]: os_name = api.GetNodeOs(hostname) # restore the workloads on the one host we tested if host.UnloadDriver(os_name, hostname, "eth") is api.types.status.FAILURE: return api.types.status.FAILURE if host.LoadDriver(os_name, hostname) is api.types.status.FAILURE: return api.types.status.FAILURE hw_config.ReAddWorkloads(hostname) return api.types.status.SUCCESS
def Trigger(tc): loops = getattr(tc.args, "loops", 1) # TOOD: add traffic to the test case # first run through unload to init nodes for node in tc.nodes: host.UnloadDriver(tc.os, node, "all") # for every node, cycle through unload/load sequence for node in tc.nodes: for i in range(0, loops): if host.UnloadDriver(tc.os, node, "eth") is api.types.status.FAILURE: return api.types.status.FAILURE if host.LoadDriver(tc.os, node) is api.types.status.FAILURE: return api.types.status.FAILURE return api.types.status.SUCCESS
def Trigger(tc): errcnt = 0 tc.intf_id = {} # assume that the driver is loaded for node in api.GetNaplesHostnames(): errcnt += testLoadedMulticast(node, 2) errcnt += testLoadedLifs(tc, node) for node in api.GetNaplesHostnames(): # set up background tests # each of these seems to take about 4 seconds, so be sure to # increase the command delays for each when another is added prepUnloadedMulticastTest(node) prepUnloadedLifs(tc, node) # unload driver and give time for the background tests to finish api.Logger.info("Start driver unload on " + node) if host.UnloadDriver(tc.os, node, "eth") is api.types.status.FAILURE: return api.types.status.FAILURE time.sleep(10) # restart the drivers and gather the results if host.LoadDriver(tc.os, node) is api.types.status.FAILURE: return api.types.status.FAILURE wl_api.ReAddWorkloads(node) api.Logger.info("Driver reload on " + node) errcnt += gatherUnloadedMulticastTest(node) errcnt += gatherUnloadedLifs(tc, node) if errcnt != 0: return api.types.status.FAILURE return api.types.status.SUCCESS
def Trigger(tc): if tc.os != host.OS_TYPE_BSD and tc.os != host.OS_TYPE_LINUX: api.Logger.info("Not implemented for %s" % tc.os) return api.types.status.IGNORED loops = getattr(tc.args, "loops", 1) # TOOD: add traffic to the test case # first run through unload to init nodes for node in tc.nodes: host.UnloadDriver(tc.os, node, "all") # for every node, cycle through unload/load sequence for node in tc.nodes: for i in range(0, loops): if host.UnloadDriver(tc.os, node, "eth") is api.types.status.FAILURE: return api.types.status.FAILURE if host.LoadDriver(tc.os, node) is api.types.status.FAILURE: return api.types.status.FAILURE return api.types.status.SUCCESS
def Setup(tc): #map iterators from IOTA infra to the enum for the toeplitz utilitys iterators = {'tcp4' : RSS.IPV4_TCP, 'udp4':RSS.IPV4_UDP, 'tcp6' : RSS.IPV6_TCP, \ 'udp6':RSS.IPV6_UDP, 'ip6':RSS.IPV6, 'ip4tcp':RSS.IPV4, 'ip4udp':RSS.IPV4, 'ip6tcp':RSS.IPV6, 'ip6udp':RSS.IPV6} tc.rss_enum = iterators.get(tc.iterators.rxflowhash, "none") # iperf options for iterators # IPv4 vs IPv6 iterator ip_proto_iterators = {'tcp4' : 'v4', 'udp4':'v4', 'tcp6' : 'v6', 'udp6':'v6', 'ip4tcp':'v4', \ 'ip4udp':'v4', 'ip6tcp':'v6', 'ip6udp':'v6'} tc.tc_ip_proto = ip_proto_iterators.get(tc.iterators.rxflowhash, "none") # UDP vs TCP iterator proto_iterators = {'tcp4' : 'tcp', 'udp4':'udp', 'tcp6':'tcp', 'udp6':'udp', 'ip4tcp':'tcp', 'ip4udp':'udp', 'ip6tcp':'tcp', 'ip6udp':'udp'} tc.proto = proto_iterators.get(tc.iterators.rxflowhash, "none") if tc.proto == "none" or tc.tc_ip_proto == "none " or tc.rss_enum == "none": api.Logger.error(f"Not able to map the iterators. {tc.iterators.rxflowhash} {tc.tc_ip_proto} {tc.proto} {tc.rss_enum}") return api.types.status.FAILURE # number of sessions iterator tc.num_sessions = int(getattr(tc.args, "num_sessions", 1)) # log which iterration is in progress: api.Logger.info(f"==================== %s ====================" % tc.rss_enum) api.Logger.info(f"ip_proto:{tc.tc_ip_proto}, proto: {tc.proto}, rss: {tc.iterators.rss}, iperf_sessions: {tc.num_sessions}") tc.nodes = api.GetWorkloadNodeHostnames() tc.os = api.GetNodeOs(tc.nodes[0]) if tc.os == 'freebsd': return api.types.status.SUCCESS # Identify the receiving node(testing RSS) # This will be the client node for iPerf # All configuration, and testing and verification will be done on this node for n in tc.nodes: if api.IsNaplesNode(n): api.Logger.info(f"Found Naples Node: [{n}]") receive_node = n break else: api.Logger.error(f"Failed to find a Naples Node!") return api.types.status.FAILURE # Get workload pars for iperf sessions workload_pairs = api.GetRemoteWorkloadPairs() if not workload_pairs: api.Logger.info("Skipping Testcase due to no workload pairs.") tc.skip = True # assign client/server node based on selected Receiving Node for pair in workload_pairs: if receive_node == pair[0].node_name: tc.client = pair[1] tc.server = pair[0] else: tc.client = pair[0] tc.server = pair[1] break # unload driver, to clear stats (server node only) # Re-add workloads if host.UnloadDriver(tc.os, tc.server.node_name, "all") is api.types.status.FAILURE: return api.types.status.FAILURE if host.LoadDriver(tc.os, tc.server.node_name) is api.types.status.FAILURE: return api.types.status.FAILURE wl_api.ReAddWorkloads(tc.server.node_name) if tc.tc_ip_proto == 'v6': tc.server_ip = ipaddress.ip_address(tc.server.ipv6_address) tc.server_ip = str(tc.server_ip.exploded) tc.client_ip = ipaddress.ip_address(tc.client.ipv6_address) tc.client_ip = str(tc.client_ip.exploded) else: tc.server_ip = tc.server.ip_address tc.client_ip = tc.client.ip_address return api.types.status.SUCCESS