def Teardown(tc): req = api.Trigger_CreateExecuteCommandsRequest() for node in tc.Nodes: api.Trigger_AddNaplesCommand( req, node, "rm -rf /update/upgrade_halt_state_machine") api.Trigger_AddNaplesCommand(req, node, "rm -rf /update/pcieport_upgdata") api.Trigger_AddNaplesCommand(req, node, "rm -rf /update/pciemgr_upgdata") api.Trigger_AddNaplesCommand(req, node, "rm -rf /update/pciemgr_upgrollback") api.Trigger_AddNaplesCommand(req, node, "rm -rf /update/nicmgr_upgstate") api.Trigger_AddNaplesCommand( req, node, "rm -rf /data/upgrade_to_same_firmware_allowed") resp = api.Trigger(req) for cmd_resp in resp.commands: api.PrintCommandResults(cmd_resp) if cmd_resp.exit_code != 0: api.Logger.error("Setup failed %s", cmd_resp.command) req = api.Trigger_CreateExecuteCommandsRequest() for n in tc.Nodes: cmd = 'curl -k -X DELETE https://' + api.GetNicIntMgmtIP( n) + ':8888/api/v1/naples/rollout/' api.Trigger_AddHostCommand(req, n, cmd) tc.resp = api.Trigger(req) for cmd in tc.resp.commands: api.PrintCommandResults(cmd) if cmd.exit_code != 0: return api.types.status.FAILURE return api.types.status.SUCCESS
def Trigger(tc): naples_nodes = api.GetNaplesHostnames() for node in naples_nodes: api.Logger.info("Start second athena_app to pick up policy.json") req = api.Trigger_CreateExecuteCommandsRequest() api.Trigger_AddNaplesCommand(req, node, "/nic/tools/start-sec-agent.sh") api.Trigger_AddNaplesCommand(req, node, "\r\n") resp = api.Trigger(req) cmd = resp.commands[0] api.PrintCommandResults(cmd) if cmd.exit_code != 0: api.Logger.error("Start second athena_app failed on node {}".format(node)) return api.types.status.FAILURE time.sleep(10) for node in naples_nodes: req = api.Trigger_CreateExecuteCommandsRequest() api.Trigger_AddNaplesCommand(req, node, "ps -aef | grep athena_app") resp = api.Trigger(req) cmd = resp.commands[0] api.PrintCommandResults(cmd) if cmd.exit_code != 0: api.Logger.error("ps failed on Node {}".format(node)) return api.types.status.FAILURE if "athena_app" not in cmd.stdout: # TODO: If athena_app is not running, run start_agent.sh manually api.Logger.error("no athena_app running on Node {}, need to start athena_app first".format(node)) return api.types.status.FAILURE athena_sec_app_pid = cmd.stdout.strip().split()[1] api.Logger.info("athena_app up and running on Node {} with PID {}".format(node, athena_sec_app_pid)) return api.types.status.SUCCESS
def Trigger(tc): req_uname = api.Trigger_CreateExecuteCommandsRequest(serial = True) req = api.Trigger_CreateExecuteCommandsRequest(serial = True) # Move driver package into position and build on naples nodes for n in tc.nodes: api.Logger.info("Building RDMA drivers and tools on: {node}".format(node=n)) if tc.os == host.OS_TYPE_LINUX: api.Trigger_AddHostCommand(req, n, "mkdir -p {path} && mv {pkg} {path} && cd {path} && tar -xma --strip-components 1 -f {pkg} && ./setup_libs.sh && ./build.sh" .format(path=tc.iota_path, pkg=tc.pkgname), timeout = 180) else: api.Trigger_AddHostCommand(req, n, "mkdir -p {path} && mv {pkg} {path} && cd {path} && tar -xm --strip-components 1 -f {pkg} && ./build.sh" .format(path=tc.iota_path, pkg=tc.pkgname), timeout = 180) api.Trigger_AddHostCommand(req_uname, n, "uname -r") # Move show_gid into position on other nodes for n in tc.other_nodes: if n in tc.nodes: continue api.Logger.info("Moving show_gid to tools on {node}" .format(node=n)) api.Trigger_AddHostCommand(req, n, "mkdir -p {path} && mv show_gid {path}" .format(path=tc.iota_path)) api.Trigger_AddHostCommand(req_uname, n, "uname -r") tc.resp_uname = api.Trigger(req_uname) tc.resp = api.Trigger(req) return api.types.status.SUCCESS
def Verify(tc): rc = api.types.status.SUCCESS req = api.Trigger_CreateExecuteCommandsRequest(serial=True) api.Trigger_AddNaplesCommand(req, tc.Nodes[0], 'sleep 3') resp = api.Trigger(req) for node, intr in intr_list: req = api.Trigger_CreateExecuteCommandsRequest(serial=True) api.Trigger_AddNaplesCommand(req, node, intr.get_count_cmd()) resp = api.Trigger(req) cmd = resp.commands[0] if cmd.exit_code != 0: api.Logger.error( "Command failed: Node {}, Interrupt {}, Field {}".format( node, intr.name(), intr.field())) return api.types.status.FAILURE expected = intr.count() + 1 value = intr.parse_count_cmd_output(cmd.stdout) if value < expected: api.Logger.error( "Node {}, Interrupt {}, Field {}, Expected {}, Got {}".format( node, intr.name(), intr.field(), expected, value)) rc = api.types.status.FAILURE return rc
def Teardown(tc): if tc.skip: return api.types.status.SUCCESS req = api.Trigger_CreateExecuteCommandsRequest() for node in tc.Nodes: api.Trigger_AddNaplesCommand(req, node, "rm -rf /data/upgrade_to_same_firmware_allowed") resp = api.Trigger(req) try: for cmd_resp in resp.commands: if cmd_resp.exit_code != 0: api.PrintCommandResults(cmd_resp) api.Logger.error("Teardown failed %s", cmd_resp.command) except: api.Logger.error("EXCEPTION occured in Naples command") return api.types.status.FAILURE req = api.Trigger_CreateExecuteCommandsRequest() for n in tc.Nodes: cmd = 'curl -k -X DELETE https://' + api.GetNicIntMgmtIP(n) + ':'+utils.GetNaplesMgmtPort()+'/api/v1/naples/rollout/' api.Trigger_AddHostCommand(req, n, cmd, timeout=100) tc.resp = api.Trigger(req) for cmd in tc.resp.commands: api.PrintCommandResults(cmd) if cmd.exit_code != 0: return api.types.status.FAILURE return api.types.status.SUCCESS
def Trigger(tc): # check that there have been packets showing up after previous tests # ip -s link show <pf> rate = tc.iterators.rate duration = 30 if __SetAndCheckRate(tc.host1, tc.pf_1, tc.vfid, rate) != api.types.status.SUCCESS: return api.types.status.ERROR servercmd = iperf.ServerCmd(server_ip=tc.remote_ip, port=7777, run_core=2) clientcmd = iperf.ClientCmd(tc.remote_ip, client_ip=tc.vf_ip, jsonOut=True, port=7777, proto='tcp', time=duration, run_core=2) sreq = api.Trigger_CreateExecuteCommandsRequest(serial=True) creq = api.Trigger_CreateExecuteCommandsRequest(serial=True) api.Trigger_AddHostCommand(sreq, tc.host2, servercmd, background=True) api.Trigger_AddHostCommand(creq, tc.host1, clientcmd, timeout=3600) server_resp = api.Trigger(sreq) if not server_resp: api.Logger.error("Unable to execute server command") return api.types.status.ERROR time.sleep(5) client_resp = api.Trigger(creq) if not client_resp: api.Logger.error("Unable to execute client command") return api.types.status.ERROR resp = client_resp.commands.pop() if resp.exit_code != 0: api.Logger.error("Iperf client failed with exit code %d" % resp.exit_code) api.PrintCommandResults(resp) return api.types.status.ERROR if not iperf.Success(resp.stdout): api.Logger.error("Iperf failed with error: %s" % iperf.Error(resp.stdout)) return api.types.status.ERROR api.Logger.info("Obs rate %sMbps" % iperf.GetSentMbps(resp.stdout)) obs_rate = float(iperf.GetSentMbps(resp.stdout)) delta = (abs(obs_rate - rate) * 100) / rate if delta > 10: api.Logger.error("Configured Tx rate %f but observed %f delta %f%%" % (rate, obs_rate, delta)) return api.types.status.FAILURE api.Trigger_TerminateAllCommands(server_resp) return api.types.status.SUCCESS
def GetHostInternalMgmtInterfaces(node, device = None): # Relay on IOTA infra to provide this information (dual-nic friendly API) if api.IsNaplesNode(node): interface_names = api.GetNaplesHostMgmtInterfaces(node, device) if interface_names: return interface_names interface_names = [] req = api.Trigger_CreateExecuteCommandsRequest(serial = True) if api.GetNodeOs(node) == OS_TYPE_LINUX: pci_bdf_list = [] #find pci bdf first for mgmt device which has deviceId as 1004 cmd = "lspci -d :1004 | cut -d' ' -f1" api.Trigger_AddHostCommand(req, node, cmd) resp = api.Trigger(req) #find the interface name for all the pci_bdfs for all the mgmt interfaces pci_bdf_list = resp.commands[0].stdout.split("\n") for pci_bdf in pci_bdf_list: if (pci_bdf != ''): cmd = "ls /sys/bus/pci/devices/0000:" + pci_bdf + "/net/" req = api.Trigger_CreateExecuteCommandsRequest(serial = True) api.Trigger_AddHostCommand(req, node, cmd) resp = api.Trigger(req) for command in resp.commands: #iface_name = None iface_name = command.stdout interface_names.append(iface_name.strip("\n")) elif api.GetNodeOs(node) == OS_TYPE_ESX: #For now hardcoding. return ["eth1"] elif api.GetNodeOs(node) == OS_TYPE_WINDOWS: entries = GetWindowsPortMapping(node) if len(entries) == 0: return [] maxbus = 0 name = "" for k, v in entries.items(): if int(v["Bus"]) > maxbus: maxbus = int(v["Bus"]) name = k return [name] else: cmd = "pciconf -l | grep chip=0x10041dd8 | cut -d'@' -f1 | sed \"s/ion/ionic/g\"" api.Trigger_AddHostCommand(req, node, cmd) resp = api.Trigger(req) for command in resp.commands: iface_name = command.stdout interface_names.append(iface_name.strip("\n")) return interface_names
def Trigger(tc): if api.IsDryrun(): return api.types.status.SUCCESS srv = tc.workloads[0] cli = tc.workloads[1] # Determine where the commands will be run - host or Naples. test_type = getattr(tc.args, "test-type", INTF_TEST_TYPE_HOST) is_naples_cmd = True if test_type == INTF_TEST_TYPE_HOST: is_naples_cmd = False srv_req = api.Trigger_CreateExecuteCommandsRequest(serial = False) cli_req = api.Trigger_CreateExecuteCommandsRequest(serial = False) proto = getattr(tc.iterators, "proto", 'tcp') number_of_iperf_threads = getattr(tc.args, "iperfthreads", 1) pktsize = getattr(tc.iterators, "pktsize", None) ipproto = getattr(tc.iterators, "ipproto", 'v4') if ipproto == 'v4': server_ip = srv.ip_address client_ip = cli.ip_address else: server_ip = srv.ipv6_address client_ip = cli.ipv6_address tc.cmd_descr = "Server: %s(%s) <--> Client: %s(%s)" %\ (srv.interface, server_ip, cli.interface, client_ip) api.Logger.info("Starting Iperf(%s/%s) test from %s" % (proto, ipproto, tc.cmd_descr)) duration = 10 for i in range(number_of_iperf_threads): if proto == 'tcp': port = api.AllocateTcpPort() else: port = api.AllocateUdpPort() iperf_server_cmd = iperf.ServerCmd(port, naples = is_naples_cmd) api.Trigger_AddCommand(srv_req, srv.node_name, srv.workload_name, iperf_server_cmd, background = True) iperf_client_cmd = iperf.ClientCmd(server_ip, port, time=duration, proto=proto, jsonOut=True, ipproto=ipproto, pktsize=pktsize, client_ip=client_ip, naples = is_naples_cmd) api.Trigger_AddCommand(cli_req, cli.node_name, cli.workload_name, iperf_client_cmd, timeout = 60) srv_resp = api.Trigger(srv_req) # Wait for iperf server to start. time.sleep(10) tc.cli_resp = api.Trigger(cli_req) # Wait for iperf clients to finish. time.sleep(2*duration) srv_resp1 = api.Trigger_TerminateAllCommands(srv_resp) return api.types.status.SUCCESS
def Trigger(tc): req = None req = api.Trigger_CreateExecuteCommandsRequest() tc.cmd_cookies = [] ping_count = getattr(tc.args, "ping_count", 20) interval = getattr(tc.args, "ping_interval", 0.01) connectivity = tc.iterators.connectivity naplesHosts = api.GetNaplesHostnames() tc_intf = (tc.iterators.interface).capitalize() if tc_intf in ['Uplink0', 'Uplink1', 'Uplinks']: setDataPortStatePerUplink(naplesHosts, tc.iterators.port_status, tc_intf) if tc_intf in ['Switchport0', 'Switchport1', 'Switchports']: switchPortOp(naplesHosts, tc.iterators.port_status, tc_intf) if connectivity == 'bgp_peer': for node in naplesHosts: for bgppeer in bgp_peer.client.Objects(node): # Dont try to ping on the down interface if tc.iterators.port_status == 'down': if tc_intf == 'Switchport0' and bgppeer.Id == 1: continue elif tc_intf == 'Switchport1' and bgppeer.Id == 2: continue cmd_cookie = "%s --> %s" %\ (str(bgppeer.LocalAddr), str(bgppeer.PeerAddr)) api.Trigger_AddNaplesCommand(req, node, \ "ping -i %f -c %d -s %d %s" % \ (interval, ping_count, tc.iterators.pktsize, \ str(bgppeer.PeerAddr))) api.Logger.info("Ping test from %s" % (cmd_cookie)) tc.cmd_cookies.append(cmd_cookie) tc.resp = api.Trigger(req) else: for node1 in naplesHosts: for node2 in naplesHosts: if node1 == node2: continue objs = device.client.Objects(node1) device1 = next(iter(objs)) objs = device.client.Objects(node2) device2 = next(iter(objs)) cmd_cookie = "%s --> %s" %\ (device1.IP, device2.IP) api.Trigger_AddNaplesCommand(req, node1, \ "ping -i %f -c %d -s %d %s" % \ (interval, ping_count, tc.iterators.pktsize, \ device2.IP)) api.Logger.info("Loopback ping test from %s" % (cmd_cookie)) tc.cmd_cookies.append(cmd_cookie) tc.resp = api.Trigger(req) return api.types.status.SUCCESS
def Trigger(tc): # move device.json cmd = "mv /device.json /nic/conf/" resp = api.RunNaplesConsoleCmd(tc.nodes[0], cmd) # load drivers cmd = "insmod /nic/bin/ionic_mnic.ko && insmod /nic/bin/mnet_uio_pdrv_genirq.ko && insmod /nic/bin/mnet.ko" resp = api.RunNaplesConsoleCmd(tc.nodes[0], cmd) # start athena app cmd = "/nic/tools/start-agent-skip-dpdk.sh" resp = api.RunNaplesConsoleCmd(tc.nodes[0], cmd) # wait for athena app to be up utils.Sleep(80) # configure int_mnic0 cmd = "ifconfig int_mnic0 " + tc.int_mnic_ip + " netmask 255.255.255.0" resp = api.RunNaplesConsoleCmd(tc.nodes[0], cmd) # run plugctl to gracefully bring up the PCI device on host req = api.Trigger_CreateExecuteCommandsRequest(serial=True) cmd = "./plugctl.sh in" api.Trigger_AddHostCommand(req, tc.bitw_node_name, cmd) resp = api.Trigger(req) cmd = resp.commands[0] api.PrintCommandResults(cmd) if cmd.exit_code != 0: api.Logger.error("Failed to gracefully bring up the PCI device on host %s" % \ tc.bitw_node_name) return api.types.status.FAILURE # get host internal mgmt intf host_intfs = naples_host.GetHostInternalMgmtInterfaces(tc.bitw_node_name) # Assuming single nic per host if len(host_intfs) == 0: api.Logger.error('Failed to get host interfaces') return api.types.status.FAILURE intf = host_intfs[0] ip_addr = str(ip_address(tc.int_mnic_ip.rstrip()) + 1) req = api.Trigger_CreateExecuteCommandsRequest(serial=True) cmd = "ifconfig " + str(intf) + " " + ip_addr + "/24 up" api.Trigger_AddHostCommand(req, tc.bitw_node_name, cmd) resp = api.Trigger(req) cmd = resp.commands[0] api.PrintCommandResults(cmd) if cmd.exit_code != 0: api.Logger.error("Failed to gracefully bring up the internal mgmt intf on host %s" % \ tc.bitw_node_name) return api.types.status.FAILURE return api.types.status.SUCCESS
def __installPenCtl(node): fullpath = api.GetTopDir() + '/' + common.PENCTL_PKG resp = api.CopyToHost(node, [fullpath], "") if resp is None: return api.types.status.FAILURE if resp.api_response.api_status != types_pb2.API_STATUS_OK: api.Logger.error("Failed to copy penctl to Node: %s" % node) return api.types.status.FAILURE fullpath = api.GetTopDir() + '/' + common.PENCTL_TOKEN_FILE resp = api.CopyToHost(node, [fullpath], "") if resp is None: return api.types.status.FAILURE if resp.api_response.api_status != types_pb2.API_STATUS_OK: api.Logger.error("Failed to copy penctl token to Node: %s" % node) return api.types.status.FAILURE req = api.Trigger_CreateExecuteCommandsRequest() api.Trigger_AddHostCommand(req, node, "tar -xvf %s" % os.path.basename(common.PENCTL_PKG) + " && sync", background = False) #Create a symlink at top level execName = __penctl_exec(node) realPath = "realpath %s/%s " % (common.PENCTL_DEST_DIR, execName) api.Trigger_AddHostCommand(req, node, realPath, background = False) resp = api.Trigger(req) for cmd in resp.commands: api.PrintCommandResults(cmd) if cmd.exit_code != 0: return api.types.status.FAILURE common.PENCTL_EXEC[node] = resp.commands[1].stdout.split("\n")[0] req = api.Trigger_CreateExecuteCommandsRequest() #Create a symlink at top level realPath = "realpath %s " % (common.PENCTL_TOKEN_FILE_NAME) api.Trigger_AddHostCommand(req, node, realPath, background = False) resp = api.Trigger(req) for cmd in resp.commands: api.PrintCommandResults(cmd) if cmd.exit_code != 0: return api.types.status.FAILURE common.PENCTL_TOKEN[node] = resp.commands[0].stdout.split("\n")[0] return api.types.status.SUCCESS
def do_lif_reset_test(node, os): for i in range(3): api.Logger.info("LIF reset and driver reload test loop %d" % i) if host.UnloadDriver(os, node, "all") is api.types.status.FAILURE: api.Logger.error("ionic unload failed loop %d" % i) return api.types.status.FAILURE if host.LoadDriver(os, node) is api.types.status.FAILURE: api.Logger.error("ionic load failed loop %d" % i) return api.types.status.FAILURE wl_api.ReAddWorkloads(node) if api.GetNaplesHostInterfaces(node) is None: api.Logger.error("No ionic interface after loop %d" % i) return api.types.status.FAILURE for intf in api.GetNaplesHostInterfaces(node): req = api.Trigger_CreateExecuteCommandsRequest(serial=True) vlan_list = getVlanList(node, intf) filter_list = getFilterList(node, intf) # Single LIF reset api.Trigger_AddHostCommand( req, node, "sysctl dev.%s.reset=1" % (host.GetNaplesSysctl(intf))) resp = api.Trigger(req) time.sleep(5) vlan_list1 = getVlanList(node, intf) filter_list1 = getFilterList(node, intf) if vlan_list != vlan_list1: api.Logger.error( "VLAN list doesn't match for %s, before: %s after: %s" % (intf, str(vlan_list), str(vlan_list1))) return api.types.status.FAILURE if filter_list != filter_list1: api.Logger.error( "Filter list doesn't match for %s, before: %s after: %s" % (intf, str(filter_list), str(filter_list1))) return api.types.status.FAILURE api.Logger.info( "Success running LIF reset test on %s VLAN: %s, Filters; %s" % (intf, str(vlan_list), str(filter_list))) # Now stress test LIF reset for intf in api.GetNaplesHostInterfaces(node): req = api.Trigger_CreateExecuteCommandsRequest(serial=True) api.Trigger_AddHostCommand( req, node, "for ((i=0;i<10;i++)); do sysctl dev.%s.reset=1; done &" % (host.GetNaplesSysctl(intf))) # Some of LIF reset will fill fail since it will be running in background # with reload of driver. resp = api.Trigger(req) return api.types.status.SUCCESS
def Trigger(tc): if tc.skip: return api.types.status.SUCCESS tc.serverCmds = [] tc.clientCmds = [] tc.cmd_descr = [] if not api.IsSimulation(): serverReq = api.Trigger_CreateAllParallelCommandsRequest() clientReq = api.Trigger_CreateAllParallelCommandsRequest() else: serverReq = api.Trigger_CreateExecuteCommandsRequest(serial = False) clientReq = api.Trigger_CreateExecuteCommandsRequest(serial = False) for idx, pairs in enumerate(tc.workload_pairs): client = pairs[0] server = pairs[1] cmd_descr = "Server: %s(%s) <--> Client: %s(%s)" %\ (server.workload_name, server.ip_address, client.workload_name, client.ip_address) tc.cmd_descr.append(cmd_descr) num_sessions = int(getattr(tc.args, "num_sessions", 1)) api.Logger.info("Starting Iperf test from %s num-sessions %d" % (cmd_descr, num_sessions)) if tc.iterators.proto == 'udp': port = api.AllocateTcpPort() serverCmd = iperf.ServerCmd(port) clientCmd = iperf.ClientCmd(server.ip_address, port, proto='udp', jsonOut=True, num_of_streams = num_sessions) else: port = api.AllocateUdpPort() serverCmd = iperf.ServerCmd(port) clientCmd = iperf.ClientCmd(server.ip_address, port, jsonOut=True, num_of_streams = num_sessions) tc.serverCmds.append(serverCmd) tc.clientCmds.append(clientCmd) api.Trigger_AddCommand(serverReq, server.node_name, server.workload_name, serverCmd, background = True) api.Trigger_AddCommand(clientReq, client.node_name, client.workload_name, clientCmd) server_resp = api.Trigger(serverReq) #Sleep for some time as bg may not have been started. time.sleep(30) tc.iperf_client_resp = api.Trigger(clientReq) #Its faster kill iperf servers #Still call terminate on all api.Trigger_TerminateAllCommands(server_resp) return api.types.status.SUCCESS
def athena_sec_app_start(node_name=None, nic_name=None, init_wait_time=INIT_WAIT_TIME_DEFAULT): node_nic_names = [] if (not node_name and nic_name) or (node_name and not nic_name): raise Exception("specify both node_name and nic_name or neither") if node_name and nic_name: node_nic_names.append((node_name, nic_name)) else: node_nic_names = get_athena_node_nic_names() for nname, nicname in node_nic_names: req = api.Trigger_CreateExecuteCommandsRequest() cmd = "/nic/tools/start-sec-agent-iota.sh" api.Trigger_AddNaplesCommand(req, nname, cmd, nicname, background=True) resp = api.Trigger(req) cmd = resp.commands[0] api.PrintCommandResults(cmd) if cmd.exit_code != 0: api.Logger.error("command to start athena sec app failed on " "node %s nic %s" % (nname, nicname)) return api.types.status.FAILURE # sleep for init to complete misc_utils.Sleep(init_wait_time) for nname, nicname in node_nic_names: req = api.Trigger_CreateExecuteCommandsRequest() cmd = "ps -aef | grep athena_app | grep soft-init | grep -v grep" api.Trigger_AddNaplesCommand(req, nname, cmd, nicname) resp = api.Trigger(req) cmd = resp.commands[0] api.PrintCommandResults(cmd) if cmd.exit_code != 0: api.Logger.error("ps failed or athena_app failed to start " "on node %s nic %s" % (nname, nicname)) return api.types.status.FAILURE if "athena_app" in cmd.stdout: athena_sec_app_pid = cmd.stdout.strip().split()[1] api.Logger.info("Athena sec app came up on node %s nic %s and " "has pid %s" % (nname, nicname, athena_sec_app_pid)) return api.types.status.SUCCESS
def __load_linux_driver(node, node_os, manifest_file): image_manifest = parser.JsonParse(manifest_file) driver_images = list(filter(lambda x: x.OS == node_os, image_manifest.Drivers))[0].Images[0] if driver_images is None: api.Logger.error("Unable to load image manifest") return api.types.status.FAILURE drImgFile = os.path.join(Gl, driver_images.drivers_pkg) api.Logger.info("Fullpath for driver image: " + drImgFile) resp = api.CopyToHost(node, [drImgFile], "") if not api.IsApiResponseOk(resp): api.Logger.error("Failed to copy %s" % drImgFile) return api.types.status.FAILURE rundir = os.path.basename(driver_images.drivers_pkg).split('.')[0] req = api.Trigger_CreateExecuteCommandsRequest(serial = True) api.Trigger_AddHostCommand(req, node, "tar -xf " + os.path.basename(driver_images.drivers_pkg)) api.Trigger_AddHostCommand(req, node, "./build.sh", rundir=rundir) resp = api.Trigger(req) if not api.IsApiResponseOk(resp): api.Logger.error("TriggerCommand for driver build failed") return api.types.status.FAILURE for cmd in resp.commands: if cmd.exit_code != 0 and cmd.command != './build.sh': # Build.sh could fail -ignored (FIXME) api.Logger.error("Failed to exec cmds to build/load new driver") return api.types.status.FAILURE api.Logger.info("New driver image is built on target host. Prepare to load") if host.UnloadDriver(node_os, node) != api.types.status.SUCCESS: api.Logger.error("Failed to unload current driver - proceeding") req = api.Trigger_CreateExecuteCommandsRequest(serial = True) if node_os == OS_TYPE_LINUX: api.Trigger_AddHostCommand(req, node, "insmod " + os.path.join(rundir, "drivers/eth/ionic/ionic.ko")) elif node_os == OS_TYPE_BSD: api.Trigger_AddHostCommand(req, node, "kldload " + os.path.join(rundir, "drivers/eth/ionic/ionic.ko")) resp = api.Trigger(req) if not api.Trigger_IsSuccess(resp): api.Logger.error("TriggerCommand for driver installation failed") return api.types.status.FAILURE return api.types.status.SUCCESS
def Trigger(tc): if api.IsDryrun(): return api.types.status.SUCCESS req1 = api.Trigger_CreateExecuteCommandsRequest(serial=True) req2 = api.Trigger_CreateExecuteCommandsRequest(serial=False) w1 = tc.workloads[0] w2 = tc.workloads[1] tc.cmd_descr = "Server: %s(%s) <--> Client: %s(%s)" %\ (w1.interface, w1.ip_address, w2.interface, w2.ip_address) api.Logger.info("Starting Iperf test from %s" % (tc.cmd_descr)) proto = getattr(tc.iterators, "proto", 'tcp') number_of_iperf_threads = getattr(tc.args, "iperfthreads", 1) pktsize = getattr(tc.iterators, "pktsize", 512) ipproto = getattr(tc.iterators, "ipproto", 'v4') for i in range(number_of_iperf_threads): if proto == 'tcp': port = api.AllocateTcpPort() else: port = api.AllocateUdpPort() iperf_server_cmd = cmd_builder.iperf_server_cmd(port=port) api.Trigger_AddCommand(req1, w1.node_name, w1.workload_name, iperf_server_cmd, background=True) iperf_client_cmd = cmd_builder.iperf_client_cmd( server_ip=w1.ip_address, port=port, proto=proto, pktsize=pktsize, ipproto=ipproto) api.Trigger_AddCommand(req2, w2.node_name, w2.workload_name, iperf_client_cmd) trig_resp1 = api.Trigger(req1) trig_resp2 = api.Trigger(req2) term_resp1 = api.Trigger_TerminateAllCommands(trig_resp1) response = api.Trigger_AggregateCommandsResponse(trig_resp1, term_resp1) tc.resp = api.Trigger_AggregateCommandsResponse(response, trig_resp2) return api.types.status.SUCCESS
def Trigger(tc): if tc.skip: return api.types.status.SUCCESS store = tc.GetBundleStore() cps = int(getattr(tc.args, "cps", 500)) attempts = int(getattr(tc.args, "attempts", 3)) sessions = int(getattr(tc.args, "num_sessions", 1)) session_time = str(getattr(tc.args, "session_time", "10")) + "s" static_arp = int(getattr(tc.args, "static_arp", False)) if static_arp: #install arp entries before running arpReqs = store["arp_ctx"] api.Trigger(arpReqs) #Spread CPS evenly clientReq = api.Trigger_CreateAllParallelCommandsRequest() cps_per_node = int(cps / len(store["client_ctxts"])) for index, (_, client) in enumerate(store["client_ctxts"].items()): jsonInput = {"connections": []} for serverIPPort in client.GetServers(): jsonInput["connections"].append({ "ServerIPPort": serverIPPort, "proto": "tcp" }) outfile = api.GetTestDataDirectory( ) + "/" + client.workload_name + "_fuz.json" with open(outfile, 'w') as fp: json.dump(jsonInput, fp) api.CopyToWorkload(client.node_name, client.workload_name, [outfile], "") clientCmd = fuz_init.FUZ_EXEC[ client.workload_name] + " -attempts " + str( attempts) + " -duration " + session_time + " -conns " + str( sessions) + " -cps " + str( cps_per_node ) + " -talk --jsonOut --jsonInput " + os.path.basename( outfile) api.Trigger_AddCommand(clientReq, client.node_name, client.workload_name, clientCmd, background=True) #Initiate connections store["client_req"] = api.Trigger(clientReq) return api.types.status.SUCCESS
def athena_sec_app_kill(node_name=None, nic_name=None): node_nic_names = [] if (not node_name and nic_name) or (node_name and not nic_name): raise Exception("specify both node_name and nic_name or neither") if node_name and nic_name: node_nic_names.append((node_name, nic_name)) else: node_nic_names = get_athena_node_nic_names() for nname, nicname in node_nic_names: req = api.Trigger_CreateExecuteCommandsRequest() cmd = "ps -aef | grep athena_app | grep soft-init | grep -v grep" api.Trigger_AddNaplesCommand(req, nname, cmd, nicname) resp = api.Trigger(req) ps_cmd_resp = resp.commands[0] api.PrintCommandResults(ps_cmd_resp) if "athena_app" in ps_cmd_resp.stdout: athena_sec_app_pid = ps_cmd_resp.stdout.strip().split()[1] api.Logger.info("athena sec app already running on node %s " "nic %s with pid %s. Killing it." % (nname, nicname, athena_sec_app_pid)) req = api.Trigger_CreateExecuteCommandsRequest() api.Trigger_AddNaplesCommand(req, nname, "pkill -n athena_app", nicname) resp = api.Trigger(req) pkill_cmd_resp = resp.commands[0] api.PrintCommandResults(pkill_cmd_resp) if pkill_cmd_resp.exit_code != 0: api.Logger.info("pkill failed for athena sec app") return api.types.status.FAILURE # sleep for kill to complete misc_utils.Sleep(ATHENA_SEC_APP_KILL_WAIT_TIME) else: api.Logger.info("athena sec app not running on node %s nic %s" % (nname, nicname)) return api.types.status.SUCCESS
def Trigger(tc): if tc.skip: return api.types.status.SUCCESS status = api.types.status.SUCCESS # clean up resources before run subif_utils.clearAll() # initialize config: subif_utils.initialize_tagged_config_workloads() # Delete existing subinterfaces __delete_subifs() time.sleep(3) # Create subinterfaces for every workload/host interface # as per <subif_count> __create_subifs() time.sleep(2) tc.workload_pairs = api.GetRemoteWorkloadPairs() tc.cmd_cookies = [] req1 = api.Trigger_CreateExecuteCommandsRequest(serial = True) __run_ping_test(req1, tc) tc.resp2 = api.Trigger(req1) result = changeMacAddrTrigger(tc) time.sleep(5) api.Logger.debug("UC MAC filter : Trigger -> Change MAC addresses result ", result) api.Logger.debug("UC MAC filter : LIF reset ") # reset the LIFs if requested lif_reset = getattr(tc.args, 'lif_reset', False) if lif_reset: if do_bsd_lif_resets(): api.Logger.error("UC MAC filter : LIF reset failed") status = api.types.status.FAILURE tc.wload_ep_set, tc.host_ep_set, tc.naples_ep_set, tc.hal_ep_set = ValidateMacRegistration(tc) req = api.Trigger_CreateExecuteCommandsRequest(serial = True) tc.workload_pairs = api.GetRemoteWorkloadPairs() __run_ping_test(req, tc) tc.resp = api.Trigger(req) return status
def __node_api_handler(self, url, json_data=None, oper=CfgOper.ADD): if oper == CfgOper.DELETE: oper = "DELETE" elif oper == CfgOper.ADD: oper = "POST" elif oper == CfgOper.UPDATE: oper = "PUT" elif oper == CfgOper.GET: oper = "GET" else: print(oper) assert (0) if GlobalOptions.debug: api.Logger.info("Url : %s" % url) cmd = None if json_data and len(json.dumps(json_data)) > 100000: filename = "/tmp/temp_config.json" with open(filename, 'w') as outfile: json.dump(json_data, outfile) req = api.Trigger_CreateAllParallelCommandsRequest() cmd = ["rm", "-rf", "temp_config.json"] cmd = " ".join(cmd) api.Trigger_AddHostCommand(req, self.host_name, cmd, timeout=3600) api.Trigger(req) resp = api.CopyToHost(self.host_name, [filename], "") if not api.IsApiResponseOk(resp): assert (0) cmd = [ "curl", "-X", oper, "-d", "@temp_config.json", "-k", "-H", "\"Content-Type:application/json\"", url ] else: cmd = [ "curl", "-X", oper, "-k", "-d", "\'" + json.dumps(json_data) + "\'" if json_data else " ", "-H", "\"Content-Type:application/json\"", url ] cmd = " ".join(cmd) req = api.Trigger_CreateAllParallelCommandsRequest() api.Trigger_AddHostCommand(req, self.host_name, cmd, timeout=3600) resp = api.Trigger(req) if GlobalOptions.debug: print(" ".join(cmd)) return resp.commands[0].stdout
def bsd_flow_ctrl(node, inf, fc_type, fc_val, pattern): req = api.Trigger_CreateExecuteCommandsRequest(serial=True) api.Trigger_AddHostCommand(req, node, 'sysctl dev.%s.flow_ctrl=%d' % (host.GetNaplesSysctl(inf), fc_type)) api.Trigger_AddHostCommand(req, node, 'sysctl dev.%s.link_pause=%d' % (host.GetNaplesSysctl(inf), fc_val)) api.Trigger_AddHostCommand(req, node, BSD_IFCONFIG_MEDIA_CMD % inf) api.Logger.info("Setting %s link type: %d value: %d pattern: %s" % (inf, fc_type, fc_val, pattern)) resp = api.Trigger(req) if resp is None: return -1 # We are interested in only last command response. cmd = resp.commands[2] if cmd.exit_code != 0: api.Logger.error("Failed exit code: %d link type: %d value: %d, stderr: %s" % (cmd.exit_code, fc_type, fc_val, cmd.stderr)) api.PrintCommandResults(cmd) return -1 if cmd.stdout.find("[\n\t]*" + pattern + "[\n\t]*") != -1: api.Logger.error("Failed link type: %d value: %d, stdout: %s" % (cmd.exit_code, fc_type, fc_val, cmd.stdout)) api.PrintCommandResults(cmd) return -1 return 0
def getLinuxStats(node, intf, pat1): stats_map = [] cmd = 'ethtool -S ' + intf + ' | grep -e ' + pat1 + ' | cut -d ":" -f 2' req = api.Trigger_CreateExecuteCommandsRequest(serial=True) #api.Logger.info("Getting stats for: %s on host: %s intf: %s" # %(cmd, node.node_name, intf)) api.Trigger_AddHostCommand(req, node.node_name, cmd) resp = api.Trigger(req) if resp is None: api.Logger.error("Failed to run: %s on host: %s intf: %s" %(cmd, node.node_name, intf)) return None cmd = resp.commands[0] if cmd.exit_code != 0: api.Logger.error( "Failed to run: %s for host: %s, stderr: %s" %(cmd, node.node_name, cmd.stderr)) api.PrintCommandResults(cmd) return None if cmd.stdout == "": api.Logger.error("Output is empty for: %s on host: %s intf: %s" %(cmd, node.node_name, intf)) api.PrintCommandResults(cmd) return None stats_map = cmd.stdout.splitlines() stats_map = list(map(int,stats_map)) return stats_map
def Trigger(tc): #============================================================== # trigger the commands #============================================================== req = api.Trigger_CreateExecuteCommandsRequest(serial=True) w1 = tc.w[0] w2 = tc.w[1] cmd = 'sysctl dev.ionic.0.qos.classification_type=' + str(tc.class_type) # Trigger Classification type config if w1.IsNaples(): api.Trigger_AddCommand(req, w1.node_name, w1.workload_name, cmd) if w2.IsNaples(): api.Trigger_AddCommand(req, w2.node_name, w2.workload_name, cmd) #============================================================== # trigger the request #============================================================== trig_resp = api.Trigger(req) term_resp = api.Trigger_TerminateAllCommands(trig_resp) tc.resp = api.Trigger_AggregateCommandsResponse(trig_resp, term_resp) return api.types.status.SUCCESS
def SetupDNSServer(server): node = server.node_name workload = server.workload_name dir_path = os.path.dirname(os.path.realpath(__file__)) zonefile = dir_path + '/' + "example.com.zone" api.Logger.info("fullpath %s" % (zonefile)) resp = api.CopyToWorkload(node, workload, [zonefile], 'dnsdir') if resp is None: return None named_conf = dir_path + '/' + "named.conf" resp = api.CopyToWorkload(node, workload, [named_conf], 'dnsdir') if resp is None: return None req = api.Trigger_CreateExecuteCommandsRequest(serial=True) api.Trigger_AddCommand(req, node, workload, "yes | cp dnsdir/named.conf /etc/") api.Trigger_AddCommand( req, node, workload, "ex -s -c \'%s/192.168.100.102/%s/g|x\' /etc/named.conf" % ("%s", server.ip_address)) api.Trigger_AddCommand(req, node, workload, "yes | cp dnsdir/example.com.zone /var/named/") api.Trigger_AddCommand(req, node, workload, "systemctl start named") api.Trigger_AddCommand(req, node, workload, "systemctl enable named") trig_resp = api.Trigger(req) term_resp = api.Trigger_TerminateAllCommands(trig_resp) for cmd in trig_resp.commands: api.PrintCommandResults(cmd) return api.types.status.SUCCESS
def getWindowsStats(node, intf, pat1): name = host.GetWindowsIntName(node.node_name, intf) cmd = "/mnt/c/Windows/Temp/drivers-windows/IonicConfig.exe DevStats -n '%s' | grep -e %s |" \ " cut -d ':' -f 2" % (name, pat1) req = api.Trigger_CreateExecuteCommandsRequest(serial=True) api.Trigger_AddHostCommand(req, node.node_name, cmd) resp = api.Trigger(req) if resp is None: api.Logger.error("Failed to run: %s on host: %s intf: %s" %(cmd, node.node_name, intf)) return None cmd = resp.commands[0] if cmd.exit_code != 0: api.Logger.error( "Failed to run: %s for host: %s, stderr: %s" %(cmd, node.node_name, cmd.stderr)) api.PrintCommandResults(cmd) return None if cmd.stdout == "": api.Logger.error("Output is empty for: %s on host: %s intf: %s" %(cmd, node.node_name, intf)) api.PrintCommandResults(cmd) return None stats_map = cmd.stdout.splitlines() stats_map = list(map(int,stats_map)) return stats_map
def pingAllRemoteWloadPairs(workload_pairs, iterators): cmd_cookies = [] req = api.Trigger_CreateExecuteCommandsRequest(serial=False) for pair in workload_pairs: w1 = pair[0] w2 = pair[1] if iterators.ipaf == 'ipv4': cmd_cookie = "ping -c 5 -i 0.2 -W 2000 -s %d -S %s %s" % ( iterators.pktsize, w1.ip_address, w2.ip_address) api.Logger.verbose( "Ping test cmd %s from %s(%s %s) --> %s(%s %s)" % (cmd_cookie, w1.workload_name, w1.ip_address, w1.interface, w2.workload_name, w2.ip_address, w2.interface)) else: cmd_cookie = "ping6 -c 5 -i 0.2 -s %d -S %s -I %s %s" % ( iterators.pktsize, w1.ipv6_address, w1.interface, w2.ipv6_address) api.Logger.verbose( "Ping test cmd %s from %s(%s %s) --> %s(%s %s)" % (cmd_cookie, w1.workload_name, w1.ipv6_address, w1.interface, w2.workload_name, w2.ipv6_address, w2.interface)) api.Trigger_AddCommand(req, w1.node_name, w1.workload_name, cmd_cookie) cmd_cookies.append(cmd_cookie) trig_resp = api.Trigger(req) term_resp = api.Trigger_TerminateAllCommands(trig_resp) resp = api.Trigger_AggregateCommandsResponse(trig_resp, term_resp) return cmd_cookies, resp
def Trigger(tc): if tc.skip: return api.types.status.SUCCESS req = None interval = "0.2" if not api.IsSimulation(): req = api.Trigger_CreateAllParallelCommandsRequest() else: req = api.Trigger_CreateExecuteCommandsRequest(serial = False) interval = "3" tc.cmd_cookies = [] for pair in tc.workload_pairs: w1 = pair[0] w2 = pair[1] if tc.iterators.ipaf == 'ipv6': cmd_cookie = "%s(%s) --> %s(%s)" %\ (w1.workload_name, w1.ipv6_address, w2.workload_name, w2.ipv6_address) api.Trigger_AddCommand(req, w1.node_name, w1.workload_name, "ping6 -i %s -c 20 -s %d %s" % (interval, tc.iterators.pktsize, w2.ipv6_address)) else: cmd_cookie = "%s(%s) --> %s(%s)" %\ (w1.workload_name, w1.ip_address, w2.workload_name, w2.ip_address) api.Trigger_AddCommand(req, w1.node_name, w1.workload_name, "ping -i %s -c 20 -s %d %s" % (interval, tc.iterators.pktsize, w2.ip_address)) api.Logger.info("Ping test from %s" % (cmd_cookie)) tc.cmd_cookies.append(cmd_cookie) tc.resp = api.Trigger(req) return api.types.status.SUCCESS
def start_single_pcap_capture(tc): try: req = api.Trigger_CreateExecuteCommandsRequest(serial=True) nodes = api.GetWorkloadNodeHostnames() tc.pcap_cmds = [] intf = api.GetRemoteWorkloadPairs()[0][0].interface tc.pcap_filename = pcap_file_name(intf) cmd = cmd_builder.tcpdump_cmd(intf, tc.pcap_filename) api.Trigger_AddHostCommand(req, n, cmd, background=True) resp = api.Trigger(req) for cmd in resp.commands: if cmd.handle == None or len(cmd.handle) == 0: api.Logger.error("Error starting pcap : %s " % cmd.command) api.Logger.error("Std Output : %s " % cmd.stdout) api.Logger.error("Std Err : %s " % cmd.stdout) return api.types.status.FAILURE api.Logger.info("Success running cmd : %s" % cmd.command) tc.pcap_trigger = resp return api.types.status.SUCCESS except: api.Logger.info("failed to start single pcap capture") api.Logger.debug( "failed to start single pcap capture. error was: {0}".format( traceback.format_exc())) return api.types.status.SUCCESS
def HitlessTriggerUpdateRequest(tc): result = api.types.status.SUCCESS if api.IsDryrun(): return result backgroun_req = api.Trigger_CreateExecuteCommandsRequest(serial=False) # start upgrade manager process for node in tc.nodes: cmd = "/nic/tools/start-upgmgr.sh -n " api.Logger.info("Starting Upgrade Manager %s" % (cmd)) api.Trigger_AddNaplesCommand(backgroun_req, node, cmd, background=True) api.Trigger(backgroun_req) # wait for upgrade manager to comeup misc_utils.Sleep(10) for node in tc.nodes: # initiate upgrade client objects # Generate Upgrade objects UpgradeClient.GenerateUpgradeObjects(node, api.GetNicMgmtIP(node)) upg_obj = UpgradeClient.GetUpgradeObject(node) upg_obj.SetPkgName(tc.pkg_name) upg_obj.SetUpgMode(upgrade_pb2.UPGRADE_MODE_HITLESS) upg_status = upg_obj.UpgradeReq() api.Logger.info( f"Hitless Upgrade request for {node} returned status {upg_status}") if upg_status != upgrade_pb2.UPGRADE_STATUS_OK: api.Logger.error(f"Failed to start upgrade manager on {node}") result = api.types.status.FAILURE continue return result
def Trigger(tc): tc.contexts = [] ctxt = IperfTestContext() ctxt.req = api.Trigger_CreateAllParallelCommandsRequest() ctxt.cmd_cookies = [] for tunnel in tc.tunnels: w1 = tunnel.ltep w2 = tunnel.rtep cmd_cookie = "Server: %s(%s) <--> Client: %s(%s)" %\ (w1.workload_name, w1.ip_address, w2.workload_name, w2.ip_address) api.Logger.info("Starting Iperf test from %s" % (cmd_cookie)) basecmd = 'iperf -p %d ' % api.AllocateTcpPort() if tc.iterators.proto == 'udp': basecmd = 'iperf -u -p %d ' % api.AllocateUdpPort() api.Trigger_AddCommand(ctxt.req, w1.node_name, w1.workload_name, "%s -s -t 300" % basecmd, background = True) api.Trigger_AddCommand(ctxt.req, w2.node_name, w2.workload_name, "%s -c %s" % (basecmd, w1.ip_address)) ctxt.cmd_cookies.append(cmd_cookie) ctxt.cmd_cookies.append(cmd_cookie) trig_resp = api.Trigger(ctxt.req) term_resp = api.Trigger_TerminateAllCommands(trig_resp) ctxt.resp = api.Trigger_AggregateCommandsResponse(trig_resp, term_resp) tc.context = ctxt return api.types.status.SUCCESS