def Trigger(tc): tc.cmd_cookies = [] req = api.Trigger_CreateExecuteCommandsRequest() for n in tc.Nodes: common.AddPenctlCommand(req, n, "update time") common.AddPenctlCommand(req, n, "show time") tc.resp = api.Trigger(req) return api.types.status.SUCCESS
def Trigger(tc): req = api.Trigger_CreateExecuteCommandsRequest() for n in tc.Nodes: common.AddPenctlCommand(req, n, "show proc-meminfo") common.AddPenctlCommand(req, n, "show system-memory-usage") tc.resp = api.Trigger(req) return api.types.status.SUCCESS
def Trigger(tc): req = api.Trigger_CreateExecuteCommandsRequest() for n in tc.Nodes: common.AddPenctlCommand(req, n, "system tech-support") common.AddPenctlCommand(req, n, "show system statistics pb detail") common.AddPenctlCommand(req, n, "show qos-class queues") tc.resp = api.Trigger(req) return api.types.status.SUCCESS
def Trigger(tc): req = api.Trigger_CreateExecuteCommandsRequest() for n in tc.Nodes: common.AddPenctlCommand(req, n, "-v") common.AddPenctlCommand(req, n, "--version") common.AddPenctlCommand(req, n, "version") common.AddPenctlCommand(req, n, "help") tc.resp = api.Trigger(req) return api.types.status.SUCCESS
def Trigger(tc): tc.cmd_cookies = [] req = api.Trigger_CreateExecuteCommandsRequest() for n in tc.Nodes: common.AddPenctlCommand(req, n, "show firmware-version") common.AddPenctlCommand(req, n, "show running-firmware") common.AddPenctlCommand(req, n, "show startup-firmware") #common.AddPenctlCommand(req, n, "update startup-firmware mainfwa") common.AddPenctlCommand(req, n, "show startup-firmware") #common.AddPenctlCommand(req, n, "update startup-firmware mainfwb") common.AddPenctlCommand(req, n, "show startup-firmware") #TODO install and save running-firmware variable/verify/etc tc.resp = api.Trigger(req) return api.types.status.SUCCESS
def StartSSH(): api.Logger.info("Starting SSH server on Nodes") req = api.Trigger_CreateExecuteCommandsRequest(serial=True) enable_sshd = "system enable-sshd" copy_key = "update ssh-pub-key -f ~/.ssh/id_rsa.pub" for n in api.GetNaplesHostnames(): #hack for now, need to set date cmd = "date -s '{}'".format( datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")) api.Trigger_AddNaplesCommand(req, n, cmd) common.AddPenctlCommand(req, n, enable_sshd) common.AddPenctlCommand(req, n, copy_key) resp = api.Trigger(req) for cmd in resp.commands: api.PrintCommandResults(cmd)
def Create(self, req): ltep_parent_mac = host_utils.GetMACAddress(self.ltep_node, self.ltep.parent_interface) rtep_parent_mac = host_utils.GetMACAddress(self.rtep_node, self.rtep.parent_interface) cmd = "update interface --name %s " % self.ltep.interface cmd += "--gw-mac %s " % rtep_parent_mac cmd += "--substrate-ip %s " % self.ltep.ip_address cmd += "--overlay-ip %s " % self.ltep.ip_address cmd += "--mpls-in %d " % self.label_in cmd += "--mpls-out %d " % self.label_out cmd += "--tunnel-dest-ip %s " % self.rtep.ip_address cmd += "--source-gw %s " % self.gateway_prefix cmd += "--ingress-bw %d " % self.policer.ingress cmd += "--egress-bw %d " % self.policer.egress cmd += "--encap %s " % self.encap cmd += "--pf-mac %s " % ltep_parent_mac cmd += "--overlay-mac %s " % self.ltep.mac_address api.Logger.info("Creating Tunnel %s ==> %s" % (self.ltep.workload_name, self.rtep.workload_name)) api.Logger.info("- command: %s" % cmd) common.AddPenctlCommand(req, self.ltep.node_name, cmd) self.__add_static_arp(req) return api.types.status.SUCCESS
def Main(step): if GlobalOptions.skip_setup: return api.types.status.SUCCESS naplesHosts = api.GetNaplesHostnames() assert(len(naplesHosts) != 0) binary_path = os.path.join(BIN_PATH, testupgapp_utils.UPGRADE_TEST_APP + "_bin", testupgapp_utils.UPGRADE_TEST_APP + ".bin") fullpath = os.path.join(api.GetTopDir(), binary_path) if not os.path.isfile(fullpath): ALT_BIN_PATH="nic/build/aarch64/iris/capri/out/" binary_path = os.path.join(ALT_BIN_PATH, testupgapp_utils.UPGRADE_TEST_APP + "_bin", testupgapp_utils.UPGRADE_TEST_APP + ".bin") for naplesHost in naplesHosts: testupgapp_utils.stopTestUpgApp(naplesHost, True) ret = utils.installBinary(naplesHost, binary_path) if ret != api.types.status.SUCCESS: api.Logger.error("Failed in test upgrade app %s copy to Naples" % binary_path) return ret req = api.Trigger_CreateExecuteCommandsRequest() for n in naplesHosts: pencommon.AddPenctlCommand(req, n, "update time") api.Trigger(req) return api.types.status.SUCCESS
def Trigger(tc): req = api.Trigger_CreateExecuteCommandsRequest() for n in tc.Nodes: # sub_command = getattr(tc.iterators, "subcmd", 'None') # api.Logger.info("cmd:%s, use_token:%s, sub_cmd:%s" \ # % (tc.iterators.command, tc.iterators.use_token, sub_command)) if 'penctl' == tc.iterators.command: if tc.iterators.use_token: cmd = " -a " + common.PENCTL_TOKEN[n] + \ " %s" % (tc.iterators.subcmd) else: cmd = " %s" % (tc.iterators.subcmd) common.AddPenctlCommand(req, n, cmd) elif 'ssh' == tc.iterators.command: cmd = ("ssh -o UserKnownHostsFile=/dev/null " "-o StrictHostKeyChecking=no root@{} ls -a /".format( common.GetNaplesMgmtIP(n))) api.Trigger_AddHostCommand(req, n, cmd) else: api.Logger.debug("Invalid cmd:%s" % (tc.iterators.command)) tc.resp = api.Trigger(req) return api.types.status.SUCCESS
def Main(step): nodes = api.GetNaplesHostnames() assert (len(nodes) != 0) api.Logger.info("Update mode to eth-dev-scale") req = api.Trigger_CreateExecuteCommandsRequest() for n in nodes: common.AddPenctlCommand( req, n, "create dsc-profile --name scale --num-lifs 16") common.AddPenctlCommand( req, n, "create dsc-profile --name default --num-lifs 1") common.AddPenctlCommand( req, n, "update dsc --managed-by host --dsc-profile scale") api.Trigger(req) return api.types.status.SUCCESS
def Trigger(tc): req = api.Trigger_CreateExecuteCommandsRequest() for n in tc.Nodes: common.AddPenctlCommand(req, n, "show dsc") tc.resp = api.Trigger(req) return api.types.status.SUCCESS
def Trigger(tc): req = api.Trigger_CreateExecuteCommandsRequest(serial=True) for n in tc.nodes: penctl.AddPenctlCommand(req, n, "show dsc") api.Logger.info("Loading RDMA driver on nodes: {0}".format(tc.nodes)) for n in tc.nodes: if tc.os == host.OS_TYPE_LINUX: api.Trigger_AddHostCommand(req, n, "modprobe ib_uverbs") api.Trigger_AddHostCommand(req, n, "modprobe rdma_ucm") api.Trigger_AddHostCommand( req, n, "insmod {path}drivers/rdma/drv/ionic/ionic_rdma.ko {opts}". format(path=tc.iota_path, opts=tc.insmod_opts)) else: if tc.insmod_opts: api.Trigger_AddHostCommand( req, n, "kenv {opts}".format(opts=tc.insmod_opts)) api.Trigger_AddHostCommand( req, n, "kldload {path}sys/modules/ionic_rdma/ionic_rdma.ko".format( path=tc.iota_path)) # allow device to register before proceeding api.Trigger_AddHostCommand(req, n, "sleep 2") repeat = int(getattr(tc.args, 'reload', 0)) if repeat: api.Logger.info( "Repeating unload + reload {repeat} times".format(repeat=repeat)) for _ in range(repeat): for n in tc.nodes: if tc.os == host.OS_TYPE_LINUX: api.Trigger_AddHostCommand(req, n, "rmmod ionic_rdma") api.Trigger_AddHostCommand( req, n, "insmod {path}drivers/rdma/drv/ionic/ionic_rdma.ko {opts}". format(path=tc.iota_path, opts=tc.insmod_opts)) else: api.Trigger_AddHostCommand(req, n, "kldunload ionic_rdma") api.Trigger_AddHostCommand( req, n, "kldload {path}sys/modules/ionic_rdma/ionic_rdma.ko". format(path=tc.iota_path)) # allow device to register before proceeding api.Trigger_AddHostCommand(req, n, "sleep 2") for n in tc.nodes: if tc.os == host.OS_TYPE_LINUX: api.Trigger_AddHostCommand(req, n, "lsmod") else: api.Trigger_AddHostCommand(req, n, "kldstat") tc.resp = api.Trigger(req) return api.types.status.SUCCESS
def Trigger(tc): req = api.Trigger_CreateExecuteCommandsRequest() for n in tc.Nodes: common.AddPenctlCommand(req, n, "%s --help" % (tc.iterators.option)) tc.resp = api.Trigger(req) return api.types.status.SUCCESS
def Delete(self, req): cmd = "delete interface --name %s " % self.ltep.interface cmd += "--encap %s " % self.encap api.Logger.info("Deleting Tunnel on interface %s" % (self.ltep.interface)) api.Logger.info("- command: %s" % cmd) common.AddPenctlCommand(req, self.ltep.node_name, cmd) self.__add_static_arp(req) return api.types.status.SUCCESS
def Trigger(tc): req = api.Trigger_CreateExecuteCommandsRequest() for n in tc.Nodes: common.AddPenctlCommand(req, n, "system tech-support --skip-core") tc.resp = api.Trigger(req) return api.types.status.SUCCESS
def VerifySessionEvents(tc): cmd_cookie = "show events" fields = ["APPROACH", "REACHED"] curr_event_count = GetPenctlEventsCount(tc) new_event_count = (curr_event_count - tc.event_count_at_start) api.Logger.info(" Event count at end : %s, New event count: %s" % (curr_event_count, new_event_count)) req = api.Trigger_CreateExecuteCommandsRequest() # show events gives out one blank line, we are interested in latest events # hence use atleast (2+1) for tail command if new_event_count < 3: new_event_count = 3 cmd = "show events --json | tail -%s | grep %s" % (new_event_count, tc.grep_str) common.AddPenctlCommand(req, tc.wc_server.node_name, cmd) resp = api.Trigger(req) cmd_resp = resp.commands[0] api.PrintCommandResults(cmd_resp) if cmd_resp.exit_code != 0: return api.types.status.FAILURE empty_lines = cmd_resp.stdout.split("\n") lines = [line for line in empty_lines if line.strip() != ""] if len(lines) != len(fields): api.Logger.error("Incorrect events len encountered : %d expected %d" % (len(lines), len(fields))) # dump last 10 events for debug purpose cmd = "show events --json | tail -10 " common.AddPenctlCommand(req, tc.wc_server.node_name, cmd) resp = api.Trigger(req) cmd_resp = resp.commands[0] api.PrintCommandResults(cmd_resp) return api.types.status.FAILURE for i in range(len(lines)): if fields[i] not in lines[i]: api.Logger.error( "Incorrect event encountered :%s expected substring : %s" % (lines[i], fields[i])) return api.types.status.FAILURE api.Logger.info("Event verification session limit for %s Success" % tc.proto) return api.types.status.SUCCESS
def Trigger(tc): tc.cmd_cookies = [] req = api.Trigger_CreateExecuteCommandsRequest() for n in tc.Nodes: common.AddPenctlCommand(req, n, "show events | tail -n 20") tc.cmd_cookies.append("No events found") #TODO: Trigger event #common.AddPenctlCommand(req, n, "show events | tail -n 20") tc.resp = api.Trigger(req) return api.types.status.SUCCESS
def Trigger(tc): req = api.Trigger_CreateExecuteCommandsRequest() for n in tc.nodes: if api.IsNaplesNode(n): common.AddPenctlCommand( req, n, "show metrics lif > metrics_lif.out.before") tc.lif_metrics_old = api.Trigger(req) common.SendTraffic(tc) req = api.Trigger_CreateExecuteCommandsRequest() for n in tc.nodes: if api.IsNaplesNode(n): common.AddPenctlCommand( req, n, "show metrics lif > metrics.lif.out.after") tc.lif_metrics_new = api.Trigger(req) return api.types.status.SUCCESS
def CollectTechSupport(tsName): try: #global __CURREN_TECHSUPPORT_CNT #__CURREN_TECHSUPPORT_CNT = __CURREN_TECHSUPPORT_CNT + 1 if GlobalOptions.pipeline in ["apulu"]: return types.status.SUCCESS Logger.info("Collecting techsupport for testsuite {0}".format(tsName)) tsName = re.sub('\W', '_', tsName) logDir = GlobalOptions.logdir if not logDir.endswith('/'): logDir += '/' logDir += 'techsupport/' if not os.path.exists(logDir): os.mkdir(logDir) nodes = api.GetNaplesHostnames() req = api.Trigger_CreateExecuteCommandsRequest() for n in nodes: Logger.info("Techsupport for node: %s" % n) common.AddPenctlCommand( req, n, "system tech-support -b %s-tech-support" % (n)) resp = api.Trigger(req) result = types.status.SUCCESS for n, cmd in zip(nodes, resp.commands): #api.PrintCommandResults(cmd) if cmd.exit_code != 0: Logger.error( "Failed to execute penctl system tech-support on node: %s. err: %d" % (n, cmd.exit_code)) result = types.status.FAILURE continue # Copy tech support tar out # TAR files are created at: pensando/iota/entities/node1_host/<test_case> ntsn = "%s-tech-support.tar.gz" % (n) resp = api.CopyFromHost(n, [ntsn], logDir) if resp == None or resp.api_response.api_status != types_pb2.API_STATUS_OK: Logger.error( "Failed to copy techsupport file %s from node: %s" % (ntsn, n)) result = types.status.FAILURE continue os.rename(logDir + ntsn, logDir + tsName + '_' + ntsn) #if __CURREN_TECHSUPPORT_CNT > __MAX_TECHSUPPORT_PER_RUN: # return types.status.CRITICAL return result except AttributeError: Logger.debug('failed to collect tech support. node list not setup yet') except: Logger.debug('failed to collect tech support. error was: {0}'.format( traceback.format_exc())) return types.status.CRITICAL
def Run(): req = api.Trigger_CreateExecuteCommandsRequest(serial=True) for n in api.GetNaplesHostnames(): enable_sshd = " -a " + common.PENCTL_TOKEN[n] + " system enable-sshd" #enable_sshd = "system enable-sshd" common.AddPenctlCommand(req, n, enable_sshd) copy_key = " -a " + common.PENCTL_TOKEN[ n] + " update ssh-pub-key -f ~/.ssh/id_rsa.pub" #copy_key = "update ssh-pub-key -f ~/.ssh/id_rsa.pub" common.AddPenctlCommand(req, n, copy_key) resp = api.Trigger(req) for cmd in resp.commands: if cmd.exit_code != 0: api.PrintCommandResults(cmd) return api.types.status.FAILURE api.Logger.info("DSC ssh is enabled") for n in api.GetNaplesHostnames(): uuid = GetNaplesUUID(n) api.SetNaplesNodeUuid(n, uuid) return api.types.status.SUCCESS
def GetPenctlEventsCount(tc): req = api.Trigger_CreateExecuteCommandsRequest() cmd = "show events --json | wc -l" common.AddPenctlCommand(req, tc.wc_server.node_name, cmd) resp = api.Trigger(req) cmd_resp = resp.commands[0] api.PrintCommandResults(cmd_resp) if cmd_resp.exit_code != 0: return api.types.status.FAILURE curr_event_count = cmd_resp.stdout.strip("\n") api.Logger.info("Event count : %s " % curr_event_count) try: curr_event_count = int(curr_event_count) except Exception as e: api.Logger.info(" Failed in retrieving the active event count") curr_event_count = 0 return curr_event_count
def Main(args): if GlobalOptions.skip_setup: # No profile change is required for skip setup return api.types.status.SUCCESS newProfile = getattr(args, 'profile', 'FEATURE_PROFILE_BASE') if newProfile != 'FEATURE_PROFILE_BASE': req = api.Trigger_CreateExecuteCommandsRequest(serial=True) for n in api.GetNaplesHostnames(): common.AddPenctlCommand( req, n, "update dsc --device-profile %s" % newProfile) resp = api.Trigger(req) if not api.Trigger_IsSuccess(resp): return api.types.status.FAILURE api.Logger.info("Naples profile changed to %s - Restarting nodes" % newProfile) return api.RestartNodes(api.GetNaplesHostnames()) else: api.Logger.info("Profile unchanged, input: %s" % newProfile) return api.types.status.SUCCESS
def Main(tc): if tc.GetStatus() == api.types.status.SUCCESS: return api.types.status.SUCCESS global __CURREN_TECHSUPPORT_CNT __CURREN_TECHSUPPORT_CNT = __CURREN_TECHSUPPORT_CNT + 1 api.Logger.info("TC: %s failed. Collecting techsupport." % (tc.Name())) nodes = api.GetNaplesHostnames() req = api.Trigger_CreateExecuteCommandsRequest() tc_dir = tc.GetLogsDir() for n in nodes: api.Logger.info("Techsupport for node: %s" % n) common.AddPenctlCommand(req, n, "system tech-support -b %s-tech-support" % (n)) tc.resp = api.Trigger(req) result = api.types.status.SUCCESS for n, cmd in zip(nodes, tc.resp.commands): api.PrintCommandResults(cmd) if cmd.exit_code != 0: api.Logger.error( "Failed to execute penctl system tech-support on node: %s. err: %d" % (n, cmd.exit_code)) result = api.types.status.FAILURE continue # Copy tech support tar out # TAR files are created at: pensando/iota/entities/node1_host/<test_case> resp = api.CopyFromHost(n, ["%s-tech-support.tar.gz" % (n)], tc_dir) if resp == None or resp.api_response.api_status != types_pb2.API_STATUS_OK: api.Logger.error("Failed to copy techsupport file from node: %s" % n) result = api.types.status.FAILURE continue if __CURREN_TECHSUPPORT_CNT > __MAX_TECHSUPPORT_PER_RUN: return api.types.status.CRITICAL return result
def Trigger(tc): try: # Get firmware version using penctl req1 = api.Trigger_CreateExecuteCommandsRequest() # Get VPD information req2 = api.Trigger_CreateExecuteCommandsRequest() for n in tc.naples_nodes: common.AddPenctlCommand(req1, n.Name(), "show firmware-version") api.Trigger_AddHostCommand(req2, n.Name(), "lspci -d 1dd8: -vv") tc.resp1 = api.Trigger(req1) tc.resp2 = api.Trigger(req2) # Create a Redfish client object tc.RF = get_redfish_obj(tc.cimc_info) tc.setting_details = get_nw_device_functions_settings_info(tc.RF) except: api.Logger.error(traceback.format_exc()) return api.types.status.ERROR return api.types.status.SUCCESS
def Main(step): if GlobalOptions.skip_setup: #No mode switch required for skeip setup return api.types.status.SUCCESS req = api.Trigger_CreateExecuteCommandsRequest(serial = True) uuidMap = api.GetNaplesNodeUuidMap() nodes = api.GetNaplesHostnames() for n in nodes: # Touch a file to indicate to NMD that the current mode is emulation cmd = "touch /data/iota-emulation" api.Trigger_AddNaplesCommand(req, n, cmd) # Make sure console is enabled CreateConfigConsoleNoAuth() api.CopyToNaples(n, [NAPLES_CONFIG_SPEC_LOCAL], "") cmd = "mv /system-config.json /sysconfig/config0/system-config.json" api.Trigger_AddNaplesCommand(req, n, cmd) if common.PenctlGetModeStatus(n) != "NETWORK" or common.PenctlGetTransitionPhaseStatus(n) != "VENICE_REGISTRATION_DONE": api.Logger.info("Host [{}] is in HOST mode. Initiating mode change.".format(n)) ret = common.SetNaplesModeOOB_Static(n, "1.1.1.1", "1.1.1.2/24") if ret == None: return api.types.status.FAILURE #hack for now, need to set date cmd = "date -s '{}'".format(datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")) api.Trigger_AddNaplesCommand(req, n, cmd) resp = api.Trigger(req) for cmd in resp.commands: api.PrintCommandResults(cmd) if cmd.exit_code != 0: return api.types.status.FAILURE num_retries = 60 reboot_nodes = [] while nodes: req = api.Trigger_CreateExecuteCommandsRequest(serial = True) for n in nodes: #hack for now, need to set date api.Logger.info("Checking Transition phase for node : %s" % n) check_state_cmd = "show naples --json" time.sleep(30) common.AddPenctlCommand(req, n, check_state_cmd) api.Trigger_AddNaplesCommand(req, n, "touch /data/no_watchdog") resp = api.Trigger(req) cmd = resp.commands[0] api.PrintCommandResults(cmd) if cmd.exit_code != 0: return api.types.status.FAILURE try: out = json.loads(cmd.stdout) except: api.Logger.error("Penctl output not in Json format {}".format(cmd.stdout)) return api.types.status.FAILURE if not hal_show_utils.IsNaplesForwardingModeClassic(n): api.Logger.info("Dataplane already in HOSTPIN mode. Skipping node [{}] for reboot.".format(n)) reboot_nodes.append(n) nodes.remove(n) elif out["status"]["transition-phase"] == "VENICE_UNREACHABLE": api.Logger.info("Reboot pending on node : %s" % n) reboot_nodes.append(n) nodes.remove(n) elif out["status"]["transition-phase"] == "VENICE_REGISTRATION_DONE": api.Logger.info("Node already transitioned : %s" % n) nodes.remove(n) else: api.Logger.info("Reboot not pending on node : %s" % n) time.sleep(1) num_retries = num_retries - 1 if num_retries == 0: api.Logger.error("Reboot pending state not transitioned complete on naples") return api.types.status.FAILURE ret = api.RestartNodes(reboot_nodes) if ret != api.types.status.SUCCESS: api.Logger.error("Node restart failed") return api.types.status.FAILURE req = api.Trigger_CreateExecuteCommandsRequest(serial = True) #enable_sshd = "system enable-sshd" #copy_key = "update ssh-pub-key -f ~/.ssh/id_rsa.pub" for n in api.GetNaplesHostnames(): #hack for now, need to set date cmd = "date -s '{}'".format(datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")) api.Trigger_AddNaplesCommand(req, n, cmd) #common.AddPenctlCommand(req, n, enable_sshd) #common.AddPenctlCommand(req, n, copy_key) resp = api.Trigger(req) for cmd in resp.commands: api.PrintCommandResults(cmd) if cmd.exit_code != 0: result = api.types.status.FAILURE #Give some time for naples to admit to venice after mode switch #This is required for now as Iota is setting time manually to make TLS happy time.sleep(30) #Check whether naples has switch mode succesfully req = api.Trigger_CreateExecuteCommandsRequest(serial = False) for n in nodes: cmd = "cat /sysconfig/config0/app-start.conf | grep hostpin" api.Trigger_AddNaplesCommand(req, n, cmd) resp = api.Trigger(req) for cmd in resp.commands: api.PrintCommandResults(cmd) if cmd.exit_code != 0: api.Logger.error("Mode switch failed on node : {}".format(cmd.node_name)) result = api.types.status.FAILURE #Verify Agent in right mode. req = api.Trigger_CreateExecuteCommandsRequest(serial = False) for n in nodes: cmd = "curl localhost:8888/api/system/info/" api.Trigger_AddNaplesCommand(req, n, cmd) # Delete the iota-emulation file created earlied for NMD. Mode change would have passed by now if it had to. cmd = "rm -f /data/iota-emulation" api.Trigger_AddNaplesCommand(req, n, cmd) resp = api.Trigger(req) for cmd in resp.commands: api.PrintCommandResults(cmd) if cmd.exit_code != 0: api.Logger.error("Agent system get failed : {}".format(cmd.node_name)) result = api.types.status.FAILURE out = None try: out = json.loads(cmd.stdout) except: api.Logger.error("Agent System get out failed {}".format(cmd.stdout)) return api.types.status.FAILURE if out["naples-mode"] != "NETWORK_MANAGED_OOB": api.Logger.error("Agent not in correct mode: {} {} ".format(cmd.node_name, out["naples-mode"])) return api.types.status.FAILURE api.Logger.info("Trying to re-enable ssh on naples nodes") return enable_ssh.Main(None)
def Trigger(tc): #Run all commands in serial req = api.Trigger_CreateExecuteCommandsRequest(serial=True) tcp_dump_req = api.Trigger_CreateExecuteCommandsRequest(serial=True) tc.cmd_cookies = [] proto = "tcp" count = 0 for tunnel in tc.tunnels: w1 = tunnel.ltep w2 = tunnel.rtep w1_class_e_addr = "" w2_class_e_addr = "" for sec_ipaddr in w1.sec_ip_addresses: if ipaddress.IPv4Address(sec_ipaddr).is_reserved: w1_class_e_addr = sec_ipaddr api.Logger.info("Class-E IP = %s" % sec_ipaddr) break for sec_ipaddr in w2.sec_ip_addresses: if ipaddress.IPv4Address(sec_ipaddr).is_reserved: w2_class_e_addr = sec_ipaddr api.Logger.info("Class-E IP = %s" % sec_ipaddr) break if w1_class_e_addr == "" or w2_class_e_addr == "": api.Logger.info("No Class-E Address found") return api.types.status.FAILURE api.Trigger_AddCommand( req, w1.node_name, w1.workload_name, "arp -s %s %s" % (w2_class_e_addr, w2.mac_address)) api.Logger.info("Add arp entry for %s" % w2_class_e_addr) tc.cmd_cookies.append("Add arp entry for %s" % w2_class_e_addr) api.Trigger_AddCommand( req, w2.node_name, w2.workload_name, "arp -s %s %s" % (w1_class_e_addr, w1.mac_address)) api.Logger.info("Add arp entry for %s" % w1_class_e_addr) tc.cmd_cookies.append("Add arp entry for %s" % w1_class_e_addr) if count % 2 == 0: # Traffic to Verify packet is encapped correctly cmd_cookie = "%s(%s) --> %s(%s)" % ( w1.workload_name, w1.ip_address, w2.workload_name, w2.ip_address) nping_cmd = utils.GetNpingCmd(proto, w2.ip_address, 2000, w1.ip_address, 1) api.Trigger_AddCommand(req, w1.node_name, w1.workload_name, nping_cmd) api.Logger.info("Send " + proto + " traffic from %s" % cmd_cookie) tc.cmd_cookies.append(proto + " positive test") # Traffic to Verify packet is encapped and decapped correctly cmd_cookie1 = "%s(%s) --> %s(%s) IP Opts Dest IP: %s" % ( w1.workload_name, w1_class_e_addr, w2.workload_name, w2.ip_address, w1.ip_address) nping_cmd = utils.GetNpingCmd(proto, w2.ip_address, 2000, w1_class_e_addr) nping_cmd += " " + GetIPOptionsStr(w1_class_e_addr, w1.ip_address) api.Trigger_AddCommand(req, w1.node_name, w1.workload_name, nping_cmd) api.Logger.info("Send " + proto + " traffic with valid IP options from %s" % cmd_cookie1) tc.cmd_cookies.append(proto + " encap decap test") # Verify error counter doesn't go up on w1 and w2 penctl_cmd = r"execute halctl show system statistics drop | grep 'Drop Vf IP'" common.AddPenctlCommand(req, w1.node_name, penctl_cmd) tc.cmd_cookies.append("Encap decap test statistics") cmd_cookie2 = "%s(%s) --> %s(%s) IP Opts Dest IP: 14.15.16.17" % ( w1.workload_name, w1.ip_address, w2.workload_name, w2_class_e_addr) # Class-E test with invalid IP options nping_cmd = utils.GetNpingCmd(proto, w2_class_e_addr, 2000, w1.ip_address) nping_cmd += " " + GetIPOptionsStr(w1_class_e_addr, "14.15.16.17") api.Trigger_AddCommand(req, w1.node_name, w1.workload_name, nping_cmd) api.Logger.info("Send " + proto + " traffic with invalid IP options from %s" % cmd_cookie2) tc.cmd_cookies.append(proto + " encap test with invalid IP options") # Verify error counter goes up on w1 penctl_cmd = r"execute halctl show system statistics drop | grep 'Drop Vf Bad'" common.AddPenctlCommand(req, w1.node_name, penctl_cmd) tc.cmd_cookies.append( "Encap test with invalid IP options statistics") if count % 2 == 1: cmd_cookie3 = "%s(%s) --> %s(%s) IP Opts Dest IP: 0.0.0.0" % ( w1.workload_name, w1.ip_address, w2.workload_name, w2_class_e_addr) # Class-E test with zero as dest IP in IP options nping_cmd = utils.GetNpingCmd(proto, w2_class_e_addr, 2000, w1.ip_address) nping_cmd += " " + GetIPOptionsStr(w1_class_e_addr, "0.0.0.0") api.Trigger_AddCommand(req, w1.node_name, w1.workload_name, nping_cmd) api.Logger.info( "Send " + proto + " traffic with zero as dest IP in IP options from %s" % cmd_cookie3) tc.cmd_cookies.append(proto + " encap test with 0.0.0.0 IP options") # Verify error counter goes up on w1 penctl_cmd = r"execute halctl show system statistics drop | grep 'Drop Vf Bad'" common.AddPenctlCommand(req, w1.node_name, penctl_cmd) tc.cmd_cookies.append( "Encap test with 0.0.0.0 IP options statistics") cmd_cookie4 = "%s(%s) --> %s(%s) IP Opts Dest IP: 224.0.0.1" % ( w1.workload_name, w1.ip_address, w2.workload_name, w2_class_e_addr) # Class-E test with multicast dest IP in IP options nping_cmd = utils.GetNpingCmd(proto, w2_class_e_addr, 2000, w1.ip_address) nping_cmd += " " + GetIPOptionsStr(w1_class_e_addr, "224.0.0.1") api.Trigger_AddCommand(req, w1.node_name, w1.workload_name, nping_cmd) api.Logger.info( "Send " + proto + " traffic with multicast dest IP in IP options from %s" % cmd_cookie4) tc.cmd_cookies.append(proto + " encap test with multicast IP options") # Verify error counter goes up on w1 penctl_cmd = r"execute halctl show system statistics drop | grep 'Drop Vf Bad'" common.AddPenctlCommand(req, w1.node_name, penctl_cmd) tc.cmd_cookies.append( "Encap test with multicast IP options statistics") count += 1 if proto is "tcp": proto = "udp" else: proto = "tcp" if count == 16: break tc.resp = api.Trigger(req) return api.types.status.SUCCESS
def checkUpdateMode(): req = api.Trigger_CreateExecuteCommandsRequest() for n in tc.Nodes: # TODO Add more negative test cases common.AddPenctlCommand(req, n, "create dsc-profile -n default -i 42") common.AddPenctlCommand(req, n, "delete dsc-profile -n default") common.AddPenctlCommand( req, n, "delete dsc-profile -n nonExistentProfile") common.AddPenctlCommand(req, n, "update dsc -o host -f scale -c a:a") common.AddPenctlCommand(req, n, "update dsc -o network -k oob -c a:a") common.AddPenctlCommand( req, n, "update dsc -o network -k inband -c a -c a1 -c a2:1") common.AddPenctlCommand(req, n, "update dsc -o a") common.AddPenctlCommand(req, n, "update dsc -k a") common.AddPenctlCommand(req, n, "update dsc -o network -k a") common.AddPenctlCommand(req, n, "update dsc -o network -k oob -m a") common.AddPenctlCommand(req, n, "update dsc -o network -n a: -k inband") common.AddPenctlCommand(req, n, "update dsc -n a: -f default") common.AddPenctlCommand(req, n, "update dsc -o network -k oob -p a") resp = api.Trigger(req) for cmd in resp.commands: api.PrintCommandResults(cmd) if cmd.exit_code != 1: return api.types.status.FAILURE for n, cmd in zip(tc.Nodes, resp.commands): cmd_resp = list(filter(None, cmd.stdout.split("\n"))) if not (x in cmd_resp for x in def_mode_set_error_strs): api.Logger.error( "Update mode response not valid : %s" % (set(def_mode_set_error_strs) - set(cmd_resp))) return api.types.status.FAILURE return api.types.status.SUCCESS
def Main(step): req = api.Trigger_CreateExecuteCommandsRequest(serial=True) venice_ips = api.GetVeniceMgmtIpAddresses() assert (venice_ips) uuidMap = api.GetNaplesNodeUuidMap() for n in api.GetNaplesHostnames(): #hack for now, need to set date cmd = "date -s '{}'".format( datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")) api.Trigger_AddNaplesCommand(req, n, cmd) api.Logger.info("Running mode switch on node : %s" % n) mode_switch_cmd = "update naples --managed-by network --management-network oob --hostname {} --primary-mac {}".format( __get_mgmt_nw(), n, uuidMap[n]) common.AddPenctlCommand(req, n, mode_switch_cmd) resp = api.Trigger(req) for cmd in resp.commands: api.PrintCommandResults(cmd) if cmd.exit_code != 0: return api.types.status.FAILURE nodes = api.GetNaplesHostnames() num_retries = 60 while nodes: req = api.Trigger_CreateExecuteCommandsRequest(serial=True) for n in nodes: #hack for now, need to set date api.Logger.info("Checking Tranisition phase for node : %s" % n) check_state_cmd = "show naples --json" common.AddPenctlCommand(req, n, check_state_cmd) resp = api.Trigger(req) cmd = resp.commands[0] api.PrintCommandResults(cmd) if cmd.exit_code != 0: return api.types.status.FAILURE try: out = json.loads(cmd.stdout) except: api.Logger.error("Penctl output not in Json format {}".format( cmd.stdout)) return api.types.status.FAILURE if out["status"]["transition-phase"] == "REBOOT_PENDING": api.Logger.info("Reboot pending on node : %s" % n) nodes.remove(n) else: api.Logger.info("Reboot not pending on node : %s" % n) time.sleep(1) num_retries = num_retries - 1 if num_retries == 0: api.Logger.error( "Reboot pending state not transitioned complete on naples") return api.types.status.FAILURE reboot_nodes = [] for n in api.GetNaplesHostnames(): reboot_nodes.append(n) ret = api.RestartNodes(reboot_nodes) if ret != api.types.status.SUCCESS: api.Logger.error("Node restart failed") return api.types.status.FAILURE req = api.Trigger_CreateExecuteCommandsRequest(serial=True) for n in api.GetNaplesHostnames(): #hack for now, need to set date cmd = "date -s '{}'".format( datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")) api.Trigger_AddNaplesCommand(req, n, cmd) resp = api.Trigger(req) for cmd in resp.commands: api.PrintCommandResults(cmd) if cmd.exit_code != 0: result = api.types.status.FAILURE #Give some time for naples to admit to venice after mode switch #This is required for now as Iota is setting time manually to make TLS happy time.sleep(30) #Check whether naples has switch mode succesfully req = api.Trigger_CreateExecuteCommandsRequest(serial=False) for n in api.GetNaplesHostnames(): cmd = "cat /sysconfig/config0/app-start.conf | grep hostpin" api.Trigger_AddNaplesCommand(req, n, cmd) resp = api.Trigger(req) for cmd in resp.commands: api.PrintCommandResults(cmd) if cmd.exit_code != 0: api.Logger.error("Mode switch failed on node : {}".format( cmd.node_name)) result = api.types.status.FAILURE #Verify Agent in right mode. req = api.Trigger_CreateExecuteCommandsRequest(serial=False) for n in api.GetNaplesHostnames(): cmd = "curl localhost:8888/api/system/info/" api.Trigger_AddNaplesCommand(req, n, cmd) resp = api.Trigger(req) venice_urls = [] for ip in api.GetVeniceMgmtIpAddresses(): venice_urls.append(ip + ":9009") for cmd in resp.commands: api.PrintCommandResults(cmd) if cmd.exit_code != 0: api.Logger.error("Agent system get failed : {}".format( cmd.node_name)) result = api.types.status.FAILURE out = None try: out = json.loads(cmd.stdout) except: api.Logger.error("Agent System get out failed {}".format( cmd.stdout)) return api.types.status.FAILURE if out["naples-mode"] != "NETWORK_MANAGED_OOB": api.Logger.error("Agent not in correct mode: {} {} ".format( cmd.node_name, out["naples-mode"])) return api.types.status.FAILURE if set(out["controller-ips"]) != set(venice_urls): api.Logger.error("Agent controller IPs don't match: {} {}".format( venice_urls, out["controller-ips"])) return api.types.status.FAILURE return api.types.status.SUCCESS