def Trigger(tc): tc.pre_data = {} tc.post_data = {} try: for _iter in range(tc.iterators.count): api.Logger.info("Iter: %d" % _iter) api.Logger.info("Issuing APC power cycle") # Getting port status before reboot tc.pre_data[_iter] = get_nw_ports_info(tc.RF) ret = api.RestartNodes([tc.node_name], 'apc') if ret != api.types.status.SUCCESS: api.Logger.info("APC power cycle failed") return api.types.status.FAILURE api.Logger.info("APC power cycle done") tc.test_node.WaitForHost() api.Logger.info("APC power cycle successful") if enable_ssh.Main(None) != api.types.status.SUCCESS: api.Logger.info("Enabling SSH failed after reboot") return api.types.status.FAILURE api.Logger.info("Enabled Naples SSH after power cycle") # Getting port status after reboot tc.post_data[_iter] = get_nw_ports_info(tc.RF) except Exception as e: api.Logger.error(str(e)) return api.types.status.ERROR return api.types.status.SUCCESS
def Setup(tc): if api.IsDryrun(): return api.types.status.SUCCESS tc.nodes = api.GetWorkloadNodeHostnames() tc.node_intfs = {} srv,cli = _get_workloads(tc) tc.workloads = [srv, cli] if getattr(tc.args, 'restart', False): ret = api.RestartNodes(tc.nodes) if ret != api.types.status.SUCCESS: api.Logger.error("Node restart failed") return api.types.status.FAILURE api.Logger.info("Setting driver features") if common.setup_features(tc) != api.types.status.SUCCESS: api.Logger.info("Setting driver features :Failed") return api.types.status.FAILURE api.Logger.info("Setting driver features : Success") if getattr(tc.args, 'capture_pcap', False): if common.start_pcap_capture(tc) != api.types.status.SUCCESS: return api.types.status.FAILURE if getattr(tc.args, 'capture_single_pcap', False): if common.start_single_pcap_capture(tc) != api.types.status.SUCCESS: return api.types.status.FAILURE return api.types.status.SUCCESS
def LoadDriver(node_names, node_os, target_version='latest'): if target_version == 'latest': api.Logger.info('Target version is latest - nothing to change') else: resp = api.DownloadAssets(release_version = target_version) if not api.IsApiResponseOk(resp): api.Logger.error("Failed to download assets for %s" % target_version) return api.types.status.FAILURE manifest_file = os.path.join(api.GetTopDir(), 'images', target_version + '.json') for node in node_names: if node_os == OS_TYPE_LINUX: if __load_linux_driver(tc, node, manifest_file) == api.types.status.SUCCESS: api.Logger.info("Release Driver %s reload on %s" % (tc.target_version, node)) else: api.Logger.error("Failed to load release driver %s reload on %s" % (tc.target_version, node)) return api.types.status.FAILURE # this is required to bring the testbed into operation state # after driver unload interfaces need to be initialized wl_api.ReAddWorkloads(node) elif tc.os == OS_TYPE_ESX: host.UnloadDriver(tc.os, node) tc.driver_changed = True if __load_esxi_driver(node, node_os, manifest_file) == api.types.status.SUCCESS: api.Logger.info("Release Driver %s reload on %s" % (tc.target_version, node)) else: api.Logger.error("Failed to load release driver %s reload on %s" % (tc.target_version, node)) return api.types.status.FAILURE api.RestartNodes([node])
def Trigger(tc): try: for _iter in range(tc.iterators.count): api.Logger.info("Iter: %d" % _iter) api.Logger.info("Issuing APC power cycle") ret = api.RestartNodes([tc.node_name], 'apc') if ret != api.types.status.SUCCESS: api.Logger.info("APC power cycle failed") return api.types.status.FAILURE api.Logger.info("APC power cycle done") tc.test_node.WaitForHost() ret = check_ncsi_conn(tc.cimc_info) if ret != api.types.status.SUCCESS: api.Logger.error("Unable to connect ot ILO in NCSI mode") return api.types.status.FAILURE api.Logger.info("APC power cycle successfull") if enable_ssh.Main(None) != api.types.status.SUCCESS: api.Logger.info("Enabling SSH failed after reboot") return api.types.status.FAILURE api.Logger.info("Enabled Naples SSH after power cycle") except Exception as e: api.Logger.error(str(e)) return api.types.status.FAILURE return api.types.status.SUCCESS
def Trigger(tc): naples_nodes = [] #for every node in the setup for node in tc.nodes: if api.IsNaplesNode(node): naples_nodes.append(node) api.Logger.info(f"Found Naples Node: [{node}]") if len(naples_nodes) == 0: api.Logger.error("Failed to find a Naples Node!") return api.types.status.FAILURE for reboot in range(tc.args.reboots): # Reboot Node. # Reboot method (APC, IPMI, OS Reboot) is passed as a testcase parameter for node in naples_nodes: api.Logger.info( f"==== Reboot Loop # {reboot} on {node}. Reboot method: {tc.iterators.reboot_method} ====" ) if api.RestartNodes( [node], tc.iterators.reboot_method) != api.types.status.SUCCESS: return api.types.status.FAILURE #raise OfflineTestbedException # Enable SSH, some checks require Naples access if enable_ssh.Main(None) != api.types.status.SUCCESS: api.Logger.error("Enabling SSH failed") return api.types.status.FAILURE # there is not a real "PCI" in IOTA for Windows. if tc.os != host.OS_TYPE_WINDOWS: api.Logger.info(f"Verifying PCI on [{node}]: ") if verify_pci.verify_errors_lspci( node, tc.os) != api.types.status.SUCCESS: api.Logger.error(f"PCIe Failure detected on {node}") return api.types.status.FAILURE #raise OfflineTestbedException if checkLinks(tc, node) is api.types.status.FAILURE: api.Logger.error("Error verifying uplink interfaces") return api.types.status.FAILURE #raise OfflineTestbedException # don't run this while we figure out how to do this in ESX if tc.os != host.OS_TYPE_ESX: # Load the ionic driver except windows if tc.os != host.OS_TYPE_WINDOWS: if host.LoadDriver(tc.os, node) is api.types.status.FAILURE: api.Logger.info("ionic already loaded") # Make sure ionic driver attached to Uplink ports. if checkDrv(node) is api.types.status.FAILURE: api.Logger.error("No ionic uplink interfaces detected") return api.types.status.FAILURE #raise OfflineTestbedException return api.types.status.SUCCESS
def recover_ilo(cimc_info, node_name): try: check_set_dedicated_mode(cimc_info) except: api.Logger.error(traceback.format_exc()) api.Logger.info("Recovery by apc power cycle") ret = api.RestartNodes([node_name], 'apc') if ret != api.types.status.SUCCESS: api.Logger.info("APC power cycle failed") return api.types.status.FAILURE api.Logger.info("APC power cycle done") check_set_dedicated_mode(cimc_info)
def Trigger(tc): req = api.Trigger_CreateExecuteCommandsRequest() for n in tc.Nodes: cmd = 'curl -k -d \'{"kind": "SmartNICRollout","meta": {"name": "test disruptive upgrade","tenant": "tenant-foo"},"spec": {"ops": [{"op": 4,"version": "0.1"}]}}\' -X POST -H "Content-Type:application/json" ' + 'https://' + api.GetNicIntMgmtIP( n) + ':8888/api/v1/naples/rollout/' api.Trigger_AddHostCommand(req, n, cmd) tc.resp = api.Trigger(req) api.Logger.info("started rollout") time.sleep(1) api.Logger.info("restarting nodes") ret = api.RestartNodes(tc.Nodes) return api.types.status.SUCCESS
def Trigger(tc): if tc.skip: return api.types.status.SUCCESS for reboot_i in range(tc.reboot_count): api.Logger.info('Begin %s num %d' % (tc.restart_mode, reboot_i)) if tc.restart_mode == 'reinit': tc.result = api.ReinitForTestsuite() else: tc.result = api.RestartNodes(tc.node_names, tc.restart_mode) if tc.result is not api.types.status.SUCCESS: break return api.types.status.SUCCESS
def Trigger(tc): #api.Logger.info("Saved IotaAgent state...") #resp = api.SaveIotaAgentState(tc.nodes) #if resp != api.types.status.SUCCESS: # api.Logger.error("Failed to save node") # return resp api.Logger.info("Restart Nodes...") tc.resp = api.RestartNodes(tc.nodes) #api.Logger.info("Restore IotaAgent...") #tc.resp = api.RestoreIotaAgentState(tc.nodes) #if tc.resp != api.types.status.SUCCESS: # api.Logger.error("Failed to restore IotaAgent after host reboot/reimage") #else: # api.Logger.info("Restore IotaAgent successful post reboot/reimage") return api.types.status.SUCCESS
def Verify(tc): for node in tc.nodes: api.Logger.info(f"Reloading {node}") if api.RestartNodes([node], 'ipmi') == api.types.status.FAILURE: return api.types.status.FAILURE # if host.LoadDriver(api.GetNodeOs(node), node) is api.types.status.FAILURE: # api.Logger.error(f"Failed to load driver on {node}") # return api.types.status.FAILURE # # api.Logger.info(f"Restoring workloads on {node}") # # time.sleep(120) # # pdb.set_trace() # wl_api.ReAddWorkloads(node) return api.types.status.SUCCESS
def RebootHost(n): nodes = [] nodes.append(n) api.Logger.info("Rebooting Host {}".format(n)) ret = api.RestartNodes(nodes) # Loop to check if the card is reachable reqPing = api.Trigger_CreateExecuteCommandsRequest(serial=True) cmd = "ping -c 2 " + api.GetMgmtIPAddress(n) api.Trigger_AddHostCommand(reqPing, n, cmd) retryCount = 0 while retryCount < 6: retryCount = retryCount + 1 respPing = api.Trigger(reqPing) if len(respPing.commands) != 1: continue if respPing.commands[0].exit_code == 0: break api.Logger.info( "Ping to the host failed. Ping Response {}".format(respPing)) time.sleep(10) if retryCount == 6: api.Logger.info( "Nodes were unreachable after reboot. {}".format(nodes)) api.types.status.FAILURE api.Logger.info("Successfully rebooted host {}".format(n)) req = api.Trigger_CreateExecuteCommandsRequest(serial=True) enable_sshd = "system enable-sshd" copy_key = "update ssh-pub-key -f ~/.ssh/id_rsa.pub" for n in nodes: # hack for now, need to set date AddPenctlCommand(req, n, enable_sshd) AddPenctlCommand(req, n, copy_key) resp = api.Trigger(req) return api.types.status.SUCCESS
def Main(args): if GlobalOptions.skip_setup: # No profile change is required for skip setup return api.types.status.SUCCESS newProfile = getattr(args, 'profile', 'FEATURE_PROFILE_BASE') if newProfile != 'FEATURE_PROFILE_BASE': req = api.Trigger_CreateExecuteCommandsRequest(serial=True) for n in api.GetNaplesHostnames(): common.AddPenctlCommand( req, n, "update dsc --device-profile %s" % newProfile) resp = api.Trigger(req) if not api.Trigger_IsSuccess(resp): return api.types.status.FAILURE api.Logger.info("Naples profile changed to %s - Restarting nodes" % newProfile) return api.RestartNodes(api.GetNaplesHostnames()) else: api.Logger.info("Profile unchanged, input: %s" % newProfile) return api.types.status.SUCCESS
def Verify(tc): if tc.resp is None: return api.types.status.FAILURE for cmd in tc.resp.commands: api.PrintCommandResults(cmd) if cmd.exit_code != 0: if PREINIT_SCRIPT_NAME in cmd.command: api.Logger.error("Failed to copy {} to {} on {}".format( PREINIT_SCRIPT_NAME, NAPLES_PREINIT_SCRIPT_PATH, tc.bitw_node_name)) return api.types.status.FAILURE if START_AGENT_SCRIPT_NAME in cmd.command: api.Logger.error("Failed to copy {} to {} on {}".format( START_AGENT_SCRIPT_NAME, NAPLES_START_AGENT_SCRIPT_PATH, tc.bitw_node_name)) return api.types.status.FAILURE # reboot the node api.Logger.info("Rebooting {}".format(tc.bitw_node_name)) return api.RestartNodes([tc.bitw_node_name], 'reboot')
def Main(step): if GlobalOptions.skip_setup: #No mode switch required for skeip setup return api.types.status.SUCCESS req = api.Trigger_CreateExecuteCommandsRequest(serial = True) uuidMap = api.GetNaplesNodeUuidMap() nodes = api.GetNaplesHostnames() for n in nodes: # Touch a file to indicate to NMD that the current mode is emulation cmd = "touch /data/iota-emulation" api.Trigger_AddNaplesCommand(req, n, cmd) # Make sure console is enabled CreateConfigConsoleNoAuth() api.CopyToNaples(n, [NAPLES_CONFIG_SPEC_LOCAL], "") cmd = "mv /system-config.json /sysconfig/config0/system-config.json" api.Trigger_AddNaplesCommand(req, n, cmd) if common.PenctlGetModeStatus(n) != "NETWORK" or common.PenctlGetTransitionPhaseStatus(n) != "VENICE_REGISTRATION_DONE": api.Logger.info("Host [{}] is in HOST mode. Initiating mode change.".format(n)) ret = common.SetNaplesModeOOB_Static(n, "1.1.1.1", "1.1.1.2/24") if ret == None: return api.types.status.FAILURE #hack for now, need to set date cmd = "date -s '{}'".format(datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")) api.Trigger_AddNaplesCommand(req, n, cmd) resp = api.Trigger(req) for cmd in resp.commands: api.PrintCommandResults(cmd) if cmd.exit_code != 0: return api.types.status.FAILURE num_retries = 60 reboot_nodes = [] while nodes: req = api.Trigger_CreateExecuteCommandsRequest(serial = True) for n in nodes: #hack for now, need to set date api.Logger.info("Checking Transition phase for node : %s" % n) check_state_cmd = "show naples --json" time.sleep(30) common.AddPenctlCommand(req, n, check_state_cmd) api.Trigger_AddNaplesCommand(req, n, "touch /data/no_watchdog") resp = api.Trigger(req) cmd = resp.commands[0] api.PrintCommandResults(cmd) if cmd.exit_code != 0: return api.types.status.FAILURE try: out = json.loads(cmd.stdout) except: api.Logger.error("Penctl output not in Json format {}".format(cmd.stdout)) return api.types.status.FAILURE if not hal_show_utils.IsNaplesForwardingModeClassic(n): api.Logger.info("Dataplane already in HOSTPIN mode. Skipping node [{}] for reboot.".format(n)) reboot_nodes.append(n) nodes.remove(n) elif out["status"]["transition-phase"] == "VENICE_UNREACHABLE": api.Logger.info("Reboot pending on node : %s" % n) reboot_nodes.append(n) nodes.remove(n) elif out["status"]["transition-phase"] == "VENICE_REGISTRATION_DONE": api.Logger.info("Node already transitioned : %s" % n) nodes.remove(n) else: api.Logger.info("Reboot not pending on node : %s" % n) time.sleep(1) num_retries = num_retries - 1 if num_retries == 0: api.Logger.error("Reboot pending state not transitioned complete on naples") return api.types.status.FAILURE ret = api.RestartNodes(reboot_nodes) if ret != api.types.status.SUCCESS: api.Logger.error("Node restart failed") return api.types.status.FAILURE req = api.Trigger_CreateExecuteCommandsRequest(serial = True) #enable_sshd = "system enable-sshd" #copy_key = "update ssh-pub-key -f ~/.ssh/id_rsa.pub" for n in api.GetNaplesHostnames(): #hack for now, need to set date cmd = "date -s '{}'".format(datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")) api.Trigger_AddNaplesCommand(req, n, cmd) #common.AddPenctlCommand(req, n, enable_sshd) #common.AddPenctlCommand(req, n, copy_key) resp = api.Trigger(req) for cmd in resp.commands: api.PrintCommandResults(cmd) if cmd.exit_code != 0: result = api.types.status.FAILURE #Give some time for naples to admit to venice after mode switch #This is required for now as Iota is setting time manually to make TLS happy time.sleep(30) #Check whether naples has switch mode succesfully req = api.Trigger_CreateExecuteCommandsRequest(serial = False) for n in nodes: cmd = "cat /sysconfig/config0/app-start.conf | grep hostpin" api.Trigger_AddNaplesCommand(req, n, cmd) resp = api.Trigger(req) for cmd in resp.commands: api.PrintCommandResults(cmd) if cmd.exit_code != 0: api.Logger.error("Mode switch failed on node : {}".format(cmd.node_name)) result = api.types.status.FAILURE #Verify Agent in right mode. req = api.Trigger_CreateExecuteCommandsRequest(serial = False) for n in nodes: cmd = "curl localhost:8888/api/system/info/" api.Trigger_AddNaplesCommand(req, n, cmd) # Delete the iota-emulation file created earlied for NMD. Mode change would have passed by now if it had to. cmd = "rm -f /data/iota-emulation" api.Trigger_AddNaplesCommand(req, n, cmd) resp = api.Trigger(req) for cmd in resp.commands: api.PrintCommandResults(cmd) if cmd.exit_code != 0: api.Logger.error("Agent system get failed : {}".format(cmd.node_name)) result = api.types.status.FAILURE out = None try: out = json.loads(cmd.stdout) except: api.Logger.error("Agent System get out failed {}".format(cmd.stdout)) return api.types.status.FAILURE if out["naples-mode"] != "NETWORK_MANAGED_OOB": api.Logger.error("Agent not in correct mode: {} {} ".format(cmd.node_name, out["naples-mode"])) return api.types.status.FAILURE api.Logger.info("Trying to re-enable ssh on naples nodes") return enable_ssh.Main(None)
def Trigger(tc): api.Logger.info("Starting CPLD Upgrade test") tc.Nodes = api.GetNaplesHostnames() api.Logger.info("The number of hosts {}".format(len(tc.Nodes))) reboot_nodes = [] for node in tc.Nodes: req = api.Trigger_CreateExecuteCommandsRequest(serial=True) api.Trigger_AddNaplesCommand(req, node, "/nic/tools/fwupdate -l") resp = api.Trigger(req) for cmd in resp.commands: req_cpldapp = api.Trigger_CreateExecuteCommandsRequest(serial=True) try: out = json.loads(cmd.stdout) except: api.Logger.error("Error using fwupdate {}".format(cmd.stdout)) return api.types.status.FAILURE revision = out["cpld"]["bitfile"]["version"] api.Logger.info("{} :: CPLD revision is {}".format( cmd.node_name, revision)) if revision == "8": api.Trigger_AddNaplesCommand( req_cpldapp, node, "LD_LIBRARY_PATH=/platform/lib/ /platform/bin/cpldapp -writeflash /data/naples100_rev9.bin", timeout=300) elif revision == "9": api.Trigger_AddNaplesCommand( req_cpldapp, node, "LD_LIBRARY_PATH=/platform/lib/ /platform/bin/cpldapp -writeflash /data/naples100_02_rev8.bin", timeout=300) else: api.Logger.error("Revision not supported continue") continue api.Trigger_AddNaplesCommand(req_cpldapp, node, "touch /tmp/cpldreset") resp_cpldapp = api.Trigger(req_cpldapp) for cmd_cpldapp in resp_cpldapp.commands: if cmd_cpldapp.exit_code == 0: api.Logger.info("Cpld written successfully ") reboot_nodes.append(node) ret = api.RestartNodes(reboot_nodes) if ret != api.types.status.SUCCESS: api.Logger.error("Node restart failed") return api.types.status.FAILURE api.Logger.info("Hosts rebooted successfully") for node_afterreboot in tc.Nodes: req_afterreboot = api.Trigger_CreateExecuteCommandsRequest(serial=True) api.Trigger_AddNaplesCommand(req_afterreboot, node_afterreboot, "/nic/tools/fwupdate -l") resp_afterreboot = api.Trigger(req_afterreboot) for cmd_afterreboot in resp_afterreboot.commands: try: out = json.loads(cmd_afterreboot.stdout) except: api.Logger.error("Error using fwupdate {}".format( cmd_afterreboot.stdout)) return api.types.status.FAILURE api.Logger.info("{} :: CPLD revision is {}".format( cmd_afterreboot.node_name, out["cpld"]["bitfile"]["version"])) return api.types.status.SUCCESS
def __reboot_nodes(): nodes = api.GetWorkloadNodeHostnames() api.RestartNodes(nodes) return api.types.status.SUCCESS
def LoadFirmware(nodes, node_os, target_version): if target_version != 'latest': resp = api.DownloadAssets(release_version = target_version) if not api.IsApiResponseOk(resp): api.Logger.error("Failed to download assets for %s" % target_version) return resp manifest_file = os.path.join(api.GetTopDir(), 'images', target_version + '.json') image_manifest = parser.JsonParse(manifest_file) fw_images = list(filter(lambda x: x.naples_type == "capri", image_manifest.Firmwares))[0] if fw_images is None: api.Logger.error("Unable to load image manifest") return api.types.status.FAILURE api.Logger.info("Fullpath for firmware image to load: %s " % fw_images.image) if node_os == OS_TYPE_LINUX: fw_version, _ = GetLinuxFwDriverVersion(node) else: fw_version = None fwImgFile = os.path.join(GlobalOptions.topdir, fw_images.image) for node in nodes: if fw_version == '1.1.1-E-15': # Naples with Fw 1.1.1-E-15 has no OOB and IntMgmt Ip is fixed resp = api.CopyToNaples(node, [fwImgFile], image_manifest.Version, via_oob=False, naples_dir="data", nic_mgmt_ip=GetNicIntMgmtIP(node)) if not api.IsApiResponseOk(resp): api.Logger.info("Failed to copy naples_fw.tar with via_oob=True") return api.types.status.FAILURE for cmd in resp.commands: if cmd.exit_code != 0: api.Logger.error("Failed to copy %s naples_fw.tar via_oob=True" % image_manifest.Version) return api.types.status.FAILURE else: resp = api.CopyToNaples(node, [fwImgFile], image_manifest.Version, via_oob=True, naples_dir="data") if not api.IsApiResponseOk(resp): api.Logger.info("Failed to copy naples_fw.tar with via_oob=True") return api.types.status.FAILURE for cmd in resp.commands: if cmd.exit_code != 0: api.Logger.error("Failed to copy %s naples_fw.tar via_oob=True" % image_manifest.Version) # Try with via_oob=False resp = api.CopyToNaples(node, [fwImgFile], image_manifest.Version, via_oob=False, naples_dir="data") if not api.Trigger_IsSuccess(resp): api.Logger.error("Failed to copy naples_fw.tar to target naples") return api.types.status.FAILURE req = api.Trigger_CreateExecuteCommandsRequest(serial = True) api.Trigger_AddNaplesCommand(req, node, "sync") api.Trigger_AddNaplesCommand(req, node, "/nic/tools/sysupdate.sh -p /data/naples_fw.tar", timeout=120) resp = api.Trigger(req) if not api.Trigger_IsSuccess(resp): api.Logger.error("sysupdate.sh cmd failed") return api.types.status.FAILURE api.RestartNodes([node]) if node_os == OS_TYPE_LINUX: return LinuxReInitMgmtIP(node) elif node_os == OS_TYPE_ESX: return ESXiReInitMgmtIP(node) return api.types.status.SUCCESS
def Setup(tc): # get node info tc.bitw_node_name = None tc.wl_node_name = None # Assuming only one bitw node and one workload node nics = store.GetTopology().GetNicsByPipeline("athena") for nic in nics: tc.bitw_node_name = nic.GetNodeName() break api.SetTestsuiteAttr("bitw_node_name", tc.bitw_node_name) workloads = api.GetWorkloads() if len(workloads) == 0: api.Logger.error('No workloads available') return api.types.status.FAILURE tc.wl_node_name = workloads[0].node_name api.SetTestsuiteAttr("wl_node_name", tc.wl_node_name) host_intfs = api.GetNaplesHostInterfaces(tc.wl_node_name) # Assuming single nic per host if len(host_intfs) != 2: api.Logger.error('Failed to get host interfaces') return api.types.status.FAILURE tc.wl = [] for wl in workloads: tc.wl.append(wl) api.Logger.info("wl: vlan: {}, mac: {}, ip: {}".format( wl.uplink_vlan, wl.mac_address, wl.ip_address)) tc.intfs = [] tc.intfs.append({ 'name': 'inb_mnic0', 'ip': str(tc.wl[0].ip_address), 'sub_ip': str(tc.wl[2].ip_address), 'vlan': str(tc.wl[2].uplink_vlan) }) tc.intfs.append({ 'name': 'inb_mnic1', 'ip': str(tc.wl[1].ip_address), 'sub_ip': str(tc.wl[3].ip_address), 'vlan': str(tc.wl[3].uplink_vlan) }) api.SetTestsuiteAttr("inb_mnic_intfs", tc.intfs) # copy device_bootstrap.json to naples bootstrap_json_fname = api.GetTopDir( ) + '/nic/conf/athena/device_bootstrap.json' api.CopyToNaples(tc.bitw_node_name, [bootstrap_json_fname], "") # write and copy pensando_pre_init.sh to naples f = open('pensando_pre_init.sh', "w") f.write('echo "copying device.json"\n') f.write('cp /data/device_bootstrap.json /nic/conf/device.json\n') f.close() api.CopyToNaples(tc.bitw_node_name, ['pensando_pre_init.sh'], "") os.remove('pensando_pre_init.sh') # move pensando_pre_init.sh to /sysconfig/config0/ and restart Athena Node req = api.Trigger_CreateExecuteCommandsRequest() cmd = "mv /pensando_pre_init.sh /sysconfig/config0/" api.Trigger_AddNaplesCommand(req, tc.bitw_node_name, cmd) cmd = "mv /device_bootstrap.json /data/" api.Trigger_AddNaplesCommand(req, tc.bitw_node_name, cmd) resp = api.Trigger(req) cmd = resp.commands[0] api.PrintCommandResults(cmd) if cmd.exit_code != 0: api.Logger.error("Bootstrap setup failed on node %s" % \ tc.bitw_node_name) return api.types.status.FAILURE # reboot the node api.Logger.info("Rebooting {}".format(tc.bitw_node_name)) return api.RestartNodes([tc.bitw_node_name], 'reboot')
def Main(step): nodes = api.GetWorkloadNodeHostnames() api.RestartNodes(nodes) return api.types.status.SUCCESS
def Main(step): req = api.Trigger_CreateExecuteCommandsRequest(serial=True) venice_ips = api.GetVeniceMgmtIpAddresses() assert (venice_ips) uuidMap = api.GetNaplesNodeUuidMap() for n in api.GetNaplesHostnames(): #hack for now, need to set date cmd = "date -s '{}'".format( datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")) api.Trigger_AddNaplesCommand(req, n, cmd) api.Logger.info("Running mode switch on node : %s" % n) mode_switch_cmd = "update naples --managed-by network --management-network oob --hostname {} --primary-mac {}".format( __get_mgmt_nw(), n, uuidMap[n]) common.AddPenctlCommand(req, n, mode_switch_cmd) resp = api.Trigger(req) for cmd in resp.commands: api.PrintCommandResults(cmd) if cmd.exit_code != 0: return api.types.status.FAILURE nodes = api.GetNaplesHostnames() num_retries = 60 while nodes: req = api.Trigger_CreateExecuteCommandsRequest(serial=True) for n in nodes: #hack for now, need to set date api.Logger.info("Checking Tranisition phase for node : %s" % n) check_state_cmd = "show naples --json" common.AddPenctlCommand(req, n, check_state_cmd) resp = api.Trigger(req) cmd = resp.commands[0] api.PrintCommandResults(cmd) if cmd.exit_code != 0: return api.types.status.FAILURE try: out = json.loads(cmd.stdout) except: api.Logger.error("Penctl output not in Json format {}".format( cmd.stdout)) return api.types.status.FAILURE if out["status"]["transition-phase"] == "REBOOT_PENDING": api.Logger.info("Reboot pending on node : %s" % n) nodes.remove(n) else: api.Logger.info("Reboot not pending on node : %s" % n) time.sleep(1) num_retries = num_retries - 1 if num_retries == 0: api.Logger.error( "Reboot pending state not transitioned complete on naples") return api.types.status.FAILURE reboot_nodes = [] for n in api.GetNaplesHostnames(): reboot_nodes.append(n) ret = api.RestartNodes(reboot_nodes) if ret != api.types.status.SUCCESS: api.Logger.error("Node restart failed") return api.types.status.FAILURE req = api.Trigger_CreateExecuteCommandsRequest(serial=True) for n in api.GetNaplesHostnames(): #hack for now, need to set date cmd = "date -s '{}'".format( datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")) api.Trigger_AddNaplesCommand(req, n, cmd) resp = api.Trigger(req) for cmd in resp.commands: api.PrintCommandResults(cmd) if cmd.exit_code != 0: result = api.types.status.FAILURE #Give some time for naples to admit to venice after mode switch #This is required for now as Iota is setting time manually to make TLS happy time.sleep(30) #Check whether naples has switch mode succesfully req = api.Trigger_CreateExecuteCommandsRequest(serial=False) for n in api.GetNaplesHostnames(): cmd = "cat /sysconfig/config0/app-start.conf | grep hostpin" api.Trigger_AddNaplesCommand(req, n, cmd) resp = api.Trigger(req) for cmd in resp.commands: api.PrintCommandResults(cmd) if cmd.exit_code != 0: api.Logger.error("Mode switch failed on node : {}".format( cmd.node_name)) result = api.types.status.FAILURE #Verify Agent in right mode. req = api.Trigger_CreateExecuteCommandsRequest(serial=False) for n in api.GetNaplesHostnames(): cmd = "curl localhost:8888/api/system/info/" api.Trigger_AddNaplesCommand(req, n, cmd) resp = api.Trigger(req) venice_urls = [] for ip in api.GetVeniceMgmtIpAddresses(): venice_urls.append(ip + ":9009") for cmd in resp.commands: api.PrintCommandResults(cmd) if cmd.exit_code != 0: api.Logger.error("Agent system get failed : {}".format( cmd.node_name)) result = api.types.status.FAILURE out = None try: out = json.loads(cmd.stdout) except: api.Logger.error("Agent System get out failed {}".format( cmd.stdout)) return api.types.status.FAILURE if out["naples-mode"] != "NETWORK_MANAGED_OOB": api.Logger.error("Agent not in correct mode: {} {} ".format( cmd.node_name, out["naples-mode"])) return api.types.status.FAILURE if set(out["controller-ips"]) != set(venice_urls): api.Logger.error("Agent controller IPs don't match: {} {}".format( venice_urls, out["controller-ips"])) return api.types.status.FAILURE return api.types.status.SUCCESS