def Trigger(tc):
tc.pre_data = {}
tc.post_data = {}
try:
for _iter in range(tc.iterators.count):
api.Logger.info("Iter: %d" % _iter)
api.Logger.info("Issuing APC power cycle")
# Getting port status before reboot
tc.pre_data[_iter] = get_nw_ports_info(tc.RF)
ret = api.RestartNodes([tc.node_name], 'apc')
if ret != api.types.status.SUCCESS:
api.Logger.info("APC power cycle failed")
return api.types.status.FAILURE
api.Logger.info("APC power cycle done")
tc.test_node.WaitForHost()
api.Logger.info("APC power cycle successful")
if enable_ssh.Main(None) != api.types.status.SUCCESS:
api.Logger.info("Enabling SSH failed after reboot")
return api.types.status.FAILURE
api.Logger.info("Enabled Naples SSH after power cycle")
# Getting port status after reboot
tc.post_data[_iter] = get_nw_ports_info(tc.RF)
except Exception as e:
api.Logger.error(str(e))
return api.types.status.ERROR
return api.types.status.SUCCESS
def Setup(tc):
if api.IsDryrun(): return api.types.status.SUCCESS
tc.nodes = api.GetWorkloadNodeHostnames()
tc.node_intfs = {}
srv,cli = _get_workloads(tc)
tc.workloads = [srv, cli]
if getattr(tc.args, 'restart', False):
ret = api.RestartNodes(tc.nodes)
if ret != api.types.status.SUCCESS:
api.Logger.error("Node restart failed")
return api.types.status.FAILURE
api.Logger.info("Setting driver features")
if common.setup_features(tc) != api.types.status.SUCCESS:
api.Logger.info("Setting driver features :Failed")
return api.types.status.FAILURE
api.Logger.info("Setting driver features : Success")
if getattr(tc.args, 'capture_pcap', False):
if common.start_pcap_capture(tc) != api.types.status.SUCCESS:
return api.types.status.FAILURE
if getattr(tc.args, 'capture_single_pcap', False):
if common.start_single_pcap_capture(tc) != api.types.status.SUCCESS:
return api.types.status.FAILURE
return api.types.status.SUCCESS
def LoadDriver(node_names, node_os, target_version='latest'):
if target_version == 'latest':
api.Logger.info('Target version is latest - nothing to change')
else:
resp = api.DownloadAssets(release_version = target_version)
if not api.IsApiResponseOk(resp):
api.Logger.error("Failed to download assets for %s" % target_version)
return api.types.status.FAILURE
manifest_file = os.path.join(api.GetTopDir(), 'images', target_version + '.json')
for node in node_names:
if node_os == OS_TYPE_LINUX:
if __load_linux_driver(tc, node, manifest_file) == api.types.status.SUCCESS:
api.Logger.info("Release Driver %s reload on %s" % (tc.target_version, node))
else:
api.Logger.error("Failed to load release driver %s reload on %s" % (tc.target_version, node))
return api.types.status.FAILURE
# this is required to bring the testbed into operation state
# after driver unload interfaces need to be initialized
wl_api.ReAddWorkloads(node)
elif tc.os == OS_TYPE_ESX:
host.UnloadDriver(tc.os, node)
tc.driver_changed = True
if __load_esxi_driver(node, node_os, manifest_file) == api.types.status.SUCCESS:
api.Logger.info("Release Driver %s reload on %s" % (tc.target_version, node))
else:
api.Logger.error("Failed to load release driver %s reload on %s" % (tc.target_version, node))
return api.types.status.FAILURE
api.RestartNodes([node])
def Trigger(tc):
try:
for _iter in range(tc.iterators.count):
api.Logger.info("Iter: %d" % _iter)
api.Logger.info("Issuing APC power cycle")
ret = api.RestartNodes([tc.node_name], 'apc')
if ret != api.types.status.SUCCESS:
api.Logger.info("APC power cycle failed")
return api.types.status.FAILURE
api.Logger.info("APC power cycle done")
tc.test_node.WaitForHost()
ret = check_ncsi_conn(tc.cimc_info)
if ret != api.types.status.SUCCESS:
api.Logger.error("Unable to connect ot ILO in NCSI mode")
return api.types.status.FAILURE
api.Logger.info("APC power cycle successfull")
if enable_ssh.Main(None) != api.types.status.SUCCESS:
api.Logger.info("Enabling SSH failed after reboot")
return api.types.status.FAILURE
api.Logger.info("Enabled Naples SSH after power cycle")
except Exception as e:
api.Logger.error(str(e))
return api.types.status.FAILURE
return api.types.status.SUCCESS
def Trigger(tc):
naples_nodes = []
#for every node in the setup
for node in tc.nodes:
if api.IsNaplesNode(node):
naples_nodes.append(node)
api.Logger.info(f"Found Naples Node: [{node}]")
if len(naples_nodes) == 0:
api.Logger.error("Failed to find a Naples Node!")
return api.types.status.FAILURE
for reboot in range(tc.args.reboots):
# Reboot Node.
# Reboot method (APC, IPMI, OS Reboot) is passed as a testcase parameter
for node in naples_nodes:
api.Logger.info(
f"==== Reboot Loop # {reboot} on {node}. Reboot method: {tc.iterators.reboot_method} ===="
)
if api.RestartNodes(
[node],
tc.iterators.reboot_method) != api.types.status.SUCCESS:
return api.types.status.FAILURE
#raise OfflineTestbedException
# Enable SSH, some checks require Naples access
if enable_ssh.Main(None) != api.types.status.SUCCESS:
api.Logger.error("Enabling SSH failed")
return api.types.status.FAILURE
# there is not a real "PCI" in IOTA for Windows.
if tc.os != host.OS_TYPE_WINDOWS:
api.Logger.info(f"Verifying PCI on [{node}]: ")
if verify_pci.verify_errors_lspci(
node, tc.os) != api.types.status.SUCCESS:
api.Logger.error(f"PCIe Failure detected on {node}")
return api.types.status.FAILURE
#raise OfflineTestbedException
if checkLinks(tc, node) is api.types.status.FAILURE:
api.Logger.error("Error verifying uplink interfaces")
return api.types.status.FAILURE
#raise OfflineTestbedException
# don't run this while we figure out how to do this in ESX
if tc.os != host.OS_TYPE_ESX:
# Load the ionic driver except windows
if tc.os != host.OS_TYPE_WINDOWS:
if host.LoadDriver(tc.os,
node) is api.types.status.FAILURE:
api.Logger.info("ionic already loaded")
# Make sure ionic driver attached to Uplink ports.
if checkDrv(node) is api.types.status.FAILURE:
api.Logger.error("No ionic uplink interfaces detected")
return api.types.status.FAILURE
#raise OfflineTestbedException
return api.types.status.SUCCESS
def recover_ilo(cimc_info, node_name):
try:
check_set_dedicated_mode(cimc_info)
except:
api.Logger.error(traceback.format_exc())
api.Logger.info("Recovery by apc power cycle")
ret = api.RestartNodes([node_name], 'apc')
if ret != api.types.status.SUCCESS:
api.Logger.info("APC power cycle failed")
return api.types.status.FAILURE
api.Logger.info("APC power cycle done")
check_set_dedicated_mode(cimc_info)
def Trigger(tc):
req = api.Trigger_CreateExecuteCommandsRequest()
for n in tc.Nodes:
cmd = 'curl -k -d \'{"kind": "SmartNICRollout","meta": {"name": "test disruptive upgrade","tenant": "tenant-foo"},"spec": {"ops": [{"op": 4,"version": "0.1"}]}}\' -X POST -H "Content-Type:application/json" ' + 'https://' + api.GetNicIntMgmtIP(
n) + ':8888/api/v1/naples/rollout/'
api.Trigger_AddHostCommand(req, n, cmd)
tc.resp = api.Trigger(req)
api.Logger.info("started rollout")
time.sleep(1)
api.Logger.info("restarting nodes")
ret = api.RestartNodes(tc.Nodes)
return api.types.status.SUCCESS
def Trigger(tc):
if tc.skip:
return api.types.status.SUCCESS
for reboot_i in range(tc.reboot_count):
api.Logger.info('Begin %s num %d' % (tc.restart_mode, reboot_i))
if tc.restart_mode == 'reinit':
tc.result = api.ReinitForTestsuite()
else:
tc.result = api.RestartNodes(tc.node_names, tc.restart_mode)
if tc.result is not api.types.status.SUCCESS:
break
return api.types.status.SUCCESS
def Trigger(tc):
#api.Logger.info("Saved IotaAgent state...")
#resp = api.SaveIotaAgentState(tc.nodes)
#if resp != api.types.status.SUCCESS:
# api.Logger.error("Failed to save node")
# return resp
api.Logger.info("Restart Nodes...")
tc.resp = api.RestartNodes(tc.nodes)
#api.Logger.info("Restore IotaAgent...")
#tc.resp = api.RestoreIotaAgentState(tc.nodes)
#if tc.resp != api.types.status.SUCCESS:
# api.Logger.error("Failed to restore IotaAgent after host reboot/reimage")
#else:
# api.Logger.info("Restore IotaAgent successful post reboot/reimage")
return api.types.status.SUCCESS
def Verify(tc):
for node in tc.nodes:
api.Logger.info(f"Reloading {node}")
if api.RestartNodes([node], 'ipmi') == api.types.status.FAILURE:
return api.types.status.FAILURE
# if host.LoadDriver(api.GetNodeOs(node), node) is api.types.status.FAILURE:
# api.Logger.error(f"Failed to load driver on {node}")
# return api.types.status.FAILURE
#
# api.Logger.info(f"Restoring workloads on {node}")
#
# time.sleep(120)
# # pdb.set_trace()
# wl_api.ReAddWorkloads(node)
return api.types.status.SUCCESS
def RebootHost(n):
nodes = []
nodes.append(n)
api.Logger.info("Rebooting Host {}".format(n))
ret = api.RestartNodes(nodes)
# Loop to check if the card is reachable
reqPing = api.Trigger_CreateExecuteCommandsRequest(serial=True)
cmd = "ping -c 2 " + api.GetMgmtIPAddress(n)
api.Trigger_AddHostCommand(reqPing, n, cmd)
retryCount = 0
while retryCount < 6:
retryCount = retryCount + 1
respPing = api.Trigger(reqPing)
if len(respPing.commands) != 1:
continue
if respPing.commands[0].exit_code == 0:
break
api.Logger.info(
"Ping to the host failed. Ping Response {}".format(respPing))
time.sleep(10)
if retryCount == 6:
api.Logger.info(
"Nodes were unreachable after reboot. {}".format(nodes))
api.types.status.FAILURE
api.Logger.info("Successfully rebooted host {}".format(n))
req = api.Trigger_CreateExecuteCommandsRequest(serial=True)
enable_sshd = "system enable-sshd"
copy_key = "update ssh-pub-key -f ~/.ssh/id_rsa.pub"
for n in nodes:
# hack for now, need to set date
AddPenctlCommand(req, n, enable_sshd)
AddPenctlCommand(req, n, copy_key)
resp = api.Trigger(req)
return api.types.status.SUCCESS
def Main(args):
if GlobalOptions.skip_setup:
# No profile change is required for skip setup
return api.types.status.SUCCESS
newProfile = getattr(args, 'profile', 'FEATURE_PROFILE_BASE')
if newProfile != 'FEATURE_PROFILE_BASE':
req = api.Trigger_CreateExecuteCommandsRequest(serial=True)
for n in api.GetNaplesHostnames():
common.AddPenctlCommand(
req, n, "update dsc --device-profile %s" % newProfile)
resp = api.Trigger(req)
if not api.Trigger_IsSuccess(resp):
return api.types.status.FAILURE
api.Logger.info("Naples profile changed to %s - Restarting nodes" %
newProfile)
return api.RestartNodes(api.GetNaplesHostnames())
else:
api.Logger.info("Profile unchanged, input: %s" % newProfile)
return api.types.status.SUCCESS
def Verify(tc):
if tc.resp is None:
return api.types.status.FAILURE
for cmd in tc.resp.commands:
api.PrintCommandResults(cmd)
if cmd.exit_code != 0:
if PREINIT_SCRIPT_NAME in cmd.command:
api.Logger.error("Failed to copy {} to {} on {}".format(
PREINIT_SCRIPT_NAME, NAPLES_PREINIT_SCRIPT_PATH,
tc.bitw_node_name))
return api.types.status.FAILURE
if START_AGENT_SCRIPT_NAME in cmd.command:
api.Logger.error("Failed to copy {} to {} on {}".format(
START_AGENT_SCRIPT_NAME, NAPLES_START_AGENT_SCRIPT_PATH,
tc.bitw_node_name))
return api.types.status.FAILURE
# reboot the node
api.Logger.info("Rebooting {}".format(tc.bitw_node_name))
return api.RestartNodes([tc.bitw_node_name], 'reboot')
def Main(step):
if GlobalOptions.skip_setup:
#No mode switch required for skeip setup
return api.types.status.SUCCESS
req = api.Trigger_CreateExecuteCommandsRequest(serial = True)
uuidMap = api.GetNaplesNodeUuidMap()
nodes = api.GetNaplesHostnames()
for n in nodes:
# Touch a file to indicate to NMD that the current mode is emulation
cmd = "touch /data/iota-emulation"
api.Trigger_AddNaplesCommand(req, n, cmd)
# Make sure console is enabled
CreateConfigConsoleNoAuth()
api.CopyToNaples(n, [NAPLES_CONFIG_SPEC_LOCAL], "")
cmd = "mv /system-config.json /sysconfig/config0/system-config.json"
api.Trigger_AddNaplesCommand(req, n, cmd)
if common.PenctlGetModeStatus(n) != "NETWORK" or common.PenctlGetTransitionPhaseStatus(n) != "VENICE_REGISTRATION_DONE":
api.Logger.info("Host [{}] is in HOST mode. Initiating mode change.".format(n))
ret = common.SetNaplesModeOOB_Static(n, "1.1.1.1", "1.1.1.2/24")
if ret == None:
return api.types.status.FAILURE
#hack for now, need to set date
cmd = "date -s '{}'".format(datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"))
api.Trigger_AddNaplesCommand(req, n, cmd)
resp = api.Trigger(req)
for cmd in resp.commands:
api.PrintCommandResults(cmd)
if cmd.exit_code != 0:
return api.types.status.FAILURE
num_retries = 60
reboot_nodes = []
while nodes:
req = api.Trigger_CreateExecuteCommandsRequest(serial = True)
for n in nodes:
#hack for now, need to set date
api.Logger.info("Checking Transition phase for node : %s" % n)
check_state_cmd = "show naples --json"
time.sleep(30)
common.AddPenctlCommand(req, n, check_state_cmd)
api.Trigger_AddNaplesCommand(req, n, "touch /data/no_watchdog")
resp = api.Trigger(req)
cmd = resp.commands[0]
api.PrintCommandResults(cmd)
if cmd.exit_code != 0:
return api.types.status.FAILURE
try:
out = json.loads(cmd.stdout)
except:
api.Logger.error("Penctl output not in Json format {}".format(cmd.stdout))
return api.types.status.FAILURE
if not hal_show_utils.IsNaplesForwardingModeClassic(n):
api.Logger.info("Dataplane already in HOSTPIN mode. Skipping node [{}] for reboot.".format(n))
reboot_nodes.append(n)
nodes.remove(n)
elif out["status"]["transition-phase"] == "VENICE_UNREACHABLE":
api.Logger.info("Reboot pending on node : %s" % n)
reboot_nodes.append(n)
nodes.remove(n)
elif out["status"]["transition-phase"] == "VENICE_REGISTRATION_DONE":
api.Logger.info("Node already transitioned : %s" % n)
nodes.remove(n)
else:
api.Logger.info("Reboot not pending on node : %s" % n)
time.sleep(1)
num_retries = num_retries - 1
if num_retries == 0:
api.Logger.error("Reboot pending state not transitioned complete on naples")
return api.types.status.FAILURE
ret = api.RestartNodes(reboot_nodes)
if ret != api.types.status.SUCCESS:
api.Logger.error("Node restart failed")
return api.types.status.FAILURE
req = api.Trigger_CreateExecuteCommandsRequest(serial = True)
#enable_sshd = "system enable-sshd"
#copy_key = "update ssh-pub-key -f ~/.ssh/id_rsa.pub"
for n in api.GetNaplesHostnames():
#hack for now, need to set date
cmd = "date -s '{}'".format(datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"))
api.Trigger_AddNaplesCommand(req, n, cmd)
#common.AddPenctlCommand(req, n, enable_sshd)
#common.AddPenctlCommand(req, n, copy_key)
resp = api.Trigger(req)
for cmd in resp.commands:
api.PrintCommandResults(cmd)
if cmd.exit_code != 0:
result = api.types.status.FAILURE
#Give some time for naples to admit to venice after mode switch
#This is required for now as Iota is setting time manually to make TLS happy
time.sleep(30)
#Check whether naples has switch mode succesfully
req = api.Trigger_CreateExecuteCommandsRequest(serial = False)
for n in nodes:
cmd = "cat /sysconfig/config0/app-start.conf | grep hostpin"
api.Trigger_AddNaplesCommand(req, n, cmd)
resp = api.Trigger(req)
for cmd in resp.commands:
api.PrintCommandResults(cmd)
if cmd.exit_code != 0:
api.Logger.error("Mode switch failed on node : {}".format(cmd.node_name))
result = api.types.status.FAILURE
#Verify Agent in right mode.
req = api.Trigger_CreateExecuteCommandsRequest(serial = False)
for n in nodes:
cmd = "curl localhost:8888/api/system/info/"
api.Trigger_AddNaplesCommand(req, n, cmd)
# Delete the iota-emulation file created earlied for NMD. Mode change would have passed by now if it had to.
cmd = "rm -f /data/iota-emulation"
api.Trigger_AddNaplesCommand(req, n, cmd)
resp = api.Trigger(req)
for cmd in resp.commands:
api.PrintCommandResults(cmd)
if cmd.exit_code != 0:
api.Logger.error("Agent system get failed : {}".format(cmd.node_name))
result = api.types.status.FAILURE
out = None
try:
out = json.loads(cmd.stdout)
except:
api.Logger.error("Agent System get out failed {}".format(cmd.stdout))
return api.types.status.FAILURE
if out["naples-mode"] != "NETWORK_MANAGED_OOB":
api.Logger.error("Agent not in correct mode: {} {} ".format(cmd.node_name, out["naples-mode"]))
return api.types.status.FAILURE
api.Logger.info("Trying to re-enable ssh on naples nodes")
return enable_ssh.Main(None)
def Trigger(tc):
api.Logger.info("Starting CPLD Upgrade test")
tc.Nodes = api.GetNaplesHostnames()
api.Logger.info("The number of hosts {}".format(len(tc.Nodes)))
reboot_nodes = []
for node in tc.Nodes:
req = api.Trigger_CreateExecuteCommandsRequest(serial=True)
api.Trigger_AddNaplesCommand(req, node, "/nic/tools/fwupdate -l")
resp = api.Trigger(req)
for cmd in resp.commands:
req_cpldapp = api.Trigger_CreateExecuteCommandsRequest(serial=True)
try:
out = json.loads(cmd.stdout)
except:
api.Logger.error("Error using fwupdate {}".format(cmd.stdout))
return api.types.status.FAILURE
revision = out["cpld"]["bitfile"]["version"]
api.Logger.info("{} :: CPLD revision is {}".format(
cmd.node_name, revision))
if revision == "8":
api.Trigger_AddNaplesCommand(
req_cpldapp,
node,
"LD_LIBRARY_PATH=/platform/lib/ /platform/bin/cpldapp -writeflash /data/naples100_rev9.bin",
timeout=300)
elif revision == "9":
api.Trigger_AddNaplesCommand(
req_cpldapp,
node,
"LD_LIBRARY_PATH=/platform/lib/ /platform/bin/cpldapp -writeflash /data/naples100_02_rev8.bin",
timeout=300)
else:
api.Logger.error("Revision not supported continue")
continue
api.Trigger_AddNaplesCommand(req_cpldapp, node,
"touch /tmp/cpldreset")
resp_cpldapp = api.Trigger(req_cpldapp)
for cmd_cpldapp in resp_cpldapp.commands:
if cmd_cpldapp.exit_code == 0:
api.Logger.info("Cpld written successfully ")
reboot_nodes.append(node)
ret = api.RestartNodes(reboot_nodes)
if ret != api.types.status.SUCCESS:
api.Logger.error("Node restart failed")
return api.types.status.FAILURE
api.Logger.info("Hosts rebooted successfully")
for node_afterreboot in tc.Nodes:
req_afterreboot = api.Trigger_CreateExecuteCommandsRequest(serial=True)
api.Trigger_AddNaplesCommand(req_afterreboot, node_afterreboot,
"/nic/tools/fwupdate -l")
resp_afterreboot = api.Trigger(req_afterreboot)
for cmd_afterreboot in resp_afterreboot.commands:
try:
out = json.loads(cmd_afterreboot.stdout)
except:
api.Logger.error("Error using fwupdate {}".format(
cmd_afterreboot.stdout))
return api.types.status.FAILURE
api.Logger.info("{} :: CPLD revision is {}".format(
cmd_afterreboot.node_name, out["cpld"]["bitfile"]["version"]))
return api.types.status.SUCCESS
def __reboot_nodes():
nodes = api.GetWorkloadNodeHostnames()
api.RestartNodes(nodes)
return api.types.status.SUCCESS
def LoadFirmware(nodes, node_os, target_version):
if target_version != 'latest':
resp = api.DownloadAssets(release_version = target_version)
if not api.IsApiResponseOk(resp):
api.Logger.error("Failed to download assets for %s" % target_version)
return resp
manifest_file = os.path.join(api.GetTopDir(), 'images', target_version + '.json')
image_manifest = parser.JsonParse(manifest_file)
fw_images = list(filter(lambda x: x.naples_type == "capri", image_manifest.Firmwares))[0]
if fw_images is None:
api.Logger.error("Unable to load image manifest")
return api.types.status.FAILURE
api.Logger.info("Fullpath for firmware image to load: %s " % fw_images.image)
if node_os == OS_TYPE_LINUX:
fw_version, _ = GetLinuxFwDriverVersion(node)
else:
fw_version = None
fwImgFile = os.path.join(GlobalOptions.topdir, fw_images.image)
for node in nodes:
if fw_version == '1.1.1-E-15':
# Naples with Fw 1.1.1-E-15 has no OOB and IntMgmt Ip is fixed
resp = api.CopyToNaples(node, [fwImgFile], image_manifest.Version,
via_oob=False, naples_dir="data",
nic_mgmt_ip=GetNicIntMgmtIP(node))
if not api.IsApiResponseOk(resp):
api.Logger.info("Failed to copy naples_fw.tar with via_oob=True")
return api.types.status.FAILURE
for cmd in resp.commands:
if cmd.exit_code != 0:
api.Logger.error("Failed to copy %s naples_fw.tar via_oob=True" % image_manifest.Version)
return api.types.status.FAILURE
else:
resp = api.CopyToNaples(node, [fwImgFile], image_manifest.Version, via_oob=True, naples_dir="data")
if not api.IsApiResponseOk(resp):
api.Logger.info("Failed to copy naples_fw.tar with via_oob=True")
return api.types.status.FAILURE
for cmd in resp.commands:
if cmd.exit_code != 0:
api.Logger.error("Failed to copy %s naples_fw.tar via_oob=True" % image_manifest.Version)
# Try with via_oob=False
resp = api.CopyToNaples(node, [fwImgFile], image_manifest.Version, via_oob=False, naples_dir="data")
if not api.Trigger_IsSuccess(resp):
api.Logger.error("Failed to copy naples_fw.tar to target naples")
return api.types.status.FAILURE
req = api.Trigger_CreateExecuteCommandsRequest(serial = True)
api.Trigger_AddNaplesCommand(req, node, "sync")
api.Trigger_AddNaplesCommand(req, node, "/nic/tools/sysupdate.sh -p /data/naples_fw.tar", timeout=120)
resp = api.Trigger(req)
if not api.Trigger_IsSuccess(resp):
api.Logger.error("sysupdate.sh cmd failed")
return api.types.status.FAILURE
api.RestartNodes([node])
if node_os == OS_TYPE_LINUX:
return LinuxReInitMgmtIP(node)
elif node_os == OS_TYPE_ESX:
return ESXiReInitMgmtIP(node)
return api.types.status.SUCCESS
def Setup(tc):
# get node info
tc.bitw_node_name = None
tc.wl_node_name = None
# Assuming only one bitw node and one workload node
nics = store.GetTopology().GetNicsByPipeline("athena")
for nic in nics:
tc.bitw_node_name = nic.GetNodeName()
break
api.SetTestsuiteAttr("bitw_node_name", tc.bitw_node_name)
workloads = api.GetWorkloads()
if len(workloads) == 0:
api.Logger.error('No workloads available')
return api.types.status.FAILURE
tc.wl_node_name = workloads[0].node_name
api.SetTestsuiteAttr("wl_node_name", tc.wl_node_name)
host_intfs = api.GetNaplesHostInterfaces(tc.wl_node_name)
# Assuming single nic per host
if len(host_intfs) != 2:
api.Logger.error('Failed to get host interfaces')
return api.types.status.FAILURE
tc.wl = []
for wl in workloads:
tc.wl.append(wl)
api.Logger.info("wl: vlan: {}, mac: {}, ip: {}".format(
wl.uplink_vlan, wl.mac_address, wl.ip_address))
tc.intfs = []
tc.intfs.append({
'name': 'inb_mnic0',
'ip': str(tc.wl[0].ip_address),
'sub_ip': str(tc.wl[2].ip_address),
'vlan': str(tc.wl[2].uplink_vlan)
})
tc.intfs.append({
'name': 'inb_mnic1',
'ip': str(tc.wl[1].ip_address),
'sub_ip': str(tc.wl[3].ip_address),
'vlan': str(tc.wl[3].uplink_vlan)
})
api.SetTestsuiteAttr("inb_mnic_intfs", tc.intfs)
# copy device_bootstrap.json to naples
bootstrap_json_fname = api.GetTopDir(
) + '/nic/conf/athena/device_bootstrap.json'
api.CopyToNaples(tc.bitw_node_name, [bootstrap_json_fname], "")
# write and copy pensando_pre_init.sh to naples
f = open('pensando_pre_init.sh', "w")
f.write('echo "copying device.json"\n')
f.write('cp /data/device_bootstrap.json /nic/conf/device.json\n')
f.close()
api.CopyToNaples(tc.bitw_node_name, ['pensando_pre_init.sh'], "")
os.remove('pensando_pre_init.sh')
# move pensando_pre_init.sh to /sysconfig/config0/ and restart Athena Node
req = api.Trigger_CreateExecuteCommandsRequest()
cmd = "mv /pensando_pre_init.sh /sysconfig/config0/"
api.Trigger_AddNaplesCommand(req, tc.bitw_node_name, cmd)
cmd = "mv /device_bootstrap.json /data/"
api.Trigger_AddNaplesCommand(req, tc.bitw_node_name, cmd)
resp = api.Trigger(req)
cmd = resp.commands[0]
api.PrintCommandResults(cmd)
if cmd.exit_code != 0:
api.Logger.error("Bootstrap setup failed on node %s" % \
tc.bitw_node_name)
return api.types.status.FAILURE
# reboot the node
api.Logger.info("Rebooting {}".format(tc.bitw_node_name))
return api.RestartNodes([tc.bitw_node_name], 'reboot')
def Main(step):
nodes = api.GetWorkloadNodeHostnames()
api.RestartNodes(nodes)
return api.types.status.SUCCESS
def Main(step):
req = api.Trigger_CreateExecuteCommandsRequest(serial=True)
venice_ips = api.GetVeniceMgmtIpAddresses()
assert (venice_ips)
uuidMap = api.GetNaplesNodeUuidMap()
for n in api.GetNaplesHostnames():
#hack for now, need to set date
cmd = "date -s '{}'".format(
datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"))
api.Trigger_AddNaplesCommand(req, n, cmd)
api.Logger.info("Running mode switch on node : %s" % n)
mode_switch_cmd = "update naples --managed-by network --management-network oob --hostname {} --primary-mac {}".format(
__get_mgmt_nw(), n, uuidMap[n])
common.AddPenctlCommand(req, n, mode_switch_cmd)
resp = api.Trigger(req)
for cmd in resp.commands:
api.PrintCommandResults(cmd)
if cmd.exit_code != 0:
return api.types.status.FAILURE
nodes = api.GetNaplesHostnames()
num_retries = 60
while nodes:
req = api.Trigger_CreateExecuteCommandsRequest(serial=True)
for n in nodes:
#hack for now, need to set date
api.Logger.info("Checking Tranisition phase for node : %s" % n)
check_state_cmd = "show naples --json"
common.AddPenctlCommand(req, n, check_state_cmd)
resp = api.Trigger(req)
cmd = resp.commands[0]
api.PrintCommandResults(cmd)
if cmd.exit_code != 0:
return api.types.status.FAILURE
try:
out = json.loads(cmd.stdout)
except:
api.Logger.error("Penctl output not in Json format {}".format(
cmd.stdout))
return api.types.status.FAILURE
if out["status"]["transition-phase"] == "REBOOT_PENDING":
api.Logger.info("Reboot pending on node : %s" % n)
nodes.remove(n)
else:
api.Logger.info("Reboot not pending on node : %s" % n)
time.sleep(1)
num_retries = num_retries - 1
if num_retries == 0:
api.Logger.error(
"Reboot pending state not transitioned complete on naples")
return api.types.status.FAILURE
reboot_nodes = []
for n in api.GetNaplesHostnames():
reboot_nodes.append(n)
ret = api.RestartNodes(reboot_nodes)
if ret != api.types.status.SUCCESS:
api.Logger.error("Node restart failed")
return api.types.status.FAILURE
req = api.Trigger_CreateExecuteCommandsRequest(serial=True)
for n in api.GetNaplesHostnames():
#hack for now, need to set date
cmd = "date -s '{}'".format(
datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"))
api.Trigger_AddNaplesCommand(req, n, cmd)
resp = api.Trigger(req)
for cmd in resp.commands:
api.PrintCommandResults(cmd)
if cmd.exit_code != 0:
result = api.types.status.FAILURE
#Give some time for naples to admit to venice after mode switch
#This is required for now as Iota is setting time manually to make TLS happy
time.sleep(30)
#Check whether naples has switch mode succesfully
req = api.Trigger_CreateExecuteCommandsRequest(serial=False)
for n in api.GetNaplesHostnames():
cmd = "cat /sysconfig/config0/app-start.conf | grep hostpin"
api.Trigger_AddNaplesCommand(req, n, cmd)
resp = api.Trigger(req)
for cmd in resp.commands:
api.PrintCommandResults(cmd)
if cmd.exit_code != 0:
api.Logger.error("Mode switch failed on node : {}".format(
cmd.node_name))
result = api.types.status.FAILURE
#Verify Agent in right mode.
req = api.Trigger_CreateExecuteCommandsRequest(serial=False)
for n in api.GetNaplesHostnames():
cmd = "curl localhost:8888/api/system/info/"
api.Trigger_AddNaplesCommand(req, n, cmd)
resp = api.Trigger(req)
venice_urls = []
for ip in api.GetVeniceMgmtIpAddresses():
venice_urls.append(ip + ":9009")
for cmd in resp.commands:
api.PrintCommandResults(cmd)
if cmd.exit_code != 0:
api.Logger.error("Agent system get failed : {}".format(
cmd.node_name))
result = api.types.status.FAILURE
out = None
try:
out = json.loads(cmd.stdout)
except:
api.Logger.error("Agent System get out failed {}".format(
cmd.stdout))
return api.types.status.FAILURE
if out["naples-mode"] != "NETWORK_MANAGED_OOB":
api.Logger.error("Agent not in correct mode: {} {} ".format(
cmd.node_name, out["naples-mode"]))
return api.types.status.FAILURE
if set(out["controller-ips"]) != set(venice_urls):
api.Logger.error("Agent controller IPs don't match: {} {}".format(
venice_urls, out["controller-ips"]))
return api.types.status.FAILURE
return api.types.status.SUCCESS
