def Trigger(tc):
tc.pre_data = {}
tc.post_data = {}
try:
for _iter in range(tc.iterators.count):
api.Logger.info("Iter: %d" % _iter)
api.Logger.info("Issuing APC power cycle")
# Getting port status before reboot
tc.pre_data[_iter] = get_nw_ports_info(tc.RF)
ret = api.RestartNodes([tc.node_name], 'apc')
if ret != api.types.status.SUCCESS:
api.Logger.info("APC power cycle failed")
return api.types.status.FAILURE
api.Logger.info("APC power cycle done")
tc.test_node.WaitForHost()
api.Logger.info("APC power cycle successful")
if enable_ssh.Main(None) != api.types.status.SUCCESS:
api.Logger.info("Enabling SSH failed after reboot")
return api.types.status.FAILURE
api.Logger.info("Enabled Naples SSH after power cycle")
# Getting port status after reboot
tc.post_data[_iter] = get_nw_ports_info(tc.RF)
except Exception as e:
api.Logger.error(str(e))
return api.types.status.ERROR
return api.types.status.SUCCESS
def Trigger(tc):
try:
for _iter in range(tc.iterators.count):
api.Logger.info("Iter: %d" % _iter)
api.Logger.info("Issuing APC power cycle")
ret = api.RestartNodes([tc.node_name], 'apc')
if ret != api.types.status.SUCCESS:
api.Logger.info("APC power cycle failed")
return api.types.status.FAILURE
api.Logger.info("APC power cycle done")
tc.test_node.WaitForHost()
ret = check_ncsi_conn(tc.cimc_info)
if ret != api.types.status.SUCCESS:
api.Logger.error("Unable to connect ot ILO in NCSI mode")
return api.types.status.FAILURE
api.Logger.info("APC power cycle successfull")
if enable_ssh.Main(None) != api.types.status.SUCCESS:
api.Logger.info("Enabling SSH failed after reboot")
return api.types.status.FAILURE
api.Logger.info("Enabled Naples SSH after power cycle")
except Exception as e:
api.Logger.error(str(e))
return api.types.status.FAILURE
return api.types.status.SUCCESS
def Verify(tc):
api.Logger.info(f"Sleeping for {tc.args.sleep} ")
time.sleep(tc.args.sleep)
if tc.resp is None:
api.Logger.error("No response for Naples Upgrade POST request")
return api.types.status.FAILURE
for cmd in tc.resp.commands:
api.PrintCommandResults(cmd)
if cmd.exit_code != 0:
return api.types.status.FAILURE
req = api.Trigger_CreateExecuteCommandsRequest()
for n in tc.Nodes:
cmd = 'curl -k https://' + api.GetNicIntMgmtIP(
n) + ':8888/api/v1/naples/rollout/'
api.Trigger_AddHostCommand(req, n, cmd)
tc.resp = api.Trigger(req)
for cmd in tc.resp.commands:
api.PrintCommandResults(cmd)
if cmd.exit_code != 0:
api.Logger.info("cmd returned failure")
return api.types.status.FAILURE
resp = json.loads(cmd.stdout)
try:
for item in resp['Status']['status']:
if not item['Op'] == 4:
api.Logger.info("opcode is bad")
return api.types.status.FAILURE
if "fail" in tc.iterators.option:
if not item['opstatus'] == 'failure':
api.Logger.info("opstatus is bad")
return api.types.status.FAILURE
if tc.iterators.option not in item['Message']:
api.Logger.info("message is bad")
return api.types.status.FAILURE
else:
if not item['opstatus'] == 'success':
api.Logger.info("opstatus is bad")
return api.types.status.FAILURE
except:
api.Logger.info("resp: ", json.dumps(resp, indent=1))
ping.TestTerminateBackgroundPing(tc, tc.pkt_size)
ping_loss_duration = ping.GetMaxPktLossDuration(tc, tc.interval)
api.Logger.info(f"Traffic dropped for {ping_loss_duration} sec")
if enable_ssh.Main(None) != api.types.status.SUCCESS:
api.Logger.error("Enabling SSH failed after upgrade")
return api.types.status.FAILURE
if upgrade_utils.VerifyUpgLog(tc.Nodes, tc.GetLogsDir()):
api.Logger.error("Failed to verify the upgrade logs")
return api.types.status.FAILURE
if verify_connectivity(tc) != api.types.status.SUCCESS:
api.Logger.error("Post trigger connectivity test failed.")
return api.types.status.FAILURE
return api.types.status.SUCCESS
def Trigger(tc):
naples_nodes = []
#for every node in the setup
for node in tc.nodes:
if api.IsNaplesNode(node):
naples_nodes.append(node)
api.Logger.info(f"Found Naples Node: [{node}]")
if len(naples_nodes) == 0:
api.Logger.error("Failed to find a Naples Node!")
return api.types.status.FAILURE
for reboot in range(tc.args.reboots):
# Reboot Node.
# Reboot method (APC, IPMI, OS Reboot) is passed as a testcase parameter
for node in naples_nodes:
api.Logger.info(
f"==== Reboot Loop # {reboot} on {node}. Reboot method: {tc.iterators.reboot_method} ===="
)
if api.RestartNodes(
[node],
tc.iterators.reboot_method) != api.types.status.SUCCESS:
return api.types.status.FAILURE
#raise OfflineTestbedException
# Enable SSH, some checks require Naples access
if enable_ssh.Main(None) != api.types.status.SUCCESS:
api.Logger.error("Enabling SSH failed")
return api.types.status.FAILURE
# there is not a real "PCI" in IOTA for Windows.
if tc.os != host.OS_TYPE_WINDOWS:
api.Logger.info(f"Verifying PCI on [{node}]: ")
if verify_pci.verify_errors_lspci(
node, tc.os) != api.types.status.SUCCESS:
api.Logger.error(f"PCIe Failure detected on {node}")
return api.types.status.FAILURE
#raise OfflineTestbedException
if checkLinks(tc, node) is api.types.status.FAILURE:
api.Logger.error("Error verifying uplink interfaces")
return api.types.status.FAILURE
#raise OfflineTestbedException
# don't run this while we figure out how to do this in ESX
if tc.os != host.OS_TYPE_ESX:
# Load the ionic driver except windows
if tc.os != host.OS_TYPE_WINDOWS:
if host.LoadDriver(tc.os,
node) is api.types.status.FAILURE:
api.Logger.info("ionic already loaded")
# Make sure ionic driver attached to Uplink ports.
if checkDrv(node) is api.types.status.FAILURE:
api.Logger.error("No ionic uplink interfaces detected")
return api.types.status.FAILURE
#raise OfflineTestbedException
return api.types.status.SUCCESS
def Verify(tc):
if install_penctl.Main(None) != api.types.status.SUCCESS:
api.Logger.info("Installing PenCtl failed after pxe install")
return api.types.status.FAILURE
if enable_ssh.Main(None) != api.types.status.SUCCESS:
api.Logger.info("Enabling SSH failed after pxe install")
return api.types.status.FAILURE
return api.types.status.SUCCESS
def Verify(tc):
if enable_ssh.Main(None) != api.types.status.SUCCESS:
api.Logger.info("Enabling SSH failed after reboot")
tc.resp = api.types.status.FAILURE
return tc.resp
def Verify(tc):
api.Logger.info("Enable SSH after the reboot tests")
if enable_ssh.Main(None) != api.types.status.SUCCESS:
api.Logger.error("Enabling SSH failed after reboot")
return api.types.status.FAILURE
return api.types.status.SUCCESS
def Main(step):
if GlobalOptions.skip_setup:
#No mode switch required for skeip setup
return api.types.status.SUCCESS
req = api.Trigger_CreateExecuteCommandsRequest(serial = True)
uuidMap = api.GetNaplesNodeUuidMap()
nodes = api.GetNaplesHostnames()
for n in nodes:
# Touch a file to indicate to NMD that the current mode is emulation
cmd = "touch /data/iota-emulation"
api.Trigger_AddNaplesCommand(req, n, cmd)
# Make sure console is enabled
CreateConfigConsoleNoAuth()
api.CopyToNaples(n, [NAPLES_CONFIG_SPEC_LOCAL], "")
cmd = "mv /system-config.json /sysconfig/config0/system-config.json"
api.Trigger_AddNaplesCommand(req, n, cmd)
if common.PenctlGetModeStatus(n) != "NETWORK" or common.PenctlGetTransitionPhaseStatus(n) != "VENICE_REGISTRATION_DONE":
api.Logger.info("Host [{}] is in HOST mode. Initiating mode change.".format(n))
ret = common.SetNaplesModeOOB_Static(n, "1.1.1.1", "1.1.1.2/24")
if ret == None:
return api.types.status.FAILURE
#hack for now, need to set date
cmd = "date -s '{}'".format(datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"))
api.Trigger_AddNaplesCommand(req, n, cmd)
resp = api.Trigger(req)
for cmd in resp.commands:
api.PrintCommandResults(cmd)
if cmd.exit_code != 0:
return api.types.status.FAILURE
num_retries = 60
reboot_nodes = []
while nodes:
req = api.Trigger_CreateExecuteCommandsRequest(serial = True)
for n in nodes:
#hack for now, need to set date
api.Logger.info("Checking Transition phase for node : %s" % n)
check_state_cmd = "show naples --json"
time.sleep(30)
common.AddPenctlCommand(req, n, check_state_cmd)
api.Trigger_AddNaplesCommand(req, n, "touch /data/no_watchdog")
resp = api.Trigger(req)
cmd = resp.commands[0]
api.PrintCommandResults(cmd)
if cmd.exit_code != 0:
return api.types.status.FAILURE
try:
out = json.loads(cmd.stdout)
except:
api.Logger.error("Penctl output not in Json format {}".format(cmd.stdout))
return api.types.status.FAILURE
if not hal_show_utils.IsNaplesForwardingModeClassic(n):
api.Logger.info("Dataplane already in HOSTPIN mode. Skipping node [{}] for reboot.".format(n))
reboot_nodes.append(n)
nodes.remove(n)
elif out["status"]["transition-phase"] == "VENICE_UNREACHABLE":
api.Logger.info("Reboot pending on node : %s" % n)
reboot_nodes.append(n)
nodes.remove(n)
elif out["status"]["transition-phase"] == "VENICE_REGISTRATION_DONE":
api.Logger.info("Node already transitioned : %s" % n)
nodes.remove(n)
else:
api.Logger.info("Reboot not pending on node : %s" % n)
time.sleep(1)
num_retries = num_retries - 1
if num_retries == 0:
api.Logger.error("Reboot pending state not transitioned complete on naples")
return api.types.status.FAILURE
ret = api.RestartNodes(reboot_nodes)
if ret != api.types.status.SUCCESS:
api.Logger.error("Node restart failed")
return api.types.status.FAILURE
req = api.Trigger_CreateExecuteCommandsRequest(serial = True)
#enable_sshd = "system enable-sshd"
#copy_key = "update ssh-pub-key -f ~/.ssh/id_rsa.pub"
for n in api.GetNaplesHostnames():
#hack for now, need to set date
cmd = "date -s '{}'".format(datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"))
api.Trigger_AddNaplesCommand(req, n, cmd)
#common.AddPenctlCommand(req, n, enable_sshd)
#common.AddPenctlCommand(req, n, copy_key)
resp = api.Trigger(req)
for cmd in resp.commands:
api.PrintCommandResults(cmd)
if cmd.exit_code != 0:
result = api.types.status.FAILURE
#Give some time for naples to admit to venice after mode switch
#This is required for now as Iota is setting time manually to make TLS happy
time.sleep(30)
#Check whether naples has switch mode succesfully
req = api.Trigger_CreateExecuteCommandsRequest(serial = False)
for n in nodes:
cmd = "cat /sysconfig/config0/app-start.conf | grep hostpin"
api.Trigger_AddNaplesCommand(req, n, cmd)
resp = api.Trigger(req)
for cmd in resp.commands:
api.PrintCommandResults(cmd)
if cmd.exit_code != 0:
api.Logger.error("Mode switch failed on node : {}".format(cmd.node_name))
result = api.types.status.FAILURE
#Verify Agent in right mode.
req = api.Trigger_CreateExecuteCommandsRequest(serial = False)
for n in nodes:
cmd = "curl localhost:8888/api/system/info/"
api.Trigger_AddNaplesCommand(req, n, cmd)
# Delete the iota-emulation file created earlied for NMD. Mode change would have passed by now if it had to.
cmd = "rm -f /data/iota-emulation"
api.Trigger_AddNaplesCommand(req, n, cmd)
resp = api.Trigger(req)
for cmd in resp.commands:
api.PrintCommandResults(cmd)
if cmd.exit_code != 0:
api.Logger.error("Agent system get failed : {}".format(cmd.node_name))
result = api.types.status.FAILURE
out = None
try:
out = json.loads(cmd.stdout)
except:
api.Logger.error("Agent System get out failed {}".format(cmd.stdout))
return api.types.status.FAILURE
if out["naples-mode"] != "NETWORK_MANAGED_OOB":
api.Logger.error("Agent not in correct mode: {} {} ".format(cmd.node_name, out["naples-mode"]))
return api.types.status.FAILURE
api.Logger.info("Trying to re-enable ssh on naples nodes")
return enable_ssh.Main(None)