def Teardown(tc):
if tc.skip:
return api.types.status.SUCCESS
req = api.Trigger_CreateExecuteCommandsRequest()
for node in tc.Nodes:
api.Trigger_AddNaplesCommand(req, node, "rm -rf /data/upgrade_to_same_firmware_allowed")
resp = api.Trigger(req)
try:
for cmd_resp in resp.commands:
if cmd_resp.exit_code != 0:
api.PrintCommandResults(cmd_resp)
api.Logger.error("Teardown failed %s", cmd_resp.command)
except:
api.Logger.error("EXCEPTION occured in Naples command")
return api.types.status.FAILURE
req = api.Trigger_CreateExecuteCommandsRequest()
for n in tc.Nodes:
cmd = 'curl -k -X DELETE https://' + api.GetNicIntMgmtIP(n) + ':'+utils.GetNaplesMgmtPort()+'/api/v1/naples/rollout/'
api.Trigger_AddHostCommand(req, n, cmd, timeout=100)
tc.resp = api.Trigger(req)
for cmd in tc.resp.commands:
api.PrintCommandResults(cmd)
if cmd.exit_code != 0:
return api.types.status.FAILURE
return api.types.status.SUCCESS
def Teardown(tc):
req = api.Trigger_CreateExecuteCommandsRequest()
for node in tc.Nodes:
api.Trigger_AddNaplesCommand(
req, node, "rm -rf /update/upgrade_halt_state_machine")
api.Trigger_AddNaplesCommand(req, node,
"rm -rf /update/pcieport_upgdata")
api.Trigger_AddNaplesCommand(req, node,
"rm -rf /update/pciemgr_upgdata")
api.Trigger_AddNaplesCommand(req, node,
"rm -rf /update/pciemgr_upgrollback")
api.Trigger_AddNaplesCommand(req, node,
"rm -rf /update/nicmgr_upgstate")
api.Trigger_AddNaplesCommand(
req, node, "rm -rf /data/upgrade_to_same_firmware_allowed")
resp = api.Trigger(req)
for cmd_resp in resp.commands:
api.PrintCommandResults(cmd_resp)
if cmd_resp.exit_code != 0:
api.Logger.error("Setup failed %s", cmd_resp.command)
req = api.Trigger_CreateExecuteCommandsRequest()
for n in tc.Nodes:
cmd = 'curl -k -X DELETE https://' + api.GetNicIntMgmtIP(
n) + ':8888/api/v1/naples/rollout/'
api.Trigger_AddHostCommand(req, n, cmd)
tc.resp = api.Trigger(req)
for cmd in tc.resp.commands:
api.PrintCommandResults(cmd)
if cmd.exit_code != 0:
return api.types.status.FAILURE
return api.types.status.SUCCESS
def Verify(tc):
api.Logger.info(f"Sleeping for {tc.args.sleep} ")
time.sleep(tc.args.sleep)
if tc.resp is None:
api.Logger.error("No response for Naples Upgrade POST request")
return api.types.status.FAILURE
for cmd in tc.resp.commands:
api.PrintCommandResults(cmd)
if cmd.exit_code != 0:
return api.types.status.FAILURE
req = api.Trigger_CreateExecuteCommandsRequest()
for n in tc.Nodes:
cmd = 'curl -k https://' + api.GetNicIntMgmtIP(
n) + ':8888/api/v1/naples/rollout/'
api.Trigger_AddHostCommand(req, n, cmd)
tc.resp = api.Trigger(req)
for cmd in tc.resp.commands:
api.PrintCommandResults(cmd)
if cmd.exit_code != 0:
api.Logger.info("cmd returned failure")
return api.types.status.FAILURE
resp = json.loads(cmd.stdout)
try:
for item in resp['Status']['status']:
if not item['Op'] == 4:
api.Logger.info("opcode is bad")
return api.types.status.FAILURE
if "fail" in tc.iterators.option:
if not item['opstatus'] == 'failure':
api.Logger.info("opstatus is bad")
return api.types.status.FAILURE
if tc.iterators.option not in item['Message']:
api.Logger.info("message is bad")
return api.types.status.FAILURE
else:
if not item['opstatus'] == 'success':
api.Logger.info("opstatus is bad")
return api.types.status.FAILURE
except:
api.Logger.info("resp: ", json.dumps(resp, indent=1))
ping.TestTerminateBackgroundPing(tc, tc.pkt_size)
ping_loss_duration = ping.GetMaxPktLossDuration(tc, tc.interval)
api.Logger.info(f"Traffic dropped for {ping_loss_duration} sec")
if enable_ssh.Main(None) != api.types.status.SUCCESS:
api.Logger.error("Enabling SSH failed after upgrade")
return api.types.status.FAILURE
if upgrade_utils.VerifyUpgLog(tc.Nodes, tc.GetLogsDir()):
api.Logger.error("Failed to verify the upgrade logs")
return api.types.status.FAILURE
if verify_connectivity(tc) != api.types.status.SUCCESS:
api.Logger.error("Post trigger connectivity test failed.")
return api.types.status.FAILURE
return api.types.status.SUCCESS
def Verify(tc):
time.sleep(tc.args.sleep)
if tc.resp is None:
return api.types.status.FAILURE
for cmd in tc.resp.commands:
api.PrintCommandResults(cmd)
if cmd.exit_code != 0:
return api.types.status.FAILURE
req = api.Trigger_CreateExecuteCommandsRequest()
for n in tc.Nodes:
cmd = 'curl -k https://' + api.GetNicIntMgmtIP(
n) + ':8888/api/v1/naples/rollout/'
api.Trigger_AddHostCommand(req, n, cmd)
tc.resp = api.Trigger(req)
for cmd in tc.resp.commands:
api.PrintCommandResults(cmd)
if netagent_cfg_api.switch_profile(push_base_profile=True) != \
api.types.status.SUCCESS:
api.Logger.warn("Failed to push base profile")
return api.types.status.FAILURE
if netagent_cfg_api.PushBaseConfig(ignore_error = False) != \
api.types.status.SUCCESS:
api.Logger.info("policy push failed")
return api.types.status.FAILURE
for cmd in tc.resp.commands:
if cmd.exit_code != 0:
api.Logger.info("cmd returned failure")
return api.types.status.FAILURE
if arping.ArPing(tc) != api.types.status.SUCCESS:
api.Logger.info("arping failed on verify")
if ping.TestPing(tc, 'local_only', 'ipv4',
64) != api.types.status.SUCCESS or ping.TestPing(
tc, 'remote_only', 'ipv4',
64) != api.types.status.SUCCESS:
api.Logger.info("ping test failed")
return api.types.status.FAILURE
resp = json.loads(cmd.stdout)
try:
for item in resp['Status']['status']:
if not item['Op'] == 4:
api.Logger.info("opcode is bad")
return api.types.status.FAILURE
else:
if not item['opstatus'] == 'success':
api.Logger.info("opstatus is bad")
return api.types.status.FAILURE
except:
api.logger.info("resp : ", json.dumps(resp, indent=4))
return api.types.status.SUCCESS
def VerifyMgmtConnectivity(nodes):
'''
Verify management connectivity.
'''
if api.IsDryrun():
return api.types.status.SUCCESS
result = api.types.status.SUCCESS
req = api.Trigger_CreateExecuteCommandsRequest(serial=False)
for node in nodes:
api.Logger.info("Checking connectivity to Naples Mgmt IP: %s" %
api.GetNicIntMgmtIP(node))
api.Trigger_AddHostCommand(
req, node, 'ping -c 5 -i 0.2 {}'.format(api.GetNicIntMgmtIP(node)))
resp = api.Trigger(req)
if not resp.commands:
return api.types.status.FAILURE
for cmd in resp.commands:
api.PrintCommandResults(cmd)
if cmd.exit_code:
result = api.types.status.FAILURE
return result
def __get_agent_cfg_nodes(node_names=None, device_names=None):
agent_node_names = node_names or api.GetNaplesHostnames()
agent_cfg_nodes = []
for node_name in agent_node_names:
assert (api.IsNaplesNode(node_name))
ip = api.GetNaplesMgmtIpAddress(node_name)
if not ip:
assert (0)
if not device_names:
device_names = api.GetDeviceNames(node_name)
for device_name in device_names:
nic_ip = api.GetNicIntMgmtIP(node_name, device_name)
agent_cfg_nodes.append(cfg_api.NewCfgNode(node_name, ip, nic_ip))
return agent_cfg_nodes
def GetSecurityPolicy(workload=None, node_name=None):
if not node_name:
node_name = workload.node_name
mgmtIp = api.GetNicIntMgmtIP(node_name)
cmd = 'curl -X GET -k -H "Content-Type:application/json" https://' + mgmtIp + ':8888/api/security/policies/'
req = api.Trigger_CreateExecuteCommandsRequest()
result = api.types.status.SUCCESS
api.Trigger_AddHostCommand(req, node_name, cmd)
api.Logger.info("Running COMMAND {}".format(cmd))
resp = api.Trigger(req)
for cmd in resp.commands:
api.Logger.info("CMD = {}", cmd)
if cmd.exit_code != 0:
return None
return cmd.stdout
def GetNaplesMgmtIP(node, device=None):
return api.GetNicIntMgmtIP(node, device)
def CopyMemStatsCheckTool(nodes=None):
if not nodes:
nodes = api.GetNaplesHostnames()
for node in nodes:
# Get the memory stats check tool
wget_cmd = "wget http://www.pixelbeat.org/scripts/ps_mem.py -O {}".format(MemStatsCheckToolHostPath)
status, resp = runHostCmd(node, wget_cmd)
if status != api.types.status.SUCCESS:
api.Logger.error("Failed to get ps_mem.py tool to naples host: %s"%node)
return api.types.status.FAILURE
copy_cmd = "chmod +x {} && sshpass -p pen123 scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no " \
"{} root@{}:{}".format(MemStatsCheckToolHostPath, MemStatsCheckToolHostPath, api.GetNicIntMgmtIP(node), MemStatsCheckToolNaplesPath)
status, resp = runHostCmd(node, copy_cmd)
if status != api.types.status.SUCCESS:
api.Logger.error("Failed to copy ps_mem.py tool to naples: %s"%node)
return api.types.status.FAILURE
api.Logger.info("Copied MemStats check tool %s successfully to %s"%(MemStatsCheckToolHostPath, nodes))
return api.types.status.SUCCESS
def CheckRolloutStatus(tc):
retries = 0
start_ts = datetime.now()
result = api.types.status.FAILURE
status_found = False
while retries < 100:
api.Logger.info("------Issuing Rollout get %s retry------"%retries)
misc_utils.Sleep(2)
retries += 1
# get rollout status
req = api.Trigger_CreateExecuteCommandsRequest(serial=False)
for n in tc.Nodes:
cmd = 'curl -k https://' + api.GetNicIntMgmtIP(n) + ':'+utils.GetNaplesMgmtPort()+'/api/v1/naples/rollout/'
api.Trigger_AddHostCommand(req, n, cmd)
api.Logger.info("Sending rollout status get request: %s"%(cmd))
tc.resp = api.Trigger(req)
try:
for cmd in tc.resp.commands:
api.PrintCommandResults(cmd)
except Exception as e:
api.Logger.error(f"Exception occured in sending rollout status get.{e}")
continue
for cmd in tc.resp.commands:
if cmd.exit_code != 0:
api.Logger.info("Rollout status get request returned failure")
continue
resp = json.loads(cmd.stdout)
try:
for item in resp['Status']['status']:
status_found = True
if not item['Op'] == 4:
api.Logger.info("opcode is bad for %s"%cmd.node_name)
result = api.types.status.FAILURE
if "fail" in tc.iterators.option:
if not item['opstatus'] == 'failure':
api.Logger.info("opstatus is bad for %s"%cmd.node_name)
result = api.types.status.FAILURE
if tc.iterators.option not in item['Message']:
api.Logger.info("message is bad")
result = api.types.status.FAILURE
else:
if not item['opstatus'] == 'success':
api.Logger.info("opstatus(%s) is bad for %s"%(item['opstatus'], cmd.node_name))
result = api.types.status.FAILURE
else:
api.Logger.info("Rollout status is SUCCESS for %s"%cmd.node_name)
result = api.types.status.SUCCESS
except Exception as e:
api.Logger.error("resp: ", json.dumps(resp, indent=1))
#api.Logger.error(f"Exception occured in parsing response: {e}")
result = api.types.status.FAILURE
continue
if status_found:
break
end_ts = datetime.now()
# find time elapsed in retrieving rollout status and adjust the wait time for traffic test.
timedelta = end_ts - start_ts
time_elapsed = timedelta.days * 24 * 3600 + timedelta.seconds
if time_elapsed < 100:
time_elapsed = 100
misc_utils.Sleep(time_elapsed)
tc.sleep = (tc.sleep - time_elapsed) if (tc.sleep > time_elapsed) else 10
return result
def Trigger(tc):
if tc.skip:
return api.types.status.SUCCESS
req = api.Trigger_CreateExecuteCommandsRequest(serial=False)
for n in tc.Nodes:
cmd = 'curl -k -d \'{"kind": "SmartNICRollout","meta": {"name": "test disruptive upgrade","tenant": "tenant-foo"},"spec": {"ops": [{"op": 4,"version": "0.1"}]}}\' -X POST -H "Content-Type:application/json" ' + 'https://' + api.GetNicIntMgmtIP(n) + ':'+utils.GetNaplesMgmtPort()+'/api/v1/naples/rollout/'
api.Logger.info("Sending rollout request cmd : %s"%cmd)
api.Trigger_AddHostCommand(req, n, cmd, timeout=30)
tc.resp = api.Trigger(req)
api.Logger.info("Rollout request processing complete")
return api.types.status.SUCCESS
def Trigger(tc):
req = api.Trigger_CreateExecuteCommandsRequest()
for n in tc.Nodes:
cmd = 'curl -k -d \'{"kind": "SmartNICRollout","meta": {"name": "test disruptive upgrade","tenant": "tenant-foo"},"spec": {"ops": [{"op": 4,"version": "0.1"}]}}\' -X POST -H "Content-Type:application/json" ' + 'https://' + api.GetNicIntMgmtIP(
n) + ':8888/api/v1/naples/rollout/'
api.Trigger_AddHostCommand(req, n, cmd)
tc.resp = api.Trigger(req)
api.Logger.info("started rollout")
time.sleep(1)
api.Logger.info("restarting nodes")
ret = api.RestartNodes(tc.Nodes)
return api.types.status.SUCCESS
def Trigger(tc):
api.Logger.info("starting ping in background ..")
ping.TestPing(tc,
"all",
'ipv4',
tc.pkt_size,
interval=tc.interval,
count=tc.count,
background=True,
hping3=True)
req = api.Trigger_CreateExecuteCommandsRequest()
for n in tc.Nodes:
cmd = 'curl -k -d \'{"kind": "SmartNICRollout","meta": {"name": "test disruptive upgrade","tenant": "tenant-foo"},"spec": {"ops": [{"op": 4,"version": "0.1"}]}}\' -X POST -H "Content-Type:application/json" ' + 'https://' + api.GetNicIntMgmtIP(
n) + ':8888/api/v1/naples/rollout/'
api.Trigger_AddHostCommand(req, n, cmd)
tc.resp = api.Trigger(req)
return api.types.status.SUCCESS