def Verify(tc):
api.Logger.info(f"Sleeping for {tc.args.sleep} ")
time.sleep(tc.args.sleep)
if tc.resp is None:
api.Logger.error("No response for Naples Upgrade POST request")
return api.types.status.FAILURE
for cmd in tc.resp.commands:
api.PrintCommandResults(cmd)
if cmd.exit_code != 0:
return api.types.status.FAILURE
req = api.Trigger_CreateExecuteCommandsRequest()
for n in tc.Nodes:
cmd = 'curl -k https://' + api.GetNicIntMgmtIP(
n) + ':8888/api/v1/naples/rollout/'
api.Trigger_AddHostCommand(req, n, cmd)
tc.resp = api.Trigger(req)
for cmd in tc.resp.commands:
api.PrintCommandResults(cmd)
if cmd.exit_code != 0:
api.Logger.info("cmd returned failure")
return api.types.status.FAILURE
resp = json.loads(cmd.stdout)
try:
for item in resp['Status']['status']:
if not item['Op'] == 4:
api.Logger.info("opcode is bad")
return api.types.status.FAILURE
if "fail" in tc.iterators.option:
if not item['opstatus'] == 'failure':
api.Logger.info("opstatus is bad")
return api.types.status.FAILURE
if tc.iterators.option not in item['Message']:
api.Logger.info("message is bad")
return api.types.status.FAILURE
else:
if not item['opstatus'] == 'success':
api.Logger.info("opstatus is bad")
return api.types.status.FAILURE
except:
api.Logger.info("resp: ", json.dumps(resp, indent=1))
ping.TestTerminateBackgroundPing(tc, tc.pkt_size)
ping_loss_duration = ping.GetMaxPktLossDuration(tc, tc.interval)
api.Logger.info(f"Traffic dropped for {ping_loss_duration} sec")
if enable_ssh.Main(None) != api.types.status.SUCCESS:
api.Logger.error("Enabling SSH failed after upgrade")
return api.types.status.FAILURE
if upgrade_utils.VerifyUpgLog(tc.Nodes, tc.GetLogsDir()):
api.Logger.error("Failed to verify the upgrade logs")
return api.types.status.FAILURE
if verify_connectivity(tc) != api.types.status.SUCCESS:
api.Logger.error("Post trigger connectivity test failed.")
return api.types.status.FAILURE
return api.types.status.SUCCESS
def ping_traffic_stop_and_verify(tc):
if not tc.background_ping:
return api.types.status.SUCCESS
if ping.TestTerminateBackgroundPing(tc, tc.pktsize,\
pktlossverif=tc.pktlossverif) != api.types.status.SUCCESS:
api.Logger.error("Failed in Ping background command termination.")
return api.types.status.FAILURE
# calculate max packet loss duration for background ping
pkt_loss_duration = ping.GetMaxPktLossDuration(tc, interval=tc.interval)
return check_max_pktloss_limit(tc, pkt_loss_duration)
def __verify_background_ping(tc):
if not GlobalOptions.dryrun and tc.bg_cmd_resp:
if ping.TestTerminateBackgroundPing(
tc, pktsize=128) != api.types.status.SUCCESS:
api.Logger.error(f"Failed in Ping background command termination.")
result = api.types.status.FAILURE
# calculate max packet loss duration for background ping
api.Logger.info(f"Checking pkt loss duration after move")
pkt_loss_duration = ping.GetMaxPktLossDuration(tc, interval=0.2)
if pkt_loss_duration != 0:
api.Logger.error(f"Packet Loss duration during flow move of is"
f" {pkt_loss_duration} sec")
# Skip packet loss check for intra subnet cases
if tc.mv_ctx['inter'] and pkt_loss_duration > 5:
api.Logger.error(f"Exceeded allowed Loss Duration " f"5 secs")
result = api.types.status.FAILURE
return api.types.status.SUCCESS
def Verify(tc):
result = api.types.status.SUCCESS
if api.IsDryrun():
# no upgrade done in case of dryrun
return result
upg_switchover_time = 70
# wait for upgrade to complete. status can be found from the presence of /update/pds_upg_status.txt
api.Logger.info(
f"Sleep for {upg_switchover_time} secs before checking for Upgrade status"
)
misc_utils.Sleep(upg_switchover_time)
if checkUpgradeStatusViaConsole(tc) != api.types.status.SUCCESS:
api.Logger.error(
"Failed in validation of Upgrade Manager completion status via Console"
)
result = api.types.status.FAILURE
if not naples_utils.EnableReachability(tc.nodes):
api.Logger.error(
f"Failed to reach naples {tc.nodes} post upgrade switchover")
result = api.types.status.FAILURE
# verify mgmt connectivity
if VerifyMgmtConnectivity(tc) != api.types.status.SUCCESS:
api.Logger.error("Failed in Mgmt Connectivity Check after Upgrade .")
result = api.types.status.FAILURE
if result != api.types.status.SUCCESS:
api.Logger.info("DUMP Upgrade Manager Logs")
# Failure could be due to upgrade failure before/after switchover or
# management connectivity failure. Hence dump the upgrade_mgr.log
# via console for debug purpose.
api.Logger.SetSkipLogPrefix(True)
for node in tc.nodes:
(resp,
exit_code) = api.RunNaplesConsoleCmd(node,
"cat /obfl/upgrademgr.log",
True)
if exit_code != 0:
api.Logger.info("Failed to dump /obfl/upgrademgr.log from "
"node: %s, exit_code:%s " % (node, exit_code))
else:
api.Logger.info("Dump /obfl/upgrademgr.log from "
"node: %s, exit_code:%s " % (node, exit_code))
lines = resp.split('\r\n')
for line in lines:
api.Logger.info(line.strip())
api.Logger.SetSkipLogPrefix(False)
return api.types.status.FAILURE
# push configs after upgrade
UpdateConfigAfterUpgrade(tc)
# verify PDS instances
if check_pds_instance(tc) != api.types.status.SUCCESS:
api.Logger.error("Failed in check_pds_instances")
result = api.types.status.FAILURE
if check_pds_agent_debug_data(tc) != api.types.status.SUCCESS:
api.Logger.error("Failed in check_pds_agent_debug_data")
result = api.types.status.FAILURE
# TODO: verify BGP Underlay (REMOVE WHEN PING API IS UPDATED)
if bgp_utils.check_underlay_bgp_peer_connectivity(
sleep_time=15, timeout_val=120) != api.types.status.SUCCESS:
api.Logger.error("Failed in underlay connectivity check")
#return api.types.status.FAILURE
# verify connectivity
if VerifyConnectivity(tc) != api.types.status.SUCCESS:
api.Logger.error("Failed in Connectivity Check after Upgrade .")
if not skip_connectivity_failure:
result = api.types.status.FAILURE
if tc.upgrade_mode:
tc.sleep = 100
# If rollout status is failure, then no need to wait for traffic test
if result == api.types.status.SUCCESS:
api.Logger.info("Sleep for %s secs for traffic test to complete" %
tc.sleep)
misc_utils.Sleep(tc.sleep)
pkt_loss_duration = 0
# terminate background traffic and calculate packet loss duration
if tc.background:
if ping.TestTerminateBackgroundPing(tc, tc.pktsize,\
pktlossverif=tc.pktlossverif) != api.types.status.SUCCESS:
api.Logger.error(
"Failed in Ping background command termination.")
result = api.types.status.FAILURE
# calculate max packet loss duration for background ping
pkt_loss_duration = ping.GetMaxPktLossDuration(
tc, interval=tc.interval)
if pkt_loss_duration != 0:
indent = "-" * 10
if tc.pktlossverif:
result = api.types.status.FAILURE
api.Logger.error(
f"{indent} Packet Loss duration during UPGRADE of {tc.nodes} is {pkt_loss_duration} secs {indent}"
)
if tc.allowed_down_time and (pkt_loss_duration >
tc.allowed_down_time):
api.Logger.error(
f"{indent} Exceeded allowed Loss Duration {tc.allowed_down_time} secs {indent}"
)
# Failing test based on longer traffic loss duration is commented for now.
# enable below line when needed.
#result = api.types.status.FAILURE
else:
api.Logger.info("No Packet Loss Found during UPGRADE Test")
if upgrade_utils.VerifyUpgLog(tc.nodes, tc.GetLogsDir()):
api.Logger.error("Failed to verify the upgrademgr logs...")
if result == api.types.status.SUCCESS:
api.Logger.info(f"Upgrade: Completed Successfully for {tc.nodes}")
else:
api.Logger.info(f"Upgrade: Failed for {tc.nodes}")
return result
def Verify(tc):
if tc.skip:
return api.types.status.SUCCESS
if tc.pktloss_verify:
if tc.background and tc.bg_cmd_resp is None:
api.Logger.error("Failed in background Ping cmd trigger")
return api.types.status.FAILURE
if tc.resp is None:
api.Logger.error("Received empty response for config request")
return api.types.status.FAILURE
else:
for cmd in tc.resp.commands:
api.PrintCommandResults(cmd)
if cmd.exit_code != 0:
api.Logger.error("Rollout request failed")
return api.types.status.FAILURE
# wait for upgrade to complete. status can be found from the presence of /update/pds_upg_status.txt
api.Logger.info("Sleep for 70 secs before checking for /update/pds_upg_status.txt")
misc_utils.Sleep(70)
status_in_progress = True
while status_in_progress:
misc_utils.Sleep(1)
req = api.Trigger_CreateExecuteCommandsRequest(serial=False)
for node in tc.Nodes:
api.Trigger_AddNaplesCommand(req, node, "grep -v in-progress /update/pds_upg_status.txt", timeout=2)
api.Logger.info("Checking for status not in-progress in file /update/pds_upg_status.txt")
resp = api.Trigger(req)
status_in_progress = False
for cmd_resp in resp.commands:
#api.PrintCommandResults(cmd_resp)
if cmd_resp.exit_code != 0:
status_in_progress = True
#api.Logger.info("File /update/pds_upg_status.txt not found")
else:
api.Logger.info("Status other than in-progress found in /update/pds_upg_status.txt")
# # push interface config updates after upgrade completes
# UpdateConfigAfterUpgrade(tc)
for i in range(10):
api.Logger.info("Sending ARPing, retry count %s"%i)
# Send Grat Arp for learning
arping.SendGratArp(tc.wloads)
misc_utils.Sleep(1)
result = CheckRolloutStatus(tc)
# ensure connectivity after upgrade
if VerifyConnectivity(tc) != api.types.status.SUCCESS:
api.Logger.error("Failed in Connectivity Check Post Upgrade.")
result = api.types.status.FAILURE
error_str = None
if tc.pktloss_verify:
# If rollout status is failure, then no need to wait for traffic test
if result == api.types.status.SUCCESS:
api.Logger.info("Sleep for %s secs for traffic test to complete"%tc.sleep)
misc_utils.Sleep(tc.sleep)
pkt_loss_duration = 0
# terminate background traffic and calculate packet loss duration
if tc.background:
if ping.TestTerminateBackgroundPing(tc, tc.pktsize,\
pktlossverif=tc.pktlossverif) != api.types.status.SUCCESS:
api.Logger.error("Failed in Ping background command termination.")
result = api.types.status.FAILURE
# calculate max packet loss duration for background ping
pkt_loss_duration = ping.GetMaxPktLossDuration(tc, interval=tc.interval)
if pkt_loss_duration != 0:
indent = "-" * 10
if tc.pktlossverif:
result = api.types.status.FAILURE
api.Logger.error(f"{indent} Packet Loss duration during UPGRADE of {tc.Nodes} is {pkt_loss_duration} secs {indent}")
if tc.allowed_down_time and (pkt_loss_duration > tc.allowed_down_time):
api.Logger.error(f"{indent} Exceeded allowed Loss Duration {tc.allowed_down_time} secs {indent}")
result = api.types.status.FAILURE
else:
api.Logger.info("No Packet Loss Found during UPGRADE Test")
if upgrade_utils.VerifyUpgLog(tc.Nodes, tc.GetLogsDir()):
api.Logger.error("Failed to verify the upgrade logs")
result = api.types.status.FAILURE
return result