def Trigger(tc):
i = 0
workloads = api.GetWorkloads(tc.node)
while i < tc.iterations:
api.Logger.info(f"Starting iteration {i}")
api.Logger.info(f"Clearing learn-db")
if not learn_utils.ClearLearnData(tc.node):
api.Logger.error("Failed to clear learn info at iteration %d" % i)
return api.types.status.FAILURE
api.Logger.info(f"Re-learning all endpoints")
if not arp_utils.SendGratArp(workloads):
api.Logger.error("Failed to send arp probes at iteration %d" % i)
return api.types.status.FAILURE
api.Logger.info(f"Sending some ARP replies")
for j in range(2):
arp_utils.SendArpReply(workloads)
misc_utils.Sleep(2)
learn_utils.DumpLearnData()
if not learn_utils.ValidateLearnInfo(tc.node):
api.Logger.error("Failed to validate learn data at iteration %d" %
i)
return api.types.status.FAILURE
i += 1
misc_utils.Sleep(tc.interval)
return api.types.status.SUCCESS
def __publish_workloads(workloads=[]):
workloads = workloads if workloads else api.GetWorkloads()
wl_list = list(filter(lambda x: x.vnic.IsOriginDiscovered() and not x.vnic.DhcpEnabled, workloads))
if not arp_utils.SendGratArp(wl_list):
return api.types.status.FAILURE
return api.types.status.SUCCESS
def Teardown(tc):
if tc.is_config_updated:
rs = config_api.RestoreObjects('Update', tc.selected_objs)
if rs is False:
api.Logger.error(
f"Teardown failed to restore objs from Update operation: {rs}")
if tc.is_config_deleted:
rs = config_api.RestoreObjects('Delete', tc.selected_objs)
if rs is False:
api.Logger.error(
f"Teardown failed to restore objs from Delete operation: {rs}")
h2h_test = getattr(tc.args, "h2h_test", False)
if h2h_test:
if not arp_utils.SendGratArp(api.GetWorkloads()):
api.Logger.error(
f"Teardown failed to send ARP to all workloads")
else:
api.Logger.info("Running overlay connectivity test")
cmd_cookies, resp = conn_utils.TriggerConnectivityTestAll(
proto="icmp", sec_ip_test_type="random")
ret = conn_utils.VerifyConnectivityTest(
"icmp", cmd_cookies, resp)
if ret != api.types.status.SUCCESS:
api.Logger.error("Connectivity verification failed.")
return api.types.status.SUCCESS
def Setup(tc):
tc.skip = False
tc.sleep = getattr(tc.args, "sleep", 200)
tc.allowed_down_time = getattr(tc.args, "allowed_down_time", 0)
tc.pktloss_verify = getattr(tc.args, "pktloss_verify", False)
tc.node_selection = tc.iterators.selection
if tc.node_selection not in ["any", "all"]:
api.Logger.error("Incorrect Node selection option {} specified. Use 'any' or 'all'".format(tc.node_selection))
tc.skip = True
return api.types.status.FAILURE
tc.Nodes = api.GetNaplesHostnames()
if tc.node_selection == "any":
tc.Nodes = [random.choice(tc.Nodes)]
req = api.Trigger_CreateExecuteCommandsRequest()
for node in tc.Nodes:
api.Trigger_AddNaplesCommand(req, node, "touch /data/upgrade_to_same_firmware_allowed")
api.Trigger_AddNaplesCommand(req, node, "rm -rf /data/techsupport/DSC_TechSupport_*")
api.Trigger_AddNaplesCommand(req, node, "rm -rf /update/pds_upg_status.txt")
api.Trigger_AddNaplesCommand(req, node, "cp /update/{} /update/{}".format(UPGRADE_NAPLES_PKG, NAPLES_PKG))
resp = api.Trigger(req)
for cmd_resp in resp.commands:
api.PrintCommandResults(cmd_resp)
if cmd_resp.exit_code != 0:
api.Logger.error("Setup failed %s", cmd_resp.command)
tc.skip = True
return api.types.status.FAILURE
if upgrade_utils.ResetUpgLog(tc.Nodes) != api.types.status.SUCCESS:
api.Logger.error("Failed in Reseting Upgrade Log files.")
return api.types.status.FAILURE
# choose workloads for connectivity/traffic test
result = ChooseWorkLoads(tc)
if result != api.types.status.SUCCESS or tc.skip:
api.Logger.error("Failed to Choose Workloads.")
return result
# Send Grat Arp for learning
arping.SendGratArp(tc.wloads)
# verify connectivity
result = VerifyConnectivity(tc)
if result != api.types.status.SUCCESS:
api.Logger.error("Failed in Connectivity Check during Setup.")
tc.skip = True
return result
# setup packet test based on pktloss verify argument
result = PacketTestSetup(tc)
if result != api.types.status.SUCCESS or tc.skip:
api.Logger.error("Failed in Packet Test setup.")
return result
return api.types.status.SUCCESS
def __modify_workload_interface(tc):
wl_api.DeleteWorkload(tc.wload)
if api.IsDryrun():
tc.wload.parent_interface = 'dryrun'
else:
tc.wload.parent_interface = intf_client.FindHostInterface(
tc.subnet.Node, tc.subnet.HostIfIdx[0]).GetInterfaceName()
tc.wload.interface = tc.wload.parent_interface
store.SetWorkloadRunning(tc.wload.workload_name)
wl_api.ReAddWorkload(tc.wload)
add_routes.AddRoutes(tc.learn_mac_obj)
arp.SendGratArp([tc.wload])
return
def MoveEpMACEntry(workload, target_subnet, ep_mac_addr, ep_ip_prefixes):
"""
Moves workload to target subnet.
"""
vnic = workload.vnic
old_subnet = vnic.SUBNET
# Update vnic information from target subnet
vnic_client.ChangeMacAddr(
vnic, objects.TemplateFieldObject(f"macaddr/{ep_mac_addr}"))
# Move vnic to target subnet
vnic_client.ChangeSubnet(vnic, target_subnet)
# Move around mapping entries
__mapping_entry_handler(vnic, ep_ip_prefixes)
# Delete the workload.
wl_api.DeleteWorkload(workload)
# Update workload object with new information
workload.node_name = target_subnet.Node
workload.ip_prefix = ep_ip_prefixes[0]
workload.ip_address = __ip_from_prefix(ep_ip_prefixes[0])
workload.sec_ip_prefixes = ep_ip_prefixes[1:]
workload.sec_ip_addresses = [
__ip_from_prefix(prefix) for prefix in ep_ip_prefixes[1:]
]
if api.GlobalOptions.dryrun:
workload.parent_interface = 'dryrun'
else:
workload.parent_interface = intf_client.FindHostInterface(
target_subnet.Node, target_subnet.HostIfIdx[0]).GetInterfaceName()
workload.interface = workload.parent_interface
workload.mac_address = vnic.MACAddr.get()
workload.vlan = vnic.VlanId()
# Set workload in running state, otherwise command trigger on workload will fail.
store.SetWorkloadRunning(workload.workload_name)
# Re add workload
wl_api.ReAddWorkload(workload)
# Add routes
__add_routes(workload)
# Send Grat ARP
arp.SendGratArp([workload])
return api.types.status.SUCCESS
def Teardown(tc):
ret = learn_utils.SetWorkloadIntfOperState(tc.workload, 'up')
if not ret:
api.Logger.error("Failed to bringup interface for workload %s%s" %
(tc.learn_mac_obj.GID(), tc.node))
return api.types.status.FAILURE
arp_utils.SendGratArp([tc.workload])
learn_utils.DumpLearnMAC(tc.node, tc.learn_mac_obj)
misc_utils.Sleep(
10) # to let remote mappings for this VNIC, sync in other nodes
if not learn_utils.ValidateLearnInfo():
api.Logger.error("Learn validation failed")
return api.types.status.FAILURE
api.Logger.verbose("Aged out Endpoints are learnt again")
return api.types.status.SUCCESS
def __relearn_endpoints():
arp_wl_list = []
dhcp_wl_list = []
for wl in api.GetWorkloads():
if wl.vnic.IsOriginDiscovered():
if wl.vnic.DhcpEnabled:
dhcp_wl_list.append(wl)
else:
arp_wl_list.append(wl)
if arp_wl_list:
if not arp_utils.SendGratArp(arp_wl_list):
return api.types.status.FAILURE
if dhcp_wl_list:
if not dhcp_utils.AcquireIPFromDhcp(dhcp_wl_list):
return api.types.status.FAILURE
return api.types.status.SUCCESS
def MoveEpIPEntry(src_node, dst_node, ep_ip_prefix):
"""
Moves IP from source workload to destination workload.
"""
ep_ip_addr = __ip_from_prefix(ep_ip_prefix)
src_primary_move = False
# Determine the source and destination workload based on prefix
src_workload = __get_workload_by_prefix(src_node, ep_ip_addr)
dst_workload = __get_workload_by_prefix(dst_node, ep_ip_addr)
api.Logger.debug(
f"Moving IP prefixes {ep_ip_prefix} {src_workload.workload_name}"
f"({src_workload.node_name}) => {dst_workload.workload_name}"
f"({dst_workload.node_name})")
# Update mapping entries on source workload and modify the IP addresses
# If moving IP is the primary, then make one of the secondary IPs as primary
# Otherwise just remove the IP
if src_workload.ip_prefix == ep_ip_prefix:
src_primary_move = True
src_workload.ip_prefix = src_workload.sec_ip_prefixes[0]
src_workload.ip_address = __ip_from_prefix(src_workload.ip_prefix)
src_workload.sec_ip_prefixes.remove(src_workload.ip_prefix)
src_workload.sec_ip_addresses.remove(src_workload.ip_address)
api.Logger.debug(
f"Changing IP address from {ep_ip_prefix} to {src_workload.ip_prefix} "
f"on {src_workload.workload_name}")
ret = __del_ip_from_workloads(src_workload, ep_ip_addr)
if ret != api.types.status.SUCCESS:
return ret
ret = __add_ip_to_workloads(src_workload, src_workload.ip_address)
if ret != api.types.status.SUCCESS:
return ret
elif ep_ip_prefix in src_workload.sec_ip_prefixes:
src_workload.sec_ip_prefixes.remove(ep_ip_prefix)
src_workload.sec_ip_addresses.remove(ep_ip_addr)
api.Logger.debug(
f"Removing IP address {ep_ip_prefix} from {src_workload.workload_name}"
)
ret = __del_ip_from_workloads(src_workload, ep_ip_addr)
if ret != api.types.status.SUCCESS:
return ret
__mapping_entry_handler(src_workload.vnic, [src_workload.ip_prefix] +
src_workload.sec_ip_prefixes)
# Update mapping entries on destination workload
# If the IP being moved was primary in the src, move it as secondary on the
# destination and vice-versa
if src_primary_move:
dst_workload.sec_ip_prefixes += [ep_ip_prefix]
dst_workload.sec_ip_addresses += [__ip_from_prefix(ep_ip_prefix)]
api.Logger.debug(
f"Adding IP address {ep_ip_prefix} to {dst_workload.workload_name}"
)
ret = __add_ip_to_workloads(dst_workload, ep_ip_addr, secondary=True)
if ret != api.types.status.SUCCESS:
return ret
else:
# Move the primary to secondary and add the moving prefix as primary
cur_pri_prefix = dst_workload.ip_prefix
dst_workload.ip_prefix = ep_ip_prefix
dst_workload.ip_address = ep_ip_addr
dst_workload.sec_ip_prefixes += [cur_pri_prefix]
dst_workload.sec_ip_addresses += [__ip_from_prefix(cur_pri_prefix)]
api.Logger.debug(
f"Changing IP address from {cur_pri_prefix} to {ep_ip_prefix} "
f"on {dst_workload.workload_name}")
ret = __add_ip_to_workloads(dst_workload, dst_workload.ip_address)
if ret != api.types.status.SUCCESS:
return ret
ret = __add_ip_to_workloads(dst_workload,
__ip_from_prefix(cur_pri_prefix),
secondary=True)
if ret != api.types.status.SUCCESS:
return ret
__mapping_entry_handler(dst_workload.vnic, [dst_workload.ip_prefix] +
dst_workload.sec_ip_prefixes)
# Delete ARP entry for the IP from the destination
host_utils.DeleteARP(dst_workload.node_name, dst_workload.workload_name,
dst_workload.interface, ep_ip_addr)
# Send Grat ARP
arp.SendGratArp([dst_workload])
return ret
def Verify(tc):
if tc.skip:
return api.types.status.SUCCESS
if tc.pktloss_verify:
if tc.background and tc.bg_cmd_resp is None:
api.Logger.error("Failed in background Ping cmd trigger")
return api.types.status.FAILURE
if tc.resp is None:
api.Logger.error("Received empty response for config request")
return api.types.status.FAILURE
else:
for cmd in tc.resp.commands:
api.PrintCommandResults(cmd)
if cmd.exit_code != 0:
api.Logger.error("Rollout request failed")
return api.types.status.FAILURE
# wait for upgrade to complete. status can be found from the presence of /update/pds_upg_status.txt
api.Logger.info("Sleep for 70 secs before checking for /update/pds_upg_status.txt")
misc_utils.Sleep(70)
status_in_progress = True
while status_in_progress:
misc_utils.Sleep(1)
req = api.Trigger_CreateExecuteCommandsRequest(serial=False)
for node in tc.Nodes:
api.Trigger_AddNaplesCommand(req, node, "grep -v in-progress /update/pds_upg_status.txt", timeout=2)
api.Logger.info("Checking for status not in-progress in file /update/pds_upg_status.txt")
resp = api.Trigger(req)
status_in_progress = False
for cmd_resp in resp.commands:
#api.PrintCommandResults(cmd_resp)
if cmd_resp.exit_code != 0:
status_in_progress = True
#api.Logger.info("File /update/pds_upg_status.txt not found")
else:
api.Logger.info("Status other than in-progress found in /update/pds_upg_status.txt")
# # push interface config updates after upgrade completes
# UpdateConfigAfterUpgrade(tc)
for i in range(10):
api.Logger.info("Sending ARPing, retry count %s"%i)
# Send Grat Arp for learning
arping.SendGratArp(tc.wloads)
misc_utils.Sleep(1)
result = CheckRolloutStatus(tc)
# ensure connectivity after upgrade
if VerifyConnectivity(tc) != api.types.status.SUCCESS:
api.Logger.error("Failed in Connectivity Check Post Upgrade.")
result = api.types.status.FAILURE
error_str = None
if tc.pktloss_verify:
# If rollout status is failure, then no need to wait for traffic test
if result == api.types.status.SUCCESS:
api.Logger.info("Sleep for %s secs for traffic test to complete"%tc.sleep)
misc_utils.Sleep(tc.sleep)
pkt_loss_duration = 0
# terminate background traffic and calculate packet loss duration
if tc.background:
if ping.TestTerminateBackgroundPing(tc, tc.pktsize,\
pktlossverif=tc.pktlossverif) != api.types.status.SUCCESS:
api.Logger.error("Failed in Ping background command termination.")
result = api.types.status.FAILURE
# calculate max packet loss duration for background ping
pkt_loss_duration = ping.GetMaxPktLossDuration(tc, interval=tc.interval)
if pkt_loss_duration != 0:
indent = "-" * 10
if tc.pktlossverif:
result = api.types.status.FAILURE
api.Logger.error(f"{indent} Packet Loss duration during UPGRADE of {tc.Nodes} is {pkt_loss_duration} secs {indent}")
if tc.allowed_down_time and (pkt_loss_duration > tc.allowed_down_time):
api.Logger.error(f"{indent} Exceeded allowed Loss Duration {tc.allowed_down_time} secs {indent}")
result = api.types.status.FAILURE
else:
api.Logger.info("No Packet Loss Found during UPGRADE Test")
if upgrade_utils.VerifyUpgLog(tc.Nodes, tc.GetLogsDir()):
api.Logger.error("Failed to verify the upgrade logs")
result = api.types.status.FAILURE
return result