def ValidateGrpcRead(self, node, getResp):
if utils.IsDryRun(): return True
numObjs = 0
for obj in getResp:
if not utils.ValidateGrpcResponse(obj):
logger.error("GRPC get request failed for ", obj)
continue
#TODO handle singleton object
resps = obj.Response
for resp in resps:
if not self.IsGrpcSpecMatching(resp.Spec):
continue
numObjs += 1
key = self.GetKeyfromSpec(resp.Spec)
cfgObj = self.GetObjectByKey(node, key)
if not utils.ValidateObject(cfgObj, resp):
logger.error("GRPC read validation failed for ", obj)
if cfgObj:
cfgObj.Show()
logger.info(f"Key:{key} Spec:{resp.Spec}")
return False
if hasattr(cfgObj, 'Status'):
cfgObj.Status.Update(resp.Status)
logger.info(f"GRPC read count {numObjs} for {self.ObjType.name} in {node}")
return (numObjs == self.GetNumHwObjects(node))
def VerifyVnicStats(self, spec):
if utils.IsDryRun(): return True
cmd = "vnic statistics --id " + self.UUID.String()
status, op = pdsctl.ExecutePdsctlShowCommand(cmd, None)
entries = op.split("---")
yamlOp = utils.LoadYaml(entries[0])
if not yamlOp: return False
statAttrs = [
'RxPackets', 'RxBytes', 'TxPackets', 'TxBytes', 'MeterRxPackets',
'MeterRxBytes', 'MeterTxPackets', 'MeterTxBytes'
]
mismatch = False
for statAttr in statAttrs:
lstatAttr = statAttr.lower()
if yamlOp['stats'][lstatAttr] != getattr(
self.Stats, statAttr) + spec[lstatAttr]:
preStat = getattr(self.Stats, statAttr)
expStat = preStat + spec[lstatAttr]
output = yamlOp['stats'][lstatAttr]
logger.info(
f"{statAttr} mismatch. PreStat: %d, ExpStat: %d, Output: %d"
% (preStat, expStat, output))
mismatch = True
if mismatch:
return False
return True
def ValidatePdsctlRead(self, node, ret, stdout):
if utils.IsDryRun(): return True
if not ret:
logger.error("pdsctl show cmd failed for ", self.ObjType)
return False
device = self.GetObjectByKey(node, 0)
if "API_STATUS_NOT_FOUND" in stdout:
if device.IsHwHabitant():
logger.error(f"GRPC get request failed for {device} with {stdout}")
device.Show()
return False
return True
# split output per object
cmdop = stdout.split("---")
for op in cmdop:
yamlOp = utils.LoadYaml(op)
if not yamlOp:
continue
device = self.GetObjectByKey(node, 0)
resp = yamlOp['response']
if not utils.ValidateObject(device, resp, yaml=True):
logger.error("pdsctl read validation failed for ", op)
device.Show()
return False
return True
def TestCaseSetup(tc):
tc.AddIgnorePacketField('UDP', 'sport')
tc.AddIgnorePacketField('UDP', 'chksum')
tc.AddIgnorePacketField('TCP', 'chksum')
if not utils.IsDryRun():
return learn.ClearLearnStatistics()
return True
def ModifyHostInterface(self, hostifidx=None):
if not utils.IsNetAgentMode():
return False
if utils.IsDryRun():
return True
if len(self.HostIfUuid) != 0:
#dissociate this from subnet first
InterfaceClient.UpdateHostInterfaces(self.Node, [self], True)
if hostifidx == None:
self.HostIfIdx = [InterfaceClient.GetHostInterface(self.Node)]
else:
self.HostIfIdx = [hostifidx]
node_uuid = EzAccessStoreClient[self.Node].GetNodeUuid(self.Node)
self.HostIfUuid = []
for ifidx in self.HostIfIdx:
self.HostIfUuid.append(utils.PdsUuid(ifidx, node_uuid=node_uuid))
vnicObjs = list(
filter(lambda x: (x.ObjType == api.ObjectTypes.VNIC),
self.Children))
for vnic in vnicObjs:
vnic.HostIfIdx = self.HostIfIdx[0]
vnic.HostIfUuid = utils.PdsUuid(vnic.HostIfIdx,
node_uuid=node_uuid)
InterfaceClient.UpdateHostInterfaces(self.Node, [self])
return True
def SetupCfgFilesForUpgrade(self, spec=None):
# For hardware nothing to setup specifically
if not utils.IsDol():
return True
if utils.IsDryRun():
return True
mode = "hitless"
self.failure_stage = None
self.failure_reason = None
args = ""
if hasattr(spec, "UpgMode"):
mode = getattr(spec, "UpgMode", "hitless")
if hasattr(spec, "failure_stage"):
self.failure_stage = getattr(spec, "failure_stage", None)
if hasattr(spec, "failure_reason"):
self.failure_reason = getattr(spec, "failure_reason", None)
logger.info("Setup Upgrade Config Files for %s mode" % mode)
logger.info("Setup Upgrade failure stage %s, failure reason %s" %
(self.failure_stage, self.failure_reason))
if self.failure_stage != None and self.failure_reason != None:
args = "%s %s" % (self.failure_stage, self.failure_reason)
# For now cfg file setup done only for hitless mode
if mode == "hitless":
# setup hitless upgrade config files
upg_setup_cmds = "apollo/test/tools/apulu/setup_hitless_upgrade_cfg_sim.sh %s" % args
if not RunCmd(upg_setup_cmds, timeout=20, background=True):
logger.error("Command Execution Failed: %s" % upg_setup_cmds)
return False
utils.Sleep(10) # setup is executed in the background.
return True
def TestCaseStepVerify(tc, step):
# this module is specific to dhcp feature only
if tc.module.feature != "learn::dhcp": return False
if utils.IsDryRun(): return True
if "LEARN_MAC_FROM_DHCP" in tc.module.name:
ep_mac = learn.EpMac(tc.config.localmapping)
return learn.ExistsOnDevice(ep_mac)
elif "LEARN_IP_FROM_DHCP" in tc.module.name:
ep_ip = learn.EpIp(tc.config.localmapping)
return learn.ExistsOnDevice(ep_ip)
elif tc.module.name == "CLEAR_IP_THEN_MAC":
ep_mac = learn.EpMac(tc.config.localmapping)
ep_ip = learn.EpIp(tc.config.localmapping)
return learn.ClearOnDevice(ep_ip) and \
not learn.ExistsOnDevice(ep_ip) and \
learn.ExistsOnDevice(ep_mac) and \
learn.ClearOnDevice(ep_mac) and \
not learn.ExistsOnDevice(ep_mac)
elif tc.module.name == "CLEAR_IP_ALL":
ep_ip = learn.EpIp(tc.config.localmapping)
return learn.ClearAllLearntIps() and \
not learn.ExistsOnDevice(ep_ip)
elif tc.module.name == "CLEAR_MAC_ALL":
ep_mac = learn.EpMac(tc.config.localmapping)
ep_ip = learn.EpIp(tc.config.localmapping)
return learn.ClearAllLearntMacs() and \
not learn.ExistsOnDevice(ep_ip) and \
not learn.ExistsOnDevice(ep_mac)
else:
return True
def ValidateGrpcRead(self, node, getResp):
if not EzAccessStoreClient[node].IsDeviceOverlayRoutingEnabled():
super().ValidateGrpcRead(node, getResp)
# TODO Override GRPC read temporarily to avoid failure
logger.info("Validate gRPC RouteObject - Overlay Routing enabled")
if utils.IsDryRun(): return True
return True
def ValidateResponse(self, resps):
if utils.IsDryRun(): return None
for r in resps:
self.Status.Update(r)
if not r.Status == upgrade_pb2.UPGRADE_STATUS_OK:
logger.error(f"Upgrade request failed with {r}")
continue
return self.Status.GetLastUpgradeStatus()
def UpgradeReq(self, ip=None):
self.Show()
if utils.IsDryRun():
return upgrade_pb2.UPGRADE_STATUS_OK
msg = self.GetGrpcUpgradeRequestMessage()
resp = api.upgradeClient[self.Node].Request(self.ObjType, 'UpgRequest',
[msg])
return self.ValidateResponse(resp)
def ReadLifs(self, node):
if utils.IsDryRun(): return
ret, op = utils.RunPdsctlShowCmd(node, "lif", yaml=True)
if not ret:
logger.error(f"show lif failed for node {node}")
return False
cmdop = op.split("---")
return cmdop
def VerifyLinkAlert(self, spec=None):
eventObj = OperEventClient.Objects(self.Node)
try:
alert = next(eventObj.GetEvents())
except:
logger.error(f"Got no event from {self.Node} for {self}")
return True if utils.IsDryRun() else False
return self.__validate_link_alert(alert)
def ValidateLearntMacEntries(self, node, ret, cli_op):
if utils.IsDryRun(): return True
if not ret:
logger.error("pdsctl show learn mac cmd failed")
return False
# split output per object
mac_entries = cli_op.split("---")
for mac in mac_entries:
yamlOp = utils.LoadYaml(mac)
if not yamlOp:
continue
yamlOp = yamlOp['macentry']['entryauto']
mac_key = yamlOp['key']['macaddr']
subnet_uuid = utils.GetYamlSpecAttr(yamlOp['key'], 'subnetid')
vnic_uuid_str = utils.List2UuidStr(
utils.GetYamlSpecAttr(yamlOp, 'vnicid'))
# verifying if the info learnt is expected from config
vnic_obj = self.GetVnicByL2MappingKey(node, mac_key, subnet_uuid,
0)
if vnic_obj == None:
logger.error(
f"vnic not found in client object store for key {mac_key} {subnet_uuid}{0}"
)
return False
# verifying if VNIC has been programmed correctly by Learn
args = "--id " + vnic_uuid_str
ret, op = utils.RunPdsctlShowCmd(node, "vnic", args, True)
if not ret:
logger.error(f"show vnic failed for vnic id {vnic_uuid_str}")
return False
cmdop = op.split("---")
logger.info("Num entries returned for vnic show id %s is %s" %
(vnic_uuid_str, (len(cmdop) - 1)))
for vnic_entry in cmdop:
yamlOp = utils.LoadYaml(vnic_entry)
if not yamlOp:
continue
vnic_spec = yamlOp['spec']
hostif = vnic_spec['hostif']
if utils.PdsUuid(
hostif).GetUuid() != vnic_obj.HostIfUuid.GetUuid():
logger.error(
f"host interface did not match for {vnic_uuid_str}")
return False
if vnic_spec['macaddress'] != vnic_obj.MACAddr.getnum():
logger.error(
f"mac address did not match for {vnic_uuid_str}")
return False
logger.info(
"Found VNIC %s entry for learn MAC MAC:%s, Subnet:%s, VNIC:%s "
%
(utils.List2UuidStr(utils.GetYamlSpecAttr(
vnic_spec, 'id')), vnic_obj.MACAddr.get(),
vnic_obj.SUBNET.UUID, vnic_uuid_str))
return True
def ConfigReplayReadyCheck(self):
if utils.IsDryRun():
return True
msg = upgrade_pb2.EmptyMsg()
resp = api.upgradeClient[self.Node].Request(self.ObjType,
'ConfigReplayReadyCheck',
[msg])
for r in resp:
return r.IsReady
def ValidateResponse(self, resps):
if utils.IsDryRun(): return None
for r in resps:
if not utils.ValidateGrpcResponse(r):
logger.error(f"TechSupport request failed with {r}")
continue
status = r.Response
self.Status.Update(status)
return self.Status.GetTechSupportFile()
def ValidatePdsctlRead(self, node, ret, stdout):
if not EzAccessStoreClient[node].IsDeviceOverlayRoutingEnabled():
super().ValidatePdsctlRead(node, ret, stdout)
# TODO Override Pdsctl read temporarily to avoid failure
logger.info("Validate Pdsctl RouteObject - Overlay Routing enabled")
if utils.IsDryRun(): return True
if not ret:
logger.error("pdsctl show cmd failed for ", self.ObjType)
return False
return True
def TestCaseSetup(tc):
tc.AddIgnorePacketField('UDP', 'sport')
tc.AddIgnorePacketField('UDP', 'chksum')
tc.AddIgnorePacketField('TCP', 'chksum')
tc.AddIgnorePacketField('IP', 'chksum') #Needed to pass NAT testcase
if not utils.IsDryRun():
utils.ReadTestcaseStats(tc.config)
if hasattr(tc.config, 'statscache') and len(tc.config.statscache):
tc.pvtdata.statscache = copy.deepcopy(tc.config.statscache)
return True
def ChangeSubnet(self, vnic, new_subnet):
logger.info(
f"Changing subnet for {vnic} {vnic.SUBNET} => {new_subnet}")
# Handle child/parent relationship
old_subnet = vnic.SUBNET
old_subnet.DeleteChild(vnic)
new_subnet.AddChild(vnic)
vnic.SUBNET = new_subnet
vnic.Vnid = vnic.SUBNET.Vnid
if not utils.IsDryRun():
node_uuid = EzAccessStoreClient[new_subnet.Node].GetNodeUuid(
new_subnet.Node)
vnic.HostIfIdx = vnic.SUBNET.HostIfIdx[0]
vnic.HostIfUuid = utils.PdsUuid(vnic.HostIfIdx,
node_uuid=node_uuid)
# Handle node change scenario
if old_subnet.Node != new_subnet.Node:
# Delete VNIC from old node
del self.Objs[vnic.Node][vnic.VnicId]
del self.__l2mapping_objs[vnic.Node][(vnic.MACAddr.getnum(),
vnic.SUBNET.UUID.GetUuid(),
vnic.VlanId())]
vnic.Node = new_subnet.Node
self.Objs[vnic.Node].update({vnic.VnicId: vnic})
self.__l2mapping_objs[vnic.Node].update({
(vnic.MACAddr.getnum(), vnic.SUBNET.UUID.GetUuid(),
vnic.VlanId()):
vnic
})
# Move children to new Node
for lmap in vnic.Children:
# Operate only on lmapping objects
if lmap.GetObjectType() != api.ObjectTypes.LMAPPING:
continue
# Move lmap entry to new Node
lmapping.client.ChangeNode(lmap, new_subnet.Node)
# Destroy rmap entry in new subnet
rmap = new_subnet.GetRemoteMappingObjectByIp(lmap.IP)
assert (rmap)
rmap.Destroy()
# Add rmap entry in old subnet
mac = "macaddr/%s" % vnic.MACAddr.get()
rmap_spec = {}
rmap_spec['rmacaddr'] = objects.TemplateFieldObject(mac)
rmap_spec['ripaddr'] = lmap.IP
ipversion = utils.IP_VERSION_6 if lmap.AddrFamily == 'IPV6' else utils.IP_VERSION_4
rmapClient.GenerateObj(old_subnet.Node, old_subnet, rmap_spec,
ipversion)
return
def ReadObjects(self, node):
logger.info(f"Reading {self.ObjType.name} Objects from {node}")
if utils.IsDryRun():
return True
if not self.GrpcRead(node):
return False
if not self.PdsctlRead(node):
return False
if not self.HttpRead(node):
return False
logger.info(f"Read & Validated {self.ObjType.name} Objects from {node}")
return True
def GetStats(self):
if utils.IsDryRun():
return
grpcmsg = nat_pb2.NatPortBlockGetRequest()
grpcmsg.Id.append(self.GetKey())
resp = api.client[self.Node].Get(api.ObjectTypes.NAT, [grpcmsg])
if resp is None:
return None
stats = NatPbStats()
stats.InUseCount = resp[0].Response[0].Stats.InUseCount
stats.SessionCount = resp[0].Response[0].Stats.SessionCount
return stats
def TestCaseStepVerify(tc, step):
if utils.IsDryRun(): return True
if "LEARN_MULTIPLE_IP_SAME_VNIC" in tc.module.name:
ep_mac = learn.EpMac(tc.config.localmapping)
ep_ip = learn.EpIp(tc.config.localmapping)
return learn.ExistsOnDevice(ep_mac) and learn.ExistsOnDevice(ep_ip)
elif tc.module.name == "LEARN_NEG_IP_NOT_IN_SUBNET":
ep_ip = learn.EpIp(tc.config.localmapping)
val_type = learn_pb2.LEARN_CHECK_IP_IN_SUBNET
drop_reason = learn.learn_pb2.LEARN_PKTDROP_REASON_LEARNING_FAIL
return VerifyCounter("validationerrors", val_type, 1) and \
VerifyCounter("dropstats", drop_reason, 1)
return True
def UpdateUnderlayObjects(self, node):
if utils.IsDryRun(): return True
cfgObjects = self.__underlay_objs[node].values()
logger.info(
f"Updating {len(cfgObjects)} underlay {self.ObjType.name} Objects in {node}"
)
result = list(map(lambda x: utils.UpdateObject(x), cfgObjects))
if not all(result):
logger.info(
f"Updating {len(cfgObjects)} underlay {self.ObjType.name} Objects FAILED in {node}"
)
return False
return True
def TestCaseStepVerify(tc, step):
if utils.IsDryRun(): return True
if "LEARN_MAC_IP_WITH_ARP" in tc.module.name:
# verify pdsctl show learn mac/ip produced correct output
# TODO: verify show vnic and show mapping output is correct
ep_mac = learn.EpMac(tc.config.localmapping)
ep_ip = learn.EpIp(tc.config.localmapping)
if learn.ExistsOnDevice(ep_mac) and learn.ExistsOnDevice(ep_ip):
stats = learn.GetLearnStatistics()
if not stats:
return False
return stats['pktsrcvd'] == 1 and \
stats['pktssent'] == 1 and \
stats['maclearnevents'][1]['count'] == 1 and \
stats['iplearnevents'][1]['count'] == 1
return False
elif tc.module.name == "RECV_ARP_PROBE_AND_REPLY":
# verify age is reflected correctly
if step.step_id == 1:
ep_ip = learn.EpIp(tc.config.localmapping)
return learn.VerifyIPAgeRefreshed(tc, ep_ip)
return True
elif tc.module.name == "RECV_ARP_PROBES_AND_TIMEOUT_MAC_IP":
# verify IP is timed out in step 3 and mac in step 4
if step.step_id == 3:
ep_ip = learn.EpIp(tc.config.localmapping)
return not learn.ExistsOnDevice(ep_ip)
elif step.step_id == 4:
ep_mac = learn.EpMac(tc.config.localmapping)
return not learn.ExistsOnDevice(ep_mac)
else:
return True
elif tc.module.name == "CLEAR_MAC":
# clear the learnt MAC
ep_mac = learn.EpMac(tc.config.localmapping)
return learn.ClearOnDevice(ep_mac) and \
not learn.ExistsOnDevice(ep_mac) and \
learn.ClearLearnStatistics()
elif tc.module.name == "LEARN_MAC_WITH_RARP":
ep_mac = learn.EpMac(tc.config.localmapping)
if learn.ExistsOnDevice(ep_mac):
stats = learn.GetLearnStatistics()
if not stats:
return False
return stats['pktsrcvd'] == 1 and \
stats['maclearnevents'][1]['count'] == 1
return False
return True
def UpdateHostInterfaces(self, node, subnets, dissociate=False):
if utils.IsDryRun() or not utils.IsNetAgentMode():
return
for subnet in subnets:
if len(subnet.HostIfIdx) == 0:
continue
hostif = self.FindHostInterface(node, subnet.HostIfIdx[0])
if hostif:
hostif.UpdateVrfAndNetwork(subnet, dissociate)
hostif.Show()
api.client[node].Update(api.ObjectTypes.INTERFACE, [hostif])
print("after update pf")
resp = api.client[node].GetHttp(api.ObjectTypes.INTERFACE)
for r in resp:
print(r)
return
def ValidateGrpcRead(self, node, getResp):
if utils.IsDryRun(): return True
device = self.GetObjectByKey(node, 0)
for obj in getResp:
if not utils.ValidateGrpcResponse(obj):
if device.IsHwHabitant():
logger.error(f"GRPC get request failed for {device} with {obj}")
device.Show()
return False
return True
resp = obj.Response
if not utils.ValidateObject(device, resp):
logger.error("GRPC read validation failed for ", obj)
device.Show()
return False
return True
def UpdateImplicit(self):
if not utils.IsNetAgentMode():
return
if utils.IsDol() and utils.IsDryRun():
return
if not self.IsOriginImplicitlyCreated():
return
# We need to read info from naples and update the DS
resp = api.client[self.Node].GetHttp(self.ObjType)
if not resp:
return
for ifinst in resp:
if self.Type == topo.InterfaceTypes.L3:
if (not ifinst['spec']['type'] == 'L3'):
continue
riid = ifinst['meta']['name']
if (self.InterfaceId != int(riid[len(riid) - 1])):
continue
elif self.Type == topo.InterfaceTypes.LOOPBACK:
if (not ifinst['spec']['type'] == 'LOOPBACK'):
continue
elif self.Type == topo.InterfaceTypes.ETH:
if (not ifinst['spec']['type'] == 'HOST_PF'):
continue
if (ifinst['status']['if-host-status']['host-ifname'] !=
self.IfName):
continue
else:
continue
# Found matching interface, get basic info
uuid_str = ifinst['meta']['uuid']
self.UUID = utils.PdsUuid(bytes.fromhex(uuid_str.replace('-','')),\
self.ObjType)
# instance found. Store meta info
self.Tenant = ifinst['meta']['tenant']
self.Namespace = ifinst['meta']['namespace']
self.SetIfNameFromAgentData(ifinst['meta']['name'])
# get ifinfo
if 'ip-address' in ifinst['spec']:
self.IpPrefix = ipaddress.ip_network(ifinst['spec']['ip-address'],\
False)
if 'vrf-name' in ifinst['spec']:
self.VrfName = ifinst['spec']['vrf-name']
return
def ReadHostInterfaceInfo(self):
if utils.IsDryRun(): return
# reads nicmgr.log and get interface info
nicmgrlog = utils.GetNicmgrLogPath()
f = open(nicmgrlog, "r")
for ifobj in self.HostIfs.values():
# get lifbase and count
pattern = ' %s: lif_base (\w+) lif_count (\w+)' % (ifobj.IfName)
for line in f:
match = re.search(pattern, line)
if match is None:
continue
lifbase = int(match.groups()[0])
lifcount = int(match.groups()[1])
ifobj.SetLifBase(lifbase)
ifobj.SetLifCount(lifcount)
break
# get devcmd addr
pattern = ' %s: regs_mem_addr (\w+) devcmd_mem_addr (\w+)' % (
ifobj.IfName)
for line in f:
match = re.search(pattern, line)
if match is None:
continue
devcmdaddr = int(match.groups()[1], base=16)
ifobj.SetDevCmdAddr(devcmdaddr)
break
# get qstate addr for all lifs under this device
for i in range(lifcount):
lif_id = lifbase + i
pattern = ' lif-%d: qtype: (\d{1}), qstate_base: (\w+)' % (
lif_id)
for line in f:
match = re.search(pattern, line)
if match is None:
continue
q_type = int(match.groups()[0])
qstate_base = int(match.groups()[1], base=16)
qstateaddr_list = ifobj.Lif2QstateMap.get(lif_id, [])
qstateaddr_list.insert(q_type, qstate_base)
ifobj.Lif2QstateMap.update({lif_id: qstateaddr_list})
if (q_type == 7):
break
f.close()
return
def VerifyUpgradeDoneStatus(self, spec=None):
if utils.IsDryRun():
return True
retry = 1200
while retry:
with open("/update/pds_upg_status.txt", 'r') as fp:
status = fp.read()
logger.info("upgrade status %s" % status)
if status.find("success") != -1:
return True
if status.find("failed") != -1:
if self.failure_stage != None:
return True
return False
retry = retry - 1
utils.Sleep(10)
return False
def ExecuteVPPCommand(command, args=None):
vppctl = __get_vppctl_path()
if args:
vppctlproc = [vppctl] + args.split() + [command]
else:
vppctlproc = [vppctl, command]
retval = True
output = ""
try:
if not utils.IsDryRun():
output = str(
subprocess.check_output(vppctlproc, stderr=subprocess.STDOUT))
except subprocess.CalledProcessError as e:
output = "Command execution failed."
retval = False
logger.info("VPP: command[%s], output[%s], retval[%d]" \
%(command, output, retval))
return retval, output
def ValidatePdsctlRead(self, node, ret, stdout):
if utils.IsDryRun(): return True
if not ret:
logger.error("pdsctl show cmd failed for ", self.ObjType)
return False
# split output per object
cmdop = stdout.split("---")
assert((len(cmdop) - 1) == self.GetNumHwObjects(node))
for op in cmdop:
yamlOp = utils.LoadYaml(op)
if not yamlOp:
continue
key = self.GetKeyfromSpec(yamlOp['spec'], yaml=True)
cfgObj = self.GetObjectByKey(node, key)
if not utils.ValidateObject(cfgObj, yamlOp, yaml=True):
logger.error("pdsctl read validation failed for ", op)
cfgObj.Show()
return False
return True
