def ValidateGrpcRead(self, node, getResp): if utils.IsDryRun(): return True numObjs = 0 for obj in getResp: if not utils.ValidateGrpcResponse(obj): logger.error("GRPC get request failed for ", obj) continue #TODO handle singleton object resps = obj.Response for resp in resps: if not self.IsGrpcSpecMatching(resp.Spec): continue numObjs += 1 key = self.GetKeyfromSpec(resp.Spec) cfgObj = self.GetObjectByKey(node, key) if not utils.ValidateObject(cfgObj, resp): logger.error("GRPC read validation failed for ", obj) if cfgObj: cfgObj.Show() logger.info(f"Key:{key} Spec:{resp.Spec}") return False if hasattr(cfgObj, 'Status'): cfgObj.Status.Update(resp.Status) logger.info(f"GRPC read count {numObjs} for {self.ObjType.name} in {node}") return (numObjs == self.GetNumHwObjects(node))
def VerifyVnicStats(self, spec): if utils.IsDryRun(): return True cmd = "vnic statistics --id " + self.UUID.String() status, op = pdsctl.ExecutePdsctlShowCommand(cmd, None) entries = op.split("---") yamlOp = utils.LoadYaml(entries[0]) if not yamlOp: return False statAttrs = [ 'RxPackets', 'RxBytes', 'TxPackets', 'TxBytes', 'MeterRxPackets', 'MeterRxBytes', 'MeterTxPackets', 'MeterTxBytes' ] mismatch = False for statAttr in statAttrs: lstatAttr = statAttr.lower() if yamlOp['stats'][lstatAttr] != getattr( self.Stats, statAttr) + spec[lstatAttr]: preStat = getattr(self.Stats, statAttr) expStat = preStat + spec[lstatAttr] output = yamlOp['stats'][lstatAttr] logger.info( f"{statAttr} mismatch. PreStat: %d, ExpStat: %d, Output: %d" % (preStat, expStat, output)) mismatch = True if mismatch: return False return True
def ValidatePdsctlRead(self, node, ret, stdout): if utils.IsDryRun(): return True if not ret: logger.error("pdsctl show cmd failed for ", self.ObjType) return False device = self.GetObjectByKey(node, 0) if "API_STATUS_NOT_FOUND" in stdout: if device.IsHwHabitant(): logger.error(f"GRPC get request failed for {device} with {stdout}") device.Show() return False return True # split output per object cmdop = stdout.split("---") for op in cmdop: yamlOp = utils.LoadYaml(op) if not yamlOp: continue device = self.GetObjectByKey(node, 0) resp = yamlOp['response'] if not utils.ValidateObject(device, resp, yaml=True): logger.error("pdsctl read validation failed for ", op) device.Show() return False return True
def TestCaseSetup(tc): tc.AddIgnorePacketField('UDP', 'sport') tc.AddIgnorePacketField('UDP', 'chksum') tc.AddIgnorePacketField('TCP', 'chksum') if not utils.IsDryRun(): return learn.ClearLearnStatistics() return True
def ModifyHostInterface(self, hostifidx=None): if not utils.IsNetAgentMode(): return False if utils.IsDryRun(): return True if len(self.HostIfUuid) != 0: #dissociate this from subnet first InterfaceClient.UpdateHostInterfaces(self.Node, [self], True) if hostifidx == None: self.HostIfIdx = [InterfaceClient.GetHostInterface(self.Node)] else: self.HostIfIdx = [hostifidx] node_uuid = EzAccessStoreClient[self.Node].GetNodeUuid(self.Node) self.HostIfUuid = [] for ifidx in self.HostIfIdx: self.HostIfUuid.append(utils.PdsUuid(ifidx, node_uuid=node_uuid)) vnicObjs = list( filter(lambda x: (x.ObjType == api.ObjectTypes.VNIC), self.Children)) for vnic in vnicObjs: vnic.HostIfIdx = self.HostIfIdx[0] vnic.HostIfUuid = utils.PdsUuid(vnic.HostIfIdx, node_uuid=node_uuid) InterfaceClient.UpdateHostInterfaces(self.Node, [self]) return True
def SetupCfgFilesForUpgrade(self, spec=None): # For hardware nothing to setup specifically if not utils.IsDol(): return True if utils.IsDryRun(): return True mode = "hitless" self.failure_stage = None self.failure_reason = None args = "" if hasattr(spec, "UpgMode"): mode = getattr(spec, "UpgMode", "hitless") if hasattr(spec, "failure_stage"): self.failure_stage = getattr(spec, "failure_stage", None) if hasattr(spec, "failure_reason"): self.failure_reason = getattr(spec, "failure_reason", None) logger.info("Setup Upgrade Config Files for %s mode" % mode) logger.info("Setup Upgrade failure stage %s, failure reason %s" % (self.failure_stage, self.failure_reason)) if self.failure_stage != None and self.failure_reason != None: args = "%s %s" % (self.failure_stage, self.failure_reason) # For now cfg file setup done only for hitless mode if mode == "hitless": # setup hitless upgrade config files upg_setup_cmds = "apollo/test/tools/apulu/setup_hitless_upgrade_cfg_sim.sh %s" % args if not RunCmd(upg_setup_cmds, timeout=20, background=True): logger.error("Command Execution Failed: %s" % upg_setup_cmds) return False utils.Sleep(10) # setup is executed in the background. return True
def TestCaseStepVerify(tc, step): # this module is specific to dhcp feature only if tc.module.feature != "learn::dhcp": return False if utils.IsDryRun(): return True if "LEARN_MAC_FROM_DHCP" in tc.module.name: ep_mac = learn.EpMac(tc.config.localmapping) return learn.ExistsOnDevice(ep_mac) elif "LEARN_IP_FROM_DHCP" in tc.module.name: ep_ip = learn.EpIp(tc.config.localmapping) return learn.ExistsOnDevice(ep_ip) elif tc.module.name == "CLEAR_IP_THEN_MAC": ep_mac = learn.EpMac(tc.config.localmapping) ep_ip = learn.EpIp(tc.config.localmapping) return learn.ClearOnDevice(ep_ip) and \ not learn.ExistsOnDevice(ep_ip) and \ learn.ExistsOnDevice(ep_mac) and \ learn.ClearOnDevice(ep_mac) and \ not learn.ExistsOnDevice(ep_mac) elif tc.module.name == "CLEAR_IP_ALL": ep_ip = learn.EpIp(tc.config.localmapping) return learn.ClearAllLearntIps() and \ not learn.ExistsOnDevice(ep_ip) elif tc.module.name == "CLEAR_MAC_ALL": ep_mac = learn.EpMac(tc.config.localmapping) ep_ip = learn.EpIp(tc.config.localmapping) return learn.ClearAllLearntMacs() and \ not learn.ExistsOnDevice(ep_ip) and \ not learn.ExistsOnDevice(ep_mac) else: return True
def ValidateGrpcRead(self, node, getResp): if not EzAccessStoreClient[node].IsDeviceOverlayRoutingEnabled(): super().ValidateGrpcRead(node, getResp) # TODO Override GRPC read temporarily to avoid failure logger.info("Validate gRPC RouteObject - Overlay Routing enabled") if utils.IsDryRun(): return True return True
def ValidateResponse(self, resps): if utils.IsDryRun(): return None for r in resps: self.Status.Update(r) if not r.Status == upgrade_pb2.UPGRADE_STATUS_OK: logger.error(f"Upgrade request failed with {r}") continue return self.Status.GetLastUpgradeStatus()
def UpgradeReq(self, ip=None): self.Show() if utils.IsDryRun(): return upgrade_pb2.UPGRADE_STATUS_OK msg = self.GetGrpcUpgradeRequestMessage() resp = api.upgradeClient[self.Node].Request(self.ObjType, 'UpgRequest', [msg]) return self.ValidateResponse(resp)
def ReadLifs(self, node): if utils.IsDryRun(): return ret, op = utils.RunPdsctlShowCmd(node, "lif", yaml=True) if not ret: logger.error(f"show lif failed for node {node}") return False cmdop = op.split("---") return cmdop
def VerifyLinkAlert(self, spec=None): eventObj = OperEventClient.Objects(self.Node) try: alert = next(eventObj.GetEvents()) except: logger.error(f"Got no event from {self.Node} for {self}") return True if utils.IsDryRun() else False return self.__validate_link_alert(alert)
def ValidateLearntMacEntries(self, node, ret, cli_op): if utils.IsDryRun(): return True if not ret: logger.error("pdsctl show learn mac cmd failed") return False # split output per object mac_entries = cli_op.split("---") for mac in mac_entries: yamlOp = utils.LoadYaml(mac) if not yamlOp: continue yamlOp = yamlOp['macentry']['entryauto'] mac_key = yamlOp['key']['macaddr'] subnet_uuid = utils.GetYamlSpecAttr(yamlOp['key'], 'subnetid') vnic_uuid_str = utils.List2UuidStr( utils.GetYamlSpecAttr(yamlOp, 'vnicid')) # verifying if the info learnt is expected from config vnic_obj = self.GetVnicByL2MappingKey(node, mac_key, subnet_uuid, 0) if vnic_obj == None: logger.error( f"vnic not found in client object store for key {mac_key} {subnet_uuid}{0}" ) return False # verifying if VNIC has been programmed correctly by Learn args = "--id " + vnic_uuid_str ret, op = utils.RunPdsctlShowCmd(node, "vnic", args, True) if not ret: logger.error(f"show vnic failed for vnic id {vnic_uuid_str}") return False cmdop = op.split("---") logger.info("Num entries returned for vnic show id %s is %s" % (vnic_uuid_str, (len(cmdop) - 1))) for vnic_entry in cmdop: yamlOp = utils.LoadYaml(vnic_entry) if not yamlOp: continue vnic_spec = yamlOp['spec'] hostif = vnic_spec['hostif'] if utils.PdsUuid( hostif).GetUuid() != vnic_obj.HostIfUuid.GetUuid(): logger.error( f"host interface did not match for {vnic_uuid_str}") return False if vnic_spec['macaddress'] != vnic_obj.MACAddr.getnum(): logger.error( f"mac address did not match for {vnic_uuid_str}") return False logger.info( "Found VNIC %s entry for learn MAC MAC:%s, Subnet:%s, VNIC:%s " % (utils.List2UuidStr(utils.GetYamlSpecAttr( vnic_spec, 'id')), vnic_obj.MACAddr.get(), vnic_obj.SUBNET.UUID, vnic_uuid_str)) return True
def ConfigReplayReadyCheck(self): if utils.IsDryRun(): return True msg = upgrade_pb2.EmptyMsg() resp = api.upgradeClient[self.Node].Request(self.ObjType, 'ConfigReplayReadyCheck', [msg]) for r in resp: return r.IsReady
def ValidateResponse(self, resps): if utils.IsDryRun(): return None for r in resps: if not utils.ValidateGrpcResponse(r): logger.error(f"TechSupport request failed with {r}") continue status = r.Response self.Status.Update(status) return self.Status.GetTechSupportFile()
def ValidatePdsctlRead(self, node, ret, stdout): if not EzAccessStoreClient[node].IsDeviceOverlayRoutingEnabled(): super().ValidatePdsctlRead(node, ret, stdout) # TODO Override Pdsctl read temporarily to avoid failure logger.info("Validate Pdsctl RouteObject - Overlay Routing enabled") if utils.IsDryRun(): return True if not ret: logger.error("pdsctl show cmd failed for ", self.ObjType) return False return True
def TestCaseSetup(tc): tc.AddIgnorePacketField('UDP', 'sport') tc.AddIgnorePacketField('UDP', 'chksum') tc.AddIgnorePacketField('TCP', 'chksum') tc.AddIgnorePacketField('IP', 'chksum') #Needed to pass NAT testcase if not utils.IsDryRun(): utils.ReadTestcaseStats(tc.config) if hasattr(tc.config, 'statscache') and len(tc.config.statscache): tc.pvtdata.statscache = copy.deepcopy(tc.config.statscache) return True
def ChangeSubnet(self, vnic, new_subnet): logger.info( f"Changing subnet for {vnic} {vnic.SUBNET} => {new_subnet}") # Handle child/parent relationship old_subnet = vnic.SUBNET old_subnet.DeleteChild(vnic) new_subnet.AddChild(vnic) vnic.SUBNET = new_subnet vnic.Vnid = vnic.SUBNET.Vnid if not utils.IsDryRun(): node_uuid = EzAccessStoreClient[new_subnet.Node].GetNodeUuid( new_subnet.Node) vnic.HostIfIdx = vnic.SUBNET.HostIfIdx[0] vnic.HostIfUuid = utils.PdsUuid(vnic.HostIfIdx, node_uuid=node_uuid) # Handle node change scenario if old_subnet.Node != new_subnet.Node: # Delete VNIC from old node del self.Objs[vnic.Node][vnic.VnicId] del self.__l2mapping_objs[vnic.Node][(vnic.MACAddr.getnum(), vnic.SUBNET.UUID.GetUuid(), vnic.VlanId())] vnic.Node = new_subnet.Node self.Objs[vnic.Node].update({vnic.VnicId: vnic}) self.__l2mapping_objs[vnic.Node].update({ (vnic.MACAddr.getnum(), vnic.SUBNET.UUID.GetUuid(), vnic.VlanId()): vnic }) # Move children to new Node for lmap in vnic.Children: # Operate only on lmapping objects if lmap.GetObjectType() != api.ObjectTypes.LMAPPING: continue # Move lmap entry to new Node lmapping.client.ChangeNode(lmap, new_subnet.Node) # Destroy rmap entry in new subnet rmap = new_subnet.GetRemoteMappingObjectByIp(lmap.IP) assert (rmap) rmap.Destroy() # Add rmap entry in old subnet mac = "macaddr/%s" % vnic.MACAddr.get() rmap_spec = {} rmap_spec['rmacaddr'] = objects.TemplateFieldObject(mac) rmap_spec['ripaddr'] = lmap.IP ipversion = utils.IP_VERSION_6 if lmap.AddrFamily == 'IPV6' else utils.IP_VERSION_4 rmapClient.GenerateObj(old_subnet.Node, old_subnet, rmap_spec, ipversion) return
def ReadObjects(self, node): logger.info(f"Reading {self.ObjType.name} Objects from {node}") if utils.IsDryRun(): return True if not self.GrpcRead(node): return False if not self.PdsctlRead(node): return False if not self.HttpRead(node): return False logger.info(f"Read & Validated {self.ObjType.name} Objects from {node}") return True
def GetStats(self): if utils.IsDryRun(): return grpcmsg = nat_pb2.NatPortBlockGetRequest() grpcmsg.Id.append(self.GetKey()) resp = api.client[self.Node].Get(api.ObjectTypes.NAT, [grpcmsg]) if resp is None: return None stats = NatPbStats() stats.InUseCount = resp[0].Response[0].Stats.InUseCount stats.SessionCount = resp[0].Response[0].Stats.SessionCount return stats
def TestCaseStepVerify(tc, step): if utils.IsDryRun(): return True if "LEARN_MULTIPLE_IP_SAME_VNIC" in tc.module.name: ep_mac = learn.EpMac(tc.config.localmapping) ep_ip = learn.EpIp(tc.config.localmapping) return learn.ExistsOnDevice(ep_mac) and learn.ExistsOnDevice(ep_ip) elif tc.module.name == "LEARN_NEG_IP_NOT_IN_SUBNET": ep_ip = learn.EpIp(tc.config.localmapping) val_type = learn_pb2.LEARN_CHECK_IP_IN_SUBNET drop_reason = learn.learn_pb2.LEARN_PKTDROP_REASON_LEARNING_FAIL return VerifyCounter("validationerrors", val_type, 1) and \ VerifyCounter("dropstats", drop_reason, 1) return True
def UpdateUnderlayObjects(self, node): if utils.IsDryRun(): return True cfgObjects = self.__underlay_objs[node].values() logger.info( f"Updating {len(cfgObjects)} underlay {self.ObjType.name} Objects in {node}" ) result = list(map(lambda x: utils.UpdateObject(x), cfgObjects)) if not all(result): logger.info( f"Updating {len(cfgObjects)} underlay {self.ObjType.name} Objects FAILED in {node}" ) return False return True
def TestCaseStepVerify(tc, step): if utils.IsDryRun(): return True if "LEARN_MAC_IP_WITH_ARP" in tc.module.name: # verify pdsctl show learn mac/ip produced correct output # TODO: verify show vnic and show mapping output is correct ep_mac = learn.EpMac(tc.config.localmapping) ep_ip = learn.EpIp(tc.config.localmapping) if learn.ExistsOnDevice(ep_mac) and learn.ExistsOnDevice(ep_ip): stats = learn.GetLearnStatistics() if not stats: return False return stats['pktsrcvd'] == 1 and \ stats['pktssent'] == 1 and \ stats['maclearnevents'][1]['count'] == 1 and \ stats['iplearnevents'][1]['count'] == 1 return False elif tc.module.name == "RECV_ARP_PROBE_AND_REPLY": # verify age is reflected correctly if step.step_id == 1: ep_ip = learn.EpIp(tc.config.localmapping) return learn.VerifyIPAgeRefreshed(tc, ep_ip) return True elif tc.module.name == "RECV_ARP_PROBES_AND_TIMEOUT_MAC_IP": # verify IP is timed out in step 3 and mac in step 4 if step.step_id == 3: ep_ip = learn.EpIp(tc.config.localmapping) return not learn.ExistsOnDevice(ep_ip) elif step.step_id == 4: ep_mac = learn.EpMac(tc.config.localmapping) return not learn.ExistsOnDevice(ep_mac) else: return True elif tc.module.name == "CLEAR_MAC": # clear the learnt MAC ep_mac = learn.EpMac(tc.config.localmapping) return learn.ClearOnDevice(ep_mac) and \ not learn.ExistsOnDevice(ep_mac) and \ learn.ClearLearnStatistics() elif tc.module.name == "LEARN_MAC_WITH_RARP": ep_mac = learn.EpMac(tc.config.localmapping) if learn.ExistsOnDevice(ep_mac): stats = learn.GetLearnStatistics() if not stats: return False return stats['pktsrcvd'] == 1 and \ stats['maclearnevents'][1]['count'] == 1 return False return True
def UpdateHostInterfaces(self, node, subnets, dissociate=False): if utils.IsDryRun() or not utils.IsNetAgentMode(): return for subnet in subnets: if len(subnet.HostIfIdx) == 0: continue hostif = self.FindHostInterface(node, subnet.HostIfIdx[0]) if hostif: hostif.UpdateVrfAndNetwork(subnet, dissociate) hostif.Show() api.client[node].Update(api.ObjectTypes.INTERFACE, [hostif]) print("after update pf") resp = api.client[node].GetHttp(api.ObjectTypes.INTERFACE) for r in resp: print(r) return
def ValidateGrpcRead(self, node, getResp): if utils.IsDryRun(): return True device = self.GetObjectByKey(node, 0) for obj in getResp: if not utils.ValidateGrpcResponse(obj): if device.IsHwHabitant(): logger.error(f"GRPC get request failed for {device} with {obj}") device.Show() return False return True resp = obj.Response if not utils.ValidateObject(device, resp): logger.error("GRPC read validation failed for ", obj) device.Show() return False return True
def UpdateImplicit(self): if not utils.IsNetAgentMode(): return if utils.IsDol() and utils.IsDryRun(): return if not self.IsOriginImplicitlyCreated(): return # We need to read info from naples and update the DS resp = api.client[self.Node].GetHttp(self.ObjType) if not resp: return for ifinst in resp: if self.Type == topo.InterfaceTypes.L3: if (not ifinst['spec']['type'] == 'L3'): continue riid = ifinst['meta']['name'] if (self.InterfaceId != int(riid[len(riid) - 1])): continue elif self.Type == topo.InterfaceTypes.LOOPBACK: if (not ifinst['spec']['type'] == 'LOOPBACK'): continue elif self.Type == topo.InterfaceTypes.ETH: if (not ifinst['spec']['type'] == 'HOST_PF'): continue if (ifinst['status']['if-host-status']['host-ifname'] != self.IfName): continue else: continue # Found matching interface, get basic info uuid_str = ifinst['meta']['uuid'] self.UUID = utils.PdsUuid(bytes.fromhex(uuid_str.replace('-','')),\ self.ObjType) # instance found. Store meta info self.Tenant = ifinst['meta']['tenant'] self.Namespace = ifinst['meta']['namespace'] self.SetIfNameFromAgentData(ifinst['meta']['name']) # get ifinfo if 'ip-address' in ifinst['spec']: self.IpPrefix = ipaddress.ip_network(ifinst['spec']['ip-address'],\ False) if 'vrf-name' in ifinst['spec']: self.VrfName = ifinst['spec']['vrf-name'] return
def ReadHostInterfaceInfo(self): if utils.IsDryRun(): return # reads nicmgr.log and get interface info nicmgrlog = utils.GetNicmgrLogPath() f = open(nicmgrlog, "r") for ifobj in self.HostIfs.values(): # get lifbase and count pattern = ' %s: lif_base (\w+) lif_count (\w+)' % (ifobj.IfName) for line in f: match = re.search(pattern, line) if match is None: continue lifbase = int(match.groups()[0]) lifcount = int(match.groups()[1]) ifobj.SetLifBase(lifbase) ifobj.SetLifCount(lifcount) break # get devcmd addr pattern = ' %s: regs_mem_addr (\w+) devcmd_mem_addr (\w+)' % ( ifobj.IfName) for line in f: match = re.search(pattern, line) if match is None: continue devcmdaddr = int(match.groups()[1], base=16) ifobj.SetDevCmdAddr(devcmdaddr) break # get qstate addr for all lifs under this device for i in range(lifcount): lif_id = lifbase + i pattern = ' lif-%d: qtype: (\d{1}), qstate_base: (\w+)' % ( lif_id) for line in f: match = re.search(pattern, line) if match is None: continue q_type = int(match.groups()[0]) qstate_base = int(match.groups()[1], base=16) qstateaddr_list = ifobj.Lif2QstateMap.get(lif_id, []) qstateaddr_list.insert(q_type, qstate_base) ifobj.Lif2QstateMap.update({lif_id: qstateaddr_list}) if (q_type == 7): break f.close() return
def VerifyUpgradeDoneStatus(self, spec=None): if utils.IsDryRun(): return True retry = 1200 while retry: with open("/update/pds_upg_status.txt", 'r') as fp: status = fp.read() logger.info("upgrade status %s" % status) if status.find("success") != -1: return True if status.find("failed") != -1: if self.failure_stage != None: return True return False retry = retry - 1 utils.Sleep(10) return False
def ExecuteVPPCommand(command, args=None): vppctl = __get_vppctl_path() if args: vppctlproc = [vppctl] + args.split() + [command] else: vppctlproc = [vppctl, command] retval = True output = "" try: if not utils.IsDryRun(): output = str( subprocess.check_output(vppctlproc, stderr=subprocess.STDOUT)) except subprocess.CalledProcessError as e: output = "Command execution failed." retval = False logger.info("VPP: command[%s], output[%s], retval[%d]" \ %(command, output, retval)) return retval, output
def ValidatePdsctlRead(self, node, ret, stdout): if utils.IsDryRun(): return True if not ret: logger.error("pdsctl show cmd failed for ", self.ObjType) return False # split output per object cmdop = stdout.split("---") assert((len(cmdop) - 1) == self.GetNumHwObjects(node)) for op in cmdop: yamlOp = utils.LoadYaml(op) if not yamlOp: continue key = self.GetKeyfromSpec(yamlOp['spec'], yaml=True) cfgObj = self.GetObjectByKey(node, key) if not utils.ValidateObject(cfgObj, yamlOp, yaml=True): logger.error("pdsctl read validation failed for ", op) cfgObj.Show() return False return True