def setUp(self): ''' get all parameters ''' self.option = self.params.get("test_opt", default='') self.tool = self.params.get("tool", default='') self.warn_msg = self.params.get("warn_msg", default='') self.pci_device = self.params.get("pci_device", default=None) if self.pci_device: if 'LOC_CODE' in self.option: location_code = pci.get_slot_from_sysfs(self.pci_device) self.option = self.option.replace('LOC_CODE', location_code) if 'INTERFACE' in self.option: adapter_type = pci.get_pci_class_name(self.pci_device) interface = pci.get_interfaces_in_pci_address( self.pci_device, adapter_type)[0] self.option = self.option.replace('INTERFACE', interface) if 'DEVICE_PATH_NAME' in self.option: adapter_type = pci.get_pci_class_name(self.pci_device) interface = pci.get_interfaces_in_pci_address( self.pci_device, adapter_type)[0] path = '/sys/class/net/%s/device/uevent' % interface output = open(path, 'r').read() for line in output.splitlines(): if "OF_FULLNAME" in line: device_path_name = line.split('=')[-1] self.option = self.option.replace('DEVICE_PATH_NAME', device_path_name) smm = SoftwareManager() if not smm.check_installed("pciutils") and not smm.install("pciutils"): self.cancel("pciutils package is need to test")
def test_eeh_basic_pe(self): """ Test to execute basic error injection on PE """ for self.addr in self.pci_addr: enter_loop = True num_of_miss = 0 num_of_hit = 0 self.pci_mem_addr = pci.get_memory_address(self.addr) self.pci_mask = pci.get_mask(self.addr) self.pci_class_name = pci.get_pci_class_name(self.addr) if self.pci_class_name == 'fc_host': self.pci_class_name = 'scsi_host' self.pci_interface = pci.get_interfaces_in_pci_address( self.addr, self.pci_class_name)[-1] self.log.info("PCI addr = %s" % self.addr) self.log.info("PCI mem_addr = %s" % self.pci_mem_addr) self.log.info("PCI mask = %s" % self.pci_mask) self.log.info("PCI class name = %s" % self.pci_class_name) self.log.info("PCI interface = %s" % self.pci_interface) while num_of_hit <= self.max_freeze: for func in self.function: self.log.info("Running error inject on pe %s function %s" % (self.addr, func)) if num_of_miss < 5: return_code = self.basic_eeh(func, self.pci_class_name, self.pci_interface, self.pci_mem_addr, self.pci_mask, self.add_cmd) if return_code == EEH_MISS: num_of_miss += 1 self.log.info("number of miss is %d" % num_of_miss) continue else: num_of_hit += 1 self.log.info("number of hit is %d" % num_of_hit) if num_of_hit <= self.max_freeze: if not self.check_eeh_pe_recovery(self.addr): self.fail("PE %s recovery failed after" "%d EEH" % (self.addr, num_of_hit)) break else: self.log.info("PE recovered successfully") else: self.log.warning("EEH inject failed for 5 times with\ function %s" % func) enter_loop = False break if not enter_loop: break else: if self.check_eeh_removed(): self.log.info("PE %s removed successfully" % self.addr) else: self.fail("PE %s not removed after max hit" % self.addr)
def is_recovered(): """ Checks if the block device adapter is recovers all its disks/paths properly after hotplug of adapter. Returns True if all disks/paths back online after adapter added Back, else False. """ def is_path_online(): path_stat = list(multipath.get_path_status(curr_path)) if path_stat[0] != 'active' or path_stat[2] != 'ready': return False return True curr_path = '' err_disks = [] if pci.get_pci_class_name(pci_addr) == 'fc_host': disks = pci.get_disks_in_pci_address(pci_addr) for disk in disks: curr_path = disk.split("/")[-1] self.log.info("curr_path=%s" % curr_path) if not wait.wait_for(is_path_online, timeout=10): self.log.info("%s failed to recover after add" % disk) err_disks.append(disk) if err_disks: self.log.info("few paths failed to recover : %s" % err_disks) return False return True
def test_eeh_basic_pe(self): """ Test to execute basic error injection on PE """ for self.addr in pci.get_pci_addresses(): self.pci_mem_addr = pci.get_memory_address(self.addr) self.pci_mask = pci.get_mask(self.addr) self.pci_class_name = pci.get_pci_class_name(self.addr) self.pci_interface = pci.get_interfaces_in_pci_address( self.addr, self.pci_class_name)[-1] self.log.info("PCI addr = %s" % self.addr) self.log.info("PCI mem_addr = %s" % self.pci_mem_addr) self.log.info("PCI mask = %s" % self.pci_mask) self.log.info("PCI class name = %s" % self.pci_class_name) self.log.info("PCI interface = %s" % self.pci_interface) for func in self.function: self.log.info("Running error injection on pe %s function %s" % (self.addr, func)) self.return_code = self.basic_eeh(func, self.pci_class_name, self.pci_interface, self.pci_mem_addr, self.pci_mask) if self.return_code == EEH_MISS: continue else: if not self.check_eeh_pe_recovery(self.addr): self.fail("PE %s recovery failed after first EEH" % self.addr) break self.log.info("PE recovered successfully")
def test_eeh_basic_pe(self): """ Test to execute basic error injection on PE """ for self.addr in pci.get_pci_addresses(): enter_loop = True num_of_miss = 0 num_of_hit = 0 self.pci_mem_addr = pci.get_memory_address(self.addr) self.pci_mask = pci.get_mask(self.addr) self.pci_class_name = pci.get_pci_class_name(self.addr) self.pci_interface = pci.get_interfaces_in_pci_address( self.addr, self.pci_class_name)[-1] self.log.info("PCI addr = %s" % self.addr) self.log.info("PCI mem_addr = %s" % self.pci_mem_addr) self.log.info("PCI mask = %s" % self.pci_mask) self.log.info("PCI class name = %s" % self.pci_class_name) self.log.info("PCI interface = %s" % self.pci_interface) while num_of_hit <= self.max_freeze: for func in self.function: self.log.info("Running error inject on pe %s function %s" % (self.addr, func)) if num_of_miss < 5: return_code = self.basic_eeh(func, self.pci_class_name, self.pci_interface, self.pci_mem_addr, self.pci_mask) if return_code == EEH_MISS: num_of_miss += 1 self.log.info("number of miss is %d" % num_of_miss) continue else: num_of_hit += 1 self.log.info("number of hit is %d" % num_of_hit) if num_of_hit <= self.max_freeze: if not self.check_eeh_pe_recovery(self.addr): self.fail("PE %s recovery failed after" "%d EEH" % (self.addr, num_of_hit)) break else: self.log.info("PE recovered successfully") else: self.log.warning("EEH inject failed for 5 times with\ function %s" % func) enter_loop = False break if not enter_loop: break else: if self.check_eeh_removed(): self.log.info("PE %s removed successfully" % self.addr) else: self.fail("PE %s not removed after max hit" % self.addr)
def verify_port_toggle_host(self, status): """ Verifies port enable/disable status change in host side for corresponding fc adapter """ for bus_id in self.pci_bus_addrs: pci_class = pci.get_pci_class_name(bus_id) intf = pci.get_interfaces_in_pci_address(bus_id, pci_class)[-1] state = genio.read_file("/sys/class/fc_host/%s/port_state" % intf).rstrip("\n") self.log.info("Host bus: %s, state: %s", bus_id, state) if state == status: self.log.info("bus: %s, port status %s got reflected", bus_id, state) else: self.fail( "port state not changed in host expected state: %s, \ actual_state: %s", status, state)
def setUp(self): """ Gets the console and set-up the machine for test """ if 'ppc' not in distro.detect().arch: self.cancel("Processor is not ppc64") eeh_enable_file = "/sys/kernel/debug/powerpc/eeh_enable" if '0x1' not in genio.read_file(eeh_enable_file).strip(): self.cancel("EEH is not enabled, please enable via FSP") self.max_freeze = self.params.get('max_freeze', default=1) self.pci_device = self.params.get('pci_device', default="") self.add_cmd = self.params.get('additional_command', default='') if not self.pci_device: self.cancel("No PCI Device specified") self.function = str(self.params.get('function')).split(" ") smm = SoftwareManager() if not smm.check_installed("pciutils") and not smm.install("pciutils"): self.cancel("pciutils package is need to test") self.mem_addr = pci.get_memory_address(self.pci_device) self.mask = pci.get_mask(self.pci_device) if self.is_baremetal(): cmd = "echo %d > /sys/kernel/debug/powerpc/eeh_max_freezes"\ % self.max_freeze process.system(cmd, ignore_status=True, shell=True) self.phb = self.pci_device.split(":", 1)[0] self.addr = genio.read_file("/sys/bus/pci/devices/%s/" "eeh_pe_config_addr" % self.pci_device) self.addr = str(self.addr).rstrip() self.err = 0 for line in process.system_output('lspci -vs %s' % self.pci_device, ignore_status=True, shell=True).decode("utf-8\ ").splitlines(): if 'Memory' in line and '64-bit, prefetchable' in line: self.err = 1 break else: self.pci_class_name = pci.get_pci_class_name(self.pci_device) if self.pci_class_name == 'fc_host': self.pci_class_name = 'scsi_host' self.pci_interface = pci.get_interfaces_in_pci_address( self.pci_device, self.pci_class_name)[-1] self.log.info("===============Testing EEH Frozen PE==================")
def hotplug_add(self, slot, pci_addr): """ Hot plug add operation and recovery check """ genio.write_file("/sys/bus/pci/slots/%s/power" % slot, "1") def is_added(): """ Returns True if pci device is added, False otherwise. """ if pci_addr not in pci.get_pci_addresses(): return False return True def is_recovered(): """ Checks if the block device adapter is recovers all its disks/paths properly after hotplug of adapter. Returns True if all disks/paths back online after adapter added Back, else False. """ def is_path_online(): path_stat = list(multipath.get_path_status(curr_path)) if path_stat[0] != 'active' or path_stat[2] != 'ready': return False return True curr_path = '' err_disks = [] if pci.get_pci_class_name(pci_addr) == 'fc_host': disks = pci.get_disks_in_pci_address(pci_addr) for disk in disks: curr_path = disk.split("/")[-1] self.log.info("curr_path=%s" % curr_path) if not wait.wait_for(is_path_online, timeout=10): self.log.info("%s failed to recover after add" % disk) err_disks.append(disk) if err_disks: self.log.info("few paths failed to recover : %s" % err_disks) return False return True def net_recovery_check(): """ Checks if the network adapter fuctionality like ping/link_state, after adapter added back. Returns True on propper Recovery, False if not. """ self.log.info("entering the net recovery check") local = LocalHost() iface = pci.get_interfaces_in_pci_address(pci_addr, 'net') networkinterface = NetworkInterface(iface[0], local) if wait.wait_for(networkinterface.is_link_up, timeout=120): if networkinterface.ping_check(self.peer_ip, count=5) is None: self.log.info("inteface is up and pinging") return True return False if wait.wait_for(is_added, timeout=30): time.sleep(45) if pci.get_pci_class_name(pci_addr) == 'net': if wait.wait_for(net_recovery_check, timeout=30): return True return False else: if wait.wait_for(is_recovered, timeout=30): return True return False