def is_added(): """ Returns True if pci device is added, False otherwise. """ if pci_addr not in pci.get_pci_addresses(): return False return True
def setUp(self): ''' Function for preliminary set-up to execute the test ''' self.err_paths = [] self.device_list = [] smm = SoftwareManager() for pkg in ["lsscsi", "pciutils"]: if not smm.check_installed(pkg) and not smm.install(pkg): self.cancel("%s is not installed" % pkg) self.wwids = self.params.get('wwids', default='') self.pci_device = self.params.get("pci_devices", default='') self.count = int(self.params.get("count", default=1)) system_pci_adress = pci.get_pci_addresses() system_wwids = multipath.get_multipath_wwids() if self.wwids: self.wwids = self.wwids.split(' ') for wwid in self.wwids: if wwid not in system_wwids: self.cancel("%s not present in the system", wwid) for path in multipath.get_paths(wwid): self.device_list.append(path) elif self.pci_device: self.pci_device = self.pci_device.split(' ') for pci_id in self.pci_device: if pci_id not in system_pci_adress: self.cancel("%s not present in the system", pci_id) cmd = "ls -l /dev/disk/by-path/" output = process.system_output(cmd).decode('utf-8') for line in output.splitlines(): if pci_id in line and 'part' not in line: self.device_list.append(line.split('/')[-1]) else: self.cancel("please provide pci adrees or wwids of scsi disk")
def check_eeh_pe_recovery(self, addr): """ Check if the PE is recovered successfully after injecting EEH """ cmd = "dmesg | grep -i 'EEH: Notify device driver to resume'; echo $?" tries = 60 for _ in range(0, tries): res = process.system_output(cmd, ignore_status=True, shell=True) if int(res[-1]) != 0: self.log.info("waiting for PE to recover %s" % self.addr) time.sleep(1) else: # EEH Recovery is not similar for all adapters. For some # adapters, specifically multipath, we see that the adapter # needs some more time to recover after the message "Notify # device driver to resume" on the dmesg. # There is no reliable way to determine this extra time # required, nor a way to determine the recovery. So, a sleep # time of 10s is introduced. time.sleep(10) break else: raise EEHRecoveryFailed("EEH recovery failed", addr) tries = 30 for _ in range(0, tries): for device in pci.get_pci_addresses(): if self.addr in device: return True time.sleep(1) return False
def test_eeh_basic_pe(self): """ Test to execute basic error injection on PE """ for self.addr in pci.get_pci_addresses(): self.pci_mem_addr = pci.get_memory_address(self.addr) self.pci_mask = pci.get_mask(self.addr) self.pci_class_name = pci.get_pci_class_name(self.addr) self.pci_interface = pci.get_interfaces_in_pci_address( self.addr, self.pci_class_name)[-1] self.log.info("PCI addr = %s" % self.addr) self.log.info("PCI mem_addr = %s" % self.pci_mem_addr) self.log.info("PCI mask = %s" % self.pci_mask) self.log.info("PCI class name = %s" % self.pci_class_name) self.log.info("PCI interface = %s" % self.pci_interface) for func in self.function: self.log.info("Running error injection on pe %s function %s" % (self.addr, func)) self.return_code = self.basic_eeh(func, self.pci_class_name, self.pci_interface, self.pci_mem_addr, self.pci_mask) if self.return_code == EEH_MISS: continue else: if not self.check_eeh_pe_recovery(self.addr): self.fail("PE %s recovery failed after first EEH" % self.addr) break self.log.info("PE recovered successfully")
def check_eeh_pe_recovery(self): """ Check if the PE is recovered successfully after injecting EEH """ cmd = "dmesg" tries = 60 for _ in range(0, tries): res = process.run(cmd, ignore_status=True, shell=True) if 'EEH: Notify device driver to resume' in res.stdout.decode("utf-8") and \ res.exit_status != 0: self.log.info("waiting for PE to recover %s", self.pci_device) time.sleep(1) else: # EEH Recovery is not similar for all adapters. For some # adapters, specifically multipath, we see that the adapter # needs some more time to recover after the message "Notify # device driver to resume" on the dmesg. # There is no reliable way to determine this extra time # required, nor a way to determine the recovery. So, a sleep # time of 10s is introduced. time.sleep(10) break else: raise EEHRecoveryFailed("EEH recovery failed", self.pci_device) tries = 30 for _ in range(0, tries): for device in pci.get_pci_addresses(): if self.pci_device in device: return True time.sleep(1) return False
def test_eeh_basic_pe(self): """ Test to execute basic error injection on PE """ for self.addr in pci.get_pci_addresses(): enter_loop = True num_of_miss = 0 num_of_hit = 0 self.pci_mem_addr = pci.get_memory_address(self.addr) self.pci_mask = pci.get_mask(self.addr) self.pci_class_name = pci.get_pci_class_name(self.addr) self.pci_interface = pci.get_interfaces_in_pci_address( self.addr, self.pci_class_name)[-1] self.log.info("PCI addr = %s" % self.addr) self.log.info("PCI mem_addr = %s" % self.pci_mem_addr) self.log.info("PCI mask = %s" % self.pci_mask) self.log.info("PCI class name = %s" % self.pci_class_name) self.log.info("PCI interface = %s" % self.pci_interface) while num_of_hit <= self.max_freeze: for func in self.function: self.log.info("Running error inject on pe %s function %s" % (self.addr, func)) if num_of_miss < 5: return_code = self.basic_eeh(func, self.pci_class_name, self.pci_interface, self.pci_mem_addr, self.pci_mask) if return_code == EEH_MISS: num_of_miss += 1 self.log.info("number of miss is %d" % num_of_miss) continue else: num_of_hit += 1 self.log.info("number of hit is %d" % num_of_hit) if num_of_hit <= self.max_freeze: if not self.check_eeh_pe_recovery(self.addr): self.fail("PE %s recovery failed after" "%d EEH" % (self.addr, num_of_hit)) break else: self.log.info("PE recovered successfully") else: self.log.warning("EEH inject failed for 5 times with\ function %s" % func) enter_loop = False break if not enter_loop: break else: if self.check_eeh_removed(): self.log.info("PE %s removed successfully" % self.addr) else: self.fail("PE %s not removed after max hit" % self.addr)
def hotplug_remove(self): """ Hot Plug remove operation """ genio.write_file("/sys/bus/pci/slots/%s/power" % self.slot, "0") time.sleep(5) if self.device in pci.get_pci_addresses(): self.return_code = 1 else: self.log.info("Adapter %s removed successfully", self.device)
def hotplug_add(self): """ Hot plug add operation """ genio.write_file("/sys/bus/pci/slots/%s/power" % self.slot, "1") time.sleep(5) if self.device not in pci.get_pci_addresses(): self.return_code = 2 else: self.log.info("Adapter %s added back successfully", self.device)
def test_eeh_basic_pe(self): """ Test to execute basic error injection on PE """ for self.addr in pci.get_pci_addresses(): enter_loop = True num_of_miss = 0 num_of_hit = 0 self.pci_mem_addr = pci.get_memory_address(self.addr) self.pci_mask = pci.get_mask(self.addr) self.pci_class_name = pci.get_pci_class_name(self.addr) self.pci_interface = pci.get_interfaces_in_pci_address( self.addr, self.pci_class_name)[-1] self.log.info("PCI addr = %s" % self.addr) self.log.info("PCI mem_addr = %s" % self.pci_mem_addr) self.log.info("PCI mask = %s" % self.pci_mask) self.log.info("PCI class name = %s" % self.pci_class_name) self.log.info("PCI interface = %s" % self.pci_interface) while num_of_hit <= self.max_freeze: for func in self.function: self.log.info("Running error inject on pe %s function %s" % (self.addr, func)) if num_of_miss < 5: return_code = self.basic_eeh(func, self.pci_class_name, self.pci_interface, self.pci_mem_addr, self.pci_mask) if return_code == EEH_MISS: num_of_miss += 1 self.log.info("number of miss is %d" % num_of_miss) continue else: num_of_hit += 1 self.log.info("number of hit is %d" % num_of_hit) if num_of_hit <= self.max_freeze: if not self.check_eeh_pe_recovery(self.addr): self.fail("PE %s recovery failed after" "%d EEH" % (self.addr, num_of_hit)) break else: self.log.info("PE recovered successfully") else: self.log.warning("EEH inject failed for 5 times with\ function %s" % func) enter_loop = False break if not enter_loop: break else: if self.check_eeh_removed(): self.log.info("PE %s removed successfully" % self.addr) else: self.fail("PE %s not removed after max hit" % self.addr)
def check_eeh_pe_recovery(self, addr): """ Check if the PE is recovered successfully after injecting EEH """ cmd = "dmesg | grep -i 'EEH: Notify device driver to resume'; echo $?" tries = 60 for _ in range(0, tries): res = process.system_output(cmd, ignore_status=True, shell=True) if int(res[-1]) != 0: self.log.info("waiting for PE to recover %s" % self.addr) time.sleep(1) else: break else: raise EEHRecoveryFailed("EEH recovery failed", addr) tries = 30 for _ in range(0, tries): for device in pci.get_pci_addresses(): if self.addr in device: return True time.sleep(1) return False
def check_eeh_pe_recovery(self): """ Check if the PE is recovered successfully after injecting EEH """ cmd = "dmesg | grep -i 'EEH: Notify device driver to resume'; echo $?" tries = 60 for _ in range(0, tries): res = process.system_output(cmd, ignore_status=True, shell=True) if int(res[-1]) != 0: self.log.info("waiting for PE to recover %s" % self.pci_device) time.sleep(1) else: break else: raise EEHRecoveryFailed("EEH recovery failed", self.pci_device) tries = 30 for _ in range(0, tries): for device in pci.get_pci_addresses(): if self.pci_device in device: return True time.sleep(1) return False
def test(self): """ Begining the test here """ pci_addrs = [] if self.module: for modl in self.module.split(' '): self.mod_list.append(modl) elif self.only_io is True: for pci_addrs in pci.get_pci_addresses(): self.mod_list.append(pci.get_driver(pci_addrs)) else: cmd = "find /lib/modules/%s/ -name /*.ko" % self.uname for line in process.getoutput(cmd).splitlines(): driver = process.getoutput(line.split('/')[-1]) self.mod_list.append(driver.split('.')[0]) for mod in self.mod_list: if self.built_in_module(mod) is True: self.mod_list.remove(mod) self.log.info("\n\n final list : %s" % self.mod_list) self.module_load_unload(list(set(self.mod_list))) if self.error_modules: self.fail("Failed Modules: %s" % self.error_modules)
def test(self): ''' Test ''' error = [] for pci_addr in pci.get_pci_addresses(): self.log.info("Checking for PCI Address: %s\n\n", pci_addr) vpd_output = pci.get_vpd(pci_addr) if vpd_output: # Slot Match if 'slot' in vpd_output: sys_slot = pci.get_slot_from_sysfs(pci_addr) if sys_slot: sys_slot = sys_slot.strip('\0') vpd_slot = vpd_output['slot'] self.log.info("Slot from sysfs: %s", sys_slot) self.log.info("Slot from lsvpd: %s", vpd_slot) if sys_slot in [sys_slot, vpd_slot[:vpd_slot.rfind('-')]]: self.log.info("=======>>> slot matches perfectly\n\n") else: error.append(pci_addr + "-> slot") self.log.info("--->>Slot Numbers not Matched\n\n") else: self.log.error("Slot info not available in vpd output\n") # Device ID match sys_pci_id_output = pci.get_pci_id_from_sysfs(pci_addr) vpd_dev_id = vpd_output['pci_id'][4:] sysfs_dev_id = sys_pci_id_output[5:-10] sysfs_sdev_id = sys_pci_id_output[15:] self.log.info("Device ID from sysfs: %s", sysfs_dev_id) self.log.info("Sub Device ID from sysfs: %s", sysfs_sdev_id) self.log.info("Device ID from vpd: %s", vpd_dev_id) if vpd_dev_id == sysfs_sdev_id or vpd_dev_id == sysfs_dev_id: self.log.info("=======>>Device ID Match Success\n\n") else: self.log.error("----->>Device ID did not Match\n\n") error.append(pci_addr + "-> Device_id") # Subvendor ID Match sysfs_subvendor_id = sys_pci_id_output[10:-5] vpd_subvendor_id = vpd_output['pci_id'][:4] self.log.info("Subvendor ID frm sysfs: %s", sysfs_subvendor_id) self.log.info("Subvendor ID from vpd : %s", vpd_subvendor_id) if sysfs_subvendor_id == vpd_subvendor_id: self.log.info("======>>>Subvendor ID Match Success\n\n") else: self.log.error("---->>Subvendor_id Not Matched\n\n") error.append(pci_addr + "-> Subvendor_id") # PCI ID Match lspci_pci_id = pci.get_pci_id(pci_addr) self.log.info(" PCI ID from Sysfs: %s", sys_pci_id_output) self.log.info("PCI ID from Vpd : %s", lspci_pci_id) if sys_pci_id_output == lspci_pci_id: self.log.info("======>>>> All PCI ID match Success\n\n") else: self.log.error("---->>>PCI info Did not Matches\n\n") error.append(pci_addr + "-> pci_id") # PCI Config Space Check if process.system("lspci -xxxx -s %s" % pci_addr, ignore_status=True, sudo=True): error.append(pci_addr + "->pci_config_space") if error: self.fail("Errors for above pci addresses: %s" % error)
def is_removed(): if pci_addr in pci.get_pci_addresses(): return False return True
def is_added(): if pci_addr not in pci.get_pci_addresses(): return False return True
def test(self): ''' Test ''' error = [] for pci_addr in pci.get_pci_addresses(): self.log.info("Checking for PCI Address: %s\n\n", pci_addr) vpd_output = pci.get_vpd(pci_addr) if vpd_output: # Slot Match if 'slot' in vpd_output: sys_slot = pci.get_slot_from_sysfs(pci_addr) if sys_slot: sys_slot = sys_slot.strip('\0') vpd_slot = vpd_output['slot'] self.log.info("Slot from sysfs: %s", sys_slot) self.log.info("Slot from lsvpd: %s", vpd_slot) if sys_slot in [sys_slot, vpd_slot[:vpd_slot.rfind('-')]]: self.log.info("=======>>> slot matches perfectly\n\n") else: error.append(pci_addr + "-> slot") self.log.info("--->>Slot Numbers not Matched\n\n") else: self.log.error("Slot info not available in vpd output\n") # Device ID match sys_pci_id_output = pci.get_pci_id_from_sysfs(pci_addr) vpd_device_id = vpd_output['pci_id'][4:] sysfs_device_id = sys_pci_id_output[5:-10] self.log.info("Device ID from sysfs: %s", sysfs_device_id) self.log.info("Device ID from vpd: %s", vpd_device_id) if vpd_device_id == sysfs_device_id: self.log.info("=======>>Device ID Match Success\n\n") else: self.log.error("----->>Device ID did not Match\n\n") error.append(pci_addr + "-> Device_id") # Subvendor ID Match sysfs_subvendor_id = sys_pci_id_output[10:-5] vpd_subvendor_id = vpd_output['pci_id'][:4] self.log.info("Subvendor ID frm sysfs: %s", sysfs_subvendor_id) self.log.info("Subvendor ID from vpd : %s", vpd_subvendor_id) if sysfs_subvendor_id == vpd_subvendor_id: self.log.info("======>>>Subvendor ID Match Success\n\n") else: self.log.error("---->>Subvendor_id Not Matched\n\n") error.append(pci_addr + "-> Subvendor_id") # PCI ID Match lspci_pci_id = pci.get_pci_id(pci_addr) self.log.info(" PCI ID from Sysfs: %s", sys_pci_id_output) self.log.info("PCI ID from Vpd : %s", lspci_pci_id) if sys_pci_id_output == lspci_pci_id: self.log.info("======>>>> All PCI ID match Success\n\n") else: self.log.error("---->>>PCI info Did not Matches\n\n") error.append(pci_addr + "-> pci_id") # PCI Config Space Check if process.system("lspci -xxxx -s %s" % pci_addr, ignore_status=True, sudo=True): error.append(pci_addr + "->pci_config_space") if error: self.fail("Errors for above pci addresses: %s" % error)
def test(self): ''' Test ''' error = [] for pci_adres in pci.get_pci_addresses(): print "Comparing for PCI Adress : ", pci_adres, "\n\n" vpd_output = pci.get_vpd(pci_adres) if vpd_output: ''' Slot Match ''' if vpd_output.has_key('slot'): sys_slot = pci.get_slot_from_sysfs(pci_adres) if sys_slot: sys_slot = sys_slot.strip('\0') vpd_slot = vpd_output['slot'] self.log.info("Slot from sysfs:%s", sys_slot) self.log.info("Slot from lsvpd:%s", vpd_slot) if sys_slot in [sys_slot, vpd_slot[:vpd_slot.rfind('-')]]: self.log.info("=======>>> slot matches perfectly\n\n") else: error.append(pci_adres + "->slot") self.log.info("--->>Slot Numbers not Matched\n\n") else: self.log.error("Slot info not available in vpd output\n") ''' Device ID match ''' sys_pci_id_output = pci.get_pci_id_from_sysfs(pci_adres) vpd_device_id = vpd_output['pci_id'][4:] sysfs_device_id = sys_pci_id_output[5:-10] self.log.info("Device ID from sysfs :%s", sysfs_device_id) self.log.info("Device ID from vpd:%s", vpd_device_id) if vpd_device_id == sysfs_device_id: self.log.info("=======>>Device ID Match Success\n\n") else: self.log.error("----->>Device ID did not Match\n\n") error.append(pci_adres + "->Device_id") ''' Subvendor ID Match ''' sysfs_subvendor_id = sys_pci_id_output[10:-5] vpd_subvendor_id = vpd_output['pci_id'][:4] self.log.info("Subvendor ID frm sysfs:%s", sysfs_subvendor_id) self.log.info("Subvendor ID from vpd :%s", vpd_subvendor_id) if sysfs_subvendor_id == vpd_subvendor_id: self.log.info("======>>>Subvendor ID Match Success\n\n") else: self.log.error("---->>Subvendor_id Not Matched\n\n") error.append(pci_adres + "->Subvendor_id") ''' PCI ID Match ''' lspci_pci_id = pci.get_pci_id(pci_adres) self.log.info(" PCI ID from Sysfs:%s", sys_pci_id_output) self.log.info("PCI ID from Vpd :%s", lspci_pci_id) if sys_pci_id_output == lspci_pci_id: self.log.info("======>>>> All PCI ID match Success\n\n") else: self.log.error("---->>>PCI info Did not Matches\n\n") error.append(pci_adres + "->pci_id") if error: self.log.error("failed pci addresses:%s", error) self.fail("Data mismatched for above pci addresses:")