def is_added():
"""
Returns True if pci device is added, False otherwise.
"""
if pci_addr not in pci.get_pci_addresses():
return False
return True
def setUp(self):
'''
Function for preliminary set-up to execute the test
'''
self.err_paths = []
self.device_list = []
smm = SoftwareManager()
for pkg in ["lsscsi", "pciutils"]:
if not smm.check_installed(pkg) and not smm.install(pkg):
self.cancel("%s is not installed" % pkg)
self.wwids = self.params.get('wwids', default='')
self.pci_device = self.params.get("pci_devices", default='')
self.count = int(self.params.get("count", default=1))
system_pci_adress = pci.get_pci_addresses()
system_wwids = multipath.get_multipath_wwids()
if self.wwids:
self.wwids = self.wwids.split(' ')
for wwid in self.wwids:
if wwid not in system_wwids:
self.cancel("%s not present in the system", wwid)
for path in multipath.get_paths(wwid):
self.device_list.append(path)
elif self.pci_device:
self.pci_device = self.pci_device.split(' ')
for pci_id in self.pci_device:
if pci_id not in system_pci_adress:
self.cancel("%s not present in the system", pci_id)
cmd = "ls -l /dev/disk/by-path/"
output = process.system_output(cmd).decode('utf-8')
for line in output.splitlines():
if pci_id in line and 'part' not in line:
self.device_list.append(line.split('/')[-1])
else:
self.cancel("please provide pci adrees or wwids of scsi disk")
def check_eeh_pe_recovery(self, addr):
"""
Check if the PE is recovered successfully after injecting EEH
"""
cmd = "dmesg | grep -i 'EEH: Notify device driver to resume'; echo $?"
tries = 60
for _ in range(0, tries):
res = process.system_output(cmd, ignore_status=True, shell=True)
if int(res[-1]) != 0:
self.log.info("waiting for PE to recover %s" % self.addr)
time.sleep(1)
else:
# EEH Recovery is not similar for all adapters. For some
# adapters, specifically multipath, we see that the adapter
# needs some more time to recover after the message "Notify
# device driver to resume" on the dmesg.
# There is no reliable way to determine this extra time
# required, nor a way to determine the recovery. So, a sleep
# time of 10s is introduced.
time.sleep(10)
break
else:
raise EEHRecoveryFailed("EEH recovery failed", addr)
tries = 30
for _ in range(0, tries):
for device in pci.get_pci_addresses():
if self.addr in device:
return True
time.sleep(1)
return False
def test_eeh_basic_pe(self):
"""
Test to execute basic error injection on PE
"""
for self.addr in pci.get_pci_addresses():
self.pci_mem_addr = pci.get_memory_address(self.addr)
self.pci_mask = pci.get_mask(self.addr)
self.pci_class_name = pci.get_pci_class_name(self.addr)
self.pci_interface = pci.get_interfaces_in_pci_address(
self.addr, self.pci_class_name)[-1]
self.log.info("PCI addr = %s" % self.addr)
self.log.info("PCI mem_addr = %s" % self.pci_mem_addr)
self.log.info("PCI mask = %s" % self.pci_mask)
self.log.info("PCI class name = %s" % self.pci_class_name)
self.log.info("PCI interface = %s" % self.pci_interface)
for func in self.function:
self.log.info("Running error injection on pe %s function %s" %
(self.addr, func))
self.return_code = self.basic_eeh(func, self.pci_class_name,
self.pci_interface,
self.pci_mem_addr,
self.pci_mask)
if self.return_code == EEH_MISS:
continue
else:
if not self.check_eeh_pe_recovery(self.addr):
self.fail("PE %s recovery failed after first EEH" %
self.addr)
break
self.log.info("PE recovered successfully")
def check_eeh_pe_recovery(self):
"""
Check if the PE is recovered successfully after injecting EEH
"""
cmd = "dmesg"
tries = 60
for _ in range(0, tries):
res = process.run(cmd, ignore_status=True, shell=True)
if 'EEH: Notify device driver to resume' in res.stdout.decode("utf-8") and \
res.exit_status != 0:
self.log.info("waiting for PE to recover %s", self.pci_device)
time.sleep(1)
else:
# EEH Recovery is not similar for all adapters. For some
# adapters, specifically multipath, we see that the adapter
# needs some more time to recover after the message "Notify
# device driver to resume" on the dmesg.
# There is no reliable way to determine this extra time
# required, nor a way to determine the recovery. So, a sleep
# time of 10s is introduced.
time.sleep(10)
break
else:
raise EEHRecoveryFailed("EEH recovery failed", self.pci_device)
tries = 30
for _ in range(0, tries):
for device in pci.get_pci_addresses():
if self.pci_device in device:
return True
time.sleep(1)
return False
def test_eeh_basic_pe(self):
"""
Test to execute basic error injection on PE
"""
for self.addr in pci.get_pci_addresses():
enter_loop = True
num_of_miss = 0
num_of_hit = 0
self.pci_mem_addr = pci.get_memory_address(self.addr)
self.pci_mask = pci.get_mask(self.addr)
self.pci_class_name = pci.get_pci_class_name(self.addr)
self.pci_interface = pci.get_interfaces_in_pci_address(
self.addr, self.pci_class_name)[-1]
self.log.info("PCI addr = %s" % self.addr)
self.log.info("PCI mem_addr = %s" % self.pci_mem_addr)
self.log.info("PCI mask = %s" % self.pci_mask)
self.log.info("PCI class name = %s" % self.pci_class_name)
self.log.info("PCI interface = %s" % self.pci_interface)
while num_of_hit <= self.max_freeze:
for func in self.function:
self.log.info("Running error inject on pe %s function %s"
% (self.addr, func))
if num_of_miss < 5:
return_code = self.basic_eeh(func,
self.pci_class_name,
self.pci_interface,
self.pci_mem_addr,
self.pci_mask)
if return_code == EEH_MISS:
num_of_miss += 1
self.log.info("number of miss is %d"
% num_of_miss)
continue
else:
num_of_hit += 1
self.log.info("number of hit is %d"
% num_of_hit)
if num_of_hit <= self.max_freeze:
if not self.check_eeh_pe_recovery(self.addr):
self.fail("PE %s recovery failed after"
"%d EEH" % (self.addr,
num_of_hit))
break
else:
self.log.info("PE recovered successfully")
else:
self.log.warning("EEH inject failed for 5 times with\
function %s" % func)
enter_loop = False
break
if not enter_loop:
break
else:
if self.check_eeh_removed():
self.log.info("PE %s removed successfully" % self.addr)
else:
self.fail("PE %s not removed after max hit" % self.addr)
def hotplug_remove(self):
"""
Hot Plug remove operation
"""
genio.write_file("/sys/bus/pci/slots/%s/power" % self.slot, "0")
time.sleep(5)
if self.device in pci.get_pci_addresses():
self.return_code = 1
else:
self.log.info("Adapter %s removed successfully", self.device)
def hotplug_add(self):
"""
Hot plug add operation
"""
genio.write_file("/sys/bus/pci/slots/%s/power" % self.slot, "1")
time.sleep(5)
if self.device not in pci.get_pci_addresses():
self.return_code = 2
else:
self.log.info("Adapter %s added back successfully", self.device)
def test_eeh_basic_pe(self):
"""
Test to execute basic error injection on PE
"""
for self.addr in pci.get_pci_addresses():
enter_loop = True
num_of_miss = 0
num_of_hit = 0
self.pci_mem_addr = pci.get_memory_address(self.addr)
self.pci_mask = pci.get_mask(self.addr)
self.pci_class_name = pci.get_pci_class_name(self.addr)
self.pci_interface = pci.get_interfaces_in_pci_address(
self.addr, self.pci_class_name)[-1]
self.log.info("PCI addr = %s" % self.addr)
self.log.info("PCI mem_addr = %s" % self.pci_mem_addr)
self.log.info("PCI mask = %s" % self.pci_mask)
self.log.info("PCI class name = %s" % self.pci_class_name)
self.log.info("PCI interface = %s" % self.pci_interface)
while num_of_hit <= self.max_freeze:
for func in self.function:
self.log.info("Running error inject on pe %s function %s" %
(self.addr, func))
if num_of_miss < 5:
return_code = self.basic_eeh(func, self.pci_class_name,
self.pci_interface,
self.pci_mem_addr,
self.pci_mask)
if return_code == EEH_MISS:
num_of_miss += 1
self.log.info("number of miss is %d" % num_of_miss)
continue
else:
num_of_hit += 1
self.log.info("number of hit is %d" % num_of_hit)
if num_of_hit <= self.max_freeze:
if not self.check_eeh_pe_recovery(self.addr):
self.fail("PE %s recovery failed after"
"%d EEH" %
(self.addr, num_of_hit))
break
else:
self.log.info("PE recovered successfully")
else:
self.log.warning("EEH inject failed for 5 times with\
function %s" % func)
enter_loop = False
break
if not enter_loop:
break
else:
if self.check_eeh_removed():
self.log.info("PE %s removed successfully" % self.addr)
else:
self.fail("PE %s not removed after max hit" % self.addr)
def check_eeh_pe_recovery(self, addr):
"""
Check if the PE is recovered successfully after injecting EEH
"""
cmd = "dmesg | grep -i 'EEH: Notify device driver to resume'; echo $?"
tries = 60
for _ in range(0, tries):
res = process.system_output(cmd, ignore_status=True, shell=True)
if int(res[-1]) != 0:
self.log.info("waiting for PE to recover %s" % self.addr)
time.sleep(1)
else:
break
else:
raise EEHRecoveryFailed("EEH recovery failed", addr)
tries = 30
for _ in range(0, tries):
for device in pci.get_pci_addresses():
if self.addr in device:
return True
time.sleep(1)
return False
def check_eeh_pe_recovery(self):
"""
Check if the PE is recovered successfully after injecting EEH
"""
cmd = "dmesg | grep -i 'EEH: Notify device driver to resume'; echo $?"
tries = 60
for _ in range(0, tries):
res = process.system_output(cmd, ignore_status=True, shell=True)
if int(res[-1]) != 0:
self.log.info("waiting for PE to recover %s" % self.pci_device)
time.sleep(1)
else:
break
else:
raise EEHRecoveryFailed("EEH recovery failed", self.pci_device)
tries = 30
for _ in range(0, tries):
for device in pci.get_pci_addresses():
if self.pci_device in device:
return True
time.sleep(1)
return False
def test(self):
"""
Begining the test here
"""
pci_addrs = []
if self.module:
for modl in self.module.split(' '):
self.mod_list.append(modl)
elif self.only_io is True:
for pci_addrs in pci.get_pci_addresses():
self.mod_list.append(pci.get_driver(pci_addrs))
else:
cmd = "find /lib/modules/%s/ -name /*.ko" % self.uname
for line in process.getoutput(cmd).splitlines():
driver = process.getoutput(line.split('/')[-1])
self.mod_list.append(driver.split('.')[0])
for mod in self.mod_list:
if self.built_in_module(mod) is True:
self.mod_list.remove(mod)
self.log.info("\n\n final list : %s" % self.mod_list)
self.module_load_unload(list(set(self.mod_list)))
if self.error_modules:
self.fail("Failed Modules: %s" % self.error_modules)
def test(self):
'''
Test
'''
error = []
for pci_addr in pci.get_pci_addresses():
self.log.info("Checking for PCI Address: %s\n\n", pci_addr)
vpd_output = pci.get_vpd(pci_addr)
if vpd_output:
# Slot Match
if 'slot' in vpd_output:
sys_slot = pci.get_slot_from_sysfs(pci_addr)
if sys_slot:
sys_slot = sys_slot.strip('\0')
vpd_slot = vpd_output['slot']
self.log.info("Slot from sysfs: %s", sys_slot)
self.log.info("Slot from lsvpd: %s", vpd_slot)
if sys_slot in [sys_slot, vpd_slot[:vpd_slot.rfind('-')]]:
self.log.info("=======>>> slot matches perfectly\n\n")
else:
error.append(pci_addr + "-> slot")
self.log.info("--->>Slot Numbers not Matched\n\n")
else:
self.log.error("Slot info not available in vpd output\n")
# Device ID match
sys_pci_id_output = pci.get_pci_id_from_sysfs(pci_addr)
vpd_dev_id = vpd_output['pci_id'][4:]
sysfs_dev_id = sys_pci_id_output[5:-10]
sysfs_sdev_id = sys_pci_id_output[15:]
self.log.info("Device ID from sysfs: %s", sysfs_dev_id)
self.log.info("Sub Device ID from sysfs: %s", sysfs_sdev_id)
self.log.info("Device ID from vpd: %s", vpd_dev_id)
if vpd_dev_id == sysfs_sdev_id or vpd_dev_id == sysfs_dev_id:
self.log.info("=======>>Device ID Match Success\n\n")
else:
self.log.error("----->>Device ID did not Match\n\n")
error.append(pci_addr + "-> Device_id")
# Subvendor ID Match
sysfs_subvendor_id = sys_pci_id_output[10:-5]
vpd_subvendor_id = vpd_output['pci_id'][:4]
self.log.info("Subvendor ID frm sysfs: %s", sysfs_subvendor_id)
self.log.info("Subvendor ID from vpd : %s", vpd_subvendor_id)
if sysfs_subvendor_id == vpd_subvendor_id:
self.log.info("======>>>Subvendor ID Match Success\n\n")
else:
self.log.error("---->>Subvendor_id Not Matched\n\n")
error.append(pci_addr + "-> Subvendor_id")
# PCI ID Match
lspci_pci_id = pci.get_pci_id(pci_addr)
self.log.info(" PCI ID from Sysfs: %s", sys_pci_id_output)
self.log.info("PCI ID from Vpd : %s", lspci_pci_id)
if sys_pci_id_output == lspci_pci_id:
self.log.info("======>>>> All PCI ID match Success\n\n")
else:
self.log.error("---->>>PCI info Did not Matches\n\n")
error.append(pci_addr + "-> pci_id")
# PCI Config Space Check
if process.system("lspci -xxxx -s %s" % pci_addr,
ignore_status=True, sudo=True):
error.append(pci_addr + "->pci_config_space")
if error:
self.fail("Errors for above pci addresses: %s" % error)
def is_removed():
if pci_addr in pci.get_pci_addresses():
return False
return True
def is_added():
if pci_addr not in pci.get_pci_addresses():
return False
return True
def test(self):
'''
Test
'''
error = []
for pci_addr in pci.get_pci_addresses():
self.log.info("Checking for PCI Address: %s\n\n", pci_addr)
vpd_output = pci.get_vpd(pci_addr)
if vpd_output:
# Slot Match
if 'slot' in vpd_output:
sys_slot = pci.get_slot_from_sysfs(pci_addr)
if sys_slot:
sys_slot = sys_slot.strip('\0')
vpd_slot = vpd_output['slot']
self.log.info("Slot from sysfs: %s", sys_slot)
self.log.info("Slot from lsvpd: %s", vpd_slot)
if sys_slot in [sys_slot, vpd_slot[:vpd_slot.rfind('-')]]:
self.log.info("=======>>> slot matches perfectly\n\n")
else:
error.append(pci_addr + "-> slot")
self.log.info("--->>Slot Numbers not Matched\n\n")
else:
self.log.error("Slot info not available in vpd output\n")
# Device ID match
sys_pci_id_output = pci.get_pci_id_from_sysfs(pci_addr)
vpd_device_id = vpd_output['pci_id'][4:]
sysfs_device_id = sys_pci_id_output[5:-10]
self.log.info("Device ID from sysfs: %s", sysfs_device_id)
self.log.info("Device ID from vpd: %s", vpd_device_id)
if vpd_device_id == sysfs_device_id:
self.log.info("=======>>Device ID Match Success\n\n")
else:
self.log.error("----->>Device ID did not Match\n\n")
error.append(pci_addr + "-> Device_id")
# Subvendor ID Match
sysfs_subvendor_id = sys_pci_id_output[10:-5]
vpd_subvendor_id = vpd_output['pci_id'][:4]
self.log.info("Subvendor ID frm sysfs: %s", sysfs_subvendor_id)
self.log.info("Subvendor ID from vpd : %s", vpd_subvendor_id)
if sysfs_subvendor_id == vpd_subvendor_id:
self.log.info("======>>>Subvendor ID Match Success\n\n")
else:
self.log.error("---->>Subvendor_id Not Matched\n\n")
error.append(pci_addr + "-> Subvendor_id")
# PCI ID Match
lspci_pci_id = pci.get_pci_id(pci_addr)
self.log.info(" PCI ID from Sysfs: %s", sys_pci_id_output)
self.log.info("PCI ID from Vpd : %s", lspci_pci_id)
if sys_pci_id_output == lspci_pci_id:
self.log.info("======>>>> All PCI ID match Success\n\n")
else:
self.log.error("---->>>PCI info Did not Matches\n\n")
error.append(pci_addr + "-> pci_id")
# PCI Config Space Check
if process.system("lspci -xxxx -s %s" % pci_addr,
ignore_status=True,
sudo=True):
error.append(pci_addr + "->pci_config_space")
if error:
self.fail("Errors for above pci addresses: %s" % error)
def test(self):
'''
Test
'''
error = []
for pci_adres in pci.get_pci_addresses():
print "Comparing for PCI Adress : ", pci_adres, "\n\n"
vpd_output = pci.get_vpd(pci_adres)
if vpd_output:
'''
Slot Match
'''
if vpd_output.has_key('slot'):
sys_slot = pci.get_slot_from_sysfs(pci_adres)
if sys_slot:
sys_slot = sys_slot.strip('\0')
vpd_slot = vpd_output['slot']
self.log.info("Slot from sysfs:%s", sys_slot)
self.log.info("Slot from lsvpd:%s", vpd_slot)
if sys_slot in [sys_slot, vpd_slot[:vpd_slot.rfind('-')]]:
self.log.info("=======>>> slot matches perfectly\n\n")
else:
error.append(pci_adres + "->slot")
self.log.info("--->>Slot Numbers not Matched\n\n")
else:
self.log.error("Slot info not available in vpd output\n")
'''
Device ID match
'''
sys_pci_id_output = pci.get_pci_id_from_sysfs(pci_adres)
vpd_device_id = vpd_output['pci_id'][4:]
sysfs_device_id = sys_pci_id_output[5:-10]
self.log.info("Device ID from sysfs :%s", sysfs_device_id)
self.log.info("Device ID from vpd:%s", vpd_device_id)
if vpd_device_id == sysfs_device_id:
self.log.info("=======>>Device ID Match Success\n\n")
else:
self.log.error("----->>Device ID did not Match\n\n")
error.append(pci_adres + "->Device_id")
'''
Subvendor ID Match
'''
sysfs_subvendor_id = sys_pci_id_output[10:-5]
vpd_subvendor_id = vpd_output['pci_id'][:4]
self.log.info("Subvendor ID frm sysfs:%s", sysfs_subvendor_id)
self.log.info("Subvendor ID from vpd :%s", vpd_subvendor_id)
if sysfs_subvendor_id == vpd_subvendor_id:
self.log.info("======>>>Subvendor ID Match Success\n\n")
else:
self.log.error("---->>Subvendor_id Not Matched\n\n")
error.append(pci_adres + "->Subvendor_id")
'''
PCI ID Match
'''
lspci_pci_id = pci.get_pci_id(pci_adres)
self.log.info(" PCI ID from Sysfs:%s", sys_pci_id_output)
self.log.info("PCI ID from Vpd :%s", lspci_pci_id)
if sys_pci_id_output == lspci_pci_id:
self.log.info("======>>>> All PCI ID match Success\n\n")
else:
self.log.error("---->>>PCI info Did not Matches\n\n")
error.append(pci_adres + "->pci_id")
if error:
self.log.error("failed pci addresses:%s", error)
self.fail("Data mismatched for above pci addresses:")
