def setUp(self):
'''
get all parameters
'''
self.option = self.params.get("test_opt", default='')
self.tool = self.params.get("tool", default='')
self.warn_msg = self.params.get("warn_msg", default='')
self.pci_device = self.params.get("pci_device", default=None)
if self.pci_device:
if 'LOC_CODE' in self.option:
location_code = pci.get_slot_from_sysfs(self.pci_device)
self.option = self.option.replace('LOC_CODE', location_code)
if 'INTERFACE' in self.option:
adapter_type = pci.get_pci_class_name(self.pci_device)
interface = pci.get_interfaces_in_pci_address(
self.pci_device, adapter_type)[0]
self.option = self.option.replace('INTERFACE', interface)
if 'DEVICE_PATH_NAME' in self.option:
adapter_type = pci.get_pci_class_name(self.pci_device)
interface = pci.get_interfaces_in_pci_address(
self.pci_device, adapter_type)[0]
path = '/sys/class/net/%s/device/uevent' % interface
output = open(path, 'r').read()
for line in output.splitlines():
if "OF_FULLNAME" in line:
device_path_name = line.split('=')[-1]
self.option = self.option.replace('DEVICE_PATH_NAME',
device_path_name)
smm = SoftwareManager()
if not smm.check_installed("pciutils") and not smm.install("pciutils"):
self.cancel("pciutils package is need to test")
def test_eeh_basic_pe(self):
"""
Test to execute basic error injection on PE
"""
for self.addr in self.pci_addr:
enter_loop = True
num_of_miss = 0
num_of_hit = 0
self.pci_mem_addr = pci.get_memory_address(self.addr)
self.pci_mask = pci.get_mask(self.addr)
self.pci_class_name = pci.get_pci_class_name(self.addr)
if self.pci_class_name == 'fc_host':
self.pci_class_name = 'scsi_host'
self.pci_interface = pci.get_interfaces_in_pci_address(
self.addr, self.pci_class_name)[-1]
self.log.info("PCI addr = %s" % self.addr)
self.log.info("PCI mem_addr = %s" % self.pci_mem_addr)
self.log.info("PCI mask = %s" % self.pci_mask)
self.log.info("PCI class name = %s" % self.pci_class_name)
self.log.info("PCI interface = %s" % self.pci_interface)
while num_of_hit <= self.max_freeze:
for func in self.function:
self.log.info("Running error inject on pe %s function %s"
% (self.addr, func))
if num_of_miss < 5:
return_code = self.basic_eeh(func,
self.pci_class_name,
self.pci_interface,
self.pci_mem_addr,
self.pci_mask,
self.add_cmd)
if return_code == EEH_MISS:
num_of_miss += 1
self.log.info("number of miss is %d"
% num_of_miss)
continue
else:
num_of_hit += 1
self.log.info("number of hit is %d"
% num_of_hit)
if num_of_hit <= self.max_freeze:
if not self.check_eeh_pe_recovery(self.addr):
self.fail("PE %s recovery failed after"
"%d EEH" % (self.addr,
num_of_hit))
break
else:
self.log.info("PE recovered successfully")
else:
self.log.warning("EEH inject failed for 5 times with\
function %s" % func)
enter_loop = False
break
if not enter_loop:
break
else:
if self.check_eeh_removed():
self.log.info("PE %s removed successfully" % self.addr)
else:
self.fail("PE %s not removed after max hit" % self.addr)
def is_recovered():
"""
Checks if the block device adapter is recovers all its disks/paths
properly after hotplug of adapter.
Returns True if all disks/paths back online after adapter added
Back, else False.
"""
def is_path_online():
path_stat = list(multipath.get_path_status(curr_path))
if path_stat[0] != 'active' or path_stat[2] != 'ready':
return False
return True
curr_path = ''
err_disks = []
if pci.get_pci_class_name(pci_addr) == 'fc_host':
disks = pci.get_disks_in_pci_address(pci_addr)
for disk in disks:
curr_path = disk.split("/")[-1]
self.log.info("curr_path=%s" % curr_path)
if not wait.wait_for(is_path_online, timeout=10):
self.log.info("%s failed to recover after add" % disk)
err_disks.append(disk)
if err_disks:
self.log.info("few paths failed to recover : %s" % err_disks)
return False
return True
def test_eeh_basic_pe(self):
"""
Test to execute basic error injection on PE
"""
for self.addr in pci.get_pci_addresses():
self.pci_mem_addr = pci.get_memory_address(self.addr)
self.pci_mask = pci.get_mask(self.addr)
self.pci_class_name = pci.get_pci_class_name(self.addr)
self.pci_interface = pci.get_interfaces_in_pci_address(
self.addr, self.pci_class_name)[-1]
self.log.info("PCI addr = %s" % self.addr)
self.log.info("PCI mem_addr = %s" % self.pci_mem_addr)
self.log.info("PCI mask = %s" % self.pci_mask)
self.log.info("PCI class name = %s" % self.pci_class_name)
self.log.info("PCI interface = %s" % self.pci_interface)
for func in self.function:
self.log.info("Running error injection on pe %s function %s" %
(self.addr, func))
self.return_code = self.basic_eeh(func, self.pci_class_name,
self.pci_interface,
self.pci_mem_addr,
self.pci_mask)
if self.return_code == EEH_MISS:
continue
else:
if not self.check_eeh_pe_recovery(self.addr):
self.fail("PE %s recovery failed after first EEH" %
self.addr)
break
self.log.info("PE recovered successfully")
def test_eeh_basic_pe(self):
"""
Test to execute basic error injection on PE
"""
for self.addr in pci.get_pci_addresses():
enter_loop = True
num_of_miss = 0
num_of_hit = 0
self.pci_mem_addr = pci.get_memory_address(self.addr)
self.pci_mask = pci.get_mask(self.addr)
self.pci_class_name = pci.get_pci_class_name(self.addr)
self.pci_interface = pci.get_interfaces_in_pci_address(
self.addr, self.pci_class_name)[-1]
self.log.info("PCI addr = %s" % self.addr)
self.log.info("PCI mem_addr = %s" % self.pci_mem_addr)
self.log.info("PCI mask = %s" % self.pci_mask)
self.log.info("PCI class name = %s" % self.pci_class_name)
self.log.info("PCI interface = %s" % self.pci_interface)
while num_of_hit <= self.max_freeze:
for func in self.function:
self.log.info("Running error inject on pe %s function %s"
% (self.addr, func))
if num_of_miss < 5:
return_code = self.basic_eeh(func,
self.pci_class_name,
self.pci_interface,
self.pci_mem_addr,
self.pci_mask)
if return_code == EEH_MISS:
num_of_miss += 1
self.log.info("number of miss is %d"
% num_of_miss)
continue
else:
num_of_hit += 1
self.log.info("number of hit is %d"
% num_of_hit)
if num_of_hit <= self.max_freeze:
if not self.check_eeh_pe_recovery(self.addr):
self.fail("PE %s recovery failed after"
"%d EEH" % (self.addr,
num_of_hit))
break
else:
self.log.info("PE recovered successfully")
else:
self.log.warning("EEH inject failed for 5 times with\
function %s" % func)
enter_loop = False
break
if not enter_loop:
break
else:
if self.check_eeh_removed():
self.log.info("PE %s removed successfully" % self.addr)
else:
self.fail("PE %s not removed after max hit" % self.addr)
def verify_port_toggle_host(self, status):
"""
Verifies port enable/disable status change in host
side for corresponding fc adapter
"""
for bus_id in self.pci_bus_addrs:
pci_class = pci.get_pci_class_name(bus_id)
intf = pci.get_interfaces_in_pci_address(bus_id, pci_class)[-1]
state = genio.read_file("/sys/class/fc_host/%s/port_state" %
intf).rstrip("\n")
self.log.info("Host bus: %s, state: %s", bus_id, state)
if state == status:
self.log.info("bus: %s, port status %s got reflected", bus_id,
state)
else:
self.fail(
"port state not changed in host expected state: %s, \
actual_state: %s", status, state)
def setUp(self):
"""
Gets the console and set-up the machine for test
"""
if 'ppc' not in distro.detect().arch:
self.cancel("Processor is not ppc64")
eeh_enable_file = "/sys/kernel/debug/powerpc/eeh_enable"
if '0x1' not in genio.read_file(eeh_enable_file).strip():
self.cancel("EEH is not enabled, please enable via FSP")
self.max_freeze = self.params.get('max_freeze', default=1)
self.pci_device = self.params.get('pci_device', default="")
self.add_cmd = self.params.get('additional_command', default='')
if not self.pci_device:
self.cancel("No PCI Device specified")
self.function = str(self.params.get('function')).split(" ")
smm = SoftwareManager()
if not smm.check_installed("pciutils") and not smm.install("pciutils"):
self.cancel("pciutils package is need to test")
self.mem_addr = pci.get_memory_address(self.pci_device)
self.mask = pci.get_mask(self.pci_device)
if self.is_baremetal():
cmd = "echo %d > /sys/kernel/debug/powerpc/eeh_max_freezes"\
% self.max_freeze
process.system(cmd, ignore_status=True, shell=True)
self.phb = self.pci_device.split(":", 1)[0]
self.addr = genio.read_file("/sys/bus/pci/devices/%s/"
"eeh_pe_config_addr" % self.pci_device)
self.addr = str(self.addr).rstrip()
self.err = 0
for line in process.system_output('lspci -vs %s' % self.pci_device,
ignore_status=True,
shell=True).decode("utf-8\
").splitlines():
if 'Memory' in line and '64-bit, prefetchable' in line:
self.err = 1
break
else:
self.pci_class_name = pci.get_pci_class_name(self.pci_device)
if self.pci_class_name == 'fc_host':
self.pci_class_name = 'scsi_host'
self.pci_interface = pci.get_interfaces_in_pci_address(
self.pci_device, self.pci_class_name)[-1]
self.log.info("===============Testing EEH Frozen PE==================")
def hotplug_add(self, slot, pci_addr):
"""
Hot plug add operation and recovery check
"""
genio.write_file("/sys/bus/pci/slots/%s/power" % slot, "1")
def is_added():
"""
Returns True if pci device is added, False otherwise.
"""
if pci_addr not in pci.get_pci_addresses():
return False
return True
def is_recovered():
"""
Checks if the block device adapter is recovers all its disks/paths
properly after hotplug of adapter.
Returns True if all disks/paths back online after adapter added
Back, else False.
"""
def is_path_online():
path_stat = list(multipath.get_path_status(curr_path))
if path_stat[0] != 'active' or path_stat[2] != 'ready':
return False
return True
curr_path = ''
err_disks = []
if pci.get_pci_class_name(pci_addr) == 'fc_host':
disks = pci.get_disks_in_pci_address(pci_addr)
for disk in disks:
curr_path = disk.split("/")[-1]
self.log.info("curr_path=%s" % curr_path)
if not wait.wait_for(is_path_online, timeout=10):
self.log.info("%s failed to recover after add" % disk)
err_disks.append(disk)
if err_disks:
self.log.info("few paths failed to recover : %s" % err_disks)
return False
return True
def net_recovery_check():
"""
Checks if the network adapter fuctionality like ping/link_state,
after adapter added back.
Returns True on propper Recovery, False if not.
"""
self.log.info("entering the net recovery check")
local = LocalHost()
iface = pci.get_interfaces_in_pci_address(pci_addr, 'net')
networkinterface = NetworkInterface(iface[0], local)
if wait.wait_for(networkinterface.is_link_up, timeout=120):
if networkinterface.ping_check(self.peer_ip, count=5) is None:
self.log.info("inteface is up and pinging")
return True
return False
if wait.wait_for(is_added, timeout=30):
time.sleep(45)
if pci.get_pci_class_name(pci_addr) == 'net':
if wait.wait_for(net_recovery_check, timeout=30):
return True
return False
else:
if wait.wait_for(is_recovered, timeout=30):
return True
return False