def hotplug_add(self, slot, pci_addr):
"""
Hot plug add operation
"""
genio.write_file("/sys/bus/pci/slots/%s/power" % slot, "1")
def is_added():
"""
Returns True if pci device is added, False otherwise.
"""
if pci_addr not in pci.get_pci_addresses():
return False
return True
def is_recovered():
"""
Compares current endpoint devices in pci address with
`pre` value, and returns True if equals pre.
False otherwise.
"""
post = len(pci.get_disks_in_pci_address(pci_addr))
post += len(pci.get_nics_in_pci_address(pci_addr))
self.log.debug("Pre: %d, Post: %d",
self.end_devices[pci_addr], post)
if post == self.end_devices[pci_addr]:
return True
return False
if not wait.wait_for(is_added, timeout=10):
return False
# Waiting for 10s per end device, for recovery.
return wait.wait_for(is_recovered,
self.end_devices[pci_addr] * 10)
def capture_vm_console_log(self, vm):
file_name = "vm_console_log_%s" % vm.name
log_file_path = os.path.join(os.environ['AVOCADO_TEST_OUTPUTDIR'],
file_name)
output = self.server_client.get_console_output(vm)
LOG.info("Capture console log for VM: %s" % vm.name)
genio.write_file(log_file_path, output)
def test(self):
'''
Function where test is executed
'''
device_list = []
cmd = "ls -l /dev/disk/by-path/"
output = process.run(cmd)
for lines in output.stdout.splitlines():
if self.pci_device in lines:
device_list.append(lines.split()[-1])
if not device_list:
self.log.warning("No devices under the given PCI device")
else:
for device_id in device_list:
device = device_id.split()[-1].strip("../*")
self.log.info("device = %s", device)
cmd = "lsscsi"
output = process.run(cmd)
for lines in output.stdout.splitlines():
if device in lines:
scsi_num = lines.split()[0].strip("[").strip("]")
self.log.info("scsi_num=%s", scsi_num)
scsi_num_seperated = scsi_num.replace(":", " ")
self.log.info("Deleting %s", scsi_num)
genio.write_file(
"/sys/block/%s/device/delete" % device, "1")
time.sleep(5)
self.log.info("%s deleted", scsi_num)
self.log.info("adding back %s", scsi_num)
process.run(
"echo scsi add-single-device %s > \
/proc/scsi/scsi", scsi_num_seperated)
time.sleep(5)
self.log.info("%s Added back", scsi_num)
def test_unbindbind(self):
"""
Performs driver unbind and bind for the Network virtualized device
"""
for device_ip, netmask, mac, peer_ip in zip(self.device_ip,
self.netmask, self.mac_id,
self.peer_ip):
device_id = self.find_device_id(mac)
try:
for _ in range(self.count):
for operation in ["unbind", "bind"]:
self.log.info(
"Running %s operation for Network \
virtualized device", operation)
genio.write_file(
os.path.join("/sys/bus/vio/drivers/ibmvnic",
operation), "%s" % device_id)
time.sleep(10)
self.log.info(
"Running a ping test to check if unbind/bind \
affected newtwork connectivity")
if not self.ping_check(device_ip, netmask, mac, peer_ip):
self.fail("Ping test failed. Network virtualized"
"unbind/bind has affected"
"Network connectivity")
except CmdError as details:
self.log.debug(str(details))
self.fail("Driver %s operation failed" % operation)
def test_clientfailover(self):
'''
Performs Client initiated failover for Network virtualized
device
'''
device_id = self.find_device_id(self.mac_id[0])
try:
for _ in range(self.count):
for val in range(int(self.backing_dev_count())):
self.log.info(
"Performing Client initiated\
failover - Attempt %s", int(val + 1))
genio.write_file(
"/sys/devices/vio/%s/failover" % device_id, "1")
time.sleep(10)
self.log.info("Running a ping test to check if failover \
affected Network connectivity")
if not self.ping_check(self.device_ip[0], self.netmask[0],
self.mac_id[0], self.peer_ip[0]):
self.fail("Ping test failed. Network virtualized \
failover has affected Network connectivity")
except CmdError as details:
self.log.debug(str(details))
self.fail("Client initiated Failover for Network virtualized \
device has failed")
def test(self):
'''
Function where test is executed
'''
self.log.info("device lists : %s " % self.device_list)
for device in self.device_list:
scsi_id = self.get_scsi_id(device)
process.run("\nlsscsi\n")
for _ in range(self.count):
self.log.info("\nDeleting %s = %s\n" % (device, scsi_id))
genio.write_file("/sys/block/%s/device/delete" % device, "1")
time.sleep(5)
if self.is_exists_scsi_device(device) is True:
self.log.info("failed to remove device: %s", device)
self.err_paths.append(device)
else:
self.log.info("\n%s = %s deleted\n" % (device, scsi_id))
self.log.info("\nadding back %s = %s\n" % (device, scsi_id))
part = "scsi add-single-device %s" % scsi_id
genio.write_file("/proc/scsi/scsi", part)
time.sleep(5)
if self.is_exists_scsi_device(device) is False:
self.log.info("failed to add Back: %s ", device)
self.err_paths.append(device)
else:
process.run("\nlsscsi\n")
self.log.info("\n%s = %s Added back\n" % (device, scsi_id))
def setUp(self):
"""
Build 'fio and ezfio'.
"""
self.disk = self.params.get('disk', default='/dev/nvme0n1')
cmd = 'ls %s' % self.disk
if process.system(cmd, ignore_status=True) is not 0:
self.cancel("%s does not exist" % self.disk)
fio_path = os.path.join(self.teststmpdir, 'fio')
fio_link = 'https://github.com/axboe/fio.git'
git.get_repo(fio_link, destination_dir=fio_path)
build.make(fio_path, make='./configure')
build.make(fio_path)
build.make(fio_path, extra_args='install')
self.ezfio_path = os.path.join(self.teststmpdir, 'ezfio')
ezfio_link = 'https://github.com/earlephilhower/ezfio.git'
git.get_repo(ezfio_link, destination_dir=self.ezfio_path)
self.utilization = self.params.get('utilization', default='100')
# aio-max-nr is 65536 by default, and test fails if QD is 256 or above
genio.write_file("/proc/sys/fs/aio-max-nr", "1048576")
smm = SoftwareManager()
# Not a package that must be installed, so not skipping.
if not smm.check_installed("sdparm") and not smm.install("sdparm"):
self.log.debug("Can not install sdparm")
pkg_list = ['libaio', 'libaio-devel']
smm = SoftwareManager()
for pkg in pkg_list:
if pkg and not smm.check_installed(pkg) and not smm.install(pkg):
self.cancel(
"Package %s is missing and could not be installed" % pkg)
self.cwd = os.getcwd()
def setUp(self):
"""
Build 'fio and ezfio'.
"""
self.disk = self.params.get('disk', default='/dev/nvme0n1')
cmd = 'ls %s' % self.disk
if process.system(cmd, ignore_status=True) is not 0:
self.cancel("%s does not exist" % self.disk)
fio_path = os.path.join(self.teststmpdir, 'fio')
fio_link = 'https://github.com/axboe/fio.git'
git.get_repo(fio_link, destination_dir=fio_path)
build.make(fio_path, make='./configure')
build.make(fio_path)
build.make(fio_path, extra_args='install')
self.ezfio_path = os.path.join(self.teststmpdir, 'ezfio')
ezfio_link = 'https://github.com/earlephilhower/ezfio.git'
git.get_repo(ezfio_link, destination_dir=self.ezfio_path)
self.utilization = self.params.get('utilization', default='100')
# aio-max-nr is 65536 by default, and test fails if QD is 256 or above
genio.write_file("/proc/sys/fs/aio-max-nr", "1048576")
smm = SoftwareManager()
# Not a package that must be installed, so not skipping.
if not smm.check_installed("sdparm") and not smm.install("sdparm"):
self.log.debug("Can not install sdparm")
self.cwd = os.getcwd()
def test(self):
'''
Function where test is executed
'''
device_list = []
cmd = "ls -l /dev/disk/by-path/"
output = process.run(cmd)
for lines in output.stdout.splitlines():
if self.pci_device in lines:
device_list.append(lines.split()[-1])
if not device_list:
self.log.warning("No devices under the given PCI device")
else:
for device_id in device_list:
device = device_id.split()[-1].strip("../*")
self.log.info("device = %s", device)
cmd = "lsscsi"
output = process.run(cmd)
for lines in output.stdout.splitlines():
if device in lines:
scsi_num = lines.split()[0].strip("[").strip("]")
self.log.info("scsi_num=%s", scsi_num)
scsi_num_seperated = scsi_num.replace(":", " ")
self.log.info("Deleting %s", scsi_num)
genio.write_file("/sys/block/%s/device/delete"
% device, "1")
time.sleep(5)
self.log.info("%s deleted", scsi_num)
self.log.info("adding back %s", scsi_num)
process.run("echo scsi add-single-device %s > \
/proc/scsi/scsi", scsi_num_seperated)
time.sleep(5)
self.log.info("%s Added back", scsi_num)
def test(self):
"""
Performs driver unbind and bind for the Network virtualized device
"""
if not self.ping_check():
self.cancel("Please make sure the network peer is configured ?")
try:
for _ in range(self.count):
for operation in ["unbind", "bind"]:
self.log.info(
"Running %s operation for Network virtualized \
device" % operation)
genio.write_file(
os.path.join("/sys/bus/vio/drivers/ibmvnic",
operation), "%s" % self.device)
time.sleep(5)
self.log.info("Running a ping test to check if unbind/bind \
affected newtwork connectivity")
if not self.ping_check():
self.fail("Ping test failed. Network virtualized \
unbind/bind has affected Network connectivity")
except CmdError as details:
self.log.debug(str(details))
self.fail("Driver %s operation failed for Network virtualized \
device %s" % (operation, self.interface))
def tearDown(self):
if self.scenario_arg in [7, 8, 9, 10, 11, 12]:
for hp_size in self.hsizes:
genio.write_file(
'/sys/kernel/mm/hugepages/hugepages-%skB/nr_hugepages' %
str(hp_size * 1024), str(0))
if self.scenario_arg not in [1, 2]:
memory.set_num_huge_pages(self.exist_pages)
def hotplug_remove(self):
"""
Hot Plug remove operation
"""
genio.write_file("/sys/bus/pci/slots/%s/power" % self.slot, "0")
time.sleep(5)
if self.device in pci.get_pci_addresses():
self.return_code = 1
else:
self.log.info("Adapter %s removed successfully", self.device)
def hotplug_add(self):
"""
Hot plug add operation
"""
genio.write_file("/sys/bus/pci/slots/%s/power" % self.slot, "1")
time.sleep(5)
if self.device not in pci.get_pci_addresses():
self.return_code = 2
else:
self.log.info("Adapter %s added back successfully", self.device)
def hotplug_add(self):
"""
Hot plug add operation
"""
genio.write_file("/sys/bus/pci/slots/%s/power" % self.slot, "1")
time.sleep(5)
cmd = "lspci -k -s %s" % self.device
if process.system_output(cmd, shell=True).strip('\n') is '':
self.return_code = 2
else:
print "Adapter %s added back successfully" % self.device
def hotplug_add(self):
"""
Hot plug add operation
"""
genio.write_file("/sys/bus/pci/slots/%s/power" % self.slot, "1")
time.sleep(5)
cmd = "lspci -k -s %s" % self.device
if process.system_output(cmd, shell=True).strip('\n') is '':
self.return_code = 2
else:
print "Adapter %s added back successfully" % self.device
def setUp(self):
smm = SoftwareManager()
dist = distro.detect()
memsize = int(memory.meminfo.MemFree.b * 0.2)
self.nr_pages = self.params.get('nr_pages',
default=memsize /
memory.get_page_size())
self.map_type = self.params.get('map_type', default='private')
self.hpage = self.params.get('h_page', default=False)
nodes = memory.numa_nodes_with_memory()
if len(nodes) < 2:
self.cancel('Test requires two numa nodes to run.'
'Node list with memory: %s' % nodes)
pkgs = ['gcc', 'make']
hp_check = 0
if self.hpage:
hp_size = memory.get_huge_page_size()
for node in nodes:
genio.write_file(
'/sys/devices/system/node/node%s/hugepages/hu'
'gepages-%skB/nr_hugepages' % (node, str(hp_size)),
str(self.nr_pages))
for node in nodes:
hp_check += int(
genio.read_file(
'/sys/devices/system/node/node%s/hugepages/hugepages-%skB'
'/nr_hugepages' % (node, str(hp_size))).strip())
if hp_check < self.nr_pages:
self.cancel('Not enough pages to be configured on nodes')
if dist.name == "Ubuntu":
pkgs.extend(
['libpthread-stubs0-dev', 'libnuma-dev', 'libhugetlbfs-dev'])
elif dist.name in ["centos", "rhel", "fedora"]:
pkgs.extend(['numactl-devel', 'libhugetlbfs-devel'])
elif dist.name == "SuSE":
pkgs.extend(['libnuma-devel'])
if dist.version >= 15:
pkgs.extend(['libhugetlbfs-devel'])
else:
pkgs.extend(['libhugetlbfs-libhugetlb-devel'])
for package in pkgs:
if not smm.check_installed(package) and not smm.install(package):
self.cancel('%s is needed for the test to be run' % package)
for file_name in [
'util.c', 'numa_test.c', 'softoffline.c', 'bench_movepages.c',
'Makefile'
]:
self.copyutil(file_name)
build.make(self.teststmpdir)
def hotplug_remove(self):
"""
Hot Plug remove operation
"""
genio.write_file("/sys/bus/pci/slots/%s/power" % self.slot, "0")
time.sleep(5)
cmd = "lspci -k -s %s" % self.device
if process.system_output(cmd, shell=True).strip('\n') is not '':
self.return_code = 1
else:
print "Adapter %s removed successfully" % self.device
def hotplug_remove(self):
"""
Hot Plug remove operation
"""
genio.write_file("/sys/bus/pci/slots/%s/power" % self.slot, "0")
time.sleep(5)
cmd = "lspci -k -s %s" % self.device
if process.system_output(cmd, shell=True).strip('\n') is not '':
self.return_code = 1
else:
print "Adapter %s removed successfully" % self.device
def test_nvmfdisconnectcfg(self):
"""
Disconnects to NVMf subsystems on the initiator
"""
pre_count = self.nvme_devs_count()
for i in range(len(self.ids)):
cmd = "nvme disconnect -n mysubsys%s" % str(i + 1)
if process.system(cmd, shell=True, ignore_status=True) != 0:
self.fail("Disconnect to mysubsys%s fails" % str(i + 1))
genio.write_file(self.nvmf_discovery_file, "")
if (pre_count - self.nvme_devs_count()) != len(self.ids):
self.fail("%d new nvme not devices removed" % len(self.ids))
def _log_modified_packages(self, path):
"""
Log any changes to installed packages.
"""
old_packages = set(self._installed_pkgs)
new_packages = set(self._get_installed_packages())
added_path = os.path.join(path, "added_packages")
added_packages = "\n".join(new_packages - old_packages) + "\n"
genio.write_file(added_path, added_packages)
removed_path = os.path.join(self.basedir, "removed_packages")
removed_packages = "\n".join(old_packages - new_packages) + "\n"
genio.write_file(removed_path, removed_packages)
def _generate_openrc_py_file(self, file_location):
content = """
import os
os.environ['OS_USERNAME'] = '******'
os.environ['OS_PASSWORD'] = '******'
os.environ['OS_AUTH_URL'] = '%s'
os.environ['OS_TENANT_NAME'] = '%s'
""" % (os.environ['OS_USERNAME'], os.environ['OS_PASSWORD'],
os.environ['OS_AUTH_URL'], os.environ['OS_TENANT_NAME'])
openrc_file = os.path.join(file_location, 'openrc.py')
genio.write_file(openrc_file, content)
def hotplug_add(self, slot, pci_addr):
"""
Hot plug add operation
"""
genio.write_file("/sys/bus/pci/slots/%s/power" % slot, "1")
def is_added():
if pci_addr not in pci.get_pci_addresses():
return False
return True
return wait.wait_for(is_added, timeout=10) or False
def test_nvmfdisconnectcfg(self):
"""
Disconnects to NVMf subsystems on the initiator
"""
pre_count = self.nvme_devs_count()
for i in range(len(self.ids)):
cmd = "nvme disconnect -n mysubsys%s" % str(i + 1)
if process.system(cmd, shell=True, ignore_status=True) != 0:
self.fail("Disconnect to mysubsys%s fails" % str(i + 1))
genio.write_file(self.nvmf_discovery_file, "")
if (pre_count - self.nvme_devs_count()) != len(self.ids):
self.fail("%d new nvme devices not removed" % len(self.ids))
def test(self):
"""
Performs driver unbind and bind for the Network virtualized device
"""
if self.device_type in ["l-lan", "vnic"]:
if self.networkinterface.ping_check(self.peer_ip,
count=5) is not None:
self.cancel(
"Please make sure the network peer is configured ?")
else:
if self.is_exists_device(self.virtual_device) is False:
self.cancel("failed to detect the test disk")
try:
for _ in range(self.count):
for operation in ["unbind", "bind"]:
self.log.info(
"Running %s operation for Network virtualized \
device" % operation)
dict = {
"vnic": "ibmvnic",
"l-lan": "ibmveth",
"v-scsi": "ibmvscsi",
"vfc-client": "ibmvfc"
}
if self.device_type in dict.keys():
param = dict[self.device_type]
genio.write_file(
os.path.join("/sys/bus/vio/drivers/%s" % param,
operation), "%s" % self.device)
time.sleep(5)
if self.device_type in ["l-lan", "vnic"]:
self.log.info(
"Running a ping test to check if unbind/bind \
affected newtwork connectivity")
if self.networkinterface.ping_check(self.peer_ip,
count=5) is not None:
self.fail("Ping test failed. Network virtualized \
unbind/bind has affected Network connectivity"
)
else:
self.log.info("checking for disk available if unbind/bind \
affected to disk")
if self.is_exists_device(self.virtual_device) is False:
self.fail("exists device test failed.unbind/bind has \
affected disk")
except CmdError as details:
self.log.debug(str(details))
self.fail("Driver %s operation failed for Network virtualized \
device %s" % (operation, self.interface))
def hotplug_remove(slot, pci_addr):
"""
Hot Plug remove operation
"""
genio.write_file("/sys/bus/pci/slots/%s/power" % slot, "0")
def is_removed():
"""
Returns True if pci device is removed, False otherwise.
"""
if pci_addr in pci.get_pci_addresses():
return False
return True
return wait.wait_for(is_removed, timeout=10) or False
def run_avocado(self):
"""
Wraps the run method, for execution inside the avocado runner.
:result: Unused param, compatibility with :class:`unittest.TestCase`.
"""
self._setup_environment_variables()
self._catch_test_status(self._run_test)
self._catch_test_status(self._tearDown)
whiteboard_file = os.path.join(self.logdir, 'whiteboard')
genio.write_file(whiteboard_file, self.whiteboard)
self.__phase = 'FINISHED'
self._tag_end()
self._report()
self.log.info("")
def setUp(self):
smm = SoftwareManager()
dist = distro.detect()
memsize = int(memory.freememtotal() * 1024 * 0.2)
self.nr_pages = self.params.get(
'nr_pages', default=memsize / memory.get_page_size())
self.map_type = self.params.get('map_type', default='private')
self.hpage = self.params.get('h_page', default=False)
nodes = memory.numa_nodes_with_memory()
if len(nodes) < 2:
self.cancel('Test requires two numa nodes to run.'
'Node list with memory: %s' % nodes)
pkgs = ['gcc', 'make']
hp_check = 0
if self.hpage:
hp_size = memory.get_huge_page_size()
for node in nodes:
genio.write_file('/sys/devices/system/node/node%s/hugepages/hu'
'gepages-%skB/nr_hugepages' %
(node, str(hp_size)), str(self.nr_pages))
for node in nodes:
hp_check += int(genio.read_file(
'/sys/devices/system/node/node%s/hugepages/hugepages-%skB'
'/nr_hugepages' % (node, str(hp_size))).strip())
if hp_check < self.nr_pages:
self.cancel('Not enough pages to be configured on nodes')
if dist.name == "Ubuntu":
pkgs.extend(['libpthread-stubs0-dev',
'libnuma-dev', 'libhugetlbfs-dev'])
elif dist.name in ["centos", "rhel", "fedora"]:
pkgs.extend(['numactl-devel', 'libhugetlbfs-devel'])
elif dist.name == "SuSE":
pkgs.extend(['libnuma-devel'])
if dist.version >= 15:
pkgs.extend(['libhugetlbfs-devel'])
else:
pkgs.extend(['libhugetlbfs-libhugetlb-devel'])
for package in pkgs:
if not smm.check_installed(package) and not smm.install(package):
self.cancel('%s is needed for the test to be run' % package)
for file_name in ['util.c', 'numa_test.c', 'Makefile']:
self.copyutil(file_name)
build.make(self.teststmpdir)
def test_nvmfconnectcfg(self):
"""
Connects to allNVMf subsystems in /etc/nvme/discovery.conf
"""
if not os.path.exists(os.path.dirname(self.nvmf_discovery_file)):
os.makedirs(os.path.dirname(self.nvmf_discovery_file))
msg = []
for i in range(len(self.ids)):
msg.append("-t rdma -a %s -s 4420 -q mysubsys%s" %
(self.peer_ips[i], str(i + 1)))
genio.write_file(self.nvmf_discovery_file, "\n".join(msg))
process.system("cat %s" % self.nvmf_discovery_file)
pre_count = self.nvme_devs_count()
cmd = "nvme connect-all"
if process.system(cmd, shell=True, ignore_status=True) != 0:
self.fail("connect-all fails")
if (self.nvme_devs_count() - pre_count) != len(self.ids):
self.fail("%d new nvme not devices added" % len(self.ids))
def test_nvmfconnectcfg(self):
"""
Connects to allNVMf subsystems in /etc/nvme/discovery.conf
"""
if not os.path.exists(os.path.dirname(self.nvmf_discovery_file)):
os.makedirs(os.path.dirname(self.nvmf_discovery_file))
msg = []
for i in range(len(self.ids)):
msg.append("-t rdma -a %s -s 4420 -q mysubsys%s" %
(self.peer_ips[i], str(i + 1)))
genio.write_file(self.nvmf_discovery_file, "\n".join(msg))
process.system("cat %s" % self.nvmf_discovery_file)
pre_count = self.nvme_devs_count()
cmd = "nvme connect-all"
if process.system(cmd, shell=True, ignore_status=True) != 0:
self.fail("connect-all fails")
if (self.nvme_devs_count() - pre_count) != len(self.ids):
self.fail("%d new nvme devices not added" % len(self.ids))
def bond_remove(self, arg1):
'''
bond_remove
'''
if arg1 == "local":
self.log.info("Removing Bonding configuration on local machine")
self.log.info("------------------------------------------------")
for ifs in self.host_interfaces:
cmd = "ip link set %s down" % ifs
if process.system(cmd, shell=True, ignore_status=True) != 0:
self.log.info("unable to bring down the interface")
if self.ib:
self.bond_ib_conf(self.bond_name, ifs, "REMOVE")
else:
genio.write_file(self.bonding_slave_file, "-%s" % ifs)
genio.write_file(self.bonding_masters_file, "-%s" % self.bond_name)
self.log.info("Removing bonding module")
linux_modules.unload_module("bonding")
time.sleep(self.sleep_time)
else:
self.log.info("Removing Bonding configuration on Peer machine")
self.log.info("------------------------------------------------")
cmd = ''
cmd += 'ip link set %s down;' % self.bond_name
for val in self.peer_interfaces:
cmd += 'ip link set %s down;' % val
for val in self.peer_interfaces:
cmd += 'ip addr flush dev %s;' % val
for val in self.peer_interfaces:
if self.ib:
self.bond_ib_conf(self.bond_name, val, "REMOVE")
else:
cmd += 'echo "-%s" > %s;' % (val, self.bonding_slave_file)
cmd += 'echo "-%s" > %s;' % (self.bond_name,
self.bonding_masters_file)
cmd += 'rmmod bonding;'
cmd += 'ip addr add %s/%s dev %s;ip link set %s up;sleep 5;'\
% (self.peer_first_ipinterface, self.net_mask[0],
self.peer_interfaces[0], self.peer_interfaces[0])
output = self.session.cmd(cmd)
if not output.exit_status == 0:
self.log.info("bond removing command failed in peer machine")
def verify_background_errors(self):
"""
Verify if there are any errors that happened on background threads.
Logs all errors in the background_errors into background-error.log and
error the test.
"""
err_file_path = os.path.join(self.logdir, BG_ERR_FILE)
bg_errors = self.background_errors.get_all()
error_messages = ["BACKGROUND ERROR LIST:"]
for index, error in enumerate(bg_errors):
error_messages.append(
"- ERROR #%d -\n%s" %
(index, "".join(traceback.format_exception(*error))))
genio.write_file(err_file_path, '\n'.join(error_messages))
if bg_errors:
msg = ["Background error"]
msg.append("s are" if len(bg_errors) > 1 else " is")
msg.append((" detected, please refer to file: "
"'%s' for more details.") % BG_ERR_FILE)
self.error(''.join(msg))
def verify_background_errors(self):
"""
Verify if there are any errors that happened on background threads.
Logs all errors in the background_errors into background-error.log and
error the test.
"""
err_file_path = os.path.join(self.logdir, BG_ERR_FILE)
bg_errors = self.background_errors.get_all()
error_messages = ["BACKGROUND ERROR LIST:"]
for index, error in enumerate(bg_errors):
error_messages.append(
"- ERROR #%d -\n%s" % (index, "".join(
traceback.format_exception(*error)
)))
genio.write_file(err_file_path, '\n'.join(error_messages))
if bg_errors:
msg = ["Background error"]
msg.append("s are" if len(bg_errors) > 1 else " is")
msg.append((" detected, please refer to file: "
"'%s' for more details.") % BG_ERR_FILE)
self.error(''.join(msg))
def collect_dmesg(output_file=None):
"""Function collect dmesg and save in file.
The dmesg operation is a privileged user task.
This function needs sudo permissions enabled on the target host
:param output_file: File use for save dmesg output if not provided it use
tmp file which located in system /tmp path
:type output_file: str
:returns: file which contain dmesg
:rtype: str
"""
if output_file is None:
_, output_file = tempfile.mkstemp(suffix=f".-{time.strftime('%Y-%m-%d:%H:%M:%S')}",
dir=tempfile.gettempdir())
dmesg = process.system_output(
"dmesg", ignore_status=True, sudo=True).decode()
genio.write_file(output_file, dmesg)
if not os.path.isfile(output_file):
raise DmesgError(f"{output_file} is not a valid file.")
return output_file
def test(self):
"""
Creates namespace on the device.
"""
genio.write_file("/sys/bus/pci/slots/%s/power" % self.slot, "0")
time.sleep(5)
cmd = "lspci -k -s %s" % self.device
if process.system_output(cmd, shell=True).strip('\n') is not '':
self.return_code = 1
else:
print "Adapter %s removed successfully" % self.device
genio.write_file("/sys/bus/pci/slots/%s/power" % self.slot, "1")
time.sleep(5)
cmd = "lspci -k -s %s" % self.device
if process.system_output(cmd, shell=True).strip('\n') is '':
self.return_code = 2
else:
print "Adapter %s added back successfully" % self.device
if self.return_code == 1:
self.fail('%s not removed' % self.device)
if self.return_code == 2:
self.fail('%s not attached back' % self.device)
def test_unbindbind(self):
"""
Performs driver unbind and bind for the Network virtualized device
"""
device_id = self.find_device_id()
try:
for _ in range(self.count):
for operation in ["unbind", "bind"]:
self.log.info("Running %s operation for Network \
virtualized device", operation)
genio.write_file(os.path.join
("/sys/bus/vio/drivers/ibmvnic",
operation), "%s" % device_id)
time.sleep(10)
self.log.info("Running a ping test to check if unbind/bind \
affected newtwork connectivity")
if not self.ping_check():
self.fail("Ping test failed. Network virtualized \
unbind/bind has affected Network connectivity")
except CmdError as details:
self.log.debug(str(details))
self.fail("Driver %s operation failed" % operation)
def test_clientfailover(self):
'''
Performs Client initiated failover for Network virtualized
device
'''
device_id = self.find_device_id()
try:
for _ in range(self.count):
for val in range(int(self.backing_dev_count())):
self.log.info("Performing Client initiated\
failover - Attempt %s", int(val+1))
genio.write_file("/sys/devices/vio/%s/failover"
% device_id, "1")
time.sleep(10)
self.log.info("Running a ping test to check if failover \
affected Network connectivity")
if not self.ping_check():
self.fail("Ping test failed. Network virtualized \
failover has affected Network connectivity")
except CmdError as details:
self.log.debug(str(details))
self.fail("Client initiated Failover for Network virtualized \
device has failed")
def bond_remove(self, arg1):
'''
bond_remove
'''
if arg1 == "local":
self.log.info("Removing Bonding configuration on local machine")
self.log.info("------------------------------------------------")
for ifs in self.host_interfaces:
cmd = "ip link set %s down" % ifs
if process.system(cmd, shell=True, ignore_status=True) != 0:
self.log.info("unable to bring down the interface")
genio.write_file(self.bonding_slave_file, "-%s" % ifs)
genio.write_file(self.bonding_masters_file, "-%s" % self.bond_name)
self.log.info("Removing bonding module")
linux_modules.unload_module("bonding")
time.sleep(self.sleep_time)
else:
self.log.info("Removing Bonding configuration on Peer machine")
self.log.info("------------------------------------------------")
cmd = ''
cmd += 'ip link set %s down;' % self.bond_name
for val in self.peer_interfaces:
cmd += 'ip link set %s down;' % val
for val in self.peer_interfaces:
cmd += 'ip addr flush dev %s;' % val
for val in self.peer_interfaces:
cmd += 'echo "-%s" > %s;' % (val, self.bonding_slave_file)
cmd += 'echo "-%s" > %s;' % (self.bond_name,
self.bonding_masters_file)
cmd += 'rmmod bonding;'
cmd += 'ip addr add %s/%s dev %s;ip link set %s up;sleep 5;'\
% (self.peer_first_ipinterface, self.net_mask[0],
self.peer_first_interface, self.peer_first_interface)
peer_cmd = "ssh %s@%s \"%s\""\
% (self.user, self.peer_first_ipinterface, cmd)
if process.system(peer_cmd, shell=True, ignore_status=True) != 0:
self.log.info("bond removing command failed in peer machine")
stacktrace.log_exc_info(sys.exc_info(), logger='avocado.test')
raise exceptions.TestSetupFail(details)
try:
testMethod()
except Exception, details:
stacktrace.log_exc_info(sys.exc_info(), logger='avocado.test')
test_exception = details
finally:
try:
self.tearDown()
except Exception, details:
stacktrace.log_exc_info(sys.exc_info(), logger='avocado.test')
cleanup_exception = details
whiteboard_file = os.path.join(self.logdir, 'whiteboard')
genio.write_file(whiteboard_file, self.whiteboard)
# pylint: disable=E0702
if test_exception is not None:
raise test_exception
elif cleanup_exception is not None:
raise exceptions.TestSetupFail(cleanup_exception)
elif stdout_check_exception is not None:
raise stdout_check_exception
elif stderr_check_exception is not None:
raise stderr_check_exception
elif self._Test__log_warn_used:
raise exceptions.TestWarn("Test passed but there were warnings "
"during execution. Check the log for "
"details.")
def bond_setup(self, arg1, arg2):
'''
bond setup
'''
if arg1 == "local":
self.log.info("Configuring Bonding on Local machine")
self.log.info("--------------------------------------")
for ifs in self.host_interfaces:
cmd = "ip addr flush dev %s" % ifs
process.system(cmd, shell=True, ignore_status=True)
for ifs in self.host_interfaces:
cmd = "ip link set %s down" % ifs
process.system(cmd, shell=True, ignore_status=True)
linux_modules.load_module("bonding")
genio.write_file(self.bonding_masters_file, "+%s" % self.bond_name)
genio.write_file("%s/bonding/mode" % self.bond_dir, arg2)
genio.write_file("%s/bonding/miimon" % self.bond_dir, "100")
genio.write_file("%s/bonding/fail_over_mac" % self.bond_dir, "2")
for val in self.host_interfaces:
genio.write_file(self.bonding_slave_file, "+%s" % val)
time.sleep(2)
bond_name_val = ''
for line in genio.read_file(self.bond_status).splitlines():
if 'Bonding Mode' in line:
bond_name_val = line.split(':')[1]
self.log.info("Trying bond mode %s [ %s ]", arg2, bond_name_val)
for ifs in self.host_interfaces:
cmd = "ip link set %s up" % ifs
if process.system(cmd, shell=True, ignore_status=True) != 0:
self.fail("unable to interface up")
cmd = "ip addr add %s/%s dev %s;ip link set %s up"\
% (self.local_ip, self.net_mask[0],
self.bond_name, self.bond_name)
process.system(cmd, shell=True, ignore_status=True)
for _ in range(0, 600, 60):
if 'state UP' in process.system_output("ip link \
show %s" % self.bond_name, shell=True):
self.log.info("Bonding setup is successful on\
local machine")
break
time.sleep(60)
else:
self.fail("Bonding setup on local machine has failed")
if self.gateway:
cmd = 'ip route add default via %s dev %s' % \
(self.gateway, self.bond_name)
process.system(cmd, shell=True, ignore_status=True)
else:
self.log.info("Configuring Bonding on Peer machine")
self.log.info("------------------------------------------")
cmd = ''
for val in self.peer_interfaces:
cmd += 'ip addr flush dev %s;' % val
for val in self.peer_interfaces:
cmd += 'ip link set %s down;' % val
cmd += 'modprobe bonding;'
cmd += 'echo +%s > %s;'\
% (self.bond_name, self.bonding_masters_file)
cmd += 'echo 0 > %s/bonding/mode;'\
% self.bond_dir
cmd += 'echo 100 > %s/bonding/miimon;'\
% self.bond_dir
cmd += 'echo 2 > %s/bonding/fail_over_mac;'\
% self.bond_dir
for val in self.peer_interfaces:
cmd += 'echo "+%s" > %s;' % (val, self.bonding_slave_file)
for val in self.peer_interfaces:
cmd += 'ip link set %s up;' % val
cmd += 'ip addr add %s/%s dev %s;ip link set %s up;sleep 5;'\
% (self.peer_first_ipinterface, self.net_mask[0],
self.bond_name, self.bond_name)
peer_cmd = "timeout %s ssh %s@%s \"%s\""\
% (self.peer_wait_time, self.user,
self.peer_first_ipinterface, cmd)
if process.system(peer_cmd, shell=True, ignore_status=True) != 0:
self.fail("bond setup command failed in peer machine")
def hotplug_add(self, slot, pci_addr):
"""
Hot plug add operation and recovery check
"""
genio.write_file("/sys/bus/pci/slots/%s/power" % slot, "1")
def is_added():
"""
Returns True if pci device is added, False otherwise.
"""
if pci_addr not in pci.get_pci_addresses():
return False
return True
def is_recovered():
"""
Checks if the block device adapter is recovers all its disks/paths
properly after hotplug of adapter.
Returns True if all disks/paths back online after adapter added
Back, else False.
"""
def is_path_online():
path_stat = list(multipath.get_path_status(curr_path))
if path_stat[0] != 'active' or path_stat[2] != 'ready':
return False
return True
curr_path = ''
err_disks = []
if pci.get_pci_class_name(pci_addr) == 'fc_host':
disks = pci.get_disks_in_pci_address(pci_addr)
for disk in disks:
curr_path = disk.split("/")[-1]
self.log.info("curr_path=%s" % curr_path)
if not wait.wait_for(is_path_online, timeout=10):
self.log.info("%s failed to recover after add" % disk)
err_disks.append(disk)
if err_disks:
self.log.info("few paths failed to recover : %s" % err_disks)
return False
return True
def net_recovery_check():
"""
Checks if the network adapter fuctionality like ping/link_state,
after adapter added back.
Returns True on propper Recovery, False if not.
"""
self.log.info("entering the net recovery check")
local = LocalHost()
iface = pci.get_interfaces_in_pci_address(pci_addr, 'net')
networkinterface = NetworkInterface(iface[0], local)
if wait.wait_for(networkinterface.is_link_up, timeout=120):
if networkinterface.ping_check(self.peer_ip, count=5) is None:
self.log.info("inteface is up and pinging")
return True
return False
if wait.wait_for(is_added, timeout=30):
time.sleep(45)
if pci.get_pci_class_name(pci_addr) == 'net':
if wait.wait_for(net_recovery_check, timeout=30):
return True
return False
else:
if wait.wait_for(is_recovered, timeout=30):
return True
return False
def bond_setup(self, arg1, arg2):
'''
bond setup
'''
if arg1 == "local":
self.log.info("Configuring Bonding on Local machine")
self.log.info("--------------------------------------")
for ifs in self.host_interfaces:
cmd = "ip addr flush dev %s" % ifs
process.system(cmd, shell=True, ignore_status=True)
for ifs in self.host_interfaces:
cmd = "ip link set %s down" % ifs
process.system(cmd, shell=True, ignore_status=True)
linux_modules.load_module("bonding")
genio.write_file(self.bonding_masters_file, "+%s" % self.bond_name)
genio.write_file("%s/bonding/mode" % self.bond_dir, arg2)
genio.write_file("%s/bonding/miimon" % self.bond_dir, self.miimon)
genio.write_file("%s/bonding/fail_over_mac" % self.bond_dir,
self.fail_over_mac)
genio.write_file("%s/bonding/downdelay" % self.bond_dir,
self.downdelay)
dict = {
'0': ['packets_per_slave', 'resend_igmp'],
'1':
['num_unsol_na', 'primary', 'primary_reselect', 'resend_igmp'],
'2': ['xmit_hash_policy'],
'4': ['lacp_rate', 'xmit_hash_policy'],
'5': [
'tlb_dynamic_lb', 'primary', 'primary_reselect',
'resend_igmp', 'xmit_hash_policy', 'lp_interval'
],
'6':
['primary', 'primary_reselect', 'resend_igmp', 'lp_interval']
}
if self.mode in dict.keys():
for param in dict[self.mode]:
param_value = self.params.get(param, default='')
if param_value:
genio.write_file(
"%s/bonding/%s" % (self.bond_dir, param),
param_value)
for val in self.host_interfaces:
if self.ib:
self.bond_ib_conf(self.bond_name, val, "ATTACH")
else:
genio.write_file(self.bonding_slave_file, "+%s" % val)
time.sleep(2)
bond_name_val = ''
for line in genio.read_file(self.bond_status).splitlines():
if 'Bonding Mode' in line:
bond_name_val = line.split(':')[1]
self.log.info("Trying bond mode %s [ %s ]", arg2, bond_name_val)
for ifs in self.host_interfaces:
cmd = "ip link set %s up" % ifs
if process.system(cmd, shell=True, ignore_status=True) != 0:
self.fail("unable to interface up")
cmd = "ip addr add %s/%s dev %s;ip link set %s up"\
% (self.local_ip, self.net_mask[0],
self.bond_name, self.bond_name)
process.system(cmd, shell=True, ignore_status=True)
for _ in range(0, 600, 60):
if 'state UP' in process.system_output(
"ip link \
show %s" % self.bond_name,
shell=True).decode("utf-8"):
self.log.info("Bonding setup is successful on\
local machine")
break
time.sleep(60)
else:
self.fail("Bonding setup on local machine has failed")
if self.gateway:
cmd = 'ip route add default via %s dev %s' % \
(self.gateway, self.bond_name)
process.system(cmd, shell=True, ignore_status=True)
else:
self.log.info("Configuring Bonding on Peer machine")
self.log.info("------------------------------------------")
cmd = ''
for val in self.peer_interfaces:
cmd += 'ip addr flush dev %s;' % val
for val in self.peer_interfaces:
cmd += 'ip link set %s down;' % val
cmd += 'modprobe bonding;'
cmd += 'echo +%s > %s;'\
% (self.bond_name, self.bonding_masters_file)
cmd += 'echo 0 > %s/bonding/mode;'\
% self.bond_dir
cmd += 'echo 100 > %s/bonding/miimon;'\
% self.bond_dir
cmd += 'echo 2 > %s/bonding/fail_over_mac;'\
% self.bond_dir
for val in self.peer_interfaces:
if self.ib:
self.bond_ib_conf(self.bond_name, val, "ATTACH")
else:
cmd += 'echo "+%s" > %s;' % (val, self.bonding_slave_file)
for val in self.peer_interfaces:
cmd += 'ip link set %s up;' % val
cmd += 'ip addr add %s/%s dev %s;ip link set %s up;sleep 5;'\
% (self.peer_first_ipinterface, self.net_mask[0],
self.bond_name, self.bond_name)
output = self.session.cmd(cmd)
if not output.exit_status == 0:
self.fail("bond setup command failed in peer machine")
def setUp(self):
'''
Build VA Test
'''
# Check for basic utilities
smm = SoftwareManager()
self.scenario_arg = int(self.params.get('scenario_arg', default=1))
self.n_chunks = nr_pages = self.n_chunks2 = self.def_chunks = 0
self.hsizes = [1024, 2]
self.hp_file = '/sys/kernel/mm/hugepages/hugepages-%skB/nr_hugepages'
page_chunker = memory.meminfo.Hugepagesize.m
if distro.detect().arch in ['ppc64', 'ppc64le']:
mmu_detect = genio.read_file(
'/proc/cpuinfo').strip().splitlines()[-1]
# Check for "Radix" as this MMU will be explicit in POWER
if 'Radix' not in mmu_detect:
self.hsizes = [16]
# For now, 16G hugepages are possible only when it is default.
# So check and add to the possible pagesize list
if page_chunker == 16384:
self.hsizes.extend([16384])
if self.scenario_arg not in range(1, 13):
self.cancel("Test need to skip as scenario will be 1-12")
elif self.scenario_arg in [7, 8, 9]:
self.log.info("Using alternate hugepages")
if len(self.hsizes) == 1:
self.cancel('Scenario is not applicable')
if memory.meminfo.Hugepagesize.m == self.hsizes[0]:
page_chunker = self.hsizes[1]
else:
page_chunker = self.hsizes[0]
self.exist_pages = memory.get_num_huge_pages()
if self.scenario_arg in [10, 11, 12]:
self.log.info("Using Multiple hugepages")
if len(self.hsizes) == 1:
self.cancel('Scenario is not applicable')
if memory.meminfo.Hugepagesize.m != self.hsizes[0]:
self.hsizes.reverse()
# Leaving half size for default pagesize
total_mem = (0.9 * memory.meminfo.MemFree.m) / 2
self.def_chunks = int(total_mem / 16384)
for hp_size in self.hsizes:
nr_pgs = int((total_mem / 2) / hp_size)
genio.write_file(self.hp_file % str(hp_size * 1024),
str(nr_pgs))
n_pages = genio.read_file(self.hp_file %
str(self.hsizes[0] * 1024)).rstrip("\n")
n_pages2 = genio.read_file(self.hp_file %
str(self.hsizes[1] * 1024)).rstrip("\n")
self.n_chunks = (int(n_pages) * self.hsizes[0]) // 16384
self.n_chunks2 = (int(n_pages2) * self.hsizes[1]) // 16384
if self.scenario_arg not in [1, 2, 10, 11, 12]:
max_hpages = int((0.9 * memory.meminfo.MemFree.m) / page_chunker)
if self.scenario_arg in [3, 4, 5, 6]:
memory.set_num_huge_pages(max_hpages)
nr_pages = memory.get_num_huge_pages()
else:
genio.write_file(self.hp_file % str(page_chunker * 1024),
str(max_hpages))
nr_pages = genio.read_file(
self.hp_file % str(page_chunker * 1024)).rstrip("\n")
self.n_chunks = (int(nr_pages) * page_chunker) // 16384
for packages in ['gcc', 'make']:
if not smm.check_installed(packages) and not smm.install(packages):
self.cancel('%s is needed for the test to be run' % packages)
shutil.copyfile(self.get_data('va_test.c'),
os.path.join(self.teststmpdir, 'va_test.c'))
shutil.copyfile(self.get_data('Makefile'),
os.path.join(self.teststmpdir, 'Makefile'))
build.make(self.teststmpdir)
stacktrace.log_exc_info(sys.exc_info(), logger='avocado.test')
raise exceptions.TestSetupFail(details)
try:
testMethod()
except Exception, details:
stacktrace.log_exc_info(sys.exc_info(), logger='avocado.test')
test_exception = details
finally:
try:
self.tearDown()
except Exception, details:
stacktrace.log_exc_info(sys.exc_info(), logger='avocado.test')
cleanup_exception = details
whiteboard_file = os.path.join(self.logdir, 'whiteboard')
genio.write_file(whiteboard_file, self.whiteboard)
# pylint: disable=E0702
if test_exception is not None:
raise test_exception
elif cleanup_exception is not None:
raise exceptions.TestSetupFail(cleanup_exception)
elif stdout_check_exception is not None:
raise stdout_check_exception
elif stderr_check_exception is not None:
raise stderr_check_exception
elif self._Test__log_warn_used:
raise exceptions.TestWarn("Test passed but there were warnings "
"during execution. Check the log for "
"details.")
def tearDown(self):
genio.write_file("/proc/sys/vm/drop_caches", "3")