def setUp(self):
self.conf = OpTestConfiguration.conf
self.cv_HOST = self.conf.host()
self.cv_SYSTEM = self.conf.system()
self.host_cmd_timeout = self.conf.args.host_cmd_timeout
self.repo = self.conf.args.git_repo
self.branch = self.conf.args.git_branch
self.home = self.conf.args.git_home
self.config = self.conf.args.git_repoconfig
self.config_path = self.conf.args.git_repoconfigpath
self.disk = self.conf.args.host_scratch_disk
self.patch = self.conf.args.git_patch
self.use_kexec = self.conf.args.use_kexec
self.append_kernel_cmdline = self.conf.args.append_kernel_cmdline
if self.config_path:
self.config = "olddefconfig"
if not self.repo:
self.fail("Provide git repo of kernel to install")
if not (self.conf.args.host_ip and self.conf.args.host_user
and self.conf.args.host_password):
self.fail(
"Provide host ip user details refer, --host-{ip,user,password}"
)
if self.disk:
OpIU = InstallUtil()
self.cv_SYSTEM.goto_state(OpSystemState.PETITBOOT_SHELL)
log.debug("Set given disk as default bootable disk")
OpIU.set_bootable_disk(self.disk)
self.console_thread = OpSOLMonitorThread(1, "console")
class RunHostTest(unittest.TestCase):
def setUp(self):
self.conf = OpTestConfiguration.conf
self.cv_SYSTEM = self.conf.system()
self.host_cmd = self.conf.args.host_cmd
self.host_cmd_file = self.conf.args.host_cmd_file
self.host_cmd_timeout = self.conf.args.host_cmd_timeout
self.host_cmd_resultpath = self.conf.args.host_cmd_resultpath
if not (self.host_cmd or self.host_cmd_file):
self.fail("Provide either --host-cmd and --host-cmd-file option")
self.resultpath = os.path.join(self.conf.output, "host-results")
if (not os.path.exists(self.resultpath)):
os.makedirs(self.resultpath)
if self.conf.args.bmc_type == 'FSP_PHYP':
self.cv_HMC = self.cv_SYSTEM.hmc
self.console_thread = OpSOLMonitorThreadVM()
else:
self.console_thread = OpSOLMonitorThread(1, "console")
self.console_thread.start()
def runTest(self):
if self.conf.args.bmc_type == 'FSP_PHYP':
self.cv_HMC.poweron_lpar(runtime=True)
else:
self.cv_SYSTEM.goto_state(OpSystemState.OS)
con = self.cv_SYSTEM.cv_HOST.get_ssh_connection()
try:
if self.host_cmd:
con.run_command(self.host_cmd, timeout=self.host_cmd_timeout)
if self.host_cmd_file:
if not os.path.isfile(self.host_cmd_file):
self.fail("Provide valid host cmd file path")
fd = open(self.host_cmd_file, "r")
for line in fd.readlines():
line = line.strip()
if "reboot" in line:
if self.conf.args.bmc_type == 'FSP_PHYP':
self.cv_HMC.poweroff_lpar()
self.cv_HMC.poweron_lpar(runtime=True)
con = self.cv_SYSTEM.cv_HOST.get_new_ssh_connection(
)
else:
self.cv_SYSTEM.goto_state(OpSystemState.OFF)
self.cv_SYSTEM.goto_state(OpSystemState.OS)
con = self.cv_SYSTEM.cv_HOST.get_ssh_connection()
continue
con.run_command(line, timeout=self.host_cmd_timeout)
finally:
if self.host_cmd_resultpath:
self.cv_SYSTEM.cv_HOST.copy_files_from_host(
self.resultpath, self.host_cmd_resultpath)
def tearDown(self):
self.console_thread.console_terminate()
def setUp(self):
self.conf = OpTestConfiguration.conf
self.cv_SYSTEM = self.conf.system()
self.host_cmd = self.conf.args.host_cmd
self.host_cmd_file = self.conf.args.host_cmd_file
self.host_cmd_timeout = self.conf.args.host_cmd_timeout
self.host_cmd_resultpath = self.conf.args.host_cmd_resultpath
if not (self.host_cmd or self.host_cmd_file):
self.fail("Provide either --host-cmd and --host-cmd-file option")
self.resultpath = os.path.join(self.conf.output, "host-results")
if (not os.path.exists(self.resultpath)):
os.makedirs(self.resultpath)
self.console_thread = OpSOLMonitorThread(1, "console")
self.console_thread.start()
def setUp(self):
self.conf = OpTestConfiguration.conf
self.cv_HOST = self.conf.host()
self.cv_SYSTEM = self.conf.system()
self.host_cmd_timeout = self.conf.args.host_cmd_timeout
self.repo = self.conf.args.git_repo
self.branch = self.conf.args.git_branch
self.home = self.conf.args.git_home
self.config = self.conf.args.git_repoconfig
self.config_path = self.conf.args.git_repoconfigpath
self.disk = self.conf.args.host_scratch_disk
self.patch = self.conf.args.git_patch
self.use_kexec = self.conf.args.use_kexec
self.append_kernel_cmdline = self.conf.args.append_kernel_cmdline
if self.config_path:
self.config = "olddefconfig"
if not self.repo:
self.fail("Provide git repo of kernel to install")
if not (self.conf.args.host_ip and self.conf.args.host_user and self.conf.args.host_password):
self.fail("Provide host ip user details refer, --host-{ip,user,password}")
if self.disk:
OpIU = InstallUtil()
self.cv_SYSTEM.goto_state(OpSystemState.PETITBOOT_SHELL)
log.debug("Set given disk as default bootable disk")
OpIU.set_bootable_disk(self.disk)
self.console_thread = OpSOLMonitorThread(1, "console")
def setUp(self):
conf = OpTestConfiguration.conf
self.cv_SYSTEM = conf.system()
self.console = self.cv_SYSTEM.console
conf = OpTestConfiguration.conf
self.hmc_user = conf.args.hmc_username
self.hmc_password = conf.args.hmc_password
self.hmc_ip = conf.args.hmc_ip
self.lpar_name = conf.args.lpar_name
self.system_name = conf.args.system_name
self.cv_HMC = self.cv_SYSTEM.hmc
# The following variables needs to be defined in
# ~/.op-test-framework.conf
self.cpu_resource = conf.args.cpu_resource
self.mem_resource = conf.args.mem_resource
self.lpar2_name = conf.args.lpar2_name
self.loop_num = conf.args.loop_num
self.console_thread = OpSOLMonitorThread(1, "console")
self.console_thread.start()
self.extended = {'loop':0,'wkld':0,'smt':0}
class RunHostTest(unittest.TestCase):
def setUp(self):
self.conf = OpTestConfiguration.conf
self.cv_SYSTEM = self.conf.system()
self.host_cmd = self.conf.args.host_cmd
self.host_cmd_file = self.conf.args.host_cmd_file
self.host_cmd_timeout = self.conf.args.host_cmd_timeout
self.host_cmd_resultpath = self.conf.args.host_cmd_resultpath
if not (self.host_cmd or self.host_cmd_file):
self.fail("Provide either --host-cmd and --host-cmd-file option")
self.resultpath = os.path.join(self.conf.output, "host-results")
if (not os.path.exists(self.resultpath)):
os.makedirs(self.resultpath)
self.console_thread = OpSOLMonitorThread(1, "console")
self.console_thread.start()
def runTest(self):
self.cv_SYSTEM.goto_state(OpSystemState.OS)
con = self.cv_SYSTEM.cv_HOST.get_ssh_connection()
try:
if self.host_cmd:
con.run_command(self.host_cmd, timeout=self.host_cmd_timeout)
if self.host_cmd_file:
if not os.path.isfile(self.host_cmd_file):
self.fail("Provide valid host cmd file path")
fd = open(self.host_cmd_file, "r")
for line in fd.readlines():
line = line.strip()
if "reboot" in line:
self.cv_SYSTEM.goto_state(OpSystemState.OFF)
self.cv_SYSTEM.goto_state(OpSystemState.OS)
con = self.cv_SYSTEM.cv_HOST.get_ssh_connection()
continue
con.run_command(line, timeout=self.host_cmd_timeout)
finally:
if self.host_cmd_resultpath:
self.cv_SYSTEM.cv_HOST.copy_files_from_host(self.resultpath,
self.host_cmd_resultpath)
def tearDown(self):
self.console_thread.console_terminate()
def setUp(self):
self.conf = OpTestConfiguration.conf
self.cv_SYSTEM = self.conf.system()
self.host_cmd = self.conf.args.host_cmd
self.host_cmd_file = self.conf.args.host_cmd_file
self.host_cmd_timeout = self.conf.args.host_cmd_timeout
self.host_cmd_resultpath = self.conf.args.host_cmd_resultpath
if not (self.host_cmd or self.host_cmd_file):
self.fail("Provide either --host-cmd and --host-cmd-file option")
self.resultpath = os.path.join(self.conf.output, "host-results")
if (not os.path.exists(self.resultpath)):
os.makedirs(self.resultpath)
self.console_thread = OpSOLMonitorThread(1, "console")
self.console_thread.start()
def setUp(self):
conf = OpTestConfiguration.conf
self.cv_SYSTEM = conf.system()
self.console = self.cv_SYSTEM.console
self.cv_HMC = self.cv_SYSTEM.hmc
self.cv_HOST = conf.host()
self.mg_system = self.cv_HMC.mg_system
self.dest_lpar = conf.args.target_lpar_name
self.pci_device = conf.args.pci_device
self.sriov = conf.args.sriov
self.num_of_dlpar = int(conf.args.num_of_dlpar)
self.util = conf.util
self.test_ip = conf.args.test_ip
self.peer_ip = conf.args.peer_ip
self.netmask = conf.args.netmask
self.console_thread = OpSOLMonitorThread(1, "console")
self.console_thread.start()
if not self.cv_HMC.is_lpar_in_managed_system(self.mg_system,
self.cv_HMC.lpar_name):
raise OpTestError("Lpar %s not found in managed system %s" %
(self.cv_HMC.lpar_name, self.mg_system))
if self.dest_lpar is not None:
if not self.cv_HMC.is_lpar_in_managed_system(
self.mg_system, self.dest_lpar):
raise OpTestError("Lpar %s not found in managed system %s" %
(self.dest_lpar, self.mg_system))
self.check_pkg_installation()
self.rsct_service_start()
if self.dest_lpar is not None:
cmd = 'lshwres -r io -m %s --rsubtype slot --filter \
lpar_names=%s -F lpar_id' % (self.mg_system, self.dest_lpar)
output = self.cv_HMC.ssh.run_command(cmd)
self.dest_lpar_id = output[0]
self.loc_code = self.get_slot_from_sysfs(self.pci_device)
self.get_slot_hw_details()
def runTest(self):
con = self.cv_SYSTEM.cv_HOST.get_ssh_connection()
try:
if self.host_cmd:
con.run_command(self.host_cmd, timeout=self.host_cmd_timeout)
if self.host_cmd_file:
if not os.path.isfile(self.host_cmd_file):
self.fail("Provide valid host cmd file path")
fd = open(self.host_cmd_file, "r")
for line in fd.readlines():
line = line.strip()
if "reboot" in line:
self.console_thread.console_terminate()
self.cv_SYSTEM.goto_state(OpSystemState.OFF)
self.cv_SYSTEM.goto_state(OpSystemState.OS)
con = self.cv_SYSTEM.cv_HOST.get_ssh_connection()
self.console_thread = OpSOLMonitorThread(1, "console")
self.console_thread.start()
continue
con.run_command(line, timeout=self.host_cmd_timeout)
finally:
if self.host_cmd_resultpath:
self.cv_SYSTEM.cv_HOST.copy_files_from_host(self.resultpath,
self.host_cmd_resultpath)
def runTest(self):
self.cv_SYSTEM.goto_state(OpSystemState.OS)
self.c = self.cv_SYSTEM.console
self.host.host_check_command("gcc", "cpupower")
self.host.host_run_command("echo 10 > /proc/sys/kernel/printk")
# tlbie test(IPI stress)
# kill any previous existing tlbie_test processes
self.host.ssh.run_command_ignore_fail("pkill -f /tmp/tlbie_test")
self.host.copy_test_file_to_host("tlbie_test.c")
self.host.host_run_command(
"gcc -pthread -o /tmp/tlbie_test /tmp/tlbie_test.c")
cmd = "/tmp/tlbie_test &"
for i in range(self.tlbie_count):
self.host.host_run_command(cmd)
# CPU Governor change tests
torture_time = self.torture_time
govs = self.get_list_of_governors()[-1].split(" ")
log.debug(govs)
cmd_list = []
for gov in govs:
if not gov:
continue
cmd = "for j in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do echo %s > $j; done" % gov
cmd_list.append(cmd)
num = 1
thread = OpSSHThreadLinearVar1(num, "Thread-%s" % num, cmd_list, 2,
torture_time, True)
thread.start()
self.thread_list.append(thread)
# CPU idle states tests, enable only one idle state at a time for certain duration
torture_time = self.torture_time
idle_states = self.get_idle_states()
for i in idle_states:
self.disable_idle_state(i)
time.sleep(0.2)
self.verify_disable_idle_state(i)
cmd_list = {}
for i in idle_states:
cmd = "for i in /sys/devices/system/cpu/cpu*/cpuidle/state%s/disable; do echo 0 > $i; done" % i
cmd_list[cmd] = 1800
cmd = "for i in /sys/devices/system/cpu/cpu*/cpuidle/state%s/disable; do echo 1 > $i; done" % i
cmd_list[cmd] = 10
log.debug(cmd_list)
num = 2
thread = OpSSHThreadLinearVar2(num, "Thread-%s" % num, cmd_list,
torture_time, True)
thread.start()
self.thread_list.append(thread)
torture_time = self.torture_time
# OCC reset reload tests
cmd_list = {
"opal-prd occ reset": 60,
"opal-prd --expert-mode htmgt-passthru 4": 10
}
num = 3
thread = OpSSHThreadLinearVar2(num, "Thread-%s" % num, cmd_list,
torture_time, True)
thread.start()
self.thread_list.append(thread)
# CPU Hotplug torture
torture_time = self.torture_time
num_avail_cores = self.host.host_get_core_count()
smt_range = ["on", "off"] + list(range(1,
self.host.host_get_smt() + 1))
log.debug("Possible smt values: %s" % smt_range)
cmd_list = []
for smt in smt_range:
cmd_list.append("ppc64_cpu --smt=%s" % str(smt))
for core in range(1, num_avail_cores + 1):
cmd_list.append("ppc64_cpu --cores-on=%s" % core)
num = 4
thread = OpSSHThreadLinearVar1(num, "Thread-%s" % num, cmd_list, 5,
torture_time, True)
thread.start()
self.thread_list.append(thread)
# Read frequency
torture_time = self.torture_time
cmd_list = ['ppc64_cpu --frequency']
num = 5
thread = OpSSHThreadLinearVar1(num, "Thread-%s" % num, cmd_list, 2,
torture_time, True)
thread.start()
self.thread_list.append(thread)
# Monitor for errors
num = 6
torture_time = self.torture_time
thread = OpSOLMonitorThread(num,
"Thread-%s" % num,
execution_time=torture_time)
thread.start()
self.thread_list.append(thread)
def lpar_migrate_test(self):
self.util.clear_dmesg()
self.check_pkg_installation()
self.lpm_setup()
if self.stressng_command:
self.execute_stressng()
for iteration in range(1, self.iterations + 1):
if not self.is_RMCActive(self.src_mg_sys):
log.info("RMC service is inactive..!")
self.rmc_service_start(self.src_mg_sys,
output_dir=os.path.join(
"logs_itr" + str(iteration),
"preForwardLPM"))
if not self.inactive_lpm:
self.check_dmesg_errors(
output_dir=os.path.join("logs_itr" +
str(iteration), "preForwardLPM"))
self.util.gather_os_logs(self.os_file_logs,
self.os_cmd_logs,
output_dir=os.path.join(
"logs_itr" + str(iteration),
"preForwardLPM"))
fwd_lpm_logs_monitor_thread = OpSOLMonitorThread(1, "console")
fwd_lpm_logs_monitor_thread.start()
cmd = ''
if self.slot_num:
cmd = self.vnic_options()
self.util.configure_host_ip(self.interface, self.interface_ip,
self.netmask, self.cv_HOST)
self.vnic_ping_test(
output_dir=os.path.join("logs_itr" +
str(iteration), "preForwardLPM"))
if self.inactive_lpm:
self.cv_HMC.poweroff_lpar()
if not self.cv_HMC.migrate_lpar(self.src_mg_sys,
self.dest_mg_sys,
self.options,
cmd,
timeout=self.lpm_timeout):
self.lpm_failed_error(self.src_mg_sys,
output_dir=os.path.join(
"logs_itr" + str(iteration),
"postForwardLPM"))
fwd_lpm_logs_monitor_thread.console_terminate()
if self.inactive_lpm:
self.cv_HMC.poweron_lpar()
time.sleep(60)
if not self.inactive_lpm:
self.check_dmesg_errors(
output_dir=os.path.join("logs_itr" +
str(iteration), "postForwardLPM"))
self.util.gather_os_logs(self.os_file_logs,
self.os_cmd_logs,
output_dir=os.path.join(
"logs_itr" + str(iteration),
"postForwardLPM"))
if not self.is_RMCActive(self.dest_mg_sys):
log.info("RMC service is inactive..!")
self.rmc_service_start(self.dest_mg_sys,
output_dir=os.path.join(
"logs_itr" + str(iteration),
"postForwardLPM"))
if self.slot_num:
cmd = self.vnic_options('remote')
self.vnic_ping_test(
output_dir=os.path.join("logs_itr" +
str(iteration), "postForwardLPM"))
if self.inactive_lpm:
self.cv_HMC.poweroff_lpar()
log.debug("Migrating lpar back to original managed system")
if not self.cv_HMC.migrate_lpar(self.dest_mg_sys,
self.src_mg_sys,
self.options,
cmd,
timeout=self.lpm_timeout):
self.lpm_failed_error(self.dest_mg_sys,
output_dir=os.path.join(
"logs_itr" + str(iteration),
"postBackwardLPM"))
if self.inactive_lpm:
self.cv_HMC.poweron_lpar()
if self.slot_num:
self.vnic_ping_test(
output_dir=os.path.join("logs_itr" +
str(iteration), "postBackwardLPM"))
if not self.inactive_lpm:
self.check_dmesg_errors(
output_dir=os.path.join("logs_itr" +
str(iteration), "postBackwardLPM"))
self.util.gather_os_logs(self.os_file_logs,
self.os_cmd_logs,
output_dir=os.path.join(
"logs_itr" + str(iteration),
"postBackwardLPM"))
class OpTestDlparIO(unittest.TestCase):
def setUp(self):
conf = OpTestConfiguration.conf
self.cv_SYSTEM = conf.system()
self.console = self.cv_SYSTEM.console
self.cv_HMC = self.cv_SYSTEM.hmc
self.cv_HOST = conf.host()
self.mg_system = self.cv_HMC.mg_system
self.dest_lpar = conf.args.target_lpar_name
self.pci_device = conf.args.pci_device
self.sriov = conf.args.sriov
self.num_of_dlpar = int(conf.args.num_of_dlpar)
self.util = conf.util
self.test_ip = conf.args.test_ip
self.peer_ip = conf.args.peer_ip
self.netmask = conf.args.netmask
self.console_thread = OpSOLMonitorThread(1, "console")
self.console_thread.start()
if not self.cv_HMC.is_lpar_in_managed_system(self.mg_system,
self.cv_HMC.lpar_name):
raise OpTestError("Lpar %s not found in managed system %s" %
(self.cv_HMC.lpar_name, self.mg_system))
if self.dest_lpar is not None:
if not self.cv_HMC.is_lpar_in_managed_system(
self.mg_system, self.dest_lpar):
raise OpTestError("Lpar %s not found in managed system %s" %
(self.dest_lpar, self.mg_system))
self.check_pkg_installation()
self.rsct_service_start()
if self.dest_lpar is not None:
cmd = 'lshwres -r io -m %s --rsubtype slot --filter \
lpar_names=%s -F lpar_id' % (self.mg_system, self.dest_lpar)
output = self.cv_HMC.ssh.run_command(cmd)
self.dest_lpar_id = output[0]
self.loc_code = self.get_slot_from_sysfs(self.pci_device)
self.get_slot_hw_details()
def adapter_check(self):
"""
Check device type , configure ip and perform ping test
"""
self.pci_device_type = self.util.get_pci_type(self.pci_device,
self.cv_HOST)
log.info("Type of PCI device is \"%s\"", self.pci_device_type)
if self.pci_device_type == "network":
self_get_interface = self.util.get_interface(
self.pci_device, self.cv_HOST)
log.info("PCI device mapped to interface: \"%s\"",
self_get_interface)
self.util.configure_host_ip(self_get_interface, self.test_ip,
self.netmask, self.cv_HOST)
if self.util.ping_test(self_get_interface, self.peer_ip,
self.cv_HOST):
return True
return False
def check_pkg_installation(self):
"""
Check required packages installed, Continue only if installed.
Raise exception if packages not installed.
"""
pkg_found = True
pkg_notfound = []
self.oslevel = self.cv_HOST.host_get_OS_Level()
lpm_pkg_list = [
"src", "rsct.core", "rsct.core.utils", "rsct.basic",
"rsct.opt.storagerm", "DynamicRM"
]
for pkg in lpm_pkg_list:
pkg_status = self.cv_HOST.host_check_pkg_installed(
self.oslevel, pkg)
if not pkg_status:
pkg_found = False
pkg_notfound.append(pkg)
if pkg_found:
return True
raise OpTestError("Install the required packages : %s" % pkg_notfound)
def rsct_service_start(self):
"""
Check if rsct and rsct_rm services are up and running, if not start the same.
"""
rc = self.cv_HOST.host_run_command("lssrc -a")
if "inoperative" in str(rc):
self.cv_HOST.host_run_command(
"startsrc -g rsct_rm; startsrc -g rsct")
rc = self.cv_HOST.host_run_command("lssrc -a")
if "inoperative" in str(rc):
raise OpTestError(
"LPM cannot continue as some of rsct services are not active"
)
def get_slot_hw_details(self):
"""
Depending upon Pyscial or SRIOV adapters get the HW slot details from HMC.
"""
if self.sriov == "yes":
cmd = "lshwres -r sriov --rsubtype logport -m %s \
--level eth --filter lpar_names=%s -F \
'adapter_id,logical_port_id,phys_port_id,lpar_id,location_code,drc_name'" \
% (self.mg_system, self.cv_HMC.lpar_name)
output = self.cv_HMC.ssh.run_command(cmd)
log.info("output = %s" % output)
for line in output:
if self.loc_code in line:
self.adapter_id = line.split(',')[0]
self.logical_port_id = line.split(',')[1]
self.phys_port_id = line.split(',')[2]
self.lpar_id = line.split(',')[3]
self.location_code = line.split(',')[4]
self.phb = line.split(',')[5].split(' ')[1]
break
log.info("lpar_id : %s, loc_code: %s", self.lpar_id, self.loc_code)
else:
cmd = 'lshwres -r io -m %s --rsubtype slot \
--filter lpar_names=%s -F drc_index,lpar_id,drc_name,bus_id' \
% (self.mg_system, self.cv_HMC.lpar_name)
output = self.cv_HMC.ssh.run_command(cmd)
log.info("output = %s" % output)
for line in output:
if self.loc_code in line:
self.drc_index = line.split(',')[0]
self.lpar_id = line.split(',')[1]
self.phb = line.split(',')[3]
break
log.info("lpar_id : %s, loc_code: %s, drc_index: %s, phb: %s",
self.lpar_id, self.loc_code, self.drc_index, self.phb)
def get_slot_from_sysfs(self, full_pci_address):
"""
Get slot form host sysfs.
"""
try:
devspec = self.cv_HOST.host_run_command(
"cat /sys/bus/pci/devices/%s/devspec" % full_pci_address)
except CommandFailed as c:
log.debug(
"CommandFailed to get devspec, probably the adapter is not attached to the system"
)
self.assertEqual(
c.exitcode, 0,
"Attach the PCI adapter before starting the test : {}".format(
c))
slot = self.cv_HOST.host_run_command(
"cat /proc/device-tree/%s/ibm,loc-code" % devspec[0])
slot_ibm = re.match(r'((\w+)[.])+(\w+)-[PC(\d+)-]*C(\d+)', slot[0])
if slot_ibm:
return slot_ibm.group()
slot_openpower = re.match(r'(\w+)[\s]*(\w+)(\d*)', slot[0])
if slot_openpower:
return slot_openpower.group()
raise OpTestError("Failed to get slot from: '%s'" % full_pci_address)
class InstallUpstreamKernel(unittest.TestCase):
def setUp(self):
self.conf = OpTestConfiguration.conf
self.cv_HOST = self.conf.host()
self.cv_SYSTEM = self.conf.system()
self.host_cmd_timeout = self.conf.args.host_cmd_timeout
self.repo = self.conf.args.git_repo
self.branch = self.conf.args.git_branch
self.home = self.conf.args.git_home
self.config = self.conf.args.git_repoconfig
self.config_path = self.conf.args.git_repoconfigpath
self.disk = self.conf.args.host_scratch_disk
self.patch = self.conf.args.git_patch
self.use_kexec = self.conf.args.use_kexec
self.append_kernel_cmdline = self.conf.args.append_kernel_cmdline
if self.config_path:
self.config = "olddefconfig"
if not self.repo:
self.fail("Provide git repo of kernel to install")
if not (self.conf.args.host_ip and self.conf.args.host_user
and self.conf.args.host_password):
self.fail(
"Provide host ip user details refer, --host-{ip,user,password}"
)
if self.disk:
OpIU = InstallUtil()
self.cv_SYSTEM.goto_state(OpSystemState.PETITBOOT_SHELL)
log.debug("Set given disk as default bootable disk")
OpIU.set_bootable_disk(self.disk)
self.console_thread = OpSOLMonitorThread(1, "console")
def runTest(self):
def is_url(path):
'''
param path: path to download
return: boolean True if given path is url False Otherwise
'''
valid_schemes = ['http', 'https', 'git', 'ftp']
if urlparse(path).scheme in valid_schemes:
return True
return False
self.cv_SYSTEM.goto_state(OpSystemState.OS)
self.console_thread.start()
try:
con = self.cv_SYSTEM.cv_HOST.get_ssh_connection()
try:
onlinecpus = int(
con.run_command("lscpu --online -e|wc -l")[-1])
except Exception:
onlinecpus = 20
log.debug("Downloading linux kernel")
# Compile and install given kernel
linux_path = os.path.join(self.home, "linux")
con.run_command("[ -d %s ] || mkdir -p %s" %
(self.home, self.home))
con.run_command("if [ -d %s ];then rm -rf %s;fi" %
(linux_path, linux_path))
con.run_command("cd %s && git clone --depth 1 %s -b %s linux" %
(self.home, self.repo, self.branch),
timeout=self.host_cmd_timeout)
con.run_command("cd %s" % linux_path)
if self.patch:
patch_file = self.patch.split("/")[-1]
if is_url(self.patch):
con.run_command("wget %s -O %s" % (self.patch, patch_file))
else:
self.cv_HOST.copy_test_file_to_host(self.patch,
dstdir=linux_path)
log.debug("Applying given patch")
con.run_command("git am %s" %
os.path.join(linux_path, patch_file))
log.debug("Downloading linux kernel config")
if self.config_path:
if is_url(self.config_path):
con.run_command("wget %s -O .config" % self.config_path)
else:
self.cv_HOST.copy_test_file_to_host(self.config_path,
dstdir=os.path.join(
linux_path,
".config"))
con.run_command("make %s" % self.config)
log.debug("Compile and install linux kernel")
con.run_command(
"make -j %d -s && make modules && make modules_install && make install"
% onlinecpus,
timeout=self.host_cmd_timeout)
if not self.use_kexec:
# FIXME: Handle distributions which do not support grub
con.run_command(
"grub2-mkconfig --output=/boot/grub2/grub.cfg")
con.run_command(
'grubby --set-default /boot/vmlinuz-`cat include/config/kernel.release 2> /dev/null`'
)
log.debug("Rebooting after kernel install...")
self.console_thread.console_terminate()
con.close()
self.cv_SYSTEM.goto_state(OpSystemState.OFF)
self.cv_SYSTEM.goto_state(OpSystemState.OS)
else:
self.console_thread.console_terminate()
cmdline = con.run_command("cat /proc/cmdline")[-1]
if self.append_kernel_cmdline:
cmdline += " %s" % self.append_kernel_cmdline
kern_rel_str = con.run_command(
"cat %s/include/config/kernel.release" % linux_path)[-1]
initrd_file = con.run_command("ls -l /boot/initr*-%s*" %
kern_rel_str)[-1].split(" ")[-1]
kexec_cmdline = "kexec --initrd %s --command-line=\"%s\" /boot/vmlinuz-%s -l" % (
initrd_file, cmdline, kern_rel_str)
# Let's makesure we set the default boot index to current kernel
# to avoid leaving host in unstable state incase boot failure
con.run_command(
'grubby --set-default /boot/vmlinuz-`uname -r 2> /dev/null`'
)
con.run_command(kexec_cmdline)
con.close()
raw_pty = self.cv_SYSTEM.console.get_console()
raw_pty.sendline("kexec -e")
raw_pty.expect("login:"******"uname -r")
log.info("Installed upstream kernel version: %s", res[-1])
if self.conf.args.host_cmd:
con.run_command(self.conf.args.host_cmd,
timeout=self.host_cmd_timeout)
self.cv_HOST.host_gather_opal_msg_log()
self.cv_HOST.host_gather_kernel_log()
finally:
if self.console_thread.isAlive():
self.console_thread.console_terminate()
def cross_hmc_migrate_test(self):
self.util.clear_dmesg()
self.check_pkg_installation()
self.lpm_setup()
if self.stressng_command:
self.execute_stressng()
for iteration in range(1, self.iterations + 1):
if not self.is_RMCActive(self.src_mg_sys):
log.info("RMC service is inactive..!")
self.rmc_service_start(self.src_mg_sys,
output_dir=os.path.join(
"logs_itr" + str(iteration),
"preForwardLPM"))
if not self.inactive_lpm:
self.check_dmesg_errors(output_dir=os.path.join(
"logs_itr" + str(iteration), "preForwardLPM"),
remote_hmc=self.remote_hmc)
self.util.gather_os_logs(self.os_file_logs,
self.os_cmd_logs,
output_dir=os.path.join(
"logs_itr" + str(iteration),
"preForwardLPM"))
fwd_lpm_logs_monitor_thread = OpSOLMonitorThread(1, "console")
fwd_lpm_logs_monitor_thread.start()
cmd = ''
if self.slot_num:
cmd = self.vnic_options()
self.util.configure_host_ip(self.interface, self.interface_ip,
self.netmask, self.cv_HOST)
self.vnic_ping_test(
output_dir=os.path.join("logs_itr" +
str(iteration), "preForwardLPM"))
if self.inactive_lpm:
self.cv_HMC.poweroff_lpar()
self.cv_HMC.cross_hmc_migration(self.src_mg_sys,
self.dest_mg_sys,
self.target_hmc_ip,
self.target_hmc_username,
self.target_hmc_password,
options=self.options,
param=cmd,
timeout=self.lpm_timeout)
log.debug("Waiting for %.2f minutes." % (self.lpm_timeout / 60))
time.sleep(self.lpm_timeout)
fwd_lpm_logs_monitor_thread.console_terminate()
if self.inactive_lpm:
self.cv_HMC.poweron_lpar(remote_hmc=self.remote_hmc)
time.sleep(60)
if not self.inactive_lpm:
self.check_dmesg_errors(output_dir=os.path.join(
"logs_itr" + str(iteration), "postForwardLPM"),
remote_hmc=self.remote_hmc)
self.util.gather_os_logs(self.os_file_logs,
self.os_cmd_logs,
output_dir=os.path.join(
"logs_itr" + str(iteration),
"postForwardLPM"))
if not self.is_RMCActive(self.dest_mg_sys, self.remote_hmc):
log.info("RMC service is inactive..!")
self.rmc_service_start(self.dest_mg_sys,
self.remote_hmc,
output_dir=os.path.join(
"logs_itr" + str(iteration),
"postForwardLPM"))
if self.slot_num:
cmd = self.vnic_options('remote')
self.vnic_ping_test(
output_dir=os.path.join("logs_itr" +
str(iteration), "postForwardLPM"))
if self.inactive_lpm:
self.cv_HMC.poweroff_lpar(remote_hmc=self.remote_hmc)
self.cv_HMC.cross_hmc_migration(self.dest_mg_sys,
self.src_mg_sys,
self.cv_HMC.hmc_ip,
self.cv_HMC.user,
self.cv_HMC.passwd,
self.remote_hmc,
options=self.options,
param=cmd,
timeout=self.lpm_timeout)
log.debug("Waiting for %.2f minutes." % (self.lpm_timeout / 60))
time.sleep(self.lpm_timeout)
if self.inactive_lpm:
self.cv_HMC.poweron_lpar()
if self.slot_num:
self.vnic_ping_test(
output_dir=os.path.join("logs_itr" +
str(iteration), "postBackwardLPM"))
if not self.inactive_lpm:
self.check_dmesg_errors(output_dir=os.path.join(
"logs_itr" + str(iteration), "postBackwardLPM"),
remote_hmc=self.remote_hmc)
self.util.gather_os_logs(self.os_file_logs,
self.os_cmd_logs,
output_dir=os.path.join(
"logs_itr" + str(iteration),
"postBackwardLPM"))
class OpTestDlpar(unittest.TestCase):
def setUp(self):
conf = OpTestConfiguration.conf
self.cv_SYSTEM = conf.system()
self.console = self.cv_SYSTEM.console
conf = OpTestConfiguration.conf
self.hmc_user = conf.args.hmc_username
self.hmc_password = conf.args.hmc_password
self.hmc_ip = conf.args.hmc_ip
self.lpar_name = conf.args.lpar_name
self.system_name = conf.args.system_name
self.cv_HMC = self.cv_SYSTEM.hmc
# The following variables needs to be defined in
# ~/.op-test-framework.conf
self.cpu_resource = conf.args.cpu_resource
self.mem_resource = conf.args.mem_resource
self.lpar2_name = conf.args.lpar2_name
self.loop_num = conf.args.loop_num
self.console_thread = OpSOLMonitorThread(1, "console")
self.console_thread.start()
self.extended = {'loop':0,'wkld':0,'smt':0}
def AddRemove(self,res_type,res_option,operation,res_num):
#Generate a random number of CPU or memory to add or remove
res_num = int(res_num)
if (res_type == "proc"):
#for x in range(res_num):
res_num = randint(1,res_num)
log.debug("Random number of resource generated for CPU is %s"%res_num)
if (res_type == "mem"):
res_mem = int(res_num/1024)
#for x in range(res_mem):
res_num = (randint(1,res_mem))*1024
log.debug("Random number of memory generated for MEM is %s"%res_num)
if(self.extended['loop'] != 1):
log.debug("Loop is not on")
self.loop_num = 1
else:
log.debug("Loop is on repeat %s %s times" %(operation,self.loop_num))
count=0
while (count < int(self.loop_num)):
log.debug("Iteration %s"%count)
if(self.extended['smt'] == 1):
log.debug("smt is on, executing the script")
self.console.run_command("nohup ./smt_script &")
if(self.extended['wkld'] == 1):
log.debug("Starting workload..")
self.console.run_command("htxscreen -f mdt.all")
self.extended['wkld'] = 0
log.debug("lshwres -r %s -m %s --level lpar --filter lpar_names=%s"
%(res_type, self.system_name, self.lpar_name))
self.cv_HMC.run_command("lshwres -r %s -m %s --level lpar --filter lpar_names=%s"
%(res_type, self.system_name, self.lpar_name))
log.debug("time chhwres -r %s -m %s -o %s -p %s %s %s"
%(res_type,self.system_name,operation,self.lpar_name,
res_option,res_num))
self.cv_HMC.run_command_ignore_fail("time chhwres -r %s -m %s -o %s -p %s %s %s"
%(res_type,self.system_name,operation,self.lpar_name,
res_option,res_num))
log.debug("time lshwres -r %s -m %s --level lpar --filter lpar_names=%s"
%(res_type,self.system_name,self.lpar_name))
self.cv_HMC.run_command("lshwres -r %s -m %s --level lpar --filter lpar_names=%s"
%(res_type,self.system_name,self.lpar_name))
count=count+1
def Move(self,res_type,res_option,res_num):
if(self.extended['loop'] != 1):
log.debug("Loop is not on")
self.loop_num = 1
else:
log.debug("Loop is on repeat %s times" %self.loop_num)
count=0
while (count < int(self.loop_num)):
log.debug("Iteration %s"%count)
if(self.extended['smt'] == 1):
log.debug("smt is on, executing the script")
self.console.run_command("nohup ./smt_script &")
if(self.extended['wkld'] == 1):
log.debug("Starting workload..")
self.console.run_command("htxscreen -f mdt.all")
self.extended['wkld'] = 0
log.debug("lshwres -r %s -m %s --level lpar --filter lpar_names=%s"
%(res_type,self.system_name,self.lpar_name))
self.cv_HMC.run_command("lshwres -r %s -m %s --level lpar --filter lpar_names=%s"
%(res_type,self.system_name,self.lpar_name))
log.debug("lshwres -r %s -m %s --level lpar --filter lpar_names=%s"
%(res_type,self.system_name,self.lpar2_name))
self.cv_HMC.run_command("lshwres -r %s -m %s --level lpar --filter lpar_names=%s"
%(res_type,self.system_name,self.lpar2_name))
log.debug("time chhwres -r %s -m %s -o m -p %s -t %s %s %s"
% (res_type,self.system_name,self.lpar_name,self.lpar2_name,res_option,res_num))
self.cv_HMC.run_command_ignore_fail("time chhwres -r %s -m %s -o m -p %s -t %s %s %s"
% (res_type,self.system_name,self.lpar_name,self.lpar2_name,res_option,res_num))
log.debug("lshwres -r %s -m %s --level lpar --filter lpar_names=%s"
% (res_type,self.system_name,self.lpar_name))
self.cv_HMC.run_command("lshwres -r %s -m %s --level lpar --filter lpar_names=%s"
% (res_type,self.system_name,self.lpar_name))
log.debug("lshwres -r %s -m %s --level lpar --filter lpar_names=%s"
% (res_type,self.system_name,self.lpar2_name))
self.cv_HMC.run_command("lshwres -r %s -m %s --level lpar --filter lpar_names=%s"
% (res_type,self.system_name,self.lpar2_name))
count = count+1
def runTest(self):
self.cv_SYSTEM.goto_state(OpSystemState.OS)
self.c = self.cv_SYSTEM.console
self.host.host_check_command("gcc", "cpupower")
self.host.host_run_command("echo 10 > /proc/sys/kernel/printk")
# tlbie test(IPI stress)
# kill any previous existing tlbie_test processes
self.host.ssh.run_command_ignore_fail("pkill -f /tmp/tlbie_test")
self.host.copy_test_file_to_host("tlbie_test.c")
self.host.host_run_command("gcc -pthread -o /tmp/tlbie_test /tmp/tlbie_test.c")
cmd = "/tmp/tlbie_test &"
for i in range(self.tlbie_count):
self.host.host_run_command(cmd)
# CPU Governor change tests
torture_time = self.torture_time
govs = self.get_list_of_governors()[-1].split(" ")
log.debug(govs)
cmd_list = []
for gov in govs:
if not gov:
continue
cmd = "for j in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do echo %s > $j; done" % gov
cmd_list.append(cmd)
num = 1
thread = OpSSHThreadLinearVar1(num, "Thread-%s" % num, cmd_list, 2, torture_time, True)
thread.start()
self.thread_list.append(thread)
# CPU idle states tests, enable only one idle state at a time for certain duration
torture_time = self.torture_time
idle_states = self.get_idle_states()
for i in idle_states:
self.disable_idle_state(i)
time.sleep(0.2)
self.verify_disable_idle_state(i)
cmd_list = {}
for i in idle_states:
cmd = "for i in /sys/devices/system/cpu/cpu*/cpuidle/state%s/disable; do echo 0 > $i; done" % i
cmd_list[cmd] = 1800
cmd = "for i in /sys/devices/system/cpu/cpu*/cpuidle/state%s/disable; do echo 1 > $i; done" % i
cmd_list[cmd] = 10
log.debug(cmd_list)
num = 2
thread = OpSSHThreadLinearVar2(num, "Thread-%s" % num, cmd_list, torture_time, True)
thread.start()
self.thread_list.append(thread)
torture_time = self.torture_time
# OCC reset reload tests
cmd_list = {"opal-prd occ reset":60, "opal-prd --expert-mode htmgt-passthru 4":10}
num = 3
thread = OpSSHThreadLinearVar2(num, "Thread-%s" % num, cmd_list, torture_time, True)
thread.start()
self.thread_list.append(thread)
# CPU Hotplug torture
torture_time = self.torture_time
num_avail_cores = self.host.host_get_core_count()
smt_range = ["on", "off"] + range(1, self.host.host_get_smt()+1)
log.debug("Possible smt values: %s" % smt_range)
cmd_list = []
for smt in smt_range:
cmd_list.append("ppc64_cpu --smt=%s" % str(smt))
for core in range(1, num_avail_cores + 1):
cmd_list.append("ppc64_cpu --cores-on=%s" % core)
num = 4
thread = OpSSHThreadLinearVar1(num, "Thread-%s" % num, cmd_list, 5, torture_time, True)
thread.start()
self.thread_list.append(thread)
# Read frequency
torture_time = self.torture_time
cmd_list = ['ppc64_cpu --frequency']
num = 5
thread = OpSSHThreadLinearVar1(num, "Thread-%s" % num, cmd_list, 2, torture_time, True)
thread.start()
self.thread_list.append(thread)
# Monitor for errors
num = 6
torture_time = self.torture_time
thread = OpSOLMonitorThread(num, "Thread-%s" % num, execution_time=torture_time)
thread.start()
self.thread_list.append(thread)
class InstallUpstreamKernel(unittest.TestCase):
def setUp(self):
self.conf = OpTestConfiguration.conf
self.cv_HOST = self.conf.host()
self.cv_SYSTEM = self.conf.system()
self.host_cmd_timeout = self.conf.args.host_cmd_timeout
self.repo = self.conf.args.git_repo
self.branch = self.conf.args.git_branch
self.home = self.conf.args.git_home
self.config = self.conf.args.git_repoconfig
self.config_path = self.conf.args.git_repoconfigpath
self.disk = self.conf.args.host_scratch_disk
self.patch = self.conf.args.git_patch
self.use_kexec = self.conf.args.use_kexec
self.append_kernel_cmdline = self.conf.args.append_kernel_cmdline
if self.config_path:
self.config = "olddefconfig"
if not self.repo:
self.fail("Provide git repo of kernel to install")
if not (self.conf.args.host_ip and self.conf.args.host_user and self.conf.args.host_password):
self.fail("Provide host ip user details refer, --host-{ip,user,password}")
if self.disk:
OpIU = InstallUtil()
self.cv_SYSTEM.goto_state(OpSystemState.PETITBOOT_SHELL)
log.debug("Set given disk as default bootable disk")
OpIU.set_bootable_disk(self.disk)
self.console_thread = OpSOLMonitorThread(1, "console")
def runTest(self):
def is_url(path):
'''
param path: path to download
return: boolean True if given path is url False Otherwise
'''
valid_schemes = ['http', 'https', 'git', 'ftp']
if urlparse(path).scheme in valid_schemes:
return True
return False
self.cv_SYSTEM.goto_state(OpSystemState.OS)
self.console_thread.start()
try:
con = self.cv_SYSTEM.cv_HOST.get_ssh_connection()
try:
onlinecpus = int(con.run_command("lscpu --online -e|wc -l")[-1])
except Exception:
onlinecpus = 20
log.debug("Downloading linux kernel")
# Compile and install given kernel
linux_path = os.path.join(self.home, "linux")
con.run_command("[ -d %s ] || mkdir -p %s" % (self.home, self.home))
con.run_command("if [ -d %s ];then rm -rf %s;fi" % (linux_path, linux_path))
con.run_command("cd %s && git clone --depth 1 %s -b %s linux" % (self.home, self.repo, self.branch), timeout=self.host_cmd_timeout)
con.run_command("cd %s" % linux_path)
if self.patch:
patch_file = self.patch.split("/")[-1]
if is_url(self.patch):
con.run_command("wget %s -O %s" % (self.patch, patch_file))
else:
self.cv_HOST.copy_test_file_to_host(self.patch, dstdir=linux_path)
log.debug("Applying given patch")
con.run_command("git am %s" % os.path.join(linux_path, patch_file))
log.debug("Downloading linux kernel config")
if self.config_path:
if is_url(self.config_path):
con.run_command("wget %s -O .config" % self.config_path)
else:
self.cv_HOST.copy_test_file_to_host(self.config_path, dstdir=os.path.join(linux_path, ".config"))
con.run_command("make %s" % self.config)
log.debug("Compile and install linux kernel")
con.run_command("make -j %d -s && make modules && make modules_install && make install" % onlinecpus, timeout=self.host_cmd_timeout)
if not self.use_kexec:
# FIXME: Handle distributions which do not support grub
con.run_command("grub2-mkconfig --output=/boot/grub2/grub.cfg")
con.run_command('grubby --set-default /boot/vmlinuz-`cat include/config/kernel.release 2> /dev/null`')
log.debug("Rebooting after kernel install...")
self.console_thread.console_terminate()
con.close()
self.cv_SYSTEM.goto_state(OpSystemState.OFF)
self.cv_SYSTEM.goto_state(OpSystemState.OS)
else:
self.console_thread.console_terminate()
cmdline = con.run_command("cat /proc/cmdline")[-1]
if self.append_kernel_cmdline:
cmdline += " %s" % self.append_kernel_cmdline
kern_rel_str = con.run_command("cat %s/include/config/kernel.release" % linux_path)[-1]
initrd_file = con.run_command("ls -l /boot/initr*-%s*" % kern_rel_str)[-1].split(" ")[-1]
kexec_cmdline = "kexec --initrd %s --command-line=\"%s\" /boot/vmlinuz-%s -l" % (initrd_file, cmdline, kern_rel_str)
# Let's makesure we set the default boot index to current kernel
# to avoid leaving host in unstable state incase boot failure
con.run_command('grubby --set-default /boot/vmlinuz-`uname -r 2> /dev/null`')
con.run_command(kexec_cmdline)
con.close()
raw_pty = self.cv_SYSTEM.console.get_console()
raw_pty.sendline("kexec -e")
raw_pty.expect("login:"******"uname -r")
log.info("Installed upstream kernel version: %s", res[-1])
if self.conf.args.host_cmd:
con.run_command(self.conf.args.host_cmd, timeout=self.host_cmd_timeout)
self.cv_HOST.host_gather_opal_msg_log()
self.cv_HOST.host_gather_kernel_log()
finally:
if self.console_thread.isAlive():
self.console_thread.console_terminate()