def output_check(nodeinfo_output):
# Check CPU model
cpu_model_nodeinfo = _check_nodeinfo(nodeinfo_output, "CPU model", 3)
cpu_model_os = utils.get_current_kernel_arch()
if not re.match(cpu_model_nodeinfo, cpu_model_os):
raise error.TestFail(
"Virsh nodeinfo output didn't match CPU model")
# Check number of CPUs
cpus_nodeinfo = _check_nodeinfo(nodeinfo_output, "CPU(s)", 2)
cpus_os = utils.count_cpus()
if int(cpus_nodeinfo) != cpus_os:
raise error.TestFail(
"Virsh nodeinfo output didn't match number of "
"CPU(s)")
# Check CPU frequency
cpu_frequency_nodeinfo = _check_nodeinfo(nodeinfo_output,
'CPU frequency', 3)
cmd = ("cat /proc/cpuinfo | grep 'cpu MHz' | head -n1 | "
"awk '{print $4}' | awk -F. '{print $1}'")
cmd_result = utils.run(cmd, ignore_status=True)
cpu_frequency_os = cmd_result.stdout.strip()
print cpu_frequency_os
if not re.match(cpu_frequency_nodeinfo, cpu_frequency_os):
raise error.TestFail("Virsh nodeinfo output didn't match CPU "
"frequency")
# Check CPU socket(s)
cpu_sockets_nodeinfo = int(
_check_nodeinfo(nodeinfo_output, 'CPU socket(s)', 3))
cmd = "grep 'physical id' /proc/cpuinfo | uniq | sort | uniq |wc -l"
cmd_result = utils.run(cmd, ignore_status=True)
cpu_NUMA_nodeinfo = _check_nodeinfo(nodeinfo_output, 'NUMA cell(s)', 3)
cpu_sockets_os = int(
cmd_result.stdout.strip()) / int(cpu_NUMA_nodeinfo)
if cpu_sockets_os != cpu_sockets_nodeinfo:
raise error.TestFail("Virsh nodeinfo output didn't match CPU "
"socket(s)")
# Check Core(s) per socket
cores_per_socket_nodeinfo = _check_nodeinfo(nodeinfo_output,
'Core(s) per socket', 4)
cmd = "grep 'cpu cores' /proc/cpuinfo | head -n1 | awk '{print $4}'"
cmd_result = utils.run(cmd, ignore_status=True)
cores_per_socket_os = cmd_result.stdout.strip()
if not re.match(cores_per_socket_nodeinfo, cores_per_socket_os):
raise error.TestFail("Virsh nodeinfo output didn't match Core(s) "
"per socket")
# Check Memory size
memory_size_nodeinfo = int(
_check_nodeinfo(nodeinfo_output, 'Memory size', 3))
memory_size_os = utils.memtotal()
if memory_size_nodeinfo != memory_size_os:
raise error.TestFail("Virsh nodeinfo output didn't match "
"Memory size")
def run_once(self, disks=None, gigabytes=None, chunk_mb=None):
"""
Runs one iteration of disktest.
@param disks: List of directories (usually mountpoints) to be passed
to the test.
@param gigabytes: Disk space that will be used for the test to run.
@param chunk_mb: Size of the portion of the disk used to run the test.
Cannot be larger than the total amount of free RAM.
"""
os.chdir(self.disk_srcdir)
if chunk_mb is None:
chunk_mb = utils.memtotal() / 1024 / 8
if disks is None:
disks = [self.tmpdir]
if gigabytes is None:
free = 100 # cap it at 100GB by default
for disk in disks:
free = min(utils.freespace(disk) / 1024**3, free)
gigabytes = free
logging.info("Resizing to %s GB", gigabytes)
sys.stdout.flush()
self.chunk_mb = chunk_mb
self.memory_mb = utils.memtotal() / 1024 / 8
if self.memory_mb > chunk_mb:
raise error.TestError("Too much RAM (%dMB) for this test to work" %
self.memory_mb)
chunks = (1024 * gigabytes) / chunk_mb
logging.info("Total of disk chunks that will be used: %s", chunks)
for i in range(chunks):
pids = []
for disk in disks:
pid = self.test_one_disk_chunk(disk, i)
pids.append(pid)
errors = []
for pid in pids:
(junk, retval) = os.waitpid(pid, 0)
if (retval != 0):
errors.append(retval)
if errors:
raise error.TestError("Errors from children: %s" % errors)
def run_once(self, disks=None, gigabytes=None, chunk_mb=None):
"""
Runs one iteration of disktest.
@param disks: List of directories (usually mountpoints) to be passed
to the test.
@param gigabytes: Disk space that will be used for the test to run.
@param chunk_mb: Size of the portion of the disk used to run the test.
Cannot be larger than the total amount of free RAM.
"""
os.chdir(self.srcdir)
if chunk_mb is None:
chunk_mb = utils.memtotal() / 1024
if disks is None:
disks = [self.tmpdir]
if gigabytes is None:
free = 100 # cap it at 100GB by default
for disk in disks:
free = min(utils.freespace(disk) / 1024**3, free)
gigabytes = free
logging.info("Resizing to %s GB", gigabytes)
sys.stdout.flush()
self.chunk_mb = chunk_mb
self.memory_mb = utils.memtotal()/1024
if self.memory_mb > chunk_mb:
raise error.TestError("Too much RAM (%dMB) for this test to work" %
self.memory_mb)
chunks = (1024 * gigabytes) / chunk_mb
logging.info("Total of disk chunks that will be used: %s", chunks)
for i in range(chunks):
pids = []
for disk in disks:
pid = self.test_one_disk_chunk(disk, i)
pids.append(pid)
errors = []
for pid in pids:
(junk, retval) = os.waitpid(pid, 0)
if (retval != 0):
errors.append(retval)
if errors:
raise error.TestError("Errors from children: %s" % errors)
def output_check(nodeinfo_output):
# Check CPU model
cpu_model_nodeinfo = _check_nodeinfo(nodeinfo_output, "CPU model", 3)
cpu_model_os = utils.get_current_kernel_arch()
if not re.match(cpu_model_nodeinfo, cpu_model_os):
raise error.TestFail("Virsh nodeinfo output didn't match CPU model")
# Check number of CPUs
cpus_nodeinfo = _check_nodeinfo(nodeinfo_output, "CPU(s)", 2)
cpus_os = utils.count_cpus()
if int(cpus_nodeinfo) != cpus_os:
raise error.TestFail("Virsh nodeinfo output didn't match number of "
"CPU(s)")
# Check CPU frequency
cpu_frequency_nodeinfo = _check_nodeinfo(nodeinfo_output, 'CPU frequency', 3)
cmd = ("cat /proc/cpuinfo | grep 'cpu MHz' | head -n1 | "
"awk '{print $4}' | awk -F. '{print $1}'")
cmd_result = utils.run(cmd, ignore_status=True)
cpu_frequency_os = cmd_result.stdout.strip()
print cpu_frequency_os
if not re.match(cpu_frequency_nodeinfo, cpu_frequency_os):
raise error.TestFail("Virsh nodeinfo output didn't match CPU "
"frequency")
# Check CPU socket(s)
cpu_sockets_nodeinfo = int(_check_nodeinfo(nodeinfo_output, 'CPU socket(s)', 3))
cmd = "grep 'physical id' /proc/cpuinfo | uniq | sort | uniq |wc -l"
cmd_result = utils.run(cmd, ignore_status=True)
cpu_NUMA_nodeinfo = _check_nodeinfo(nodeinfo_output, 'NUMA cell(s)', 3)
cpu_sockets_os = int(cmd_result.stdout.strip())/int(cpu_NUMA_nodeinfo)
if cpu_sockets_os != cpu_sockets_nodeinfo:
raise error.TestFail("Virsh nodeinfo output didn't match CPU "
"socket(s)")
# Check Core(s) per socket
cores_per_socket_nodeinfo = _check_nodeinfo(nodeinfo_output, 'Core(s) per socket', 4)
cmd = "grep 'cpu cores' /proc/cpuinfo | head -n1 | awk '{print $4}'"
cmd_result = utils.run(cmd, ignore_status=True)
cores_per_socket_os = cmd_result.stdout.strip()
if not re.match(cores_per_socket_nodeinfo, cores_per_socket_os):
raise error.TestFail("Virsh nodeinfo output didn't match Core(s) "
"per socket")
# Check Memory size
memory_size_nodeinfo = int(_check_nodeinfo(nodeinfo_output, 'Memory size', 3))
memory_size_os = utils.memtotal()
if memory_size_nodeinfo != memory_size_os:
raise error.TestFail("Virsh nodeinfo output didn't match "
"Memory size")
def run_the_test(self, iterations):
utils.write_one_line("/proc/sys/vm/dirty_ratio", "4")
utils.write_one_line("/proc/sys/vm/dirty_background_ratio", "2")
cmd = os.path.join(self.srcdir, "linus_stress")
args = "%d" % (utils.memtotal() / 32)
profilers = self.job.profilers
if profilers.present():
profilers.start(self)
for i in range(iterations):
utils.system(cmd + " " + args)
if profilers.present():
profilers.stop(self)
profilers.report(self)
def run_the_test(self, iterations):
utils.write_one_line('/proc/sys/vm/dirty_ratio', '4')
utils.write_one_line('/proc/sys/vm/dirty_background_ratio', '2')
cmd = os.path.join(self.srcdir, 'linus_stress')
args = "%d" % (utils.memtotal() / 32)
profilers = self.job.profilers
if profilers.present():
profilers.start(self)
for i in range(iterations):
utils.system(cmd + ' ' + args)
if profilers.present():
profilers.stop(self)
profilers.report(self)
def discover_container_style():
global super_root_path, cpuset_prefix
global mem_isolation_on, fake_numa_containers
global node_mbytes, root_container_bytes
if super_root_path != '':
return # already looked up
if os.path.exists('/dev/cgroup/tasks'):
# running on 2.6.26 or later kernel with containers on:
super_root_path = '/dev/cgroup'
cpuset_prefix = 'cpuset.'
if get_boot_numa():
mem_isolation_on = fake_numa_containers = True
else: # memcg containers IFF compiled-in & mounted & non-fakenuma boot
fake_numa_containers = False
mem_isolation_on = os.path.exists(
'/dev/cgroup/memory.limit_in_bytes')
# TODO: handle possibility of where memcg is mounted as its own
# cgroup hierarchy, separate from cpuset??
elif os.path.exists('/dev/cpuset/tasks'):
# running on 2.6.18 kernel with containers on:
super_root_path = '/dev/cpuset'
cpuset_prefix = ''
mem_isolation_on = fake_numa_containers = get_boot_numa() != ''
else:
# neither cpuset nor cgroup filesystem active:
super_root_path = None
cpuset_prefix = 'no_cpusets_or_cgroups_exist'
mem_isolation_on = fake_numa_containers = False
logging.debug('mem_isolation: %s', mem_isolation_on)
logging.debug('fake_numa_containers: %s', fake_numa_containers)
if fake_numa_containers:
node_mbytes = int(mbytes_per_mem_node())
elif mem_isolation_on: # memcg-style containers
# For now, limit total of all containers to using just 98% of system's
# visible total ram, to avoid oom events at system level, and avoid
# page reclaim overhead from going above kswapd highwater mark.
system_visible_pages = utils.memtotal() >> 2
usable_pages = int(system_visible_pages * 0.98)
root_container_bytes = usable_pages << 12
logging.debug('root_container_bytes: %s',
utils.human_format(root_container_bytes))
def discover_container_style():
global super_root_path, cpuset_prefix
global mem_isolation_on, fake_numa_containers
global node_mbytes, root_container_bytes
if super_root_path != '':
return # already looked up
if os.path.exists('/dev/cgroup/tasks'):
# running on 2.6.26 or later kernel with containers on:
super_root_path = '/dev/cgroup'
cpuset_prefix = 'cpuset.'
if get_boot_numa():
mem_isolation_on = fake_numa_containers = True
else: # memcg containers IFF compiled-in & mounted & non-fakenuma boot
fake_numa_containers = False
mem_isolation_on = os.path.exists(
'/dev/cgroup/memory.limit_in_bytes')
# TODO: handle possibility of where memcg is mounted as its own
# cgroup hierarchy, separate from cpuset??
elif os.path.exists('/dev/cpuset/tasks'):
# running on 2.6.18 kernel with containers on:
super_root_path = '/dev/cpuset'
cpuset_prefix = ''
mem_isolation_on = fake_numa_containers = get_boot_numa() != ''
else:
# neither cpuset nor cgroup filesystem active:
super_root_path = None
cpuset_prefix = 'no_cpusets_or_cgroups_exist'
mem_isolation_on = fake_numa_containers = False
logging.debug('mem_isolation: %s', mem_isolation_on)
logging.debug('fake_numa_containers: %s', fake_numa_containers)
if fake_numa_containers:
node_mbytes = int(mbytes_per_mem_node())
elif mem_isolation_on: # memcg-style containers
# For now, limit total of all containers to using just 98% of system's
# visible total ram, to avoid oom events at system level, and avoid
# page reclaim overhead from going above kswapd highwater mark.
system_visible_pages = utils.memtotal() >> 2
usable_pages = int(system_visible_pages * 0.98)
root_container_bytes = usable_pages << 12
logging.debug('root_container_bytes: %s',
utils.human_format(root_container_bytes))
def execute(self, testdir = None, iterations = 10000):
if not testdir:
testdir = self.tmpdir
os.chdir(testdir)
file = os.path.join(testdir, 'foo')
# Want to use 3/4 of all memory for each of
# bash-shared-mapping and usemem
kilobytes = (3 * utils.memtotal()) / 4
# Want two usemem -m megabytes in parallel in background.
pid = [None, None]
usemem = os.path.join(self.srcdir, 'usemem')
args = ('usemem', '-N', '-m', '%d' % (kilobytes / 1024))
# print_to_tty ('2 x ' + ' '.join(args))
for i in (0,1):
pid[i] = os.spawnv(os.P_NOWAIT, usemem, args)
cmd = "%s/bash-shared-mapping %s %d -t %d -n %d" % \
(self.srcdir, file, kilobytes,
utils.count_cpus(), iterations)
os.system(cmd)
for i in (0, 1):
os.kill(pid[i], signal.SIGKILL)
def execute(self, testdir=None, iterations=10000):
if not testdir:
testdir = self.tmpdir
os.chdir(testdir)
file = os.path.join(testdir, 'foo')
# Want to use 3/4 of all memory for each of
# bash-shared-mapping and usemem
kilobytes = (3 * utils.memtotal()) / 4
# Want two usemem -m megabytes in parallel in background.
pid = [None, None]
usemem = os.path.join(self.srcdir, 'usemem')
args = ('usemem', '-N', '-m', '%d' % (kilobytes / 1024))
# print_to_tty ('2 x ' + ' '.join(args))
for i in (0, 1):
pid[i] = os.spawnv(os.P_NOWAIT, usemem, args)
cmd = "%s/bash-shared-mapping %s %d -t %d -n %d" % \
(self.srcdir, file, kilobytes,
utils.count_cpus(), iterations)
os.system(cmd)
for i in (0, 1):
os.kill(pid[i], signal.SIGKILL)
utils.run("echo 'never' > %s" % e_rh)
new_ksm = True
else:
try:
utils.run("modprobe ksm")
utils.run("ksmctl start 5000 100")
except error.CmdError, e:
raise error.TestFail("Failed to load KSM: %s" % e)
# host_reserve: mem reserve kept for the host system to run
host_reserve = int(params.get("ksm_host_reserve", -1))
if (host_reserve == -1):
# default host_reserve = MemAvailable + one_minimal_guest(128MB)
# later we add 64MB per additional guest
host_reserve = (
(utils.memtotal() - utils.read_from_meminfo("MemFree")) / 1024 +
128)
# using default reserve
_host_reserve = True
else:
_host_reserve = False
# guest_reserve: mem reserve kept to avoid guest OS to kill processes
guest_reserve = int(params.get("ksm_guest_reserve", -1))
if (guest_reserve == -1):
# default guest_reserve = minimal_system_mem(256MB)
# later we add tmpfs overhead
guest_reserve = 256
# using default reserve
_guest_reserve = True
else:
if os.path.exists(e_rh):
utils.run("echo 'never' > %s" % e_rh)
new_ksm = True
else:
try:
utils.run("modprobe ksm")
utils.run("ksmctl start 5000 100")
except error.CmdError, e:
raise error.TestFail("Failed to load KSM: %s" % e)
# host_reserve: mem reserve kept for the host system to run
host_reserve = int(params.get("ksm_host_reserve", -1))
if (host_reserve == -1):
# default host_reserve = MemAvailable + one_minimal_guest(128MB)
# later we add 64MB per additional guest
host_reserve = ((utils.memtotal() - utils.read_from_meminfo("MemFree"))
/ 1024 + 128)
# using default reserve
_host_reserve = True
else:
_host_reserve = False
# guest_reserve: mem reserve kept to avoid guest OS to kill processes
guest_reserve = int(params.get("ksm_guest_reserve", -1))
if (guest_reserve == -1):
# default guest_reserve = minimal_system_mem(256MB)
# later we add tmpfs overhead
guest_reserve = 256
# using default reserve
_guest_reserve = True
else:
def run_stress_kernel_compile(tests, params, env):
"""
Boot VMs and run kernel compile inside VM parallel.
1) Boot up VMs:
Every VM has 4G vmem, the total vmem of VMs' are
$overcommit times as host's mem.
2) Launch kernel compile inside every guest.
@param test: QEMU test object.
@param params: Dictionary with the test parameters.
@param env: Dictionary with test environment.
"""
def kernelcompile(session, vm_name):
vm = env.get_vm(vm_name)
ip = vm.get_address()
path = params.get("download_url")
logging.info("kernel path = %s" % path)
get_kernel_cmd = "wget %s" % path
try:
status, output = session.cmd_status_output(get_kernel_cmd,
timeout=240)
if status != 0:
logging.error(output)
raise error.TestFail("Fail to download the kernel"
" in %s" % vm_name)
else:
logging.info("Completed download the kernel src"
" in %s" %vm_name)
test_cmd = params.get("test_cmd")
status, output = session.cmd_status_output(test_cmd, timeout=1200)
if status != 0:
logging.error(output)
finally:
status, _ = utils_test.ping(ip, count=10, timeout=30)
if status != 0:
raise error.TestFail("vm no response, pls check serial log")
over_c = float(params.get("overcommit", 1.5))
guest_number = int(params.get("guest_number", "1"))
if guest_number < 1:
logging.warn("At least boot up one guest for this test,"
" set up guest number to 1")
guest_number = 1
for tag in range(1, guest_number):
params["vms"] += " stress_guest_%s" % tag
mem_host = utils.memtotal() / 1024
vmem = int(mem_host * over_c / guest_number)
if vmem < 256:
raise error.TestNAError("The memory size set for guest is too small."
" Please try less than %s guests"
" in this host." % guest_number)
params["mem"] = vmem
params["start_vm"] = "yes"
login_timeout = int(params.get("login_timeout", 360))
env_process.preprocess(tests, params, env)
sessions_info = []
for vm_name in params["vms"].split():
vm = env.get_vm(vm_name)
vm.verify_alive()
session = vm.wait_for_login(timeout=login_timeout)
if not session:
raise error.TestFail("Could not log into guest %s" % vm_name)
sessions_info.append([session, vm_name])
# run kernel compile in vms
try:
logging.info("run kernel compile in vms")
bg_threads = []
for session_info in sessions_info:
session = session_info[0]
vm_name = session_info[1]
bg_thread = utils_test.BackgroundTest(kernelcompile,
(session, vm_name))
bg_thread.start()
bg_threads.append(bg_thread)
completed = False
while not completed:
completed = True
for bg_thread in bg_threads:
if bg_thread.is_alive():
completed = False
finally:
try:
for bg_thread in bg_threads:
if bg_thread:
bg_thread.join()
finally:
for session_info in sessions_info:
session_info[0].close()
def run_virsh_nodememstats(test, params, env):
"""
Test the command virsh nodememstats
(1) Call the virsh nodememstats command
(2) Get the output
(3) Check the against /proc/meminfo output
(4) Call the virsh nodememstats command with an unexpected option
(5) Call the virsh nodememstats command with libvirtd service stop
"""
# Initialize the variables
expected = {}
actual = {}
deltas = []
name_stats = ['total', 'free', 'buffers', 'cached']
itr = int(params.get("itr"))
def virsh_check_nodememtats(actual_stats, expected_stats, delta):
"""
Check the nodememstats output value with /proc/meminfo value
"""
delta_stats = {}
for name in name_stats:
delta_stats[name] = abs(actual_stats[name] - expected_stats[name])
if 'total' in name:
if not delta_stats[name] == 0:
raise error.TestFail("Command 'virsh nodememstats' not"
" succeeded as the value for %s is "
"deviated by %d\nThe total memory "
"value is deviating-check"
% (name, delta_stats[name]))
else:
if delta_stats[name] > delta:
raise error.TestFail("Command 'virsh nodememstats' not "
"succeeded as the value for %s"
" is deviated by %d"
% (name, delta_stats[name]))
return delta_stats
# Prepare libvirtd service
check_libvirtd = params.has_key("libvirtd")
if check_libvirtd:
libvirtd = params.get("libvirtd")
if libvirtd == "off":
libvirt_vm.service_libvirtd_control("stop")
# Get the option for the test case
option = params.get("virsh_nodememstats_options")
# Run test case for 10 iterations
# (default can be changed in subtests.cfg file)
# and print the final statistics
for i in range(itr):
output = virsh.nodememstats(option)
# Get the status of the virsh command executed
status = output.exit_status
# Get status_error option for the test case
status_error = params.get("status_error")
if status_error == "yes":
if status == 0:
if libvirtd == "off":
libvirt_vm.service_libvirtd_control("start")
raise error.TestFail("Command 'virsh nodememstats' "
"succeeded with libvirtd service"
" stopped, incorrect")
else:
raise error.TestFail("Command 'virsh nodememstats %s' "
"succeeded (incorrect command)"
% option)
elif status_error == "no":
if status == 0:
# From the beginning of a line, group 1 is one or
# more word-characters, followed by zero or more
# whitespace characters and a ':', then one or
# more whitespace characters, followed by group 2,
# which is one or more digit characters,
# then one or more whitespace characters followed by
# a literal 'kB' or 'KiB' sequence, e.g as below
# total : 3809340 kB
# total : 3809340 KiB
# Normalise the value to MBs
regex_obj = re.compile(r"^(\w+)\s*:\s+(\d+)\s\w+")
expected = {}
for line in output.stdout.split('\n'):
match_obj = regex_obj.search(line)
# Due to the extra space in the list
if match_obj is not None:
name = match_obj.group(1)
value = match_obj.group(2)
expected[name] = int(value) / 1024
# Get the actual value from /proc/meminfo and normalise to MBs
actual['total'] = int(utils.memtotal()) / 1024
actual['free'] = int(utils.freememtotal()) / 1024
actual['buffers'] = int(utils.read_from_meminfo('Buffers'))/1024
actual['cached'] = int(utils.read_from_meminfo('Cached')) / 1024
# Currently the delta value is kept at 200 MB this can be
# tuned based on the accuracy
# Check subtests.cfg for more details
delta = int(params.get("delta"))
output = virsh_check_nodememtats(actual, expected, delta)
deltas.append(output)
else:
raise error.TestFail("Command virsh nodememstats %s not "
"succeeded:\n%s" % (option, status))
# Recover libvirtd service start
if libvirtd == "off":
libvirt_vm.service_libvirtd_control("start")
# Print the deviated values for all iterations
if status_error == "no":
logging.debug("The following is the deviations from "
"the actual(/proc/meminfo) and expected"
" value(output of virsh nodememstats)")
for i in range(itr):
logging.debug("iteration %d:", i)
for index, name in enumerate(name_stats):
logging.debug("%19s : %d", name, deltas[i][name])
