def raw_ping(command, timeout, session, output_func):
"""
Low-level ping command execution.
:param command: Ping command.
:param timeout: Timeout of the ping command.
:param session: Local executon hint or session to execute the ping command.
"""
if session is None:
process = aexpect.run_bg(command,
output_func=output_func,
timeout=timeout)
# Send SIGINT signal to notify the timeout of running ping process,
# Because ping have the ability to catch the SIGINT signal so we can
# always get the packet loss ratio even if timeout.
if process.is_alive():
utils_misc.kill_process_tree(process.get_pid(), signal.SIGINT)
status = process.get_status()
output = process.get_output()
process.close()
return status, output
else:
output = ""
try:
output = session.cmd_output(command,
timeout=timeout,
print_func=output_func)
except aexpect.ShellTimeoutError:
# Send ctrl+c (SIGINT) through ssh session
session.send("\003")
try:
output2 = session.read_up_to_prompt(print_func=output_func)
output += output2
except aexpect.ExpectTimeoutError, e:
output += e.output
# We also need to use this session to query the return value
session.send("\003")
session.sendline(session.status_test_command)
try:
o2 = session.read_up_to_prompt()
except aexpect.ExpectError:
status = -1
else:
try:
status = int(re.findall("\d+", o2)[0])
except Exception:
status = -1
return status, output
def raw_ping(command, timeout, session, output_func):
"""
Low-level ping command execution.
:param command: Ping command.
:param timeout: Timeout of the ping command.
:param session: Local executon hint or session to execute the ping command.
"""
if session is None:
process = aexpect.run_bg(command, output_func=output_func,
timeout=timeout)
# Send SIGINT signal to notify the timeout of running ping process,
# Because ping have the ability to catch the SIGINT signal so we can
# always get the packet loss ratio even if timeout.
if process.is_alive():
utils_misc.kill_process_tree(process.get_pid(), signal.SIGINT)
status = process.get_status()
output = process.get_output()
process.close()
return status, output
else:
output = ""
try:
output = session.cmd_output(command, timeout=timeout,
print_func=output_func)
except aexpect.ShellTimeoutError:
# Send ctrl+c (SIGINT) through ssh session
session.send("\003")
try:
output2 = session.read_up_to_prompt(print_func=output_func)
output += output2
except aexpect.ExpectTimeoutError, e:
output += e.output
# We also need to use this session to query the return value
session.send("\003")
session.sendline(session.status_test_command)
try:
o2 = session.read_up_to_prompt()
except aexpect.ExpectError:
status = -1
else:
try:
status = int(re.findall("\d+", o2)[0])
except Exception:
status = -1
return status, output
if copy_only:
return ("%s/autotest-local --args=\"%s\" --verbose %s/control" %
(destination_autotest_path, control_args,
destination_autotest_path))
# Run the test
logging.info("Running autotest control file %s on guest, timeout %ss",
os.path.basename(control_path), timeout)
# Start a background job to run server process if needed.
server_process = None
if server_control_path:
command = ("%s %s --verbose -t %s" % (autotest_local_path,
server_control_path,
os.path.basename(server_control_path)))
server_process = aexpect.run_bg(command)
try:
bg = None
try:
logging.info("---------------- Test output ----------------")
if migrate_background:
mig_timeout = float(params.get("mig_timeout", "3600"))
mig_protocol = params.get("migration_protocol", "tcp")
bg = utils.InterruptedThread(session.cmd_output,
kwargs={
'cmd': "./autotest --args="
"\"%s\" control" %
(control_args),
'timeout': timeout,
def run_timedrift(test, params, env):
"""
Time drift test (mainly for Windows guests):
1) Log into a guest.
2) Take a time reading from the guest and host.
3) Run load on the guest and host.
4) Take a second time reading.
5) Stop the load and rest for a while.
6) Take a third time reading.
7) If the drift immediately after load is higher than a user-
specified value (in %), fail.
If the drift after the rest period is higher than a user-specified value,
fail.
@param test: QEMU test object.
@param params: Dictionary with test parameters.
@param env: Dictionary with the test environment.
"""
# Helper functions
def set_cpu_affinity(pid, mask):
"""
Set the CPU affinity of all threads of the process with PID pid.
Do this recursively for all child processes as well.
@param pid: The process ID.
@param mask: The CPU affinity mask.
@return: A dict containing the previous mask for each thread.
"""
tids = commands.getoutput("ps -L --pid=%s -o lwp=" % pid).split()
prev_masks = {}
for tid in tids:
prev_mask = commands.getoutput("taskset -p %s" % tid).split()[-1]
prev_masks[tid] = prev_mask
commands.getoutput("taskset -p %s %s" % (mask, tid))
children = commands.getoutput("ps --ppid=%s -o pid=" % pid).split()
for child in children:
prev_masks.update(set_cpu_affinity(child, mask))
return prev_masks
def restore_cpu_affinity(prev_masks):
"""
Restore the CPU affinity of several threads.
@param prev_masks: A dict containing TIDs as keys and masks as values.
"""
for tid, mask in prev_masks.items():
commands.getoutput("taskset -p %s %s" % (mask, tid))
vm = env.get_vm(params["main_vm"])
vm.verify_alive()
timeout = int(params.get("login_timeout", 360))
session = vm.wait_for_login(timeout=timeout)
# Collect test parameters:
# Command to run to get the current time
time_command = params.get("time_command")
# Filter which should match a string to be passed to time.strptime()
time_filter_re = params.get("time_filter_re")
# Time format for time.strptime()
time_format = params.get("time_format")
guest_load_command = params.get("guest_load_command")
guest_load_stop_command = params.get("guest_load_stop_command")
host_load_command = params.get("host_load_command")
guest_load_instances = int(params.get("guest_load_instances", "1"))
host_load_instances = int(params.get("host_load_instances", "0"))
# CPU affinity mask for taskset
cpu_mask = params.get("cpu_mask", "0xFF")
load_duration = float(params.get("load_duration", "30"))
rest_duration = float(params.get("rest_duration", "10"))
drift_threshold = float(params.get("drift_threshold", "200"))
drift_threshold_after_rest = float(params.get("drift_threshold_after_rest",
"200"))
guest_load_sessions = []
host_load_sessions = []
try:
# Set the VM's CPU affinity
prev_affinity = set_cpu_affinity(vm.get_shell_pid(), cpu_mask)
try:
# Open shell sessions with the guest
logging.info("Starting load on guest...")
for i in range(guest_load_instances):
load_session = vm.login()
# Set output func to None to stop it from being called so we
# can change the callback function and the parameters it takes
# with no problems
load_session.set_output_func(None)
load_session.set_output_params(())
load_session.set_output_prefix("(guest load %d) " % i)
load_session.set_output_func(logging.debug)
guest_load_sessions.append(load_session)
# Get time before load
# (ht stands for host time, gt stands for guest time)
(ht0, gt0) = utils_test.get_time(session,
time_command,
time_filter_re,
time_format)
# Run some load on the guest
for load_session in guest_load_sessions:
load_session.sendline(guest_load_command)
# Run some load on the host
logging.info("Starting load on host...")
for i in range(host_load_instances):
host_load_sessions.append(
aexpect.run_bg(host_load_command,
output_func=logging.debug,
output_prefix="(host load %d) " % i,
timeout=0.5))
# Set the CPU affinity of the load process
pid = host_load_sessions[-1].get_pid()
set_cpu_affinity(pid, cpu_mask)
# Sleep for a while (during load)
logging.info("Sleeping for %s seconds...", load_duration)
time.sleep(load_duration)
# Get time delta after load
(ht1, gt1) = utils_test.get_time(session,
time_command,
time_filter_re,
time_format)
# Report results
host_delta = ht1 - ht0
guest_delta = gt1 - gt0
drift = 100.0 * (host_delta - guest_delta) / host_delta
logging.info("Host duration: %.2f", host_delta)
logging.info("Guest duration: %.2f", guest_delta)
logging.info("Drift: %.2f%%", drift)
finally:
logging.info("Cleaning up...")
# Restore the VM's CPU affinity
restore_cpu_affinity(prev_affinity)
# Stop the guest load
if guest_load_stop_command:
session.cmd_output(guest_load_stop_command)
# Close all load shell sessions
for load_session in guest_load_sessions:
load_session.close()
for load_session in host_load_sessions:
load_session.close()
# Sleep again (rest)
logging.info("Sleeping for %s seconds...", rest_duration)
time.sleep(rest_duration)
# Get time after rest
(ht2, gt2) = utils_test.get_time(session,
time_command,
time_filter_re,
time_format)
finally:
session.close()
# Report results
host_delta_total = ht2 - ht0
guest_delta_total = gt2 - gt0
drift_total = 100.0 * (host_delta_total - guest_delta_total) / host_delta
logging.info("Total host duration including rest: %.2f", host_delta_total)
logging.info("Total guest duration including rest: %.2f", guest_delta_total)
logging.info("Total drift after rest: %.2f%%", drift_total)
# Fail the test if necessary
if abs(drift) > drift_threshold:
raise error.TestFail("Time drift too large: %.2f%%" % drift)
if abs(drift_total) > drift_threshold_after_rest:
raise error.TestFail("Time drift too large after rest period: %.2f%%"
% drift_total)
def run_time_manage(test, params, env):
"""
Time manage test:
1) Generate stress in host.
2) Run atleast 15 vms with "driftfix=slew" option
3) Reboot the guest.
4) Repeat the step 3 for all guests and check whether the guest
responds properly(not any watchdog reported).
5) TODO: Improve the way of checking the response and
run some stress inside guest too.
6) Continue the step 4 for 10 iterations and
record the guest/host realtime, calculate drift in time for
each iterations.
7) Print the drift values for all sessions
8) TODO: Validate if the drift value has to be within defined value
:param test: QEMU test object.
:param params: Dictionary with test parameters.
:param env: Dictionary with the test environment.
"""
# Checking the main vm is alive
vm = env.get_vm(params["main_vm"])
vm.verify_alive()
timeout = int(params.get("login_timeout", 360))
session = vm.wait_for_login(timeout=timeout)
# Collect test parameters
login_timeout = float(params.get("login_timeout", 240))
host_load_command = params["host_load_command"]
host_load_kill_command = params["host_load_kill_command"]
time_command = params["time_command"]
time_filter_re = params["time_filter_re"]
time_format = params["time_format"]
# Intialize the variables
itr = 0
num = 2
host_load_sessions = []
sessions = [session]
prev_time = []
curr_time = []
timedrift = []
totaldrift = []
vmnames = ["virt-tests-vm1"]
# Run some load on the host
logging.info("Starting load on host.")
host_load_sessions.append(
aexpect.run_bg(host_load_command,
output_func=logging.debug,
output_prefix="host load ",
timeout=0.5))
# Boot the VMs
try:
while num <= int(params["max_vms"]):
# Clone vm according to the first one
vm_name = "virt-tests-vm%d" % num
vmnames.append(vm_name)
vm_params = vm.params.copy()
curr_vm = vm.clone(vm_name, vm_params)
env.register_vm(vm_name, curr_vm)
env_process.preprocess_vm(test, vm_params, env, vm_name)
params["vms"] += " " + vm_name
sessions.append(curr_vm.wait_for_login(timeout=login_timeout))
logging.info("Guest #%d booted up successfully", num)
# Check whether all previous shell sessions are responsive
error.context("checking responsiveness of the booted guest")
for se in sessions:
se.cmd(params["alive_test_cmd"])
num += 1
while itr <= int(params["max_itrs"]):
for vmid, se in enumerate(sessions):
# Get the respective vm object
vmname = "virt-tests-vm%d" % (vmid + 1)
vm = env.get_vm(vmname)
# Run current iteration
logging.info("Rebooting:vm%d iteration %d " %
((vmid + 1), itr))
se = vm.reboot(se, timeout=timeout)
# Remember the current changed session
sessions[vmid] = se
error.context("checking responsiveness of guest")
se.cmd(params["alive_test_cmd"])
if itr == 0:
(ht0, gt0) = utils_test.get_time(se, time_command,
time_filter_re,
time_format)
prev_time.append((ht0, gt0))
else:
(ht1, gt1) = utils_test.get_time(se, time_command,
time_filter_re,
time_format)
curr_time.append((ht1, gt1))
if itr != 0:
for i in range(int(params["max_vms"])):
hdelta = curr_time[i][0] - prev_time[i][0]
gdelta = curr_time[i][1] - prev_time[i][1]
drift = "%.2f" % (100.0 * (hdelta - gdelta) / hdelta)
timedrift.append(drift)
totaldrift.append(timedrift)
prev_time = curr_time
timedrift = []
curr_time = []
# Wait for some time before next iteration
time.sleep(30)
itr += 1
logging.info("The time drift values for all VM sessions/iterations")
logging.info("VM-Name:%s" % vmnames)
for idx, value in enumerate(totaldrift):
logging.info("itr-%2d:%s" % (idx + 1, value))
finally:
for se in sessions:
# Closing all the sessions.
se.close()
logging.info("killing load on host.")
host_load_sessions.append(
aexpect.run_bg(host_load_kill_command,
output_func=logging.debug,
output_prefix="host load kill",
timeout=0.5))
def run_timedrift(test, params, env):
"""
Time drift test (mainly for Windows guests):
1) Log into a guest.
2) Take a time reading from the guest and host.
3) Run load on the guest and host.
4) Take a second time reading.
5) Stop the load and rest for a while.
6) Take a third time reading.
7) If the drift immediately after load is higher than a user-
specified value (in %), fail.
If the drift after the rest period is higher than a user-specified value,
fail.
@param test: QEMU test object.
@param params: Dictionary with test parameters.
@param env: Dictionary with the test environment.
"""
# Helper functions
def set_cpu_affinity(pid, mask):
"""
Set the CPU affinity of all threads of the process with PID pid.
Do this recursively for all child processes as well.
@param pid: The process ID.
@param mask: The CPU affinity mask.
@return: A dict containing the previous mask for each thread.
"""
tids = commands.getoutput("ps -L --pid=%s -o lwp=" % pid).split()
prev_masks = {}
for tid in tids:
prev_mask = commands.getoutput("taskset -p %s" % tid).split()[-1]
prev_masks[tid] = prev_mask
commands.getoutput("taskset -p %s %s" % (mask, tid))
children = commands.getoutput("ps --ppid=%s -o pid=" % pid).split()
for child in children:
prev_masks.update(set_cpu_affinity(child, mask))
return prev_masks
def restore_cpu_affinity(prev_masks):
"""
Restore the CPU affinity of several threads.
@param prev_masks: A dict containing TIDs as keys and masks as values.
"""
for tid, mask in prev_masks.items():
commands.getoutput("taskset -p %s %s" % (mask, tid))
vm = env.get_vm(params["main_vm"])
vm.verify_alive()
timeout = int(params.get("login_timeout", 360))
session = vm.wait_for_login(timeout=timeout)
# Collect test parameters:
# Command to run to get the current time
time_command = params.get("time_command")
# Filter which should match a string to be passed to time.strptime()
time_filter_re = params.get("time_filter_re")
# Time format for time.strptime()
time_format = params.get("time_format")
guest_load_command = params.get("guest_load_command")
guest_load_stop_command = params.get("guest_load_stop_command")
host_load_command = params.get("host_load_command")
guest_load_instances = int(params.get("guest_load_instances", "1"))
host_load_instances = int(params.get("host_load_instances", "0"))
# CPU affinity mask for taskset
cpu_mask = params.get("cpu_mask", "0xFF")
load_duration = float(params.get("load_duration", "30"))
rest_duration = float(params.get("rest_duration", "10"))
drift_threshold = float(params.get("drift_threshold", "200"))
drift_threshold_after_rest = float(
params.get("drift_threshold_after_rest", "200"))
guest_load_sessions = []
host_load_sessions = []
try:
# Set the VM's CPU affinity
prev_affinity = set_cpu_affinity(vm.get_shell_pid(), cpu_mask)
try:
# Open shell sessions with the guest
logging.info("Starting load on guest...")
for i in range(guest_load_instances):
load_session = vm.login()
# Set output func to None to stop it from being called so we
# can change the callback function and the parameters it takes
# with no problems
load_session.set_output_func(None)
load_session.set_output_params(())
load_session.set_output_prefix("(guest load %d) " % i)
load_session.set_output_func(logging.debug)
guest_load_sessions.append(load_session)
# Get time before load
# (ht stands for host time, gt stands for guest time)
(ht0, gt0) = utils_test.get_time(session, time_command,
time_filter_re, time_format)
# Run some load on the guest
for load_session in guest_load_sessions:
load_session.sendline(guest_load_command)
# Run some load on the host
logging.info("Starting load on host...")
for i in range(host_load_instances):
host_load_sessions.append(
aexpect.run_bg(host_load_command,
output_func=logging.debug,
output_prefix="(host load %d) " % i,
timeout=0.5))
# Set the CPU affinity of the load process
pid = host_load_sessions[-1].get_pid()
set_cpu_affinity(pid, cpu_mask)
# Sleep for a while (during load)
logging.info("Sleeping for %s seconds...", load_duration)
time.sleep(load_duration)
# Get time delta after load
(ht1, gt1) = utils_test.get_time(session, time_command,
time_filter_re, time_format)
# Report results
host_delta = ht1 - ht0
guest_delta = gt1 - gt0
drift = 100.0 * (host_delta - guest_delta) / host_delta
logging.info("Host duration: %.2f", host_delta)
logging.info("Guest duration: %.2f", guest_delta)
logging.info("Drift: %.2f%%", drift)
finally:
logging.info("Cleaning up...")
# Restore the VM's CPU affinity
restore_cpu_affinity(prev_affinity)
# Stop the guest load
if guest_load_stop_command:
session.cmd_output(guest_load_stop_command)
# Close all load shell sessions
for load_session in guest_load_sessions:
load_session.close()
for load_session in host_load_sessions:
load_session.close()
# Sleep again (rest)
logging.info("Sleeping for %s seconds...", rest_duration)
time.sleep(rest_duration)
# Get time after rest
(ht2, gt2) = utils_test.get_time(session, time_command, time_filter_re,
time_format)
finally:
session.close()
# Report results
host_delta_total = ht2 - ht0
guest_delta_total = gt2 - gt0
drift_total = 100.0 * (host_delta_total - guest_delta_total) / host_delta
logging.info("Total host duration including rest: %.2f", host_delta_total)
logging.info("Total guest duration including rest: %.2f",
guest_delta_total)
logging.info("Total drift after rest: %.2f%%", drift_total)
# Fail the test if necessary
if abs(drift) > drift_threshold:
raise error.TestFail("Time drift too large: %.2f%%" % drift)
if abs(drift_total) > drift_threshold_after_rest:
raise error.TestFail("Time drift too large after rest period: %.2f%%" %
drift_total)
def run(test, params, env):
"""
Time manage test:
1) Generate stress in host.
2) Run atleast 15 vms with "driftfix=slew" option
3) Reboot the guest.
4) Repeat the step 3 for all guests and check whether the guest
responds properly(not any watchdog reported).
5) TODO: Improve the way of checking the response and
run some stress inside guest too.
6) Continue the step 4 for 10 iterations and
record the guest/host realtime, calculate drift in time for
each iterations.
7) Print the drift values for all sessions
8) TODO: Validate if the drift value has to be within defined value
:param test: QEMU test object.
:param params: Dictionary with test parameters.
:param env: Dictionary with the test environment.
"""
# Checking the main vm is alive
vm = env.get_vm(params["main_vm"])
vm.verify_alive()
timeout = int(params.get("login_timeout", 360))
session = vm.wait_for_login(timeout=timeout)
# Collect test parameters
login_timeout = float(params.get("login_timeout", 240))
host_load_command = params["host_load_command"]
host_load_kill_command = params["host_load_kill_command"]
time_command = params["time_command"]
time_filter_re = params["time_filter_re"]
time_format = params["time_format"]
# Intialize the variables
itr = 0
num = 2
host_load_sessions = []
sessions = [session]
prev_time = []
curr_time = []
timedrift = []
totaldrift = []
vmnames = ["virt-tests-vm1"]
# Run some load on the host
logging.info("Starting load on host.")
host_load_sessions.append(aexpect.run_bg(host_load_command,
output_func=logging.debug,
output_prefix="host load ",
timeout=0.5))
# Boot the VMs
try:
while num <= int(params["max_vms"]):
# Clone vm according to the first one
vm_name = "virt-tests-vm%d" % num
vmnames.append(vm_name)
vm_params = vm.params.copy()
curr_vm = vm.clone(vm_name, vm_params)
env.register_vm(vm_name, curr_vm)
env_process.preprocess_vm(test, vm_params, env, vm_name)
params["vms"] += " " + vm_name
sessions.append(curr_vm.wait_for_login(timeout=login_timeout))
logging.info("Guest #%d booted up successfully", num)
# Check whether all previous shell sessions are responsive
error.context("checking responsiveness of the booted guest")
for se in sessions:
se.cmd(params["alive_test_cmd"])
num += 1
while itr <= int(params["max_itrs"]):
for vmid, se in enumerate(sessions):
# Get the respective vm object
vmname = "virt-tests-vm%d" % (vmid + 1)
vm = env.get_vm(vmname)
# Run current iteration
logging.info(
"Rebooting:vm%d iteration %d " % ((vmid + 1), itr))
se = vm.reboot(se, timeout=timeout)
# Remember the current changed session
sessions[vmid] = se
error.context("checking responsiveness of guest")
se.cmd(params["alive_test_cmd"])
if itr == 0:
(ht0, gt0) = utils_test.get_time(se, time_command,
time_filter_re, time_format)
prev_time.append((ht0, gt0))
else:
(ht1, gt1) = utils_test.get_time(se, time_command,
time_filter_re, time_format)
curr_time.append((ht1, gt1))
if itr != 0:
for i in range(int(params["max_vms"])):
hdelta = curr_time[i][0] - prev_time[i][0]
gdelta = curr_time[i][1] - prev_time[i][1]
drift = "%.2f" % (100.0 * (hdelta - gdelta) / hdelta)
timedrift.append(drift)
totaldrift.append(timedrift)
prev_time = curr_time
timedrift = []
curr_time = []
# Wait for some time before next iteration
time.sleep(30)
itr += 1
logging.info("The time drift values for all VM sessions/iterations")
logging.info("VM-Name:%s" % vmnames)
for idx, value in enumerate(totaldrift):
logging.info("itr-%2d:%s" % (idx + 1, value))
finally:
for se in sessions:
# Closing all the sessions.
se.close()
logging.info("killing load on host.")
host_load_sessions.append(aexpect.run_bg(host_load_kill_command,
output_func=logging.debug,
output_prefix="host load kill",
timeout=0.5))
