def raw_ping(command, timeout, session, output_func): """ Low-level ping command execution. :param command: Ping command. :param timeout: Timeout of the ping command. :param session: Local executon hint or session to execute the ping command. """ if session is None: process = aexpect.run_bg(command, output_func=output_func, timeout=timeout) # Send SIGINT signal to notify the timeout of running ping process, # Because ping have the ability to catch the SIGINT signal so we can # always get the packet loss ratio even if timeout. if process.is_alive(): utils_misc.kill_process_tree(process.get_pid(), signal.SIGINT) status = process.get_status() output = process.get_output() process.close() return status, output else: output = "" try: output = session.cmd_output(command, timeout=timeout, print_func=output_func) except aexpect.ShellTimeoutError: # Send ctrl+c (SIGINT) through ssh session session.send("\003") try: output2 = session.read_up_to_prompt(print_func=output_func) output += output2 except aexpect.ExpectTimeoutError, e: output += e.output # We also need to use this session to query the return value session.send("\003") session.sendline(session.status_test_command) try: o2 = session.read_up_to_prompt() except aexpect.ExpectError: status = -1 else: try: status = int(re.findall("\d+", o2)[0]) except Exception: status = -1 return status, output
if copy_only: return ("%s/autotest-local --args=\"%s\" --verbose %s/control" % (destination_autotest_path, control_args, destination_autotest_path)) # Run the test logging.info("Running autotest control file %s on guest, timeout %ss", os.path.basename(control_path), timeout) # Start a background job to run server process if needed. server_process = None if server_control_path: command = ("%s %s --verbose -t %s" % (autotest_local_path, server_control_path, os.path.basename(server_control_path))) server_process = aexpect.run_bg(command) try: bg = None try: logging.info("---------------- Test output ----------------") if migrate_background: mig_timeout = float(params.get("mig_timeout", "3600")) mig_protocol = params.get("migration_protocol", "tcp") bg = utils.InterruptedThread(session.cmd_output, kwargs={ 'cmd': "./autotest --args=" "\"%s\" control" % (control_args), 'timeout': timeout,
def run_timedrift(test, params, env): """ Time drift test (mainly for Windows guests): 1) Log into a guest. 2) Take a time reading from the guest and host. 3) Run load on the guest and host. 4) Take a second time reading. 5) Stop the load and rest for a while. 6) Take a third time reading. 7) If the drift immediately after load is higher than a user- specified value (in %), fail. If the drift after the rest period is higher than a user-specified value, fail. @param test: QEMU test object. @param params: Dictionary with test parameters. @param env: Dictionary with the test environment. """ # Helper functions def set_cpu_affinity(pid, mask): """ Set the CPU affinity of all threads of the process with PID pid. Do this recursively for all child processes as well. @param pid: The process ID. @param mask: The CPU affinity mask. @return: A dict containing the previous mask for each thread. """ tids = commands.getoutput("ps -L --pid=%s -o lwp=" % pid).split() prev_masks = {} for tid in tids: prev_mask = commands.getoutput("taskset -p %s" % tid).split()[-1] prev_masks[tid] = prev_mask commands.getoutput("taskset -p %s %s" % (mask, tid)) children = commands.getoutput("ps --ppid=%s -o pid=" % pid).split() for child in children: prev_masks.update(set_cpu_affinity(child, mask)) return prev_masks def restore_cpu_affinity(prev_masks): """ Restore the CPU affinity of several threads. @param prev_masks: A dict containing TIDs as keys and masks as values. """ for tid, mask in prev_masks.items(): commands.getoutput("taskset -p %s %s" % (mask, tid)) vm = env.get_vm(params["main_vm"]) vm.verify_alive() timeout = int(params.get("login_timeout", 360)) session = vm.wait_for_login(timeout=timeout) # Collect test parameters: # Command to run to get the current time time_command = params.get("time_command") # Filter which should match a string to be passed to time.strptime() time_filter_re = params.get("time_filter_re") # Time format for time.strptime() time_format = params.get("time_format") guest_load_command = params.get("guest_load_command") guest_load_stop_command = params.get("guest_load_stop_command") host_load_command = params.get("host_load_command") guest_load_instances = int(params.get("guest_load_instances", "1")) host_load_instances = int(params.get("host_load_instances", "0")) # CPU affinity mask for taskset cpu_mask = params.get("cpu_mask", "0xFF") load_duration = float(params.get("load_duration", "30")) rest_duration = float(params.get("rest_duration", "10")) drift_threshold = float(params.get("drift_threshold", "200")) drift_threshold_after_rest = float(params.get("drift_threshold_after_rest", "200")) guest_load_sessions = [] host_load_sessions = [] try: # Set the VM's CPU affinity prev_affinity = set_cpu_affinity(vm.get_shell_pid(), cpu_mask) try: # Open shell sessions with the guest logging.info("Starting load on guest...") for i in range(guest_load_instances): load_session = vm.login() # Set output func to None to stop it from being called so we # can change the callback function and the parameters it takes # with no problems load_session.set_output_func(None) load_session.set_output_params(()) load_session.set_output_prefix("(guest load %d) " % i) load_session.set_output_func(logging.debug) guest_load_sessions.append(load_session) # Get time before load # (ht stands for host time, gt stands for guest time) (ht0, gt0) = utils_test.get_time(session, time_command, time_filter_re, time_format) # Run some load on the guest for load_session in guest_load_sessions: load_session.sendline(guest_load_command) # Run some load on the host logging.info("Starting load on host...") for i in range(host_load_instances): host_load_sessions.append( aexpect.run_bg(host_load_command, output_func=logging.debug, output_prefix="(host load %d) " % i, timeout=0.5)) # Set the CPU affinity of the load process pid = host_load_sessions[-1].get_pid() set_cpu_affinity(pid, cpu_mask) # Sleep for a while (during load) logging.info("Sleeping for %s seconds...", load_duration) time.sleep(load_duration) # Get time delta after load (ht1, gt1) = utils_test.get_time(session, time_command, time_filter_re, time_format) # Report results host_delta = ht1 - ht0 guest_delta = gt1 - gt0 drift = 100.0 * (host_delta - guest_delta) / host_delta logging.info("Host duration: %.2f", host_delta) logging.info("Guest duration: %.2f", guest_delta) logging.info("Drift: %.2f%%", drift) finally: logging.info("Cleaning up...") # Restore the VM's CPU affinity restore_cpu_affinity(prev_affinity) # Stop the guest load if guest_load_stop_command: session.cmd_output(guest_load_stop_command) # Close all load shell sessions for load_session in guest_load_sessions: load_session.close() for load_session in host_load_sessions: load_session.close() # Sleep again (rest) logging.info("Sleeping for %s seconds...", rest_duration) time.sleep(rest_duration) # Get time after rest (ht2, gt2) = utils_test.get_time(session, time_command, time_filter_re, time_format) finally: session.close() # Report results host_delta_total = ht2 - ht0 guest_delta_total = gt2 - gt0 drift_total = 100.0 * (host_delta_total - guest_delta_total) / host_delta logging.info("Total host duration including rest: %.2f", host_delta_total) logging.info("Total guest duration including rest: %.2f", guest_delta_total) logging.info("Total drift after rest: %.2f%%", drift_total) # Fail the test if necessary if abs(drift) > drift_threshold: raise error.TestFail("Time drift too large: %.2f%%" % drift) if abs(drift_total) > drift_threshold_after_rest: raise error.TestFail("Time drift too large after rest period: %.2f%%" % drift_total)
def run_time_manage(test, params, env): """ Time manage test: 1) Generate stress in host. 2) Run atleast 15 vms with "driftfix=slew" option 3) Reboot the guest. 4) Repeat the step 3 for all guests and check whether the guest responds properly(not any watchdog reported). 5) TODO: Improve the way of checking the response and run some stress inside guest too. 6) Continue the step 4 for 10 iterations and record the guest/host realtime, calculate drift in time for each iterations. 7) Print the drift values for all sessions 8) TODO: Validate if the drift value has to be within defined value :param test: QEMU test object. :param params: Dictionary with test parameters. :param env: Dictionary with the test environment. """ # Checking the main vm is alive vm = env.get_vm(params["main_vm"]) vm.verify_alive() timeout = int(params.get("login_timeout", 360)) session = vm.wait_for_login(timeout=timeout) # Collect test parameters login_timeout = float(params.get("login_timeout", 240)) host_load_command = params["host_load_command"] host_load_kill_command = params["host_load_kill_command"] time_command = params["time_command"] time_filter_re = params["time_filter_re"] time_format = params["time_format"] # Intialize the variables itr = 0 num = 2 host_load_sessions = [] sessions = [session] prev_time = [] curr_time = [] timedrift = [] totaldrift = [] vmnames = ["virt-tests-vm1"] # Run some load on the host logging.info("Starting load on host.") host_load_sessions.append( aexpect.run_bg(host_load_command, output_func=logging.debug, output_prefix="host load ", timeout=0.5)) # Boot the VMs try: while num <= int(params["max_vms"]): # Clone vm according to the first one vm_name = "virt-tests-vm%d" % num vmnames.append(vm_name) vm_params = vm.params.copy() curr_vm = vm.clone(vm_name, vm_params) env.register_vm(vm_name, curr_vm) env_process.preprocess_vm(test, vm_params, env, vm_name) params["vms"] += " " + vm_name sessions.append(curr_vm.wait_for_login(timeout=login_timeout)) logging.info("Guest #%d booted up successfully", num) # Check whether all previous shell sessions are responsive error.context("checking responsiveness of the booted guest") for se in sessions: se.cmd(params["alive_test_cmd"]) num += 1 while itr <= int(params["max_itrs"]): for vmid, se in enumerate(sessions): # Get the respective vm object vmname = "virt-tests-vm%d" % (vmid + 1) vm = env.get_vm(vmname) # Run current iteration logging.info("Rebooting:vm%d iteration %d " % ((vmid + 1), itr)) se = vm.reboot(se, timeout=timeout) # Remember the current changed session sessions[vmid] = se error.context("checking responsiveness of guest") se.cmd(params["alive_test_cmd"]) if itr == 0: (ht0, gt0) = utils_test.get_time(se, time_command, time_filter_re, time_format) prev_time.append((ht0, gt0)) else: (ht1, gt1) = utils_test.get_time(se, time_command, time_filter_re, time_format) curr_time.append((ht1, gt1)) if itr != 0: for i in range(int(params["max_vms"])): hdelta = curr_time[i][0] - prev_time[i][0] gdelta = curr_time[i][1] - prev_time[i][1] drift = "%.2f" % (100.0 * (hdelta - gdelta) / hdelta) timedrift.append(drift) totaldrift.append(timedrift) prev_time = curr_time timedrift = [] curr_time = [] # Wait for some time before next iteration time.sleep(30) itr += 1 logging.info("The time drift values for all VM sessions/iterations") logging.info("VM-Name:%s" % vmnames) for idx, value in enumerate(totaldrift): logging.info("itr-%2d:%s" % (idx + 1, value)) finally: for se in sessions: # Closing all the sessions. se.close() logging.info("killing load on host.") host_load_sessions.append( aexpect.run_bg(host_load_kill_command, output_func=logging.debug, output_prefix="host load kill", timeout=0.5))
def run_timedrift(test, params, env): """ Time drift test (mainly for Windows guests): 1) Log into a guest. 2) Take a time reading from the guest and host. 3) Run load on the guest and host. 4) Take a second time reading. 5) Stop the load and rest for a while. 6) Take a third time reading. 7) If the drift immediately after load is higher than a user- specified value (in %), fail. If the drift after the rest period is higher than a user-specified value, fail. @param test: QEMU test object. @param params: Dictionary with test parameters. @param env: Dictionary with the test environment. """ # Helper functions def set_cpu_affinity(pid, mask): """ Set the CPU affinity of all threads of the process with PID pid. Do this recursively for all child processes as well. @param pid: The process ID. @param mask: The CPU affinity mask. @return: A dict containing the previous mask for each thread. """ tids = commands.getoutput("ps -L --pid=%s -o lwp=" % pid).split() prev_masks = {} for tid in tids: prev_mask = commands.getoutput("taskset -p %s" % tid).split()[-1] prev_masks[tid] = prev_mask commands.getoutput("taskset -p %s %s" % (mask, tid)) children = commands.getoutput("ps --ppid=%s -o pid=" % pid).split() for child in children: prev_masks.update(set_cpu_affinity(child, mask)) return prev_masks def restore_cpu_affinity(prev_masks): """ Restore the CPU affinity of several threads. @param prev_masks: A dict containing TIDs as keys and masks as values. """ for tid, mask in prev_masks.items(): commands.getoutput("taskset -p %s %s" % (mask, tid)) vm = env.get_vm(params["main_vm"]) vm.verify_alive() timeout = int(params.get("login_timeout", 360)) session = vm.wait_for_login(timeout=timeout) # Collect test parameters: # Command to run to get the current time time_command = params.get("time_command") # Filter which should match a string to be passed to time.strptime() time_filter_re = params.get("time_filter_re") # Time format for time.strptime() time_format = params.get("time_format") guest_load_command = params.get("guest_load_command") guest_load_stop_command = params.get("guest_load_stop_command") host_load_command = params.get("host_load_command") guest_load_instances = int(params.get("guest_load_instances", "1")) host_load_instances = int(params.get("host_load_instances", "0")) # CPU affinity mask for taskset cpu_mask = params.get("cpu_mask", "0xFF") load_duration = float(params.get("load_duration", "30")) rest_duration = float(params.get("rest_duration", "10")) drift_threshold = float(params.get("drift_threshold", "200")) drift_threshold_after_rest = float( params.get("drift_threshold_after_rest", "200")) guest_load_sessions = [] host_load_sessions = [] try: # Set the VM's CPU affinity prev_affinity = set_cpu_affinity(vm.get_shell_pid(), cpu_mask) try: # Open shell sessions with the guest logging.info("Starting load on guest...") for i in range(guest_load_instances): load_session = vm.login() # Set output func to None to stop it from being called so we # can change the callback function and the parameters it takes # with no problems load_session.set_output_func(None) load_session.set_output_params(()) load_session.set_output_prefix("(guest load %d) " % i) load_session.set_output_func(logging.debug) guest_load_sessions.append(load_session) # Get time before load # (ht stands for host time, gt stands for guest time) (ht0, gt0) = utils_test.get_time(session, time_command, time_filter_re, time_format) # Run some load on the guest for load_session in guest_load_sessions: load_session.sendline(guest_load_command) # Run some load on the host logging.info("Starting load on host...") for i in range(host_load_instances): host_load_sessions.append( aexpect.run_bg(host_load_command, output_func=logging.debug, output_prefix="(host load %d) " % i, timeout=0.5)) # Set the CPU affinity of the load process pid = host_load_sessions[-1].get_pid() set_cpu_affinity(pid, cpu_mask) # Sleep for a while (during load) logging.info("Sleeping for %s seconds...", load_duration) time.sleep(load_duration) # Get time delta after load (ht1, gt1) = utils_test.get_time(session, time_command, time_filter_re, time_format) # Report results host_delta = ht1 - ht0 guest_delta = gt1 - gt0 drift = 100.0 * (host_delta - guest_delta) / host_delta logging.info("Host duration: %.2f", host_delta) logging.info("Guest duration: %.2f", guest_delta) logging.info("Drift: %.2f%%", drift) finally: logging.info("Cleaning up...") # Restore the VM's CPU affinity restore_cpu_affinity(prev_affinity) # Stop the guest load if guest_load_stop_command: session.cmd_output(guest_load_stop_command) # Close all load shell sessions for load_session in guest_load_sessions: load_session.close() for load_session in host_load_sessions: load_session.close() # Sleep again (rest) logging.info("Sleeping for %s seconds...", rest_duration) time.sleep(rest_duration) # Get time after rest (ht2, gt2) = utils_test.get_time(session, time_command, time_filter_re, time_format) finally: session.close() # Report results host_delta_total = ht2 - ht0 guest_delta_total = gt2 - gt0 drift_total = 100.0 * (host_delta_total - guest_delta_total) / host_delta logging.info("Total host duration including rest: %.2f", host_delta_total) logging.info("Total guest duration including rest: %.2f", guest_delta_total) logging.info("Total drift after rest: %.2f%%", drift_total) # Fail the test if necessary if abs(drift) > drift_threshold: raise error.TestFail("Time drift too large: %.2f%%" % drift) if abs(drift_total) > drift_threshold_after_rest: raise error.TestFail("Time drift too large after rest period: %.2f%%" % drift_total)
def run(test, params, env): """ Time manage test: 1) Generate stress in host. 2) Run atleast 15 vms with "driftfix=slew" option 3) Reboot the guest. 4) Repeat the step 3 for all guests and check whether the guest responds properly(not any watchdog reported). 5) TODO: Improve the way of checking the response and run some stress inside guest too. 6) Continue the step 4 for 10 iterations and record the guest/host realtime, calculate drift in time for each iterations. 7) Print the drift values for all sessions 8) TODO: Validate if the drift value has to be within defined value :param test: QEMU test object. :param params: Dictionary with test parameters. :param env: Dictionary with the test environment. """ # Checking the main vm is alive vm = env.get_vm(params["main_vm"]) vm.verify_alive() timeout = int(params.get("login_timeout", 360)) session = vm.wait_for_login(timeout=timeout) # Collect test parameters login_timeout = float(params.get("login_timeout", 240)) host_load_command = params["host_load_command"] host_load_kill_command = params["host_load_kill_command"] time_command = params["time_command"] time_filter_re = params["time_filter_re"] time_format = params["time_format"] # Intialize the variables itr = 0 num = 2 host_load_sessions = [] sessions = [session] prev_time = [] curr_time = [] timedrift = [] totaldrift = [] vmnames = ["virt-tests-vm1"] # Run some load on the host logging.info("Starting load on host.") host_load_sessions.append(aexpect.run_bg(host_load_command, output_func=logging.debug, output_prefix="host load ", timeout=0.5)) # Boot the VMs try: while num <= int(params["max_vms"]): # Clone vm according to the first one vm_name = "virt-tests-vm%d" % num vmnames.append(vm_name) vm_params = vm.params.copy() curr_vm = vm.clone(vm_name, vm_params) env.register_vm(vm_name, curr_vm) env_process.preprocess_vm(test, vm_params, env, vm_name) params["vms"] += " " + vm_name sessions.append(curr_vm.wait_for_login(timeout=login_timeout)) logging.info("Guest #%d booted up successfully", num) # Check whether all previous shell sessions are responsive error.context("checking responsiveness of the booted guest") for se in sessions: se.cmd(params["alive_test_cmd"]) num += 1 while itr <= int(params["max_itrs"]): for vmid, se in enumerate(sessions): # Get the respective vm object vmname = "virt-tests-vm%d" % (vmid + 1) vm = env.get_vm(vmname) # Run current iteration logging.info( "Rebooting:vm%d iteration %d " % ((vmid + 1), itr)) se = vm.reboot(se, timeout=timeout) # Remember the current changed session sessions[vmid] = se error.context("checking responsiveness of guest") se.cmd(params["alive_test_cmd"]) if itr == 0: (ht0, gt0) = utils_test.get_time(se, time_command, time_filter_re, time_format) prev_time.append((ht0, gt0)) else: (ht1, gt1) = utils_test.get_time(se, time_command, time_filter_re, time_format) curr_time.append((ht1, gt1)) if itr != 0: for i in range(int(params["max_vms"])): hdelta = curr_time[i][0] - prev_time[i][0] gdelta = curr_time[i][1] - prev_time[i][1] drift = "%.2f" % (100.0 * (hdelta - gdelta) / hdelta) timedrift.append(drift) totaldrift.append(timedrift) prev_time = curr_time timedrift = [] curr_time = [] # Wait for some time before next iteration time.sleep(30) itr += 1 logging.info("The time drift values for all VM sessions/iterations") logging.info("VM-Name:%s" % vmnames) for idx, value in enumerate(totaldrift): logging.info("itr-%2d:%s" % (idx + 1, value)) finally: for se in sessions: # Closing all the sessions. se.close() logging.info("killing load on host.") host_load_sessions.append(aexpect.run_bg(host_load_kill_command, output_func=logging.debug, output_prefix="host load kill", timeout=0.5))