Esempio n. 1
0
def run(test, params, env):
    """
    PXE test:

    1) Boot up guest from NIC(from pxe/gpxe server)
    2) Snoop the tftp packet in the tap device
    3) Analyzing the tcpdump result

    :param test: QEMU test object.
    :param params: Dictionary with the test parameters.
    :param env: Dictionary with test environment.
    """
    error_context.context("Try to boot from NIC", logging.info)
    vm = env.get_vm(params["main_vm"])
    vm.verify_alive()
    timeout = int(params.get("pxe_timeout", 60))

    error_context.context("Snoop packet in the tap device", logging.info)
    tcpdump_cmd = "tcpdump -nli %s" % vm.get_ifname()
    try:
        tcpdump_process = aexpect.run_bg(command=tcpdump_cmd,
                                         output_func=logging.debug,
                                         output_prefix="(pxe capture) ")
        if not tcpdump_process.read_until_output_matches(['tftp'],
                                                         timeout=timeout):
            test.fail("Couldn't find any TFTP packets after %s seconds" %
                      timeout)
        logging.info("Found TFTP packet")
    finally:
        try:
            tcpdump_process.kill()
        except:
            pass
Esempio n. 2
0
 def get_procs():
     procs = []
     for x in range(0, int(nums_cpu)):
         pipefile = '/tmp/virtio-trace/trace-path-cpu{}.out'.format(x)
         proc = aexpect.run_bg('cat %s' % pipefile)
         procs.append(proc)
     return procs
Esempio n. 3
0
def raw_ping(command, timeout, session, output_func):
    """
    Low-level ping command execution.

    @param command: Ping command.
    @param timeout: Timeout of the ping command.
    @param session: Local executon hint or session to execute the ping command.
    """
    if session is None:
        process = aexpect.run_bg(command,
                                 output_func=output_func,
                                 timeout=timeout)

        # Send SIGINT signal to notify the timeout of running ping process,
        # Because ping have the ability to catch the SIGINT signal so we can
        # always get the packet loss ratio even if timeout.
        if process.is_alive():
            virt_utils.kill_process_tree(process.get_pid(), signal.SIGINT)

        status = process.get_status()
        output = process.get_output()

        process.close()
        return status, output
    else:
        output = ""
        try:
            output = session.cmd_output(command,
                                        timeout=timeout,
                                        print_func=output_func)
        except aexpect.ShellTimeoutError:
            # Send ctrl+c (SIGINT) through ssh session
            session.send("\003")
            try:
                output2 = session.read_up_to_prompt(print_func=output_func)
                output += output2
            except aexpect.ExpectTimeoutError, e:
                output += e.output
                # We also need to use this session to query the return value
                session.send("\003")

        session.sendline(session.status_test_command)
        try:
            o2 = session.read_up_to_prompt()
        except aexpect.ExpectError:
            status = -1
        else:
            try:
                status = int(re.findall("\d+", o2)[0])
            except Exception:
                status = -1

        return status, output
Esempio n. 4
0
def raw_ping(command, timeout, session, output_func):
    """
    Low-level ping command execution.

    @param command: Ping command.
    @param timeout: Timeout of the ping command.
    @param session: Local executon hint or session to execute the ping command.
    """
    if session is None:
        process = aexpect.run_bg(command, output_func=output_func,
                                        timeout=timeout)

        # Send SIGINT signal to notify the timeout of running ping process,
        # Because ping have the ability to catch the SIGINT signal so we can
        # always get the packet loss ratio even if timeout.
        if process.is_alive():
            virt_utils.kill_process_tree(process.get_pid(), signal.SIGINT)

        status = process.get_status()
        output = process.get_output()

        process.close()
        return status, output
    else:
        output = ""
        try:
            output = session.cmd_output(command, timeout=timeout,
                                        print_func=output_func)
        except aexpect.ShellTimeoutError:
            # Send ctrl+c (SIGINT) through ssh session
            session.send("\003")
            try:
                output2 = session.read_up_to_prompt(print_func=output_func)
                output += output2
            except aexpect.ExpectTimeoutError, e:
                output += e.output
                # We also need to use this session to query the return value
                session.send("\003")

        session.sendline(session.status_test_command)
        try:
            o2 = session.read_up_to_prompt()
        except aexpect.ExpectError:
            status = -1
        else:
            try:
                status = int(re.findall("\d+", o2)[0])
            except Exception:
                status = -1

        return status, output
Esempio n. 5
0
def run(test, params, env):
    """
    Time drift test (mainly for Windows guests):

    1) Log into a guest.
    2) Take a time reading from the guest and host.
    3) Run load on the guest and host.
    4) Take a second time reading.
    5) Stop the load and rest for a while.
    6) Take a third time reading.
    7) If the drift immediately after load is higher than a user-
    specified value (in %), fail.
    If the drift after the rest period is higher than a user-specified value,
    fail.

    :param test: QEMU test object.
    :param params: Dictionary with test parameters.
    :param env: Dictionary with the test environment.
    """

    # Helper functions
    def set_cpu_affinity(pid, mask):
        """
        Set the CPU affinity of all threads of the process with PID pid.
        Do this recursively for all child processes as well.

        :param pid: The process ID.
        :param mask: The CPU affinity mask.
        :return: A dict containing the previous mask for each thread.
        """
        tids = decode_to_text(
            process.system_output("ps -L --pid=%s -o lwp=" % pid,
                                  verbose=False,
                                  ignore_status=True)).split()
        prev_masks = {}
        for tid in tids:
            prev_mask = decode_to_text(
                process.system_output("taskset -p %s" % tid,
                                      verbose=False)).split()[-1]
            prev_masks[tid] = prev_mask
            process.system("taskset -p %s %s" % (mask, tid), verbose=False)
        children = decode_to_text(
            process.system_output("ps --ppid=%s -o pid=" % pid,
                                  verbose=False,
                                  ignore_status=True)).split()
        for child in children:
            prev_masks.update(set_cpu_affinity(child, mask))
        return prev_masks

    def restore_cpu_affinity(prev_masks):
        """
        Restore the CPU affinity of several threads.

        :param prev_masks: A dict containing TIDs as keys and masks as values.
        """
        for tid, mask in prev_masks.items():
            process.system("taskset -p %s %s" % (mask, tid),
                           verbose=False,
                           ignore_status=True)

    vm = env.get_vm(params["main_vm"])
    vm.verify_alive()

    boot_option_added = params.get("boot_option_added")
    boot_option_removed = params.get("boot_option_removed")
    if boot_option_added or boot_option_removed:
        utils_test.update_boot_option(vm,
                                      args_removed=boot_option_removed,
                                      args_added=boot_option_added)

    if params["os_type"] == "windows":
        utils_time.sync_timezone_win(vm)

    timeout = int(params.get("login_timeout", 360))
    session = vm.wait_for_serial_login(timeout=timeout)

    # Collect test parameters:
    # Command to run to get the current time
    time_command = params["time_command"]
    # Filter which should match a string to be passed to time.strptime()
    time_filter_re = params["time_filter_re"]
    # Time format for time.strptime()
    time_format = params["time_format"]
    guest_load_command = params["guest_load_command"]
    guest_load_stop_command = params["guest_load_stop_command"]
    host_load_command = params["host_load_command"]
    guest_load_instances = params["guest_load_instances"]
    host_load_instances = params["host_load_instances"]
    if not guest_load_instances and not host_load_instances:
        host_load_instances = cpu.total_cpus_count()
        guest_load_instances = vm.get_cpu_count()
    else:
        host_load_instances = int(host_load_instances)
        guest_load_instances = int(guest_load_instances)
    # CPU affinity mask for taskset
    cpu_mask = int(params.get("cpu_mask", "0xFF"), 16)
    load_duration = float(params.get("load_duration", "30"))
    rest_duration = float(params.get("rest_duration", "10"))
    drift_threshold = float(params.get("drift_threshold", "200"))
    drift_threshold_after_rest = float(
        params.get("drift_threshold_after_rest", "200"))
    test_duration = float(params.get("test_duration", "60"))
    interval_gettime = float(params.get("interval_gettime", "20"))
    guest_load_sessions = []
    host_load_sessions = []

    try:
        # Set the VM's CPU affinity
        prev_affinity = set_cpu_affinity(vm.get_shell_pid(), cpu_mask)

        try:
            # Open shell sessions with the guest
            logging.info("Starting load on guest...")
            for i in range(guest_load_instances):
                load_session = vm.wait_for_login(timeout=timeout)
                # Set output func to None to stop it from being called so we
                # can change the callback function and the parameters it takes
                # with no problems
                load_session.set_output_func(None)
                load_session.set_output_params(())
                load_session.set_output_prefix("(guest load %d) " % i)
                load_session.set_output_func(logging.debug)
                guest_load_sessions.append(load_session)

            # Get time before load
            # (ht stands for host time, gt stands for guest time)
            (ht0, gt0) = utils_test.get_time(session, time_command,
                                             time_filter_re, time_format)

            # Run some load on the guest
            if params["os_type"] == "linux":
                for i, load_session in enumerate(guest_load_sessions):
                    load_session.sendline(guest_load_command % i)
            else:
                for load_session in guest_load_sessions:
                    load_session.sendline(guest_load_command)

            # Run some load on the host
            logging.info("Starting load on host...")
            for i in range(host_load_instances):
                load_cmd = aexpect.run_bg(host_load_command,
                                          output_func=logging.debug,
                                          output_prefix="(host load %d) " % i,
                                          timeout=0.5)
                host_load_sessions.append(load_cmd)
                # Set the CPU affinity of the load process
                pid = load_cmd.get_pid()
                set_cpu_affinity(pid, cpu_mask << i)

            # Sleep for a while (during load)
            logging.info("Sleeping for %s seconds...", load_duration)
            time.sleep(load_duration)

            start_time = time.time()
            while (time.time() - start_time) < test_duration:
                # Get time delta after load
                (ht1, gt1) = utils_test.get_time(session, time_command,
                                                 time_filter_re, time_format)

                # Report results
                host_delta = ht1 - ht0
                guest_delta = gt1 - gt0
                drift = 100.0 * (host_delta - guest_delta) / host_delta
                logging.info("Host duration: %.2f", host_delta)
                logging.info("Guest duration: %.2f", guest_delta)
                logging.info("Drift: %.2f%%", drift)
                time.sleep(interval_gettime)

        finally:
            logging.info("Cleaning up...")
            # Restore the VM's CPU affinity
            restore_cpu_affinity(prev_affinity)
            # Stop the guest load
            if guest_load_stop_command:
                session.cmd_output(guest_load_stop_command)
            # Close all load shell sessions
            for load_session in guest_load_sessions:
                load_session.close()
            for load_session in host_load_sessions:
                load_session.close()

        # Sleep again (rest)
        logging.info("Sleeping for %s seconds...", rest_duration)
        time.sleep(rest_duration)

        # Get time after rest
        (ht2, gt2) = utils_test.get_time(session, time_command, time_filter_re,
                                         time_format)

    finally:
        session.close()
        # remove flags add for this test.
        if boot_option_added or boot_option_removed:
            utils_test.update_boot_option(vm,
                                          args_removed=boot_option_added,
                                          args_added=boot_option_removed)

    # Report results
    host_delta_total = ht2 - ht0
    guest_delta_total = gt2 - gt0
    drift_total = 100.0 * (host_delta_total - guest_delta_total) / host_delta
    logging.info("Total host duration including rest: %.2f", host_delta_total)
    logging.info("Total guest duration including rest: %.2f",
                 guest_delta_total)
    logging.info("Total drift after rest: %.2f%%", drift_total)

    # Fail the test if necessary
    if abs(drift) > drift_threshold:
        test.fail("Time drift too large: %.2f%%" % drift)
    if abs(drift_total) > drift_threshold_after_rest:
        test.fail("Time drift too large after rest period: %.2f%%" %
                  drift_total)
Esempio n. 6
0
def run(test, params, env):
    """
    Time manage test:

    1) Generate stress in host.
    2) Run atleast 15 vms with "driftfix=slew" option
    3) Reboot the guest.
    4) Repeat the step 3 for all guests and check whether the guest
       responds properly(not any watchdog reported).
    5) TODO: Improve the way of checking the response and
        run some stress inside guest too.
    6) Continue the step 4 for 10 iterations and
       record the guest/host realtime, calculate drift in time for
       each iterations.
    7) Print the drift values for all sessions
    8) TODO: Validate if the drift value has to be within defined value

    :param test: QEMU test object.
    :param params: Dictionary with test parameters.
    :param env: Dictionary with the test environment.
    """
    # Checking the main vm is alive
    vm = env.get_vm(params["main_vm"])
    vm.verify_alive()
    timeout = int(params.get("login_timeout", 360))
    session = vm.wait_for_login(timeout=timeout)

    # Collect test parameters
    login_timeout = float(params.get("login_timeout", 240))
    host_load_command = params["host_load_command"]
    host_load_kill_command = params["host_load_kill_command"]
    time_command = params["time_command"]
    time_filter_re = params["time_filter_re"]
    time_format = params["time_format"]

    # Intialize the variables
    itr = 0
    num = 2
    host_load_sessions = []
    sessions = [session]
    prev_time = []
    curr_time = []
    timedrift = []
    totaldrift = []
    vmnames = ["virt-tests-vm1"]

    # Run some load on the host
    logging.info("Starting load on host.")
    host_load_sessions.append(aexpect.run_bg(host_load_command,
                                             output_func=logging.debug,
                                             output_prefix="host load ",
                                             timeout=0.5))
    # Boot the VMs
    try:
        while num <= int(params["max_vms"]):
            # Clone vm according to the first one
            vm_name = "virt-tests-vm%d" % num
            vmnames.append(vm_name)
            vm_params = vm.params.copy()
            curr_vm = vm.clone(vm_name, vm_params)
            env.register_vm(vm_name, curr_vm)
            env_process.preprocess_vm(test, vm_params, env, vm_name)
            params["vms"] += " " + vm_name

            sessions.append(curr_vm.wait_for_login(timeout=login_timeout))
            logging.info("Guest #%d booted up successfully", num)

            # Check whether all previous shell sessions are responsive
            error.context("checking responsiveness of the booted guest")
            for se in sessions:
                se.cmd(params["alive_test_cmd"])
            num += 1

        while itr <= int(params["max_itrs"]):
            for vmid, se in enumerate(sessions):
                # Get the respective vm object
                vmname = "virt-tests-vm%d" % (vmid + 1)
                vm = env.get_vm(vmname)
                # Run current iteration
                logging.info(
                    "Rebooting:vm%d iteration %d " % ((vmid + 1), itr))
                se = vm.reboot(se, timeout=timeout)
                # Remember the current changed session
                sessions[vmid] = se
                error.context("checking responsiveness of guest")
                se.cmd(params["alive_test_cmd"])
                if itr == 0:
                    (ht0, gt0) = utils_test.get_time(se, time_command,
                                                     time_filter_re, time_format)
                    prev_time.append((ht0, gt0))
                else:
                    (ht1, gt1) = utils_test.get_time(se, time_command,
                                                     time_filter_re, time_format)
                    curr_time.append((ht1, gt1))
            if itr != 0:
                for i in range(int(params["max_vms"])):
                    hdelta = curr_time[i][0] - prev_time[i][0]
                    gdelta = curr_time[i][1] - prev_time[i][1]
                    drift = "%.2f" % (100.0 * (hdelta - gdelta) / hdelta)
                    timedrift.append(drift)
                totaldrift.append(timedrift)
                prev_time = curr_time
                timedrift = []
                curr_time = []
            # Wait for some time before next iteration
            time.sleep(30)
            itr += 1

        logging.info("The time drift values for all VM sessions/iterations")
        logging.info("VM-Name:%s" % vmnames)
        for idx, value in enumerate(totaldrift):
            logging.info("itr-%2d:%s" % (idx + 1, value))

    finally:
        for se in sessions:
            # Closing all the sessions.
            se.close()
        logging.info("killing load on host.")
        host_load_sessions.append(aexpect.run_bg(host_load_kill_command,
                                                 output_func=logging.debug,
                                                 output_prefix="host load kill",
                                                 timeout=0.5))
Esempio n. 7
0
def run(test, params, env):
    """
    Time manage test:

    1) Generate stress in host.
    2) Run atleast 15 vms with "driftfix=slew" option
    3) Reboot the guest.
    4) Repeat the step 3 for all guests and check whether the guest
       responds properly(not any watchdog reported).
    5) TODO: Improve the way of checking the response and
        run some stress inside guest too.
    6) Continue the step 4 for 10 iterations and
       record the guest/host realtime, calculate drift in time for
       each iterations.
    7) Print the drift values for all sessions
    8) TODO: Validate if the drift value has to be within defined value

    :param test: QEMU test object.
    :param params: Dictionary with test parameters.
    :param env: Dictionary with the test environment.
    """
    # Checking the main vm is alive
    vm = env.get_vm(params["main_vm"])
    vm.verify_alive()
    timeout = int(params.get("login_timeout", 360))
    session = vm.wait_for_login(timeout=timeout)

    # Collect test parameters
    login_timeout = float(params.get("login_timeout", 240))
    host_load_command = params["host_load_command"]
    host_load_kill_command = params["host_load_kill_command"]
    time_command = params["time_command"]
    time_filter_re = params["time_filter_re"]
    time_format = params["time_format"]

    # Intialize the variables
    itr = 0
    num = 2
    host_load_sessions = []
    sessions = [session]
    prev_time = []
    curr_time = []
    timedrift = []
    totaldrift = []
    vmnames = ["virt-tests-vm1"]

    # Run some load on the host
    logging.info("Starting load on host.")
    host_load_sessions.append(
        aexpect.run_bg(host_load_command,
                       output_func=logging.debug,
                       output_prefix="host load ",
                       timeout=0.5))
    # Boot the VMs
    try:
        while num <= int(params["max_vms"]):
            # Clone vm according to the first one
            vm_name = "virt-tests-vm%d" % num
            vmnames.append(vm_name)
            vm_params = vm.params.copy()
            curr_vm = vm.clone(vm_name, vm_params)
            env.register_vm(vm_name, curr_vm)
            env_process.preprocess_vm(test, vm_params, env, vm_name)
            params["vms"] += " " + vm_name

            sessions.append(curr_vm.wait_for_login(timeout=login_timeout))
            logging.info("Guest #%d booted up successfully", num)

            # Check whether all previous shell sessions are responsive
            error.context("checking responsiveness of the booted guest")
            for se in sessions:
                se.cmd(params["alive_test_cmd"])
            num += 1

        while itr <= int(params["max_itrs"]):
            for vmid, se in enumerate(sessions):
                # Get the respective vm object
                vmname = "virt-tests-vm%d" % (vmid + 1)
                vm = env.get_vm(vmname)
                # Run current iteration
                logging.info("Rebooting:vm%d iteration %d " %
                             ((vmid + 1), itr))
                se = vm.reboot(se, timeout=timeout)
                # Remember the current changed session
                sessions[vmid] = se
                error.context("checking responsiveness of guest")
                se.cmd(params["alive_test_cmd"])
                if itr == 0:
                    (ht0, gt0) = utils_test.get_time(se, time_command,
                                                     time_filter_re,
                                                     time_format)
                    prev_time.append((ht0, gt0))
                else:
                    (ht1, gt1) = utils_test.get_time(se, time_command,
                                                     time_filter_re,
                                                     time_format)
                    curr_time.append((ht1, gt1))
            if itr != 0:
                for i in range(int(params["max_vms"])):
                    hdelta = curr_time[i][0] - prev_time[i][0]
                    gdelta = curr_time[i][1] - prev_time[i][1]
                    drift = "%.2f" % (100.0 * (hdelta - gdelta) / hdelta)
                    timedrift.append(drift)
                totaldrift.append(timedrift)
                prev_time = curr_time
                timedrift = []
                curr_time = []
            # Wait for some time before next iteration
            time.sleep(30)
            itr += 1

        logging.info("The time drift values for all VM sessions/iterations")
        logging.info("VM-Name:%s" % vmnames)
        for idx, value in enumerate(totaldrift):
            logging.info("itr-%2d:%s" % (idx + 1, value))

    finally:
        for se in sessions:
            # Closing all the sessions.
            se.close()
        logging.info("killing load on host.")
        host_load_sessions.append(
            aexpect.run_bg(host_load_kill_command,
                           output_func=logging.debug,
                           output_prefix="host load kill",
                           timeout=0.5))
Esempio n. 8
0
def run(test, params, env):
    """
    Test native TLS encryption on chardev TCP transports
    Scenario 1:
        a. Run gnutls server
        b. Launch QEMU with a serial port as TLS client
        c. Check the server endpoint output
    Scenario 2:
        a. Launch QEMU with a serial port as TLS server
        b. Run gnutls client to connect TLS server
        c. Check the client endpoint output
    Scenario 3:
        a. Launch QEMU with a serial port as TLS server
        b. Execute 'cat /dev/ttyS0' in guest which boot from step 1
        c. Launch QEMU with a serial port as TLS client
        d. Check the output of step b
    :param test: QEMU test object
    :param params: Dictionary with the test parameters
    :param env: Dictionary with test environment.
    """
    clean_cmd = params["clean_cmd"]
    try:
        setup_certs(params)
        expected_msg = params["expected_msg"]
        hostname = process.run('hostname',
                               ignore_status=False,
                               shell=True,
                               verbose=True).stdout_text.strip()
        port = str(utils_misc.find_free_ports(5000, 9999, 1, hostname)[0])

        # Scenario 1
        gnutls_cmd_server = params.get("gnutls_cmd_server")
        if gnutls_cmd_server:
            gnutls_cmd_server = gnutls_cmd_server % port
            params["extra_params"] = params["extra_params"] % (hostname, port)
            error_context.context("Run gnutls server ...", logging.info)
            tls_server = aexpect.run_bg(gnutls_cmd_server)
            params['start_vm'] = 'yes'
            vm_name = params['main_vm']
            error_context.context(
                "Launch QEMU with a serial port as TLS client", logging.info)
            env_process.preprocess_vm(test, params, env, vm_name)
            if not utils_misc.wait_for(
                    lambda: expected_msg in tls_server.get_output(),
                    first=5,
                    timeout=15):
                test.fail("TLS server can't connect client succssfully.")

        # Scenario 2
        gnutls_cmd_client = params.get("gnutls_cmd_client")
        if gnutls_cmd_client:
            gnutls_cmd_client = gnutls_cmd_client % (port, hostname)
            params["extra_params"] = params["extra_params"] % (hostname, port)
            params['start_vm'] = 'yes'
            vm_name = params['main_vm']
            error_context.context(
                "Launch QEMU with a serial port as TLS server", logging.info)
            env_process.preprocess_vm(test, params, env, vm_name)
            error_context.context("Run gnutls client to connect TLS server",
                                  logging.info)
            tls_client = aexpect.run_bg(gnutls_cmd_client)
            if not utils_misc.wait_for(
                    lambda: expected_msg in tls_client.get_output(),
                    first=5,
                    timeout=15):
                test.fail("TLS client can't connect server succssfully.")

        # Scenario 3:
        guest_cmd = params.get("guest_cmd")
        if guest_cmd:
            params["start_vm"] = "yes"
            vms = params.get("vms").split()
            params["extra_params"] = params["extra_params_%s" %
                                            vms[0]] % (hostname, port)
            error_context.context(
                "Launch QEMU with a serial port as TLS server", logging.info)
            env_process.preprocess_vm(test, params, env, vms[0])
            vm1 = env.get_vm(vms[0])
            session_vm1 = vm1.wait_for_login()
            session_vm1.cmd(guest_cmd)
            params["extra_params"] = params["extra_params_%s" %
                                            vms[1]] % (hostname, port)
            error_context.context(
                "Launch QEMU with a serial port as TLS client", logging.info)
            env_process.preprocess_vm(test, params, env, vms[1])
            try:
                session_vm1.read_until_output_matches([expected_msg],
                                                      timeout=15)
            except aexpect.ExpectError:
                test.fail("Can't connect TLS client inside TLS server guest.")
            vm2 = env.get_vm(vms[1])
            session_vm1.close()
            vm1.destroy()
            vm2.destroy()
    finally:
        gnutls_pid = process.getoutput("pgrep -f gnutls", shell=True)
        if gnutls_pid:
            process.run("pkill -9 gnutls")
        process.run(clean_cmd)
Esempio n. 9
0
def run(test, params, env):
    """
    Time drift test (mainly for Windows guests):

    1) Log into a guest.
    2) Take a time reading from the guest and host.
    3) Run load on the guest and host.
    4) Take a second time reading.
    5) Stop the load and rest for a while.
    6) Take a third time reading.
    7) If the drift immediately after load is higher than a user-
    specified value (in %), fail.
    If the drift after the rest period is higher than a user-specified value,
    fail.

    :param test: QEMU test object.
    :param params: Dictionary with test parameters.
    :param env: Dictionary with the test environment.
    """
    # Helper functions
    def set_cpu_affinity(pid, mask):
        """
        Set the CPU affinity of all threads of the process with PID pid.
        Do this recursively for all child processes as well.

        :param pid: The process ID.
        :param mask: The CPU affinity mask.
        :return: A dict containing the previous mask for each thread.
        """
        tids = commands.getoutput("ps -L --pid=%s -o lwp=" % pid).split()
        prev_masks = {}
        for tid in tids:
            prev_mask = commands.getoutput("taskset -p %s" % tid).split()[-1]
            prev_masks[tid] = prev_mask
            commands.getoutput("taskset -p %s %s" % (mask, tid))
        children = commands.getoutput("ps --ppid=%s -o pid=" % pid).split()
        for child in children:
            prev_masks.update(set_cpu_affinity(child, mask))
        return prev_masks

    def restore_cpu_affinity(prev_masks):
        """
        Restore the CPU affinity of several threads.

        :param prev_masks: A dict containing TIDs as keys and masks as values.
        """
        for tid, mask in prev_masks.items():
            commands.getoutput("taskset -p %s %s" % (mask, tid))

    vm = env.get_vm(params["main_vm"])
    vm.verify_alive()

    boot_option_added = params.get("boot_option_added")
    boot_option_removed = params.get("boot_option_removed")
    if boot_option_added or boot_option_removed:
        utils_test.update_boot_option(vm,
                                      args_removed=boot_option_removed,
                                      args_added=boot_option_added)

    timeout = int(params.get("login_timeout", 360))
    session = vm.wait_for_login(timeout=timeout)

    # Collect test parameters:
    # Command to run to get the current time
    time_command = params["time_command"]
    # Filter which should match a string to be passed to time.strptime()
    time_filter_re = params["time_filter_re"]
    # Time format for time.strptime()
    time_format = params["time_format"]
    guest_load_command = params["guest_load_command"]
    guest_load_stop_command = params["guest_load_stop_command"]
    host_load_command = params["host_load_command"]
    guest_load_instances = int(params.get("guest_load_instances", "1"))
    host_load_instances = int(params.get("host_load_instances", "0"))
    # CPU affinity mask for taskset
    cpu_mask = params.get("cpu_mask", "0xFF")
    load_duration = float(params.get("load_duration", "30"))
    rest_duration = float(params.get("rest_duration", "10"))
    drift_threshold = float(params.get("drift_threshold", "200"))
    drift_threshold_after_rest = float(params.get("drift_threshold_after_rest",
                                                  "200"))
    test_duration = float(params.get("test_duration", "60"))
    interval_gettime = float(params.get("interval_gettime", "20"))
    guest_load_sessions = []
    host_load_sessions = []

    try:
        # Set the VM's CPU affinity
        prev_affinity = set_cpu_affinity(vm.get_shell_pid(), cpu_mask)

        try:
            # Open shell sessions with the guest
            logging.info("Starting load on guest...")
            for i in range(guest_load_instances):
                load_session = vm.login()
                # Set output func to None to stop it from being called so we
                # can change the callback function and the parameters it takes
                # with no problems
                load_session.set_output_func(None)
                load_session.set_output_params(())
                load_session.set_output_prefix("(guest load %d) " % i)
                load_session.set_output_func(logging.debug)
                guest_load_sessions.append(load_session)

            # Get time before load
            # (ht stands for host time, gt stands for guest time)
            (ht0, gt0) = utils_test.get_time(session,
                                             time_command,
                                             time_filter_re,
                                             time_format)

            # Run some load on the guest
            for load_session in guest_load_sessions:
                load_session.sendline(guest_load_command)

            # Run some load on the host
            logging.info("Starting load on host...")
            for i in range(host_load_instances):
                load_cmd = aexpect.run_bg(host_load_command,
                                          output_func=logging.debug,
                                          output_prefix="(host load %d) " % i,
                                          timeout=0.5)
                host_load_sessions.append(load_cmd)
                # Set the CPU affinity of the load process
                pid = load_cmd.get_pid()
                set_cpu_affinity(pid, cpu_mask)

            # Sleep for a while (during load)
            logging.info("Sleeping for %s seconds...", load_duration)
            time.sleep(load_duration)

            start_time = time.time()
            while (time.time() - start_time) < test_duration:
                # Get time delta after load
                (ht1, gt1) = utils_test.get_time(session,
                                                 time_command,
                                                 time_filter_re,
                                                 time_format)

                # Report results
                host_delta = ht1 - ht0
                guest_delta = gt1 - gt0
                drift = 100.0 * (host_delta - guest_delta) / host_delta
                logging.info("Host duration: %.2f", host_delta)
                logging.info("Guest duration: %.2f", guest_delta)
                logging.info("Drift: %.2f%%", drift)
                time.sleep(interval_gettime)

        finally:
            logging.info("Cleaning up...")
            # Restore the VM's CPU affinity
            restore_cpu_affinity(prev_affinity)
            # Stop the guest load
            if guest_load_stop_command:
                session.cmd_output(guest_load_stop_command)
            # Close all load shell sessions
            for load_session in guest_load_sessions:
                load_session.close()
            for load_session in host_load_sessions:
                load_session.close()

        # Sleep again (rest)
        logging.info("Sleeping for %s seconds...", rest_duration)
        time.sleep(rest_duration)

        # Get time after rest
        (ht2, gt2) = utils_test.get_time(session,
                                         time_command,
                                         time_filter_re,
                                         time_format)

    finally:
        session.close()
        # remove flags add for this test.
        if boot_option_added or boot_option_removed:
            utils_test.update_boot_option(vm,
                                          args_removed=boot_option_added,
                                          args_added=boot_option_removed)

    # Report results
    host_delta_total = ht2 - ht0
    guest_delta_total = gt2 - gt0
    drift_total = 100.0 * (host_delta_total - guest_delta_total) / host_delta
    logging.info("Total host duration including rest: %.2f", host_delta_total)
    logging.info(
        "Total guest duration including rest: %.2f", guest_delta_total)
    logging.info("Total drift after rest: %.2f%%", drift_total)

    # Fail the test if necessary
    if abs(drift) > drift_threshold:
        raise error.TestFail("Time drift too large: %.2f%%" % drift)
    if abs(drift_total) > drift_threshold_after_rest:
        raise error.TestFail("Time drift too large after rest period: %.2f%%"
                             % drift_total)