def test(self):
cpu_model, extra_flags = parse_cpu_model()
logging.debug("Run tests with cpu model %s.", (cpu_model))
flags = HgFlags(cpu_model, extra_flags)
(self.vm, session) = start_guest_with_cpuflags(cpu_model, smp)
def encap(timeout):
random.seed()
begin = time.time()
end = begin
if smp > 1:
while end - begin < 60:
cpu = random.randint(1, smp - 1)
if random.randint(0, 1):
disable_cpu(session, cpu, True)
else:
disable_cpu(session, cpu, False)
end = time.time()
return True
else:
logging.warning("For this test is necessary smp > 1.")
return False
timeout = 60
test_flags = flags.guest_flags
if all_host_supported_flags == "yes":
test_flags = flags.all_possible_guest_flags
result = virt_utils.parallel([(encap, [timeout]), (run_stress, [self.vm, timeout, test_flags])])
if not (result[0] and result[1]):
raise error.TestFail("Stress tests failed before" " end of testing.")
def test(self):
cpu_model, extra_flags = parse_cpu_model()
logging.debug("Run tests with cpu model %s.", (cpu_model))
flags = HgFlags(cpu_model, extra_flags)
(self.vm, session) = start_guest_with_cpuflags(cpu_model, smp)
def encap(timeout):
random.seed()
begin = time.time()
end = begin
if smp > 1:
while end - begin < 60:
cpu = random.randint(1, smp - 1)
if random.randint(0, 1):
disable_cpu(session, cpu, True)
else:
disable_cpu(session, cpu, False)
end = time.time()
return True
else:
logging.warning("For this test is necessary smp > 1.")
return False
timeout = 60
test_flags = flags.guest_flags
if all_host_supported_flags == "yes":
test_flags = flags.all_possible_guest_flags
result = virt_utils.parallel([(encap, [timeout]),
(run_stress,
[self.vm, timeout, test_flags])])
if not (result[0] and result[1]):
raise error.TestFail("Stress tests failed before"
" end of testing.")
def run_whql_submission(test, params, env):
"""
WHQL submission test:
1) Log into the client machines and into a DTM server machine
2) Copy the automation program binary (dsso_test_binary) to the server machine
3) Run the automation program
4) Pass the program all relevant parameters (e.g. device_data)
5) Wait for the program to terminate
6) Parse and report job results
(logs and HTML reports are placed in test.debugdir)
@param test: kvm test object
@param params: Dictionary with the test parameters
@param env: Dictionary with test environment.
"""
# Log into all client VMs
login_timeout = int(params.get("login_timeout", 360))
vms = []
sessions = []
for vm_name in params.objects("vms"):
vms.append(env.get_vm(vm_name))
vms[-1].verify_alive()
sessions.append(vms[-1].wait_for_login(timeout=login_timeout))
# Make sure all NICs of all client VMs are up
for vm in vms:
nics = vm.params.objects("nics")
for nic_index in range(len(nics)):
s = vm.wait_for_login(nic_index, 600)
s.close()
# Collect parameters
server_address = params.get("server_address")
server_shell_port = int(params.get("server_shell_port"))
server_file_transfer_port = int(params.get("server_file_transfer_port"))
server_studio_path = params.get("server_studio_path", "%programfiles%\\ " "Microsoft Driver Test Manager\\Studio")
dsso_test_binary = params.get("dsso_test_binary", "deps/whql_submission_15.exe")
dsso_test_binary = virt_utils.get_path(test.virtdir, dsso_test_binary)
dsso_delete_machine_binary = params.get("dsso_delete_machine_binary", "deps/whql_delete_machine_15.exe")
dsso_delete_machine_binary = virt_utils.get_path(test.virtdir, dsso_delete_machine_binary)
test_timeout = float(params.get("test_timeout", 600))
# Copy dsso binaries to the server
for filename in dsso_test_binary, dsso_delete_machine_binary:
rss_client.upload(server_address, server_file_transfer_port, filename, server_studio_path, timeout=60)
# Open a shell session with the server
server_session = virt_remote.remote_login(
"nc", server_address, server_shell_port, "", "", sessions[0].prompt, sessions[0].linesep
)
server_session.set_status_test_command(sessions[0].status_test_command)
# Get the computer names of the server and clients
cmd = "echo %computername%"
server_name = server_session.cmd_output(cmd).strip()
client_names = [session.cmd_output(cmd).strip() for session in sessions]
# Delete all client machines from the server's data store
server_session.cmd("cd %s" % server_studio_path)
for client_name in client_names:
cmd = "%s %s %s" % (os.path.basename(dsso_delete_machine_binary), server_name, client_name)
server_session.cmd(cmd, print_func=logging.debug)
# Reboot the client machines
sessions = virt_utils.parallel((vm.reboot, (session,)) for vm, session in zip(vms, sessions))
# Check the NICs again
for vm in vms:
nics = vm.params.objects("nics")
for nic_index in range(len(nics)):
s = vm.wait_for_login(nic_index, 600)
s.close()
# Run whql_pre_command and close the sessions
if params.get("whql_pre_command"):
for session in sessions:
session.cmd(params.get("whql_pre_command"), int(params.get("whql_pre_command_timeout", 600)))
session.close()
# Run the automation program on the server
pool_name = "%s_pool" % client_names[0]
submission_name = "%s_%s" % (client_names[0], params.get("submission_name"))
cmd = "%s %s %s %s %s %s" % (
os.path.basename(dsso_test_binary),
server_name,
pool_name,
submission_name,
test_timeout,
" ".join(client_names),
)
server_session.sendline(cmd)
# Helper function: wait for a given prompt and raise an exception if an
# error occurs
def find_prompt(prompt):
m, o = server_session.read_until_last_line_matches(
[prompt, server_session.prompt], print_func=logging.info, timeout=600
)
if m != 0:
errors = re.findall("^Error:.*$", o, re.I | re.M)
if errors:
raise error.TestError(errors[0])
else:
raise error.TestError("Error running automation program: " "could not find '%s' prompt" % prompt)
# Tell the automation program which device to test
find_prompt("Device to test:")
server_session.sendline(params.get("test_device"))
# Tell the automation program which jobs to run
find_prompt("Jobs to run:")
server_session.sendline(params.get("job_filter", ".*"))
# Set submission DeviceData
find_prompt("DeviceData name:")
for dd in params.objects("device_data"):
dd_params = params.object_params(dd)
if dd_params.get("dd_name") and dd_params.get("dd_data"):
server_session.sendline(dd_params.get("dd_name"))
server_session.sendline(dd_params.get("dd_data"))
server_session.sendline()
# Set submission descriptors
find_prompt("Descriptor path:")
for desc in params.objects("descriptors"):
desc_params = params.object_params(desc)
if desc_params.get("desc_path"):
server_session.sendline(desc_params.get("desc_path"))
server_session.sendline()
# Set machine dimensions for each client machine
for vm_name in params.objects("vms"):
vm_params = params.object_params(vm_name)
find_prompt(r"Dimension name\b.*:")
for dp in vm_params.objects("dimensions"):
dp_params = vm_params.object_params(dp)
if dp_params.get("dim_name") and dp_params.get("dim_value"):
server_session.sendline(dp_params.get("dim_name"))
server_session.sendline(dp_params.get("dim_value"))
server_session.sendline()
# Set extra parameters for tests that require them (e.g. NDISTest)
for vm_name in params.objects("vms"):
vm_params = params.object_params(vm_name)
find_prompt(r"Parameter name\b.*:")
for dp in vm_params.objects("device_params"):
dp_params = vm_params.object_params(dp)
if dp_params.get("dp_name") and dp_params.get("dp_regex"):
server_session.sendline(dp_params.get("dp_name"))
server_session.sendline(dp_params.get("dp_regex"))
# Make sure the prompt appears again (if the device isn't found
# the automation program will terminate)
find_prompt(r"Parameter name\b.*:")
server_session.sendline()
# Wait for the automation program to terminate
try:
o = server_session.read_up_to_prompt(print_func=logging.info, timeout=test_timeout + 300)
# (test_timeout + 300 is used here because the automation program is
# supposed to terminate cleanly on its own when test_timeout expires)
done = True
except aexpect.ExpectError, e:
o = e.output
done = False
name_length = max(len(r["job"]) for r in results)
fmt = "%%-6s %%-%ds %%-15s %%-8s %%-8s %%-8s %%-15s" % name_length
f = open(os.path.join(test.debugdir, "summary"), "w")
print_summary_line(f, fmt % ("ID", "Job", "Status", "Pass", "Fail", "NotRun", "NotApplicable"))
print_summary_line(f, fmt % ("--", "---", "------", "----", "----", "------", "-------------"))
for r in results:
print_summary_line(
f, fmt % (r["id"], r["job"], r["status"], r["pass"], r["fail"], r["notrun"], r["notapplicable"])
)
f.close()
logging.info("(see logs and HTML reports in %s)", test.debugdir)
# Kill the client VMs and fail if the automation program did not terminate
# on time
if not done:
virt_utils.parallel(vm.destroy for vm in vms)
raise error.TestFail("The automation program did not terminate " "on time")
# Fail if there are failed or incomplete jobs (kill the client VMs if there
# are incomplete jobs)
failed_jobs = [r["job"] for r in results if r["status"].lower() == "investigate"]
running_jobs = [r["job"] for r in results if r["status"].lower() == "inprogress"]
errors = []
if failed_jobs:
errors += ["Jobs failed: %s." % failed_jobs]
if running_jobs:
for vm in vms:
vm.destroy()
errors += ["Jobs did not complete on time: %s." % running_jobs]
if errors:
raise error.TestFail(" ".join(errors))
def run_whql_submission(test, params, env):
"""
WHQL submission test:
1) Log into the client machines and into a DTM server machine
2) Copy the automation program binary (dsso_test_binary) to the server machine
3) Run the automation program
4) Pass the program all relevant parameters (e.g. device_data)
5) Wait for the program to terminate
6) Parse and report job results
(logs and HTML reports are placed in test.debugdir)
@param test: kvm test object
@param params: Dictionary with the test parameters
@param env: Dictionary with test environment.
"""
# Log into all client VMs
login_timeout = int(params.get("login_timeout", 360))
vms = []
sessions = []
for vm_name in params.objects("vms"):
vms.append(env.get_vm(vm_name))
vms[-1].verify_alive()
sessions.append(vms[-1].wait_for_login(timeout=login_timeout))
# Make sure all NICs of all client VMs are up
for vm in vms:
nics = vm.params.objects("nics")
for nic_index in range(len(nics)):
s = vm.wait_for_login(nic_index, 600)
s.close()
# Collect parameters
server_address = params.get("server_address")
server_shell_port = int(params.get("server_shell_port"))
server_file_transfer_port = int(params.get("server_file_transfer_port"))
server_studio_path = params.get("server_studio_path", "%programfiles%\\ "
"Microsoft Driver Test Manager\\Studio")
dsso_test_binary = params.get("dsso_test_binary",
"deps/whql_submission_15.exe")
dsso_test_binary = virt_utils.get_path(test.virtdir, dsso_test_binary)
dsso_delete_machine_binary = params.get("dsso_delete_machine_binary",
"deps/whql_delete_machine_15.exe")
dsso_delete_machine_binary = virt_utils.get_path(test.virtdir,
dsso_delete_machine_binary)
test_timeout = float(params.get("test_timeout", 600))
# Copy dsso binaries to the server
for filename in dsso_test_binary, dsso_delete_machine_binary:
rss_client.upload(server_address, server_file_transfer_port,
filename, server_studio_path, timeout=60)
# Open a shell session with the server
server_session = virt_remote.remote_login("nc", server_address,
server_shell_port, "", "",
sessions[0].prompt,
sessions[0].linesep)
server_session.set_status_test_command(sessions[0].status_test_command)
# Get the computer names of the server and clients
cmd = "echo %computername%"
server_name = server_session.cmd_output(cmd).strip()
client_names = [session.cmd_output(cmd).strip() for session in sessions]
# Delete all client machines from the server's data store
server_session.cmd("cd %s" % server_studio_path)
for client_name in client_names:
cmd = "%s %s %s" % (os.path.basename(dsso_delete_machine_binary),
server_name, client_name)
server_session.cmd(cmd, print_func=logging.debug)
# Reboot the client machines
sessions = virt_utils.parallel((vm.reboot, (session,))
for vm, session in zip(vms, sessions))
# Check the NICs again
for vm in vms:
nics = vm.params.objects("nics")
for nic_index in range(len(nics)):
s = vm.wait_for_login(nic_index, 600)
s.close()
# Run whql_pre_command and close the sessions
if params.get("whql_pre_command"):
for session in sessions:
session.cmd(params.get("whql_pre_command"),
int(params.get("whql_pre_command_timeout", 600)))
session.close()
# Run the automation program on the server
pool_name = "%s_pool" % client_names[0]
submission_name = "%s_%s" % (client_names[0],
params.get("submission_name"))
cmd = "%s %s %s %s %s %s" % (os.path.basename(dsso_test_binary),
server_name, pool_name, submission_name,
test_timeout, " ".join(client_names))
server_session.sendline(cmd)
# Helper function: wait for a given prompt and raise an exception if an
# error occurs
def find_prompt(prompt):
m, o = server_session.read_until_last_line_matches(
[prompt, server_session.prompt], print_func=logging.info,
timeout=600)
if m != 0:
errors = re.findall("^Error:.*$", o, re.I | re.M)
if errors:
raise error.TestError(errors[0])
else:
raise error.TestError("Error running automation program: "
"could not find '%s' prompt" % prompt)
# Tell the automation program which device to test
find_prompt("Device to test:")
server_session.sendline(params.get("test_device"))
# Tell the automation program which jobs to run
find_prompt("Jobs to run:")
server_session.sendline(params.get("job_filter", ".*"))
# Set submission DeviceData
find_prompt("DeviceData name:")
for dd in params.objects("device_data"):
dd_params = params.object_params(dd)
if dd_params.get("dd_name") and dd_params.get("dd_data"):
server_session.sendline(dd_params.get("dd_name"))
server_session.sendline(dd_params.get("dd_data"))
server_session.sendline()
# Set submission descriptors
find_prompt("Descriptor path:")
for desc in params.objects("descriptors"):
desc_params = params.object_params(desc)
if desc_params.get("desc_path"):
server_session.sendline(desc_params.get("desc_path"))
server_session.sendline()
# Set machine dimensions for each client machine
for vm_name in params.objects("vms"):
vm_params = params.object_params(vm_name)
find_prompt(r"Dimension name\b.*:")
for dp in vm_params.objects("dimensions"):
dp_params = vm_params.object_params(dp)
if dp_params.get("dim_name") and dp_params.get("dim_value"):
server_session.sendline(dp_params.get("dim_name"))
server_session.sendline(dp_params.get("dim_value"))
server_session.sendline()
# Set extra parameters for tests that require them (e.g. NDISTest)
for vm_name in params.objects("vms"):
vm_params = params.object_params(vm_name)
find_prompt(r"Parameter name\b.*:")
for dp in vm_params.objects("device_params"):
dp_params = vm_params.object_params(dp)
if dp_params.get("dp_name") and dp_params.get("dp_regex"):
server_session.sendline(dp_params.get("dp_name"))
server_session.sendline(dp_params.get("dp_regex"))
# Make sure the prompt appears again (if the device isn't found
# the automation program will terminate)
find_prompt(r"Parameter name\b.*:")
server_session.sendline()
# Wait for the automation program to terminate
try:
o = server_session.read_up_to_prompt(print_func=logging.info,
timeout=test_timeout + 300)
# (test_timeout + 300 is used here because the automation program is
# supposed to terminate cleanly on its own when test_timeout expires)
done = True
except aexpect.ExpectError, e:
o = e.output
done = False
f = open(os.path.join(test.debugdir, "summary"), "w")
print_summary_line(f, fmt % ("ID", "Job", "Status", "Pass", "Fail",
"NotRun", "NotApplicable"))
print_summary_line(f, fmt % ("--", "---", "------", "----", "----",
"------", "-------------"))
for r in results:
print_summary_line(f, fmt % (r["id"], r["job"], r["status"],
r["pass"], r["fail"], r["notrun"],
r["notapplicable"]))
f.close()
logging.info("(see logs and HTML reports in %s)", test.debugdir)
# Kill the client VMs and fail if the automation program did not terminate
# on time
if not done:
virt_utils.parallel(vm.destroy for vm in vms)
raise error.TestFail("The automation program did not terminate "
"on time")
# Fail if there are failed or incomplete jobs (kill the client VMs if there
# are incomplete jobs)
failed_jobs = [r["job"] for r in results
if r["status"].lower() == "investigate"]
running_jobs = [r["job"] for r in results
if r["status"].lower() == "inprogress"]
errors = []
if failed_jobs:
errors += ["Jobs failed: %s." % failed_jobs]
if running_jobs:
for vm in vms:
vm.destroy()
