def run_whql_submission(test, params, env):
"""
WHQL submission test:
1) Log into the client machines and into a DTM server machine
2) Copy the automation program binary (dsso_test_binary) to the server machine
3) Run the automation program
4) Pass the program all relevant parameters (e.g. device_data)
5) Wait for the program to terminate
6) Parse and report job results
(logs and HTML reports are placed in test.debugdir)
:param test: kvm test object
:param params: Dictionary with the test parameters
:param env: Dictionary with test environment.
"""
# Log into all client VMs
login_timeout = int(params.get("login_timeout", 360))
vms = []
sessions = []
for vm_name in params.objects("vms"):
vms.append(env.get_vm(vm_name))
vms[-1].verify_alive()
sessions.append(vms[-1].wait_for_login(timeout=login_timeout))
# Make sure all NICs of all client VMs are up
for vm in vms:
nics = vm.params.objects("nics")
for nic_index in range(len(nics)):
s = vm.wait_for_login(nic_index, 600)
s.close()
# Collect parameters
server_address = params.get("server_address")
server_shell_port = int(params.get("server_shell_port"))
server_file_transfer_port = int(params.get("server_file_transfer_port"))
server_studio_path = params.get("server_studio_path", "%programfiles%\\ "
"Microsoft Driver Test Manager\\Studio")
dsso_test_binary = params.get("dsso_test_binary",
"deps/whql_submission_15.exe")
dsso_test_binary = utils_misc.get_path(test.virtdir, dsso_test_binary)
dsso_delete_machine_binary = params.get("dsso_delete_machine_binary",
"deps/whql_delete_machine_15.exe")
dsso_delete_machine_binary = utils_misc.get_path(test.virtdir,
dsso_delete_machine_binary)
test_timeout = float(params.get("test_timeout", 600))
# Copy dsso binaries to the server
for filename in dsso_test_binary, dsso_delete_machine_binary:
rss_client.upload(server_address, server_file_transfer_port,
filename, server_studio_path, timeout=60)
# Open a shell session with the server
server_session = remote.remote_login("nc", server_address,
server_shell_port, "", "",
sessions[0].prompt,
sessions[0].linesep)
server_session.set_status_test_command(sessions[0].status_test_command)
# Get the computer names of the server and clients
cmd = "echo %computername%"
server_name = server_session.cmd_output(cmd).strip()
client_names = [session.cmd_output(cmd).strip() for session in sessions]
# Delete all client machines from the server's data store
server_session.cmd("cd %s" % server_studio_path)
for client_name in client_names:
cmd = "%s %s %s" % (os.path.basename(dsso_delete_machine_binary),
server_name, client_name)
server_session.cmd(cmd, print_func=logging.debug)
# Reboot the client machines
sessions = utils_misc.parallel((vm.reboot, (session,))
for vm, session in zip(vms, sessions))
# Check the NICs again
for vm in vms:
nics = vm.params.objects("nics")
for nic_index in range(len(nics)):
s = vm.wait_for_login(nic_index, 600)
s.close()
# Run whql_pre_command and close the sessions
if params.get("whql_pre_command"):
for session in sessions:
session.cmd(params.get("whql_pre_command"),
int(params.get("whql_pre_command_timeout", 600)))
session.close()
# Run the automation program on the server
pool_name = "%s_pool" % client_names[0]
submission_name = "%s_%s" % (client_names[0],
params.get("submission_name"))
cmd = "%s %s %s %s %s %s" % (os.path.basename(dsso_test_binary),
server_name, pool_name, submission_name,
test_timeout, " ".join(client_names))
server_session.sendline(cmd)
# Helper function: wait for a given prompt and raise an exception if an
# error occurs
def find_prompt(prompt):
m, o = server_session.read_until_last_line_matches(
[prompt, server_session.prompt], print_func=logging.info,
timeout=600)
if m != 0:
errors = re.findall("^Error:.*$", o, re.I | re.M)
if errors:
raise error.TestError(errors[0])
else:
raise error.TestError("Error running automation program: "
"could not find '%s' prompt" % prompt)
# Tell the automation program which device to test
find_prompt("Device to test:")
server_session.sendline(params.get("test_device"))
# Tell the automation program which jobs to run
find_prompt("Jobs to run:")
server_session.sendline(params.get("job_filter", ".*"))
# Set submission DeviceData
find_prompt("DeviceData name:")
for dd in params.objects("device_data"):
dd_params = params.object_params(dd)
if dd_params.get("dd_name") and dd_params.get("dd_data"):
server_session.sendline(dd_params.get("dd_name"))
server_session.sendline(dd_params.get("dd_data"))
server_session.sendline()
# Set submission descriptors
find_prompt("Descriptor path:")
for desc in params.objects("descriptors"):
desc_params = params.object_params(desc)
if desc_params.get("desc_path"):
server_session.sendline(desc_params.get("desc_path"))
server_session.sendline()
# Set machine dimensions for each client machine
for vm_name in params.objects("vms"):
vm_params = params.object_params(vm_name)
find_prompt(r"Dimension name\b.*:")
for dp in vm_params.objects("dimensions"):
dp_params = vm_params.object_params(dp)
if dp_params.get("dim_name") and dp_params.get("dim_value"):
server_session.sendline(dp_params.get("dim_name"))
server_session.sendline(dp_params.get("dim_value"))
server_session.sendline()
# Set extra parameters for tests that require them (e.g. NDISTest)
for vm_name in params.objects("vms"):
vm_params = params.object_params(vm_name)
find_prompt(r"Parameter name\b.*:")
for dp in vm_params.objects("device_params"):
dp_params = vm_params.object_params(dp)
if dp_params.get("dp_name") and dp_params.get("dp_regex"):
server_session.sendline(dp_params.get("dp_name"))
server_session.sendline(dp_params.get("dp_regex"))
# Make sure the prompt appears again (if the device isn't found
# the automation program will terminate)
find_prompt(r"Parameter name\b.*:")
server_session.sendline()
# Wait for the automation program to terminate
try:
o = server_session.read_up_to_prompt(print_func=logging.info,
timeout=test_timeout + 300)
# (test_timeout + 300 is used here because the automation program is
# supposed to terminate cleanly on its own when test_timeout expires)
done = True
except aexpect.ExpectError, e:
o = e.output
done = False
def run(test, params, env):
"""
WHQL submission test:
1) Log into the client machines and into a DTM server machine
2) Copy the automation program binary (dsso_test_binary) to the server machine
3) Run the automation program
4) Pass the program all relevant parameters (e.g. device_data)
5) Wait for the program to terminate
6) Parse and report job results
(logs and HTML reports are placed in test.debugdir)
:param test: kvm test object
:param params: Dictionary with the test parameters
:param env: Dictionary with test environment.
"""
# Log into all client VMs
login_timeout = int(params.get("login_timeout", 360))
vms = []
sessions = []
for vm_name in params.objects("vms"):
vms.append(env.get_vm(vm_name))
vms[-1].verify_alive()
sessions.append(vms[-1].wait_for_login(timeout=login_timeout))
# Make sure all NICs of all client VMs are up
for vm in vms:
nics = vm.params.objects("nics")
for nic_index in range(len(nics)):
s = vm.wait_for_login(nic_index, 600)
s.close()
# Collect parameters
server_address = params.get("server_address")
server_shell_port = int(params.get("server_shell_port"))
server_file_transfer_port = int(params.get("server_file_transfer_port"))
server_studio_path = params.get(
"server_studio_path", "%programfiles%\\ "
"Microsoft Driver Test Manager\\Studio")
dsso_test_binary = params.get("dsso_test_binary",
"deps/whql_submission_15.exe")
dsso_test_binary = utils_misc.get_path(test.virtdir, dsso_test_binary)
dsso_delete_machine_binary = params.get("dsso_delete_machine_binary",
"deps/whql_delete_machine_15.exe")
dsso_delete_machine_binary = utils_misc.get_path(
test.virtdir, dsso_delete_machine_binary)
test_timeout = float(params.get("test_timeout", 600))
# Copy dsso binaries to the server
for filename in dsso_test_binary, dsso_delete_machine_binary:
rss_client.upload(server_address,
server_file_transfer_port,
filename,
server_studio_path,
timeout=60)
# Open a shell session with the server
server_session = remote.remote_login("nc", server_address,
server_shell_port, "", "",
sessions[0].prompt,
sessions[0].linesep)
server_session.set_status_test_command(sessions[0].status_test_command)
# Get the computer names of the server and clients
cmd = "echo %computername%"
server_name = server_session.cmd_output(cmd).strip()
client_names = [session.cmd_output(cmd).strip() for session in sessions]
# Delete all client machines from the server's data store
server_session.cmd("cd %s" % server_studio_path)
for client_name in client_names:
cmd = "%s %s %s" % (os.path.basename(dsso_delete_machine_binary),
server_name, client_name)
server_session.cmd(cmd, print_func=logging.debug)
# Reboot the client machines
sessions = utils_misc.parallel(
(vm.reboot, (session, )) for vm, session in zip(vms, sessions))
# Check the NICs again
for vm in vms:
nics = vm.params.objects("nics")
for nic_index in range(len(nics)):
s = vm.wait_for_login(nic_index, 600)
s.close()
# Run whql_pre_command and close the sessions
if params.get("whql_pre_command"):
for session in sessions:
session.cmd(params.get("whql_pre_command"),
int(params.get("whql_pre_command_timeout", 600)))
session.close()
# Run the automation program on the server
pool_name = "%s_pool" % client_names[0]
submission_name = "%s_%s" % (client_names[0],
params.get("submission_name"))
cmd = "%s %s %s %s %s %s" % (os.path.basename(dsso_test_binary),
server_name, pool_name, submission_name,
test_timeout, " ".join(client_names))
server_session.sendline(cmd)
# Helper function: wait for a given prompt and raise an exception if an
# error occurs
def find_prompt(prompt):
m, o = server_session.read_until_last_line_matches(
[prompt, server_session.prompt],
print_func=logging.info,
timeout=600)
if m != 0:
errors = re.findall("^Error:.*$", o, re.I | re.M)
if errors:
raise error.TestError(errors[0])
else:
raise error.TestError("Error running automation program: "
"could not find '%s' prompt" % prompt)
# Tell the automation program which device to test
find_prompt("Device to test:")
server_session.sendline(params.get("test_device"))
# Tell the automation program which jobs to run
find_prompt("Jobs to run:")
server_session.sendline(params.get("job_filter", ".*"))
# Set submission DeviceData
find_prompt("DeviceData name:")
for dd in params.objects("device_data"):
dd_params = params.object_params(dd)
if dd_params.get("dd_name") and dd_params.get("dd_data"):
server_session.sendline(dd_params.get("dd_name"))
server_session.sendline(dd_params.get("dd_data"))
server_session.sendline()
# Set submission descriptors
find_prompt("Descriptor path:")
for desc in params.objects("descriptors"):
desc_params = params.object_params(desc)
if desc_params.get("desc_path"):
server_session.sendline(desc_params.get("desc_path"))
server_session.sendline()
# Set machine dimensions for each client machine
for vm_name in params.objects("vms"):
vm_params = params.object_params(vm_name)
find_prompt(r"Dimension name\b.*:")
for dp in vm_params.objects("dimensions"):
dp_params = vm_params.object_params(dp)
if dp_params.get("dim_name") and dp_params.get("dim_value"):
server_session.sendline(dp_params.get("dim_name"))
server_session.sendline(dp_params.get("dim_value"))
server_session.sendline()
# Set extra parameters for tests that require them (e.g. NDISTest)
for vm_name in params.objects("vms"):
vm_params = params.object_params(vm_name)
find_prompt(r"Parameter name\b.*:")
for dp in vm_params.objects("device_params"):
dp_params = vm_params.object_params(dp)
if dp_params.get("dp_name") and dp_params.get("dp_regex"):
server_session.sendline(dp_params.get("dp_name"))
server_session.sendline(dp_params.get("dp_regex"))
# Make sure the prompt appears again (if the device isn't found
# the automation program will terminate)
find_prompt(r"Parameter name\b.*:")
server_session.sendline()
# Wait for the automation program to terminate
try:
o = server_session.read_up_to_prompt(print_func=logging.info,
timeout=test_timeout + 300)
# (test_timeout + 300 is used here because the automation program is
# supposed to terminate cleanly on its own when test_timeout expires)
done = True
except aexpect.ExpectError, e:
o = e.output
done = False
def run_whql_client_install(test, params, env):
"""
WHQL DTM client installation:
1) Log into the guest (the client machine) and into a DTM server machine
2) Stop the DTM client service (wttsvc) on the client machine
3) Delete the client machine from the server's data store
4) Rename the client machine (give it a randomly generated name)
5) Move the client machine into the server's workgroup
6) Reboot the client machine
7) Install the DTM client software
8) Setup auto logon for the user created by the installation
(normally DTMLLUAdminUser)
9) Reboot again
:param test: kvm test object
:param params: Dictionary with the test parameters
:param env: Dictionary with test environment.
"""
vm = env.get_vm(params["main_vm"])
vm.verify_alive()
session = vm.wait_for_login(timeout=int(params.get("login_timeout", 360)))
# Collect test params
server_address = params.get("server_address")
server_shell_port = int(params.get("server_shell_port"))
server_file_transfer_port = int(params.get("server_file_transfer_port"))
server_studio_path = params.get("server_studio_path", "%programfiles%\\ "
"Microsoft Driver Test Manager\\Studio")
server_username = params.get("server_username")
server_password = params.get("server_password")
client_username = params.get("client_username")
client_password = params.get("client_password")
dsso_delete_machine_binary = params.get("dsso_delete_machine_binary",
"deps/whql_delete_machine_15.exe")
dsso_delete_machine_binary = utils_misc.get_path(test.bindir,
dsso_delete_machine_binary)
install_timeout = float(params.get("install_timeout", 600))
install_cmd = params.get("install_cmd")
wtt_services = params.get("wtt_services")
# Stop WTT service(s) on client
for svc in wtt_services.split():
utils_test.stop_windows_service(session, svc)
# Copy dsso_delete_machine_binary to server
rss_client.upload(server_address, server_file_transfer_port,
dsso_delete_machine_binary, server_studio_path,
timeout=60)
# Open a shell session with server
server_session = remote.remote_login("nc", server_address,
server_shell_port, "", "",
session.prompt, session.linesep)
server_session.set_status_test_command(session.status_test_command)
# Get server and client information
cmd = "echo %computername%"
server_name = server_session.cmd_output(cmd).strip()
client_name = session.cmd_output(cmd).strip()
cmd = "wmic computersystem get domain"
server_workgroup = server_session.cmd_output(cmd).strip()
server_workgroup = server_workgroup.splitlines()[-1]
regkey = r"HKLM\SYSTEM\CurrentControlSet\Services\Tcpip\Parameters"
cmd = "reg query %s /v Domain" % regkey
o = server_session.cmd_output(cmd).strip().splitlines()[-1]
try:
server_dns_suffix = o.split(None, 2)[2]
except IndexError:
server_dns_suffix = ""
# Delete the client machine from the server's data store (if it's there)
server_session.cmd("cd %s" % server_studio_path)
cmd = "%s %s %s" % (os.path.basename(dsso_delete_machine_binary),
server_name, client_name)
server_session.cmd(cmd, print_func=logging.info)
server_session.close()
# Rename the client machine
client_name = "autotest_%s" % utils_misc.generate_random_string(4)
logging.info("Renaming client machine to '%s'", client_name)
cmd = ('wmic computersystem where name="%%computername%%" rename name="%s"'
% client_name)
session.cmd(cmd, timeout=600)
# Join the server's workgroup
logging.info("Joining workgroup '%s'", server_workgroup)
cmd = ('wmic computersystem where name="%%computername%%" call '
'joindomainorworkgroup name="%s"' % server_workgroup)
session.cmd(cmd, timeout=600)
# Set the client machine's DNS suffix
logging.info("Setting DNS suffix to '%s'", server_dns_suffix)
cmd = 'reg add %s /v Domain /d "%s" /f' % (regkey, server_dns_suffix)
session.cmd(cmd, timeout=300)
# Reboot
session = vm.reboot(session)
# Access shared resources on the server machine
logging.info("Attempting to access remote share on server")
cmd = r"net use \\%s /user:%s %s" % (server_name, server_username,
server_password)
end_time = time.time() + 120
while time.time() < end_time:
try:
session.cmd(cmd)
break
except Exception:
pass
time.sleep(5)
else:
raise error.TestError("Could not access server share from client "
"machine")
# Install
logging.info("Installing DTM client (timeout=%ds)", install_timeout)
install_cmd = r"cmd /c \\%s\%s" % (server_name, install_cmd.lstrip("\\"))
session.cmd(install_cmd, timeout=install_timeout)
# Setup auto logon
logging.info("Setting up auto logon for user '%s'", client_username)
cmd = ('reg add '
'"HKLM\\Software\\Microsoft\\Windows NT\\CurrentVersion\\winlogon" '
'/v "%s" /d "%s" /t REG_SZ /f')
session.cmd(cmd % ("AutoAdminLogon", "1"))
session.cmd(cmd % ("DefaultUserName", client_username))
session.cmd(cmd % ("DefaultPassword", client_password))
# Reboot one more time
session = vm.reboot(session)
session.close()
def run_whql_client_install(test, params, env):
"""
WHQL DTM client installation:
1) Log into the guest (the client machine) and into a DTM server machine
2) Stop the DTM client service (wttsvc) on the client machine
3) Delete the client machine from the server's data store
4) Rename the client machine (give it a randomly generated name)
5) Move the client machine into the server's workgroup
6) Reboot the client machine
7) Install the DTM client software
8) Setup auto logon for the user created by the installation
(normally DTMLLUAdminUser)
9) Reboot again
@param test: kvm test object
@param params: Dictionary with the test parameters
@param env: Dictionary with test environment.
"""
vm = env.get_vm(params["main_vm"])
vm.verify_alive()
session = vm.wait_for_login(timeout=int(params.get("login_timeout", 360)))
# Collect test params
server_address = params.get("server_address")
server_shell_port = int(params.get("server_shell_port"))
server_file_transfer_port = int(params.get("server_file_transfer_port"))
server_studio_path = params.get("server_studio_path", "%programfiles%\\ "
"Microsoft Driver Test Manager\\Studio")
server_username = params.get("server_username")
server_password = params.get("server_password")
client_username = params.get("client_username")
client_password = params.get("client_password")
dsso_delete_machine_binary = params.get("dsso_delete_machine_binary",
"deps/whql_delete_machine_15.exe")
dsso_delete_machine_binary = utils_misc.get_path(test.bindir,
dsso_delete_machine_binary)
install_timeout = float(params.get("install_timeout", 600))
install_cmd = params.get("install_cmd")
wtt_services = params.get("wtt_services")
# Stop WTT service(s) on client
for svc in wtt_services.split():
utils_test.stop_windows_service(session, svc)
# Copy dsso_delete_machine_binary to server
rss_client.upload(server_address, server_file_transfer_port,
dsso_delete_machine_binary, server_studio_path,
timeout=60)
# Open a shell session with server
server_session = remote.remote_login("nc", server_address,
server_shell_port, "", "",
session.prompt, session.linesep)
server_session.set_status_test_command(session.status_test_command)
# Get server and client information
cmd = "echo %computername%"
server_name = server_session.cmd_output(cmd).strip()
client_name = session.cmd_output(cmd).strip()
cmd = "wmic computersystem get domain"
server_workgroup = server_session.cmd_output(cmd).strip()
server_workgroup = server_workgroup.splitlines()[-1]
regkey = r"HKLM\SYSTEM\CurrentControlSet\Services\Tcpip\Parameters"
cmd = "reg query %s /v Domain" % regkey
o = server_session.cmd_output(cmd).strip().splitlines()[-1]
try:
server_dns_suffix = o.split(None, 2)[2]
except IndexError:
server_dns_suffix = ""
# Delete the client machine from the server's data store (if it's there)
server_session.cmd("cd %s" % server_studio_path)
cmd = "%s %s %s" % (os.path.basename(dsso_delete_machine_binary),
server_name, client_name)
server_session.cmd(cmd, print_func=logging.info)
server_session.close()
# Rename the client machine
client_name = "autotest_%s" % utils_misc.generate_random_string(4)
logging.info("Renaming client machine to '%s'", client_name)
cmd = ('wmic computersystem where name="%%computername%%" rename name="%s"'
% client_name)
session.cmd(cmd, timeout=600)
# Join the server's workgroup
logging.info("Joining workgroup '%s'", server_workgroup)
cmd = ('wmic computersystem where name="%%computername%%" call '
'joindomainorworkgroup name="%s"' % server_workgroup)
session.cmd(cmd, timeout=600)
# Set the client machine's DNS suffix
logging.info("Setting DNS suffix to '%s'", server_dns_suffix)
cmd = 'reg add %s /v Domain /d "%s" /f' % (regkey, server_dns_suffix)
session.cmd(cmd, timeout=300)
# Reboot
session = vm.reboot(session)
# Access shared resources on the server machine
logging.info("Attempting to access remote share on server")
cmd = r"net use \\%s /user:%s %s" % (server_name, server_username,
server_password)
end_time = time.time() + 120
while time.time() < end_time:
try:
session.cmd(cmd)
break
except Exception:
pass
time.sleep(5)
else:
raise error.TestError("Could not access server share from client "
"machine")
# Install
logging.info("Installing DTM client (timeout=%ds)", install_timeout)
install_cmd = r"cmd /c \\%s\%s" % (server_name, install_cmd.lstrip("\\"))
session.cmd(install_cmd, timeout=install_timeout)
# Setup auto logon
logging.info("Setting up auto logon for user '%s'", client_username)
cmd = ('reg add '
'"HKLM\\Software\\Microsoft\\Windows NT\\CurrentVersion\\winlogon" '
'/v "%s" /d "%s" /t REG_SZ /f')
session.cmd(cmd % ("AutoAdminLogon", "1"))
session.cmd(cmd % ("DefaultUserName", client_username))
session.cmd(cmd % ("DefaultPassword", client_password))
# Reboot one more time
session = vm.reboot(session)
session.close()
def run(test, params, env):
"""
WHQL submission test:
1) Log into the client machines and into a DTM server machine
2) Copy the automation program binary (dsso_test_binary) to the server machine
3) Run the automation program
4) Pass the program all relevant parameters (e.g. device_data)
5) Wait for the program to terminate
6) Parse and report job results
(logs and HTML reports are placed in test.debugdir)
:param test: kvm test object
:param params: Dictionary with the test parameters
:param env: Dictionary with test environment.
"""
# Log into all client VMs
login_timeout = int(params.get("login_timeout", 360))
vms = []
sessions = []
for vm_name in params.objects("vms"):
vms.append(env.get_vm(vm_name))
vms[-1].verify_alive()
sessions.append(vms[-1].wait_for_login(timeout=login_timeout))
# Make sure all NICs of all client VMs are up
for vm in vms:
nics = vm.params.objects("nics")
for nic_index in range(len(nics)):
s = vm.wait_for_login(nic_index, 600)
s.close()
# Collect parameters
server_address = params.get("server_address")
server_shell_port = int(params.get("server_shell_port"))
server_file_transfer_port = int(params.get("server_file_transfer_port"))
server_studio_path = params.get("server_studio_path", "%programfiles%\\ "
"Microsoft Driver Test Manager\\Studio")
dsso_test_binary = params.get("dsso_test_binary",
"deps/whql_submission_15.exe")
dsso_test_binary = utils_misc.get_path(test.virtdir, dsso_test_binary)
dsso_delete_machine_binary = params.get("dsso_delete_machine_binary",
"deps/whql_delete_machine_15.exe")
dsso_delete_machine_binary = utils_misc.get_path(test.virtdir,
dsso_delete_machine_binary)
test_timeout = float(params.get("test_timeout", 600))
# Copy dsso binaries to the server
for filename in dsso_test_binary, dsso_delete_machine_binary:
rss_client.upload(server_address, server_file_transfer_port,
filename, server_studio_path, timeout=60)
# Open a shell session with the server
server_session = remote.remote_login("nc", server_address,
server_shell_port, "", "",
sessions[0].prompt,
sessions[0].linesep)
server_session.set_status_test_command(sessions[0].status_test_command)
# Get the computer names of the server and clients
cmd = "echo %computername%"
server_name = server_session.cmd_output(cmd).strip()
client_names = [session.cmd_output(cmd).strip() for session in sessions]
# Delete all client machines from the server's data store
server_session.cmd("cd %s" % server_studio_path)
for client_name in client_names:
cmd = "%s %s %s" % (os.path.basename(dsso_delete_machine_binary),
server_name, client_name)
server_session.cmd(cmd, print_func=logging.debug)
# Reboot the client machines
sessions = utils_misc.parallel((vm.reboot, (session,))
for vm, session in zip(vms, sessions))
# Check the NICs again
for vm in vms:
nics = vm.params.objects("nics")
for nic_index in range(len(nics)):
s = vm.wait_for_login(nic_index, 600)
s.close()
# Run whql_pre_command and close the sessions
if params.get("whql_pre_command"):
for session in sessions:
session.cmd(params.get("whql_pre_command"),
int(params.get("whql_pre_command_timeout", 600)))
session.close()
# Run the automation program on the server
pool_name = "%s_pool" % client_names[0]
submission_name = "%s_%s" % (client_names[0],
params.get("submission_name"))
cmd = "%s %s %s %s %s %s" % (os.path.basename(dsso_test_binary),
server_name, pool_name, submission_name,
test_timeout, " ".join(client_names))
server_session.sendline(cmd)
# Helper function: wait for a given prompt and raise an exception if an
# error occurs
def find_prompt(test, prompt):
m, o = server_session.read_until_last_line_matches(
[prompt, server_session.prompt], print_func=logging.info,
timeout=600)
if m != 0:
errors = re.findall("^Error:.*$", o, re.I | re.M)
if errors:
test.error(errors[0])
else:
test.error("Error running automation program: "
"could not find '%s' prompt" % prompt)
# Tell the automation program which device to test
find_prompt(test, "Device to test:")
server_session.sendline(params.get("test_device"))
# Tell the automation program which jobs to run
find_prompt(test, "Jobs to run:")
server_session.sendline(params.get("job_filter", ".*"))
# Set submission DeviceData
find_prompt(test, "DeviceData name:")
for dd in params.objects("device_data"):
dd_params = params.object_params(dd)
if dd_params.get("dd_name") and dd_params.get("dd_data"):
server_session.sendline(dd_params.get("dd_name"))
server_session.sendline(dd_params.get("dd_data"))
server_session.sendline()
# Set submission descriptors
find_prompt(test, "Descriptor path:")
for desc in params.objects("descriptors"):
desc_params = params.object_params(desc)
if desc_params.get("desc_path"):
server_session.sendline(desc_params.get("desc_path"))
server_session.sendline()
# Set machine dimensions for each client machine
for vm_name in params.objects("vms"):
vm_params = params.object_params(vm_name)
find_prompt(test, r"Dimension name\b.*:")
for dp in vm_params.objects("dimensions"):
dp_params = vm_params.object_params(dp)
if dp_params.get("dim_name") and dp_params.get("dim_value"):
server_session.sendline(dp_params.get("dim_name"))
server_session.sendline(dp_params.get("dim_value"))
server_session.sendline()
# Set extra parameters for tests that require them (e.g. NDISTest)
for vm_name in params.objects("vms"):
vm_params = params.object_params(vm_name)
find_prompt(test, r"Parameter name\b.*:")
for dp in vm_params.objects("device_params"):
dp_params = vm_params.object_params(dp)
if dp_params.get("dp_name") and dp_params.get("dp_regex"):
server_session.sendline(dp_params.get("dp_name"))
server_session.sendline(dp_params.get("dp_regex"))
# Make sure the prompt appears again (if the device isn't found
# the automation program will terminate)
find_prompt(test, r"Parameter name\b.*:")
server_session.sendline()
# Wait for the automation program to terminate
try:
o = server_session.read_up_to_prompt(print_func=logging.info,
timeout=test_timeout + 300)
# (test_timeout + 300 is used here because the automation program is
# supposed to terminate cleanly on its own when test_timeout expires)
done = True
except aexpect.ExpectError as e:
o = e.output
done = False
server_session.close()
# Look for test results in the automation program's output
result_summaries = re.findall(r"---- \[.*?\] ----", o, re.DOTALL)
if not result_summaries:
test.error("The automation program did not return any results")
results = result_summaries[-1].strip("-")
results = eval("".join(results.splitlines()))
# Download logs and HTML reports from the server
for r in results:
if "report" in r:
try:
rss_client.download(server_address,
server_file_transfer_port,
r["report"], test.debugdir)
except rss_client.FileTransferNotFoundError:
pass
if "logs" in r:
try:
rss_client.download(server_address,
server_file_transfer_port,
r["logs"], test.debugdir)
except rss_client.FileTransferNotFoundError:
pass
else:
try:
# Create symlinks to test log dirs to make it easier
# to access them (their original names are not human
# readable)
link_name = "logs_%s" % r["report"].split("\\")[-1]
link_name = link_name.replace(" ", "_")
link_name = link_name.replace("/", "_")
os.symlink(r["logs"].split("\\")[-1],
os.path.join(test.debugdir, link_name))
except (KeyError, OSError):
pass
# Print result summary (both to the regular logs and to a file named
# 'summary' in test.debugdir)
def print_summary_line(f, line):
logging.info(line)
f.write(line + "\n")
if results:
# Make sure all results have the required keys
for r in results:
r["id"] = str(r.get("id"))
r["job"] = str(r.get("job"))
r["status"] = str(r.get("status"))
r["pass"] = int(r.get("pass", 0))
r["fail"] = int(r.get("fail", 0))
r["notrun"] = int(r.get("notrun", 0))
r["notapplicable"] = int(r.get("notapplicable", 0))
# Sort the results by failures and total test count in descending order
results = [(r["fail"],
r["pass"] + r["fail"] + r["notrun"] + r["notapplicable"],
r) for r in results]
results.sort(reverse=True)
results = [r[-1] for r in results]
# Print results
logging.info("")
logging.info("Result summary:")
name_length = max(len(r["job"]) for r in results)
fmt = "%%-6s %%-%ds %%-15s %%-8s %%-8s %%-8s %%-15s" % name_length
f = open(os.path.join(test.debugdir, "summary"), "w")
print_summary_line(f, fmt % ("ID", "Job", "Status", "Pass", "Fail",
"NotRun", "NotApplicable"))
print_summary_line(f, fmt % ("--", "---", "------", "----", "----",
"------", "-------------"))
for r in results:
print_summary_line(f, fmt % (r["id"], r["job"], r["status"],
r["pass"], r["fail"], r["notrun"],
r["notapplicable"]))
f.close()
logging.info("(see logs and HTML reports in %s)", test.debugdir)
# Kill the client VMs and fail if the automation program did not terminate
# on time
if not done:
utils_misc.parallel(vm.destroy for vm in vms)
test.fail("The automation program did not terminate on time")
# Fail if there are failed or incomplete jobs (kill the client VMs if there
# are incomplete jobs)
failed_jobs = [r["job"] for r in results
if r["status"].lower() == "investigate"]
running_jobs = [r["job"] for r in results
if r["status"].lower() == "inprogress"]
errors = []
if failed_jobs:
errors += ["Jobs failed: %s." % failed_jobs]
if running_jobs:
for vm in vms:
vm.destroy()
errors += ["Jobs did not complete on time: %s." % running_jobs]
if errors:
test.fail(" ".join(errors))
