def RunTests(self):
"""Runs tests on a single device.
Returns:
A TestResults object.
"""
try:
self.test_package.CreateTestRunnerScript(self._gtest_filter,
self._test_arguments)
self.test_results = self.test_package.RunTestsAndListResults()
except errors.DeviceUnresponsiveError as e:
# Make sure this device is not attached
if android_commands.IsDeviceAttached(self.device):
raise e
# TODO(frankf): We should report these as "skipped" not "failures".
# Wrap the results
logging.warning(e)
failed_tests = []
for t in self._gtest_filter.split(':'):
failed_tests += [BaseTestResult(t, '')]
self.test_results = TestResults.FromRun(
failed=failed_tests, device_exception=self.device)
return self.test_results
def RunTest(self, test):
test_results = base_test_result.TestRunResults()
if not test:
return test_results, None
try:
self.test_package.ClearApplicationState()
self.test_package.CreateTestRunnerScript(test,
self._test_arguments)
test_results = self.test_package.RunTestsAndListResults()
except errors.DeviceUnresponsiveError as e:
# Make sure this device is not attached
logging.warning(e)
if android_commands.IsDeviceAttached(self.device):
raise
finally:
self.CleanupSpawningServerState()
# Calculate unknown test results.
all_tests = set(test.split(':'))
all_tests_ran = set([t.GetName() for t in test_results.GetAll()])
unknown_tests = all_tests - all_tests_ran
test_results.AddResults([
base_test_result.BaseTestResult(
t, base_test_result.ResultType.UNKNOWN) for t in unknown_tests
])
retry = ':'.join([t.GetName() for t in test_results.GetNotPass()])
return test_results, retry
def RunTest(self, test):
"""Runs a test on a single device.
Args:
test: a gtest filter string to run.
Returns:
Tuple: (TestResults, test to retry or None)
"""
test_results = test_result.TestResults()
if not test:
return test_results, None
try:
self.test_package.ClearApplicationState()
self.test_package.CreateTestRunnerScript(test,
self._test_arguments)
test_results = self.test_package.RunTestsAndListResults()
except errors.DeviceUnresponsiveError as e:
# Make sure this device is not attached
logging.warning(e)
if android_commands.IsDeviceAttached(self.device):
raise
# Calculate unknown test results.
# TODO(frankf): Do not break TestResults encapsulation.
all_tests = set(test.split(':'))
all_tests_ran = set([t.name for t in test_results.GetAll()])
unknown_tests = all_tests - all_tests_ran
test_results.unknown = [
test_result.BaseTestResult(t, '') for t in unknown_tests
]
retry = ':'.join([t.name for t in test_results.GetAllBroken()])
return test_results, retry
def RunTests(self):
"""Runs tests on a single device.
Returns:
A TestResults object.
"""
if self._gtest_filter:
try:
self.test_package.CreateTestRunnerScript(
self._gtest_filter, self._test_arguments)
self.test_results = self.test_package.RunTestsAndListResults()
except errors.DeviceUnresponsiveError as e:
# Make sure this device is not attached
logging.warning(e)
if android_commands.IsDeviceAttached(self.device):
raise e
self.test_results.device_exception = device_exception
# Calculate unknown test results.
finally:
# TODO(frankf): Do not break TestResults encapsulation.
all_tests = set(self._gtest_filter.split(':'))
all_tests_ran = set(
[t.name for t in self.test_results.GetAll()])
unknown_tests = all_tests - all_tests_ran
self.test_results.unknown = [
BaseTestResult(t, '') for t in unknown_tests
]
return self.test_results
def _RunTestsFromQueue(runner, test_collection, out_results, watcher,
num_retries):
"""Runs tests from the test_collection until empty using the given runner.
Adds TestRunResults objects to the out_results list and may add tests to the
out_retry list.
Args:
runner: A TestRunner object used to run the tests.
test_collection: A _TestCollection from which to get _Test objects to run.
out_results: A list to add TestRunResults to.
watcher: A watchdog_timer.WatchdogTimer object, used as a shared timeout.
num_retries: Number of retries for a test.
"""
for test in test_collection:
watcher.Reset()
try:
if not android_commands.IsDeviceAttached(runner.device):
# Device is unresponsive, stop handling tests on this device.
msg = 'Device %s is unresponsive.' % runner.device
logging.warning(msg)
raise android_commands.errors.DeviceUnresponsiveError(msg)
result, retry = runner.RunTest(test.test)
test.tries += 1
if retry and test.tries <= num_retries:
# Retry non-passing results, only record passing results.
pass_results = base_test_result.TestRunResults()
pass_results.AddResults(result.GetPass())
out_results.append(pass_results)
logging.warning('Will retry test, try #%s.' % test.tries)
test_collection.add(_Test(test=retry, tries=test.tries))
else:
# All tests passed or retry limit reached. Either way, record results.
out_results.append(result)
except android_commands.errors.DeviceUnresponsiveError:
# Device is unresponsive, stop handling tests on this device and ensure
# current test gets runs by another device. Don't reraise this exception
# on the main thread.
test_collection.add(test)
return
except:
# An unhandleable exception, ensure tests get run by another device and
# reraise this exception on the main thread.
test_collection.add(test)
raise
finally:
# Retries count as separate tasks so always mark the popped test as done.
test_collection.test_completed()
def _RunTestsFromQueue(runner,
test_collection,
out_results,
watcher,
num_retries,
tag_results_with_device=False):
"""Runs tests from the test_collection until empty using the given runner.
Adds TestRunResults objects to the out_results list and may add tests to the
out_retry list.
Args:
runner: A TestRunner object used to run the tests.
test_collection: A _TestCollection from which to get _Test objects to run.
out_results: A list to add TestRunResults to.
watcher: A watchdog_timer.WatchdogTimer object, used as a shared timeout.
num_retries: Number of retries for a test.
tag_results_with_device: If True, appends the name of the device on which
the test was run to the test name. Used when replicating to identify
which device ran each copy of the test, and to ensure each copy of the
test is recorded separately.
"""
def TagTestRunResults(test_run_results):
"""Tags all results with the last 4 digits of the device id.
Used when replicating tests to distinguish the same tests run on different
devices. We use a set to store test results, so the hash (generated from
name and tag) must be unique to be considered different results.
"""
new_test_run_results = base_test_result.TestRunResults()
for test_result in test_run_results.GetAll():
test_result.SetName('%s_%s' %
(runner.device[-4:], test_result.GetName()))
new_test_run_results.AddResult(test_result)
return new_test_run_results
for test in test_collection:
watcher.Reset()
try:
if not android_commands.IsDeviceAttached(runner.device):
# Device is unresponsive, stop handling tests on this device.
msg = 'Device %s is unresponsive.' % runner.device
logging.warning(msg)
raise android_commands.errors.DeviceUnresponsiveError(msg)
result, retry = runner.RunTest(test.test)
if tag_results_with_device:
result = TagTestRunResults(result)
test.tries += 1
if retry and test.tries <= num_retries:
# Retry non-passing results, only record passing results.
pass_results = base_test_result.TestRunResults()
pass_results.AddResults(result.GetPass())
out_results.append(pass_results)
logging.warning('Will retry test, try #%s.' % test.tries)
test_collection.add(_Test(test=retry, tries=test.tries))
else:
# All tests passed or retry limit reached. Either way, record results.
out_results.append(result)
except:
# An unhandleable exception, ensure tests get run by another device and
# reraise this exception on the main thread.
test_collection.add(test)
raise
finally:
# Retries count as separate tasks so always mark the popped test as done.
test_collection.test_completed()