def RebaselineTests(self):
"""Runs all available tests, restarting in case of failures."""
if self._gtest_filter:
all_tests = set(self._gtest_filter.split(':'))
else:
all_tests = set(self.test_package.GetAllTests())
failed_results = set()
executed_results = set()
while True:
executed_names = set([f.name for f in executed_results])
self._gtest_filter = ':'.join(all_tests - executed_names)
self.RunTestsWithFilter()
failed_results.update(self.test_results.crashed,
self.test_results.failed)
executed_results.update(self.test_results.crashed,
self.test_results.failed,
self.test_results.ok)
executed_names = set([f.name for f in executed_results])
logging.info('*' * 80)
logging.info(self.device)
logging.info('Executed: ' + str(len(executed_names)) + ' of ' +
str(len(all_tests)))
logging.info('Failed so far: ' + str(len(failed_results)) + ' ' +
str([f.name for f in failed_results]))
logging.info('Remaining: ' + str(len(all_tests - executed_names)) + ' ' +
str(all_tests - executed_names))
logging.info('*' * 80)
if executed_names == all_tests:
break
self.test_results = TestResults.FromRun(
ok=list(executed_results - failed_results),
failed=list(failed_results))
def RunTests(self):
"""Runs all tests (in rebaseline mode, runs each test in isolation).
Returns:
A TestResults object.
"""
try:
if self.test_package.rebaseline:
self.RebaselineTests()
else:
if not self._gtest_filter:
self._gtest_filter = (
'-' + ':'.join(self.GetDisabledTests()) + ':' +
':'.join([
'*.' + x + '*'
for x in self.test_package.GetDisabledPrefixes()
]))
self.RunTestsWithFilter()
except errors.DeviceUnresponsiveError as e:
# Make sure this device is not attached
if android_commands.IsDeviceAttached(self.device):
raise e
# Wrap the results
logging.warning(e)
failed_tests = []
for t in self._gtest_filter.split(':'):
failed_tests += [BaseTestResult(t, '')]
self.test_results = TestResults.FromRun(
failed=failed_tests, device_exception=self.device)
return self.test_results
def RunTests(self):
"""Runs tests on a single device.
Returns:
A TestResults object.
"""
try:
self.test_package.CreateTestRunnerScript(self._gtest_filter,
self._test_arguments)
self.test_results = self.test_package.RunTestsAndListResults()
except errors.DeviceUnresponsiveError as e:
# Make sure this device is not attached
if android_commands.IsDeviceAttached(self.device):
raise e
# TODO(frankf): We should report these as "skipped" not "failures".
# Wrap the results
logging.warning(e)
failed_tests = []
for t in self._gtest_filter.split(':'):
failed_tests += [BaseTestResult(t, '')]
self.test_results = TestResults.FromRun(
failed=failed_tests, device_exception=self.device)
return self.test_results
def RunTests(device,
test_suite,
gtest_filter,
test_arguments,
rebaseline,
timeout,
performance_test,
cleanup_test_files,
tool,
log_dump_name,
apk,
annotate=False):
"""Runs the tests.
Args:
device: Device to run the tests.
test_suite: A specific test suite to run, empty to run all.
gtest_filter: A gtest_filter flag.
test_arguments: Additional arguments to pass to the test binary.
rebaseline: Whether or not to run tests in isolation and update the filter.
timeout: Timeout for each test.
performance_test: Whether or not performance test(s).
cleanup_test_files: Whether or not to cleanup test files on device.
tool: Name of the Valgrind tool.
log_dump_name: Name of log dump file.
apk: boolean to state if we are using the apk based test runner
annotate: should we print buildbot-style annotations?
Returns:
A TestResults object.
"""
results = []
if test_suite:
global _TEST_SUITES
if (not os.path.exists(test_suite)
and not os.path.splitext(test_suite)[1] == '.apk'):
logging.critical('Unrecognized test suite %s, supported: %s' %
(test_suite, _TEST_SUITES))
if test_suite in _TEST_SUITES:
logging.critical(
'(Remember to include the path: out/Release/%s)',
test_suite)
return TestResults.FromRun(failed=[BaseTestResult(test_suite, '')])
fully_qualified_test_suites = [test_suite]
else:
fully_qualified_test_suites = FullyQualifiedTestSuites(apk)
debug_info_list = []
print 'Known suites: ' + str(_TEST_SUITES)
print 'Running these: ' + str(fully_qualified_test_suites)
for t in fully_qualified_test_suites:
if annotate:
print '@@@BUILD_STEP Test suite %s@@@' % os.path.basename(t)
test = SingleTestRunner(device, t, gtest_filter, test_arguments,
timeout, rebaseline, performance_test,
cleanup_test_files, tool, 0,
not not log_dump_name)
test.Run()
results += [test.test_results]
# Collect debug info.
debug_info_list += [test.dump_debug_info]
if rebaseline:
test.UpdateFilter(test.test_results.failed)
test.test_results.LogFull()
# Zip all debug info outputs into a file named by log_dump_name.
debug_info.GTestDebugInfo.ZipAndCleanResults(
os.path.join(run_tests_helper.CHROME_DIR, 'out', 'Release',
'debug_info_dumps'), log_dump_name,
[d for d in debug_info_list if d])
if annotate:
PrintAnnotationForTestResults(test.test_results)
return TestResults.FromTestResults(results)
def _WatchTestOutput(self, p):
"""Watches the test output.
Args:
p: the process generating output as created by pexpect.spawn.
"""
ok_tests = []
failed_tests = []
crashed_tests = []
timed_out = False
overall_fail = False
re_run = re.compile('\[ RUN \] ?(.*)\r\n')
# APK tests rely on the END tag.
re_end = re.compile('\[ END \] ?(.*)\r\n')
# Signal handlers are installed before starting tests
# to output the CRASHED marker when a crash happens.
re_crash = re.compile('\[ CRASHED \](.*)\r\n')
re_fail = re.compile('\[ FAILED \] ?(.*)\r\n')
re_runner_fail = re.compile('\[ RUNNER_FAILED \] ?(.*)\r\n')
re_ok = re.compile('\[ OK \] ?(.*)\r\n')
(io_stats_before, ready_to_continue) = self._BeginGetIOStats()
while ready_to_continue:
found = p.expect([re_run, pexpect.EOF, re_end, re_runner_fail],
timeout=self.timeout)
if found == 1: # matched pexpect.EOF
break
if found == 2: # matched END.
break
if found == 3: # RUNNER_FAILED
logging.error('RUNNER_FAILED')
overall_fail = True
break
if self.dump_debug_info:
self.dump_debug_info.TakeScreenshot('_Test_Start_Run_')
full_test_name = p.match.group(1)
found = p.expect(
[re_ok, re_fail, re_crash, pexpect.EOF, pexpect.TIMEOUT],
timeout=self.timeout)
if found == 0: # re_ok
ok_tests += [
BaseTestResult(full_test_name.replace('\r', ''), p.before)
]
continue
if found == 2: # re_crash
crashed_tests += [
BaseTestResult(full_test_name.replace('\r', ''), p.before)
]
overall_fail = True
break
# The test failed.
failed_tests += [
BaseTestResult(full_test_name.replace('\r', ''), p.before)
]
if found >= 3:
# The test bailed out (i.e., didn't print OK or FAIL).
if found == 4: # pexpect.TIMEOUT
logging.error('Test terminated after %d second timeout.',
self.timeout)
timed_out = True
break
p.close()
if not self.rebaseline and ready_to_continue:
ok_tests += self._EndGetIOStats(io_stats_before)
ret_code = self._GetGTestReturnCode()
if ret_code:
failed_tests += [
BaseTestResult(
'gtest exit code: %d' % ret_code, 'pexpect.before: %s'
'\npexpect.after: %s' % (p.before, p.after))
]
# Create TestResults and return
return TestResults.FromRun(ok=ok_tests,
failed=failed_tests,
crashed=crashed_tests,
timed_out=timed_out,
overall_fail=overall_fail)
def _WatchTestOutput(self, p):
"""Watches the test output.
Args:
p: the process generating output as created by pexpect.spawn.
"""
ok_tests = []
failed_tests = []
crashed_tests = []
timed_out = False
overall_fail = False
# Test case statuses.
re_run = re.compile('\[ RUN \] ?(.*)\r\n')
re_fail = re.compile('\[ FAILED \] ?(.*)\r\n')
re_ok = re.compile('\[ OK \] ?(.*?) .*\r\n')
# Test run statuses.
re_passed = re.compile('\[ PASSED \] ?(.*)\r\n')
re_runner_fail = re.compile('\[ RUNNER_FAILED \] ?(.*)\r\n')
# Signal handlers are installed before starting tests
# to output the CRASHED marker when a crash happens.
re_crash = re.compile('\[ CRASHED \](.*)\r\n')
try:
while True:
found = p.expect([re_run, re_passed, re_runner_fail],
timeout=self.timeout)
if found == 1: # re_passed
break
elif found == 2: # re_runner_fail
overall_fail = True
break
else: # re_run
if self.dump_debug_info:
self.dump_debug_info.TakeScreenshot('_Test_Start_Run_')
full_test_name = p.match.group(1).replace('\r', '')
found = p.expect([re_ok, re_fail, re_crash], timeout=self.timeout)
if found == 0: # re_ok
if full_test_name == p.match.group(1).replace('\r', ''):
ok_tests += [BaseTestResult(full_test_name, p.before)]
elif found == 2: # re_crash
crashed_tests += [BaseTestResult(full_test_name, p.before)]
overall_fail = True
break
else: # re_fail
failed_tests += [BaseTestResult(full_test_name, p.before)]
except pexpect.EOF:
logging.error('Test terminated - EOF')
raise errors.DeviceUnresponsiveError('Device may be offline')
except pexpect.TIMEOUT:
logging.error('Test terminated after %d second timeout.',
self.timeout)
timed_out = True
finally:
p.close()
ret_code = self._GetGTestReturnCode()
if ret_code:
logging.critical(
'gtest exit code: %d\npexpect.before: %s\npexpect.after: %s',
ret_code, p.before, p.after)
overall_fail = True
# Create TestResults and return
return TestResults.FromRun(ok=ok_tests, failed=failed_tests,
crashed=crashed_tests, timed_out=timed_out,
overall_fail=overall_fail)
