def RebaselineTests(self):
"""Runs all available tests, restarting in case of failures."""
if self._gtest_filter:
all_tests = set(self._gtest_filter.split(':'))
else:
all_tests = set(self.test_package.GetAllTests())
failed_results = set()
executed_results = set()
while True:
executed_names = set([f.name for f in executed_results])
self._gtest_filter = ':'.join(all_tests - executed_names)
self.RunTestsWithFilter()
failed_results.update(self.test_results.crashed,
self.test_results.failed)
executed_results.update(self.test_results.crashed,
self.test_results.failed,
self.test_results.ok)
executed_names = set([f.name for f in executed_results])
logging.info('*' * 80)
logging.info(self.device)
logging.info('Executed: ' + str(len(executed_names)) + ' of ' +
str(len(all_tests)))
logging.info('Failed so far: ' + str(len(failed_results)) + ' ' +
str([f.name for f in failed_results]))
logging.info('Remaining: ' + str(len(all_tests - executed_names)) +
' ' + str(all_tests - executed_names))
logging.info('*' * 80)
if executed_names == all_tests:
break
self.test_results = TestResults.FromOkAndFailed(
list(executed_results - failed_results), list(failed_results))
def RunTests(device, test_suite, gtest_filter, test_arguments, rebaseline,
timeout, performance_test, cleanup_test_files, tool,
log_dump_name):
"""Runs the tests.
Args:
device: Device to run the tests.
test_suite: A specific test suite to run, empty to run all.
gtest_filter: A gtest_filter flag.
test_arguments: Additional arguments to pass to the test binary.
rebaseline: Whether or not to run tests in isolation and update the filter.
timeout: Timeout for each test.
performance_test: Whether or not performance test(s).
cleanup_test_files: Whether or not to cleanup test files on device.
tool: Name of the Valgrind tool.
log_dump_name: Name of log dump file.
Returns:
A TestResults object.
"""
results = []
if test_suite:
global _TEST_SUITES
if not os.path.exists(test_suite):
logging.critical('Unrecognized test suite, supported: %s' %
_TEST_SUITES)
if test_suite in _TEST_SUITES:
logging.critical('(Remember to include the path: out/Release/%s)',
test_suite)
return TestResults.FromOkAndFailed([], [BaseTestResult(test_suite, '')])
_TEST_SUITES = [test_suite]
else:
# If not specified, assume the test suites are in out/Release
test_suite_dir = os.path.abspath(os.path.join(run_tests_helper.CHROME_DIR,
'out', 'Release'))
_TEST_SUITES = [os.path.join(test_suite_dir, t) for t in _TEST_SUITES]
debug_info_list = []
for t in _TEST_SUITES:
test = SingleTestRunner(device, t, gtest_filter, test_arguments,
timeout, rebaseline, performance_test,
cleanup_test_files, tool, not not log_dump_name)
test.RunTests()
results += [test.test_results]
# Collect debug info.
debug_info_list += [test.dump_debug_info]
if rebaseline:
test.UpdateFilter(test.test_results.failed)
elif test.test_results.failed:
# Stop running test if encountering failed test.
test.test_results.LogFull()
break
# Zip all debug info outputs into a file named by log_dump_name.
debug_info.GTestDebugInfo.ZipAndCleanResults(
os.path.join(run_tests_helper.CHROME_DIR, 'out', 'Release',
'debug_info_dumps'),
log_dump_name, [d for d in debug_info_list if d])
return TestResults.FromTestResults(results)
def _WatchTestOutput(self, p):
"""Watches the test output.
Args:
p: the process generating output as created by pexpect.spawn.
"""
ok_tests = []
failed_tests = []
timed_out = False
overall_fail = False
re_run = re.compile('\[ RUN \] ?(.*)\r\n')
# APK tests rely on the END tag.
re_end = re.compile('\[ END \] ?(.*)\r\n')
re_fail = re.compile('\[ FAILED \] ?(.*)\r\n')
re_runner_fail = re.compile('\[ RUNNER_FAILED \] ?(.*)\r\n')
re_ok = re.compile('\[ OK \] ?(.*)\r\n')
(io_stats_before, ready_to_continue) = self._BeginGetIOStats()
while ready_to_continue:
found = p.expect([re_run, pexpect.EOF, re_end, re_runner_fail],
timeout=self.timeout)
if found == 1: # matched pexpect.EOF
break
if found == 2: # matched END.
break
if found == 3: # RUNNER_FAILED
logging.error('RUNNER_FAILED')
overall_fail = True
break
if self.dump_debug_info:
self.dump_debug_info.TakeScreenshot('_Test_Start_Run_')
full_test_name = p.match.group(1)
found = p.expect([re_ok, re_fail, pexpect.EOF, pexpect.TIMEOUT],
timeout=self.timeout)
if found == 0: # re_ok
ok_tests += [BaseTestResult(full_test_name.replace('\r', ''),
p.before)]
continue
failed_tests += [BaseTestResult(full_test_name.replace('\r', ''),
p.before)]
if found >= 2:
# The test crashed / bailed out (i.e., didn't print OK or FAIL).
if found == 3: # pexpect.TIMEOUT
logging.error('Test terminated after %d second timeout.',
self.timeout)
timed_out = True
break
p.close()
if not self.rebaseline and ready_to_continue:
ok_tests += self._EndGetIOStats(io_stats_before)
ret_code = self._GetGTestReturnCode()
if ret_code:
failed_tests += [BaseTestResult('gtest exit code: %d' % ret_code,
'pexpect.before: %s'
'\npexpect.after: %s'
% (p.before,
p.after))]
return TestResults.FromOkAndFailed(ok_tests, failed_tests,
timed_out, overall_fail)
def RunTests(device, test_suite, gtest_filter, test_arguments, rebaseline,
timeout, performance_test, cleanup_test_files, tool,
log_dump_name, apk, annotate=False):
"""Runs the tests.
Args:
device: Device to run the tests.
test_suite: A specific test suite to run, empty to run all.
gtest_filter: A gtest_filter flag.
test_arguments: Additional arguments to pass to the test binary.
rebaseline: Whether or not to run tests in isolation and update the filter.
timeout: Timeout for each test.
performance_test: Whether or not performance test(s).
cleanup_test_files: Whether or not to cleanup test files on device.
tool: Name of the Valgrind tool.
log_dump_name: Name of log dump file.
apk: boolean to state if we are using the apk based test runner
annotate: should we print buildbot-style annotations?
Returns:
A TestResults object.
"""
results = []
if test_suite:
global _TEST_SUITES
if (not os.path.exists(test_suite) and
not os.path.splitext(test_suite)[1] == '.apk'):
logging.critical('Unrecognized test suite %s, supported: %s' %
(test_suite, _TEST_SUITES))
if test_suite in _TEST_SUITES:
logging.critical('(Remember to include the path: out/Release/%s)',
test_suite)
return TestResults.FromOkAndFailed([], [BaseTestResult(test_suite, '')],
False, False)
fully_qualified_test_suites = [test_suite]
else:
fully_qualified_test_suites = FullyQualifiedTestSuites(apk)
debug_info_list = []
print 'Known suites: ' + str(_TEST_SUITES)
print 'Running these: ' + str(fully_qualified_test_suites)
for t in fully_qualified_test_suites:
if annotate:
print '@@@BUILD_STEP Test suite %s@@@' % os.path.basename(t)
test = SingleTestRunner(device, t, gtest_filter, test_arguments,
timeout, rebaseline, performance_test,
cleanup_test_files, tool, 0, not not log_dump_name)
test.Run()
results += [test.test_results]
# Collect debug info.
debug_info_list += [test.dump_debug_info]
if rebaseline:
test.UpdateFilter(test.test_results.failed)
test.test_results.LogFull()
# Zip all debug info outputs into a file named by log_dump_name.
debug_info.GTestDebugInfo.ZipAndCleanResults(
os.path.join(run_tests_helper.CHROME_DIR, 'out', 'Release',
'debug_info_dumps'),
log_dump_name, [d for d in debug_info_list if d])
if annotate:
if test.test_results.timed_out:
print '@@@STEP_WARNINGS@@@'
elif test.test_results.failed:
print '@@@STEP_FAILURE@@@'
elif test.test_results.overall_fail:
print '@@@STEP_FAILURE@@@'
else:
print 'Step success!' # No annotation needed
return TestResults.FromTestResults(results)
