def test_interrupt_if_at_failure_limits(self):
port = Mock() # FIXME: This should be a tighter mock.
port.TEST_PATH_SEPARATOR = '/'
port._filesystem = MockFileSystem()
manager = Manager(port=port, options=MockOptions(), printer=Mock())
manager._options = MockOptions(exit_after_n_failures=None, exit_after_n_crashes_or_timeouts=None)
result_summary = ResultSummary(expectations=Mock(), test_files=[])
result_summary.unexpected_failures = 100
result_summary.unexpected_crashes = 50
result_summary.unexpected_timeouts = 50
# No exception when the exit_after* options are None.
manager._interrupt_if_at_failure_limits(result_summary)
# No exception when we haven't hit the limit yet.
manager._options.exit_after_n_failures = 101
manager._options.exit_after_n_crashes_or_timeouts = 101
manager._interrupt_if_at_failure_limits(result_summary)
# Interrupt if we've exceeded either limit:
manager._options.exit_after_n_crashes_or_timeouts = 10
self.assertRaises(TestRunInterruptedException, manager._interrupt_if_at_failure_limits, result_summary)
manager._options.exit_after_n_crashes_or_timeouts = None
manager._options.exit_after_n_failures = 10
exception = self.assertRaises(TestRunInterruptedException, manager._interrupt_if_at_failure_limits, result_summary)
def test_interrupt_if_at_failure_limits(self):
port = Mock() # FIXME: This should be a tighter mock.
port.TEST_PATH_SEPARATOR = '/'
port._filesystem = MockFileSystem()
manager = Manager(port=port, options=MockOptions(), printer=Mock())
manager._options = MockOptions(exit_after_n_failures=None,
exit_after_n_crashes_or_timeouts=None)
result_summary = ResultSummary(expectations=Mock(), test_files=[])
result_summary.unexpected_failures = 100
result_summary.unexpected_crashes = 50
result_summary.unexpected_timeouts = 50
# No exception when the exit_after* options are None.
manager._interrupt_if_at_failure_limits(result_summary)
# No exception when we haven't hit the limit yet.
manager._options.exit_after_n_failures = 101
manager._options.exit_after_n_crashes_or_timeouts = 101
manager._interrupt_if_at_failure_limits(result_summary)
# Interrupt if we've exceeded either limit:
manager._options.exit_after_n_crashes_or_timeouts = 10
self.assertRaises(TestRunInterruptedException,
manager._interrupt_if_at_failure_limits,
result_summary)
manager._options.exit_after_n_crashes_or_timeouts = None
manager._options.exit_after_n_failures = 10
exception = self.assertRaises(TestRunInterruptedException,
manager._interrupt_if_at_failure_limits,
result_summary)
def test_interrupt_if_at_failure_limits(self):
runner = self._runner()
runner._options.exit_after_n_failures = None
runner._options.exit_after_n_crashes_or_times = None
test_names = ['passes/text.html', 'passes/image.html']
runner._test_inputs = [TestInput(test_name, 6000) for test_name in test_names]
result_summary = ResultSummary(TestExpectations(runner._port, test_names), len(test_names))
result_summary.unexpected_failures = 100
result_summary.unexpected_crashes = 50
result_summary.unexpected_timeouts = 50
# No exception when the exit_after* options are None.
runner._interrupt_if_at_failure_limits(result_summary)
# No exception when we haven't hit the limit yet.
runner._options.exit_after_n_failures = 101
runner._options.exit_after_n_crashes_or_timeouts = 101
runner._interrupt_if_at_failure_limits(result_summary)
# Interrupt if we've exceeded either limit:
runner._options.exit_after_n_crashes_or_timeouts = 10
self.assertRaises(TestRunInterruptedException, runner._interrupt_if_at_failure_limits, result_summary)
self.assertEqual(result_summary.results['passes/text.html'].type, test_expectations.SKIP)
self.assertEqual(result_summary.results['passes/image.html'].type, test_expectations.SKIP)
runner._options.exit_after_n_crashes_or_timeouts = None
runner._options.exit_after_n_failures = 10
exception = self.assertRaises(TestRunInterruptedException, runner._interrupt_if_at_failure_limits, result_summary)
def run_tests(self, expectations, test_inputs, tests_to_skip, num_workers, needs_http, needs_websockets, retrying):
self._expectations = expectations
self._test_inputs = test_inputs
self._needs_http = needs_http
self._needs_websockets = needs_websockets
self._retrying = retrying
result_summary = ResultSummary(self._expectations, len(test_inputs) + len(tests_to_skip))
self._current_result_summary = result_summary
self._remaining_locked_shards = []
self._has_http_lock = False
self._printer.num_tests = len(test_inputs)
self._printer.num_completed = 0
if not retrying:
self._printer.print_expected(result_summary, self._expectations.get_tests_with_result_type)
for test_name in set(tests_to_skip):
result = test_results.TestResult(test_name)
result.type = test_expectations.SKIP
result_summary.add(result, expected=True, test_is_slow=self._test_is_slow(test_name))
self._printer.write_update('Sharding tests ...')
locked_shards, unlocked_shards = self._sharder.shard_tests(test_inputs, int(self._options.child_processes), self._options.fully_parallel)
# FIXME: We don't have a good way to coordinate the workers so that
# they don't try to run the shards that need a lock if we don't actually
# have the lock. The easiest solution at the moment is to grab the
# lock at the beginning of the run, and then run all of the locked
# shards first. This minimizes the time spent holding the lock, but
# means that we won't be running tests while we're waiting for the lock.
# If this becomes a problem in practice we'll need to change this.
all_shards = locked_shards + unlocked_shards
self._remaining_locked_shards = locked_shards
if self._port.requires_http_server() or (locked_shards and self._options.http):
self.start_servers_with_lock(2 * min(num_workers, len(locked_shards)))
num_workers = min(num_workers, len(all_shards))
self._printer.print_workers_and_shards(num_workers, len(all_shards), len(locked_shards))
if self._options.dry_run:
return result_summary
self._printer.write_update('Starting %s ...' % grammar.pluralize('worker', num_workers))
try:
with message_pool.get(self, self._worker_factory, num_workers, self._port.worker_startup_delay_secs(), self._port.host) as pool:
pool.run(('test_list', shard.name, shard.test_inputs) for shard in all_shards)
except TestRunInterruptedException, e:
_log.warning(e.reason)
result_summary.interrupted = True
def test_update_summary_with_result(self):
host = MockHost()
port = host.port_factory.get('test-win-xp')
test = 'failures/expected/reftest.html'
expectations = TestExpectations(
port,
tests=[test],
expectations='WONTFIX : failures/expected/reftest.html = IMAGE',
test_config=port.test_configuration())
# Reftests expected to be image mismatch should be respected when pixel_tests=False.
manager = Manager(port=port,
options=MockOptions(
pixel_tests=False,
exit_after_n_failures=None,
exit_after_n_crashes_or_timeouts=None),
printer=Mock())
manager._expectations = expectations
result_summary = ResultSummary(expectations=expectations,
test_files=[test])
result = TestResult(
test_name=test,
failures=[test_failures.FailureReftestMismatchDidNotOccur()])
manager._update_summary_with_result(result_summary, result)
self.assertEquals(1, result_summary.expected)
self.assertEquals(0, result_summary.unexpected)
def _prepare_lists(self):
tests_to_skip = self._finder.skip_tests(self._paths, self._test_names,
self._expectations,
self._http_tests())
self._test_names = [
test for test in self._test_names if test not in tests_to_skip
]
# Create a sorted list of test files so the subset chunk,
# if used, contains alphabetically consecutive tests.
if self._options.order == 'natural':
self._test_names.sort(key=self._port.test_key)
elif self._options.order == 'random':
random.shuffle(self._test_names)
self._test_names, tests_in_other_chunks = self._finder.split_into_chunks(
self._test_names)
self._expectations.add_skipped_tests(tests_in_other_chunks)
tests_to_skip.update(tests_in_other_chunks)
if self._options.repeat_each > 1:
list_with_repetitions = []
for test in self._test_names:
list_with_repetitions += ([test] * self._options.repeat_each)
self._test_names = list_with_repetitions
if self._options.iterations > 1:
self._test_names = self._test_names * self._options.iterations
iterations = self._options.repeat_each * self._options.iterations
return ResultSummary(self._expectations, set(self._test_names),
iterations, tests_to_skip)
def _prepare_lists(self, paths, test_names):
tests_to_skip = self._finder.skip_tests(paths, test_names, self._expectations, self._http_tests(test_names))
tests_to_run = [test for test in test_names if test not in tests_to_skip]
# Create a sorted list of test files so the subset chunk,
# if used, contains alphabetically consecutive tests.
if self._options.order == 'natural':
tests_to_run.sort(key=self._port.test_key)
elif self._options.order == 'random':
random.shuffle(tests_to_run)
tests_to_run, tests_in_other_chunks = self._finder.split_into_chunks(tests_to_run)
self._expectations.add_skipped_tests(tests_in_other_chunks)
tests_to_skip.update(tests_in_other_chunks)
summary = ResultSummary(self._expectations, len(tests_to_run) * self._options.repeat_each * self._options.iterations + len(tests_to_skip))
for test_name in set(tests_to_skip):
result = test_results.TestResult(test_name)
result.type = test_expectations.SKIP
summary.add(result, expected=True, test_is_slow=self._test_is_slow(test_name))
return summary, tests_to_run
def test_update_summary_with_result(self):
# Reftests expected to be image mismatch should be respected when pixel_tests=False.
runner = self._runner()
runner._options.pixel_tests = False
test = 'failures/expected/reftest.html'
expectations = TestExpectations(runner._port, tests=[test])
runner._expectations = expectations
result_summary = ResultSummary(expectations, [test], 1, set())
result = TestResult(
test_name=test,
failures=[test_failures.FailureReftestMismatchDidNotOccur()],
reftest_type=['!='])
runner._update_summary_with_result(result_summary, result)
self.assertEqual(1, result_summary.expected)
self.assertEqual(0, result_summary.unexpected)
result_summary = ResultSummary(expectations, [test], 1, set())
result = TestResult(test_name=test, failures=[], reftest_type=['=='])
runner._update_summary_with_result(result_summary, result)
self.assertEqual(0, result_summary.expected)
self.assertEqual(1, result_summary.unexpected)
def test_look_for_new_crash_logs(self):
def get_manager_with_tests(test_names):
host = MockHost()
port = host.port_factory.get('test-mac-leopard')
manager = Manager(port, options=MockOptions(test_list=None, http=True, max_locked_shards=1), printer=Mock())
manager._collect_tests(test_names)
return manager
host = MockHost()
port = host.port_factory.get('test-mac-leopard')
tests = ['failures/expected/crash.html']
expectations = test_expectations.TestExpectations(port, tests)
rs = ResultSummary(expectations, tests, 1, set())
manager = get_manager_with_tests(tests)
manager._look_for_new_crash_logs(rs, time.time())
def _result_summary(self, runner, tests):
return ResultSummary(TestExpectations(runner._port, tests), tests, 1,
set())
def get_result_summary(self, port, test_names, expectations_str):
port.expectations_dict = lambda: {'': expectations_str}
expectations = test_expectations.TestExpectations(port, test_names)
return test_names, ResultSummary(expectations, test_names, 1, set()), expectations
def run(self, args):
"""Run all our tests on all our test files and return the number of unexpected results (0 == success)."""
self._printer.write_update("Collecting tests ...")
try:
self._paths, self._test_names = self._collect_tests(args)
except IOError as exception:
# This is raised if --test-list doesn't exist
return -1
self._printer.write_update("Parsing expectations ...")
self._expectations = test_expectations.TestExpectations(
self._port, self._test_names)
num_all_test_files_found = len(self._test_names)
result_summary = self._prepare_lists()
# Check to make sure we're not skipping every test.
if not self._test_names:
_log.critical('No tests to run.')
return -1
self._printer.print_found(num_all_test_files_found,
len(self._test_names),
self._options.repeat_each,
self._options.iterations)
self._printer.print_expected(
result_summary, self._expectations.get_tests_with_result_type)
if not self._set_up_run():
return -1
start_time = time.time()
interrupted, keyboard_interrupted, thread_timings, test_timings, individual_test_timings = \
self._run_tests(self._test_names, result_summary, int(self._options.child_processes))
# We exclude the crashes from the list of results to retry, because
# we want to treat even a potentially flaky crash as an error.
failures = self._get_failures(
result_summary,
include_crashes=self._port.should_retry_crashes(),
include_missing=False)
retry_summary = result_summary
while (len(failures) and self._options.retry_failures
and not self._retrying and not interrupted
and not keyboard_interrupted):
_log.info('')
_log.info("Retrying %d unexpected failure(s) ..." % len(failures))
_log.info('')
self._retrying = True
retry_summary = ResultSummary(self._expectations, failures.keys(),
1, set())
# Note that we intentionally ignore the return value here.
self._run_tests(failures.keys(), retry_summary, 1)
failures = self._get_failures(retry_summary,
include_crashes=True,
include_missing=True)
end_time = time.time()
# Some crash logs can take a long time to be written out so look
# for new logs after the test run finishes.
self._look_for_new_crash_logs(result_summary, start_time)
self._look_for_new_crash_logs(retry_summary, start_time)
self._clean_up_run()
unexpected_results = summarize_results(self._port,
self._expectations,
result_summary,
retry_summary,
individual_test_timings,
only_unexpected=True,
interrupted=interrupted)
self._printer.print_results(end_time - start_time, thread_timings,
test_timings, individual_test_timings,
result_summary, unexpected_results)
# Re-raise a KeyboardInterrupt if necessary so the caller can handle it.
if keyboard_interrupted:
raise KeyboardInterrupt
# FIXME: remove record_results. It's just used for testing. There's no need
# for it to be a commandline argument.
if (self._options.record_results and not self._options.dry_run
and not keyboard_interrupted):
self._port.print_leaks_summary()
# Write the same data to log files and upload generated JSON files to appengine server.
summarized_results = summarize_results(self._port,
self._expectations,
result_summary,
retry_summary,
individual_test_timings,
only_unexpected=False,
interrupted=interrupted)
self._upload_json_files(summarized_results, result_summary,
individual_test_timings)
# Write the summary to disk (results.html) and display it if requested.
if not self._options.dry_run:
self._copy_results_html_file()
if self._options.show_results:
self._show_results_html_file(result_summary)
return self._port.exit_code_from_summarized_results(unexpected_results)
