def parse_webdriver_expectations(host, port):
expectations_path = port.path_to_webdriver_expectations_file()
file_contents = host.filesystem.read_text_file(expectations_path)
expectations_dict = {expectations_path: file_contents}
expectations = test_expectations.TestExpectations(
port, expectations_dict=expectations_dict)
return expectations
def test_summarized_results_with_iterations(self):
test_name = 'passes/text.html'
expectations = test_expectations.TestExpectations(self.port)
initial_results = test_run_results.TestRunResults(
expectations, 3, None)
initial_results.add(get_result(test_name, ResultType.Crash), False,
False)
initial_results.add(get_result(test_name, ResultType.Failure), False,
False)
initial_results.add(get_result(test_name, ResultType.Timeout), False,
False)
all_retry_results = [
test_run_results.TestRunResults(expectations, 2, None)
]
all_retry_results[0].add(get_result(test_name, ResultType.Failure),
False, False)
all_retry_results[0].add(get_result(test_name, ResultType.Failure),
False, False)
summary = test_run_results.summarize_results(self.port, expectations,
initial_results,
all_retry_results)
self.assertEquals(summary['tests']['passes']['text.html']['expected'],
'PASS')
self.assertEquals(summary['tests']['passes']['text.html']['actual'],
'CRASH FAIL TIMEOUT FAIL FAIL')
self.assertEquals(summary['num_flaky'], 0)
self.assertEquals(summary['num_passes'], 0)
self.assertEquals(summary['num_regressions'], 1)
def test_timeout_then_unexpected_pass(self):
test_name = 'failures/expected/text.html'
expectations = test_expectations.TestExpectations(self.port)
initial_results = test_run_results.TestRunResults(
expectations, 1, None)
initial_results.add(
get_result(test_name, ResultType.Timeout, run_time=1), False,
False)
all_retry_results = [
test_run_results.TestRunResults(expectations, 1, None),
test_run_results.TestRunResults(expectations, 1, None),
test_run_results.TestRunResults(expectations, 1, None)
]
all_retry_results[0].add(
get_result(test_name, ResultType.Failure, run_time=0.1), False,
False)
all_retry_results[1].add(
get_result(test_name, ResultType.Pass, run_time=0.1), False, False)
all_retry_results[2].add(
get_result(test_name, ResultType.Pass, run_time=0.1), False, False)
summary = test_run_results.summarize_results(self.port, expectations,
initial_results,
all_retry_results)
self.assertIn('is_unexpected',
summary['tests']['failures']['expected']['text.html'])
self.assertEquals(
summary['tests']['failures']['expected']['text.html']['expected'],
'FAIL')
self.assertEquals(
summary['tests']['failures']['expected']['text.html']['actual'],
'TIMEOUT FAIL PASS PASS')
self.assertEquals(summary['num_passes'], 1)
self.assertEquals(summary['num_regressions'], 0)
self.assertEquals(summary['num_flaky'], 0)
def test_summarized_results_flaky_pass_after_first_retry(self):
test_name = 'passes/text.html'
expectations = test_expectations.TestExpectations(self.port)
initial_results = test_run_results.TestRunResults(
expectations, 1, None)
initial_results.add(get_result(test_name, ResultType.Crash), False,
False)
all_retry_results = [
test_run_results.TestRunResults(expectations, 1, None),
test_run_results.TestRunResults(expectations, 1, None),
test_run_results.TestRunResults(expectations, 1, None)
]
all_retry_results[0].add(get_result(test_name, ResultType.Timeout),
False, False)
all_retry_results[1].add(get_result(test_name, ResultType.Pass), True,
False)
all_retry_results[2].add(get_result(test_name, ResultType.Pass), True,
False)
summary = test_run_results.summarize_results(self.port, expectations,
initial_results,
all_retry_results)
self.assertTrue(
'is_unexpected' not in summary['tests']['passes']['text.html'])
self.assertEquals(summary['tests']['passes']['text.html']['expected'],
'PASS')
self.assertEquals(summary['tests']['passes']['text.html']['actual'],
'CRASH TIMEOUT PASS PASS')
self.assertEquals(summary['num_flaky'], 1)
self.assertEquals(summary['num_passes'], 0)
self.assertEquals(summary['num_regressions'], 0)
def test_summarized_results_flaky_pass_after_first_retry(self):
test_name = 'passes/text.html'
expectations = test_expectations.TestExpectations(
self.port, [test_name])
initial_results = test_run_results.TestRunResults(expectations, 1)
initial_results.add(get_result(test_name, test_expectations.CRASH),
False, False)
all_retry_results = [
test_run_results.TestRunResults(expectations, 1),
test_run_results.TestRunResults(expectations, 1),
test_run_results.TestRunResults(expectations, 1)
]
all_retry_results[0].add(
get_result(test_name, test_expectations.TIMEOUT), False, False)
all_retry_results[1].add(get_result(test_name, test_expectations.PASS),
True, False)
all_retry_results[2].add(get_result(test_name, test_expectations.PASS),
True, False)
summary = test_run_results.summarize_results(
self.port,
expectations,
initial_results,
all_retry_results,
enabled_pixel_tests_in_retry=True)
self.assertTrue(
'is_unexpected' not in summary['tests']['passes']['text.html'])
self.assertEquals(summary['tests']['passes']['text.html']['expected'],
'PASS')
self.assertEquals(summary['tests']['passes']['text.html']['actual'],
'CRASH TIMEOUT PASS PASS')
self.assertEquals(summary['num_flaky'], 1)
self.assertEquals(summary['num_passes'], 0)
self.assertEquals(summary['num_regressions'], 0)
def setUp(self):
expectations = test_expectations.TestExpectations(
MockHost().port_factory.get(port_name='test'))
self.results = test_run_results.TestRunResults(expectations, 1, None)
self.test = get_result('failures/expected/text.html',
ResultType.Timeout,
run_time=1)
def test_summarized_results_with_iterations(self):
test_name = 'passes/text.html'
expectations = test_expectations.TestExpectations(
self.port, [test_name])
initial_results = test_run_results.TestRunResults(expectations, 3)
initial_results.add(get_result(test_name, test_expectations.CRASH),
False, False)
initial_results.add(get_result(test_name, test_expectations.IMAGE),
False, False)
initial_results.add(get_result(test_name, test_expectations.TIMEOUT),
False, False)
all_retry_results = [test_run_results.TestRunResults(expectations, 2)]
all_retry_results[0].add(get_result(test_name, test_expectations.TEXT),
False, False)
all_retry_results[0].add(get_result(test_name, test_expectations.LEAK),
False, False)
summary = test_run_results.summarize_results(
self.port,
expectations,
initial_results,
all_retry_results,
enabled_pixel_tests_in_retry=True)
print summary
self.assertEquals(summary['tests']['passes']['text.html']['expected'],
'PASS')
self.assertEquals(summary['tests']['passes']['text.html']['actual'],
'CRASH IMAGE TIMEOUT TEXT LEAK')
self.assertEquals(summary['num_flaky'], 0)
self.assertEquals(summary['num_passes'], 0)
self.assertEquals(summary['num_regressions'], 1)
def run_results(port, extra_skipped_tests=None):
tests = [
'passes/text.html', 'failures/expected/timeout.html',
'failures/expected/crash.html', 'failures/expected/leak.html',
'failures/expected/keyboard.html', 'failures/expected/audio.html',
'failures/expected/text.html', 'passes/skipped/skip.html'
]
expectations = test_expectations.TestExpectations(port, tests)
if extra_skipped_tests:
expectations.add_extra_skipped_tests(extra_skipped_tests)
return test_run_results.TestRunResults(expectations, len(tests))
def lint(host, options):
ports_to_lint = [
host.port_factory.get(name)
for name in host.port_factory.all_port_names(options.platform)
]
files_linted = set()
# In general, the set of TestExpectation files should be the same for
# all ports. However, the method used to list expectations files is
# in Port, and the TestExpectations constructor takes a Port.
# Perhaps this function could be changed to just use one Port
# (the default Port for this host) and it would work the same.
failures = []
wpt_overrides_exps_path = host.filesystem.join(
ports_to_lint[0].web_tests_dir(), 'WPTOverrideExpectations')
web_gpu_exps_path = host.filesystem.join(ports_to_lint[0].web_tests_dir(),
'WebGPUExpectations')
paths = [wpt_overrides_exps_path, web_gpu_exps_path]
expectations_dict = {}
for path in paths:
if host.filesystem.exists(path):
expectations_dict[path] = host.filesystem.read_text_file(path)
for port in ports_to_lint:
expectations_dict.update(port.all_expectations_dict())
for path in port.extra_expectations_files():
if host.filesystem.exists(path):
expectations_dict[path] = host.filesystem.read_text_file(path)
for path, content in expectations_dict.items():
try:
test_expectations.TestExpectations(
ports_to_lint[0],
expectations_dict={path: content},
is_lint_mode=True)
except test_expectations.ParseError as error:
_log.error('')
for warning in error.warnings:
_log.error(warning)
failures.append('%s: %s' % (path, warning))
_log.error('')
for lineno, line in enumerate(content.split('\n'), 1):
if line.strip().startswith('Bug('):
error = ((
"%s:%d Expectation '%s' has the Bug(...) token, "
"The token has been removed in the new expectations format"
) % (host.filesystem.basename(path), lineno, line))
_log.error(error)
failures.append(error)
_log.error('')
return failures
def run_results(port, extra_skipped_tests=None):
tests = [
'passes/text.html', 'failures/expected/timeout.html',
'failures/expected/crash.html', 'failures/expected/leak.html',
'failures/expected/keyboard.html', 'failures/expected/audio.html',
'failures/expected/text.html', 'passes/skipped/skip.html'
]
expectations = test_expectations.TestExpectations(port)
if extra_skipped_tests:
extra_expectations = '# results: [ Skip ]'
for test in extra_skipped_tests:
extra_expectations += '\n%s [ Skip ]' % test
expectations.merge_raw_expectations(extra_expectations)
return test_run_results.TestRunResults(expectations, len(tests), None)
def test_look_for_new_crash_logs(self):
def get_manager():
host = MockHost()
port = host.port_factory.get('test-mac-mac10.10')
manager = Manager(port,
options=optparse.Values({
'test_list': None,
'http': True,
'max_locked_shards': 1
}),
printer=FakePrinter())
return manager
host = MockHost()
port = host.port_factory.get('test-mac-mac10.10')
tests = ['failures/expected/crash.html']
expectations = test_expectations.TestExpectations(port)
run_results = TestRunResults(expectations, len(tests), None)
manager = get_manager()
manager._look_for_new_crash_logs(run_results, time.time())
def test_timeout_then_unexpected_pass(self):
test_name = 'failures/expected/text.html'
expectations = test_expectations.TestExpectations(self.port, [test_name])
initial_results = test_run_results.TestRunResults(expectations, 1)
initial_results.add(get_result(test_name, test_expectations.TIMEOUT, run_time=1), False, False)
all_retry_results = [test_run_results.TestRunResults(expectations, 1),
test_run_results.TestRunResults(expectations, 1),
test_run_results.TestRunResults(expectations, 1)]
all_retry_results[0].add(get_result(test_name, test_expectations.LEAK, run_time=0.1), False, False)
all_retry_results[1].add(get_result(test_name, test_expectations.PASS, run_time=0.1), False, False)
all_retry_results[2].add(get_result(test_name, test_expectations.PASS, run_time=0.1), False, False)
summary = test_run_results.summarize_results(
self.port, expectations, initial_results, all_retry_results,
enabled_pixel_tests_in_retry=True)
self.assertIn('is_unexpected', summary['tests']['failures']['expected']['text.html'])
self.assertEquals(summary['tests']['failures']['expected']['text.html']['expected'], 'FAIL')
self.assertEquals(summary['tests']['failures']['expected']['text.html']['actual'], 'TIMEOUT LEAK PASS PASS')
self.assertEquals(summary['num_passes'], 1)
self.assertEquals(summary['num_regressions'], 0)
self.assertEquals(summary['num_flaky'], 0)
def lint(host, options):
ports_to_lint = [
host.port_factory.get(name)
for name in host.port_factory.all_port_names(options.platform)
]
files_linted = set()
# In general, the set of TestExpectation files should be the same for
# all ports. However, the method used to list expectations files is
# in Port, and the TestExpectations constructor takes a Port.
# Perhaps this function could be changed to just use one Port
# (the default Port for this host) and it would work the same.
failures = []
for port_to_lint in ports_to_lint:
expectations_dict = port_to_lint.all_expectations_dict()
for path in port_to_lint.extra_expectations_files():
if host.filesystem.exists(path):
expectations_dict[path] = host.filesystem.read_text_file(path)
for expectations_file in expectations_dict:
if expectations_file in files_linted:
continue
try:
test_expectations.TestExpectations(
port_to_lint,
expectations_dict={
expectations_file: expectations_dict[expectations_file]
},
is_lint_mode=True)
except test_expectations.ParseError as error:
_log.error('')
for warning in error.warnings:
_log.error(warning)
failures.append('%s: %s' % (expectations_file, warning))
_log.error('')
files_linted.add(expectations_file)
return failures
def test_skip_tests_expectations(self):
"""Tests that tests are skipped based on to expectations and options."""
host = MockHost()
port = host.port_factory.get('test-win-win7', None)
all_tests = [
'fast/css/passes.html',
'fast/css/fails.html',
'fast/css/times_out.html',
'fast/css/skip.html',
]
# Patch port.tests() to return our tests
port.tests = lambda paths: paths or all_tests
options = optparse.Values({
'no_expectations': False,
'enable_sanitizer': False,
'skipped': 'default',
'skip_timeouts': False,
'skip_failing_tests': False,
})
finder = web_test_finder.WebTestFinder(port, options)
expectations = test_expectations.TestExpectations(port)
expectations.merge_raw_expectations(
('# results: [ Failure Timeout Skip ]'
'\nfast/css/fails.html [ Failure ]'
'\nfast/css/times_out.html [ Timeout ]'
'\nfast/css/skip.html [ Skip ]'))
# When run with default settings, we only skip the tests marked Skip.
tests = finder.skip_tests([], all_tests, expectations)
self.assertEqual(tests, set(['fast/css/skip.html']))
# Specify test on the command line; by default should not skip.
tests = finder.skip_tests(['fast/css/skip.html'], all_tests,
expectations)
self.assertEqual(tests, set())
# Specify test on the command line, but always skip.
finder._options.skipped = 'always'
tests = finder.skip_tests(['fast/css/skip.html'], all_tests,
expectations)
self.assertEqual(tests, set(['fast/css/skip.html']))
finder._options.skipped = 'default'
# Only run skip tests, aka skip all non-skipped tests.
finder._options.skipped = 'only'
tests = finder.skip_tests([], all_tests, expectations)
self.assertEqual(
tests,
set([
'fast/css/passes.html', 'fast/css/fails.html',
'fast/css/times_out.html'
]))
finder._options.skipped = 'default'
# Ignore any skip entries, aka never skip anything.
finder._options.skipped = 'ignore'
tests = finder.skip_tests([], all_tests, expectations)
self.assertEqual(tests, set())
finder._options.skipped = 'default'
# Skip tests that are marked TIMEOUT.
finder._options.skip_timeouts = True
tests = finder.skip_tests([], all_tests, expectations)
self.assertEqual(
tests, set(['fast/css/times_out.html', 'fast/css/skip.html']))
finder._options.skip_timeouts = False
# Skip tests that are marked FAILURE
finder._options.skip_failing_tests = True
tests = finder.skip_tests([], all_tests, expectations)
self.assertEqual(tests,
set(['fast/css/fails.html', 'fast/css/skip.html']))
finder._options.skip_failing_tests = False
# Disable expectations entirely; nothing should be skipped by default.
finder._options.no_expectations = True
tests = finder.skip_tests([], all_tests, expectations)
self.assertEqual(tests, set())
def test_skip_tests_idlharness(self):
"""Tests that idlharness tests are skipped on MSAN/ASAN runs.
See https://crbug.com/856601
"""
host = MockHost()
port = host.port_factory.get('test-win-win7', None)
non_idlharness_test = 'external/wpt/dir1/dir2/foo.html'
idlharness_test_1 = 'external/wpt/dir1/dir2/idlharness.any.html'
idlharness_test_2 = 'external/wpt/dir1/dir2/idlharness.any.worker.html'
all_tests = [
non_idlharness_test,
idlharness_test_1,
idlharness_test_2,
]
# Patch port.tests() to return our tests
port.tests = lambda paths: paths or all_tests
options = optparse.Values({
'no_expectations': False,
'enable_sanitizer': False,
'skipped': 'default',
'skip_timeouts': False,
'skip_failing_tests': False,
})
finder = web_test_finder.WebTestFinder(port, options)
# Default case; not MSAN/ASAN so should not skip anything.
expectations = test_expectations.TestExpectations(port)
tests = finder.skip_tests([], all_tests, expectations)
self.assertEqual(tests, set())
for test in all_tests:
self.assertTrue(
expectations.get_expectations(test).is_default_pass)
# MSAN/ASAN, with no paths specified explicitly, so should skip both
# idlharness tests.
expectations = test_expectations.TestExpectations(port)
finder._options.enable_sanitizer = True
tests = finder.skip_tests([], all_tests, expectations)
self.assertEqual(tests, set([idlharness_test_1, idlharness_test_2]))
self.assertTrue(
expectations.get_expectations(non_idlharness_test).is_default_pass)
self.assertEquals(
expectations.get_expectations(idlharness_test_1).results, {'SKIP'})
self.assertEquals(
expectations.get_expectations(idlharness_test_2).results, {'SKIP'})
# Disable expectations entirely; we should still skip the idlharness
# tests but shouldn't touch the expectations parameter.
finder._options.no_expectations = True
tests = finder.skip_tests([], all_tests, None)
self.assertEqual(tests, set([idlharness_test_1, idlharness_test_2]))
# MSAN/ASAN, with one of the tests specified explicitly (and
# --skipped=default), so should skip only the unspecified test.
expectations = test_expectations.TestExpectations(port)
tests = finder.skip_tests([idlharness_test_1], all_tests, expectations)
self.assertEqual(tests, set([idlharness_test_2]))
# Although we will run the test because it was specified explicitly, it
# is still *expected* to Skip. This is consistent with how entries in
# TestExpectations work.
self.assertTrue(
expectations.get_expectations(non_idlharness_test).is_default_pass)
self.assertEquals(
expectations.get_expectations(idlharness_test_1).results, {'SKIP'})
self.assertEquals(
expectations.get_expectations(idlharness_test_2).results, {'SKIP'})
def run(self, args):
"""Runs the tests and return a RunDetails object with the results."""
start_time = time.time()
self._printer.write_update('Collecting tests ...')
running_all_tests = False
try:
paths, all_test_names, running_all_tests = self._collect_tests(
args)
except IOError:
# This is raised if --test-list doesn't exist
return test_run_results.RunDetails(
exit_code=exit_codes.NO_TESTS_EXIT_STATUS)
test_names = self._finder.split_into_chunks(all_test_names)
if self._options.order == 'natural':
test_names.sort(key=self._port.test_key)
elif self._options.order == 'random':
test_names.sort()
random.Random(self._options.seed).shuffle(test_names)
elif self._options.order == 'none':
# Restore the test order to user specified order.
# base.tests() may change the order as it returns tests in the
# real, external/wpt, virtual order.
if paths:
test_names = self._restore_order(paths, test_names)
if not self._options.no_expectations:
self._printer.write_update('Parsing expectations ...')
self._expectations = test_expectations.TestExpectations(self._port)
tests_to_run, tests_to_skip = self._prepare_lists(paths, test_names)
self._printer.print_found(len(all_test_names), len(test_names),
len(tests_to_run), self._options.repeat_each,
self._options.iterations)
# Check to make sure we're not skipping every test.
if not tests_to_run:
msg = 'No tests to run.'
if self._options.zero_tests_executed_ok:
_log.info(msg)
# Keep executing to produce valid (but empty) results.
else:
_log.critical(msg)
code = exit_codes.NO_TESTS_EXIT_STATUS
return test_run_results.RunDetails(exit_code=code)
exit_code = self._set_up_run(tests_to_run)
if exit_code:
return test_run_results.RunDetails(exit_code=exit_code)
if self._options.num_retries is None:
# If --test-list is passed, or if no test narrowing is specified,
# default to 3 retries. Otherwise [e.g. if tests are being passed by
# name], default to 0 retries.
if self._options.test_list or len(paths) < len(test_names):
self._options.num_retries = 3
else:
self._options.num_retries = 0
should_retry_failures = self._options.num_retries > 0
try:
self._start_servers(tests_to_run)
if self._options.watch:
run_results = self._run_test_loop(tests_to_run, tests_to_skip)
else:
run_results = self._run_test_once(tests_to_run, tests_to_skip,
should_retry_failures)
initial_results, all_retry_results = run_results
finally:
self._stop_servers()
self._clean_up_run()
if self._options.no_expectations:
return test_run_results.RunDetails(0, [], [], initial_results,
all_retry_results)
# Some crash logs can take a long time to be written out so look
# for new logs after the test run finishes.
self._printer.write_update('Looking for new crash logs ...')
self._look_for_new_crash_logs(initial_results, start_time)
for retry_attempt_results in all_retry_results:
self._look_for_new_crash_logs(retry_attempt_results, start_time)
self._printer.write_update('Summarizing results ...')
summarized_full_results = test_run_results.summarize_results(
self._port, self._expectations, initial_results, all_retry_results)
summarized_failing_results = test_run_results.summarize_results(
self._port,
self._expectations,
initial_results,
all_retry_results,
only_include_failing=True)
exit_code = summarized_failing_results['num_regressions']
if exit_code > exit_codes.MAX_FAILURES_EXIT_STATUS:
_log.warning('num regressions (%d) exceeds max exit status (%d)',
exit_code, exit_codes.MAX_FAILURES_EXIT_STATUS)
exit_code = exit_codes.MAX_FAILURES_EXIT_STATUS
if not self._options.dry_run:
self._write_json_files(summarized_full_results,
summarized_failing_results, initial_results,
running_all_tests)
self._upload_json_files()
self._copy_results_html_file(self._artifacts_directory,
'results.html')
if initial_results.keyboard_interrupted:
exit_code = exit_codes.INTERRUPTED_EXIT_STATUS
else:
if initial_results.interrupted:
exit_code = exit_codes.EARLY_EXIT_STATUS
if (self._options.show_results
and (exit_code or initial_results.total_failures)):
self._port.show_results_html_file(
self._filesystem.join(self._artifacts_directory,
'results.html'))
self._printer.print_results(time.time() - start_time,
initial_results)
return test_run_results.RunDetails(exit_code, summarized_full_results,
summarized_failing_results,
initial_results, all_retry_results)
def run(self, args):
"""Runs the tests and return a RunDetails object with the results."""
start_time = time.time()
self._printer.write_update('Collecting tests ...')
running_all_tests = False
if not args or any('external' in path for path in args):
self._printer.write_update('Generating MANIFEST.json for web-platform-tests ...')
WPTManifest.ensure_manifest(self._port.host)
self._printer.write_update('Completed generating manifest.')
self._printer.write_update('Collecting tests ...')
try:
paths, all_test_names, running_all_tests = self._collect_tests(args)
except IOError:
# This is raised if --test-list doesn't exist
return test_run_results.RunDetails(exit_code=exit_codes.NO_TESTS_EXIT_STATUS)
test_names, tests_in_other_chunks = self._finder.split_into_chunks(all_test_names)
if self._options.order == 'natural':
test_names.sort(key=self._port.test_key)
elif self._options.order == 'random':
test_names.sort()
random.Random(self._options.seed).shuffle(test_names)
self._printer.write_update('Parsing expectations ...')
self._expectations = test_expectations.TestExpectations(self._port, test_names)
tests_to_run, tests_to_skip = self._prepare_lists(paths, test_names)
self._expectations.remove_tests_from_expectations(tests_in_other_chunks)
self._printer.print_found(
len(all_test_names), len(test_names), len(tests_to_run),
self._options.repeat_each, self._options.iterations)
# Check to make sure we're not skipping every test.
if not tests_to_run:
msg = 'No tests to run.'
if self._options.zero_tests_executed_ok:
_log.info(msg)
# Keep executing to produce valid (but empty) results.
else:
_log.critical(msg)
code = exit_codes.NO_TESTS_EXIT_STATUS
return test_run_results.RunDetails(exit_code=code)
exit_code = self._set_up_run(tests_to_run)
if exit_code:
return test_run_results.RunDetails(exit_code=exit_code)
if self._options.num_retries is None:
# Don't retry failures if an explicit list of tests was passed in.
should_retry_failures = len(paths) < len(test_names)
# Retry failures 3 times by default.
if should_retry_failures:
self._options.num_retries = 3
else:
should_retry_failures = self._options.num_retries > 0
try:
self._start_servers(tests_to_run)
if self._options.watch:
run_results = self._run_test_loop(tests_to_run, tests_to_skip)
else:
run_results = self._run_test_once(tests_to_run, tests_to_skip, should_retry_failures)
initial_results, all_retry_results, enabled_pixel_tests_in_retry = run_results
finally:
self._stop_servers()
self._clean_up_run()
# Some crash logs can take a long time to be written out so look
# for new logs after the test run finishes.
self._printer.write_update('Looking for new crash logs ...')
self._look_for_new_crash_logs(initial_results, start_time)
for retry_attempt_results in all_retry_results:
self._look_for_new_crash_logs(retry_attempt_results, start_time)
self._printer.write_update('Summarizing results ...')
summarized_full_results = test_run_results.summarize_results(
self._port, self._expectations, initial_results, all_retry_results,
enabled_pixel_tests_in_retry)
summarized_failing_results = test_run_results.summarize_results(
self._port, self._expectations, initial_results, all_retry_results,
enabled_pixel_tests_in_retry, only_include_failing=True)
exit_code = summarized_failing_results['num_regressions']
if exit_code > exit_codes.MAX_FAILURES_EXIT_STATUS:
_log.warning('num regressions (%d) exceeds max exit status (%d)',
exit_code, exit_codes.MAX_FAILURES_EXIT_STATUS)
exit_code = exit_codes.MAX_FAILURES_EXIT_STATUS
if not self._options.dry_run:
self._write_json_files(summarized_full_results, summarized_failing_results, initial_results, running_all_tests)
self._upload_json_files()
self._copy_results_html_file(self._results_directory, 'results.html')
self._copy_results_html_file(self._results_directory, 'legacy-results.html')
if initial_results.keyboard_interrupted:
exit_code = exit_codes.INTERRUPTED_EXIT_STATUS
else:
if initial_results.interrupted:
exit_code = exit_codes.EARLY_EXIT_STATUS
if self._options.show_results and (exit_code or initial_results.total_failures):
self._port.show_results_html_file(
self._filesystem.join(self._results_directory, 'results.html'))
self._printer.print_results(time.time() - start_time, initial_results)
return test_run_results.RunDetails(
exit_code, summarized_full_results, summarized_failing_results,
initial_results, all_retry_results, enabled_pixel_tests_in_retry)
def lint(host, options):
ports_to_lint = [host.port_factory.get(name) for name in host.port_factory.all_port_names(options.platform)]
files_linted = set()
# In general, the set of TestExpectation files should be the same for
# all ports. However, the method used to list expectations files is
# in Port, and the TestExpectations constructor takes a Port.
# Perhaps this function could be changed to just use one Port
# (the default Port for this host) and it would work the same.
failures = []
wpt_overrides_exps_path = host.filesystem.join(
ports_to_lint[0].web_tests_dir(), 'WPTOverrideExpectations')
web_gpu_exps_path = host.filesystem.join(
ports_to_lint[0].web_tests_dir(), 'WebGPUExpectations')
paths = [wpt_overrides_exps_path, web_gpu_exps_path]
expectations_dict = {}
all_system_specifiers = set()
all_build_specifiers = set(ports_to_lint[0].ALL_BUILD_TYPES)
for path in paths:
if host.filesystem.exists(path):
expectations_dict[path] = host.filesystem.read_text_file(path)
for port in ports_to_lint:
expectations_dict.update(port.all_expectations_dict())
config_macro_dict = port.configuration_specifier_macros()
if config_macro_dict:
all_system_specifiers.update({s.lower() for s in config_macro_dict.keys()})
all_system_specifiers.update(
{s.lower() for s in reduce(lambda x, y: x + y, config_macro_dict.values())})
for path in port.extra_expectations_files():
if host.filesystem.exists(path):
expectations_dict[path] = host.filesystem.read_text_file(path)
for path, content in expectations_dict.items():
try:
test_expectations.TestExpectations(
ports_to_lint[0],
expectations_dict={path: content},
is_lint_mode=True)
except test_expectations.ParseError as error:
_log.error('')
for warning in error.warnings:
_log.error(warning)
failures.append('%s: %s' % (path, warning))
_log.error('')
exp_lines = content.split('\n')
for lineno, line in enumerate(exp_lines, 1):
if line.strip().startswith('Bug('):
error = (("%s:%d Expectation '%s' has the Bug(...) token, "
"The token has been removed in the new expectations format") %
(host.filesystem.basename(path), lineno, line))
_log.error(error)
failures.append(error)
_log.error('')
for lineno, line in enumerate(exp_lines, 1):
if line.strip().startswith('#') or not line.strip():
continue
exp_line = TestExpectationLine.tokenize_line(
host.filesystem.basename(path), line, lineno, ports_to_lint[0])
specifiers = set(s.lower() for s in exp_line.specifiers)
system_intersection = specifiers & all_system_specifiers
build_intersection = specifiers & all_build_specifiers
for intersection in [system_intersection, build_intersection]:
if len(intersection) < 2:
continue
error = (("%s:%d Expectation '%s' has multiple specifiers that are mutually exclusive.\n"
"The mutually exclusive specifiers are %s") %
(host.filesystem.basename(path), lineno, line, ', '.join(intersection)))
_log.error(error)
_log.error('')
failures.append(error)
return failures
