def skip_tests(self, paths, all_tests_list, expectations): """Given a list of tests, returns the ones that should be skipped. A test may be skipped for many reasons, depending on the expectation files and options selected. The most obvious is SKIP entries in TestExpectations, but we also e.g. skip idlharness tests on MSAN/ASAN due to https://crbug.com/856601. Args: paths: the paths passed on the command-line to run_web_tests.py all_tests_list: all tests that we are considering running expectations: parsed TestExpectations data Returns: a set of tests that should be skipped (not run). """ all_tests = set(all_tests_list) tests_to_skip = set() for test in all_tests: # We always skip idlharness tests for MSAN/ASAN, even when running # with --no-expectations (https://crbug.com/856601). Note we will # run the test anyway if it is explicitly specified on the command # line; paths are removed from the skip list after this loop. if self._options.enable_sanitizer and Port.is_wpt_idlharness_test( test): tests_to_skip.update({test}) continue if self._options.no_expectations: # do not skip anything from TestExpectations continue expected_results = expectations.get_expectations(test).results if ResultType.Skip in expected_results: tests_to_skip.update({test}) if self._options.skip_timeouts and ResultType.Timeout in expected_results: tests_to_skip.update({test}) if self._options.skip_failing_tests and ResultType.Failure in expected_results: tests_to_skip.update({test}) if self._options.skipped == 'only': tests_to_skip = all_tests - tests_to_skip elif self._options.skipped == 'ignore': tests_to_skip = set() elif self._options.skipped != 'always': # make sure we're explicitly running any tests passed on the command line; equivalent to 'default'. tests_to_skip -= set(paths) return tests_to_skip
def skip_tests(self, paths, all_tests_list, expectations): """Given a list of tests, returns the ones that should be skipped. A test may be skipped for many reasons, depending on the expectation files and options selected. The most obvious is SKIP entries in TestExpectations, but we also e.g. skip idlharness tests on MSAN/ASAN due to https://crbug.com/856601. Note that for programmatically added SKIPs, this function will modify the input expectations to include the SKIP expectation (but not write it to disk) Args: paths: the paths passed on the command-line to run_web_tests.py all_tests_list: all tests that we are considering running expectations: parsed TestExpectations data Returns: a set of tests that should be skipped (not run). """ all_tests = set(all_tests_list) tests_to_skip = set() idlharness_skips = set() for test in all_tests: # We always skip idlharness tests for MSAN/ASAN, even when running # with --no-expectations (https://crbug.com/856601). Note we will # run the test anyway if it is explicitly specified on the command # line; paths are removed from the skip list after this loop. if self._options.enable_sanitizer and Port.is_wpt_idlharness_test( test): tests_to_skip.update({test}) idlharness_skips.update({test}) continue if self._options.no_expectations: # do not skip anything from TestExpectations continue expected_results = expectations.get_expectations(test).results if ResultType.Skip in expected_results: tests_to_skip.update({test}) if self._options.skip_timeouts and ResultType.Timeout in expected_results: tests_to_skip.update({test}) if self._options.skip_failing_tests and ResultType.Failure in expected_results: tests_to_skip.update({test}) # Idlharness tests are skipped programmatically on MSAN/ASAN, so we have # to add them to the expectations to avoid reporting unexpected skips. if idlharness_skips and expectations is not None: raw_expectations = '# results: [ Skip ]\n' for test in idlharness_skips: raw_expectations += typ_types.Expectation( reason="crbug.com/856601", test=test, results=[ResultType.Skip]).to_string() + '\n' expectations.merge_raw_expectations(raw_expectations) if self._options.skipped == 'only': tests_to_skip = all_tests - tests_to_skip elif self._options.skipped == 'ignore': tests_to_skip = set() elif self._options.skipped != 'always': # make sure we're explicitly running any tests passed on the command line; equivalent to 'default'. tests_to_skip -= set(paths) return tests_to_skip