def __init__(self, host, port_name=None, **kwargs):
Port.__init__(self, host, port_name or TestPort.default_port_name, **kwargs)
self._tests = unit_test_list()
self._flakes = set()
# FIXME: crbug.com/279494. This needs to be in the "real web tests
# dir" in a mock filesystem, rather than outside of the checkout, so
# that tests that want to write to a TestExpectations file can share
# this between "test" ports and "real" ports. This is the result of
# rebaseline_unittest.py having tests that refer to "real" port names
# and real builders instead of fake builders that point back to the
# test ports. rebaseline_unittest.py needs to not mix both "real" ports
# and "test" ports
self._generic_expectations_path = WEB_TEST_DIR + '/TestExpectations'
self._results_directory = None
self._operating_system = 'mac'
if self._name.startswith('test-win'):
self._operating_system = 'win'
elif self._name.startswith('test-linux'):
self._operating_system = 'linux'
self.port_name = self._operating_system
version_map = {
'test-win-win7': 'win7',
'test-win-win10': 'win10',
'test-mac-mac10.10': 'mac10.10',
'test-mac-mac10.11': 'mac10.11',
'test-linux-precise': 'precise',
'test-linux-trusty': 'trusty',
}
self._version = version_map[self._name]
if self._operating_system == 'linux':
self._architecture = 'x86_64'
self.all_systems = (('mac10.10', 'x86'),
('mac10.11', 'x86'),
('win7', 'x86'),
('win10', 'x86'),
('precise', 'x86_64'),
('trusty', 'x86_64'))
self.all_build_types = ('debug', 'release')
# To avoid surprises when introducing new macros, these are
# intentionally fixed in time.
self.configuration_specifier_macros_dict = {
'mac': ['mac10.10', 'mac10.11'],
'win': ['win7', 'win10'],
'linux': ['precise', 'trusty']
}
def skip_tests(self, paths, all_tests_list, expectations):
"""Given a list of tests, returns the ones that should be skipped.
A test may be skipped for many reasons, depending on the expectation
files and options selected. The most obvious is SKIP entries in
TestExpectations, but we also e.g. skip idlharness tests on MSAN/ASAN
due to https://crbug.com/856601.
Args:
paths: the paths passed on the command-line to run_web_tests.py
all_tests_list: all tests that we are considering running
expectations: parsed TestExpectations data
Returns: a set of tests that should be skipped (not run).
"""
all_tests = set(all_tests_list)
tests_to_skip = set()
for test in all_tests:
# We always skip idlharness tests for MSAN/ASAN, even when running
# with --no-expectations (https://crbug.com/856601). Note we will
# run the test anyway if it is explicitly specified on the command
# line; paths are removed from the skip list after this loop.
if self._options.enable_sanitizer and Port.is_wpt_idlharness_test(
test):
tests_to_skip.update({test})
continue
if self._options.no_expectations:
# do not skip anything from TestExpectations
continue
expected_results = expectations.get_expectations(test).results
if ResultType.Skip in expected_results:
tests_to_skip.update({test})
if self._options.skip_timeouts and ResultType.Timeout in expected_results:
tests_to_skip.update({test})
if self._options.skip_failing_tests and ResultType.Failure in expected_results:
tests_to_skip.update({test})
if self._options.skipped == 'only':
tests_to_skip = all_tests - tests_to_skip
elif self._options.skipped == 'ignore':
tests_to_skip = set()
elif self._options.skipped != 'always':
# make sure we're explicitly running any tests passed on the command line; equivalent to 'default'.
tests_to_skip -= set(paths)
return tests_to_skip
def test_repeated_test_artifacts(self):
host = MockSystemHost()
port = Port(host, 'baseport')
artifacts = Artifacts('/dir', host.filesystem, repeat_tests=True)
def init_test_failure(test_failure):
test_failure.port = port
test_failure.filesystem = host.filesystem
test_failure.test_name = 'foo.html'
test_failure.result_directory = '/dir'
pass_with_stderr = PassWithStderr(
DriverOutput(None, None, None, None, error=b'pass with stderr'))
init_test_failure(pass_with_stderr)
crash = FailureCrash(
DriverOutput(None,
None,
None,
None,
crash=True,
error=b'crash stderr'))
init_test_failure(crash)
timeout = FailureTimeout(
DriverOutput(None, None, None, None, error=b'timeout with stderr'))
init_test_failure(timeout)
pass_with_stderr.create_artifacts(artifacts)
self.assertEqual('pass with stderr',
host.filesystem.read_text_file('/dir/foo-stderr.txt'))
crash.create_artifacts(artifacts)
self.assertEqual('crash stderr',
host.filesystem.read_text_file('/dir/foo-stderr.txt'))
timeout.create_artifacts(artifacts)
self.assertEqual('timeout with stderr',
host.filesystem.read_text_file('/dir/foo-stderr.txt'))
pass_with_stderr.create_artifacts(artifacts)
self.assertEqual('timeout with stderr',
host.filesystem.read_text_file('/dir/foo-stderr.txt'))
def skip_tests(self, paths, all_tests_list, expectations):
"""Given a list of tests, returns the ones that should be skipped.
A test may be skipped for many reasons, depending on the expectation
files and options selected. The most obvious is SKIP entries in
TestExpectations, but we also e.g. skip idlharness tests on MSAN/ASAN
due to https://crbug.com/856601. Note that for programmatically added
SKIPs, this function will modify the input expectations to include the
SKIP expectation (but not write it to disk)
Args:
paths: the paths passed on the command-line to run_web_tests.py
all_tests_list: all tests that we are considering running
expectations: parsed TestExpectations data
Returns: a set of tests that should be skipped (not run).
"""
all_tests = set(all_tests_list)
tests_to_skip = set()
idlharness_skips = set()
for test in all_tests:
# We always skip idlharness tests for MSAN/ASAN, even when running
# with --no-expectations (https://crbug.com/856601). Note we will
# run the test anyway if it is explicitly specified on the command
# line; paths are removed from the skip list after this loop.
if self._options.enable_sanitizer and Port.is_wpt_idlharness_test(
test):
tests_to_skip.update({test})
idlharness_skips.update({test})
continue
if self._options.no_expectations:
# do not skip anything from TestExpectations
continue
expected_results = expectations.get_expectations(test).results
if ResultType.Skip in expected_results:
tests_to_skip.update({test})
if self._options.skip_timeouts and ResultType.Timeout in expected_results:
tests_to_skip.update({test})
if self._options.skip_failing_tests and ResultType.Failure in expected_results:
tests_to_skip.update({test})
# Idlharness tests are skipped programmatically on MSAN/ASAN, so we have
# to add them to the expectations to avoid reporting unexpected skips.
if idlharness_skips and expectations is not None:
raw_expectations = '# results: [ Skip ]\n'
for test in idlharness_skips:
raw_expectations += typ_types.Expectation(
reason="crbug.com/856601",
test=test,
results=[ResultType.Skip]).to_string() + '\n'
expectations.merge_raw_expectations(raw_expectations)
if self._options.skipped == 'only':
tests_to_skip = all_tests - tests_to_skip
elif self._options.skipped == 'ignore':
tests_to_skip = set()
elif self._options.skipped != 'always':
# make sure we're explicitly running any tests passed on the command line; equivalent to 'default'.
tests_to_skip -= set(paths)
return tests_to_skip
def make_port(self):
return Port(MockSystemHost(), 'test', optparse.Values({'configuration': 'Release'}))
