def test_skips_test_port_doesnt_skip_smoke_tests(self):
port = self.make_port(with_tests=True)
port.default_smoke_test_only = lambda: False
self.assertFalse(port.skips_test(
'failures/expected/image.html',
generic_expectations=TestExpectations(port, include_overrides=False),
full_expectations=TestExpectations(port, include_overrides=True)))
def _update_expectations_files(self, lines_to_remove):
# FIXME: This routine is way too expensive. We're creating N ports and N TestExpectations
# objects and (re-)writing the actual expectations file N times, for each test we update.
# We should be able to update everything in memory, once, and then write the file out a single time.
for test in lines_to_remove:
for builder in lines_to_remove[test]:
port = self._tool.port_factory.get_from_builder_name(builder)
path = port.path_to_generic_test_expectations_file()
expectations = TestExpectations(port, include_overrides=False)
for test_configuration in port.all_test_configurations():
if test_configuration.version == port.test_configuration(
).version:
expectationsString = expectations.remove_configuration_from_test(
test, test_configuration)
self._tool.filesystem.write_text_file(path, expectationsString)
for port_name in self._tool.port_factory.all_port_names():
port = self._tool.port_factory.get(port_name)
generic_expectations = TestExpectations(
port, tests=[test], include_overrides=False)
if self._port_skips_test(port, test, generic_expectations):
for test_configuration in port.all_test_configurations():
if test_configuration.version == port.test_configuration(
).version:
expectationsString = generic_expectations.remove_configuration_from_test(
test, test_configuration)
generic_path = port.path_to_generic_test_expectations_file(
)
self._tool.filesystem.write_text_file(
generic_path, expectationsString)
def _update_expectations_files(self, lines_to_remove):
# FIXME: This routine is way too expensive. We're creating O(n ports) TestExpectations objects.
# This is slow and uses a lot of memory.
tests = lines_to_remove.keys()
to_remove = []
# This is so we remove lines for builders that skip this test, e.g. Android skips most
# tests and we don't want to leave stray [ Android ] lines in TestExpectations..
# This is only necessary for "webkit-patch rebaseline" and for rebaselining expected
# failures from garden-o-matic. rebaseline-expectations and auto-rebaseline will always
# pass the exact set of ports to rebaseline.
for port_name in self._tool.port_factory.all_port_names():
port = self._tool.port_factory.get(port_name)
generic_expectations = TestExpectations(port, tests=tests, include_overrides=False)
full_expectations = TestExpectations(port, tests=tests, include_overrides=True)
for test in tests:
if port.skips_test(test, generic_expectations, full_expectations):
for test_configuration in port.all_test_configurations():
if test_configuration.version == port.test_configuration().version:
to_remove.append((test, test_configuration))
for test in lines_to_remove:
for builder in lines_to_remove[test]:
port = self._tool.port_factory.get_from_builder_name(builder)
for test_configuration in port.all_test_configurations():
if test_configuration.version == port.test_configuration().version:
to_remove.append((test, test_configuration))
port = self._tool.port_factory.get()
expectations = TestExpectations(port, include_overrides=False)
expectations_string = expectations.remove_configurations(to_remove)
path = port.path_to_generic_test_expectations_file()
self._tool.filesystem.write_text_file(path, expectations_string)
def test_skips_test_in_smoke_tests(self):
port = self.make_port(with_tests=True)
port.default_smoke_test_only = lambda: True
port.host.filesystem.write_text_file(port.path_to_smoke_tests_file(), 'passes/text.html\n')
self.assertTrue(port.skips_test(
'failures/expected/image.html',
generic_expectations=TestExpectations(port, include_overrides=False),
full_expectations=TestExpectations(port, include_overrides=True)))
def _copy_existing_baseline(self, port_name, test_name, suffix):
"""Copies the baseline for the given builder to all "predecessor" directories."""
baseline_directory = self._tool.port_factory.get(
port_name).baseline_version_dir()
ports = [
self._port_for_primary_baseline(baseline) for baseline in
self._immediate_predecessors_in_fallback(baseline_directory)
]
old_baselines = []
new_baselines = []
# Need to gather all the baseline paths before modifying the filesystem since
# the modifications can affect the results of port.expected_filename.
for port in ports:
old_baseline = port.expected_filename(test_name, '.' + suffix)
if not self._tool.filesystem.exists(old_baseline):
_log.debug('No existing baseline for %s.', test_name)
continue
new_baseline = self._tool.filesystem.join(
port.baseline_version_dir(),
self._file_name_for_expected_result(test_name, suffix))
if self._tool.filesystem.exists(new_baseline):
_log.debug('Existing baseline at %s, not copying over it.',
new_baseline)
continue
generic_expectations = TestExpectations(port,
tests=[test_name],
include_overrides=False)
full_expectations = TestExpectations(port,
tests=[test_name],
include_overrides=True)
# TODO(qyearsley): Change Port.skips_test so that this can be simplified.
if SKIP in full_expectations.get_expectations(test_name):
_log.debug('%s is skipped (perhaps temporarily) on %s.',
test_name, port.name())
continue
if port.skips_test(test_name, generic_expectations,
full_expectations):
_log.debug('%s is skipped on %s.', test_name, port.name())
continue
old_baselines.append(old_baseline)
new_baselines.append(new_baseline)
for i in range(len(old_baselines)):
old_baseline = old_baselines[i]
new_baseline = new_baselines[i]
_log.debug('Copying baseline from %s to %s.', old_baseline,
new_baseline)
self._tool.filesystem.maybe_make_directory(
self._tool.filesystem.dirname(new_baseline))
self._tool.filesystem.copyfile(old_baseline, new_baseline)
def test_skips_test_skip_in_full_expectations(self):
port = self.make_port(with_tests=True)
port.default_smoke_test_only = lambda: False
port.host.filesystem.write_text_file(
port.host.filesystem.join(port.layout_tests_dir(), 'NeverFixTests'),
'Bug(test) failures/expected/image.html [ WontFix ]\n')
self.assertTrue(port.skips_test(
'failures/expected/image.html',
generic_expectations=TestExpectations(port, include_overrides=False),
full_expectations=TestExpectations(port, include_overrides=True)))
def skipped_specifiers(self, test_name):
"""Returns a list of platform specifiers for which the test is skipped."""
# TODO(qyearsley): Change Port.skips_test so that this can be simplified.
specifiers = []
for port in self.all_try_builder_ports():
generic_expectations = TestExpectations(port, tests=[test_name], include_overrides=False)
full_expectations = TestExpectations(port, tests=[test_name], include_overrides=True)
if port.skips_test(test_name, generic_expectations, full_expectations):
specifiers.append(self.host.builders.version_specifier_for_port_name(port.name()))
return specifiers
def _update_expectations_files(self, port_name):
port = self._tool.port_factory.get(port_name)
expectations = TestExpectations(port)
for path in port.expectations_dict():
if self._tool.filesystem.exists(path):
self._tool.filesystem.write_text_file(path, expectations.remove_rebaselined_tests(expectations.get_rebaselining_failures(), path))
def _update_expectations_files(self, lines_to_remove):
tests = lines_to_remove.keys()
to_remove = []
# This is so we remove lines for builders that skip this test.
# For example, Android skips most tests and we don't want to leave
# stray [ Android ] lines in TestExpectations.
# This is only necessary for "blink_tool.py rebaseline".
for port_name in self._tool.port_factory.all_port_names():
port = self._tool.port_factory.get(port_name)
for test in tests:
if port.skips_test(test):
for test_configuration in port.all_test_configurations():
if test_configuration.version == port.test_configuration().version:
to_remove.append((test, test_configuration))
for test in lines_to_remove:
for port_name in lines_to_remove[test]:
port = self._tool.port_factory.get(port_name)
for test_configuration in port.all_test_configurations():
if test_configuration.version == port.test_configuration().version:
to_remove.append((test, test_configuration))
port = self._tool.port_factory.get()
expectations = TestExpectations(port, include_overrides=False)
expectations_string = expectations.remove_configurations(to_remove)
path = port.path_to_generic_test_expectations_file()
self._tool.filesystem.write_text_file(path, expectations_string)
def _tests_to_rebaseline(port):
tests_to_rebaseline = []
for path, value in port.expectations_dict().items():
expectations = TestExpectations(port, include_overrides=False, expectations_dict={path: value})
for test in expectations.get_rebaselining_failures():
tests_to_rebaseline.append(test)
return tests_to_rebaseline
def _model(self, options, port_name, tests):
port = self._tool.port_factory.get(port_name, options)
expectations_path = port.path_to_test_expectations_file()
if not expectations_path in self._expectation_models:
self._expectation_models[expectations_path] = TestExpectations(
port, tests).model()
return self._expectation_models[expectations_path]
def update_test_expectations(self, deleted_tests, renamed_tests):
"""Updates the TestExpectations file entries for tests that have been deleted or renamed."""
port = self.host.port_factory.get()
test_expectations = TestExpectations(port, include_overrides=False)
# Tests for which files don't exist aren't stored in TestExpectationsModel,
# so methods like TestExpectations.remove_expectation_line don't work; instead
# we can run through the TestExpectationLine objects that were parsed.
# FIXME: This won't work for removed or renamed directories with test expectations
# that are directories rather than individual tests.
new_lines = []
changed_lines = []
for expectation_line in test_expectations.expectations():
if expectation_line.name in deleted_tests:
continue
if expectation_line.name in renamed_tests:
expectation_line.name = renamed_tests[expectation_line.name]
# Upon parsing the file, a "path does not exist" warning is expected
# to be there for tests that have been renamed, and if there are warnings,
# then the original string is used. If the warnings are reset, then the
# expectation line is re-serialized when output.
expectation_line.warnings = []
changed_lines.append(expectation_line)
new_lines.append(expectation_line)
self.host.filesystem.write_text_file(
port.path_to_generic_test_expectations_file(),
TestExpectations.list_to_string(
new_lines, reconstitute_only_these=changed_lines))
def test_update_summary_with_result(self):
host = MockHost()
port = host.port_factory.get('test-win-xp')
test = 'failures/expected/reftest.html'
expectations = TestExpectations(
port,
tests=[test],
expectations='WONTFIX : failures/expected/reftest.html = IMAGE',
test_config=port.test_configuration())
# Reftests expected to be image mismatch should be respected when pixel_tests=False.
manager = Manager(port=port,
options=MockOptions(
pixel_tests=False,
exit_after_n_failures=None,
exit_after_n_crashes_or_timeouts=None),
printer=Mock())
manager._expectations = expectations
result_summary = ResultSummary(expectations=expectations,
test_files=[test])
result = TestResult(
test_name=test,
failures=[test_failures.FailureReftestMismatchDidNotOccur()])
manager._update_summary_with_result(result_summary, result)
self.assertEquals(1, result_summary.expected)
self.assertEquals(0, result_summary.unexpected)
def _tests_to_rebaseline(self, port):
tests_to_rebaseline = {}
expectations = TestExpectations(port, include_overrides=True)
expectations.parse_all_expectations()
for test in expectations.get_rebaselining_failures():
tests_to_rebaseline[test] = TestExpectations.suffixes_for_expectations(expectations.model().get_expectations(test))
return tests_to_rebaseline
def _update_expectations(self):
"""Updates all test expectations that are affected by the move.
"""
_log.info('Updating expectations')
test_expectations = TestExpectations(self._port, include_overrides=False, model_all_expectations=True)
for expectation in self._get_expectations(test_expectations.model(), self._origin):
path = expectation.path
if self._is_child_path(self._origin, path):
# If the existing expectation is a child of the moved path, we simply replace it
# with an expectation for the updated path.
new_path = self._move_path(path, self._origin, self._destination)
_log.debug('Updating expectation for %s to %s' % (path, new_path))
test_expectations.remove_expectation_line(path)
test_expectations.add_expectation_line(testsMover._clone_expectation_line_for_path(expectation, new_path))
else:
# If the existing expectation is not a child of the moved path, we have to leave it
# in place. But we also add a new expectation for the destination path.
new_path = self._destination
_log.warning('Copying expectation for %s to %s. You should check that these expectations are still correct.' %
(path, new_path))
test_expectations.add_expectation_line(testsMover._clone_expectation_line_for_path(expectation, new_path))
expectations_file = self._port.path_to_generic_test_expectations_file()
self._filesystem.write_text_file(expectations_file,
TestExpectations.list_to_string(test_expectations._expectations, reconstitute_only_these=[]))
self._scm.add(self._filesystem.relpath(expectations_file, self._scm.checkout_root))
def _tests_to_rebaseline(port):
tests_to_rebaseline = {}
for path, value in port.expectations_dict().items():
expectations = TestExpectations(port, include_overrides=False, expectations_dict={path: value})
for test in expectations.get_rebaselining_failures():
suffixes = TestExpectations.suffixes_for_expectations(expectations.get_expectations(test))
tests_to_rebaseline[test] = suffixes or BASELINE_SUFFIX_LIST
return tests_to_rebaseline
def _tests_to_rebaseline(self, port):
tests_to_rebaseline = {}
expectations = TestExpectations(port, include_overrides=True)
for test in expectations.get_rebaselining_failures():
suffixes = TestExpectations.suffixes_for_expectations(
expectations.get_expectations(test))
tests_to_rebaseline[test] = suffixes or BASELINE_SUFFIX_LIST
return tests_to_rebaseline
def _update_expectations_file(self, builder_name, test_name):
port = self._tool.port_factory.get_from_builder_name(builder_name)
expectations = TestExpectations(port, include_overrides=False)
for test_configuration in port.all_test_configurations():
if test_configuration.version == port.test_configuration().version:
expectationsString = expectations.remove_configuration_from_test(test_name, test_configuration)
self._tool.filesystem.write_text_file(port.path_to_test_expectations_file(), expectationsString)
def _update_expectations_file(self, port_name):
port = self._tool.port_factory.get(port_name)
# FIXME: This will intentionally skip over any REBASELINE expectations that were in an overrides file.
# This is not good, but avoids having the overrides getting written into the main file.
# See https://bugs.webkit.org/show_bug.cgi?id=88456 for context. This will no longer be needed
# once we properly support cascading expectations files.
expectations = TestExpectations(port, include_overrides=False)
path = port.path_to_test_expectations_file()
self._tool.filesystem.write_text_file(path, expectations.remove_rebaselined_tests(expectations.get_rebaselining_failures()))
def _make_test_baseline_set(self, tests):
test_baseline_set = TestBaselineSet(self._tool)
for builder_name in self._release_builders():
port_name = self._tool.builders.port_name_for_builder_name(builder_name)
port = self._tool.port_factory.get(port_name)
expectations = TestExpectations(port, include_overrides=True)
for test in expectations.get_needs_rebaseline_failures():
if test not in tests:
continue
test_baseline_set.add(test, Build(builder_name))
return test_baseline_set
def _update_expectations_files(self, port_name):
port = self._tool.port_factory.get(port_name)
expectations = TestExpectations(port)
rebaseline_tests = expectations.get_rebaselining_failures()
filtered_rebaseline_tests = [test for test in rebaseline_tests if not port.reference_files(test)]
for path in port.expectations_dict():
if self._tool.filesystem.exists(path):
self._tool.filesystem.write_text_file(path, expectations.remove_rebaselined_tests(filtered_rebaseline_tests, path))
def _model(self, options, port_name, tests):
port = self._tool.port_factory.get(port_name, options)
expectations_path = port.path_to_test_expectations_file()
if not expectations_path in self._expectation_models:
lint_mode = False
self._expectation_models[expectations_path] = TestExpectations(
port, tests, port.test_expectations(),
port.test_configuration(), lint_mode,
port.test_expectations_overrides(),
port.skipped_layout_tests(tests)).model()
return self._expectation_models[expectations_path]
def _update_expectations_files(self, lines_to_remove):
for test in lines_to_remove:
for builder in lines_to_remove[test]:
port = self._tool.port_factory.get_from_builder_name(builder)
path = port.path_to_generic_test_expectations_file()
expectations = TestExpectations(port, include_overrides=False)
for test_configuration in port.all_test_configurations():
if test_configuration.version == port.test_configuration(
).version:
expectationsString = expectations.remove_configuration_from_test(
test, test_configuration)
self._tool.filesystem.write_text_file(path, expectationsString)
def _port_skips_test(self, port, test, generic_expectations):
fs = port.host.filesystem
if port.default_smoke_test_only():
smoke_test_filename = fs.join(port.layout_tests_dir(),
'SmokeTests')
if fs.exists(
smoke_test_filename) and test not in fs.read_text_file(
smoke_test_filename):
return True
full_expectations = TestExpectations(port,
tests=[test],
include_overrides=True)
return (SKIP in full_expectations.get_expectations(test)
and SKIP not in generic_expectations.get_expectations(test))
def _copy_existing_baseline(self, builder_name, test_name, suffix):
baseline_directory = self._baseline_directory(builder_name)
ports = [
self._port_for_primary_baseline(baseline) for baseline in
self._immediate_predecessors_in_fallback(baseline_directory)
]
old_baselines = []
new_baselines = []
# Need to gather all the baseline paths before modifying the filesystem since
# the modifications can affect the results of port.expected_filename.
for port in ports:
old_baseline = port.expected_filename(test_name, "." + suffix)
if not self._tool.filesystem.exists(old_baseline):
_log.debug("No existing baseline for %s." % test_name)
continue
new_baseline = self._tool.filesystem.join(
port.baseline_path(),
self._file_name_for_expected_result(test_name, suffix))
if self._tool.filesystem.exists(new_baseline):
_log.debug("Existing baseline at %s, not copying over it." %
new_baseline)
continue
expectations = TestExpectations(port, [test_name])
if SKIP in expectations.get_expectations(test_name):
_log.debug("%s is skipped on %s." % (test_name, port.name()))
continue
old_baselines.append(old_baseline)
new_baselines.append(new_baseline)
for i in range(len(old_baselines)):
old_baseline = old_baselines[i]
new_baseline = new_baselines[i]
_log.debug("Copying baseline from %s to %s." %
(old_baseline, new_baseline))
self._tool.filesystem.maybe_make_directory(
self._tool.filesystem.dirname(new_baseline))
self._tool.filesystem.copyfile(old_baseline, new_baseline)
if not self._tool.scm().exists(new_baseline):
self._add_to_scm_later(new_baseline)
def _expectations_to_remove(self):
"""Computes and returns the expectation lines that should be removed.
returns a list of TestExpectationLines that can be removed from the test expectations
file. The result is memoized so that subsequent calls will not recompute the result.
"""
if self._expectations_to_remove_list is not None:
return self._expectations_to_remove_list
self._builder_results_by_path = self._get_builder_results_by_path()
self._expectations_to_remove_list = []
test_expectations = TestExpectations(
self._port, include_overrides=False).expectations()
for expectation in test_expectations:
if self._can_delete_line(expectation):
self._expectations_to_remove_list.append(expectation)
return self._expectations_to_remove_list
def get_test_prefix_list(self, tests):
test_prefix_list = {}
lines_to_remove = {}
for builder_name in self._release_builders():
port_name = builders.port_name_for_builder_name(builder_name)
port = self._tool.port_factory.get(port_name)
expectations = TestExpectations(port, include_overrides=True)
for test in expectations.get_needs_rebaseline_failures():
if test not in tests:
continue
if test not in test_prefix_list:
lines_to_remove[test] = []
test_prefix_list[test] = {}
lines_to_remove[test].append(builder_name)
test_prefix_list[test][builder_name] = BASELINE_SUFFIX_LIST
return test_prefix_list, lines_to_remove
def execute(self, options, args, tool):
if not options.paths and not args and not options.all:
print "You must either specify one or more test paths or --all."
return
if options.platform:
port_names = fnmatch.filter(tool.port_factory.all_port_names(),
options.platform)
if not port_names:
default_port = tool.port_factory.get(options.platform)
if default_port:
port_names = [default_port.name()]
else:
print "No port names match '%s'" % options.platform
return
else:
default_port = tool.port_factory.get(port_names[0])
else:
default_port = tool.port_factory.get(options=options)
port_names = [default_port.name()]
if options.paths:
files = default_port.expectations_files()
layout_tests_dir = default_port.layout_tests_dir()
for file in files:
if file.startswith(layout_tests_dir):
file = file.replace(layout_tests_dir, 'LayoutTests')
print file
return
tests = set(default_port.tests(args))
for port_name in port_names:
port = tool.port_factory.get(port_name, options)
model = TestExpectations(port, tests).model()
tests_to_print = self._filter_tests(options, model, tests)
lines = [
model.get_expectation_line(test)
for test in sorted(tests_to_print)
]
if port_name != port_names[0]:
print
print '\n'.join(self._format_lines(options, port_name, lines))
def _update_expectations_file(self, builder_name, test_name):
port = self._tool.port_factory.get_from_builder_name(builder_name)
# Since rebaseline-test-internal can be called multiple times in parallel,
# we need to ensure that we're not trying to update the expectations file
# concurrently as well.
# FIXME: We should rework the code to not need this; maybe just download
# the files in parallel and rebaseline local files serially?
try:
path = port.path_to_generic_test_expectations_file()
lock = self._tool.make_file_lock(path + '.lock')
lock.acquire_lock()
expectations = TestExpectations(port, include_overrides=False)
for test_configuration in port.all_test_configurations():
if test_configuration.version == port.test_configuration().version:
expectationsString = expectations.remove_configuration_from_test(test_name, test_configuration)
self._tool.filesystem.write_text_file(path, expectationsString)
finally:
lock.release_lock()
def get_updated_test_expectations(self):
"""Filters out passing lines from TestExpectations file.
Reads the current TestExpectations file and, using results from the
build bots, removes lines that are passing. That is, removes lines that
were not needed to keep the bots green.
Returns:
A TestExpectations object with the passing lines filtered out.
"""
test_expectations = TestExpectations(self._port, include_overrides=False).expectations()
for expectation in self._expectations_to_remove():
index = test_expectations.index(expectation)
test_expectations.remove(expectation)
# Remove associated comments and whitespace if we've removed the last expectation under
# a comment block. Only remove a comment block if it's not separated from the test
# expectation line by whitespace.
self._remove_associated_comments_and_whitespace(test_expectations, index)
return test_expectations
