def compare_output(self, port, filename, output, test_args, configuration):
"""Implementation of CompareOutput that checks the output text against
the expected text from the LayoutTest directory."""
failures = []
# If we're generating a new baseline, we pass.
if test_args.new_baseline or test_args.reset_results:
# Although all test_shell/DumpRenderTree output should be utf-8,
# we do not ever decode it inside run-webkit-tests. For some tests
# DumpRenderTree may not output utf-8 text (e.g. webarchives).
self._save_baseline_data(filename, output, ".txt", encoding=None,
generate_new_baseline=test_args.new_baseline)
return failures
# Normalize text to diff
output = self._get_normalized_output_text(output)
expected = self._get_normalized_expected_text(filename)
# Write output files for new tests, too.
if port.compare_text(output, expected):
# Text doesn't match, write output files.
self.write_output_files(port, filename, ".txt", output,
expected, encoding=None,
print_text_diffs=True)
if expected == '':
failures.append(test_failures.FailureMissingResult())
else:
failures.append(test_failures.FailureTextMismatch(True))
return failures
def _compare_text(self, actual_text, expected_text):
failures = []
if self._port.compare_text(
self._get_normalized_output_text(actual_text),
# Assuming expected_text is already normalized.
expected_text):
if expected_text == '':
failures.append(test_failures.FailureMissingResult())
else:
failures.append(test_failures.FailureTextMismatch())
return failures
def _compare_text(self, actual_text, expected_text):
failures = []
if (expected_text and actual_text and
# Assuming expected_text is already normalized.
self._port.compare_text(
self._get_normalized_output_text(actual_text),
expected_text)):
failures.append(test_failures.FailureTextMismatch())
elif actual_text and not expected_text:
failures.append(test_failures.FailureMissingResult())
return failures
def _failures_from_fail_row(self, row):
# Look at all anchors in this row, and guess what type
# of new-run-webkit-test failures they equate to.
failures = set()
test_name = None
for anchor in row.findAll("a"):
anchor_text = unicode(anchor.string)
if not test_name:
test_name = anchor_text
continue
if anchor_text in ["expected image", "image diffs"] or '%' in anchor_text:
failures.add(test_failures.FailureImageHashMismatch())
elif anchor_text in ["expected", "actual", "diff", "pretty diff"]:
failures.add(test_failures.FailureTextMismatch())
else:
log("Unhandled link text in results.html parsing: %s. Please file a bug against webkitpy." % anchor_text)
# FIXME: Its possible the row contained no links due to ORWT brokeness.
# We should probably assume some type of failure anyway.
return failures
def _mock_test_result(self, testname):
return test_results.TestResult(testname,
[test_failures.FailureTextMismatch()])