def test_failing_tests_message(self):
# Needed to define port_name, used in AbstractEarlyWarningSystem.__init__
class TestEWS(AbstractEarlyWarningSystem):
port_name = "win" # Needs to be a port which port/factory understands.
ews = TestEWS()
ews.bind_to_tool(MockTool())
ews._options = MockOptions(port=None, confirm=False)
OutputCapture().assert_outputs(
self,
ews.begin_work_queue,
expected_logs=self._default_begin_work_queue_logs(ews.name))
task = Mock()
task.results_from_patch_test_run = lambda a: LayoutTestResults(
[
test_results.TestResult("foo.html",
failures=
[test_failures.FailureTextMismatch()]),
test_results.TestResult(
"bar.html", failures=[test_failures.FailureTextMismatch()])
],
did_exceed_test_failure_limit=False)
task.results_from_test_run_without_patch = lambda a: LayoutTestResults(
[], did_exceed_test_failure_limit=False)
patch = ews._tool.bugs.fetch_attachment(10000)
self.assertMultiLineEqual(ews._failing_tests_message(task, patch),
"New failing tests:\nbar.html\nfoo.html")
def _failure_types_from_actual_result(self, actual):
# FIXME: There doesn't seem to be a full list of all possible values of
# 'actual' anywhere. However JSONLayoutResultsGenerator.FAILURE_TO_CHAR
# is a useful reference as that's for "old" style results.json files
if actual == test_expectations.PASS:
return []
elif actual == test_expectations.TEXT:
return [test_failures.FailureTextMismatch()]
elif actual == test_expectations.IMAGE:
return [test_failures.FailureImageHashMismatch()]
elif actual == test_expectations.IMAGE_PLUS_TEXT:
return [
test_failures.FailureImageHashMismatch(),
test_failures.FailureTextMismatch()
]
elif actual == test_expectations.AUDIO:
return [test_failures.FailureAudioMismatch()]
elif actual == test_expectations.TIMEOUT:
return [test_failures.FailureTimeout()]
elif actual == test_expectations.CRASH:
# NOTE: We don't know what process crashed from the json, just that a process crashed.
return [test_failures.FailureCrash()]
elif actual == test_expectations.MISSING:
return [
test_failures.FailureMissingResult(),
test_failures.FailureMissingImageHash(),
test_failures.FailureMissingImage()
]
else:
log("Failed to handle: %s" % self._result_dict['actual'])
return []
def test_failing_tests_message(self):
ews = TestEWS()
results = lambda a: LayoutTestResults([
test_results.TestResult(
"foo.html", failures=[test_failures.FailureTextMismatch()]),
test_results.TestResult(
"bar.html", failures=[test_failures.FailureTextMismatch()])
],
did_exceed_test_failure_limit=
False)
message = "New failing tests:\nfoo.html\nbar.html"
self._test_message(ews, results, message)
def results_from_patch_test_run(self, patch):
return LayoutTestResults([
test_results.TestResult(
"mock_test_name.html",
failures=[test_failures.FailureTextMismatch()])
],
did_exceed_test_failure_limit=False)
def test_basic(self):
expected_results = [
test_results.TestResult("svg/dynamic-updates/SVGFEDropShadowElement-dom-stdDeviation-attr.html", [test_failures.FailureImageHashMismatch()], 0),
test_results.TestResult("fast/dom/prototype-inheritance.html", [test_failures.FailureTextMismatch(), test_failures.FailureImageHashMismatch(), test_failures.FailureAudioMismatch()], 0),
]
parsed_results = ParsedJSONResults(self._example_full_results_json)
self.assertEqual(expected_results, parsed_results.test_results())
self.assertTrue(parsed_results.did_exceed_test_failure_limit())
def _failure_types_from_actual_result(self, actual):
# FIXME: There doesn't seem to be a full list of all possible values of
# 'actual' anywhere. However JSONLayoutResultsGenerator.FAILURE_TO_CHAR
# is a useful reference as that's for "old" style results.json files
#
# FIXME: TEXT, IMAGE_PLUS_TEXT, and AUDIO are obsolete but we keep them for
# now so that we can parse old results.json files.
if actual == test_expectations.PASS:
return []
elif actual == test_expectations.FAIL:
return [
test_failures.FailureTextMismatch(),
test_failures.FailureImageHashMismatch(),
test_failures.FailureAudioMismatch()
]
elif actual == test_expectations.TEXT:
return [test_failures.FailureTextMismatch()]
elif actual == test_expectations.IMAGE:
return [test_failures.FailureImageHashMismatch()]
elif actual == test_expectations.IMAGE_PLUS_TEXT:
return [
test_failures.FailureImageHashMismatch(),
test_failures.FailureTextMismatch()
]
elif actual == test_expectations.AUDIO:
return [test_failures.FailureAudioMismatch()]
elif actual == test_expectations.TIMEOUT:
return [test_failures.FailureTimeout()]
elif actual == test_expectations.CRASH:
# NOTE: We don't know what process crashed from the json, just that a process crashed.
return [test_failures.FailureCrash()]
elif actual == test_expectations.LEAK:
urls = []
for url_dict in self._result_dict['leaks']:
urls.append(url_dict['document'])
return [test_failures.FailureDocumentLeak(urls)]
elif actual == test_expectations.MISSING:
return [
test_failures.FailureMissingResult(),
test_failures.FailureMissingImageHash(),
test_failures.FailureMissingImage()
]
else:
_log.warning("Failed to handle: %s" % self._result_dict['actual'])
return []
def _compare_text(self, actual_text, expected_text):
failures = []
if (expected_text and actual_text and
# Assuming expected_text is already normalized.
self._port.compare_text(self._get_normalized_output_text(actual_text), expected_text)):
failures.append(test_failures.FailureTextMismatch())
elif actual_text and not expected_text:
failures.append(test_failures.FailureMissingResult())
return failures
def _compare_text(self, expected_text, actual_text):
failures = []
if (expected_text is not None and actual_text is not None and
# Assuming expected_text is already normalized.
self._port.do_text_results_differ(expected_text, self._get_normalized_output_text(actual_text))):
failures.append(test_failures.FailureTextMismatch())
elif actual_text is not None and expected_text is None:
failures.append(test_failures.FailureMissingResult())
return failures
def _compare_text(self, expected_text, actual_text):
if not actual_text:
return []
if not expected_text:
return [test_failures.FailureMissingResult()]
normalized_actual_text = self._get_normalized_output_text(actual_text)
# Assuming expected_text is already normalized.
if not self._port.do_text_results_differ(expected_text,
normalized_actual_text):
return []
# Determine the text mismatch type
def remove_chars(text, chars):
for char in chars:
text = text.replace(char, '')
return text
def is_ng_name_mismatch(expected, actual):
if 'LayoutNGBlockFlow' not in actual:
return False
if not self._is_render_tree(actual) and not self._is_layer_tree(
actual):
return False
processed = actual.replace('LayoutNGBlockFlow',
'LayoutBlockFlow').replace(
'LayoutNGListItem',
'LayoutListItem')
return not self._port.do_text_results_differ(expected, processed)
# LayoutNG name mismatch (e.g., LayoutBlockFlow vs. LayoutNGBlockFlow)
# is treated as pass
if is_ng_name_mismatch(expected_text, normalized_actual_text):
return []
# General text mismatch
if self._port.do_text_results_differ(
remove_chars(expected_text, ' \t\n'),
remove_chars(normalized_actual_text, ' \t\n')):
return [test_failures.FailureTextMismatch()]
# Space-only mismatch
if not self._port.do_text_results_differ(
remove_chars(expected_text, ' \t'),
remove_chars(normalized_actual_text, ' \t')):
return [test_failures.FailureSpacesAndTabsTextMismatch()]
# Newline-only mismatch
if not self._port.do_text_results_differ(
remove_chars(expected_text, '\n'),
remove_chars(normalized_actual_text, '\n')):
return [test_failures.FailureLineBreaksTextMismatch()]
# Spaces and newlines
return [test_failures.FailureSpaceTabLineBreakTextMismatch()]
def test_basic(self):
expected_results = [
test_results.TestResult(
"svg/dynamic-updates/SVGFEDropShadowElement-dom-stdDeviation-attr.html",
[test_failures.FailureImageHashMismatch()], 0),
test_results.TestResult("fast/dom/prototype-inheritance.html",
[test_failures.FailureTextMismatch()], 0),
]
results = ResultsJSONParser.parse_results_json(
self._example_full_results_json)
self.assertEqual(expected_results, results)
def test_basic(self):
self.maxDiff = None
expected_results = [
test_results.TestResult("svg/dynamic-updates/SVGFEDropShadowElement-dom-stdDeviation-attr.html", [test_failures.FailureImageHashMismatch()], 0),
test_results.TestResult("fast/dom/prototype-inheritance.html", [test_failures.FailureTextMismatch(), test_failures.FailureImageHashMismatch(), test_failures.FailureAudioMismatch()], 0),
test_results.TestResult("fast/dom/prototype-strawberry.html", [test_failures.FailureDocumentLeak(['file:///Volumes/Data/worker/webkit/build/LayoutTests/fast/dom/prototype-strawberry.html'])], 0),
]
expected_results.sort(key=lambda result: result.test_name)
parsed_results = ParsedJSONResults(self._example_full_results_json)
self.assertEqual(expected_results, parsed_results.test_results())
self.assertTrue(parsed_results.did_exceed_test_failure_limit())
def _failure_types_from_actual_result(self, actual):
# FIXME: There doesn't seem to be a full list of all possible values of
# 'actual' anywhere.
#
# FIXME: TEXT, IMAGE_PLUS_TEXT, and AUDIO are obsolete but we keep them for
# now so that we can parse old results.json files.
if actual == test_expectations.PASS:
return []
elif actual == test_expectations.FAIL:
return [
test_failures.FailureTextMismatch(),
test_failures.FailureImageHashMismatch(),
test_failures.FailureAudioMismatch()
]
elif actual == test_expectations.TEXT:
return [test_failures.FailureTextMismatch()]
elif actual == test_expectations.IMAGE:
return [test_failures.FailureImageHashMismatch()]
elif actual == test_expectations.IMAGE_PLUS_TEXT:
return [
test_failures.FailureImageHashMismatch(),
test_failures.FailureTextMismatch()
]
elif actual == test_expectations.AUDIO:
return [test_failures.FailureAudioMismatch()]
elif actual == test_expectations.TIMEOUT:
return [test_failures.FailureTimeout()]
elif actual == test_expectations.CRASH:
# NOTE: We don't know what process crashed from the json, just that a process crashed.
return [test_failures.FailureCrash()]
elif actual == test_expectations.MISSING:
return [
test_failures.FailureMissingResult(),
test_failures.FailureMissingImageHash(),
test_failures.FailureMissingImage()
]
else:
_log.warning("Failed to handle: %s" % self._result_dict['actual'])
return []
def _compare_text(self, expected_text, actual_text):
failures = []
if self._options.ignore_render_tree_dump_results and actual_text and self._render_tree_dump_pattern.match(actual_text):
return failures
if (expected_text and actual_text and
# Assuming expected_text is already normalized.
self._port.do_text_results_differ(expected_text, self._get_normalized_output_text(actual_text))):
failures.append(test_failures.FailureTextMismatch())
elif actual_text and not expected_text:
failures.append(test_failures.FailureMissingResult())
elif not actual_text and expected_text:
failures.append(test_failures.FailureNoOutput())
return failures
def get_result(test_name, result_type=test_expectations.PASS, run_time=0):
failures = []
if result_type == test_expectations.TIMEOUT:
failures = [test_failures.FailureTimeout()]
elif result_type == test_expectations.AUDIO:
failures = [test_failures.FailureAudioMismatch()]
elif result_type == test_expectations.TEXT:
failures = [test_failures.FailureTextMismatch()]
elif result_type == test_expectations.IMAGE:
failures = [test_failures.FailureImageHashMismatch()]
elif result_type == test_expectations.CRASH:
failures = [test_failures.FailureCrash()]
elif result_type == test_expectations.LEAK:
failures = [test_failures.FailureLeak()]
return test_results.TestResult(test_name, failures=failures, test_run_time=run_time)
def _failures_from_fail_row(self, row):
# Look at all anchors in this row, and guess what type
# of new-run-webkit-test failures they equate to.
failures = set()
test_name = None
for anchor in row.findAll("a"):
anchor_text = unicode(anchor.string)
if not test_name:
test_name = anchor_text
continue
if anchor_text in ["expected image", "image diffs"] or '%' in anchor_text:
failures.add(test_failures.FailureImageHashMismatch())
elif anchor_text in ["expected", "actual", "diff", "pretty diff"]:
failures.add(test_failures.FailureTextMismatch())
else:
log("Unhandled link text in results.html parsing: %s. Please file a bug against webkitpy." % anchor_text)
# FIXME: Its possible the row contained no links due to ORWT brokeness.
# We should probably assume some type of failure anyway.
return failures
def _compare_text(self, expected_text, actual_text):
if not actual_text:
return []
if not expected_text:
return [test_failures.FailureMissingResult()]
normalized_actual_text = self._get_normalized_output_text(actual_text)
# Assuming expected_text is already normalized.
if not self._port.do_text_results_differ(expected_text, normalized_actual_text):
return []
# Determine the text mismatch type
def remove_chars(text, chars):
for char in chars:
text = text.replace(char, '')
return text
# General text mismatch
if self._port.do_text_results_differ(
remove_chars(expected_text, ' \t\n'),
remove_chars(normalized_actual_text, ' \t\n')):
return [test_failures.FailureTextMismatch()]
# Space-only mismatch
if not self._port.do_text_results_differ(
remove_chars(expected_text, ' \t'),
remove_chars(normalized_actual_text, ' \t')):
return [test_failures.FailureSpacesAndTabsTextMismatch()]
# Newline-only mismatch
if not self._port.do_text_results_differ(
remove_chars(expected_text, '\n'),
remove_chars(normalized_actual_text, '\n')):
return [test_failures.FailureLineBreaksTextMismatch()]
# Spaces and newlines
return [test_failures.FailureSpaceTabLineBreakTextMismatch()]
def _mock_test_result(self, testname):
return test_results.TestResult(testname,
[test_failures.FailureTextMismatch()])
