def test_failing_tests_message(self): # Needed to define port_name, used in AbstractEarlyWarningSystem.__init__ class TestEWS(AbstractEarlyWarningSystem): port_name = "win" # Needs to be a port which port/factory understands. ews = TestEWS() ews.bind_to_tool(MockTool()) ews._options = MockOptions(port=None, confirm=False) OutputCapture().assert_outputs( self, ews.begin_work_queue, expected_logs=self._default_begin_work_queue_logs(ews.name)) task = Mock() task.results_from_patch_test_run = lambda a: LayoutTestResults( [ test_results.TestResult("foo.html", failures= [test_failures.FailureTextMismatch()]), test_results.TestResult( "bar.html", failures=[test_failures.FailureTextMismatch()]) ], did_exceed_test_failure_limit=False) task.results_from_test_run_without_patch = lambda a: LayoutTestResults( [], did_exceed_test_failure_limit=False) patch = ews._tool.bugs.fetch_attachment(10000) self.assertMultiLineEqual(ews._failing_tests_message(task, patch), "New failing tests:\nbar.html\nfoo.html")
def _failure_types_from_actual_result(self, actual): # FIXME: There doesn't seem to be a full list of all possible values of # 'actual' anywhere. However JSONLayoutResultsGenerator.FAILURE_TO_CHAR # is a useful reference as that's for "old" style results.json files if actual == test_expectations.PASS: return [] elif actual == test_expectations.TEXT: return [test_failures.FailureTextMismatch()] elif actual == test_expectations.IMAGE: return [test_failures.FailureImageHashMismatch()] elif actual == test_expectations.IMAGE_PLUS_TEXT: return [ test_failures.FailureImageHashMismatch(), test_failures.FailureTextMismatch() ] elif actual == test_expectations.AUDIO: return [test_failures.FailureAudioMismatch()] elif actual == test_expectations.TIMEOUT: return [test_failures.FailureTimeout()] elif actual == test_expectations.CRASH: # NOTE: We don't know what process crashed from the json, just that a process crashed. return [test_failures.FailureCrash()] elif actual == test_expectations.MISSING: return [ test_failures.FailureMissingResult(), test_failures.FailureMissingImageHash(), test_failures.FailureMissingImage() ] else: log("Failed to handle: %s" % self._result_dict['actual']) return []
def test_failing_tests_message(self): ews = TestEWS() results = lambda a: LayoutTestResults([ test_results.TestResult( "foo.html", failures=[test_failures.FailureTextMismatch()]), test_results.TestResult( "bar.html", failures=[test_failures.FailureTextMismatch()]) ], did_exceed_test_failure_limit= False) message = "New failing tests:\nfoo.html\nbar.html" self._test_message(ews, results, message)
def results_from_patch_test_run(self, patch): return LayoutTestResults([ test_results.TestResult( "mock_test_name.html", failures=[test_failures.FailureTextMismatch()]) ], did_exceed_test_failure_limit=False)
def test_basic(self): expected_results = [ test_results.TestResult("svg/dynamic-updates/SVGFEDropShadowElement-dom-stdDeviation-attr.html", [test_failures.FailureImageHashMismatch()], 0), test_results.TestResult("fast/dom/prototype-inheritance.html", [test_failures.FailureTextMismatch(), test_failures.FailureImageHashMismatch(), test_failures.FailureAudioMismatch()], 0), ] parsed_results = ParsedJSONResults(self._example_full_results_json) self.assertEqual(expected_results, parsed_results.test_results()) self.assertTrue(parsed_results.did_exceed_test_failure_limit())
def _failure_types_from_actual_result(self, actual): # FIXME: There doesn't seem to be a full list of all possible values of # 'actual' anywhere. However JSONLayoutResultsGenerator.FAILURE_TO_CHAR # is a useful reference as that's for "old" style results.json files # # FIXME: TEXT, IMAGE_PLUS_TEXT, and AUDIO are obsolete but we keep them for # now so that we can parse old results.json files. if actual == test_expectations.PASS: return [] elif actual == test_expectations.FAIL: return [ test_failures.FailureTextMismatch(), test_failures.FailureImageHashMismatch(), test_failures.FailureAudioMismatch() ] elif actual == test_expectations.TEXT: return [test_failures.FailureTextMismatch()] elif actual == test_expectations.IMAGE: return [test_failures.FailureImageHashMismatch()] elif actual == test_expectations.IMAGE_PLUS_TEXT: return [ test_failures.FailureImageHashMismatch(), test_failures.FailureTextMismatch() ] elif actual == test_expectations.AUDIO: return [test_failures.FailureAudioMismatch()] elif actual == test_expectations.TIMEOUT: return [test_failures.FailureTimeout()] elif actual == test_expectations.CRASH: # NOTE: We don't know what process crashed from the json, just that a process crashed. return [test_failures.FailureCrash()] elif actual == test_expectations.LEAK: urls = [] for url_dict in self._result_dict['leaks']: urls.append(url_dict['document']) return [test_failures.FailureDocumentLeak(urls)] elif actual == test_expectations.MISSING: return [ test_failures.FailureMissingResult(), test_failures.FailureMissingImageHash(), test_failures.FailureMissingImage() ] else: _log.warning("Failed to handle: %s" % self._result_dict['actual']) return []
def _compare_text(self, actual_text, expected_text): failures = [] if (expected_text and actual_text and # Assuming expected_text is already normalized. self._port.compare_text(self._get_normalized_output_text(actual_text), expected_text)): failures.append(test_failures.FailureTextMismatch()) elif actual_text and not expected_text: failures.append(test_failures.FailureMissingResult()) return failures
def _compare_text(self, expected_text, actual_text): failures = [] if (expected_text is not None and actual_text is not None and # Assuming expected_text is already normalized. self._port.do_text_results_differ(expected_text, self._get_normalized_output_text(actual_text))): failures.append(test_failures.FailureTextMismatch()) elif actual_text is not None and expected_text is None: failures.append(test_failures.FailureMissingResult()) return failures
def _compare_text(self, expected_text, actual_text): if not actual_text: return [] if not expected_text: return [test_failures.FailureMissingResult()] normalized_actual_text = self._get_normalized_output_text(actual_text) # Assuming expected_text is already normalized. if not self._port.do_text_results_differ(expected_text, normalized_actual_text): return [] # Determine the text mismatch type def remove_chars(text, chars): for char in chars: text = text.replace(char, '') return text def is_ng_name_mismatch(expected, actual): if 'LayoutNGBlockFlow' not in actual: return False if not self._is_render_tree(actual) and not self._is_layer_tree( actual): return False processed = actual.replace('LayoutNGBlockFlow', 'LayoutBlockFlow').replace( 'LayoutNGListItem', 'LayoutListItem') return not self._port.do_text_results_differ(expected, processed) # LayoutNG name mismatch (e.g., LayoutBlockFlow vs. LayoutNGBlockFlow) # is treated as pass if is_ng_name_mismatch(expected_text, normalized_actual_text): return [] # General text mismatch if self._port.do_text_results_differ( remove_chars(expected_text, ' \t\n'), remove_chars(normalized_actual_text, ' \t\n')): return [test_failures.FailureTextMismatch()] # Space-only mismatch if not self._port.do_text_results_differ( remove_chars(expected_text, ' \t'), remove_chars(normalized_actual_text, ' \t')): return [test_failures.FailureSpacesAndTabsTextMismatch()] # Newline-only mismatch if not self._port.do_text_results_differ( remove_chars(expected_text, '\n'), remove_chars(normalized_actual_text, '\n')): return [test_failures.FailureLineBreaksTextMismatch()] # Spaces and newlines return [test_failures.FailureSpaceTabLineBreakTextMismatch()]
def test_basic(self): expected_results = [ test_results.TestResult( "svg/dynamic-updates/SVGFEDropShadowElement-dom-stdDeviation-attr.html", [test_failures.FailureImageHashMismatch()], 0), test_results.TestResult("fast/dom/prototype-inheritance.html", [test_failures.FailureTextMismatch()], 0), ] results = ResultsJSONParser.parse_results_json( self._example_full_results_json) self.assertEqual(expected_results, results)
def test_basic(self): self.maxDiff = None expected_results = [ test_results.TestResult("svg/dynamic-updates/SVGFEDropShadowElement-dom-stdDeviation-attr.html", [test_failures.FailureImageHashMismatch()], 0), test_results.TestResult("fast/dom/prototype-inheritance.html", [test_failures.FailureTextMismatch(), test_failures.FailureImageHashMismatch(), test_failures.FailureAudioMismatch()], 0), test_results.TestResult("fast/dom/prototype-strawberry.html", [test_failures.FailureDocumentLeak(['file:///Volumes/Data/worker/webkit/build/LayoutTests/fast/dom/prototype-strawberry.html'])], 0), ] expected_results.sort(key=lambda result: result.test_name) parsed_results = ParsedJSONResults(self._example_full_results_json) self.assertEqual(expected_results, parsed_results.test_results()) self.assertTrue(parsed_results.did_exceed_test_failure_limit())
def _failure_types_from_actual_result(self, actual): # FIXME: There doesn't seem to be a full list of all possible values of # 'actual' anywhere. # # FIXME: TEXT, IMAGE_PLUS_TEXT, and AUDIO are obsolete but we keep them for # now so that we can parse old results.json files. if actual == test_expectations.PASS: return [] elif actual == test_expectations.FAIL: return [ test_failures.FailureTextMismatch(), test_failures.FailureImageHashMismatch(), test_failures.FailureAudioMismatch() ] elif actual == test_expectations.TEXT: return [test_failures.FailureTextMismatch()] elif actual == test_expectations.IMAGE: return [test_failures.FailureImageHashMismatch()] elif actual == test_expectations.IMAGE_PLUS_TEXT: return [ test_failures.FailureImageHashMismatch(), test_failures.FailureTextMismatch() ] elif actual == test_expectations.AUDIO: return [test_failures.FailureAudioMismatch()] elif actual == test_expectations.TIMEOUT: return [test_failures.FailureTimeout()] elif actual == test_expectations.CRASH: # NOTE: We don't know what process crashed from the json, just that a process crashed. return [test_failures.FailureCrash()] elif actual == test_expectations.MISSING: return [ test_failures.FailureMissingResult(), test_failures.FailureMissingImageHash(), test_failures.FailureMissingImage() ] else: _log.warning("Failed to handle: %s" % self._result_dict['actual']) return []
def _compare_text(self, expected_text, actual_text): failures = [] if self._options.ignore_render_tree_dump_results and actual_text and self._render_tree_dump_pattern.match(actual_text): return failures if (expected_text and actual_text and # Assuming expected_text is already normalized. self._port.do_text_results_differ(expected_text, self._get_normalized_output_text(actual_text))): failures.append(test_failures.FailureTextMismatch()) elif actual_text and not expected_text: failures.append(test_failures.FailureMissingResult()) elif not actual_text and expected_text: failures.append(test_failures.FailureNoOutput()) return failures
def get_result(test_name, result_type=test_expectations.PASS, run_time=0): failures = [] if result_type == test_expectations.TIMEOUT: failures = [test_failures.FailureTimeout()] elif result_type == test_expectations.AUDIO: failures = [test_failures.FailureAudioMismatch()] elif result_type == test_expectations.TEXT: failures = [test_failures.FailureTextMismatch()] elif result_type == test_expectations.IMAGE: failures = [test_failures.FailureImageHashMismatch()] elif result_type == test_expectations.CRASH: failures = [test_failures.FailureCrash()] elif result_type == test_expectations.LEAK: failures = [test_failures.FailureLeak()] return test_results.TestResult(test_name, failures=failures, test_run_time=run_time)
def _failures_from_fail_row(self, row): # Look at all anchors in this row, and guess what type # of new-run-webkit-test failures they equate to. failures = set() test_name = None for anchor in row.findAll("a"): anchor_text = unicode(anchor.string) if not test_name: test_name = anchor_text continue if anchor_text in ["expected image", "image diffs"] or '%' in anchor_text: failures.add(test_failures.FailureImageHashMismatch()) elif anchor_text in ["expected", "actual", "diff", "pretty diff"]: failures.add(test_failures.FailureTextMismatch()) else: log("Unhandled link text in results.html parsing: %s. Please file a bug against webkitpy." % anchor_text) # FIXME: Its possible the row contained no links due to ORWT brokeness. # We should probably assume some type of failure anyway. return failures
def _compare_text(self, expected_text, actual_text): if not actual_text: return [] if not expected_text: return [test_failures.FailureMissingResult()] normalized_actual_text = self._get_normalized_output_text(actual_text) # Assuming expected_text is already normalized. if not self._port.do_text_results_differ(expected_text, normalized_actual_text): return [] # Determine the text mismatch type def remove_chars(text, chars): for char in chars: text = text.replace(char, '') return text # General text mismatch if self._port.do_text_results_differ( remove_chars(expected_text, ' \t\n'), remove_chars(normalized_actual_text, ' \t\n')): return [test_failures.FailureTextMismatch()] # Space-only mismatch if not self._port.do_text_results_differ( remove_chars(expected_text, ' \t'), remove_chars(normalized_actual_text, ' \t')): return [test_failures.FailureSpacesAndTabsTextMismatch()] # Newline-only mismatch if not self._port.do_text_results_differ( remove_chars(expected_text, '\n'), remove_chars(normalized_actual_text, '\n')): return [test_failures.FailureLineBreaksTextMismatch()] # Spaces and newlines return [test_failures.FailureSpaceTabLineBreakTextMismatch()]
def _mock_test_result(self, testname): return test_results.TestResult(testname, [test_failures.FailureTextMismatch()])