def test_interpret_test_failures(self): test_dict = test_run_results._interpret_test_failures( [test_failures.FailureImageHashMismatch(diff_percent=0.42)]) self.assertEqual(test_dict['image_diff_percent'], 0.42) test_dict = test_run_results._interpret_test_failures([ test_failures.FailureReftestMismatch( self.port.abspath_for_test('foo/reftest-expected.html')) ]) self.assertIn('image_diff_percent', test_dict) test_dict = test_run_results._interpret_test_failures([ test_failures.FailureReftestMismatchDidNotOccur( self.port.abspath_for_test( 'foo/reftest-expected-mismatch.html')) ]) self.assertEqual(len(test_dict), 0) test_dict = test_run_results._interpret_test_failures( [test_failures.FailureMissingAudio()]) self.assertIn('is_missing_audio', test_dict) test_dict = test_run_results._interpret_test_failures( [test_failures.FailureMissingResult()]) self.assertIn('is_missing_text', test_dict) test_dict = test_run_results._interpret_test_failures( [test_failures.FailureMissingImage()]) self.assertIn('is_missing_image', test_dict) test_dict = test_run_results._interpret_test_failures( [test_failures.FailureMissingImageHash()]) self.assertIn('is_missing_image', test_dict)
def test_parse_layout_test_results(self): failures = [test_failures.FailureMissingResult(), test_failures.FailureMissingImageHash(), test_failures.FailureMissingImage()] testname = 'fast/repaint/no-caret-repaint-in-non-content-editable-element.html' expected_results = [test_results.TestResult(testname, failures)] results = ORWTResultsHTMLParser.parse_results_html(self._example_results_html) self.assertEqual(expected_results, results)
def _failure_types_from_actual_result(self, actual): # FIXME: There doesn't seem to be a full list of all possible values of # 'actual' anywhere. However JSONLayoutResultsGenerator.FAILURE_TO_CHAR # is a useful reference as that's for "old" style results.json files if actual == test_expectations.PASS: return [] elif actual == test_expectations.TEXT: return [test_failures.FailureTextMismatch()] elif actual == test_expectations.IMAGE: return [test_failures.FailureImageHashMismatch()] elif actual == test_expectations.IMAGE_PLUS_TEXT: return [ test_failures.FailureImageHashMismatch(), test_failures.FailureTextMismatch() ] elif actual == test_expectations.AUDIO: return [test_failures.FailureAudioMismatch()] elif actual == test_expectations.TIMEOUT: return [test_failures.FailureTimeout()] elif actual == test_expectations.CRASH: # NOTE: We don't know what process crashed from the json, just that a process crashed. return [test_failures.FailureCrash()] elif actual == test_expectations.MISSING: return [ test_failures.FailureMissingResult(), test_failures.FailureMissingImageHash(), test_failures.FailureMissingImage() ] else: log("Failed to handle: %s" % self._result_dict['actual']) return []
def _compare_text(self, expected_text, actual_text): failures = [] if (expected_text is not None and actual_text is not None and # Assuming expected_text is already normalized. self._port.do_text_results_differ(expected_text, self._get_normalized_output_text(actual_text))): failures.append(test_failures.FailureTextMismatch()) elif actual_text is not None and expected_text is None: failures.append(test_failures.FailureMissingResult()) return failures
def _compare_text(self, actual_text, expected_text): failures = [] if (expected_text and actual_text and # Assuming expected_text is already normalized. self._port.compare_text(self._get_normalized_output_text(actual_text), expected_text)): failures.append(test_failures.FailureTextMismatch()) elif actual_text and not expected_text: failures.append(test_failures.FailureMissingResult()) return failures
def _compare_text(self, expected_text, actual_text): if not actual_text: return [] if not expected_text: return [test_failures.FailureMissingResult()] normalized_actual_text = self._get_normalized_output_text(actual_text) # Assuming expected_text is already normalized. if not self._port.do_text_results_differ(expected_text, normalized_actual_text): return [] # Determine the text mismatch type def remove_chars(text, chars): for char in chars: text = text.replace(char, '') return text def is_ng_name_mismatch(expected, actual): if 'LayoutNGBlockFlow' not in actual: return False if not self._is_render_tree(actual) and not self._is_layer_tree( actual): return False processed = actual.replace('LayoutNGBlockFlow', 'LayoutBlockFlow').replace( 'LayoutNGListItem', 'LayoutListItem') return not self._port.do_text_results_differ(expected, processed) # LayoutNG name mismatch (e.g., LayoutBlockFlow vs. LayoutNGBlockFlow) # is treated as pass if is_ng_name_mismatch(expected_text, normalized_actual_text): return [] # General text mismatch if self._port.do_text_results_differ( remove_chars(expected_text, ' \t\n'), remove_chars(normalized_actual_text, ' \t\n')): return [test_failures.FailureTextMismatch()] # Space-only mismatch if not self._port.do_text_results_differ( remove_chars(expected_text, ' \t'), remove_chars(normalized_actual_text, ' \t')): return [test_failures.FailureSpacesAndTabsTextMismatch()] # Newline-only mismatch if not self._port.do_text_results_differ( remove_chars(expected_text, '\n'), remove_chars(normalized_actual_text, '\n')): return [test_failures.FailureLineBreaksTextMismatch()] # Spaces and newlines return [test_failures.FailureSpaceTabLineBreakTextMismatch()]
def _failures_from_row(cls, row, table_title): if table_title == cls.fail_key: return cls._failures_from_fail_row(row) if table_title == cls.crash_key: return [test_failures.FailureCrash()] if table_title == cls.webprocess_crash_key: return [test_failures.FailureCrash(process_name="WebProcess")] if table_title == cls.timeout_key: return [test_failures.FailureTimeout()] if table_title == cls.missing_key: return [test_failures.FailureMissingResult(), test_failures.FailureMissingImageHash(), test_failures.FailureMissingImage()] return None
def _compare_text(self, expected_text, actual_text): failures = [] if self._options.ignore_render_tree_dump_results and actual_text and self._render_tree_dump_pattern.match(actual_text): return failures if (expected_text and actual_text and # Assuming expected_text is already normalized. self._port.do_text_results_differ(expected_text, self._get_normalized_output_text(actual_text))): failures.append(test_failures.FailureTextMismatch()) elif actual_text and not expected_text: failures.append(test_failures.FailureMissingResult()) elif not actual_text and expected_text: failures.append(test_failures.FailureNoOutput()) return failures
def _failure_types_from_actual_result(self, actual): # FIXME: There doesn't seem to be a full list of all possible values of # 'actual' anywhere. However JSONLayoutResultsGenerator.FAILURE_TO_CHAR # is a useful reference as that's for "old" style results.json files # # FIXME: TEXT, IMAGE_PLUS_TEXT, and AUDIO are obsolete but we keep them for # now so that we can parse old results.json files. if actual == test_expectations.PASS: return [] elif actual == test_expectations.FAIL: return [ test_failures.FailureTextMismatch(), test_failures.FailureImageHashMismatch(), test_failures.FailureAudioMismatch() ] elif actual == test_expectations.TEXT: return [test_failures.FailureTextMismatch()] elif actual == test_expectations.IMAGE: return [test_failures.FailureImageHashMismatch()] elif actual == test_expectations.IMAGE_PLUS_TEXT: return [ test_failures.FailureImageHashMismatch(), test_failures.FailureTextMismatch() ] elif actual == test_expectations.AUDIO: return [test_failures.FailureAudioMismatch()] elif actual == test_expectations.TIMEOUT: return [test_failures.FailureTimeout()] elif actual == test_expectations.CRASH: # NOTE: We don't know what process crashed from the json, just that a process crashed. return [test_failures.FailureCrash()] elif actual == test_expectations.LEAK: urls = [] for url_dict in self._result_dict['leaks']: urls.append(url_dict['document']) return [test_failures.FailureDocumentLeak(urls)] elif actual == test_expectations.MISSING: return [ test_failures.FailureMissingResult(), test_failures.FailureMissingImageHash(), test_failures.FailureMissingImage() ] else: _log.warning("Failed to handle: %s" % self._result_dict['actual']) return []
def _failure_types_from_actual_result(self, actual): # FIXME: There doesn't seem to be a full list of all possible values of # 'actual' anywhere. # # FIXME: TEXT, IMAGE_PLUS_TEXT, and AUDIO are obsolete but we keep them for # now so that we can parse old results.json files. if actual == test_expectations.PASS: return [] elif actual == test_expectations.FAIL: return [ test_failures.FailureTextMismatch(), test_failures.FailureImageHashMismatch(), test_failures.FailureAudioMismatch() ] elif actual == test_expectations.TEXT: return [test_failures.FailureTextMismatch()] elif actual == test_expectations.IMAGE: return [test_failures.FailureImageHashMismatch()] elif actual == test_expectations.IMAGE_PLUS_TEXT: return [ test_failures.FailureImageHashMismatch(), test_failures.FailureTextMismatch() ] elif actual == test_expectations.AUDIO: return [test_failures.FailureAudioMismatch()] elif actual == test_expectations.TIMEOUT: return [test_failures.FailureTimeout()] elif actual == test_expectations.CRASH: # NOTE: We don't know what process crashed from the json, just that a process crashed. return [test_failures.FailureCrash()] elif actual == test_expectations.MISSING: return [ test_failures.FailureMissingResult(), test_failures.FailureMissingImageHash(), test_failures.FailureMissingImage() ] else: _log.warning("Failed to handle: %s" % self._result_dict['actual']) return []
def _compare_text(self, expected_text, actual_text): if not actual_text: return [] if not expected_text: return [test_failures.FailureMissingResult()] normalized_actual_text = self._get_normalized_output_text(actual_text) # Assuming expected_text is already normalized. if not self._port.do_text_results_differ(expected_text, normalized_actual_text): return [] # Determine the text mismatch type def remove_chars(text, chars): for char in chars: text = text.replace(char, '') return text # General text mismatch if self._port.do_text_results_differ( remove_chars(expected_text, ' \t\n'), remove_chars(normalized_actual_text, ' \t\n')): return [test_failures.FailureTextMismatch()] # Space-only mismatch if not self._port.do_text_results_differ( remove_chars(expected_text, ' \t'), remove_chars(normalized_actual_text, ' \t')): return [test_failures.FailureSpacesAndTabsTextMismatch()] # Newline-only mismatch if not self._port.do_text_results_differ( remove_chars(expected_text, '\n'), remove_chars(normalized_actual_text, '\n')): return [test_failures.FailureLineBreaksTextMismatch()] # Spaces and newlines return [test_failures.FailureSpaceTabLineBreakTextMismatch()]