def test_interpret_test_failures(self): test_dict = test_run_results._interpret_test_failures([ test_failures.FailureReftestMismatchDidNotOccur( self._actual_output, self._expected_output, self.port.abspath_for_test( 'foo/reftest-expected-mismatch.html')) ]) self.assertEqual(len(test_dict), 0) test_dict = test_run_results._interpret_test_failures([ test_failures.FailureMissingAudio(self._actual_output, self._expected_output) ]) self.assertIn('is_missing_audio', test_dict) test_dict = test_run_results._interpret_test_failures([ test_failures.FailureMissingResult(self._actual_output, self._expected_output) ]) self.assertIn('is_missing_text', test_dict) test_dict = test_run_results._interpret_test_failures([ test_failures.FailureMissingImage(self._actual_output, self._expected_output) ]) self.assertIn('is_missing_image', test_dict) test_dict = test_run_results._interpret_test_failures([ test_failures.FailureMissingImageHash(self._actual_output, self._expected_output) ]) self.assertIn('is_missing_image', test_dict)
def _check_extra_and_missing_baselines(self, expected_driver_output, driver_output): failures = [] if driver_output.text: if self._is_all_pass_testharness_text_not_needing_baseline( driver_output.text): if self._report_extra_baseline( driver_output, '.txt', 'is a all-pass testharness test'): # TODO(wangxianzhu): Make this a failure. pass elif testharness_results.is_testharness_output(driver_output.text): # We only need -expected.txt for a testharness test when we # expect it to fail or produce additional console output (when # -expected.txt is optional), so don't report missing # -expected.txt for testharness tests. pass elif self._reference_files: # A reftest's -expected.txt is optional. TODO(wangxianzhu): May # let reftests use the standard baseline existence rule. pass elif not expected_driver_output.text: failures.append(test_failures.FailureMissingResult()) elif self._report_extra_baseline(driver_output, '.txt', 'does not produce text result'): failures.append(test_failures.FailureTextMismatch()) if driver_output.image_hash: if self._reference_files: if self._report_extra_baseline(driver_output, '.png', 'is a reftest'): # TODO(wangxianzhu): Make this a failure. pass else: if not expected_driver_output.image: failures.append(test_failures.FailureMissingImage()) if not expected_driver_output.image_hash: failures.append(test_failures.FailureMissingImageHash()) elif self._report_extra_baseline(driver_output, '.png', 'does not produce image result'): failures.append(test_failures.FailureImageHashMismatch()) if driver_output.audio: if not expected_driver_output.audio: failures.append(test_failures.FailureMissingAudio()) elif self._report_extra_baseline(driver_output, '.wav', 'does not produce audio result'): failures.append(test_failures.FailureAudioMismatch()) return failures
def _compare_text(self, expected_text, actual_text): if not actual_text: return [] if not expected_text: return [test_failures.FailureMissingResult()] normalized_actual_text = self._get_normalized_output_text(actual_text) # Assuming expected_text is already normalized. if not self._port.do_text_results_differ(expected_text, normalized_actual_text): return [] # Determine the text mismatch type def remove_chars(text, chars): for char in chars: text = text.replace(char, '') return text def remove_ng_text(results): processed = re.sub(r'LayoutNG(BlockFlow|ListItem|TableCell)', r'Layout\1', results) # LayoutTableCaption doesn't override LayoutBlockFlow::GetName, so # render tree dumps have "LayoutBlockFlow" for captions. processed = re.sub('LayoutNGTableCaption', 'LayoutBlockFlow', processed) return processed def is_ng_name_mismatch(expected, actual): if not re.search( "LayoutNG(BlockFlow|ListItem|TableCaption|TableCell)", actual): return False if not self._is_render_tree(actual) and not self._is_layer_tree( actual): return False # There's a mix of NG and legacy names in both expected and actual, # so just remove NG from both. return not self._port.do_text_results_differ( remove_ng_text(expected), remove_ng_text(actual)) # LayoutNG name mismatch (e.g., LayoutBlockFlow vs. LayoutNGBlockFlow) # is treated as pass if is_ng_name_mismatch(expected_text, normalized_actual_text): return [] # General text mismatch if self._port.do_text_results_differ( remove_chars(expected_text, ' \t\n'), remove_chars(normalized_actual_text, ' \t\n')): return [test_failures.FailureTextMismatch()] # Space-only mismatch if not self._port.do_text_results_differ( remove_chars(expected_text, ' \t'), remove_chars(normalized_actual_text, ' \t')): return [test_failures.FailureSpacesAndTabsTextMismatch()] # Newline-only mismatch if not self._port.do_text_results_differ( remove_chars(expected_text, '\n'), remove_chars(normalized_actual_text, '\n')): return [test_failures.FailureLineBreaksTextMismatch()] # Spaces and newlines return [test_failures.FailureSpaceTabLineBreakTextMismatch()]