def test_interpret_test_failures(self): test_dict = test_run_results._interpret_test_failures( [test_failures.FailureImageHashMismatch(diff_percent=0.42)]) self.assertEqual(test_dict['image_diff_percent'], 0.42) test_dict = test_run_results._interpret_test_failures([ test_failures.FailureReftestMismatch( self.port.abspath_for_test('foo/reftest-expected.html')) ]) self.assertIn('image_diff_percent', test_dict) test_dict = test_run_results._interpret_test_failures([ test_failures.FailureReftestMismatchDidNotOccur( self.port.abspath_for_test( 'foo/reftest-expected-mismatch.html')) ]) self.assertEqual(len(test_dict), 0) test_dict = test_run_results._interpret_test_failures( [test_failures.FailureMissingAudio()]) self.assertIn('is_missing_audio', test_dict) test_dict = test_run_results._interpret_test_failures( [test_failures.FailureMissingResult()]) self.assertIn('is_missing_text', test_dict) test_dict = test_run_results._interpret_test_failures( [test_failures.FailureMissingImage()]) self.assertIn('is_missing_image', test_dict) test_dict = test_run_results._interpret_test_failures( [test_failures.FailureMissingImageHash()]) self.assertIn('is_missing_image', test_dict)
def _compare_image(self, expected_driver_output, driver_output): failures = [] # If we didn't produce a hash file, this test must be text-only. if driver_output.image_hash is None: return failures if not expected_driver_output.image: failures.append(test_failures.FailureMissingImage()) elif not expected_driver_output.image_hash: failures.append(test_failures.FailureMissingImageHash()) elif driver_output.image_hash != expected_driver_output.image_hash: diff_result = self._port.diff_image(expected_driver_output.image, driver_output.image) error_string = diff_result[2] if error_string: _log.warning(' %s : %s' % (self._test_name, error_string)) failures.append(test_failures.FailureImageHashMismatch()) driver_output.error = (driver_output.error or '') + error_string else: driver_output.image_diff = diff_result[0] if driver_output.image_diff: failures.append( test_failures.FailureImageHashMismatch(diff_result[1])) else: # See https://bugs.webkit.org/show_bug.cgi?id=69444 for why this isn't a full failure. _log.warning( ' %s -> pixel hash failed (but diff passed)' % self._test_name) return failures
def _failure_types_from_actual_result(self, actual): # FIXME: There doesn't seem to be a full list of all possible values of # 'actual' anywhere. However JSONLayoutResultsGenerator.FAILURE_TO_CHAR # is a useful reference as that's for "old" style results.json files if actual == test_expectations.PASS: return [] elif actual == test_expectations.TEXT: return [test_failures.FailureTextMismatch()] elif actual == test_expectations.IMAGE: return [test_failures.FailureImageHashMismatch()] elif actual == test_expectations.IMAGE_PLUS_TEXT: return [ test_failures.FailureImageHashMismatch(), test_failures.FailureTextMismatch() ] elif actual == test_expectations.AUDIO: return [test_failures.FailureAudioMismatch()] elif actual == test_expectations.TIMEOUT: return [test_failures.FailureTimeout()] elif actual == test_expectations.CRASH: # NOTE: We don't know what process crashed from the json, just that a process crashed. return [test_failures.FailureCrash()] elif actual == test_expectations.MISSING: return [ test_failures.FailureMissingResult(), test_failures.FailureMissingImageHash(), test_failures.FailureMissingImage() ] else: log("Failed to handle: %s" % self._result_dict['actual']) return []
def test_parse_layout_test_results(self): failures = [ test_failures.FailureMissingResult(), test_failures.FailureMissingImageHash(), test_failures.FailureMissingImage() ] testname = 'fast/repaint/no-caret-repaint-in-non-content-editable-element.html' expected_results = [test_results.TestResult(testname, failures)] results = ORWTResultsHTMLParser.parse_results_html( self._example_results_html) self.assertEqual(expected_results, results)
def _compare_image(self, driver_output, expected_driver_output): failures = [] # If we didn't produce a hash file, this test must be text-only. if driver_output.image_hash is None: return failures if not expected_driver_output.image: failures.append(test_failures.FailureMissingImage()) elif not expected_driver_output.image_hash: failures.append(test_failures.FailureMissingImageHash()) elif driver_output.image_hash != expected_driver_output.image_hash: driver_output.image_diff = self._port.diff_image( driver_output.image, expected_driver_output.image) if driver_output.image_diff: failures.append(test_failures.FailureImageHashMismatch()) return failures
def _failures_from_row(cls, row, table_title): if table_title == cls.fail_key: return cls._failures_from_fail_row(row) if table_title == cls.crash_key: return [test_failures.FailureCrash()] if table_title == cls.webprocess_crash_key: return [test_failures.FailureCrash(process_name="WebProcess")] if table_title == cls.timeout_key: return [test_failures.FailureTimeout()] if table_title == cls.missing_key: return [ test_failures.FailureMissingResult(), test_failures.FailureMissingImageHash(), test_failures.FailureMissingImage() ] return None
def _failure_types_from_actual_result(self, actual): # FIXME: There doesn't seem to be a full list of all possible values of # 'actual' anywhere. However JSONLayoutResultsGenerator.FAILURE_TO_CHAR # is a useful reference as that's for "old" style results.json files # # FIXME: TEXT, IMAGE_PLUS_TEXT, and AUDIO are obsolete but we keep them for # now so that we can parse old results.json files. if actual == test_expectations.PASS: return [] elif actual == test_expectations.FAIL: return [ test_failures.FailureTextMismatch(), test_failures.FailureImageHashMismatch(), test_failures.FailureAudioMismatch() ] elif actual == test_expectations.TEXT: return [test_failures.FailureTextMismatch()] elif actual == test_expectations.IMAGE: return [test_failures.FailureImageHashMismatch()] elif actual == test_expectations.IMAGE_PLUS_TEXT: return [ test_failures.FailureImageHashMismatch(), test_failures.FailureTextMismatch() ] elif actual == test_expectations.AUDIO: return [test_failures.FailureAudioMismatch()] elif actual == test_expectations.TIMEOUT: return [test_failures.FailureTimeout()] elif actual == test_expectations.CRASH: # NOTE: We don't know what process crashed from the json, just that a process crashed. return [test_failures.FailureCrash()] elif actual == test_expectations.LEAK: urls = [] for url_dict in self._result_dict['leaks']: urls.append(url_dict['document']) return [test_failures.FailureDocumentLeak(urls)] elif actual == test_expectations.MISSING: return [ test_failures.FailureMissingResult(), test_failures.FailureMissingImageHash(), test_failures.FailureMissingImage() ] else: _log.warning("Failed to handle: %s" % self._result_dict['actual']) return []
def _failure_types_from_actual_result(self, actual): # FIXME: There doesn't seem to be a full list of all possible values of # 'actual' anywhere. # # FIXME: TEXT, IMAGE_PLUS_TEXT, and AUDIO are obsolete but we keep them for # now so that we can parse old results.json files. if actual == test_expectations.PASS: return [] elif actual == test_expectations.FAIL: return [ test_failures.FailureTextMismatch(), test_failures.FailureImageHashMismatch(), test_failures.FailureAudioMismatch() ] elif actual == test_expectations.TEXT: return [test_failures.FailureTextMismatch()] elif actual == test_expectations.IMAGE: return [test_failures.FailureImageHashMismatch()] elif actual == test_expectations.IMAGE_PLUS_TEXT: return [ test_failures.FailureImageHashMismatch(), test_failures.FailureTextMismatch() ] elif actual == test_expectations.AUDIO: return [test_failures.FailureAudioMismatch()] elif actual == test_expectations.TIMEOUT: return [test_failures.FailureTimeout()] elif actual == test_expectations.CRASH: # NOTE: We don't know what process crashed from the json, just that a process crashed. return [test_failures.FailureCrash()] elif actual == test_expectations.MISSING: return [ test_failures.FailureMissingResult(), test_failures.FailureMissingImageHash(), test_failures.FailureMissingImage() ] else: _log.warning("Failed to handle: %s" % self._result_dict['actual']) return []