def test_interpret_test_failures(self):
test_dict = test_run_results._interpret_test_failures(
[test_failures.FailureImageHashMismatch(diff_percent=0.42)])
self.assertEqual(test_dict['image_diff_percent'], 0.42)
test_dict = test_run_results._interpret_test_failures([
test_failures.FailureReftestMismatch(
self.port.abspath_for_test('foo/reftest-expected.html'))
])
self.assertIn('image_diff_percent', test_dict)
test_dict = test_run_results._interpret_test_failures([
test_failures.FailureReftestMismatchDidNotOccur(
self.port.abspath_for_test(
'foo/reftest-expected-mismatch.html'))
])
self.assertEqual(len(test_dict), 0)
test_dict = test_run_results._interpret_test_failures(
[test_failures.FailureMissingAudio()])
self.assertIn('is_missing_audio', test_dict)
test_dict = test_run_results._interpret_test_failures(
[test_failures.FailureMissingResult()])
self.assertIn('is_missing_text', test_dict)
test_dict = test_run_results._interpret_test_failures(
[test_failures.FailureMissingImage()])
self.assertIn('is_missing_image', test_dict)
test_dict = test_run_results._interpret_test_failures(
[test_failures.FailureMissingImageHash()])
self.assertIn('is_missing_image', test_dict)
def _compare_image(self, expected_driver_output, driver_output):
failures = []
# If we didn't produce a hash file, this test must be text-only.
if driver_output.image_hash is None:
return failures
if not expected_driver_output.image:
failures.append(test_failures.FailureMissingImage())
elif not expected_driver_output.image_hash:
failures.append(test_failures.FailureMissingImageHash())
elif driver_output.image_hash != expected_driver_output.image_hash:
diff_result = self._port.diff_image(expected_driver_output.image,
driver_output.image)
error_string = diff_result[2]
if error_string:
_log.warning(' %s : %s' % (self._test_name, error_string))
failures.append(test_failures.FailureImageHashMismatch())
driver_output.error = (driver_output.error
or '') + error_string
else:
driver_output.image_diff = diff_result[0]
if driver_output.image_diff:
failures.append(
test_failures.FailureImageHashMismatch(diff_result[1]))
else:
# See https://bugs.webkit.org/show_bug.cgi?id=69444 for why this isn't a full failure.
_log.warning(
' %s -> pixel hash failed (but diff passed)' %
self._test_name)
return failures
def _failure_types_from_actual_result(self, actual):
# FIXME: There doesn't seem to be a full list of all possible values of
# 'actual' anywhere. However JSONLayoutResultsGenerator.FAILURE_TO_CHAR
# is a useful reference as that's for "old" style results.json files
if actual == test_expectations.PASS:
return []
elif actual == test_expectations.TEXT:
return [test_failures.FailureTextMismatch()]
elif actual == test_expectations.IMAGE:
return [test_failures.FailureImageHashMismatch()]
elif actual == test_expectations.IMAGE_PLUS_TEXT:
return [
test_failures.FailureImageHashMismatch(),
test_failures.FailureTextMismatch()
]
elif actual == test_expectations.AUDIO:
return [test_failures.FailureAudioMismatch()]
elif actual == test_expectations.TIMEOUT:
return [test_failures.FailureTimeout()]
elif actual == test_expectations.CRASH:
# NOTE: We don't know what process crashed from the json, just that a process crashed.
return [test_failures.FailureCrash()]
elif actual == test_expectations.MISSING:
return [
test_failures.FailureMissingResult(),
test_failures.FailureMissingImageHash(),
test_failures.FailureMissingImage()
]
else:
log("Failed to handle: %s" % self._result_dict['actual'])
return []
def test_parse_layout_test_results(self):
failures = [
test_failures.FailureMissingResult(),
test_failures.FailureMissingImageHash(),
test_failures.FailureMissingImage()
]
testname = 'fast/repaint/no-caret-repaint-in-non-content-editable-element.html'
expected_results = [test_results.TestResult(testname, failures)]
results = ORWTResultsHTMLParser.parse_results_html(
self._example_results_html)
self.assertEqual(expected_results, results)
def _compare_image(self, driver_output, expected_driver_output):
failures = []
# If we didn't produce a hash file, this test must be text-only.
if driver_output.image_hash is None:
return failures
if not expected_driver_output.image:
failures.append(test_failures.FailureMissingImage())
elif not expected_driver_output.image_hash:
failures.append(test_failures.FailureMissingImageHash())
elif driver_output.image_hash != expected_driver_output.image_hash:
driver_output.image_diff = self._port.diff_image(
driver_output.image, expected_driver_output.image)
if driver_output.image_diff:
failures.append(test_failures.FailureImageHashMismatch())
return failures
def _failures_from_row(cls, row, table_title):
if table_title == cls.fail_key:
return cls._failures_from_fail_row(row)
if table_title == cls.crash_key:
return [test_failures.FailureCrash()]
if table_title == cls.webprocess_crash_key:
return [test_failures.FailureCrash(process_name="WebProcess")]
if table_title == cls.timeout_key:
return [test_failures.FailureTimeout()]
if table_title == cls.missing_key:
return [
test_failures.FailureMissingResult(),
test_failures.FailureMissingImageHash(),
test_failures.FailureMissingImage()
]
return None
def _failure_types_from_actual_result(self, actual):
# FIXME: There doesn't seem to be a full list of all possible values of
# 'actual' anywhere. However JSONLayoutResultsGenerator.FAILURE_TO_CHAR
# is a useful reference as that's for "old" style results.json files
#
# FIXME: TEXT, IMAGE_PLUS_TEXT, and AUDIO are obsolete but we keep them for
# now so that we can parse old results.json files.
if actual == test_expectations.PASS:
return []
elif actual == test_expectations.FAIL:
return [
test_failures.FailureTextMismatch(),
test_failures.FailureImageHashMismatch(),
test_failures.FailureAudioMismatch()
]
elif actual == test_expectations.TEXT:
return [test_failures.FailureTextMismatch()]
elif actual == test_expectations.IMAGE:
return [test_failures.FailureImageHashMismatch()]
elif actual == test_expectations.IMAGE_PLUS_TEXT:
return [
test_failures.FailureImageHashMismatch(),
test_failures.FailureTextMismatch()
]
elif actual == test_expectations.AUDIO:
return [test_failures.FailureAudioMismatch()]
elif actual == test_expectations.TIMEOUT:
return [test_failures.FailureTimeout()]
elif actual == test_expectations.CRASH:
# NOTE: We don't know what process crashed from the json, just that a process crashed.
return [test_failures.FailureCrash()]
elif actual == test_expectations.LEAK:
urls = []
for url_dict in self._result_dict['leaks']:
urls.append(url_dict['document'])
return [test_failures.FailureDocumentLeak(urls)]
elif actual == test_expectations.MISSING:
return [
test_failures.FailureMissingResult(),
test_failures.FailureMissingImageHash(),
test_failures.FailureMissingImage()
]
else:
_log.warning("Failed to handle: %s" % self._result_dict['actual'])
return []
def _failure_types_from_actual_result(self, actual):
# FIXME: There doesn't seem to be a full list of all possible values of
# 'actual' anywhere.
#
# FIXME: TEXT, IMAGE_PLUS_TEXT, and AUDIO are obsolete but we keep them for
# now so that we can parse old results.json files.
if actual == test_expectations.PASS:
return []
elif actual == test_expectations.FAIL:
return [
test_failures.FailureTextMismatch(),
test_failures.FailureImageHashMismatch(),
test_failures.FailureAudioMismatch()
]
elif actual == test_expectations.TEXT:
return [test_failures.FailureTextMismatch()]
elif actual == test_expectations.IMAGE:
return [test_failures.FailureImageHashMismatch()]
elif actual == test_expectations.IMAGE_PLUS_TEXT:
return [
test_failures.FailureImageHashMismatch(),
test_failures.FailureTextMismatch()
]
elif actual == test_expectations.AUDIO:
return [test_failures.FailureAudioMismatch()]
elif actual == test_expectations.TIMEOUT:
return [test_failures.FailureTimeout()]
elif actual == test_expectations.CRASH:
# NOTE: We don't know what process crashed from the json, just that a process crashed.
return [test_failures.FailureCrash()]
elif actual == test_expectations.MISSING:
return [
test_failures.FailureMissingResult(),
test_failures.FailureMissingImageHash(),
test_failures.FailureMissingImage()
]
else:
_log.warning("Failed to handle: %s" % self._result_dict['actual'])
return []
