def _failure_types_from_actual_result(self, actual):
# FIXME: There doesn't seem to be a full list of all possible values of
# 'actual' anywhere. However JSONLayoutResultsGenerator.FAILURE_TO_CHAR
# is a useful reference as that's for "old" style results.json files
if actual == test_expectations.PASS:
return []
elif actual == test_expectations.TEXT:
return [test_failures.FailureTextMismatch()]
elif actual == test_expectations.IMAGE:
return [test_failures.FailureImageHashMismatch()]
elif actual == test_expectations.IMAGE_PLUS_TEXT:
return [
test_failures.FailureImageHashMismatch(),
test_failures.FailureTextMismatch()
]
elif actual == test_expectations.AUDIO:
return [test_failures.FailureAudioMismatch()]
elif actual == test_expectations.TIMEOUT:
return [test_failures.FailureTimeout()]
elif actual == test_expectations.CRASH:
# NOTE: We don't know what process crashed from the json, just that a process crashed.
return [test_failures.FailureCrash()]
elif actual == test_expectations.MISSING:
return [
test_failures.FailureMissingResult(),
test_failures.FailureMissingImageHash(),
test_failures.FailureMissingImage()
]
else:
log("Failed to handle: %s" % self._result_dict['actual'])
return []
def _compare_audio(self, expected_audio, actual_audio):
failures = []
if (expected_audio and actual_audio and
self._port.do_audio_results_differ(expected_audio, actual_audio)):
failures.append(test_failures.FailureAudioMismatch())
elif actual_audio and not expected_audio:
failures.append(test_failures.FailureMissingAudio())
return failures
def _failure_types_from_actual_result(self, actual):
# FIXME: There doesn't seem to be a full list of all possible values of
# 'actual' anywhere. However JSONLayoutResultsGenerator.FAILURE_TO_CHAR
# is a useful reference as that's for "old" style results.json files
#
# FIXME: TEXT, IMAGE_PLUS_TEXT, and AUDIO are obsolete but we keep them for
# now so that we can parse old results.json files.
if actual == test_expectations.PASS:
return []
elif actual == test_expectations.FAIL:
return [
test_failures.FailureTextMismatch(),
test_failures.FailureImageHashMismatch(),
test_failures.FailureAudioMismatch()
]
elif actual == test_expectations.TEXT:
return [test_failures.FailureTextMismatch()]
elif actual == test_expectations.IMAGE:
return [test_failures.FailureImageHashMismatch()]
elif actual == test_expectations.IMAGE_PLUS_TEXT:
return [
test_failures.FailureImageHashMismatch(),
test_failures.FailureTextMismatch()
]
elif actual == test_expectations.AUDIO:
return [test_failures.FailureAudioMismatch()]
elif actual == test_expectations.TIMEOUT:
return [test_failures.FailureTimeout()]
elif actual == test_expectations.CRASH:
# NOTE: We don't know what process crashed from the json, just that a process crashed.
return [test_failures.FailureCrash()]
elif actual == test_expectations.LEAK:
urls = []
for url_dict in self._result_dict['leaks']:
urls.append(url_dict['document'])
return [test_failures.FailureDocumentLeak(urls)]
elif actual == test_expectations.MISSING:
return [
test_failures.FailureMissingResult(),
test_failures.FailureMissingImageHash(),
test_failures.FailureMissingImage()
]
else:
_log.warning("Failed to handle: %s" % self._result_dict['actual'])
return []
def get_result(test_name, result_type=test_expectations.PASS, run_time=0):
failures = []
if result_type == test_expectations.TIMEOUT:
failures = [test_failures.FailureTimeout()]
elif result_type == test_expectations.AUDIO:
failures = [test_failures.FailureAudioMismatch()]
elif result_type == test_expectations.CRASH:
failures = [test_failures.FailureCrash()]
return test_results.TestResult(test_name, failures=failures, test_run_time=run_time)
def _failure_types_from_actual_result(self, actual):
# FIXME: There doesn't seem to be a full list of all possible values of
# 'actual' anywhere.
#
# FIXME: TEXT, IMAGE_PLUS_TEXT, and AUDIO are obsolete but we keep them for
# now so that we can parse old results.json files.
if actual == test_expectations.PASS:
return []
elif actual == test_expectations.FAIL:
return [
test_failures.FailureTextMismatch(),
test_failures.FailureImageHashMismatch(),
test_failures.FailureAudioMismatch()
]
elif actual == test_expectations.TEXT:
return [test_failures.FailureTextMismatch()]
elif actual == test_expectations.IMAGE:
return [test_failures.FailureImageHashMismatch()]
elif actual == test_expectations.IMAGE_PLUS_TEXT:
return [
test_failures.FailureImageHashMismatch(),
test_failures.FailureTextMismatch()
]
elif actual == test_expectations.AUDIO:
return [test_failures.FailureAudioMismatch()]
elif actual == test_expectations.TIMEOUT:
return [test_failures.FailureTimeout()]
elif actual == test_expectations.CRASH:
# NOTE: We don't know what process crashed from the json, just that a process crashed.
return [test_failures.FailureCrash()]
elif actual == test_expectations.MISSING:
return [
test_failures.FailureMissingResult(),
test_failures.FailureMissingImageHash(),
test_failures.FailureMissingImage()
]
else:
_log.warning("Failed to handle: %s" % self._result_dict['actual'])
return []
def test_basic(self):
expected_results = [
test_results.TestResult(
"svg/dynamic-updates/SVGFEDropShadowElement-dom-stdDeviation-attr.html",
[test_failures.FailureImageHashMismatch()], 0),
test_results.TestResult("fast/dom/prototype-inheritance.html", [
test_failures.FailureTextMismatch(),
test_failures.FailureImageHashMismatch(),
test_failures.FailureAudioMismatch()
], 0),
]
results = ResultsJSONParser.parse_results_json(
self._example_full_results_json)
self.assertEqual(expected_results, results)
def get_result(test_name, result_type=test_expectations.PASS, run_time=0):
failures = []
if result_type == test_expectations.TIMEOUT:
failures = [test_failures.FailureTimeout()]
elif result_type == test_expectations.AUDIO:
failures = [test_failures.FailureAudioMismatch()]
elif result_type == test_expectations.CRASH:
failures = [test_failures.FailureCrash()]
elif result_type == test_expectations.LEAK:
failures = [
test_failures.FailureDocumentLeak(
['http://localhost:8000/failures/expected/leak.html'])
]
return test_results.TestResult(test_name,
failures=failures,
test_run_time=run_time)
def test_basic(self):
expected_results = [
test_results.TestResult(
"svg/dynamic-updates/SVGFEDropShadowElement-dom-stdDeviation-attr.html",
[test_failures.FailureImageHashMismatch()], 0),
test_results.TestResult("fast/dom/prototype-inheritance.html", [
test_failures.FailureTextMismatch(),
test_failures.FailureImageHashMismatch(),
test_failures.FailureAudioMismatch()
], 0),
test_results.TestResult("fast/dom/prototype-strawberry.html", [
test_failures.FailureDocumentLeak([
'file:///Volumes/Data/slave/webkit/build/LayoutTests/fast/dom/prototype-strawberry.html'
])
], 0),
]
parsed_results = ParsedJSONResults(self._example_full_results_json)
self.assertEqual(expected_results, parsed_results.test_results())
self.assertTrue(parsed_results.did_exceed_test_failure_limit())