def _compare_output_with_reference(self, reference_driver_output, actual_driver_output, reference_filename, mismatch):
has_stderr = reference_driver_output.has_stderr() or actual_driver_output.has_stderr()
failures = []
failures.extend(self._handle_error(actual_driver_output))
if failures:
# Don't continue any more if we already have crash or timeout.
return TestResult(self._test_name, failures, 0, has_stderr)
failures.extend(self._handle_error(reference_driver_output, reference_filename=reference_filename))
if failures:
return TestResult(self._test_name, failures, 0, has_stderr, pid=actual_driver_output.pid)
if not actual_driver_output.image_hash:
failures.append(test_failures.FailureReftestNoImageGenerated(reference_filename))
elif not reference_driver_output.image_hash:
failures.append(test_failures.FailureReftestNoReferenceImageGenerated(reference_filename))
elif mismatch:
if reference_driver_output.image_hash == actual_driver_output.image_hash:
failures.append(test_failures.FailureReftestMismatchDidNotOccur(reference_filename))
elif reference_driver_output.image_hash != actual_driver_output.image_hash:
diff, err_str = self._port.diff_image(reference_driver_output.image, actual_driver_output.image)
if diff:
failures.append(test_failures.FailureReftestMismatch(reference_filename))
elif err_str:
_log.error(err_str)
else:
_log.warning(" %s -> ref test hashes didn't match but diff passed", self._test_name)
return TestResult(self._test_name, failures, 0, has_stderr, pid=actual_driver_output.pid)
def test_reftest_image_missing(self):
failure = test_failures.FailureReftestNoImageGenerated()
failure.reference_filename = '/src/exists-expected.html'
files = {'/src/exists-expected.html': 'yup'}
written_files = self.run_test(failures=[failure], files=files)
self.assertEqual(written_files, {'/tmp/exists-expected.html': 'yup'})
failure = test_failures.FailureReftestNoReferenceImageGenerated()
failure.reference_filename = '/src/exists-expected.html'
files = {'/src/exists-expected.html': 'yup'}
written_files = self.run_test(failures=[failure], files=files)
self.assertEqual(written_files, {'/tmp/exists-expected.html': 'yup'})