def _compare_output_with_reference(self, reference_driver_output, actual_driver_output, reference_filename, mismatch):
total_test_time = reference_driver_output.test_time + actual_driver_output.test_time
has_stderr = reference_driver_output.has_stderr() or actual_driver_output.has_stderr()
failures = []
failures.extend(self._handle_error(actual_driver_output))
if failures:
# Don't continue any more if we already have crash or timeout.
return TestResult(self._test_name, failures, total_test_time, has_stderr)
failures.extend(self._handle_error(reference_driver_output, reference_filename=reference_filename))
if failures:
return TestResult(self._test_name, failures, total_test_time, has_stderr, pid=actual_driver_output.pid)
if not reference_driver_output.image_hash and not actual_driver_output.image_hash:
failures.append(test_failures.FailureReftestNoImagesGenerated(reference_filename))
elif mismatch:
# Calling image_hash is considered unnecessary for expected mismatch ref tests.
if reference_driver_output.image_hash == actual_driver_output.image_hash:
failures.append(test_failures.FailureReftestMismatchDidNotOccur(reference_filename))
elif reference_driver_output.image_hash != actual_driver_output.image_hash:
# ImageDiff has a hard coded color distance threshold even though tolerance=0 is specified.
diff_result = self._port.diff_image(reference_driver_output.image, actual_driver_output.image, tolerance=0)
error_string = diff_result[2]
if error_string:
_log.warning(' %s : %s' % (self._test_name, error_string))
failures.append(test_failures.FailureReftestMismatch(reference_filename))
actual_driver_output.error = (actual_driver_output.error or '') + error_string
elif diff_result[0]:
failures.append(test_failures.FailureReftestMismatch(reference_filename))
return TestResult(self._test_name, failures, total_test_time, has_stderr, pid=actual_driver_output.pid)
def _compare_output_with_reference(self, reference_driver_output, actual_driver_output, reference_filename, mismatch):
total_test_time = reference_driver_output.test_time + actual_driver_output.test_time
has_stderr = reference_driver_output.has_stderr() or actual_driver_output.has_stderr()
failures = []
failures.extend(self._handle_error(actual_driver_output))
if failures:
# Don't continue any more if we already have crash or timeout.
return TestResult(self._test_name, failures, total_test_time, has_stderr)
failures.extend(self._handle_error(reference_driver_output, reference_filename=reference_filename))
if failures:
return TestResult(self._test_name, failures, total_test_time, has_stderr, pid=actual_driver_output.pid)
if not reference_driver_output.image_hash and not actual_driver_output.image_hash:
failures.append(test_failures.FailureReftestNoImagesGenerated(reference_filename))
elif mismatch:
if reference_driver_output.image_hash == actual_driver_output.image_hash:
diff, err_str = self._port.diff_image(reference_driver_output.image, actual_driver_output.image)
if not diff:
failures.append(test_failures.FailureReftestMismatchDidNotOccur(reference_filename))
elif err_str:
_log.error(err_str)
else:
_log.warning(" %s -> ref test hashes matched but diff failed" % self._test_name)
elif reference_driver_output.image_hash != actual_driver_output.image_hash:
diff, err_str = self._port.diff_image(reference_driver_output.image, actual_driver_output.image)
if diff:
failures.append(test_failures.FailureReftestMismatch(reference_filename))
elif err_str:
_log.error(err_str)
else:
_log.warning(" %s -> ref test hashes didn't match but diff passed" % self._test_name)
return TestResult(self._test_name, failures, total_test_time, has_stderr, pid=actual_driver_output.pid)
def _compare_output_with_reference(self, driver_output1, driver_output2,
reference_filename, mismatch):
total_test_time = driver_output1.test_time + driver_output2.test_time
has_stderr = driver_output1.has_stderr() or driver_output2.has_stderr()
failures = []
failures.extend(self._handle_error(driver_output1))
if failures:
# Don't continue any more if we already have crash or timeout.
return TestResult(self._test_name, failures, total_test_time,
has_stderr)
failures.extend(
self._handle_error(driver_output2,
reference_filename=reference_filename))
if failures:
return TestResult(self._test_name, failures, total_test_time,
has_stderr)
if not driver_output1.image_hash and not driver_output2.image_hash:
failures.append(
test_failures.FailureReftestNoImagesGenerated(
reference_filename))
elif mismatch:
if driver_output1.image_hash == driver_output2.image_hash:
failures.append(
test_failures.FailureReftestMismatchDidNotOccur(
reference_filename))
elif driver_output1.image_hash != driver_output2.image_hash:
failures.append(
test_failures.FailureReftestMismatch(reference_filename))
return TestResult(self._test_name, failures, total_test_time,
has_stderr)
