def _compare_output_with_reference(self, reference_driver_output, actual_driver_output, reference_filename, mismatch): total_test_time = reference_driver_output.test_time + actual_driver_output.test_time has_stderr = reference_driver_output.has_stderr() or actual_driver_output.has_stderr() failures = [] failures.extend(self._handle_error(actual_driver_output)) if failures: # Don't continue any more if we already have crash or timeout. return TestResult(self._test_name, failures, total_test_time, has_stderr) failures.extend(self._handle_error(reference_driver_output, reference_filename=reference_filename)) if failures: return TestResult(self._test_name, failures, total_test_time, has_stderr, pid=actual_driver_output.pid) if not reference_driver_output.image_hash and not actual_driver_output.image_hash: failures.append(test_failures.FailureReftestNoImagesGenerated(reference_filename)) elif mismatch: # Calling image_hash is considered unnecessary for expected mismatch ref tests. if reference_driver_output.image_hash == actual_driver_output.image_hash: failures.append(test_failures.FailureReftestMismatchDidNotOccur(reference_filename)) elif reference_driver_output.image_hash != actual_driver_output.image_hash: # ImageDiff has a hard coded color distance threshold even though tolerance=0 is specified. diff_result = self._port.diff_image(reference_driver_output.image, actual_driver_output.image, tolerance=0) error_string = diff_result[2] if error_string: _log.warning(' %s : %s' % (self._test_name, error_string)) failures.append(test_failures.FailureReftestMismatch(reference_filename)) actual_driver_output.error = (actual_driver_output.error or '') + error_string elif diff_result[0]: failures.append(test_failures.FailureReftestMismatch(reference_filename)) return TestResult(self._test_name, failures, total_test_time, has_stderr, pid=actual_driver_output.pid)
def _compare_output_with_reference(self, reference_driver_output, actual_driver_output, reference_filename, mismatch): total_test_time = reference_driver_output.test_time + actual_driver_output.test_time has_stderr = reference_driver_output.has_stderr() or actual_driver_output.has_stderr() failures = [] failures.extend(self._handle_error(actual_driver_output)) if failures: # Don't continue any more if we already have crash or timeout. return TestResult(self._test_name, failures, total_test_time, has_stderr) failures.extend(self._handle_error(reference_driver_output, reference_filename=reference_filename)) if failures: return TestResult(self._test_name, failures, total_test_time, has_stderr, pid=actual_driver_output.pid) if not reference_driver_output.image_hash and not actual_driver_output.image_hash: failures.append(test_failures.FailureReftestNoImagesGenerated(reference_filename)) elif mismatch: if reference_driver_output.image_hash == actual_driver_output.image_hash: diff, err_str = self._port.diff_image(reference_driver_output.image, actual_driver_output.image) if not diff: failures.append(test_failures.FailureReftestMismatchDidNotOccur(reference_filename)) elif err_str: _log.error(err_str) else: _log.warning(" %s -> ref test hashes matched but diff failed" % self._test_name) elif reference_driver_output.image_hash != actual_driver_output.image_hash: diff, err_str = self._port.diff_image(reference_driver_output.image, actual_driver_output.image) if diff: failures.append(test_failures.FailureReftestMismatch(reference_filename)) elif err_str: _log.error(err_str) else: _log.warning(" %s -> ref test hashes didn't match but diff passed" % self._test_name) return TestResult(self._test_name, failures, total_test_time, has_stderr, pid=actual_driver_output.pid)
def _compare_output_with_reference(self, driver_output1, driver_output2, reference_filename, mismatch): total_test_time = driver_output1.test_time + driver_output2.test_time has_stderr = driver_output1.has_stderr() or driver_output2.has_stderr() failures = [] failures.extend(self._handle_error(driver_output1)) if failures: # Don't continue any more if we already have crash or timeout. return TestResult(self._test_name, failures, total_test_time, has_stderr) failures.extend( self._handle_error(driver_output2, reference_filename=reference_filename)) if failures: return TestResult(self._test_name, failures, total_test_time, has_stderr) if not driver_output1.image_hash and not driver_output2.image_hash: failures.append( test_failures.FailureReftestNoImagesGenerated( reference_filename)) elif mismatch: if driver_output1.image_hash == driver_output2.image_hash: failures.append( test_failures.FailureReftestMismatchDidNotOccur( reference_filename)) elif driver_output1.image_hash != driver_output2.image_hash: failures.append( test_failures.FailureReftestMismatch(reference_filename)) return TestResult(self._test_name, failures, total_test_time, has_stderr)