def _compare_output_with_reference(self, reference_driver_output, actual_driver_output, reference_filename, mismatch): has_stderr = reference_driver_output.has_stderr() or actual_driver_output.has_stderr() failures = [] failures.extend(self._handle_error(actual_driver_output)) if failures: # Don't continue any more if we already have crash or timeout. return TestResult(self._test_name, failures, 0, has_stderr, crash_site=actual_driver_output.crash_site) failures.extend(self._handle_error(reference_driver_output, reference_filename=reference_filename)) if failures: return TestResult(self._test_name, failures, 0, has_stderr, pid=actual_driver_output.pid, crash_site=reference_driver_output.crash_site) if not actual_driver_output.image_hash: failures.append(test_failures.FailureReftestNoImageGenerated(reference_filename)) elif not reference_driver_output.image_hash: failures.append(test_failures.FailureReftestNoReferenceImageGenerated(reference_filename)) elif mismatch: if reference_driver_output.image_hash == actual_driver_output.image_hash: failures.append(test_failures.FailureReftestMismatchDidNotOccur(reference_filename)) elif reference_driver_output.image_hash != actual_driver_output.image_hash: diff, err_str = self._port.diff_image(reference_driver_output.image, actual_driver_output.image) if diff: failures.append(test_failures.FailureReftestMismatch(reference_filename)) elif err_str: _log.error(err_str) else: _log.warning(" %s -> ref test hashes didn't match but diff passed", self._test_name) return TestResult(self._test_name, failures, 0, has_stderr, pid=actual_driver_output.pid)
def _compare_output_with_reference(self, reference_driver_output, actual_driver_output, reference_filename, mismatch): failures = [] # Don't continue any more if we already have crash failures.extend(self._handle_error(actual_driver_output)) if failures: return failures failures.extend( self._handle_error( reference_driver_output, reference_filename=reference_filename)) if failures: return failures if not actual_driver_output.image_hash: failures.append( test_failures.FailureReftestNoImageGenerated( actual_driver_output, reference_driver_output, reference_filename)) elif not reference_driver_output.image_hash: failures.append( test_failures.FailureReftestNoReferenceImageGenerated( actual_driver_output, reference_driver_output, reference_filename)) elif mismatch: if reference_driver_output.image_hash == actual_driver_output.image_hash: failures.append( test_failures.FailureReftestMismatchDidNotOccur( actual_driver_output, reference_driver_output, reference_filename)) elif reference_driver_output.image_hash != actual_driver_output.image_hash: diff, err_str = self._port.diff_image( reference_driver_output.image, actual_driver_output.image) if diff: actual_driver_output.image_diff = diff if err_str: _log.error(err_str) actual_driver_output.error = (actual_driver_output.error or '') + err_str if diff or err_str: failures.append( test_failures.FailureReftestMismatch( actual_driver_output, reference_driver_output, reference_filename)) elif err_str: # TODO(rmhasan) Should we include this error message in the artifacts ? _log.error(' %s : %s', self._test_name, err_str) else: _log.warning( " %s -> ref test hashes didn't match but diff passed", self._test_name) return failures
def test_reference_is_missing(self): failure = test_failures.FailureReftestMismatch() failure.reference_filename = 'notfound.html' fs = self.run_test(failures=[failure], files={}) self.assertEqual(fs.written_files, {}) failure = test_failures.FailureReftestMismatchDidNotOccur() failure.reference_filename = 'notfound.html' fs = self.run_test(failures=[failure], files={}) self.assertEqual(fs.written_files, {})
def test_reference_exists(self): failure = test_failures.FailureReftestMismatch() failure.reference_filename = '/src/exists-expected.html' files = {'/src/exists-expected.html': 'yup'} written_files = self.run_test(failures=[failure], files=files) self.assertEqual(written_files, {'/tmp/exists-expected.html': 'yup'}) failure = test_failures.FailureReftestMismatchDidNotOccur() failure.reference_filename = '/src/exists-expected-mismatch.html' files = {'/src/exists-expected-mismatch.html': 'yup'} written_files = self.run_test(failures=[failure], files=files) self.assertEqual(written_files, {'/tmp/exists-expected-mismatch.html': 'yup'})