def setUp(self): self._actual_output = DriverOutput(text=None, image=None, image_hash=None, audio=None) self._expected_output = DriverOutput(text=None, image=None, image_hash=None, audio=None)
def get_result(test_name, result_type=test_expectations.PASS, run_time=0): failures = [] dummy_1, dummy_2 = DriverOutput(None, None, None, None), DriverOutput(None, None, None, None) if result_type == test_expectations.TIMEOUT: failures = [test_failures.FailureTimeout(dummy_1)] elif result_type == test_expectations.CRASH: failures = [test_failures.FailureCrash(dummy_1)] elif result_type == test_expectations.FAIL: failures = [test_failures.TestFailure(dummy_1, dummy_2)] return test_results.TestResult(test_name, failures=failures, test_run_time=run_time)
def get_result(test_name, result_type=ResultType.Pass, run_time=0): failures = [] dummy_1, dummy_2 = DriverOutput(None, None, None, None), DriverOutput( None, None, None, None) if result_type == ResultType.Timeout: failures = [test_failures.FailureTimeout(dummy_1)] elif result_type == ResultType.Crash: failures = [test_failures.FailureCrash(dummy_1)] elif result_type == ResultType.Failure: failures = [test_failures.TestFailure(dummy_1, dummy_2)] return test_results.TestResult(test_name, failures=failures, test_run_time=run_time)
def output_for_test(self, test_input, is_reftest): port = self._port actual_text = port.expected_text(test_input.test_name) actual_audio = port.expected_audio(test_input.test_name) actual_image = None actual_checksum = None if is_reftest: # Make up some output for reftests. actual_text = 'reference text\n' actual_checksum = 'mock-checksum' actual_image = 'blank' if test_input.test_name.endswith('-mismatch.html'): actual_text = 'not reference text\n' actual_checksum = 'not-mock-checksum' actual_image = 'not blank' elif test_input.image_hash: actual_checksum = port.expected_checksum(test_input.test_name) actual_image = port.expected_image(test_input.test_name) if self._options.actual_directory: actual_path = port.host.filesystem.join(self._options.actual_directory, test_input.test_name) root, _ = port.host.filesystem.splitext(actual_path) text_path = root + '-actual.txt' if port.host.filesystem.exists(text_path): actual_text = port.host.filesystem.read_binary_file(text_path) audio_path = root + '-actual.wav' if port.host.filesystem.exists(audio_path): actual_audio = port.host.filesystem.read_binary_file(audio_path) image_path = root + '-actual.png' if port.host.filesystem.exists(image_path): actual_image = port.host.filesystem.read_binary_file(image_path) with port.host.filesystem.open_binary_file_for_reading(image_path) as filehandle: actual_checksum = read_checksum_from_png.read_checksum(filehandle) return DriverOutput(actual_text, actual_image, actual_checksum, actual_audio)
def run_test(self, failures=None, files=None, filename='foo.html'): failures = failures or [] host = MockSystemHost() host.filesystem.files = files or {} port = TestPort(host=host, port_name='test-mac-mac10.11', options=optparse.Values()) actual_output = DriverOutput(text='', image=None, image_hash=None, audio=None) expected_output = DriverOutput(text='', image=None, image_hash=None, audio=None) write_test_result(host.filesystem, port, '/tmp', filename, actual_output, expected_output, failures) return host.filesystem
def get_result(test_name, result_type=test_expectations.PASS, run_time=0): failures = [] dummy_1, dummy_2 = DriverOutput(None, None, None, None), DriverOutput( None, None, None, None) if result_type == test_expectations.TIMEOUT: failures = [test_failures.FailureTimeout(dummy_1)] elif result_type == test_expectations.AUDIO: failures = [test_failures.FailureAudioMismatch(dummy_1, dummy_2)] elif result_type == test_expectations.TEXT: failures = [test_failures.FailureTextMismatch(dummy_1, dummy_2)] elif result_type == test_expectations.IMAGE: failures = [test_failures.FailureImageHashMismatch(dummy_1, dummy_2)] elif result_type == test_expectations.CRASH: failures = [test_failures.FailureCrash(dummy_1)] elif result_type == test_expectations.LEAK: failures = [test_failures.FailureLeak(dummy_1)] return test_results.TestResult(test_name, failures=failures, test_run_time=run_time)
def test_repeated_test_artifacts(self): host = MockSystemHost() port = Port(host, 'baseport') artifacts = Artifacts('/dir', host.filesystem, repeat_tests=True) def init_test_failure(test_failure): test_failure.port = port test_failure.filesystem = host.filesystem test_failure.test_name = 'foo.html' test_failure.result_directory = '/dir' pass_with_stderr = PassWithStderr( DriverOutput(None, None, None, None, error=b'pass with stderr')) init_test_failure(pass_with_stderr) crash = FailureCrash( DriverOutput(None, None, None, None, crash=True, error=b'crash stderr')) init_test_failure(crash) timeout = FailureTimeout( DriverOutput(None, None, None, None, error=b'timeout with stderr')) init_test_failure(timeout) pass_with_stderr.create_artifacts(artifacts) self.assertEqual('pass with stderr', host.filesystem.read_text_file('/dir/foo-stderr.txt')) crash.create_artifacts(artifacts) self.assertEqual('crash stderr', host.filesystem.read_text_file('/dir/foo-stderr.txt')) timeout.create_artifacts(artifacts) self.assertEqual('timeout with stderr', host.filesystem.read_text_file('/dir/foo-stderr.txt')) pass_with_stderr.create_artifacts(artifacts) self.assertEqual('timeout with stderr', host.filesystem.read_text_file('/dir/foo-stderr.txt'))
def test_results_multiple(self): driver_output = DriverOutput(None, None, None, None) failure_crash = [ test_failures.FailureCrash(driver_output, None), test_failures.TestFailure(driver_output, None) ] failure_timeout = [ test_failures.FailureTimeout(driver_output, None), test_failures.TestFailure(driver_output, None) ] failure_early_exit = [ test_failures.FailureEarlyExit(driver_output, None), test_failures.TestFailure(driver_output, None) ] # Should not raise an exception for CRASH and FAIL. TestResult('foo', failures=failure_crash) # Should not raise an exception for TIMEOUT and FAIL. TestResult('foo', failures=failure_timeout) with self.assertRaises(AssertionError): TestResult('foo', failures=failure_early_exit)
def run_test(self, driver_input): if not self.started: self.started = True self.pid = TestDriver.next_pid TestDriver.next_pid += 1 start_time = time.time() test_name = driver_input.test_name test_args = driver_input.args or [] test = self._port._tests[test_name] if test.keyboard: raise KeyboardInterrupt if test.exception: raise ValueError('exception from ' + test_name) if test.device_failure: raise DeviceFailure('device failure in ' + test_name) audio = None actual_text = test.actual_text crash = test.crash web_process_crash = test.web_process_crash leak = test.leak if 'flaky/text.html' in test_name and not test_name in self._port._flakes: self._port._flakes.add(test_name) actual_text = 'flaky text failure' if 'crash_then_text.html' in test_name: if test_name in self._port._flakes: actual_text = 'text failure' else: self._port._flakes.add(test_name) crashed_process_name = self._port.driver_name() crashed_pid = 1 crash = True if 'text_then_crash.html' in test_name: if test_name in self._port._flakes: crashed_process_name = self._port.driver_name() crashed_pid = 1 crash = True else: self._port._flakes.add(test_name) actual_text = 'text failure' if actual_text and test_args and test_name == 'passes/args.html': actual_text = actual_text + ' ' + ' '.join(test_args) if test.actual_audio: audio = base64.b64decode(test.actual_audio) crashed_process_name = None crashed_pid = None leak_log = '' if leak: leak_log = 'leak detected' crash_log = '' if crash: crashed_process_name = self._port.driver_name() crashed_pid = 1 crash_log = 'crash log' elif web_process_crash: crashed_process_name = 'WebProcess' crashed_pid = 2 crash_log = 'web process crash log' if crashed_process_name: crash_logs = CrashLogs(self._port.host) crash_log = crash_logs.find_newest_log(crashed_process_name, None) or crash_log if 'crash-reftest.html' in test_name: crashed_process_name = self._port.driver_name() crashed_pid = 3 crash = True crash_log = 'reftest crash log' if test.actual_checksum == driver_input.image_hash: image = None else: image = test.actual_image return DriverOutput(actual_text, image, test.actual_checksum, audio, crash=(crash or web_process_crash), crashed_process_name=crashed_process_name, crashed_pid=crashed_pid, crash_log=crash_log, test_time=time.time() - start_time, timeout=test.timeout, error=test.error, pid=self.pid, leak=test.leak, leak_log=leak_log)
def _expected_driver_output(self): return DriverOutput(self._port.expected_text(self._test_name), self._port.expected_image(self._test_name), self._port.expected_checksum(self._test_name), self._port.expected_audio(self._test_name))
def _run_reftest(self): test_output = self._driver.run_test(self._driver_input()) total_test_time = test_output.test_time expected_text = self._port.expected_text(self._test_name) expected_text_output = DriverOutput(text=expected_text, image=None, image_hash=None, audio=None) # This _compare_output compares text if expected text exists, ignores # image, checks for extra baselines, and generates crash or timeout # failures if needed. compare_text_failures = self._compare_output(expected_text_output, test_output) # If the test crashed, or timed out, or a leak was detected, there's no point # in running the reference at all. This can save a lot of execution time if we # have a lot of crashes or timeouts. if test_output.crash or test_output.timeout or test_output.leak: return build_test_result(test_output, self._test_name, retry_attempt=self._retry_attempt, failures=compare_text_failures, test_run_time=test_output.test_time, pid=test_output.pid, crash_site=test_output.crash_site) # A reftest can have multiple match references and multiple mismatch references; # the test fails if any mismatch matches and all of the matches don't match. # To minimize the number of references we have to check, we run all of the # mismatches first, then the matches, and short-circuit out as soon as we can. # Note that sorting by the expectation sorts "!=" before "==" so this is easy # to do. expected_output = None reference_test_names = [] reftest_failures = [] args = self._port.args_for_test(self._test_name) # sort self._reference_files to put mismatch tests first for expectation, reference_filename in sorted(self._reference_files): reference_test_name = self._port.relative_test_filename( reference_filename) reference_test_names.append(reference_test_name) driver_input = DriverInput(reference_test_name, self._timeout_ms, image_hash=test_output.image_hash, args=args) expected_output = self._driver.run_test(driver_input) total_test_time += expected_output.test_time reftest_failures = self._compare_output_with_reference( expected_output, test_output, reference_filename, expectation == '!=') if ((expectation == '!=' and reftest_failures) or (expectation == '==' and not reftest_failures)): break assert expected_output # Combine compare_text_result and test_result expected_output.text = expected_text_output.text failures = reftest_failures + compare_text_failures # FIXME: We don't really deal with a mix of reftest types properly. We # pass in a set() to reftest_type and only really handle the first of # the references in the result. reftest_type = list( set([ reference_file[0] for reference_file in self._reference_files ])) return build_test_result(test_output, self._test_name, retry_attempt=self._retry_attempt, failures=failures, test_run_time=total_test_time, reftest_type=reftest_type, pid=test_output.pid, crash_site=test_output.crash_site, references=reference_test_names)
def setUp(self): host = MockHost() self.port = host.port_factory.get(port_name='test') self._actual_output = DriverOutput(None, None, None, None) self._expected_output = DriverOutput(None, None, None, None)
def test_results_has_repaint_overlay(self): driver_output = DriverOutput('"invalidations": [', None, None, None) failures = [test_failures.FailureTextMismatch(driver_output, None)] result = TestResult('foo', failures=failures) self.assertTrue(result.has_repaint_overlay)
def test_results_has_stderr(self): driver_output = DriverOutput(None, None, None, None, error='error') failures = [test_failures.FailureCrash(driver_output, None)] result = TestResult('foo', failures=failures) self.assertTrue(result.has_stderr)
def _run_reftest(self): test_output = self._driver.run_test(self._driver_input(), self._stop_when_done) total_test_time = test_output.test_time expected_output = None test_result = None expected_text = self._port.expected_text(self._test_name) expected_text_output = DriverOutput(text=expected_text, image=None, image_hash=None, audio=None) # If the test crashed, or timed out, there's no point in running the reference at all. # This can save a lot of execution time if we have a lot of crashes or timeouts. if test_output.crash or test_output.timeout: test_result = self._compare_output(expected_text_output, test_output) if test_output.crash: test_result_writer.write_test_result( self._filesystem, self._port, self._results_directory, self._test_name, test_output, expected_text_output, test_result.failures) return test_result # A reftest can have multiple match references and multiple mismatch references; # the test fails if any mismatch matches and all of the matches don't match. # To minimize the number of references we have to check, we run all of the mismatches first, # then the matches, and short-circuit out as soon as we can. # Note that sorting by the expectation sorts "!=" before "==" so this is easy to do. putAllMismatchBeforeMatch = sorted reference_test_names = [] for expectation, reference_filename in putAllMismatchBeforeMatch( self._reference_files): if self._port.lookup_virtual_test_base(self._test_name): args = self._port.lookup_virtual_reference_args( self._test_name) else: args = self._port.lookup_physical_reference_args( self._test_name) reference_test_name = self._port.relative_test_filename( reference_filename) reference_test_names.append(reference_test_name) driver_input = DriverInput(reference_test_name, self._timeout_ms, image_hash=test_output.image_hash, should_run_pixel_test=True, args=args) expected_output = self._reference_driver.run_test( driver_input, self._stop_when_done) total_test_time += expected_output.test_time test_result = self._compare_output_with_reference( expected_output, test_output, reference_filename, expectation == '!=') if (expectation == '!=' and test_result.failures) or ( expectation == '==' and not test_result.failures): break assert expected_output if expected_text: text_output = DriverOutput(text=test_output.text, image=None, image_hash=None, audio=None) text_compare_result = self._compare_output(expected_text_output, text_output) test_result.failures.extend(text_compare_result.failures) test_result.has_repaint_overlay = text_compare_result.has_repaint_overlay expected_output.text = expected_text_output.text test_result_writer.write_test_result(self._filesystem, self._port, self._results_directory, self._test_name, test_output, expected_output, test_result.failures) # FIXME: We don't really deal with a mix of reftest types properly. We pass in a set() to reftest_type # and only really handle the first of the references in the result. reftest_type = list( set([ reference_file[0] for reference_file in self._reference_files ])) return TestResult(self._test_name, test_result.failures, total_test_time, test_result.has_stderr, reftest_type=reftest_type, pid=test_result.pid, crash_site=test_result.crash_site, references=reference_test_names, has_repaint_overlay=test_result.has_repaint_overlay)