def testHasNewFailures(self): files = self.GetTestFiles() failures = self.GetTestFailures() # no changes, no new failures cf = compare_failures.CompareFailures(files, failures, set(), set(), self.GetTmpDir(), False) self.failUnless(not cf.HasNewFailures()) # test goes from passing to failing pass_file = os.path.join(path_utils.LayoutTestsDir(), 'fast', 'pass1.html') failures[pass_file] = [test_failures.FailureTextMismatch(None)] cf = compare_failures.CompareFailures(files, failures, set(), set(), self.GetTmpDir(), False) self.failUnless(cf.HasNewFailures()) # Failing to passing failures = self.GetTestFailures() failure_file = os.path.join(path_utils.LayoutTestsDir(), 'fast', 'bar', 'fail2.html') del failures[failure_file] cf = compare_failures.CompareFailures(files, failures, set(), set(), self.GetTmpDir(), False) self.failUnless(not cf.HasNewFailures()) # A new test that fails, this doesn't count as a new failure. new_test_file = os.path.join(path_utils.LayoutTestsDir(), "new-test.html") files.add(new_test_file) failures = self.GetTestFailures() failures[new_test_file] = [test_failures.FailureCrash()] cf = compare_failures.CompareFailures(files, failures, set(), set(), self.GetTmpDir(), False) self.failUnless(not cf.HasNewFailures())
def testFailureToCrash(self): """When there's a new crash, we don't add it to the baseline or remove it from the failure list.""" failures = self.GetTestFailures() # Test case where we don't update new baseline. A failure moving to a # crash shouldn't be added to the expected-crashes.txt file. failure_file = os.path.join(path_utils.LayoutTestsDir(), 'fast', 'foo', 'fail1.html') failures[failure_file] = [test_failures.FailureCrash()] self.CheckNoChanges(failures)
def GetTestFailures(self): """Get a dictionary representing the crashes and failures in the expected-*.txt files.""" failures = {} for filename in self.GetTestFiles(): if filename.find('crash') != -1: failures[filename] = [test_failures.FailureCrash()] elif filename.find('fail') != -1: failures[filename] = [test_failures.FailureTextMismatch(None)] return failures
def testPassingToFailure(self): """When there's a new failure, we don't add it to the baseline.""" failures = self.GetTestFailures() # Test case where we don't update new baseline. We have a new failure, # but it shouldn't be added to the expected-failures.txt file. pass_file = os.path.join(path_utils.LayoutTestsDir(), 'fast', 'pass1.html') failures[pass_file] = [test_failures.FailureTextMismatch(None)] self.CheckNoChanges(failures) # Same thing as before: pass -> crash failures[pass_file] = [test_failures.FailureCrash()] self.CheckNoChanges(failures)
def testNewTestCrash(self): """After a merge, we need to update new crashing tests properly.""" files = self.GetTestFiles() new_test_file = os.path.join(path_utils.LayoutTestsDir(), "new-test.html") files.add(new_test_file) failures = self.GetTestFailures() # New test file crashing failures[new_test_file] = [test_failures.FailureCrash()] cf = compare_failures.CompareFailures(files, failures, set(), set(), self.GetTmpDir(), False) cf.UpdateFailuresOnDisk() self.CheckOutputWithExpectedFiles('expected-passing.txt', 'expected-failures.txt', 'expected-crashes-new-test.txt')
def process_output(port, test_info, test_types, test_args, configuration, output_dir, crash, timeout, test_run_time, actual_checksum, output, error): """Receives the output from a DumpRenderTree process, subjects it to a number of tests, and returns a list of failure types the test produced. Args: port: port-specific hooks proc: an active DumpRenderTree process test_info: Object containing the test filename, uri and timeout test_types: list of test types to subject the output to test_args: arguments to be passed to each test configuration: Debug or Release output_dir: directory to put crash stack traces into Returns: a TestResult object """ failures = [] # Some test args, such as the image hash, may be added or changed on a # test-by-test basis. local_test_args = copy.copy(test_args) local_test_args.hash = actual_checksum if crash: failures.append(test_failures.FailureCrash()) if timeout: failures.append(test_failures.FailureTimeout()) if crash: _log.debug("Stacktrace for %s:\n%s" % (test_info.filename, error)) # Strip off "file://" since RelativeTestFilename expects # filesystem paths. filename = os.path.join( output_dir, port.relative_test_filename(test_info.filename)) filename = os.path.splitext(filename)[0] + "-stack.txt" port.maybe_make_directory(os.path.split(filename)[0]) with codecs.open(filename, "wb", "utf-8") as file: file.write(error) elif error: _log.debug("Previous test output stderr lines:\n%s" % error) # Check the output and save the results. start_time = time.time() time_for_diffs = {} for test_type in test_types: start_diff_time = time.time() new_failures = test_type.compare_output(port, test_info.filename, output, local_test_args, configuration) # Don't add any more failures if we already have a crash, so we don't # double-report those tests. We do double-report for timeouts since # we still want to see the text and image output. if not crash: failures.extend(new_failures) time_for_diffs[test_type.__class__.__name__] = (time.time() - start_diff_time) total_time_for_all_diffs = time.time() - start_diff_time return TestResult(test_info.filename, failures, test_run_time, total_time_for_all_diffs, time_for_diffs)