def testHasNewFailures(self): files = self.GetTestFiles() failures = self.GetTestFailures() # no changes, no new failures cf = compare_failures.CompareFailures(files, failures, set(), set(), self.GetTmpDir(), False) self.failUnless(not cf.HasNewFailures()) # test goes from passing to failing pass_file = os.path.join(path_utils.LayoutTestsDir(), 'fast', 'pass1.html') failures[pass_file] = [test_failures.FailureTextMismatch(None)] cf = compare_failures.CompareFailures(files, failures, set(), set(), self.GetTmpDir(), False) self.failUnless(cf.HasNewFailures()) # Failing to passing failures = self.GetTestFailures() failure_file = os.path.join(path_utils.LayoutTestsDir(), 'fast', 'bar', 'fail2.html') del failures[failure_file] cf = compare_failures.CompareFailures(files, failures, set(), set(), self.GetTmpDir(), False) self.failUnless(not cf.HasNewFailures()) # A new test that fails, this doesn't count as a new failure. new_test_file = os.path.join(path_utils.LayoutTestsDir(), "new-test.html") files.add(new_test_file) failures = self.GetTestFailures() failures[new_test_file] = [test_failures.FailureCrash()] cf = compare_failures.CompareFailures(files, failures, set(), set(), self.GetTmpDir(), False) self.failUnless(not cf.HasNewFailures())
def CheckNoChanges(self, failures): """Verify that none of the expected-*.txt files have changed.""" cf = compare_failures.CompareFailures(self.GetTestFiles(), failures, set(), set(), self.GetTmpDir(), False) cf.UpdateFailuresOnDisk() self.CheckOutputWithExpectedFiles('expected-passing.txt', 'expected-failures.txt', 'expected-crashes.txt')
def testCrashToFailure(self): """This is better than before, so we should update both lists.""" failures = self.GetTestFailures() crash_file = os.path.join(path_utils.LayoutTestsDir(), 'fast', 'bar', 'betz', 'crash3.html') failures[crash_file] = [test_failures.FailureTextMismatch(None)] cf = compare_failures.CompareFailures(self.GetTestFiles(), failures, set(), set(), self.GetTmpDir(), False) cf.UpdateFailuresOnDisk() self.CheckOutputWithExpectedFiles('expected-passing.txt', 'expected-failures-new-crash.txt', 'expected-crashes-new-passing.txt')
def testCrashToPassing(self): """This is better than before, so we update the crashes file.""" failures = self.GetTestFailures() crash_file = os.path.join(path_utils.LayoutTestsDir(), 'fast', 'bar', 'betz', 'crash3.html') del failures[crash_file] cf = compare_failures.CompareFailures(self.GetTestFiles(), failures, set(), set(), self.GetTmpDir(), False) cf.UpdateFailuresOnDisk() self.CheckOutputWithExpectedFiles('expected-passing-new-passing.txt', 'expected-failures.txt', 'expected-crashes-new-passing.txt')
def testNewTestPass(self): """After a merge, we need to update new passing tests properly.""" files = self.GetTestFiles() new_test_file = os.path.join(path_utils.LayoutTestsDir(), "new-test.html") files.add(new_test_file) failures = self.GetTestFailures() # New test file passing cf = compare_failures.CompareFailures(files, failures, set(), set(), self.GetTmpDir(), False) cf.UpdateFailuresOnDisk() self.CheckOutputWithExpectedFiles('expected-passing-new-test.txt', 'expected-failures.txt', 'expected-crashes.txt')
def testFailureToPassing(self): """This is better than before, so we should update the failure list.""" failures = self.GetTestFailures() # Remove one of the failing test cases from the failures dictionary. This # makes failure_file considered to be passing. failure_file = os.path.join(path_utils.LayoutTestsDir(), 'fast', 'bar', 'fail2.html') del failures[failure_file] cf = compare_failures.CompareFailures(self.GetTestFiles(), failures, set(), set(), self.GetTmpDir(), False) cf.UpdateFailuresOnDisk() self.CheckOutputWithExpectedFiles('expected-passing-new-passing2.txt', 'expected-failures-new-passing.txt', 'expected-crashes.txt')
def testGenerateNewBaseline(self): """Test the generation of new expected-*.txt files when either they don't exist or the user explicitly asks to make new files.""" failures = self.GetTestFailures() # Test to make sure we generate baseline files if the file doesn't exist. os.remove(os.path.join(self.GetTmpDir(), 'expected-passing.txt')) os.remove(os.path.join(self.GetTmpDir(), 'expected-failures.txt')) os.remove(os.path.join(self.GetTmpDir(), 'expected-crashes.txt')) # Test force generation of new baseline files with a new failure and one # less passing. pass_file = os.path.join(path_utils.LayoutTestsDir(), 'fast', 'pass1.html') failures[pass_file] = [test_failures.FailureTextMismatch(None)] cf = compare_failures.CompareFailures(self.GetTestFiles(), failures, set(), set(), self.GetTmpDir(), False) cf.UpdateFailuresOnDisk() self.CheckOutputWithExpectedFiles('expected-passing-new-baseline.txt', 'expected-failures-added.txt', 'expected-crashes.txt')