def testHasNewFailures(self):
files = self.GetTestFiles()
failures = self.GetTestFailures()
# no changes, no new failures
cf = compare_failures.CompareFailures(files, failures, set(), set(),
self.GetTmpDir(), False)
self.failUnless(not cf.HasNewFailures())
# test goes from passing to failing
pass_file = os.path.join(path_utils.LayoutTestsDir(), 'fast',
'pass1.html')
failures[pass_file] = [test_failures.FailureTextMismatch(None)]
cf = compare_failures.CompareFailures(files, failures, set(), set(),
self.GetTmpDir(), False)
self.failUnless(cf.HasNewFailures())
# Failing to passing
failures = self.GetTestFailures()
failure_file = os.path.join(path_utils.LayoutTestsDir(), 'fast', 'bar',
'fail2.html')
del failures[failure_file]
cf = compare_failures.CompareFailures(files, failures, set(), set(),
self.GetTmpDir(), False)
self.failUnless(not cf.HasNewFailures())
# A new test that fails, this doesn't count as a new failure.
new_test_file = os.path.join(path_utils.LayoutTestsDir(),
"new-test.html")
files.add(new_test_file)
failures = self.GetTestFailures()
failures[new_test_file] = [test_failures.FailureCrash()]
cf = compare_failures.CompareFailures(files, failures, set(), set(),
self.GetTmpDir(), False)
self.failUnless(not cf.HasNewFailures())
def CheckNoChanges(self, failures):
"""Verify that none of the expected-*.txt files have changed."""
cf = compare_failures.CompareFailures(self.GetTestFiles(), failures,
set(), set(), self.GetTmpDir(),
False)
cf.UpdateFailuresOnDisk()
self.CheckOutputWithExpectedFiles('expected-passing.txt',
'expected-failures.txt',
'expected-crashes.txt')
def testCrashToFailure(self):
"""This is better than before, so we should update both lists."""
failures = self.GetTestFailures()
crash_file = os.path.join(path_utils.LayoutTestsDir(), 'fast', 'bar',
'betz', 'crash3.html')
failures[crash_file] = [test_failures.FailureTextMismatch(None)]
cf = compare_failures.CompareFailures(self.GetTestFiles(), failures,
set(), set(), self.GetTmpDir(),
False)
cf.UpdateFailuresOnDisk()
self.CheckOutputWithExpectedFiles('expected-passing.txt',
'expected-failures-new-crash.txt',
'expected-crashes-new-passing.txt')
def testCrashToPassing(self):
"""This is better than before, so we update the crashes file."""
failures = self.GetTestFailures()
crash_file = os.path.join(path_utils.LayoutTestsDir(), 'fast', 'bar',
'betz', 'crash3.html')
del failures[crash_file]
cf = compare_failures.CompareFailures(self.GetTestFiles(), failures,
set(), set(), self.GetTmpDir(),
False)
cf.UpdateFailuresOnDisk()
self.CheckOutputWithExpectedFiles('expected-passing-new-passing.txt',
'expected-failures.txt',
'expected-crashes-new-passing.txt')
def testNewTestPass(self):
"""After a merge, we need to update new passing tests properly."""
files = self.GetTestFiles()
new_test_file = os.path.join(path_utils.LayoutTestsDir(),
"new-test.html")
files.add(new_test_file)
failures = self.GetTestFailures()
# New test file passing
cf = compare_failures.CompareFailures(files, failures, set(), set(),
self.GetTmpDir(), False)
cf.UpdateFailuresOnDisk()
self.CheckOutputWithExpectedFiles('expected-passing-new-test.txt',
'expected-failures.txt',
'expected-crashes.txt')
def testFailureToPassing(self):
"""This is better than before, so we should update the failure list."""
failures = self.GetTestFailures()
# Remove one of the failing test cases from the failures dictionary. This
# makes failure_file considered to be passing.
failure_file = os.path.join(path_utils.LayoutTestsDir(), 'fast', 'bar',
'fail2.html')
del failures[failure_file]
cf = compare_failures.CompareFailures(self.GetTestFiles(), failures,
set(), set(), self.GetTmpDir(),
False)
cf.UpdateFailuresOnDisk()
self.CheckOutputWithExpectedFiles('expected-passing-new-passing2.txt',
'expected-failures-new-passing.txt',
'expected-crashes.txt')
def testGenerateNewBaseline(self):
"""Test the generation of new expected-*.txt files when either they don't
exist or the user explicitly asks to make new files."""
failures = self.GetTestFailures()
# Test to make sure we generate baseline files if the file doesn't exist.
os.remove(os.path.join(self.GetTmpDir(), 'expected-passing.txt'))
os.remove(os.path.join(self.GetTmpDir(), 'expected-failures.txt'))
os.remove(os.path.join(self.GetTmpDir(), 'expected-crashes.txt'))
# Test force generation of new baseline files with a new failure and one
# less passing.
pass_file = os.path.join(path_utils.LayoutTestsDir(), 'fast',
'pass1.html')
failures[pass_file] = [test_failures.FailureTextMismatch(None)]
cf = compare_failures.CompareFailures(self.GetTestFiles(), failures,
set(), set(), self.GetTmpDir(),
False)
cf.UpdateFailuresOnDisk()
self.CheckOutputWithExpectedFiles('expected-passing-new-baseline.txt',
'expected-failures-added.txt',
'expected-crashes.txt')