Example #1
0
 def test_filter_testcases_by_status_first_pass(self):
   suite = TestSuite('foo', 'bar')
   suite.tests = [
     TestCase(suite, 'foo/bar', 'foo/bar'),
     TestCase(suite, 'baz/bar', 'baz/bar'),
   ]
   suite.rules = {
     '': {
       'foo/bar': set(['PASS', 'SKIP']),
       'baz/bar': set(['PASS', 'FAIL']),
     },
   }
   suite.prefix_rules = {
     '': {
       'baz/': set(['PASS', 'SLOW']),
     },
   }
   suite.FilterTestCasesByStatus()
   self.assertEquals(
       [TestCase(suite, 'baz/bar', 'baz/bar')],
       suite.tests,
   )
   outcomes = suite.GetStatusFileOutcomes(suite.tests[0].name,
                                          suite.tests[0].variant)
   self.assertEquals(set(['PASS', 'FAIL', 'SLOW']), outcomes)
Example #2
0
  def test_filter_testcases_by_status_second_pass(self):
    suite = TestSuite('foo', 'bar')

    test1 = TestCase(suite, 'foo/bar', 'foo/bar')
    test2 = TestCase(suite, 'baz/bar', 'baz/bar')

    suite.tests = [
      test1.CopyAddingFlags(variant='default', flags=[]),
      test1.CopyAddingFlags(variant='stress', flags=['-v']),
      test2.CopyAddingFlags(variant='default', flags=[]),
      test2.CopyAddingFlags(variant='stress', flags=['-v']),
    ]

    suite.rules = {
      '': {
        'foo/bar': set(['PREV']),
      },
      'default': {
        'foo/bar': set(['PASS', 'SKIP']),
        'baz/bar': set(['PASS', 'FAIL']),
      },
      'stress': {
        'baz/bar': set(['SKIP']),
      },
    }
    suite.prefix_rules = {
      '': {
        'baz/': set(['PREV']),
      },
      'default': {
        'baz/': set(['PASS', 'SLOW']),
      },
      'stress': {
        'foo/': set(['PASS', 'SLOW']),
      },
    }
    suite.FilterTestCasesByStatus()
    self.assertEquals(
        [
          TestCase(suite, 'foo/bar', 'foo/bar', flags=['-v']),
          TestCase(suite, 'baz/bar', 'baz/bar'),
        ],
        suite.tests,
    )

    self.assertEquals(
        set(['PREV', 'PASS', 'SLOW']),
        suite.GetStatusFileOutcomes(suite.tests[0].name,
                                    suite.tests[0].variant),
    )
    self.assertEquals(
        set(['PREV', 'PASS', 'FAIL', 'SLOW']),
        suite.GetStatusFileOutcomes(suite.tests[1].name,
                                    suite.tests[1].variant),
    )
  def test_filter_testcases_by_status_second_pass(self):
    suite = TestSuite('foo', 'bar')

    test1 = TestCase(suite, 'foo/bar')
    test2 = TestCase(suite, 'baz/bar')

    # Contrived outcomes from filtering by variant-independent rules.
    test1.outcomes = set(['PREV'])
    test2.outcomes = set(['PREV'])

    suite.tests = [
      test1.CopyAddingFlags(variant='default', flags=[]),
      test1.CopyAddingFlags(variant='stress', flags=['-v']),
      test2.CopyAddingFlags(variant='default', flags=[]),
      test2.CopyAddingFlags(variant='stress', flags=['-v']),
    ]

    suite.rules = {
      'default': {
        'foo/bar': set(['PASS', 'SKIP']),
        'baz/bar': set(['PASS', 'FAIL']),
      },
      'stress': {
        'baz/bar': set(['SKIP']),
      },
    }
    suite.wildcards = {
      'default': {
        'baz/*': set(['PASS', 'SLOW']),
      },
      'stress': {
        'foo/*': set(['PASS', 'SLOW']),
      },
    }
    suite.FilterTestCasesByStatus(warn_unused_rules=False, variants=True)
    self.assertEquals(
        [
          TestCase(suite, 'foo/bar', flags=['-v']),
          TestCase(suite, 'baz/bar'),
        ],
        suite.tests,
    )

    self.assertEquals(
        set(['PASS', 'SLOW', 'PREV']),
        suite.tests[0].outcomes,
    )
    self.assertEquals(
        set(['PASS', 'FAIL', 'SLOW', 'PREV']),
        suite.tests[1].outcomes,
    )
Example #4
0
    def test_fail_ok_outcome(self):
        suite = TestSuite('foo', 'bar')
        suite.rules = {
            '': {
                'foo/bar': set(['FAIL_OK']),
                'baz/bar': set(['FAIL']),
            },
        }
        suite.prefix_rules = {}
        suite.tests = [
            TestCase(suite, 'foo/bar', 'foo/bar'),
            TestCase(suite, 'baz/bar', 'baz/bar'),
        ]

        for t in suite.tests:
            self.assertEquals(['FAIL'], t.expected_outcomes)
Example #5
0
  def test_fail_ok_outcome(self):
    suite = TestSuite('foo', 'bar')
    suite.rules = {
      '': {
        'foo/bar': set(['FAIL_OK']),
        'baz/bar': set(['FAIL']),
      },
    }
    suite.prefix_rules = {}
    suite.tests = [
      TestCase(suite, 'foo/bar', 'foo/bar'),
      TestCase(suite, 'baz/bar', 'baz/bar'),
    ]

    for t in suite.tests:
      self.assertEquals(['FAIL'], t.expected_outcomes)
    def test_filter_testcases_by_status_second_pass(self):
        suite = TestSuite('foo', 'bar')

        suite.rules = {
            '': {
                'foo/bar': set(['PREV']),
            },
            'default': {
                'foo/bar': set(['PASS', 'SKIP']),
                'baz/bar': set(['PASS', 'FAIL']),
            },
            'stress': {
                'baz/bar': set(['SKIP']),
            },
        }
        suite.prefix_rules = {
            '': {
                'baz/': set(['PREV']),
            },
            'default': {
                'baz/': set(['PASS', 'SLOW']),
            },
            'stress': {
                'foo/': set(['PASS', 'SLOW']),
            },
        }

        test1 = TestCase(suite, 'foo/bar', 'foo/bar')
        test2 = TestCase(suite, 'baz/bar', 'baz/bar')
        suite.tests = [
            test1.create_variant(variant='default', flags=[]),
            test1.create_variant(variant='stress', flags=['-v']),
            test2.create_variant(variant='default', flags=[]),
            test2.create_variant(variant='stress', flags=['-v']),
        ]

        suite.FilterTestCasesByStatus()
        self.assertEquals(
            [
                TestCase(suite, 'foo/bar', 'foo/bar').create_variant(
                    None, ['-v']),
                TestCase(suite, 'baz/bar', 'baz/bar'),
            ],
            suite.tests,
        )

        self.assertEquals(
            set(['PREV', 'PASS', 'SLOW']),
            suite.GetStatusFileOutcomes(suite.tests[0].name,
                                        suite.tests[0].variant),
        )
        self.assertEquals(
            set(['PREV', 'PASS', 'FAIL', 'SLOW']),
            suite.GetStatusFileOutcomes(suite.tests[1].name,
                                        suite.tests[1].variant),
        )
Example #7
0
 def test_filter_testcases_by_status_first_pass(self):
     suite = TestSuite('foo', 'bar')
     suite.tests = [
         TestCase(suite, 'foo/bar', 'foo/bar'),
         TestCase(suite, 'baz/bar', 'baz/bar'),
     ]
     suite.rules = {
         '': {
             'foo/bar': set(['PASS', 'SKIP']),
             'baz/bar': set(['PASS', 'FAIL']),
         },
     }
     suite.prefix_rules = {
         '': {
             'baz/': set(['PASS', 'SLOW']),
         },
     }
     suite.FilterTestCasesByStatus()
     self.assertEquals(
         [TestCase(suite, 'baz/bar', 'baz/bar')],
         suite.tests,
     )
     outcomes = suite.GetStatusFileOutcomes(suite.tests[0].name,
                                            suite.tests[0].variant)
     self.assertEquals(set(['PASS', 'FAIL', 'SLOW']), outcomes)
Example #8
0
    def testLoadingTestSuite(self):
        suite = TestSuite.LoadTestSuite(self.test_root, self.test_config)

        self.assertEquals(suite.name, "fake_testsuite")
        self.assertEquals(suite.test_config, self.test_config)

        # Verify that no tests are loaded yet.
        self.assertIsNone(suite.tests)
        self.assertIsNone(suite.statusfile)
 def test_filter_testcases_by_status_first_pass(self):
   suite = TestSuite('foo', 'bar')
   suite.tests = [
     TestCase(suite, 'foo/bar'),
     TestCase(suite, 'baz/bar'),
   ]
   suite.rules = {
     '': {
       'foo/bar': set(['PASS', 'SKIP']),
       'baz/bar': set(['PASS', 'FAIL']),
     },
   }
   suite.wildcards = {
     '': {
       'baz/*': set(['PASS', 'SLOW']),
     },
   }
   suite.FilterTestCasesByStatus(warn_unused_rules=False)
   self.assertEquals(
       [TestCase(suite, 'baz/bar')],
       suite.tests,
   )
   self.assertEquals(set(['PASS', 'FAIL', 'SLOW']), suite.tests[0].outcomes)
  def setUp(self):
    test_dir = os.path.dirname(__file__)
    self.test_root = os.path.join(test_dir, "fake_testsuite")
    self.test_config = TestConfig(
        command_prefix=[],
        extra_flags=[],
        isolates=False,
        mode_flags=[],
        no_harness=False,
        noi18n=False,
        random_seed=0,
        run_skipped=False,
        shell_dir='fake_testsuite/fake_d8',
        timeout=10,
        verbose=False,
    )

    self.suite = TestSuite.Load(self.test_root, self.test_config)
Example #11
0
    def test_filter_testcases_by_status_second_pass(self):
        suite = TestSuite('foo', 'bar')

        test1 = TestCase(suite, 'foo/bar')
        test2 = TestCase(suite, 'baz/bar')

        # Contrived outcomes from filtering by variant-independent rules.
        test1.outcomes = set(['PREV'])
        test2.outcomes = set(['PREV'])

        suite.tests = [
            test1.CopyAddingFlags(variant='default', flags=[]),
            test1.CopyAddingFlags(variant='stress', flags=['-v']),
            test2.CopyAddingFlags(variant='default', flags=[]),
            test2.CopyAddingFlags(variant='stress', flags=['-v']),
        ]

        suite.rules = {
            'default': {
                'foo/bar': set(['PASS', 'SKIP']),
                'baz/bar': set(['PASS', 'FAIL']),
            },
            'stress': {
                'baz/bar': set(['SKIP']),
            },
        }
        suite.wildcards = {
            'default': {
                'baz/*': set(['PASS', 'SLOW']),
            },
            'stress': {
                'foo/*': set(['PASS', 'SLOW']),
            },
        }
        suite.FilterTestCasesByStatus(warn_unused_rules=False, variants=True)
        self.assertEquals(
            [
                TestCase(suite, 'foo/bar', flags=['-v']),
                TestCase(suite, 'baz/bar'),
            ],
            suite.tests,
        )

        self.assertEquals(
            set(['PASS', 'SLOW', 'PREV']),
            suite.tests[0].outcomes,
        )
        self.assertEquals(
            set(['PASS', 'FAIL', 'SLOW', 'PREV']),
            suite.tests[1].outcomes,
        )
Example #12
0
 def test_filter_testcases_by_status_first_pass(self):
     suite = TestSuite('foo', 'bar')
     suite.tests = [
         TestCase(suite, 'foo/bar'),
         TestCase(suite, 'baz/bar'),
     ]
     suite.rules = {
         '': {
             'foo/bar': set(['PASS', 'SKIP']),
             'baz/bar': set(['PASS', 'FAIL']),
         },
     }
     suite.wildcards = {
         '': {
             'baz/*': set(['PASS', 'SLOW']),
         },
     }
     suite.FilterTestCasesByStatus(warn_unused_rules=False)
     self.assertEquals(
         [TestCase(suite, 'baz/bar')],
         suite.tests,
     )
     self.assertEquals(set(['PASS', 'FAIL', 'SLOW']),
                       suite.tests[0].outcomes)