def test_find_fastest_tests_excludes_deleted_tests(self): host = MockHost() port = host.port_factory.get('test-win-win7', None) all_tests = [ 'fast/css/1.html', 'fast/css/2.html', ] port.tests = lambda paths: paths or all_tests finder = web_test_finder.WebTestFinder(port, {}) finder._times_trie = lambda: { 'fast': { 'css': { '1.html': 1, '2.html': 2, 'non-existant.html': 1, } }, } tests = finder.find_tests(fastest_percentile=90, args=[]) self.assertEqual(set(tests[1]), set(['fast/css/1.html']))
def test_find_fastest_tests(self): host = MockHost() port = host.port_factory.get('test-win-win7', None) all_tests = [ 'path/test.html', 'new/test.html', 'fast/css/1.html', 'fast/css/2.html', 'fast/css/3.html', 'fast/css/skip1.html', 'fast/css/skip2.html', 'fast/css/skip3.html', 'fast/css/skip4.html', 'fast/css/skip5.html', ] port.tests = lambda paths: paths or all_tests finder = web_test_finder.WebTestFinder(port, {}) finder._times_trie = lambda: { 'fast': { 'css': { '1.html': 1, '2.html': 2, '3.html': 3, 'skip1.html': 0, 'skip2.html': 0, 'skip3.html': 0, 'skip4.html': 0, 'skip5.html': 0, } }, 'path': { 'test.html': 4, } } tests = finder.find_tests(fastest_percentile=50, args=[]) self.assertEqual( set(tests[1]), set(['fast/css/1.html', 'fast/css/2.html', 'new/test.html'])) tests = finder.find_tests(fastest_percentile=50, args=['path/test.html']) self.assertEqual( set(tests[1]), set([ 'fast/css/1.html', 'fast/css/2.html', 'path/test.html', 'new/test.html' ])) tests = finder.find_tests(args=[]) self.assertEqual(tests[1], all_tests) tests = finder.find_tests(args=['path/test.html']) self.assertEqual(tests[1], ['path/test.html'])
def test_skip_tests_idlharness(self): """Tests that idlharness tests are skipped on MSAN/ASAN runs. See https://crbug.com/856601 """ host = MockHost() port = host.port_factory.get('test-win-win7', None) all_tests = [ 'external/wpt/dir1/dir2/foo.html', 'external/wpt/dir1/dir2/idlharness.any.html', 'external/wpt/dir1/dir2/idlharness.any.worker.html', ] # Patch port.tests() to return our tests port.tests = lambda paths: paths or all_tests options = optparse.Values({ 'no_expectations': True, 'enable_sanitizer': False, 'skipped': 'default', }) finder = web_test_finder.WebTestFinder(port, options) # Default case; not MSAN/ASAN so should not skip anything. tests = finder.skip_tests([], all_tests, None) self.assertEqual(tests, set()) # MSAN/ASAN, with no paths specified explicitly, so should skip both # idlharness tests. finder._options.enable_sanitizer = True tests = finder.skip_tests([], all_tests, None) self.assertEqual( tests, set([ 'external/wpt/dir1/dir2/idlharness.any.html', 'external/wpt/dir1/dir2/idlharness.any.worker.html' ])) # MSAN/ASAN, with one of the tests specified explicitly (and # --skipped=default), so should skip only the unspecified test. tests = finder.skip_tests( ['external/wpt/dir1/dir2/idlharness.any.html'], all_tests, None) self.assertEqual( tests, set(['external/wpt/dir1/dir2/idlharness.any.worker.html']))
def test_skip_tests_expectations(self): """Tests that tests are skipped based on to expectations and options.""" host = MockHost() port = host.port_factory.get('test-win-win7', None) all_tests = [ 'fast/css/passes.html', 'fast/css/fails.html', 'fast/css/times_out.html', 'fast/css/skip.html', ] # Patch port.tests() to return our tests port.tests = lambda paths: paths or all_tests options = optparse.Values({ 'no_expectations': False, 'enable_sanitizer': False, 'skipped': 'default', 'skip_timeouts': False, 'skip_failing_tests': False, }) finder = web_test_finder.WebTestFinder(port, options) expectations = test_expectations.TestExpectations(port) expectations.merge_raw_expectations( ('# results: [ Failure Timeout Skip ]' '\nfast/css/fails.html [ Failure ]' '\nfast/css/times_out.html [ Timeout ]' '\nfast/css/skip.html [ Skip ]')) # When run with default settings, we only skip the tests marked Skip. tests = finder.skip_tests([], all_tests, expectations) self.assertEqual(tests, set(['fast/css/skip.html'])) # Specify test on the command line; by default should not skip. tests = finder.skip_tests(['fast/css/skip.html'], all_tests, expectations) self.assertEqual(tests, set()) # Specify test on the command line, but always skip. finder._options.skipped = 'always' tests = finder.skip_tests(['fast/css/skip.html'], all_tests, expectations) self.assertEqual(tests, set(['fast/css/skip.html'])) finder._options.skipped = 'default' # Only run skip tests, aka skip all non-skipped tests. finder._options.skipped = 'only' tests = finder.skip_tests([], all_tests, expectations) self.assertEqual( tests, set([ 'fast/css/passes.html', 'fast/css/fails.html', 'fast/css/times_out.html' ])) finder._options.skipped = 'default' # Ignore any skip entries, aka never skip anything. finder._options.skipped = 'ignore' tests = finder.skip_tests([], all_tests, expectations) self.assertEqual(tests, set()) finder._options.skipped = 'default' # Skip tests that are marked TIMEOUT. finder._options.skip_timeouts = True tests = finder.skip_tests([], all_tests, expectations) self.assertEqual( tests, set(['fast/css/times_out.html', 'fast/css/skip.html'])) finder._options.skip_timeouts = False # Skip tests that are marked FAILURE finder._options.skip_failing_tests = True tests = finder.skip_tests([], all_tests, expectations) self.assertEqual(tests, set(['fast/css/fails.html', 'fast/css/skip.html'])) finder._options.skip_failing_tests = False # Disable expectations entirely; nothing should be skipped by default. finder._options.no_expectations = True tests = finder.skip_tests([], all_tests, expectations) self.assertEqual(tests, set())
def test_skip_tests_idlharness(self): """Tests that idlharness tests are skipped on MSAN/ASAN runs. See https://crbug.com/856601 """ host = MockHost() port = host.port_factory.get('test-win-win7', None) non_idlharness_test = 'external/wpt/dir1/dir2/foo.html' idlharness_test_1 = 'external/wpt/dir1/dir2/idlharness.any.html' idlharness_test_2 = 'external/wpt/dir1/dir2/idlharness.any.worker.html' all_tests = [ non_idlharness_test, idlharness_test_1, idlharness_test_2, ] # Patch port.tests() to return our tests port.tests = lambda paths: paths or all_tests options = optparse.Values({ 'no_expectations': False, 'enable_sanitizer': False, 'skipped': 'default', 'skip_timeouts': False, 'skip_failing_tests': False, }) finder = web_test_finder.WebTestFinder(port, options) # Default case; not MSAN/ASAN so should not skip anything. expectations = test_expectations.TestExpectations(port) tests = finder.skip_tests([], all_tests, expectations) self.assertEqual(tests, set()) for test in all_tests: self.assertTrue( expectations.get_expectations(test).is_default_pass) # MSAN/ASAN, with no paths specified explicitly, so should skip both # idlharness tests. expectations = test_expectations.TestExpectations(port) finder._options.enable_sanitizer = True tests = finder.skip_tests([], all_tests, expectations) self.assertEqual(tests, set([idlharness_test_1, idlharness_test_2])) self.assertTrue( expectations.get_expectations(non_idlharness_test).is_default_pass) self.assertEquals( expectations.get_expectations(idlharness_test_1).results, {'SKIP'}) self.assertEquals( expectations.get_expectations(idlharness_test_2).results, {'SKIP'}) # Disable expectations entirely; we should still skip the idlharness # tests but shouldn't touch the expectations parameter. finder._options.no_expectations = True tests = finder.skip_tests([], all_tests, None) self.assertEqual(tests, set([idlharness_test_1, idlharness_test_2])) # MSAN/ASAN, with one of the tests specified explicitly (and # --skipped=default), so should skip only the unspecified test. expectations = test_expectations.TestExpectations(port) tests = finder.skip_tests([idlharness_test_1], all_tests, expectations) self.assertEqual(tests, set([idlharness_test_2])) # Although we will run the test because it was specified explicitly, it # is still *expected* to Skip. This is consistent with how entries in # TestExpectations work. self.assertTrue( expectations.get_expectations(non_idlharness_test).is_default_pass) self.assertEquals( expectations.get_expectations(idlharness_test_1).results, {'SKIP'}) self.assertEquals( expectations.get_expectations(idlharness_test_2).results, {'SKIP'})