class SharderTests(unittest.TestCase): test_list = [ "http/tests/websocket/tests/unicode.htm", "animations/keyframes.html", "http/tests/security/view-source-no-refresh.html", "http/tests/websocket/tests/websocket-protocol-ignored.html", "fast/css/display-none-inline-style-change-crash.html", "http/tests/xmlhttprequest/supported-xml-content-types.html", "dom/html/level2/html/HTMLAnchorElement03.html", "ietestcenter/Javascript/11.1.5_4-4-c-1.html", "dom/html/level2/html/HTMLAnchorElement06.html", ] def get_test_input(self, test_file): return TestInput(Test(test_file), needs_servers=(test_file.startswith('http'))) def get_shards(self, num_workers, fully_parallel, test_list=None): port = TestPort(MockSystemHost()) self.sharder = Sharder(port.split_test) test_list = test_list or self.test_list return self.sharder.shard_tests([self.get_test_input(test) for test in test_list], num_workers, fully_parallel) def assert_shards(self, actual_shards, expected_shard_names): self.assertEqual(len(actual_shards), len(expected_shard_names)) for i, shard in enumerate(actual_shards): expected_shard_name, expected_test_names = expected_shard_names[i] self.assertEqual(shard.name, expected_shard_name) self.assertEqual([test_input.test_name for test_input in shard.test_inputs], expected_test_names) def test_shard_by_dir(self): result = self.get_shards(num_workers=2, fully_parallel=False) self.assert_shards(result, [('animations', ['animations/keyframes.html']), ('dom/html/level2/html', ['dom/html/level2/html/HTMLAnchorElement03.html', 'dom/html/level2/html/HTMLAnchorElement06.html']), ('fast/css', ['fast/css/display-none-inline-style-change-crash.html']), ('http/tests/security', ['http/tests/security/view-source-no-refresh.html']), ('http/tests/websocket/tests', ['http/tests/websocket/tests/unicode.htm', 'http/tests/websocket/tests/websocket-protocol-ignored.html']), ('http/tests/xmlhttprequest', ['http/tests/xmlhttprequest/supported-xml-content-types.html']), ('ietestcenter/Javascript', ['ietestcenter/Javascript/11.1.5_4-4-c-1.html'])]) def test_shard_every_file(self): result = self.get_shards(num_workers=2, fully_parallel=True) self.assert_shards(result, [('.', ['http/tests/websocket/tests/unicode.htm']), ('.', ['animations/keyframes.html']), ('.', ['http/tests/security/view-source-no-refresh.html']), ('.', ['http/tests/websocket/tests/websocket-protocol-ignored.html']), ('.', ['fast/css/display-none-inline-style-change-crash.html']), ('.', ['http/tests/xmlhttprequest/supported-xml-content-types.html']), ('.', ['dom/html/level2/html/HTMLAnchorElement03.html']), ('.', ['ietestcenter/Javascript/11.1.5_4-4-c-1.html']), ('.', ['dom/html/level2/html/HTMLAnchorElement06.html'])])
class SharderTests(unittest.TestCase): test_list = [ "http/tests/websocket/tests/unicode.htm", "animations/keyframes.html", "http/tests/security/view-source-no-refresh.html", "http/tests/websocket/tests/websocket-protocol-ignored.html", "fast/css/display-none-inline-style-change-crash.html", "http/tests/xmlhttprequest/supported-xml-content-types.html", "dom/html/level2/html/HTMLAnchorElement03.html", "ietestcenter/Javascript/11.1.5_4-4-c-1.html", "dom/html/level2/html/HTMLAnchorElement06.html", ] def get_test_input(self, test_file): return TestInput(test_file, needs_servers=(test_file.startswith('http'))) def get_shards(self, num_workers, fully_parallel, test_list=None): port = TestPort(MockSystemHost()) self.sharder = Sharder(port.split_test) test_list = test_list or self.test_list return self.sharder.shard_tests([self.get_test_input(test) for test in test_list], num_workers, fully_parallel) def assert_shards(self, actual_shards, expected_shard_names): self.assertEqual(len(actual_shards), len(expected_shard_names)) for i, shard in enumerate(actual_shards): expected_shard_name, expected_test_names = expected_shard_names[i] self.assertEqual(shard.name, expected_shard_name) self.assertEqual([test_input.test_name for test_input in shard.test_inputs], expected_test_names) def test_shard_by_dir(self): result = self.get_shards(num_workers=2, fully_parallel=False) self.assert_shards(result, [('animations', ['animations/keyframes.html']), ('dom/html/level2/html', ['dom/html/level2/html/HTMLAnchorElement03.html', 'dom/html/level2/html/HTMLAnchorElement06.html']), ('fast/css', ['fast/css/display-none-inline-style-change-crash.html']), ('http/tests/security', ['http/tests/security/view-source-no-refresh.html']), ('http/tests/websocket/tests', ['http/tests/websocket/tests/unicode.htm', 'http/tests/websocket/tests/websocket-protocol-ignored.html']), ('http/tests/xmlhttprequest', ['http/tests/xmlhttprequest/supported-xml-content-types.html']), ('ietestcenter/Javascript', ['ietestcenter/Javascript/11.1.5_4-4-c-1.html'])]) def test_shard_every_file(self): result = self.get_shards(num_workers=2, fully_parallel=True) self.assert_shards(result, [('.', ['http/tests/websocket/tests/unicode.htm']), ('.', ['animations/keyframes.html']), ('.', ['http/tests/security/view-source-no-refresh.html']), ('.', ['http/tests/websocket/tests/websocket-protocol-ignored.html']), ('.', ['fast/css/display-none-inline-style-change-crash.html']), ('.', ['http/tests/xmlhttprequest/supported-xml-content-types.html']), ('.', ['dom/html/level2/html/HTMLAnchorElement03.html']), ('.', ['ietestcenter/Javascript/11.1.5_4-4-c-1.html']), ('.', ['dom/html/level2/html/HTMLAnchorElement06.html'])])
class SharderTests(unittest.TestCase): test_list = [ "http/tests/websocket/tests/unicode.htm", "animations/keyframes.html", "http/tests/security/view-source-no-refresh.html", "http/tests/websocket/tests/websocket-protocol-ignored.html", "fast/css/display-none-inline-style-change-crash.html", "http/tests/xmlhttprequest/supported-xml-content-types.html", "dom/html/level2/html/HTMLAnchorElement03.html", "ietestcenter/Javascript/11.1.5_4-4-c-1.html", "dom/html/level2/html/HTMLAnchorElement06.html", "perf/object-keys.html", "virtual/threaded/dir/test.html", "virtual/threaded/fast/foo/test.html", ] def get_test_input(self, test_file): return TestInput(test_file, requires_lock=(test_file.startswith('http') or test_file.startswith('perf'))) def get_shards(self, num_workers, fully_parallel, run_singly, test_list=None, max_locked_shards=1): port = TestPort(MockSystemHost()) self.sharder = Sharder(port.split_test, max_locked_shards) test_list = test_list or self.test_list return self.sharder.shard_tests([self.get_test_input(test) for test in test_list], num_workers, fully_parallel, run_singly) def assert_shards(self, actual_shards, expected_shard_names): self.assertEqual(len(actual_shards), len(expected_shard_names)) for i, shard in enumerate(actual_shards): expected_shard_name, expected_test_names = expected_shard_names[i] self.assertEqual(shard.name, expected_shard_name) self.assertEqual([test_input.test_name for test_input in shard.test_inputs], expected_test_names) def test_shard_by_dir(self): locked, unlocked = self.get_shards(num_workers=2, fully_parallel=False, run_singly=False) # Note that although there are tests in multiple dirs that need locks, # they are crammed into a single shard in order to reduce the # of # workers hitting the server at once. self.assert_shards(locked, [('locked_shard_1', ['http/tests/security/view-source-no-refresh.html', 'http/tests/websocket/tests/unicode.htm', 'http/tests/websocket/tests/websocket-protocol-ignored.html', 'http/tests/xmlhttprequest/supported-xml-content-types.html', 'perf/object-keys.html'])]) self.assert_shards(unlocked, [('virtual/threaded/dir', ['virtual/threaded/dir/test.html']), ('virtual/threaded/fast/foo', ['virtual/threaded/fast/foo/test.html']), ('animations', ['animations/keyframes.html']), ('dom/html/level2/html', ['dom/html/level2/html/HTMLAnchorElement03.html', 'dom/html/level2/html/HTMLAnchorElement06.html']), ('fast/css', ['fast/css/display-none-inline-style-change-crash.html']), ('ietestcenter/Javascript', ['ietestcenter/Javascript/11.1.5_4-4-c-1.html'])]) def test_shard_every_file(self): locked, unlocked = self.get_shards(num_workers=2, fully_parallel=True, max_locked_shards=2, run_singly=False) self.assert_shards(locked, [('locked_shard_1', ['http/tests/websocket/tests/unicode.htm', 'http/tests/security/view-source-no-refresh.html', 'http/tests/websocket/tests/websocket-protocol-ignored.html']), ('locked_shard_2', ['http/tests/xmlhttprequest/supported-xml-content-types.html', 'perf/object-keys.html'])]), self.assert_shards(unlocked, [('virtual/threaded/dir', ['virtual/threaded/dir/test.html']), ('virtual/threaded/fast/foo', ['virtual/threaded/fast/foo/test.html']), ('.', ['animations/keyframes.html']), ('.', ['fast/css/display-none-inline-style-change-crash.html']), ('.', ['dom/html/level2/html/HTMLAnchorElement03.html']), ('.', ['ietestcenter/Javascript/11.1.5_4-4-c-1.html']), ('.', ['dom/html/level2/html/HTMLAnchorElement06.html'])]) def test_shard_in_two(self): locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False, run_singly=False) self.assert_shards(locked, [('locked_tests', ['http/tests/websocket/tests/unicode.htm', 'http/tests/security/view-source-no-refresh.html', 'http/tests/websocket/tests/websocket-protocol-ignored.html', 'http/tests/xmlhttprequest/supported-xml-content-types.html', 'perf/object-keys.html'])]) self.assert_shards(unlocked, [('unlocked_tests', ['animations/keyframes.html', 'fast/css/display-none-inline-style-change-crash.html', 'dom/html/level2/html/HTMLAnchorElement03.html', 'ietestcenter/Javascript/11.1.5_4-4-c-1.html', 'dom/html/level2/html/HTMLAnchorElement06.html', 'virtual/threaded/dir/test.html', 'virtual/threaded/fast/foo/test.html'])]) def test_shard_in_two_has_no_locked_shards(self): locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False, run_singly=False, test_list=['animations/keyframe.html']) self.assertEqual(len(locked), 0) self.assertEqual(len(unlocked), 1) def test_shard_in_two_has_no_unlocked_shards(self): locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False, run_singly=False, test_list=['http/tests/websocket/tests/unicode.htm']) self.assertEqual(len(locked), 1) self.assertEqual(len(unlocked), 0) def test_multiple_locked_shards(self): locked, unlocked = self.get_shards(num_workers=4, fully_parallel=False, max_locked_shards=2, run_singly=False) self.assert_shards(locked, [('locked_shard_1', ['http/tests/security/view-source-no-refresh.html', 'http/tests/websocket/tests/unicode.htm', 'http/tests/websocket/tests/websocket-protocol-ignored.html']), ('locked_shard_2', ['http/tests/xmlhttprequest/supported-xml-content-types.html', 'perf/object-keys.html'])]) locked, unlocked = self.get_shards(num_workers=4, fully_parallel=False, run_singly=False) self.assert_shards(locked, [('locked_shard_1', ['http/tests/security/view-source-no-refresh.html', 'http/tests/websocket/tests/unicode.htm', 'http/tests/websocket/tests/websocket-protocol-ignored.html', 'http/tests/xmlhttprequest/supported-xml-content-types.html', 'perf/object-keys.html'])]) def test_virtual_shards(self): # With run_singly=False, we try to keep all of the tests in a virtual suite together even # when fully_parallel=True, so that we don't restart every time the command line args change. locked, unlocked = self.get_shards(num_workers=2, fully_parallel=True, max_locked_shards=2, run_singly=False, test_list=['virtual/foo/bar1.html', 'virtual/foo/bar2.html']) self.assert_shards(unlocked, [('virtual/foo', ['virtual/foo/bar1.html', 'virtual/foo/bar2.html'])]) # But, with run_singly=True, we have to restart every time anyway, so we want full parallelism. locked, unlocked = self.get_shards(num_workers=2, fully_parallel=True, max_locked_shards=2, run_singly=True, test_list=['virtual/foo/bar1.html', 'virtual/foo/bar2.html']) self.assert_shards(unlocked, [('.', ['virtual/foo/bar1.html']), ('.', ['virtual/foo/bar2.html'])])
class SharderTests(unittest.TestCase): test_list = [ "http/tests/websocket/tests/unicode.htm", "animations/keyframes.html", "http/tests/security/view-source-no-refresh.html", "http/tests/websocket/tests/websocket-protocol-ignored.html", "fast/css/display-none-inline-style-change-crash.html", "http/tests/xmlhttprequest/supported-xml-content-types.html", "dom/html/level2/html/HTMLAnchorElement03.html", "dom/html/level2/html/HTMLAnchorElement06.html", "perf/object-keys.html", "virtual/threaded/dir/test.html", "virtual/threaded/fast/foo/test.html", ] def get_test_input(self, test_file): return TestInput(test_file, requires_lock=(test_file.startswith('http') or test_file.startswith('perf'))) def get_shards(self, num_workers, fully_parallel, run_singly, test_list=None, max_locked_shards=1): port = TestPort(MockSystemHost()) self.sharder = Sharder(port.split_test, max_locked_shards) test_list = test_list or self.test_list return self.sharder.shard_tests( [self.get_test_input(test) for test in test_list], num_workers, fully_parallel, run_singly) def assert_shards(self, actual_shards, expected_shard_names): self.assertEqual(len(actual_shards), len(expected_shard_names)) for i, shard in enumerate(actual_shards): expected_shard_name, expected_test_names = expected_shard_names[i] self.assertEqual(shard.name, expected_shard_name) self.assertEqual( [test_input.test_name for test_input in shard.test_inputs], expected_test_names) def test_shard_by_dir(self): locked, unlocked = self.get_shards(num_workers=2, fully_parallel=False, run_singly=False) # Note that although there are tests in multiple dirs that need locks, # they are crammed into a single shard in order to reduce the # of # workers hitting the server at once. self.assert_shards(locked, [('locked_shard_1', [ 'http/tests/security/view-source-no-refresh.html', 'http/tests/websocket/tests/unicode.htm', 'http/tests/websocket/tests/websocket-protocol-ignored.html', 'http/tests/xmlhttprequest/supported-xml-content-types.html', 'perf/object-keys.html' ])]) self.assert_shards( unlocked, [('virtual/threaded/dir', ['virtual/threaded/dir/test.html']), ('virtual/threaded/fast/foo', ['virtual/threaded/fast/foo/test.html']), ('animations', ['animations/keyframes.html']), ('dom/html/level2/html', [ 'dom/html/level2/html/HTMLAnchorElement03.html', 'dom/html/level2/html/HTMLAnchorElement06.html' ]), ('fast/css', ['fast/css/display-none-inline-style-change-crash.html'])]) def test_shard_every_file(self): locked, unlocked = self.get_shards(num_workers=2, fully_parallel=True, max_locked_shards=2, run_singly=False) self.assert_shards( locked, [('locked_shard_1', [ 'http/tests/websocket/tests/unicode.htm', 'http/tests/security/view-source-no-refresh.html', 'http/tests/websocket/tests/websocket-protocol-ignored.html' ]), ('locked_shard_2', [ 'http/tests/xmlhttprequest/supported-xml-content-types.html', 'perf/object-keys.html' ])]), self.assert_shards( unlocked, [('virtual/threaded/dir', ['virtual/threaded/dir/test.html']), ('virtual/threaded/fast/foo', ['virtual/threaded/fast/foo/test.html']), ('.', ['animations/keyframes.html']), ('.', ['fast/css/display-none-inline-style-change-crash.html']), ('.', ['dom/html/level2/html/HTMLAnchorElement03.html']), ('.', ['dom/html/level2/html/HTMLAnchorElement06.html'])]) def test_shard_in_two(self): locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False, run_singly=False) self.assert_shards(locked, [('locked_tests', [ 'http/tests/websocket/tests/unicode.htm', 'http/tests/security/view-source-no-refresh.html', 'http/tests/websocket/tests/websocket-protocol-ignored.html', 'http/tests/xmlhttprequest/supported-xml-content-types.html', 'perf/object-keys.html' ])]) self.assert_shards(unlocked, [('unlocked_tests', [ 'animations/keyframes.html', 'fast/css/display-none-inline-style-change-crash.html', 'dom/html/level2/html/HTMLAnchorElement03.html', 'dom/html/level2/html/HTMLAnchorElement06.html', 'virtual/threaded/dir/test.html', 'virtual/threaded/fast/foo/test.html' ])]) def test_shard_in_two_has_no_locked_shards(self): locked, unlocked = self.get_shards( num_workers=1, fully_parallel=False, run_singly=False, test_list=['animations/keyframe.html']) self.assertEqual(len(locked), 0) self.assertEqual(len(unlocked), 1) def test_shard_in_two_has_no_unlocked_shards(self): locked, unlocked = self.get_shards( num_workers=1, fully_parallel=False, run_singly=False, test_list=['http/tests/websocket/tests/unicode.htm']) self.assertEqual(len(locked), 1) self.assertEqual(len(unlocked), 0) def test_multiple_locked_shards(self): locked, unlocked = self.get_shards(num_workers=4, fully_parallel=False, max_locked_shards=2, run_singly=False) self.assert_shards( locked, [('locked_shard_1', [ 'http/tests/security/view-source-no-refresh.html', 'http/tests/websocket/tests/unicode.htm', 'http/tests/websocket/tests/websocket-protocol-ignored.html' ]), ('locked_shard_2', [ 'http/tests/xmlhttprequest/supported-xml-content-types.html', 'perf/object-keys.html' ])]) locked, unlocked = self.get_shards(num_workers=4, fully_parallel=False, run_singly=False) self.assert_shards(locked, [('locked_shard_1', [ 'http/tests/security/view-source-no-refresh.html', 'http/tests/websocket/tests/unicode.htm', 'http/tests/websocket/tests/websocket-protocol-ignored.html', 'http/tests/xmlhttprequest/supported-xml-content-types.html', 'perf/object-keys.html' ])]) def test_virtual_shards(self): # With run_singly=False, we try to keep all of the tests in a virtual suite together even # when fully_parallel=True, so that we don't restart every time the command line args change. locked, unlocked = self.get_shards( num_workers=2, fully_parallel=True, max_locked_shards=2, run_singly=False, test_list=['virtual/foo/bar1.html', 'virtual/foo/bar2.html']) self.assert_shards(unlocked, [ ('virtual/foo', ['virtual/foo/bar1.html', 'virtual/foo/bar2.html']) ]) # But, with run_singly=True, we have to restart every time anyway, so we want full parallelism. locked, unlocked = self.get_shards( num_workers=2, fully_parallel=True, max_locked_shards=2, run_singly=True, test_list=['virtual/foo/bar1.html', 'virtual/foo/bar2.html']) self.assert_shards(unlocked, [('.', ['virtual/foo/bar1.html']), ('.', ['virtual/foo/bar2.html'])])
class SharderTests(unittest.TestCase): test_list = [ "http/tests/websocket/tests/unicode.htm", "animations/keyframes.html", "http/tests/security/view-source-no-refresh.html", "http/tests/websocket/tests/websocket-protocol-ignored.html", "fast/css/display-none-inline-style-change-crash.html", "http/tests/xmlhttprequest/supported-xml-content-types.html", "dom/html/level2/html/HTMLAnchorElement03.html", "ietestcenter/Javascript/11.1.5_4-4-c-1.html", "dom/html/level2/html/HTMLAnchorElement06.html", "perf/object-keys.html", ] ref_tests = [ "http/tests/security/view-source-no-refresh.html", "http/tests/websocket/tests/websocket-protocol-ignored.html", "ietestcenter/Javascript/11.1.5_4-4-c-1.html", "dom/html/level2/html/HTMLAnchorElement06.html", ] def get_test_input(self, test_file): return TestInput(test_file, requires_lock=(test_file.startswith('http') or test_file.startswith('perf')), reference_files=(['foo'] if test_file in self.ref_tests else None)) def get_shards(self, num_workers, fully_parallel, shard_ref_tests=False, test_list=None, max_locked_shards=1): def split(test_name): idx = test_name.rfind('/') if idx != -1: return (test_name[0:idx], test_name[idx + 1:]) self.sharder = Sharder(split, '/', max_locked_shards) test_list = test_list or self.test_list return self.sharder.shard_tests([self.get_test_input(test) for test in test_list], num_workers, fully_parallel, shard_ref_tests) def assert_shards(self, actual_shards, expected_shard_names): self.assertEquals(len(actual_shards), len(expected_shard_names)) for i, shard in enumerate(actual_shards): expected_shard_name, expected_test_names = expected_shard_names[i] self.assertEquals(shard.name, expected_shard_name) self.assertEquals([test_input.test_name for test_input in shard.test_inputs], expected_test_names) def test_shard_by_dir(self): locked, unlocked = self.get_shards(num_workers=2, fully_parallel=False) # Note that although there are tests in multiple dirs that need locks, # they are crammed into a single shard in order to reduce the # of # workers hitting the server at once. self.assert_shards(locked, [('locked_shard_1', ['http/tests/security/view-source-no-refresh.html', 'http/tests/websocket/tests/unicode.htm', 'http/tests/websocket/tests/websocket-protocol-ignored.html', 'http/tests/xmlhttprequest/supported-xml-content-types.html', 'perf/object-keys.html'])]) self.assert_shards(unlocked, [('animations', ['animations/keyframes.html']), ('dom/html/level2/html', ['dom/html/level2/html/HTMLAnchorElement03.html', 'dom/html/level2/html/HTMLAnchorElement06.html']), ('fast/css', ['fast/css/display-none-inline-style-change-crash.html']), ('ietestcenter/Javascript', ['ietestcenter/Javascript/11.1.5_4-4-c-1.html'])]) def test_shard_by_dir_sharding_ref_tests(self): locked, unlocked = self.get_shards(num_workers=2, fully_parallel=False, shard_ref_tests=True) # Note that although there are tests in multiple dirs that need locks, # they are crammed into a single shard in order to reduce the # of # workers hitting the server at once. self.assert_shards(locked, [('locked_shard_1', ['http/tests/websocket/tests/unicode.htm', 'http/tests/xmlhttprequest/supported-xml-content-types.html', 'perf/object-keys.html', 'http/tests/security/view-source-no-refresh.html', 'http/tests/websocket/tests/websocket-protocol-ignored.html'])]) self.assert_shards(unlocked, [('animations', ['animations/keyframes.html']), ('dom/html/level2/html', ['dom/html/level2/html/HTMLAnchorElement03.html']), ('fast/css', ['fast/css/display-none-inline-style-change-crash.html']), ('~ref:dom/html/level2/html', ['dom/html/level2/html/HTMLAnchorElement06.html']), ('~ref:ietestcenter/Javascript', ['ietestcenter/Javascript/11.1.5_4-4-c-1.html'])]) def test_shard_every_file(self): locked, unlocked = self.get_shards(num_workers=2, fully_parallel=True) self.assert_shards(locked, [('.', ['http/tests/websocket/tests/unicode.htm']), ('.', ['http/tests/security/view-source-no-refresh.html']), ('.', ['http/tests/websocket/tests/websocket-protocol-ignored.html']), ('.', ['http/tests/xmlhttprequest/supported-xml-content-types.html']), ('.', ['perf/object-keys.html'])]), self.assert_shards(unlocked, [('.', ['animations/keyframes.html']), ('.', ['fast/css/display-none-inline-style-change-crash.html']), ('.', ['dom/html/level2/html/HTMLAnchorElement03.html']), ('.', ['ietestcenter/Javascript/11.1.5_4-4-c-1.html']), ('.', ['dom/html/level2/html/HTMLAnchorElement06.html'])]) def test_shard_in_two(self): locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False) self.assert_shards(locked, [('locked_tests', ['http/tests/websocket/tests/unicode.htm', 'http/tests/security/view-source-no-refresh.html', 'http/tests/websocket/tests/websocket-protocol-ignored.html', 'http/tests/xmlhttprequest/supported-xml-content-types.html', 'perf/object-keys.html'])]) self.assert_shards(unlocked, [('unlocked_tests', ['animations/keyframes.html', 'fast/css/display-none-inline-style-change-crash.html', 'dom/html/level2/html/HTMLAnchorElement03.html', 'ietestcenter/Javascript/11.1.5_4-4-c-1.html', 'dom/html/level2/html/HTMLAnchorElement06.html'])]) def test_shard_in_two_sharding_ref_tests(self): locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False, shard_ref_tests=True) self.assert_shards(locked, [('locked_tests', ['http/tests/websocket/tests/unicode.htm', 'http/tests/xmlhttprequest/supported-xml-content-types.html', 'perf/object-keys.html', 'http/tests/security/view-source-no-refresh.html', 'http/tests/websocket/tests/websocket-protocol-ignored.html'])]) self.assert_shards(unlocked, [('unlocked_tests', ['animations/keyframes.html', 'fast/css/display-none-inline-style-change-crash.html', 'dom/html/level2/html/HTMLAnchorElement03.html', 'ietestcenter/Javascript/11.1.5_4-4-c-1.html', 'dom/html/level2/html/HTMLAnchorElement06.html'])]) def test_shard_in_two_has_no_locked_shards(self): locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False, test_list=['animations/keyframe.html']) self.assertEquals(len(locked), 0) self.assertEquals(len(unlocked), 1) def test_shard_in_two_has_no_unlocked_shards(self): locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False, test_list=['http/tests/websocket/tests/unicode.htm']) self.assertEquals(len(locked), 1) self.assertEquals(len(unlocked), 0) def test_multiple_locked_shards(self): locked, unlocked = self.get_shards(num_workers=4, fully_parallel=False, max_locked_shards=2) self.assert_shards(locked, [('locked_shard_1', ['http/tests/security/view-source-no-refresh.html', 'http/tests/websocket/tests/unicode.htm', 'http/tests/websocket/tests/websocket-protocol-ignored.html']), ('locked_shard_2', ['http/tests/xmlhttprequest/supported-xml-content-types.html', 'perf/object-keys.html'])]) locked, unlocked = self.get_shards(num_workers=4, fully_parallel=False) self.assert_shards(locked, [('locked_shard_1', ['http/tests/security/view-source-no-refresh.html', 'http/tests/websocket/tests/unicode.htm', 'http/tests/websocket/tests/websocket-protocol-ignored.html', 'http/tests/xmlhttprequest/supported-xml-content-types.html', 'perf/object-keys.html'])])
class SharderTests(unittest.TestCase): test_list = [ "http/tests/websocket/tests/unicode.htm", "animations/keyframes.html", "http/tests/security/view-source-no-refresh.html", "http/tests/websocket/tests/websocket-protocol-ignored.html", "fast/css/display-none-inline-style-change-crash.html", "http/tests/xmlhttprequest/supported-xml-content-types.html", "dom/html/level2/html/HTMLAnchorElement03.html", "ietestcenter/Javascript/11.1.5_4-4-c-1.html", "dom/html/level2/html/HTMLAnchorElement06.html", "perf/object-keys.html", ] def get_test_input(self, test_file): return TestInput(test_file, requires_lock=(test_file.startswith('http') or test_file.startswith('perf'))) def get_shards(self, num_workers, fully_parallel, test_list=None, max_locked_shards=1): port = TestPort(MockSystemHost()) self.sharder = Sharder(port.split_test, max_locked_shards) test_list = test_list or self.test_list return self.sharder.shard_tests( [self.get_test_input(test) for test in test_list], num_workers, fully_parallel) def assert_shards(self, actual_shards, expected_shard_names): self.assertEqual(len(actual_shards), len(expected_shard_names)) for i, shard in enumerate(actual_shards): expected_shard_name, expected_test_names = expected_shard_names[i] self.assertEqual(shard.name, expected_shard_name) self.assertEqual( [test_input.test_name for test_input in shard.test_inputs], expected_test_names) def test_shard_by_dir(self): locked, unlocked = self.get_shards(num_workers=2, fully_parallel=False) # Note that although there are tests in multiple dirs that need locks, # they are crammed into a single shard in order to reduce the # of # workers hitting the server at once. self.assert_shards(locked, [('locked_shard_1', [ 'http/tests/security/view-source-no-refresh.html', 'http/tests/websocket/tests/unicode.htm', 'http/tests/websocket/tests/websocket-protocol-ignored.html', 'http/tests/xmlhttprequest/supported-xml-content-types.html', 'perf/object-keys.html' ])]) self.assert_shards( unlocked, [('animations', ['animations/keyframes.html']), ('dom/html/level2/html', [ 'dom/html/level2/html/HTMLAnchorElement03.html', 'dom/html/level2/html/HTMLAnchorElement06.html' ]), ('fast/css', ['fast/css/display-none-inline-style-change-crash.html']), ('ietestcenter/Javascript', ['ietestcenter/Javascript/11.1.5_4-4-c-1.html'])]) def test_shard_every_file(self): locked, unlocked = self.get_shards(num_workers=2, fully_parallel=True) self.assert_shards( locked, [('.', ['http/tests/websocket/tests/unicode.htm']), ('.', ['http/tests/security/view-source-no-refresh.html']), ('.', ['http/tests/websocket/tests/websocket-protocol-ignored.html']), ('.', ['http/tests/xmlhttprequest/supported-xml-content-types.html']), ('.', ['perf/object-keys.html'])]), self.assert_shards( unlocked, [('.', ['animations/keyframes.html']), ('.', ['fast/css/display-none-inline-style-change-crash.html']), ('.', ['dom/html/level2/html/HTMLAnchorElement03.html']), ('.', ['ietestcenter/Javascript/11.1.5_4-4-c-1.html']), ('.', ['dom/html/level2/html/HTMLAnchorElement06.html'])]) def test_shard_in_two(self): locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False) self.assert_shards(locked, [('locked_tests', [ 'http/tests/websocket/tests/unicode.htm', 'http/tests/security/view-source-no-refresh.html', 'http/tests/websocket/tests/websocket-protocol-ignored.html', 'http/tests/xmlhttprequest/supported-xml-content-types.html', 'perf/object-keys.html' ])]) self.assert_shards(unlocked, [('unlocked_tests', [ 'animations/keyframes.html', 'fast/css/display-none-inline-style-change-crash.html', 'dom/html/level2/html/HTMLAnchorElement03.html', 'ietestcenter/Javascript/11.1.5_4-4-c-1.html', 'dom/html/level2/html/HTMLAnchorElement06.html' ])]) def test_shard_in_two_has_no_locked_shards(self): locked, unlocked = self.get_shards( num_workers=1, fully_parallel=False, test_list=['animations/keyframe.html']) self.assertEqual(len(locked), 0) self.assertEqual(len(unlocked), 1) def test_shard_in_two_has_no_unlocked_shards(self): locked, unlocked = self.get_shards( num_workers=1, fully_parallel=False, test_list=['http/tests/websocket/tests/unicode.htm']) self.assertEqual(len(locked), 1) self.assertEqual(len(unlocked), 0) def test_multiple_locked_shards(self): locked, unlocked = self.get_shards(num_workers=4, fully_parallel=False, max_locked_shards=2) self.assert_shards( locked, [('locked_shard_1', [ 'http/tests/security/view-source-no-refresh.html', 'http/tests/websocket/tests/unicode.htm', 'http/tests/websocket/tests/websocket-protocol-ignored.html' ]), ('locked_shard_2', [ 'http/tests/xmlhttprequest/supported-xml-content-types.html', 'perf/object-keys.html' ])]) locked, unlocked = self.get_shards(num_workers=4, fully_parallel=False) self.assert_shards(locked, [('locked_shard_1', [ 'http/tests/security/view-source-no-refresh.html', 'http/tests/websocket/tests/unicode.htm', 'http/tests/websocket/tests/websocket-protocol-ignored.html', 'http/tests/xmlhttprequest/supported-xml-content-types.html', 'perf/object-keys.html' ])])
class SharderTests(unittest.TestCase): test_list = [ "http/tests/websocket/tests/unicode.htm", "animations/keyframes.html", "http/tests/security/view-source-no-refresh.html", "http/tests/websocket/tests/websocket-protocol-ignored.html", "fast/css/display-none-inline-style-change-crash.html", "http/tests/xmlhttprequest/supported-xml-content-types.html", "dom/html/level2/html/HTMLAnchorElement03.html", "ietestcenter/Javascript/11.1.5_4-4-c-1.html", "dom/html/level2/html/HTMLAnchorElement06.html", "perf/object-keys.html", ] def get_test_input(self, test_file): return TestInput(test_file, requires_lock=(test_file.startswith("http") or test_file.startswith("perf"))) def get_shards(self, num_workers, fully_parallel, test_list=None, max_locked_shards=1): port = TestPort(MockSystemHost()) self.sharder = Sharder(port.split_test, max_locked_shards) test_list = test_list or self.test_list return self.sharder.shard_tests([self.get_test_input(test) for test in test_list], num_workers, fully_parallel) def assert_shards(self, actual_shards, expected_shard_names): self.assertEqual(len(actual_shards), len(expected_shard_names)) for i, shard in enumerate(actual_shards): expected_shard_name, expected_test_names = expected_shard_names[i] self.assertEqual(shard.name, expected_shard_name) self.assertEqual([test_input.test_name for test_input in shard.test_inputs], expected_test_names) def test_shard_by_dir(self): locked, unlocked = self.get_shards(num_workers=2, fully_parallel=False) # Note that although there are tests in multiple dirs that need locks, # they are crammed into a single shard in order to reduce the # of # workers hitting the server at once. self.assert_shards( locked, [ ( "locked_shard_1", [ "http/tests/security/view-source-no-refresh.html", "http/tests/websocket/tests/unicode.htm", "http/tests/websocket/tests/websocket-protocol-ignored.html", "http/tests/xmlhttprequest/supported-xml-content-types.html", "perf/object-keys.html", ], ) ], ) self.assert_shards( unlocked, [ ("animations", ["animations/keyframes.html"]), ( "dom/html/level2/html", ["dom/html/level2/html/HTMLAnchorElement03.html", "dom/html/level2/html/HTMLAnchorElement06.html"], ), ("fast/css", ["fast/css/display-none-inline-style-change-crash.html"]), ("ietestcenter/Javascript", ["ietestcenter/Javascript/11.1.5_4-4-c-1.html"]), ], ) def test_shard_every_file(self): locked, unlocked = self.get_shards(num_workers=2, fully_parallel=True) self.assert_shards( locked, [ (".", ["http/tests/websocket/tests/unicode.htm"]), (".", ["http/tests/security/view-source-no-refresh.html"]), (".", ["http/tests/websocket/tests/websocket-protocol-ignored.html"]), (".", ["http/tests/xmlhttprequest/supported-xml-content-types.html"]), (".", ["perf/object-keys.html"]), ], ), self.assert_shards( unlocked, [ (".", ["animations/keyframes.html"]), (".", ["fast/css/display-none-inline-style-change-crash.html"]), (".", ["dom/html/level2/html/HTMLAnchorElement03.html"]), (".", ["ietestcenter/Javascript/11.1.5_4-4-c-1.html"]), (".", ["dom/html/level2/html/HTMLAnchorElement06.html"]), ], ) def test_shard_in_two(self): locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False) self.assert_shards( locked, [ ( "locked_tests", [ "http/tests/websocket/tests/unicode.htm", "http/tests/security/view-source-no-refresh.html", "http/tests/websocket/tests/websocket-protocol-ignored.html", "http/tests/xmlhttprequest/supported-xml-content-types.html", "perf/object-keys.html", ], ) ], ) self.assert_shards( unlocked, [ ( "unlocked_tests", [ "animations/keyframes.html", "fast/css/display-none-inline-style-change-crash.html", "dom/html/level2/html/HTMLAnchorElement03.html", "ietestcenter/Javascript/11.1.5_4-4-c-1.html", "dom/html/level2/html/HTMLAnchorElement06.html", ], ) ], ) def test_shard_in_two_has_no_locked_shards(self): locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False, test_list=["animations/keyframe.html"]) self.assertEqual(len(locked), 0) self.assertEqual(len(unlocked), 1) def test_shard_in_two_has_no_unlocked_shards(self): locked, unlocked = self.get_shards( num_workers=1, fully_parallel=False, test_list=["http/tests/websocket/tests/unicode.htm"] ) self.assertEqual(len(locked), 1) self.assertEqual(len(unlocked), 0) def test_multiple_locked_shards(self): locked, unlocked = self.get_shards(num_workers=4, fully_parallel=False, max_locked_shards=2) self.assert_shards( locked, [ ( "locked_shard_1", [ "http/tests/security/view-source-no-refresh.html", "http/tests/websocket/tests/unicode.htm", "http/tests/websocket/tests/websocket-protocol-ignored.html", ], ), ( "locked_shard_2", ["http/tests/xmlhttprequest/supported-xml-content-types.html", "perf/object-keys.html"], ), ], ) locked, unlocked = self.get_shards(num_workers=4, fully_parallel=False) self.assert_shards( locked, [ ( "locked_shard_1", [ "http/tests/security/view-source-no-refresh.html", "http/tests/websocket/tests/unicode.htm", "http/tests/websocket/tests/websocket-protocol-ignored.html", "http/tests/xmlhttprequest/supported-xml-content-types.html", "perf/object-keys.html", ], ) ], )