def setUp(self):
def split(test_name):
idx = test_name.rfind('/')
if idx != -1:
return (test_name[0:idx], test_name[idx + 1:])
self.sharder = Sharder(split, '/', 1)
def get_shards(self, num_workers, fully_parallel, test_list=None):
port = TestPort(MockSystemHost())
self.sharder = Sharder(port.split_test)
test_list = test_list or self.test_list
return self.sharder.shard_tests(
[self.get_test_input(test) for test in test_list], num_workers,
fully_parallel)
def get_shards(self,
num_workers,
fully_parallel,
test_list=None,
max_locked_shards=1):
def split(test_name):
idx = test_name.rfind('/')
if idx != -1:
return (test_name[0:idx], test_name[idx + 1:])
self.sharder = Sharder(split, '/', max_locked_shards)
test_list = test_list or self.test_list
return self.sharder.shard_tests(
[self.get_test_input(test) for test in test_list], num_workers,
fully_parallel)
def setUp(self):
def split(test_name):
idx = test_name.rfind('/')
if idx != -1:
return (test_name[0:idx], test_name[idx + 1:])
self.sharder = Sharder(split, '/', 1)
def get_shards(self, num_workers, fully_parallel, shard_ref_tests=False, test_list=None, max_locked_shards=1):
def split(test_name):
idx = test_name.rfind('/')
if idx != -1:
return (test_name[0:idx], test_name[idx + 1:])
self.sharder = Sharder(split, '/', max_locked_shards)
test_list = test_list or self.test_list
return self.sharder.shard_tests([self.get_test_input(test) for test in test_list], num_workers, fully_parallel, shard_ref_tests)
class SharderTests(unittest.TestCase):
test_list = [
"http/tests/websocket/tests/unicode.htm",
"animations/keyframes.html",
"http/tests/security/view-source-no-refresh.html",
"http/tests/websocket/tests/websocket-protocol-ignored.html",
"fast/css/display-none-inline-style-change-crash.html",
"http/tests/xmlhttprequest/supported-xml-content-types.html",
"dom/html/level2/html/HTMLAnchorElement03.html",
"ietestcenter/Javascript/11.1.5_4-4-c-1.html",
"dom/html/level2/html/HTMLAnchorElement06.html",
]
def get_test_input(self, test_file):
return TestInput(Test(test_file), needs_servers=(test_file.startswith('http')))
def get_shards(self, num_workers, fully_parallel, test_list=None):
port = TestPort(MockSystemHost())
self.sharder = Sharder(port.split_test)
test_list = test_list or self.test_list
return self.sharder.shard_tests([self.get_test_input(test) for test in test_list], num_workers, fully_parallel)
def assert_shards(self, actual_shards, expected_shard_names):
self.assertEqual(len(actual_shards), len(expected_shard_names))
for i, shard in enumerate(actual_shards):
expected_shard_name, expected_test_names = expected_shard_names[i]
self.assertEqual(shard.name, expected_shard_name)
self.assertEqual([test_input.test_name for test_input in shard.test_inputs],
expected_test_names)
def test_shard_by_dir(self):
result = self.get_shards(num_workers=2, fully_parallel=False)
self.assert_shards(result,
[('animations', ['animations/keyframes.html']),
('dom/html/level2/html', ['dom/html/level2/html/HTMLAnchorElement03.html',
'dom/html/level2/html/HTMLAnchorElement06.html']),
('fast/css', ['fast/css/display-none-inline-style-change-crash.html']),
('http/tests/security', ['http/tests/security/view-source-no-refresh.html']),
('http/tests/websocket/tests', ['http/tests/websocket/tests/unicode.htm', 'http/tests/websocket/tests/websocket-protocol-ignored.html']),
('http/tests/xmlhttprequest', ['http/tests/xmlhttprequest/supported-xml-content-types.html']),
('ietestcenter/Javascript', ['ietestcenter/Javascript/11.1.5_4-4-c-1.html'])])
def test_shard_every_file(self):
result = self.get_shards(num_workers=2, fully_parallel=True)
self.assert_shards(result,
[('.', ['http/tests/websocket/tests/unicode.htm']),
('.', ['animations/keyframes.html']),
('.', ['http/tests/security/view-source-no-refresh.html']),
('.', ['http/tests/websocket/tests/websocket-protocol-ignored.html']),
('.', ['fast/css/display-none-inline-style-change-crash.html']),
('.', ['http/tests/xmlhttprequest/supported-xml-content-types.html']),
('.', ['dom/html/level2/html/HTMLAnchorElement03.html']),
('.', ['ietestcenter/Javascript/11.1.5_4-4-c-1.html']),
('.', ['dom/html/level2/html/HTMLAnchorElement06.html'])])
class SharderTests(unittest.TestCase):
test_list = [
"http/tests/websocket/tests/unicode.htm",
"animations/keyframes.html",
"http/tests/security/view-source-no-refresh.html",
"http/tests/websocket/tests/websocket-protocol-ignored.html",
"fast/css/display-none-inline-style-change-crash.html",
"http/tests/xmlhttprequest/supported-xml-content-types.html",
"dom/html/level2/html/HTMLAnchorElement03.html",
"ietestcenter/Javascript/11.1.5_4-4-c-1.html",
"dom/html/level2/html/HTMLAnchorElement06.html",
]
def get_test_input(self, test_file):
return TestInput(test_file, needs_servers=(test_file.startswith('http')))
def get_shards(self, num_workers, fully_parallel, test_list=None):
port = TestPort(MockSystemHost())
self.sharder = Sharder(port.split_test)
test_list = test_list or self.test_list
return self.sharder.shard_tests([self.get_test_input(test) for test in test_list], num_workers, fully_parallel)
def assert_shards(self, actual_shards, expected_shard_names):
self.assertEqual(len(actual_shards), len(expected_shard_names))
for i, shard in enumerate(actual_shards):
expected_shard_name, expected_test_names = expected_shard_names[i]
self.assertEqual(shard.name, expected_shard_name)
self.assertEqual([test_input.test_name for test_input in shard.test_inputs],
expected_test_names)
def test_shard_by_dir(self):
result = self.get_shards(num_workers=2, fully_parallel=False)
self.assert_shards(result,
[('animations', ['animations/keyframes.html']),
('dom/html/level2/html', ['dom/html/level2/html/HTMLAnchorElement03.html',
'dom/html/level2/html/HTMLAnchorElement06.html']),
('fast/css', ['fast/css/display-none-inline-style-change-crash.html']),
('http/tests/security', ['http/tests/security/view-source-no-refresh.html']),
('http/tests/websocket/tests', ['http/tests/websocket/tests/unicode.htm', 'http/tests/websocket/tests/websocket-protocol-ignored.html']),
('http/tests/xmlhttprequest', ['http/tests/xmlhttprequest/supported-xml-content-types.html']),
('ietestcenter/Javascript', ['ietestcenter/Javascript/11.1.5_4-4-c-1.html'])])
def test_shard_every_file(self):
result = self.get_shards(num_workers=2, fully_parallel=True)
self.assert_shards(result,
[('.', ['http/tests/websocket/tests/unicode.htm']),
('.', ['animations/keyframes.html']),
('.', ['http/tests/security/view-source-no-refresh.html']),
('.', ['http/tests/websocket/tests/websocket-protocol-ignored.html']),
('.', ['fast/css/display-none-inline-style-change-crash.html']),
('.', ['http/tests/xmlhttprequest/supported-xml-content-types.html']),
('.', ['dom/html/level2/html/HTMLAnchorElement03.html']),
('.', ['ietestcenter/Javascript/11.1.5_4-4-c-1.html']),
('.', ['dom/html/level2/html/HTMLAnchorElement06.html'])])
class KeyCompareTest(unittest.TestCase):
def setUp(self):
def split(test_name):
idx = test_name.rfind('/')
if idx != -1:
return (test_name[0:idx], test_name[idx + 1:])
self.sharder = Sharder(split, '/', 1)
def assert_cmp(self, x, y, result):
self.assertEquals(cmp(self.sharder.test_key(x), self.sharder.test_key(y)), result)
def test_test_key(self):
self.assert_cmp('/a', '/a', 0)
self.assert_cmp('/a', '/b', -1)
self.assert_cmp('/a2', '/a10', -1)
self.assert_cmp('/a2/foo', '/a10/foo', -1)
self.assert_cmp('/a/foo11', '/a/foo2', 1)
self.assert_cmp('/ab', '/a/a/b', -1)
self.assert_cmp('/a/a/b', '/ab', 1)
self.assert_cmp('/foo-bar/baz', '/foo/baz', -1)
class KeyCompareTest(unittest.TestCase):
def setUp(self):
def split(test_name):
idx = test_name.rfind('/')
if idx != -1:
return (test_name[0:idx], test_name[idx + 1:])
self.sharder = Sharder(split, '/', 1)
def assert_cmp(self, x, y, result):
self.assertEquals(
cmp(self.sharder.test_key(x), self.sharder.test_key(y)), result)
def test_test_key(self):
self.assert_cmp('/a', '/a', 0)
self.assert_cmp('/a', '/b', -1)
self.assert_cmp('/a2', '/a10', -1)
self.assert_cmp('/a2/foo', '/a10/foo', -1)
self.assert_cmp('/a/foo11', '/a/foo2', 1)
self.assert_cmp('/ab', '/a/a/b', -1)
self.assert_cmp('/a/a/b', '/ab', 1)
self.assert_cmp('/foo-bar/baz', '/foo/baz', -1)
class SharderTests(unittest.TestCase):
test_list = [
"http/tests/websocket/tests/unicode.htm",
"animations/keyframes.html",
"http/tests/security/view-source-no-refresh.html",
"http/tests/websocket/tests/websocket-protocol-ignored.html",
"fast/css/display-none-inline-style-change-crash.html",
"http/tests/xmlhttprequest/supported-xml-content-types.html",
"dom/html/level2/html/HTMLAnchorElement03.html",
"dom/html/level2/html/HTMLAnchorElement06.html",
"perf/object-keys.html",
"virtual/threaded/dir/test.html",
"virtual/threaded/fast/foo/test.html",
]
def get_test_input(self, test_file):
return TestInput(test_file,
requires_lock=(test_file.startswith('http')
or test_file.startswith('perf')))
def get_shards(self,
num_workers,
fully_parallel,
run_singly,
test_list=None,
max_locked_shards=1):
port = TestPort(MockSystemHost())
self.sharder = Sharder(port.split_test, max_locked_shards)
test_list = test_list or self.test_list
return self.sharder.shard_tests(
[self.get_test_input(test) for test in test_list], num_workers,
fully_parallel, run_singly)
def assert_shards(self, actual_shards, expected_shard_names):
self.assertEqual(len(actual_shards), len(expected_shard_names))
for i, shard in enumerate(actual_shards):
expected_shard_name, expected_test_names = expected_shard_names[i]
self.assertEqual(shard.name, expected_shard_name)
self.assertEqual(
[test_input.test_name for test_input in shard.test_inputs],
expected_test_names)
def test_shard_by_dir(self):
locked, unlocked = self.get_shards(num_workers=2,
fully_parallel=False,
run_singly=False)
# Note that although there are tests in multiple dirs that need locks,
# they are crammed into a single shard in order to reduce the # of
# workers hitting the server at once.
self.assert_shards(locked, [('locked_shard_1', [
'http/tests/security/view-source-no-refresh.html',
'http/tests/websocket/tests/unicode.htm',
'http/tests/websocket/tests/websocket-protocol-ignored.html',
'http/tests/xmlhttprequest/supported-xml-content-types.html',
'perf/object-keys.html'
])])
self.assert_shards(
unlocked,
[('virtual/threaded/dir', ['virtual/threaded/dir/test.html']),
('virtual/threaded/fast/foo',
['virtual/threaded/fast/foo/test.html']),
('animations', ['animations/keyframes.html']),
('dom/html/level2/html', [
'dom/html/level2/html/HTMLAnchorElement03.html',
'dom/html/level2/html/HTMLAnchorElement06.html'
]),
('fast/css',
['fast/css/display-none-inline-style-change-crash.html'])])
def test_shard_every_file(self):
locked, unlocked = self.get_shards(num_workers=2,
fully_parallel=True,
max_locked_shards=2,
run_singly=False)
self.assert_shards(
locked,
[('locked_shard_1', [
'http/tests/websocket/tests/unicode.htm',
'http/tests/security/view-source-no-refresh.html',
'http/tests/websocket/tests/websocket-protocol-ignored.html'
]),
('locked_shard_2', [
'http/tests/xmlhttprequest/supported-xml-content-types.html',
'perf/object-keys.html'
])]),
self.assert_shards(
unlocked,
[('virtual/threaded/dir', ['virtual/threaded/dir/test.html']),
('virtual/threaded/fast/foo',
['virtual/threaded/fast/foo/test.html']),
('.', ['animations/keyframes.html']),
('.', ['fast/css/display-none-inline-style-change-crash.html']),
('.', ['dom/html/level2/html/HTMLAnchorElement03.html']),
('.', ['dom/html/level2/html/HTMLAnchorElement06.html'])])
def test_shard_in_two(self):
locked, unlocked = self.get_shards(num_workers=1,
fully_parallel=False,
run_singly=False)
self.assert_shards(locked, [('locked_tests', [
'http/tests/websocket/tests/unicode.htm',
'http/tests/security/view-source-no-refresh.html',
'http/tests/websocket/tests/websocket-protocol-ignored.html',
'http/tests/xmlhttprequest/supported-xml-content-types.html',
'perf/object-keys.html'
])])
self.assert_shards(unlocked, [('unlocked_tests', [
'animations/keyframes.html',
'fast/css/display-none-inline-style-change-crash.html',
'dom/html/level2/html/HTMLAnchorElement03.html',
'dom/html/level2/html/HTMLAnchorElement06.html',
'virtual/threaded/dir/test.html',
'virtual/threaded/fast/foo/test.html'
])])
def test_shard_in_two_has_no_locked_shards(self):
locked, unlocked = self.get_shards(
num_workers=1,
fully_parallel=False,
run_singly=False,
test_list=['animations/keyframe.html'])
self.assertEqual(len(locked), 0)
self.assertEqual(len(unlocked), 1)
def test_shard_in_two_has_no_unlocked_shards(self):
locked, unlocked = self.get_shards(
num_workers=1,
fully_parallel=False,
run_singly=False,
test_list=['http/tests/websocket/tests/unicode.htm'])
self.assertEqual(len(locked), 1)
self.assertEqual(len(unlocked), 0)
def test_multiple_locked_shards(self):
locked, unlocked = self.get_shards(num_workers=4,
fully_parallel=False,
max_locked_shards=2,
run_singly=False)
self.assert_shards(
locked,
[('locked_shard_1', [
'http/tests/security/view-source-no-refresh.html',
'http/tests/websocket/tests/unicode.htm',
'http/tests/websocket/tests/websocket-protocol-ignored.html'
]),
('locked_shard_2', [
'http/tests/xmlhttprequest/supported-xml-content-types.html',
'perf/object-keys.html'
])])
locked, unlocked = self.get_shards(num_workers=4,
fully_parallel=False,
run_singly=False)
self.assert_shards(locked, [('locked_shard_1', [
'http/tests/security/view-source-no-refresh.html',
'http/tests/websocket/tests/unicode.htm',
'http/tests/websocket/tests/websocket-protocol-ignored.html',
'http/tests/xmlhttprequest/supported-xml-content-types.html',
'perf/object-keys.html'
])])
def test_virtual_shards(self):
# With run_singly=False, we try to keep all of the tests in a virtual suite together even
# when fully_parallel=True, so that we don't restart every time the command line args change.
locked, unlocked = self.get_shards(
num_workers=2,
fully_parallel=True,
max_locked_shards=2,
run_singly=False,
test_list=['virtual/foo/bar1.html', 'virtual/foo/bar2.html'])
self.assert_shards(unlocked, [
('virtual/foo', ['virtual/foo/bar1.html', 'virtual/foo/bar2.html'])
])
# But, with run_singly=True, we have to restart every time anyway, so we want full parallelism.
locked, unlocked = self.get_shards(
num_workers=2,
fully_parallel=True,
max_locked_shards=2,
run_singly=True,
test_list=['virtual/foo/bar1.html', 'virtual/foo/bar2.html'])
self.assert_shards(unlocked, [('.', ['virtual/foo/bar1.html']),
('.', ['virtual/foo/bar2.html'])])
def assert_cmp(self, x, y, result):
self.assertEquals(cmp(Sharder.natural_sort_key(x), Sharder.natural_sort_key(y)), result)
class SharderTests(unittest.TestCase):
test_list = [
"http/tests/websocket/tests/unicode.htm",
"animations/keyframes.html",
"http/tests/security/view-source-no-refresh.html",
"http/tests/websocket/tests/websocket-protocol-ignored.html",
"fast/css/display-none-inline-style-change-crash.html",
"http/tests/xmlhttprequest/supported-xml-content-types.html",
"dom/html/level2/html/HTMLAnchorElement03.html",
"ietestcenter/Javascript/11.1.5_4-4-c-1.html",
"dom/html/level2/html/HTMLAnchorElement06.html",
"perf/object-keys.html",
]
def get_test_input(self, test_file):
return TestInput(test_file,
requires_lock=(test_file.startswith('http')
or test_file.startswith('perf')))
def get_shards(self,
num_workers,
fully_parallel,
test_list=None,
max_locked_shards=1):
port = TestPort(MockSystemHost())
self.sharder = Sharder(port.split_test, max_locked_shards)
test_list = test_list or self.test_list
return self.sharder.shard_tests(
[self.get_test_input(test) for test in test_list], num_workers,
fully_parallel)
def assert_shards(self, actual_shards, expected_shard_names):
self.assertEqual(len(actual_shards), len(expected_shard_names))
for i, shard in enumerate(actual_shards):
expected_shard_name, expected_test_names = expected_shard_names[i]
self.assertEqual(shard.name, expected_shard_name)
self.assertEqual(
[test_input.test_name for test_input in shard.test_inputs],
expected_test_names)
def test_shard_by_dir(self):
locked, unlocked = self.get_shards(num_workers=2, fully_parallel=False)
# Note that although there are tests in multiple dirs that need locks,
# they are crammed into a single shard in order to reduce the # of
# workers hitting the server at once.
self.assert_shards(locked, [('locked_shard_1', [
'http/tests/security/view-source-no-refresh.html',
'http/tests/websocket/tests/unicode.htm',
'http/tests/websocket/tests/websocket-protocol-ignored.html',
'http/tests/xmlhttprequest/supported-xml-content-types.html',
'perf/object-keys.html'
])])
self.assert_shards(
unlocked,
[('animations', ['animations/keyframes.html']),
('dom/html/level2/html', [
'dom/html/level2/html/HTMLAnchorElement03.html',
'dom/html/level2/html/HTMLAnchorElement06.html'
]),
('fast/css',
['fast/css/display-none-inline-style-change-crash.html']),
('ietestcenter/Javascript',
['ietestcenter/Javascript/11.1.5_4-4-c-1.html'])])
def test_shard_every_file(self):
locked, unlocked = self.get_shards(num_workers=2, fully_parallel=True)
self.assert_shards(
locked,
[('.', ['http/tests/websocket/tests/unicode.htm']),
('.', ['http/tests/security/view-source-no-refresh.html']),
('.',
['http/tests/websocket/tests/websocket-protocol-ignored.html']),
('.',
['http/tests/xmlhttprequest/supported-xml-content-types.html']),
('.', ['perf/object-keys.html'])]),
self.assert_shards(
unlocked,
[('.', ['animations/keyframes.html']),
('.', ['fast/css/display-none-inline-style-change-crash.html']),
('.', ['dom/html/level2/html/HTMLAnchorElement03.html']),
('.', ['ietestcenter/Javascript/11.1.5_4-4-c-1.html']),
('.', ['dom/html/level2/html/HTMLAnchorElement06.html'])])
def test_shard_in_two(self):
locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False)
self.assert_shards(locked, [('locked_tests', [
'http/tests/websocket/tests/unicode.htm',
'http/tests/security/view-source-no-refresh.html',
'http/tests/websocket/tests/websocket-protocol-ignored.html',
'http/tests/xmlhttprequest/supported-xml-content-types.html',
'perf/object-keys.html'
])])
self.assert_shards(unlocked, [('unlocked_tests', [
'animations/keyframes.html',
'fast/css/display-none-inline-style-change-crash.html',
'dom/html/level2/html/HTMLAnchorElement03.html',
'ietestcenter/Javascript/11.1.5_4-4-c-1.html',
'dom/html/level2/html/HTMLAnchorElement06.html'
])])
def test_shard_in_two_has_no_locked_shards(self):
locked, unlocked = self.get_shards(
num_workers=1,
fully_parallel=False,
test_list=['animations/keyframe.html'])
self.assertEqual(len(locked), 0)
self.assertEqual(len(unlocked), 1)
def test_shard_in_two_has_no_unlocked_shards(self):
locked, unlocked = self.get_shards(
num_workers=1,
fully_parallel=False,
test_list=['http/tests/websocket/tests/unicode.htm'])
self.assertEqual(len(locked), 1)
self.assertEqual(len(unlocked), 0)
def test_multiple_locked_shards(self):
locked, unlocked = self.get_shards(num_workers=4,
fully_parallel=False,
max_locked_shards=2)
self.assert_shards(
locked,
[('locked_shard_1', [
'http/tests/security/view-source-no-refresh.html',
'http/tests/websocket/tests/unicode.htm',
'http/tests/websocket/tests/websocket-protocol-ignored.html'
]),
('locked_shard_2', [
'http/tests/xmlhttprequest/supported-xml-content-types.html',
'perf/object-keys.html'
])])
locked, unlocked = self.get_shards(num_workers=4, fully_parallel=False)
self.assert_shards(locked, [('locked_shard_1', [
'http/tests/security/view-source-no-refresh.html',
'http/tests/websocket/tests/unicode.htm',
'http/tests/websocket/tests/websocket-protocol-ignored.html',
'http/tests/xmlhttprequest/supported-xml-content-types.html',
'perf/object-keys.html'
])])
class SharderTests(unittest.TestCase):
test_list = [
"http/tests/websocket/tests/unicode.htm",
"animations/keyframes.html",
"http/tests/security/view-source-no-refresh.html",
"http/tests/websocket/tests/websocket-protocol-ignored.html",
"fast/css/display-none-inline-style-change-crash.html",
"http/tests/xmlhttprequest/supported-xml-content-types.html",
"dom/html/level2/html/HTMLAnchorElement03.html",
"ietestcenter/Javascript/11.1.5_4-4-c-1.html",
"dom/html/level2/html/HTMLAnchorElement06.html",
"perf/object-keys.html",
]
def get_test_input(self, test_file):
return TestInput(test_file, requires_lock=(test_file.startswith("http") or test_file.startswith("perf")))
def get_shards(self, num_workers, fully_parallel, test_list=None, max_locked_shards=1):
port = TestPort(MockSystemHost())
self.sharder = Sharder(port.split_test, max_locked_shards)
test_list = test_list or self.test_list
return self.sharder.shard_tests([self.get_test_input(test) for test in test_list], num_workers, fully_parallel)
def assert_shards(self, actual_shards, expected_shard_names):
self.assertEqual(len(actual_shards), len(expected_shard_names))
for i, shard in enumerate(actual_shards):
expected_shard_name, expected_test_names = expected_shard_names[i]
self.assertEqual(shard.name, expected_shard_name)
self.assertEqual([test_input.test_name for test_input in shard.test_inputs], expected_test_names)
def test_shard_by_dir(self):
locked, unlocked = self.get_shards(num_workers=2, fully_parallel=False)
# Note that although there are tests in multiple dirs that need locks,
# they are crammed into a single shard in order to reduce the # of
# workers hitting the server at once.
self.assert_shards(
locked,
[
(
"locked_shard_1",
[
"http/tests/security/view-source-no-refresh.html",
"http/tests/websocket/tests/unicode.htm",
"http/tests/websocket/tests/websocket-protocol-ignored.html",
"http/tests/xmlhttprequest/supported-xml-content-types.html",
"perf/object-keys.html",
],
)
],
)
self.assert_shards(
unlocked,
[
("animations", ["animations/keyframes.html"]),
(
"dom/html/level2/html",
["dom/html/level2/html/HTMLAnchorElement03.html", "dom/html/level2/html/HTMLAnchorElement06.html"],
),
("fast/css", ["fast/css/display-none-inline-style-change-crash.html"]),
("ietestcenter/Javascript", ["ietestcenter/Javascript/11.1.5_4-4-c-1.html"]),
],
)
def test_shard_every_file(self):
locked, unlocked = self.get_shards(num_workers=2, fully_parallel=True)
self.assert_shards(
locked,
[
(".", ["http/tests/websocket/tests/unicode.htm"]),
(".", ["http/tests/security/view-source-no-refresh.html"]),
(".", ["http/tests/websocket/tests/websocket-protocol-ignored.html"]),
(".", ["http/tests/xmlhttprequest/supported-xml-content-types.html"]),
(".", ["perf/object-keys.html"]),
],
),
self.assert_shards(
unlocked,
[
(".", ["animations/keyframes.html"]),
(".", ["fast/css/display-none-inline-style-change-crash.html"]),
(".", ["dom/html/level2/html/HTMLAnchorElement03.html"]),
(".", ["ietestcenter/Javascript/11.1.5_4-4-c-1.html"]),
(".", ["dom/html/level2/html/HTMLAnchorElement06.html"]),
],
)
def test_shard_in_two(self):
locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False)
self.assert_shards(
locked,
[
(
"locked_tests",
[
"http/tests/websocket/tests/unicode.htm",
"http/tests/security/view-source-no-refresh.html",
"http/tests/websocket/tests/websocket-protocol-ignored.html",
"http/tests/xmlhttprequest/supported-xml-content-types.html",
"perf/object-keys.html",
],
)
],
)
self.assert_shards(
unlocked,
[
(
"unlocked_tests",
[
"animations/keyframes.html",
"fast/css/display-none-inline-style-change-crash.html",
"dom/html/level2/html/HTMLAnchorElement03.html",
"ietestcenter/Javascript/11.1.5_4-4-c-1.html",
"dom/html/level2/html/HTMLAnchorElement06.html",
],
)
],
)
def test_shard_in_two_has_no_locked_shards(self):
locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False, test_list=["animations/keyframe.html"])
self.assertEqual(len(locked), 0)
self.assertEqual(len(unlocked), 1)
def test_shard_in_two_has_no_unlocked_shards(self):
locked, unlocked = self.get_shards(
num_workers=1, fully_parallel=False, test_list=["http/tests/websocket/tests/unicode.htm"]
)
self.assertEqual(len(locked), 1)
self.assertEqual(len(unlocked), 0)
def test_multiple_locked_shards(self):
locked, unlocked = self.get_shards(num_workers=4, fully_parallel=False, max_locked_shards=2)
self.assert_shards(
locked,
[
(
"locked_shard_1",
[
"http/tests/security/view-source-no-refresh.html",
"http/tests/websocket/tests/unicode.htm",
"http/tests/websocket/tests/websocket-protocol-ignored.html",
],
),
(
"locked_shard_2",
["http/tests/xmlhttprequest/supported-xml-content-types.html", "perf/object-keys.html"],
),
],
)
locked, unlocked = self.get_shards(num_workers=4, fully_parallel=False)
self.assert_shards(
locked,
[
(
"locked_shard_1",
[
"http/tests/security/view-source-no-refresh.html",
"http/tests/websocket/tests/unicode.htm",
"http/tests/websocket/tests/websocket-protocol-ignored.html",
"http/tests/xmlhttprequest/supported-xml-content-types.html",
"perf/object-keys.html",
],
)
],
)
def assert_cmp(self, x, y, result):
self.assertEquals(
cmp(Sharder.natural_sort_key(x), Sharder.natural_sort_key(y)),
result)
class SharderTests(unittest.TestCase):
test_list = [
"http/tests/websocket/tests/unicode.htm",
"animations/keyframes.html",
"http/tests/security/view-source-no-refresh.html",
"http/tests/websocket/tests/websocket-protocol-ignored.html",
"fast/css/display-none-inline-style-change-crash.html",
"http/tests/xmlhttprequest/supported-xml-content-types.html",
"dom/html/level2/html/HTMLAnchorElement03.html",
"ietestcenter/Javascript/11.1.5_4-4-c-1.html",
"dom/html/level2/html/HTMLAnchorElement06.html",
"perf/object-keys.html",
"virtual/threaded/dir/test.html",
"virtual/threaded/fast/foo/test.html",
]
def get_test_input(self, test_file):
return TestInput(test_file, requires_lock=(test_file.startswith('http') or test_file.startswith('perf')))
def get_shards(self, num_workers, fully_parallel, run_singly, test_list=None, max_locked_shards=1):
port = TestPort(MockSystemHost())
self.sharder = Sharder(port.split_test, max_locked_shards)
test_list = test_list or self.test_list
return self.sharder.shard_tests([self.get_test_input(test) for test in test_list],
num_workers, fully_parallel, run_singly)
def assert_shards(self, actual_shards, expected_shard_names):
self.assertEqual(len(actual_shards), len(expected_shard_names))
for i, shard in enumerate(actual_shards):
expected_shard_name, expected_test_names = expected_shard_names[i]
self.assertEqual(shard.name, expected_shard_name)
self.assertEqual([test_input.test_name for test_input in shard.test_inputs],
expected_test_names)
def test_shard_by_dir(self):
locked, unlocked = self.get_shards(num_workers=2, fully_parallel=False, run_singly=False)
# Note that although there are tests in multiple dirs that need locks,
# they are crammed into a single shard in order to reduce the # of
# workers hitting the server at once.
self.assert_shards(locked,
[('locked_shard_1',
['http/tests/security/view-source-no-refresh.html',
'http/tests/websocket/tests/unicode.htm',
'http/tests/websocket/tests/websocket-protocol-ignored.html',
'http/tests/xmlhttprequest/supported-xml-content-types.html',
'perf/object-keys.html'])])
self.assert_shards(unlocked,
[('virtual/threaded/dir', ['virtual/threaded/dir/test.html']),
('virtual/threaded/fast/foo', ['virtual/threaded/fast/foo/test.html']),
('animations', ['animations/keyframes.html']),
('dom/html/level2/html', ['dom/html/level2/html/HTMLAnchorElement03.html',
'dom/html/level2/html/HTMLAnchorElement06.html']),
('fast/css', ['fast/css/display-none-inline-style-change-crash.html']),
('ietestcenter/Javascript', ['ietestcenter/Javascript/11.1.5_4-4-c-1.html'])])
def test_shard_every_file(self):
locked, unlocked = self.get_shards(num_workers=2, fully_parallel=True, max_locked_shards=2, run_singly=False)
self.assert_shards(locked,
[('locked_shard_1',
['http/tests/websocket/tests/unicode.htm',
'http/tests/security/view-source-no-refresh.html',
'http/tests/websocket/tests/websocket-protocol-ignored.html']),
('locked_shard_2',
['http/tests/xmlhttprequest/supported-xml-content-types.html',
'perf/object-keys.html'])]),
self.assert_shards(unlocked,
[('virtual/threaded/dir', ['virtual/threaded/dir/test.html']),
('virtual/threaded/fast/foo', ['virtual/threaded/fast/foo/test.html']),
('.', ['animations/keyframes.html']),
('.', ['fast/css/display-none-inline-style-change-crash.html']),
('.', ['dom/html/level2/html/HTMLAnchorElement03.html']),
('.', ['ietestcenter/Javascript/11.1.5_4-4-c-1.html']),
('.', ['dom/html/level2/html/HTMLAnchorElement06.html'])])
def test_shard_in_two(self):
locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False, run_singly=False)
self.assert_shards(locked,
[('locked_tests',
['http/tests/websocket/tests/unicode.htm',
'http/tests/security/view-source-no-refresh.html',
'http/tests/websocket/tests/websocket-protocol-ignored.html',
'http/tests/xmlhttprequest/supported-xml-content-types.html',
'perf/object-keys.html'])])
self.assert_shards(unlocked,
[('unlocked_tests',
['animations/keyframes.html',
'fast/css/display-none-inline-style-change-crash.html',
'dom/html/level2/html/HTMLAnchorElement03.html',
'ietestcenter/Javascript/11.1.5_4-4-c-1.html',
'dom/html/level2/html/HTMLAnchorElement06.html',
'virtual/threaded/dir/test.html',
'virtual/threaded/fast/foo/test.html'])])
def test_shard_in_two_has_no_locked_shards(self):
locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False, run_singly=False,
test_list=['animations/keyframe.html'])
self.assertEqual(len(locked), 0)
self.assertEqual(len(unlocked), 1)
def test_shard_in_two_has_no_unlocked_shards(self):
locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False, run_singly=False,
test_list=['http/tests/websocket/tests/unicode.htm'])
self.assertEqual(len(locked), 1)
self.assertEqual(len(unlocked), 0)
def test_multiple_locked_shards(self):
locked, unlocked = self.get_shards(num_workers=4, fully_parallel=False, max_locked_shards=2, run_singly=False)
self.assert_shards(locked,
[('locked_shard_1',
['http/tests/security/view-source-no-refresh.html',
'http/tests/websocket/tests/unicode.htm',
'http/tests/websocket/tests/websocket-protocol-ignored.html']),
('locked_shard_2',
['http/tests/xmlhttprequest/supported-xml-content-types.html',
'perf/object-keys.html'])])
locked, unlocked = self.get_shards(num_workers=4, fully_parallel=False, run_singly=False)
self.assert_shards(locked,
[('locked_shard_1',
['http/tests/security/view-source-no-refresh.html',
'http/tests/websocket/tests/unicode.htm',
'http/tests/websocket/tests/websocket-protocol-ignored.html',
'http/tests/xmlhttprequest/supported-xml-content-types.html',
'perf/object-keys.html'])])
def test_virtual_shards(self):
# With run_singly=False, we try to keep all of the tests in a virtual suite together even
# when fully_parallel=True, so that we don't restart every time the command line args change.
locked, unlocked = self.get_shards(num_workers=2, fully_parallel=True, max_locked_shards=2, run_singly=False,
test_list=['virtual/foo/bar1.html', 'virtual/foo/bar2.html'])
self.assert_shards(unlocked,
[('virtual/foo', ['virtual/foo/bar1.html', 'virtual/foo/bar2.html'])])
# But, with run_singly=True, we have to restart every time anyway, so we want full parallelism.
locked, unlocked = self.get_shards(num_workers=2, fully_parallel=True, max_locked_shards=2, run_singly=True,
test_list=['virtual/foo/bar1.html', 'virtual/foo/bar2.html'])
self.assert_shards(unlocked,
[('.', ['virtual/foo/bar1.html']),
('.', ['virtual/foo/bar2.html'])])
def get_shards(self, num_workers, fully_parallel, run_singly, test_list=None, max_locked_shards=1):
port = TestPort(MockSystemHost())
self.sharder = Sharder(port.split_test, max_locked_shards)
test_list = test_list or self.test_list
return self.sharder.shard_tests([self.get_test_input(test) for test in test_list],
num_workers, fully_parallel, run_singly)
class SharderTests(unittest.TestCase):
test_list = [
"http/tests/websocket/tests/unicode.htm",
"animations/keyframes.html",
"http/tests/security/view-source-no-refresh.html",
"http/tests/websocket/tests/websocket-protocol-ignored.html",
"fast/css/display-none-inline-style-change-crash.html",
"http/tests/xmlhttprequest/supported-xml-content-types.html",
"dom/html/level2/html/HTMLAnchorElement03.html",
"ietestcenter/Javascript/11.1.5_4-4-c-1.html",
"dom/html/level2/html/HTMLAnchorElement06.html",
"perf/object-keys.html",
]
ref_tests = [
"http/tests/security/view-source-no-refresh.html",
"http/tests/websocket/tests/websocket-protocol-ignored.html",
"ietestcenter/Javascript/11.1.5_4-4-c-1.html",
"dom/html/level2/html/HTMLAnchorElement06.html",
]
def get_test_input(self, test_file):
return TestInput(test_file, requires_lock=(test_file.startswith('http') or test_file.startswith('perf')),
reference_files=(['foo'] if test_file in self.ref_tests else None))
def get_shards(self, num_workers, fully_parallel, shard_ref_tests=False, test_list=None, max_locked_shards=1):
def split(test_name):
idx = test_name.rfind('/')
if idx != -1:
return (test_name[0:idx], test_name[idx + 1:])
self.sharder = Sharder(split, '/', max_locked_shards)
test_list = test_list or self.test_list
return self.sharder.shard_tests([self.get_test_input(test) for test in test_list], num_workers, fully_parallel, shard_ref_tests)
def assert_shards(self, actual_shards, expected_shard_names):
self.assertEquals(len(actual_shards), len(expected_shard_names))
for i, shard in enumerate(actual_shards):
expected_shard_name, expected_test_names = expected_shard_names[i]
self.assertEquals(shard.name, expected_shard_name)
self.assertEquals([test_input.test_name for test_input in shard.test_inputs],
expected_test_names)
def test_shard_by_dir(self):
locked, unlocked = self.get_shards(num_workers=2, fully_parallel=False)
# Note that although there are tests in multiple dirs that need locks,
# they are crammed into a single shard in order to reduce the # of
# workers hitting the server at once.
self.assert_shards(locked,
[('locked_shard_1',
['http/tests/security/view-source-no-refresh.html',
'http/tests/websocket/tests/unicode.htm',
'http/tests/websocket/tests/websocket-protocol-ignored.html',
'http/tests/xmlhttprequest/supported-xml-content-types.html',
'perf/object-keys.html'])])
self.assert_shards(unlocked,
[('animations', ['animations/keyframes.html']),
('dom/html/level2/html', ['dom/html/level2/html/HTMLAnchorElement03.html',
'dom/html/level2/html/HTMLAnchorElement06.html']),
('fast/css', ['fast/css/display-none-inline-style-change-crash.html']),
('ietestcenter/Javascript', ['ietestcenter/Javascript/11.1.5_4-4-c-1.html'])])
def test_shard_by_dir_sharding_ref_tests(self):
locked, unlocked = self.get_shards(num_workers=2, fully_parallel=False, shard_ref_tests=True)
# Note that although there are tests in multiple dirs that need locks,
# they are crammed into a single shard in order to reduce the # of
# workers hitting the server at once.
self.assert_shards(locked,
[('locked_shard_1',
['http/tests/websocket/tests/unicode.htm',
'http/tests/xmlhttprequest/supported-xml-content-types.html',
'perf/object-keys.html',
'http/tests/security/view-source-no-refresh.html',
'http/tests/websocket/tests/websocket-protocol-ignored.html'])])
self.assert_shards(unlocked,
[('animations', ['animations/keyframes.html']),
('dom/html/level2/html', ['dom/html/level2/html/HTMLAnchorElement03.html']),
('fast/css', ['fast/css/display-none-inline-style-change-crash.html']),
('~ref:dom/html/level2/html', ['dom/html/level2/html/HTMLAnchorElement06.html']),
('~ref:ietestcenter/Javascript', ['ietestcenter/Javascript/11.1.5_4-4-c-1.html'])])
def test_shard_every_file(self):
locked, unlocked = self.get_shards(num_workers=2, fully_parallel=True)
self.assert_shards(locked,
[('.', ['http/tests/websocket/tests/unicode.htm']),
('.', ['http/tests/security/view-source-no-refresh.html']),
('.', ['http/tests/websocket/tests/websocket-protocol-ignored.html']),
('.', ['http/tests/xmlhttprequest/supported-xml-content-types.html']),
('.', ['perf/object-keys.html'])]),
self.assert_shards(unlocked,
[('.', ['animations/keyframes.html']),
('.', ['fast/css/display-none-inline-style-change-crash.html']),
('.', ['dom/html/level2/html/HTMLAnchorElement03.html']),
('.', ['ietestcenter/Javascript/11.1.5_4-4-c-1.html']),
('.', ['dom/html/level2/html/HTMLAnchorElement06.html'])])
def test_shard_in_two(self):
locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False)
self.assert_shards(locked,
[('locked_tests',
['http/tests/websocket/tests/unicode.htm',
'http/tests/security/view-source-no-refresh.html',
'http/tests/websocket/tests/websocket-protocol-ignored.html',
'http/tests/xmlhttprequest/supported-xml-content-types.html',
'perf/object-keys.html'])])
self.assert_shards(unlocked,
[('unlocked_tests',
['animations/keyframes.html',
'fast/css/display-none-inline-style-change-crash.html',
'dom/html/level2/html/HTMLAnchorElement03.html',
'ietestcenter/Javascript/11.1.5_4-4-c-1.html',
'dom/html/level2/html/HTMLAnchorElement06.html'])])
def test_shard_in_two_sharding_ref_tests(self):
locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False, shard_ref_tests=True)
self.assert_shards(locked,
[('locked_tests',
['http/tests/websocket/tests/unicode.htm',
'http/tests/xmlhttprequest/supported-xml-content-types.html',
'perf/object-keys.html',
'http/tests/security/view-source-no-refresh.html',
'http/tests/websocket/tests/websocket-protocol-ignored.html'])])
self.assert_shards(unlocked,
[('unlocked_tests',
['animations/keyframes.html',
'fast/css/display-none-inline-style-change-crash.html',
'dom/html/level2/html/HTMLAnchorElement03.html',
'ietestcenter/Javascript/11.1.5_4-4-c-1.html',
'dom/html/level2/html/HTMLAnchorElement06.html'])])
def test_shard_in_two_has_no_locked_shards(self):
locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False,
test_list=['animations/keyframe.html'])
self.assertEquals(len(locked), 0)
self.assertEquals(len(unlocked), 1)
def test_shard_in_two_has_no_unlocked_shards(self):
locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False,
test_list=['http/tests/websocket/tests/unicode.htm'])
self.assertEquals(len(locked), 1)
self.assertEquals(len(unlocked), 0)
def test_multiple_locked_shards(self):
locked, unlocked = self.get_shards(num_workers=4, fully_parallel=False, max_locked_shards=2)
self.assert_shards(locked,
[('locked_shard_1',
['http/tests/security/view-source-no-refresh.html',
'http/tests/websocket/tests/unicode.htm',
'http/tests/websocket/tests/websocket-protocol-ignored.html']),
('locked_shard_2',
['http/tests/xmlhttprequest/supported-xml-content-types.html',
'perf/object-keys.html'])])
locked, unlocked = self.get_shards(num_workers=4, fully_parallel=False)
self.assert_shards(locked,
[('locked_shard_1',
['http/tests/security/view-source-no-refresh.html',
'http/tests/websocket/tests/unicode.htm',
'http/tests/websocket/tests/websocket-protocol-ignored.html',
'http/tests/xmlhttprequest/supported-xml-content-types.html',
'perf/object-keys.html'])])