def test_coverage_works(self):
# This is awkward; by design, running test-webkitpy -c will
# create a .coverage file in tools, so we need to be
# careful not to clobber an existing one, and to clean up.
# FIXME: This design needs to change since it means we can't actually
# run this method itself under coverage properly.
filesystem = FileSystem()
executive = Executive()
module_path = filesystem.path_to_module(self.__module__)
script_dir = module_path[0:module_path.find('webkitpy') - 1]
coverage_file = filesystem.join(script_dir, '.coverage')
coverage_file_orig = None
if filesystem.exists(coverage_file):
coverage_file_orig = coverage_file + '.orig'
filesystem.move(coverage_file, coverage_file_orig)
try:
proc = executive.popen([sys.executable, filesystem.join(script_dir, 'test-webkitpy'), '-c', STUBS_CLASS + '.test_empty'],
stdout=executive.PIPE, stderr=executive.PIPE)
out, _ = proc.communicate()
retcode = proc.returncode
self.assertEqual(retcode, 0)
self.assertIn('Cover', out)
finally:
if coverage_file_orig:
filesystem.move(coverage_file_orig, coverage_file)
elif filesystem.exists(coverage_file):
filesystem.remove(coverage_file)
def test_coverage_works(self):
# This is awkward; by design, running test-webkitpy -c will
# create a .coverage file in tools, so we need to be
# careful not to clobber an existing one, and to clean up.
# FIXME: This design needs to change since it means we can't actually
# run this method itself under coverage properly.
filesystem = FileSystem()
executive = Executive()
module_path = filesystem.path_to_module(self.__module__)
script_dir = module_path[0:module_path.find('webkitpy') - 1]
coverage_file = filesystem.join(script_dir, '.coverage')
coverage_file_orig = None
if filesystem.exists(coverage_file):
coverage_file_orig = coverage_file + '.orig'
filesystem.move(coverage_file, coverage_file_orig)
try:
proc = executive.popen([
sys.executable,
filesystem.join(script_dir, 'test-webkitpy'), '-c',
STUBS_CLASS + '.test_empty'
],
stdout=executive.PIPE,
stderr=executive.PIPE)
out, _ = proc.communicate()
retcode = proc.returncode
self.assertEqual(retcode, 0)
self.assertIn('Cover', out)
finally:
if coverage_file_orig:
filesystem.move(coverage_file_orig, coverage_file)
elif filesystem.exists(coverage_file):
filesystem.remove(coverage_file)
def test_auto_stringify_args(self):
executive = Executive()
executive.run_command(command_line('echo', 1))
with executive.popen(command_line('echo', 1),
stdout=executive.PIPE) as process:
process.wait()
self.assertEqual('echo 1',
executive.command_for_printing(['echo', 1]))
def integration_test_coverage_works(self):
filesystem = FileSystem()
executive = Executive()
module_path = filesystem.path_to_module(self.__module__)
script_dir = module_path[0:module_path.find('webkitpy') - 1]
proc = executive.popen([sys.executable, filesystem.join(script_dir, 'test-webkitpy'), '-c', STUBS_CLASS + '.test_empty'],
stdout=executive.PIPE, stderr=executive.PIPE)
out, _ = proc.communicate()
retcode = proc.returncode
self.assertEqual(retcode, 0)
self.assertIn('Cover', out)
def integration_test_coverage_works(self):
filesystem = FileSystem()
executive = Executive()
module_path = filesystem.path_to_module(self.__module__)
script_dir = module_path[0:module_path.find('webkitpy') - 1]
proc = executive.popen([
sys.executable,
filesystem.join(script_dir, 'test-webkitpy'), '-c',
STUBS_CLASS + '.test_empty'
],
stdout=executive.PIPE,
stderr=executive.PIPE)
out, _ = proc.communicate()
retcode = proc.returncode
self.assertEqual(retcode, 0)
self.assertIn('Cover', out)
def serial_test_kill_all(self):
executive = Executive()
with executive.popen(never_ending_command(), stdout=subprocess.PIPE) as process:
self.assertIsNone(process.poll()) # Process is running
executive.kill_all(never_ending_command()[0])
# Note: Can't use a ternary since signal.SIGTERM is undefined for sys.platform == "win32"
if sys.platform == "cygwin":
expected_exit_code = 0 # os.kill results in exit(0) for this process.
self.assertEqual(process.wait(), expected_exit_code)
elif sys.platform.startswith('win'):
# FIXME: https://bugs.webkit.org/show_bug.cgi?id=54790
# We seem to get either 0 or 1 here for some reason.
self.assertIn(process.wait(), (0, 1))
else:
expected_exit_code = -signal.SIGTERM
self.assertEqual(process.wait(), expected_exit_code)
# Killing again should fail silently.
executive.kill_all(never_ending_command()[0])
def serial_test_kill_process(self):
executive = Executive()
with executive.popen(never_ending_command(), stdout=subprocess.PIPE) as process:
self.assertEqual(process.poll(), None) # Process is running
executive.kill_process(process.pid)
# Note: Can't use a ternary since signal.SIGKILL is undefined for sys.platform == "win32"
if sys.platform.startswith('win'):
# FIXME: https://bugs.webkit.org/show_bug.cgi?id=54790
# We seem to get either 0 or 1 here for some reason.
self.assertIn(process.wait(), (0, 1))
elif sys.platform == "cygwin":
# FIXME: https://bugs.webkit.org/show_bug.cgi?id=98196
# cygwin seems to give us either SIGABRT or SIGKILL
# Native Windows (via Cygwin) returns ENOTBLK (-15)
self.assertIn(process.wait(), (-signal.SIGABRT, -signal.SIGKILL, -15))
else:
expected_exit_code = -signal.SIGTERM
self.assertEqual(process.wait(), expected_exit_code)
# Killing again should fail silently.
executive.kill_process(process.pid)
def test_popen_args(self):
executive = Executive()
# Explicitly naming the 'args' argument should not thow an exception.
with executive.popen(args=command_line('echo', 1), stdout=executive.PIPE) as process:
process.wait()
def test_popen_args(self):
executive = Executive()
# Explicitly naming the 'args' argument should not thow an exception.
executive.popen(args=command_line('echo', 1), stdout=executive.PIPE).wait()
def test_auto_stringify_args(self):
executive = Executive()
executive.run_command(command_line('echo', 1))
executive.popen(command_line('echo', 1), stdout=executive.PIPE).wait()
self.assertEqual('echo 1', executive.command_for_printing(['echo', 1]))
class Bisector(object):
def __init__(self, tests, is_debug):
self.executive = Executive()
self.tests = tests
self.expected_failure = tests[-1]
self.is_debug = is_debug
self.webkit_finder = WebKitFinder(FileSystem())
def bisect(self):
if self.test_fails_in_isolation():
self.buckets = [Bucket([self.expected_failure])]
print '%s fails when run in isolation.' % self.expected_failure
self.print_result()
return 0
if not self.test_fails(self.tests):
_log.error('%s does not fail', self.expected_failure)
return 1
# Split the list of test into buckets. Each bucket has at least one test required to cause
# the expected failure at the end. Split buckets in half until there are only buckets left
# with one item in them.
self.buckets = [
Bucket(self.tests[:-1]),
Bucket([self.expected_failure])
]
while not self.is_done():
self.print_progress()
self.split_largest_bucket()
self.print_result()
self.verify_non_flaky()
return 0
def test_fails_in_isolation(self):
return self.test_bucket_list_fails([Bucket([self.expected_failure])])
def verify_non_flaky(self):
print 'Verifying the failure is not flaky by running 10 times.'
count_failures = 0
for _ in range(0, 10):
if self.test_bucket_list_fails(self.buckets):
count_failures += 1
print 'Failed %d/10 times' % count_failures
def print_progress(self):
count = 0
for bucket in self.buckets:
count += len(bucket.tests)
print '%d tests left, %d buckets' % (count, len(self.buckets))
def print_result(self):
tests = []
for bucket in self.buckets:
tests += bucket.tests
extra_args = ' --debug' if self.is_debug else ''
print 'run-webkit-tests%s --child-processes=1 --order=none %s' % (
extra_args, ' '.join(tests))
def is_done(self):
for bucket in self.buckets:
if bucket.size() > 1:
return False
return True
def split_largest_bucket(self):
index = 0
largest_index = 0
largest_size = 0
for bucket in self.buckets:
if bucket.size() > largest_size:
largest_index = index
largest_size = bucket.size()
index += 1
bucket_to_split = self.buckets[largest_index]
halfway_point = int(largest_size / 2)
first_half = Bucket(bucket_to_split.tests[:halfway_point])
second_half = Bucket(bucket_to_split.tests[halfway_point:])
buckets_before = self.buckets[:largest_index]
buckets_after = self.buckets[largest_index + 1:]
# Do the second half first because it tends to be faster because the http tests are front-loaded and slow.
new_buckets = buckets_before + [second_half] + buckets_after
if self.test_bucket_list_fails(new_buckets):
self.buckets = new_buckets
return
new_buckets = buckets_before + [first_half] + buckets_after
if self.test_bucket_list_fails(new_buckets):
self.buckets = new_buckets
return
self.buckets = buckets_before + [first_half, second_half
] + buckets_after
def test_bucket_list_fails(self, buckets):
tests = []
for bucket in buckets:
tests += bucket.tests
return self.test_fails(tests)
def test_fails(self, tests):
extra_args = ['--debug'] if self.is_debug else []
path_to_run_webkit_tests = self.webkit_finder.path_from_webkit_base(
'Tools', 'Scripts', 'run-webkit-tests')
output = self.executive.popen([
path_to_run_webkit_tests, '--child-processes', '1', '--order',
'none', '--no-retry', '--no-show-results', '--verbose'
] + extra_args + tests,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
failure_string = self.expected_failure + ' failed'
if failure_string in output.stderr.read():
return True
return False
def test_popen_args(self):
executive = Executive()
# Explicitly naming the 'args' argument should not thow an exception.
executive.popen(args=command_line('echo', 1)).wait()
def test_auto_stringify_args(self):
executive = Executive()
executive.run_command(command_line('echo', 1))
executive.popen(command_line('echo', 1)).wait()
class Bisector(object):
def __init__(self, tests, is_debug):
self.executive = Executive()
self.tests = tests
self.expected_failure = tests[-1]
self.is_debug = is_debug
self.webkit_finder = WebKitFinder(FileSystem())
def bisect(self):
if self.test_fails_in_isolation():
self.buckets = [Bucket([self.expected_failure])]
print '%s fails when run in isolation.' % self.expected_failure
self.print_result()
return 0
if not self.test_fails(self.tests):
_log.error('%s does not fail' % self.expected_failure)
return 1
# Split the list of test into buckets. Each bucket has at least one test required to cause
# the expected failure at the end. Split buckets in half until there are only buckets left
# with one item in them.
self.buckets = [Bucket(self.tests[:-1]), Bucket([self.expected_failure])]
while not self.is_done():
self.print_progress()
self.split_largest_bucket()
self.print_result()
self.verify_non_flaky()
return 0
def test_fails_in_isolation(self):
return self.test_bucket_list_fails([Bucket([self.expected_failure])])
def verify_non_flaky(self):
print 'Verifying the failure is not flaky by running 10 times.'
count_failures = 0
for i in range(0, 10):
if self.test_bucket_list_fails(self.buckets):
count_failures += 1
print 'Failed %d/10 times' % count_failures
def print_progress(self):
count = 0
for bucket in self.buckets:
count += len(bucket.tests)
print '%d tests left, %d buckets' % (count, len(self.buckets))
def print_result(self):
tests = []
for bucket in self.buckets:
tests += bucket.tests
extra_args = ' --debug' if self.is_debug else ''
print 'run-webkit-tests%s --child-processes=1 --order=none %s' % (extra_args, " ".join(tests))
def is_done(self):
for bucket in self.buckets:
if bucket.size() > 1:
return False
return True
def split_largest_bucket(self):
index = 0
largest_index = 0
largest_size = 0
for bucket in self.buckets:
if bucket.size() > largest_size:
largest_index = index
largest_size = bucket.size()
index += 1
bucket_to_split = self.buckets[largest_index]
halfway_point = int(largest_size / 2)
first_half = Bucket(bucket_to_split.tests[:halfway_point])
second_half = Bucket(bucket_to_split.tests[halfway_point:])
buckets_before = self.buckets[:largest_index]
buckets_after = self.buckets[largest_index + 1:]
# Do the second half first because it tends to be faster because the http tests are front-loaded and slow.
new_buckets = buckets_before + [second_half] + buckets_after
if self.test_bucket_list_fails(new_buckets):
self.buckets = new_buckets
return
new_buckets = buckets_before + [first_half] + buckets_after
if self.test_bucket_list_fails(new_buckets):
self.buckets = new_buckets
return
self.buckets = buckets_before + [first_half, second_half] + buckets_after
def test_bucket_list_fails(self, buckets):
tests = []
for bucket in buckets:
tests += bucket.tests
return self.test_fails(tests)
def test_fails(self, tests):
extra_args = ['--debug'] if self.is_debug else []
path_to_run_webkit_tests = self.webkit_finder.path_from_webkit_base('tools', 'test_sky')
output = self.executive.popen([path_to_run_webkit_tests, '--child-processes', '1', '--order', 'none', '--no-retry', '--no-show-results', '--verbose'] + extra_args + tests, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
failure_string = self.expected_failure + ' failed'
if failure_string in output.stderr.read():
return True
return False
def test_auto_stringify_args(self):
executive = Executive()
executive.run_command(command_line('echo', 1))
executive.popen(command_line('echo', 1)).wait()
