def test_auto_stringify_args(self): executive = Executive() executive.run_command(command_line('echo', 1)) executive.popen(command_line('echo', 1), stdout=executive.PIPE).wait() self.assertEqual('echo 1', executive.command_for_printing(['echo', 1]))
def test_popen_args(self): executive = Executive() # Explicitly naming the 'args' argument should not throw an exception. executive.popen(args=command_line('echo', 1), stdout=executive.PIPE).wait()
class Bisector(object): def __init__(self, tests, is_debug): self.executive = Executive() self.tests = tests self.expected_failure = tests[-1] self.is_debug = is_debug self.path_finder = PathFinder(FileSystem()) def bisect(self): if self.test_fails_in_isolation(): self.buckets = [Bucket([self.expected_failure])] print '%s fails when run in isolation.' % self.expected_failure self.print_result() return 0 if not self.test_fails(self.tests): _log.error('%s does not fail', self.expected_failure) return 1 # Split the list of test into buckets. Each bucket has at least one test required to cause # the expected failure at the end. Split buckets in half until there are only buckets left # with one item in them. self.buckets = [ Bucket(self.tests[:-1]), Bucket([self.expected_failure]) ] while not self.is_done(): self.print_progress() self.split_largest_bucket() self.print_result() self.verify_non_flaky() return 0 def test_fails_in_isolation(self): return self.test_bucket_list_fails([Bucket([self.expected_failure])]) def verify_non_flaky(self): print 'Verifying the failure is not flaky by running 10 times.' count_failures = 0 for _ in range(0, 10): if self.test_bucket_list_fails(self.buckets): count_failures += 1 print 'Failed %d/10 times' % count_failures def print_progress(self): count = 0 for bucket in self.buckets: count += len(bucket.tests) print '%d tests left, %d buckets' % (count, len(self.buckets)) def print_result(self): tests = [] for bucket in self.buckets: tests += bucket.tests extra_args = ' --debug' if self.is_debug else '' print 'run_web_tests.py%s --child-processes=1 --order=none %s' % ( extra_args, ' '.join(tests)) def is_done(self): for bucket in self.buckets: if bucket.size() > 1: return False return True def split_largest_bucket(self): index = 0 largest_index = 0 largest_size = 0 for bucket in self.buckets: if bucket.size() > largest_size: largest_index = index largest_size = bucket.size() index += 1 bucket_to_split = self.buckets[largest_index] halfway_point = int(largest_size / 2) first_half = Bucket(bucket_to_split.tests[:halfway_point]) second_half = Bucket(bucket_to_split.tests[halfway_point:]) buckets_before = self.buckets[:largest_index] buckets_after = self.buckets[largest_index + 1:] # Do the second half first because it tends to be faster because the http tests are front-loaded and slow. new_buckets = buckets_before + [second_half] + buckets_after if self.test_bucket_list_fails(new_buckets): self.buckets = new_buckets return new_buckets = buckets_before + [first_half] + buckets_after if self.test_bucket_list_fails(new_buckets): self.buckets = new_buckets return self.buckets = buckets_before + [first_half, second_half ] + buckets_after def test_bucket_list_fails(self, buckets): tests = [] for bucket in buckets: tests += bucket.tests return self.test_fails(tests) def test_fails(self, tests): extra_args = ['--debug'] if self.is_debug else [] path_to_run_webkit_tests = self.path_finder.path_from_tools_scripts( 'run_web_tests.py') output = self.executive.popen([ path_to_run_webkit_tests, '--child-processes', '1', '--order', 'none', '--no-retry', '--no-show-results', '--verbose' ] + extra_args + tests, stdout=subprocess.PIPE, stderr=subprocess.PIPE) failure_string = self.expected_failure + ' failed' if failure_string in output.stderr.read(): return True return False