def test_coverage_works(self): # This is awkward; by design, running test-webkitpy -c will # create a .coverage file in tools, so we need to be # careful not to clobber an existing one, and to clean up. # FIXME: This design needs to change since it means we can't actually # run this method itself under coverage properly. filesystem = FileSystem() executive = Executive() module_path = filesystem.path_to_module(self.__module__) script_dir = module_path[0:module_path.find('webkitpy') - 1] coverage_file = filesystem.join(script_dir, '.coverage') coverage_file_orig = None if filesystem.exists(coverage_file): coverage_file_orig = coverage_file + '.orig' filesystem.move(coverage_file, coverage_file_orig) try: proc = executive.popen([sys.executable, filesystem.join(script_dir, 'test-webkitpy'), '-c', STUBS_CLASS + '.test_empty'], stdout=executive.PIPE, stderr=executive.PIPE) out, _ = proc.communicate() retcode = proc.returncode self.assertEqual(retcode, 0) self.assertIn('Cover', out) finally: if coverage_file_orig: filesystem.move(coverage_file_orig, coverage_file) elif filesystem.exists(coverage_file): filesystem.remove(coverage_file)
def test_coverage_works(self): # This is awkward; by design, running test-webkitpy -c will # create a .coverage file in tools, so we need to be # careful not to clobber an existing one, and to clean up. # FIXME: This design needs to change since it means we can't actually # run this method itself under coverage properly. filesystem = FileSystem() executive = Executive() module_path = filesystem.path_to_module(self.__module__) script_dir = module_path[0:module_path.find('webkitpy') - 1] coverage_file = filesystem.join(script_dir, '.coverage') coverage_file_orig = None if filesystem.exists(coverage_file): coverage_file_orig = coverage_file + '.orig' filesystem.move(coverage_file, coverage_file_orig) try: proc = executive.popen([ sys.executable, filesystem.join(script_dir, 'test-webkitpy'), '-c', STUBS_CLASS + '.test_empty' ], stdout=executive.PIPE, stderr=executive.PIPE) out, _ = proc.communicate() retcode = proc.returncode self.assertEqual(retcode, 0) self.assertIn('Cover', out) finally: if coverage_file_orig: filesystem.move(coverage_file_orig, coverage_file) elif filesystem.exists(coverage_file): filesystem.remove(coverage_file)
def test_auto_stringify_args(self): executive = Executive() executive.run_command(command_line('echo', 1)) with executive.popen(command_line('echo', 1), stdout=executive.PIPE) as process: process.wait() self.assertEqual('echo 1', executive.command_for_printing(['echo', 1]))
def integration_test_coverage_works(self): filesystem = FileSystem() executive = Executive() module_path = filesystem.path_to_module(self.__module__) script_dir = module_path[0:module_path.find('webkitpy') - 1] proc = executive.popen([sys.executable, filesystem.join(script_dir, 'test-webkitpy'), '-c', STUBS_CLASS + '.test_empty'], stdout=executive.PIPE, stderr=executive.PIPE) out, _ = proc.communicate() retcode = proc.returncode self.assertEqual(retcode, 0) self.assertIn('Cover', out)
def integration_test_coverage_works(self): filesystem = FileSystem() executive = Executive() module_path = filesystem.path_to_module(self.__module__) script_dir = module_path[0:module_path.find('webkitpy') - 1] proc = executive.popen([ sys.executable, filesystem.join(script_dir, 'test-webkitpy'), '-c', STUBS_CLASS + '.test_empty' ], stdout=executive.PIPE, stderr=executive.PIPE) out, _ = proc.communicate() retcode = proc.returncode self.assertEqual(retcode, 0) self.assertIn('Cover', out)
def serial_test_kill_all(self): executive = Executive() with executive.popen(never_ending_command(), stdout=subprocess.PIPE) as process: self.assertIsNone(process.poll()) # Process is running executive.kill_all(never_ending_command()[0]) # Note: Can't use a ternary since signal.SIGTERM is undefined for sys.platform == "win32" if sys.platform == "cygwin": expected_exit_code = 0 # os.kill results in exit(0) for this process. self.assertEqual(process.wait(), expected_exit_code) elif sys.platform.startswith('win'): # FIXME: https://bugs.webkit.org/show_bug.cgi?id=54790 # We seem to get either 0 or 1 here for some reason. self.assertIn(process.wait(), (0, 1)) else: expected_exit_code = -signal.SIGTERM self.assertEqual(process.wait(), expected_exit_code) # Killing again should fail silently. executive.kill_all(never_ending_command()[0])
def serial_test_kill_process(self): executive = Executive() with executive.popen(never_ending_command(), stdout=subprocess.PIPE) as process: self.assertEqual(process.poll(), None) # Process is running executive.kill_process(process.pid) # Note: Can't use a ternary since signal.SIGKILL is undefined for sys.platform == "win32" if sys.platform.startswith('win'): # FIXME: https://bugs.webkit.org/show_bug.cgi?id=54790 # We seem to get either 0 or 1 here for some reason. self.assertIn(process.wait(), (0, 1)) elif sys.platform == "cygwin": # FIXME: https://bugs.webkit.org/show_bug.cgi?id=98196 # cygwin seems to give us either SIGABRT or SIGKILL # Native Windows (via Cygwin) returns ENOTBLK (-15) self.assertIn(process.wait(), (-signal.SIGABRT, -signal.SIGKILL, -15)) else: expected_exit_code = -signal.SIGTERM self.assertEqual(process.wait(), expected_exit_code) # Killing again should fail silently. executive.kill_process(process.pid)
def test_popen_args(self): executive = Executive() # Explicitly naming the 'args' argument should not thow an exception. with executive.popen(args=command_line('echo', 1), stdout=executive.PIPE) as process: process.wait()
def test_popen_args(self): executive = Executive() # Explicitly naming the 'args' argument should not thow an exception. executive.popen(args=command_line('echo', 1), stdout=executive.PIPE).wait()
def test_auto_stringify_args(self): executive = Executive() executive.run_command(command_line('echo', 1)) executive.popen(command_line('echo', 1), stdout=executive.PIPE).wait() self.assertEqual('echo 1', executive.command_for_printing(['echo', 1]))
class Bisector(object): def __init__(self, tests, is_debug): self.executive = Executive() self.tests = tests self.expected_failure = tests[-1] self.is_debug = is_debug self.webkit_finder = WebKitFinder(FileSystem()) def bisect(self): if self.test_fails_in_isolation(): self.buckets = [Bucket([self.expected_failure])] print '%s fails when run in isolation.' % self.expected_failure self.print_result() return 0 if not self.test_fails(self.tests): _log.error('%s does not fail', self.expected_failure) return 1 # Split the list of test into buckets. Each bucket has at least one test required to cause # the expected failure at the end. Split buckets in half until there are only buckets left # with one item in them. self.buckets = [ Bucket(self.tests[:-1]), Bucket([self.expected_failure]) ] while not self.is_done(): self.print_progress() self.split_largest_bucket() self.print_result() self.verify_non_flaky() return 0 def test_fails_in_isolation(self): return self.test_bucket_list_fails([Bucket([self.expected_failure])]) def verify_non_flaky(self): print 'Verifying the failure is not flaky by running 10 times.' count_failures = 0 for _ in range(0, 10): if self.test_bucket_list_fails(self.buckets): count_failures += 1 print 'Failed %d/10 times' % count_failures def print_progress(self): count = 0 for bucket in self.buckets: count += len(bucket.tests) print '%d tests left, %d buckets' % (count, len(self.buckets)) def print_result(self): tests = [] for bucket in self.buckets: tests += bucket.tests extra_args = ' --debug' if self.is_debug else '' print 'run-webkit-tests%s --child-processes=1 --order=none %s' % ( extra_args, ' '.join(tests)) def is_done(self): for bucket in self.buckets: if bucket.size() > 1: return False return True def split_largest_bucket(self): index = 0 largest_index = 0 largest_size = 0 for bucket in self.buckets: if bucket.size() > largest_size: largest_index = index largest_size = bucket.size() index += 1 bucket_to_split = self.buckets[largest_index] halfway_point = int(largest_size / 2) first_half = Bucket(bucket_to_split.tests[:halfway_point]) second_half = Bucket(bucket_to_split.tests[halfway_point:]) buckets_before = self.buckets[:largest_index] buckets_after = self.buckets[largest_index + 1:] # Do the second half first because it tends to be faster because the http tests are front-loaded and slow. new_buckets = buckets_before + [second_half] + buckets_after if self.test_bucket_list_fails(new_buckets): self.buckets = new_buckets return new_buckets = buckets_before + [first_half] + buckets_after if self.test_bucket_list_fails(new_buckets): self.buckets = new_buckets return self.buckets = buckets_before + [first_half, second_half ] + buckets_after def test_bucket_list_fails(self, buckets): tests = [] for bucket in buckets: tests += bucket.tests return self.test_fails(tests) def test_fails(self, tests): extra_args = ['--debug'] if self.is_debug else [] path_to_run_webkit_tests = self.webkit_finder.path_from_webkit_base( 'Tools', 'Scripts', 'run-webkit-tests') output = self.executive.popen([ path_to_run_webkit_tests, '--child-processes', '1', '--order', 'none', '--no-retry', '--no-show-results', '--verbose' ] + extra_args + tests, stdout=subprocess.PIPE, stderr=subprocess.PIPE) failure_string = self.expected_failure + ' failed' if failure_string in output.stderr.read(): return True return False
def test_popen_args(self): executive = Executive() # Explicitly naming the 'args' argument should not thow an exception. executive.popen(args=command_line('echo', 1)).wait()
def test_auto_stringify_args(self): executive = Executive() executive.run_command(command_line('echo', 1)) executive.popen(command_line('echo', 1)).wait()
class Bisector(object): def __init__(self, tests, is_debug): self.executive = Executive() self.tests = tests self.expected_failure = tests[-1] self.is_debug = is_debug self.webkit_finder = WebKitFinder(FileSystem()) def bisect(self): if self.test_fails_in_isolation(): self.buckets = [Bucket([self.expected_failure])] print '%s fails when run in isolation.' % self.expected_failure self.print_result() return 0 if not self.test_fails(self.tests): _log.error('%s does not fail' % self.expected_failure) return 1 # Split the list of test into buckets. Each bucket has at least one test required to cause # the expected failure at the end. Split buckets in half until there are only buckets left # with one item in them. self.buckets = [Bucket(self.tests[:-1]), Bucket([self.expected_failure])] while not self.is_done(): self.print_progress() self.split_largest_bucket() self.print_result() self.verify_non_flaky() return 0 def test_fails_in_isolation(self): return self.test_bucket_list_fails([Bucket([self.expected_failure])]) def verify_non_flaky(self): print 'Verifying the failure is not flaky by running 10 times.' count_failures = 0 for i in range(0, 10): if self.test_bucket_list_fails(self.buckets): count_failures += 1 print 'Failed %d/10 times' % count_failures def print_progress(self): count = 0 for bucket in self.buckets: count += len(bucket.tests) print '%d tests left, %d buckets' % (count, len(self.buckets)) def print_result(self): tests = [] for bucket in self.buckets: tests += bucket.tests extra_args = ' --debug' if self.is_debug else '' print 'run-webkit-tests%s --child-processes=1 --order=none %s' % (extra_args, " ".join(tests)) def is_done(self): for bucket in self.buckets: if bucket.size() > 1: return False return True def split_largest_bucket(self): index = 0 largest_index = 0 largest_size = 0 for bucket in self.buckets: if bucket.size() > largest_size: largest_index = index largest_size = bucket.size() index += 1 bucket_to_split = self.buckets[largest_index] halfway_point = int(largest_size / 2) first_half = Bucket(bucket_to_split.tests[:halfway_point]) second_half = Bucket(bucket_to_split.tests[halfway_point:]) buckets_before = self.buckets[:largest_index] buckets_after = self.buckets[largest_index + 1:] # Do the second half first because it tends to be faster because the http tests are front-loaded and slow. new_buckets = buckets_before + [second_half] + buckets_after if self.test_bucket_list_fails(new_buckets): self.buckets = new_buckets return new_buckets = buckets_before + [first_half] + buckets_after if self.test_bucket_list_fails(new_buckets): self.buckets = new_buckets return self.buckets = buckets_before + [first_half, second_half] + buckets_after def test_bucket_list_fails(self, buckets): tests = [] for bucket in buckets: tests += bucket.tests return self.test_fails(tests) def test_fails(self, tests): extra_args = ['--debug'] if self.is_debug else [] path_to_run_webkit_tests = self.webkit_finder.path_from_webkit_base('tools', 'test_sky') output = self.executive.popen([path_to_run_webkit_tests, '--child-processes', '1', '--order', 'none', '--no-retry', '--no-show-results', '--verbose'] + extra_args + tests, stdout=subprocess.PIPE, stderr=subprocess.PIPE) failure_string = self.expected_failure + ' failed' if failure_string in output.stderr.read(): return True return False