def run_tests(self, expectations, test_inputs, tests_to_skip, num_workers, retry_attempt): self._expectations = expectations self._test_inputs = test_inputs self._retry_attempt = retry_attempt self._shards_to_redo = [] # FIXME: rename all variables to test_run_results or some such ... run_results = TestRunResults(self._expectations, len(test_inputs) + len(tests_to_skip)) self._current_run_results = run_results self._printer.num_tests = len(test_inputs) self._printer.num_completed = 0 if retry_attempt < 1: self._printer.print_expected(run_results, self._expectations.get_tests_with_result_type) for test_name in set(tests_to_skip): result = test_results.TestResult(test_name) result.type = test_expectations.SKIP run_results.add(result, expected=True, test_is_slow=self._test_is_slow(test_name)) self._printer.write_update('Sharding tests ...') locked_shards, unlocked_shards = self._sharder.shard_tests(test_inputs, int(self._options.child_processes), self._options.fully_parallel, self._options.run_singly or (self._options.batch_size == 1)) # We don't have a good way to coordinate the workers so that they don't # try to run the shards that need a lock. The easiest solution is to # run all of the locked shards first. all_shards = locked_shards + unlocked_shards num_workers = min(num_workers, len(all_shards)) if retry_attempt < 1: self._printer.print_workers_and_shards(num_workers, len(all_shards), len(locked_shards)) if self._options.dry_run: return run_results self._printer.write_update('Starting %s ...' % grammar.pluralize('worker', num_workers)) start_time = time.time() try: with message_pool.get(self, self._worker_factory, num_workers, self._port.host) as pool: pool.run(('test_list', shard.name, shard.test_inputs) for shard in all_shards) if self._shards_to_redo: num_workers -= len(self._shards_to_redo) if num_workers > 0: with message_pool.get(self, self._worker_factory, num_workers, self._port.host) as pool: pool.run(('test_list', shard.name, shard.test_inputs) for shard in self._shards_to_redo) except TestRunInterruptedException, e: _log.warning(e.reason) run_results.interrupted = True
def run_tests(self, expectations, test_inputs, tests_to_skip, num_workers, retrying): self._expectations = expectations self._test_inputs = test_inputs self._retrying = retrying # FIXME: rename all variables to test_run_results or some such ... run_results = TestRunResults(self._expectations, len(test_inputs) + len(tests_to_skip)) self._current_run_results = run_results self._printer.num_tests = len(test_inputs) self._printer.num_started = 0 if not retrying: self._printer.print_expected( run_results, self._expectations.model().get_tests_with_result_type) for test_name in set(tests_to_skip): result = test_results.TestResult(test_name) result.type = test_expectations.SKIP run_results.add(result, expected=True, test_is_slow=self._test_is_slow(test_name)) self._printer.write_update('Sharding tests ...') all_shards = self._sharder.shard_tests( test_inputs, int(self._options.child_processes), self._options.fully_parallel) self._printer.print_workers_and_shards(num_workers, len(all_shards)) if self._options.dry_run: return run_results self._printer.write_update('Starting %s ...' % grammar.pluralize(num_workers, "worker")) try: with message_pool.get(self, self._worker_factory, num_workers, self._port.worker_startup_delay_secs(), self._port.host) as pool: pool.run(('test_list', shard.name, shard.test_inputs) for shard in all_shards) except TestRunInterruptedException as e: _log.warning(e.reason) run_results.interrupted = True except KeyboardInterrupt: self._printer.flush() self._printer.writeln('Interrupted, exiting ...') run_results.keyboard_interrupted = True except Exception as e: _log.debug('%s("%s") raised, exiting' % (e.__class__.__name__, str(e))) raise return run_results
def run_tests( self, expectations, test_inputs, tests_to_skip, num_workers, needs_http, needs_websockets, needs_web_platform_test_server, retrying, ): self._expectations = expectations self._test_inputs = test_inputs self._needs_http = needs_http self._needs_websockets = needs_websockets self._needs_web_platform_test_server = needs_web_platform_test_server self._retrying = retrying # FIXME: rename all variables to test_run_results or some such ... run_results = TestRunResults(self._expectations, len(test_inputs) + len(tests_to_skip)) self._current_run_results = run_results self._printer.num_tests = len(test_inputs) self._printer.num_started = 0 if not retrying: self._printer.print_expected(run_results, self._expectations.model().get_tests_with_result_type) for test_name in set(tests_to_skip): result = test_results.TestResult(test_name) result.type = test_expectations.SKIP run_results.add(result, expected=True, test_is_slow=self._test_is_slow(test_name)) self._printer.write_update("Sharding tests ...") all_shards = self._sharder.shard_tests( test_inputs, int(self._options.child_processes), self._options.fully_parallel ) if (self._needs_http and self._options.http) or self._needs_web_platform_test_server: self.start_servers() num_workers = min(num_workers, len(all_shards)) self._printer.print_workers_and_shards(num_workers, len(all_shards)) if self._options.dry_run: return run_results self._printer.write_update("Starting %s ..." % grammar.pluralize(num_workers, "worker")) try: with message_pool.get( self, self._worker_factory, num_workers, self._port.worker_startup_delay_secs(), self._port.host ) as pool: pool.run(("test_list", shard.name, shard.test_inputs) for shard in all_shards) except TestRunInterruptedException, e: _log.warning(e.reason) run_results.interrupted = True
def run_tests(self, expectations, test_inputs, tests_to_skip, num_workers, needs_http, needs_websockets, needs_web_platform_test_server, retrying): self._expectations = expectations self._test_inputs = test_inputs self._needs_http = needs_http self._needs_websockets = needs_websockets self._needs_web_platform_test_server = needs_web_platform_test_server self._retrying = retrying # FIXME: rename all variables to test_run_results or some such ... run_results = TestRunResults(self._expectations, len(test_inputs) + len(tests_to_skip)) self._current_run_results = run_results self._printer.num_tests = len(test_inputs) self._printer.num_started = 0 if not retrying: self._printer.print_expected( run_results, self._expectations.model().get_tests_with_result_type) for test_name in set(tests_to_skip): result = test_results.TestResult(test_name) result.type = test_expectations.SKIP run_results.add(result, expected=True, test_is_slow=self._test_is_slow(test_name)) self._printer.write_update('Sharding tests ...') all_shards = self._sharder.shard_tests( test_inputs, int(self._options.child_processes), self._options.fully_parallel) if (self._needs_http and self._options.http) or self._needs_web_platform_test_server: self.start_servers() num_workers = min(num_workers, len(all_shards)) self._printer.print_workers_and_shards(num_workers, len(all_shards)) if self._options.dry_run: return run_results self._printer.write_update('Starting %s ...' % grammar.pluralize(num_workers, "worker")) try: with message_pool.get(self, self._worker_factory, num_workers, self._port.worker_startup_delay_secs(), self._port.host) as pool: pool.run(('test_list', shard.name, shard.test_inputs) for shard in all_shards) except TestRunInterruptedException, e: _log.warning(e.reason) run_results.interrupted = True
def run_tests(self, test_inputs, expectations, result_summary, num_workers, needs_http, needs_websockets, retrying): """Returns a tuple of (thread_timings, test_timings, individual_test_timings): thread_timings is a list of dicts with the total runtime of each thread with 'name', 'num_tests', 'total_time' properties test_timings is a list of timings for each sharded subdirectory of the form [time, directory_name, num_tests] individual_test_timings is a list of run times for each test in the form {filename:filename, test_run_time:test_run_time} """ self._current_result_summary = result_summary self._expectations = expectations self._needs_http = needs_http self._needs_websockets = needs_websockets self._retrying = retrying self._test_files_list = [test_input.test_name for test_input in test_inputs] self._printer.num_tests = len(self._test_files_list) self._printer.num_completed = 0 self._all_results = [] self._group_stats = {} self._worker_stats = {} self._has_http_lock = False self._remaining_locked_shards = [] self._printer.write_update('Sharding tests ...') locked_shards, unlocked_shards = self._sharder.shard_tests(test_inputs, int(self._options.child_processes), self._options.fully_parallel) # FIXME: We don't have a good way to coordinate the workers so that # they don't try to run the shards that need a lock if we don't actually # have the lock. The easiest solution at the moment is to grab the # lock at the beginning of the run, and then run all of the locked # shards first. This minimizes the time spent holding the lock, but # means that we won't be running tests while we're waiting for the lock. # If this becomes a problem in practice we'll need to change this. all_shards = locked_shards + unlocked_shards self._remaining_locked_shards = locked_shards if self._port.requires_http_server() or (locked_shards and self._options.http): self.start_servers_with_lock(2 * min(num_workers, len(locked_shards))) num_workers = min(num_workers, len(all_shards)) self._printer.print_workers_and_shards(num_workers, len(all_shards), len(locked_shards)) if self._options.dry_run: return (self._worker_stats.values(), self._group_stats, self._all_results) self._printer.write_update('Starting %s ...' % grammar.pluralize('worker', num_workers)) try: with message_pool.get(self, self._worker_factory, num_workers, self._port.worker_startup_delay_secs(), self._port.host) as pool: pool.run(('test_list', shard.name, shard.test_inputs) for shard in all_shards) except TestRunInterruptedException, e: _log.warning(e.reason) result_summary.interrupted = True
def run(self, tests, num_workers): if not tests: return self.printer.write_update('Sharding tests ...') shards = Runner._shard_tests(tests) with message_pool.get( self, lambda caller: _Worker(caller, self.port, shards), min(num_workers, len(shards))) as pool: pool.run(('test', shard) for shard, _ in shards.iteritems())
def run(self, suite): run_start_time = time.time() all_test_names = self.all_test_names(suite) self.printer.num_tests = len(all_test_names) with message_pool.get(self, self.worker_factory, int(self.options.child_processes)) as pool: pool.run(('test', test_name) for test_name in all_test_names) self.printer.print_result(self.result, time.time() - run_start_time) return self.result
def run_tests(self, expectations, test_inputs, tests_to_skip, num_workers, needs_http, needs_websockets, retrying): self._expectations = expectations self._test_inputs = test_inputs self._needs_http = needs_http self._needs_websockets = needs_websockets self._retrying = retrying # FIXME: rename all variables to test_run_results or some such ... run_results = TestRunResults(self._expectations, len(test_inputs) + len(tests_to_skip)) self._current_run_results = run_results self._remaining_locked_shards = [] self._has_http_lock = False self._printer.num_tests = len(test_inputs) self._printer.num_started = 0 if not retrying: self._printer.print_expected(run_results, self._expectations.get_tests_with_result_type) for test_name in set(tests_to_skip): result = test_results.TestResult(test_name) result.type = test_expectations.SKIP run_results.add(result, expected=True, test_is_slow=self._test_is_slow(test_name)) self._printer.write_update('Sharding tests ...') locked_shards, unlocked_shards = self._sharder.shard_tests(test_inputs, int(self._options.child_processes), self._options.fully_parallel) # FIXME: We don't have a good way to coordinate the workers so that # they don't try to run the shards that need a lock if we don't actually # have the lock. The easiest solution at the moment is to grab the # lock at the beginning of the run, and then run all of the locked # shards first. This minimizes the time spent holding the lock, but # means that we won't be running tests while we're waiting for the lock. # If this becomes a problem in practice we'll need to change this. all_shards = locked_shards + unlocked_shards self._remaining_locked_shards = locked_shards if locked_shards and self._options.http: self.start_servers_with_lock(2 * min(num_workers, len(locked_shards))) num_workers = min(num_workers, len(all_shards)) self._printer.print_workers_and_shards(num_workers, len(all_shards), len(locked_shards)) if self._options.dry_run: return run_results self._printer.write_update('Starting %s ...' % grammar.pluralize('worker', num_workers)) try: with message_pool.get(self, self._worker_factory, num_workers, self._port.worker_startup_delay_secs(), self._port.host) as pool: pool.run(('test_list', shard.name, shard.test_inputs) for shard in all_shards) except TestRunInterruptedException, e: _log.warning(e.reason) run_results.interrupted = True
def run_tests(self, expectations, test_inputs, tests_to_skip, num_workers, needs_http, needs_websockets, retrying): self._expectations = expectations self._test_inputs = test_inputs self._needs_http = needs_http self._needs_websockets = needs_websockets self._retrying = retrying # FIXME: rename all variables to test_run_results or some such ... run_results = TestRunResults(self._expectations, len(test_inputs) + len(tests_to_skip)) self._current_run_results = run_results self._remaining_locked_shards = [] self._has_http_lock = False self._printer.num_tests = len(test_inputs) self._printer.num_started = 0 if not retrying: self._printer.print_expected(run_results, self._expectations.model().get_tests_with_result_type) for test_name in set(tests_to_skip): result = test_results.TestResult(test_name) result.type = test_expectations.SKIP run_results.add(result, expected=True, test_is_slow=self._test_is_slow(test_name)) self._printer.write_update('Sharding tests ...') locked_shards, unlocked_shards = self._sharder.shard_tests(test_inputs, int(self._options.child_processes), self._options.fully_parallel) # FIXME: We don't have a good way to coordinate the workers so that # they don't try to run the shards that need a lock if we don't actually # have the lock. The easiest solution at the moment is to grab the # lock at the beginning of the run, and then run all of the locked # shards first. This minimizes the time spent holding the lock, but # means that we won't be running tests while we're waiting for the lock. # If this becomes a problem in practice we'll need to change this. all_shards = locked_shards + unlocked_shards self._remaining_locked_shards = locked_shards if locked_shards and self._options.http: self.start_servers_with_lock(2 * min(num_workers, len(locked_shards))) num_workers = min(num_workers, len(all_shards)) self._printer.print_workers_and_shards(num_workers, len(all_shards), len(locked_shards)) if self._options.dry_run: return run_results self._printer.write_update('Starting %s ...' % grammar.pluralize(num_workers, "worker")) try: with message_pool.get(self, self._worker_factory, num_workers, self._port.worker_startup_delay_secs(), self._port.host) as pool: pool.run(('test_list', shard.name, shard.test_inputs) for shard in all_shards) except TestRunInterruptedException, e: _log.warning(e.reason) run_results.interrupted = True
def run(self, tests, num_workers): if not tests: return self.printer.write_update('Sharding tests ...') shards = Runner._shard_tests(tests) original_level = server_process_logger.level server_process_logger.setLevel(logging.CRITICAL) try: self._num_workers = min(num_workers, len(shards)) with message_pool.get(self, lambda caller: _Worker(caller, self.port, shards), self._num_workers) as pool: pool.run(('test', shard) for shard, _ in shards.iteritems()) finally: server_process_logger.setLevel(original_level)
def run(self, test_names, num_workers): if not test_names: return num_workers = min(num_workers, len(test_names)) with message_pool.get(self, self.worker_factory, num_workers) as pool: pool.run(('test', test_name) for test_name in test_names)
def run_tests(self, test_inputs, expectations, result_summary, num_workers, needs_http, needs_websockets, retrying): """Returns a tuple of (interrupted, keyboard_interrupted, thread_timings, test_timings, individual_test_timings): interrupted is whether the run was interrupted keyboard_interrupted is whether the interruption was because someone typed Ctrl^C thread_timings is a list of dicts with the total runtime of each thread with 'name', 'num_tests', 'total_time' properties test_timings is a list of timings for each sharded subdirectory of the form [time, directory_name, num_tests] individual_test_timings is a list of run times for each test in the form {filename:filename, test_run_time:test_run_time} result_summary: summary object to populate with the results """ self._current_result_summary = result_summary self._expectations = expectations self._needs_http = needs_http self._needs_websockets = needs_websockets self._retrying = retrying self._test_files_list = [ test_input.test_name for test_input in test_inputs ] self._printer.num_tests = len(self._test_files_list) self._printer.num_completed = 0 self._all_results = [] self._group_stats = {} self._worker_stats = {} self._has_http_lock = False self._remaining_locked_shards = [] keyboard_interrupted = False interrupted = False self._printer.write_update('Sharding tests ...') locked_shards, unlocked_shards = self._sharder.shard_tests( test_inputs, int(self._options.child_processes), self._options.fully_parallel) # FIXME: We don't have a good way to coordinate the workers so that # they don't try to run the shards that need a lock if we don't actually # have the lock. The easiest solution at the moment is to grab the # lock at the beginning of the run, and then run all of the locked # shards first. This minimizes the time spent holding the lock, but # means that we won't be running tests while we're waiting for the lock. # If this becomes a problem in practice we'll need to change this. all_shards = locked_shards + unlocked_shards self._remaining_locked_shards = locked_shards if self._port.requires_http_server() or (locked_shards and self._options.http): self.start_servers_with_lock(2 * min(num_workers, len(locked_shards))) num_workers = min(num_workers, len(all_shards)) self._printer.print_workers_and_shards(num_workers, len(all_shards), len(locked_shards)) if self._options.dry_run: return (keyboard_interrupted, interrupted, self._worker_stats.values(), self._group_stats, self._all_results) self._printer.write_update('Starting %s ...' % grammar.pluralize('worker', num_workers)) try: with message_pool.get(self, self._worker_factory, num_workers, self._port.worker_startup_delay_secs(), self._port.host) as pool: pool.run(('test_list', shard.name, shard.test_inputs) for shard in all_shards) except KeyboardInterrupt: self._printer.flush() self._printer.writeln('Interrupted, exiting ...') keyboard_interrupted = True except TestRunInterruptedException, e: _log.warning(e.reason) interrupted = True