def run_tests(self, expectations, test_inputs, tests_to_skip, num_workers, retry_attempt):
self._expectations = expectations
self._test_inputs = test_inputs
self._retry_attempt = retry_attempt
self._shards_to_redo = []
# FIXME: rename all variables to test_run_results or some such ...
run_results = TestRunResults(self._expectations, len(test_inputs) + len(tests_to_skip))
self._current_run_results = run_results
self._printer.num_tests = len(test_inputs)
self._printer.num_completed = 0
if retry_attempt < 1:
self._printer.print_expected(run_results, self._expectations.get_tests_with_result_type)
for test_name in set(tests_to_skip):
result = test_results.TestResult(test_name)
result.type = test_expectations.SKIP
run_results.add(result, expected=True, test_is_slow=self._test_is_slow(test_name))
self._printer.write_update('Sharding tests ...')
locked_shards, unlocked_shards = self._sharder.shard_tests(test_inputs,
int(self._options.child_processes), self._options.fully_parallel,
self._options.run_singly or (self._options.batch_size == 1))
# We don't have a good way to coordinate the workers so that they don't
# try to run the shards that need a lock. The easiest solution is to
# run all of the locked shards first.
all_shards = locked_shards + unlocked_shards
num_workers = min(num_workers, len(all_shards))
if retry_attempt < 1:
self._printer.print_workers_and_shards(num_workers, len(all_shards), len(locked_shards))
if self._options.dry_run:
return run_results
self._printer.write_update('Starting %s ...' % grammar.pluralize('worker', num_workers))
start_time = time.time()
try:
with message_pool.get(self, self._worker_factory, num_workers, self._port.host) as pool:
pool.run(('test_list', shard.name, shard.test_inputs) for shard in all_shards)
if self._shards_to_redo:
num_workers -= len(self._shards_to_redo)
if num_workers > 0:
with message_pool.get(self, self._worker_factory, num_workers, self._port.host) as pool:
pool.run(('test_list', shard.name, shard.test_inputs) for shard in self._shards_to_redo)
except TestRunInterruptedException, e:
_log.warning(e.reason)
run_results.interrupted = True
def run_tests(self, expectations, test_inputs, tests_to_skip, num_workers,
retrying):
self._expectations = expectations
self._test_inputs = test_inputs
self._retrying = retrying
# FIXME: rename all variables to test_run_results or some such ...
run_results = TestRunResults(self._expectations,
len(test_inputs) + len(tests_to_skip))
self._current_run_results = run_results
self._printer.num_tests = len(test_inputs)
self._printer.num_started = 0
if not retrying:
self._printer.print_expected(
run_results,
self._expectations.model().get_tests_with_result_type)
for test_name in set(tests_to_skip):
result = test_results.TestResult(test_name)
result.type = test_expectations.SKIP
run_results.add(result,
expected=True,
test_is_slow=self._test_is_slow(test_name))
self._printer.write_update('Sharding tests ...')
all_shards = self._sharder.shard_tests(
test_inputs, int(self._options.child_processes),
self._options.fully_parallel)
self._printer.print_workers_and_shards(num_workers, len(all_shards))
if self._options.dry_run:
return run_results
self._printer.write_update('Starting %s ...' %
grammar.pluralize(num_workers, "worker"))
try:
with message_pool.get(self, self._worker_factory, num_workers,
self._port.worker_startup_delay_secs(),
self._port.host) as pool:
pool.run(('test_list', shard.name, shard.test_inputs)
for shard in all_shards)
except TestRunInterruptedException as e:
_log.warning(e.reason)
run_results.interrupted = True
except KeyboardInterrupt:
self._printer.flush()
self._printer.writeln('Interrupted, exiting ...')
run_results.keyboard_interrupted = True
except Exception as e:
_log.debug('%s("%s") raised, exiting' %
(e.__class__.__name__, str(e)))
raise
return run_results
def run_tests(
self,
expectations,
test_inputs,
tests_to_skip,
num_workers,
needs_http,
needs_websockets,
needs_web_platform_test_server,
retrying,
):
self._expectations = expectations
self._test_inputs = test_inputs
self._needs_http = needs_http
self._needs_websockets = needs_websockets
self._needs_web_platform_test_server = needs_web_platform_test_server
self._retrying = retrying
# FIXME: rename all variables to test_run_results or some such ...
run_results = TestRunResults(self._expectations, len(test_inputs) + len(tests_to_skip))
self._current_run_results = run_results
self._printer.num_tests = len(test_inputs)
self._printer.num_started = 0
if not retrying:
self._printer.print_expected(run_results, self._expectations.model().get_tests_with_result_type)
for test_name in set(tests_to_skip):
result = test_results.TestResult(test_name)
result.type = test_expectations.SKIP
run_results.add(result, expected=True, test_is_slow=self._test_is_slow(test_name))
self._printer.write_update("Sharding tests ...")
all_shards = self._sharder.shard_tests(
test_inputs, int(self._options.child_processes), self._options.fully_parallel
)
if (self._needs_http and self._options.http) or self._needs_web_platform_test_server:
self.start_servers()
num_workers = min(num_workers, len(all_shards))
self._printer.print_workers_and_shards(num_workers, len(all_shards))
if self._options.dry_run:
return run_results
self._printer.write_update("Starting %s ..." % grammar.pluralize(num_workers, "worker"))
try:
with message_pool.get(
self, self._worker_factory, num_workers, self._port.worker_startup_delay_secs(), self._port.host
) as pool:
pool.run(("test_list", shard.name, shard.test_inputs) for shard in all_shards)
except TestRunInterruptedException, e:
_log.warning(e.reason)
run_results.interrupted = True
def run_tests(self, expectations, test_inputs, tests_to_skip, num_workers,
needs_http, needs_websockets, needs_web_platform_test_server,
retrying):
self._expectations = expectations
self._test_inputs = test_inputs
self._needs_http = needs_http
self._needs_websockets = needs_websockets
self._needs_web_platform_test_server = needs_web_platform_test_server
self._retrying = retrying
# FIXME: rename all variables to test_run_results or some such ...
run_results = TestRunResults(self._expectations,
len(test_inputs) + len(tests_to_skip))
self._current_run_results = run_results
self._printer.num_tests = len(test_inputs)
self._printer.num_started = 0
if not retrying:
self._printer.print_expected(
run_results,
self._expectations.model().get_tests_with_result_type)
for test_name in set(tests_to_skip):
result = test_results.TestResult(test_name)
result.type = test_expectations.SKIP
run_results.add(result,
expected=True,
test_is_slow=self._test_is_slow(test_name))
self._printer.write_update('Sharding tests ...')
all_shards = self._sharder.shard_tests(
test_inputs, int(self._options.child_processes),
self._options.fully_parallel)
if (self._needs_http and
self._options.http) or self._needs_web_platform_test_server:
self.start_servers()
num_workers = min(num_workers, len(all_shards))
self._printer.print_workers_and_shards(num_workers, len(all_shards))
if self._options.dry_run:
return run_results
self._printer.write_update('Starting %s ...' %
grammar.pluralize(num_workers, "worker"))
try:
with message_pool.get(self, self._worker_factory, num_workers,
self._port.worker_startup_delay_secs(),
self._port.host) as pool:
pool.run(('test_list', shard.name, shard.test_inputs)
for shard in all_shards)
except TestRunInterruptedException, e:
_log.warning(e.reason)
run_results.interrupted = True
def run_tests(self, test_inputs, expectations, result_summary, num_workers, needs_http, needs_websockets, retrying):
"""Returns a tuple of (thread_timings, test_timings, individual_test_timings):
thread_timings is a list of dicts with the total runtime
of each thread with 'name', 'num_tests', 'total_time' properties
test_timings is a list of timings for each sharded subdirectory
of the form [time, directory_name, num_tests]
individual_test_timings is a list of run times for each test
in the form {filename:filename, test_run_time:test_run_time}
"""
self._current_result_summary = result_summary
self._expectations = expectations
self._needs_http = needs_http
self._needs_websockets = needs_websockets
self._retrying = retrying
self._test_files_list = [test_input.test_name for test_input in test_inputs]
self._printer.num_tests = len(self._test_files_list)
self._printer.num_completed = 0
self._all_results = []
self._group_stats = {}
self._worker_stats = {}
self._has_http_lock = False
self._remaining_locked_shards = []
self._printer.write_update('Sharding tests ...')
locked_shards, unlocked_shards = self._sharder.shard_tests(test_inputs, int(self._options.child_processes), self._options.fully_parallel)
# FIXME: We don't have a good way to coordinate the workers so that
# they don't try to run the shards that need a lock if we don't actually
# have the lock. The easiest solution at the moment is to grab the
# lock at the beginning of the run, and then run all of the locked
# shards first. This minimizes the time spent holding the lock, but
# means that we won't be running tests while we're waiting for the lock.
# If this becomes a problem in practice we'll need to change this.
all_shards = locked_shards + unlocked_shards
self._remaining_locked_shards = locked_shards
if self._port.requires_http_server() or (locked_shards and self._options.http):
self.start_servers_with_lock(2 * min(num_workers, len(locked_shards)))
num_workers = min(num_workers, len(all_shards))
self._printer.print_workers_and_shards(num_workers, len(all_shards), len(locked_shards))
if self._options.dry_run:
return (self._worker_stats.values(), self._group_stats, self._all_results)
self._printer.write_update('Starting %s ...' % grammar.pluralize('worker', num_workers))
try:
with message_pool.get(self, self._worker_factory, num_workers, self._port.worker_startup_delay_secs(), self._port.host) as pool:
pool.run(('test_list', shard.name, shard.test_inputs) for shard in all_shards)
except TestRunInterruptedException, e:
_log.warning(e.reason)
result_summary.interrupted = True
def run(self, tests, num_workers):
if not tests:
return
self.printer.write_update('Sharding tests ...')
shards = Runner._shard_tests(tests)
with message_pool.get(
self, lambda caller: _Worker(caller, self.port, shards),
min(num_workers, len(shards))) as pool:
pool.run(('test', shard) for shard, _ in shards.iteritems())
def run(self, suite):
run_start_time = time.time()
all_test_names = self.all_test_names(suite)
self.printer.num_tests = len(all_test_names)
with message_pool.get(self, self.worker_factory,
int(self.options.child_processes)) as pool:
pool.run(('test', test_name) for test_name in all_test_names)
self.printer.print_result(self.result, time.time() - run_start_time)
return self.result
def run_tests(self, expectations, test_inputs, tests_to_skip, num_workers, needs_http, needs_websockets, retrying):
self._expectations = expectations
self._test_inputs = test_inputs
self._needs_http = needs_http
self._needs_websockets = needs_websockets
self._retrying = retrying
# FIXME: rename all variables to test_run_results or some such ...
run_results = TestRunResults(self._expectations, len(test_inputs) + len(tests_to_skip))
self._current_run_results = run_results
self._remaining_locked_shards = []
self._has_http_lock = False
self._printer.num_tests = len(test_inputs)
self._printer.num_started = 0
if not retrying:
self._printer.print_expected(run_results, self._expectations.get_tests_with_result_type)
for test_name in set(tests_to_skip):
result = test_results.TestResult(test_name)
result.type = test_expectations.SKIP
run_results.add(result, expected=True, test_is_slow=self._test_is_slow(test_name))
self._printer.write_update('Sharding tests ...')
locked_shards, unlocked_shards = self._sharder.shard_tests(test_inputs, int(self._options.child_processes), self._options.fully_parallel)
# FIXME: We don't have a good way to coordinate the workers so that
# they don't try to run the shards that need a lock if we don't actually
# have the lock. The easiest solution at the moment is to grab the
# lock at the beginning of the run, and then run all of the locked
# shards first. This minimizes the time spent holding the lock, but
# means that we won't be running tests while we're waiting for the lock.
# If this becomes a problem in practice we'll need to change this.
all_shards = locked_shards + unlocked_shards
self._remaining_locked_shards = locked_shards
if locked_shards and self._options.http:
self.start_servers_with_lock(2 * min(num_workers, len(locked_shards)))
num_workers = min(num_workers, len(all_shards))
self._printer.print_workers_and_shards(num_workers, len(all_shards), len(locked_shards))
if self._options.dry_run:
return run_results
self._printer.write_update('Starting %s ...' % grammar.pluralize('worker', num_workers))
try:
with message_pool.get(self, self._worker_factory, num_workers, self._port.worker_startup_delay_secs(), self._port.host) as pool:
pool.run(('test_list', shard.name, shard.test_inputs) for shard in all_shards)
except TestRunInterruptedException, e:
_log.warning(e.reason)
run_results.interrupted = True
def run_tests(self, expectations, test_inputs, tests_to_skip, num_workers, needs_http, needs_websockets, retrying):
self._expectations = expectations
self._test_inputs = test_inputs
self._needs_http = needs_http
self._needs_websockets = needs_websockets
self._retrying = retrying
# FIXME: rename all variables to test_run_results or some such ...
run_results = TestRunResults(self._expectations, len(test_inputs) + len(tests_to_skip))
self._current_run_results = run_results
self._remaining_locked_shards = []
self._has_http_lock = False
self._printer.num_tests = len(test_inputs)
self._printer.num_started = 0
if not retrying:
self._printer.print_expected(run_results, self._expectations.model().get_tests_with_result_type)
for test_name in set(tests_to_skip):
result = test_results.TestResult(test_name)
result.type = test_expectations.SKIP
run_results.add(result, expected=True, test_is_slow=self._test_is_slow(test_name))
self._printer.write_update('Sharding tests ...')
locked_shards, unlocked_shards = self._sharder.shard_tests(test_inputs, int(self._options.child_processes), self._options.fully_parallel)
# FIXME: We don't have a good way to coordinate the workers so that
# they don't try to run the shards that need a lock if we don't actually
# have the lock. The easiest solution at the moment is to grab the
# lock at the beginning of the run, and then run all of the locked
# shards first. This minimizes the time spent holding the lock, but
# means that we won't be running tests while we're waiting for the lock.
# If this becomes a problem in practice we'll need to change this.
all_shards = locked_shards + unlocked_shards
self._remaining_locked_shards = locked_shards
if locked_shards and self._options.http:
self.start_servers_with_lock(2 * min(num_workers, len(locked_shards)))
num_workers = min(num_workers, len(all_shards))
self._printer.print_workers_and_shards(num_workers, len(all_shards), len(locked_shards))
if self._options.dry_run:
return run_results
self._printer.write_update('Starting %s ...' % grammar.pluralize(num_workers, "worker"))
try:
with message_pool.get(self, self._worker_factory, num_workers, self._port.worker_startup_delay_secs(), self._port.host) as pool:
pool.run(('test_list', shard.name, shard.test_inputs) for shard in all_shards)
except TestRunInterruptedException, e:
_log.warning(e.reason)
run_results.interrupted = True
def run(self, tests, num_workers):
if not tests:
return
self.printer.write_update('Sharding tests ...')
shards = Runner._shard_tests(tests)
original_level = server_process_logger.level
server_process_logger.setLevel(logging.CRITICAL)
try:
self._num_workers = min(num_workers, len(shards))
with message_pool.get(self, lambda caller: _Worker(caller, self.port, shards), self._num_workers) as pool:
pool.run(('test', shard) for shard, _ in shards.iteritems())
finally:
server_process_logger.setLevel(original_level)
def run(self, test_names, num_workers):
if not test_names:
return
num_workers = min(num_workers, len(test_names))
with message_pool.get(self, self.worker_factory, num_workers) as pool:
pool.run(('test', test_name) for test_name in test_names)
def run_tests(self, test_inputs, expectations, result_summary, num_workers,
needs_http, needs_websockets, retrying):
"""Returns a tuple of (interrupted, keyboard_interrupted, thread_timings, test_timings, individual_test_timings):
interrupted is whether the run was interrupted
keyboard_interrupted is whether the interruption was because someone typed Ctrl^C
thread_timings is a list of dicts with the total runtime
of each thread with 'name', 'num_tests', 'total_time' properties
test_timings is a list of timings for each sharded subdirectory
of the form [time, directory_name, num_tests]
individual_test_timings is a list of run times for each test
in the form {filename:filename, test_run_time:test_run_time}
result_summary: summary object to populate with the results
"""
self._current_result_summary = result_summary
self._expectations = expectations
self._needs_http = needs_http
self._needs_websockets = needs_websockets
self._retrying = retrying
self._test_files_list = [
test_input.test_name for test_input in test_inputs
]
self._printer.num_tests = len(self._test_files_list)
self._printer.num_completed = 0
self._all_results = []
self._group_stats = {}
self._worker_stats = {}
self._has_http_lock = False
self._remaining_locked_shards = []
keyboard_interrupted = False
interrupted = False
self._printer.write_update('Sharding tests ...')
locked_shards, unlocked_shards = self._sharder.shard_tests(
test_inputs, int(self._options.child_processes),
self._options.fully_parallel)
# FIXME: We don't have a good way to coordinate the workers so that
# they don't try to run the shards that need a lock if we don't actually
# have the lock. The easiest solution at the moment is to grab the
# lock at the beginning of the run, and then run all of the locked
# shards first. This minimizes the time spent holding the lock, but
# means that we won't be running tests while we're waiting for the lock.
# If this becomes a problem in practice we'll need to change this.
all_shards = locked_shards + unlocked_shards
self._remaining_locked_shards = locked_shards
if self._port.requires_http_server() or (locked_shards
and self._options.http):
self.start_servers_with_lock(2 *
min(num_workers, len(locked_shards)))
num_workers = min(num_workers, len(all_shards))
self._printer.print_workers_and_shards(num_workers, len(all_shards),
len(locked_shards))
if self._options.dry_run:
return (keyboard_interrupted, interrupted,
self._worker_stats.values(), self._group_stats,
self._all_results)
self._printer.write_update('Starting %s ...' %
grammar.pluralize('worker', num_workers))
try:
with message_pool.get(self, self._worker_factory, num_workers,
self._port.worker_startup_delay_secs(),
self._port.host) as pool:
pool.run(('test_list', shard.name, shard.test_inputs)
for shard in all_shards)
except KeyboardInterrupt:
self._printer.flush()
self._printer.writeln('Interrupted, exiting ...')
keyboard_interrupted = True
except TestRunInterruptedException, e:
_log.warning(e.reason)
interrupted = True