def check_arguments_and_execute(self, options, args, tool=None):
if len(args) < len(self.required_arguments):
_log.error(
"%s required, %s provided. Provided: %s Required: %s\nSee '%s help %s' for usage.",
pluralize('argument', len(self.required_arguments)),
pluralize('argument', len(args)), "'%s'" % ' '.join(args),
' '.join(self.required_arguments), tool.name(), self.name)
return 1
return self.execute(options, args, tool) or 0
def print_summary(self, total_time, run_results):
if self._options.timing:
parallel_time = sum(
result.total_run_time
for result in run_results.results_by_name.values())
# There is serial overhead in layout_test_runner.run() that we can't easily account for when
# really running in parallel, but taking the min() ensures that in the worst case
# (if parallel time is less than run_time) we do account for it.
serial_time = total_time - min(run_results.run_time, parallel_time)
speedup = (parallel_time + serial_time) / total_time
timing_summary = ' in %.2fs (%.2fs in rwt, %.2gx)' % (
total_time, serial_time, speedup)
else:
timing_summary = ''
total = run_results.total - run_results.expected_skips
expected = run_results.expected - run_results.expected_skips
unexpected = run_results.unexpected
incomplete = total - expected - unexpected
incomplete_str = ''
if incomplete:
self._print_default('')
incomplete_str = " (%d didn't run)" % incomplete
if self._options.verbose or self._options.debug_rwt_logging or unexpected:
self.writeln('')
expected_summary_str = ''
if run_results.expected_failures > 0:
expected_summary_str = " (%d passed, %d didn't)" % (
expected - run_results.expected_failures,
run_results.expected_failures)
summary = ''
if unexpected == 0:
if expected == total:
if expected > 1:
summary = 'All %d tests ran as expected%s%s.' % (
expected, expected_summary_str, timing_summary)
else:
summary = 'The test ran as expected%s%s.' % (
expected_summary_str, timing_summary)
else:
summary = '%s ran as expected%s%s%s.' % (grammar.pluralize(
'test', expected), expected_summary_str, incomplete_str,
timing_summary)
self._print_quiet(summary)
else:
self._print_quiet(
"%s ran as expected%s, %d didn't%s%s:" %
(grammar.pluralize('test', expected), expected_summary_str,
unexpected, incomplete_str, timing_summary))
for test_name in sorted(run_results.unexpected_results_by_name):
self._print_quiet(' %s' % test_name)
def _run_test_once(self, tests_to_run, tests_to_skip, should_retry_failures):
num_workers = self._port.num_workers(int(self._options.child_processes))
initial_results = self._run_tests(
tests_to_run, tests_to_skip, self._options.repeat_each, self._options.iterations,
num_workers)
# Don't retry failures when interrupted by user or failures limit exception.
should_retry_failures = should_retry_failures and not (
initial_results.interrupted or initial_results.keyboard_interrupted)
tests_to_retry = self._tests_to_retry(initial_results)
all_retry_results = []
if should_retry_failures and tests_to_retry:
for retry_attempt in xrange(1, self._options.num_retries + 1):
if not tests_to_retry:
break
_log.info('')
_log.info('Retrying %s, attempt %d of %d...',
grammar.pluralize('unexpected failure', len(tests_to_retry)),
retry_attempt, self._options.num_retries)
retry_results = self._run_tests(tests_to_retry,
tests_to_skip=set(),
repeat_each=1,
iterations=1,
num_workers=num_workers,
retry_attempt=retry_attempt)
all_retry_results.append(retry_results)
tests_to_retry = self._tests_to_retry(retry_results)
return (initial_results, all_retry_results)
def print_found(self, num_all_test_files, num_shard_test_files, num_to_run, repeat_each, iterations):
found_str = 'Found %s' % grammar.pluralize('test', num_shard_test_files)
if num_all_test_files != num_shard_test_files:
found_str += ' (total %d)' % num_all_test_files
found_str += '; running %d' % num_to_run
if repeat_each * iterations > 1:
found_str += ' (%d times each: --repeat-each=%d --iterations=%d)' % (repeat_each * iterations, repeat_each, iterations)
found_str += ', skipping %d' % (num_shard_test_files - num_to_run)
self._print_default(found_str + '.')
def print_workers_and_shards(self, port, num_workers, num_shards, num_locked_shards):
driver_name = port.driver_name()
if num_workers == 1:
self._print_default('Running 1 %s.' % driver_name)
self._print_debug('(%s).' % grammar.pluralize('shard', num_shards))
else:
self._print_default('Running %d %ss in parallel.' % (num_workers, driver_name))
self._print_debug('(%d shards; %d locked).' % (num_shards, num_locked_shards))
self._print_default('')
def run_tests(self, expectations, test_inputs, tests_to_skip, num_workers,
retry_attempt):
batch_size = self._options.derived_batch_size
# If we're retrying a test, then it's because we think it might be flaky
# and rerunning it might provide a different result. We must restart
# content shell to get a valid result, as otherwise state can leak
# from previous tests. To do so, we set a batch size of 1, as that
# prevents content shell reuse.
if not self._options.must_use_derived_batch_size and retry_attempt >= 1:
batch_size = 1
self._expectations = expectations
self._test_inputs = test_inputs
test_run_results = TestRunResults(
self._expectations,
len(test_inputs) + len(tests_to_skip),
self._test_result_sink,
)
self._current_run_results = test_run_results
self._printer.num_tests = len(test_inputs)
self._printer.num_completed = 0
for test_name in set(tests_to_skip):
result = test_results.TestResult(test_name)
result.type = ResultType.Skip
test_run_results.add(
result,
expected=True,
test_is_slow=self._test_is_slow(test_name))
self._printer.write_update('Sharding tests ...')
locked_shards, unlocked_shards = self._sharder.shard_tests(
test_inputs, int(self._options.child_processes),
self._options.fully_parallel, self._options.virtual_parallel,
batch_size == 1)
self._reorder_tests_by_args(locked_shards)
self._reorder_tests_by_args(unlocked_shards)
# We don't have a good way to coordinate the workers so that they don't
# try to run the shards that need a lock. The easiest solution is to
# run all of the locked shards first.
all_shards = locked_shards + unlocked_shards
num_workers = min(num_workers, len(all_shards))
if retry_attempt < 1:
self._printer.print_workers_and_shards(self._port, num_workers,
len(all_shards),
len(locked_shards))
if self._options.dry_run:
return test_run_results
self._printer.write_update(
'Starting %s ...' % grammar.pluralize('worker', num_workers))
start_time = time.time()
try:
with message_pool.get(self, self._worker_factory, num_workers,
self._port.host) as pool:
pool.run(('test_list', shard.name, shard.test_inputs,
batch_size) for shard in all_shards)
if self._shards_to_redo:
num_workers -= len(self._shards_to_redo)
if num_workers > 0:
with message_pool.get(self, self._worker_factory,
num_workers,
self._port.host) as pool:
pool.run(('test_list', shard.name, shard.test_inputs,
batch_size)
for shard in self._shards_to_redo)
else:
self._mark_interrupted_tests_as_skipped(
self._current_run_results)
raise TestRunInterruptedException(
'All workers have device failures. Exiting.')
except TestRunInterruptedException as error:
_log.warning(error.reason)
test_run_results.interrupted = True
except KeyboardInterrupt:
self._printer.flush()
self._printer.writeln('Interrupted, exiting ...')
test_run_results.keyboard_interrupted = True
except Exception as error:
_log.debug('%s("%s") raised, exiting', error.__class__.__name__,
error)
raise
finally:
test_run_results.run_time = time.time() - start_time
return test_run_results
def run_tests(self, expectations, test_inputs, tests_to_skip, num_workers,
retry_attempt):
self._expectations = expectations
self._test_inputs = test_inputs
self._retry_attempt = retry_attempt
test_run_results = TestRunResults(
self._expectations,
len(test_inputs) + len(tests_to_skip))
self._current_run_results = test_run_results
self._printer.num_tests = len(test_inputs)
self._printer.num_completed = 0
if retry_attempt < 1:
self._printer.print_expected(
test_run_results,
self._expectations.get_tests_with_result_type)
for test_name in set(tests_to_skip):
result = test_results.TestResult(test_name)
result.type = test_expectations.SKIP
test_run_results.add(result,
expected=True,
test_is_slow=self._test_is_slow(test_name))
self._printer.write_update('Sharding tests ...')
locked_shards, unlocked_shards = self._sharder.shard_tests(
test_inputs, int(self._options.child_processes),
self._options.fully_parallel, self._options.batch_size == 1)
self._reorder_tests_by_args(locked_shards)
self._reorder_tests_by_args(unlocked_shards)
# We don't have a good way to coordinate the workers so that they don't
# try to run the shards that need a lock. The easiest solution is to
# run all of the locked shards first.
all_shards = locked_shards + unlocked_shards
num_workers = min(num_workers, len(all_shards))
if retry_attempt < 1:
self._printer.print_workers_and_shards(num_workers,
len(all_shards),
len(locked_shards))
if self._options.dry_run:
return test_run_results
self._printer.write_update('Starting %s ...' %
grammar.pluralize('worker', num_workers))
start_time = time.time()
try:
with message_pool.get(self, self._worker_factory, num_workers,
self._port.host) as pool:
pool.run(('test_list', shard.name, shard.test_inputs)
for shard in all_shards)
if self._shards_to_redo:
num_workers -= len(self._shards_to_redo)
if num_workers > 0:
with message_pool.get(self, self._worker_factory,
num_workers,
self._port.host) as pool:
pool.run(('test_list', shard.name, shard.test_inputs)
for shard in self._shards_to_redo)
except TestRunInterruptedException as error:
_log.warning(error.reason)
test_run_results.interrupted = True
except KeyboardInterrupt:
self._printer.flush()
self._printer.writeln('Interrupted, exiting ...')
test_run_results.keyboard_interrupted = True
except Exception as error:
_log.debug('%s("%s") raised, exiting', error.__class__.__name__,
error)
raise
finally:
test_run_results.run_time = time.time() - start_time
return test_run_results
def test_pluralize_two_ends_with_sh(self):
self.assertEqual('2 crashes', grammar.pluralize('crash', 2))
def test_pluralize_two(self):
self.assertEqual('2 tests', grammar.pluralize('test', 2))
def test_pluralize_one(self):
self.assertEqual('1 test', grammar.pluralize('test', 1))
def test_pluralize_zero(self):
self.assertEqual('0 tests', grammar.pluralize('test', 0))
