def launch(self):
"""Launches tests using xcodebuild."""
overall_launch_command_result = ResultCollection()
shards = self.shards
running_tests = set(self.egtests_app.get_all_tests())
# total number of attempts is self.retries+1
for attempt in range(self.retries + 1):
# Erase all simulators per each attempt
if iossim_util.is_device_with_udid_simulator(self.udid):
# kill all running simulators to prevent possible memory leaks
test_runner.SimulatorTestRunner.kill_simulators()
shutdown_all_simulators()
shutdown_all_simulators(XTDEVICE_FOLDER)
erase_all_simulators()
erase_all_simulators(XTDEVICE_FOLDER)
outdir_attempt = os.path.join(self.out_dir, 'attempt_%d' % attempt)
cmd_list = self.egtests_app.command(outdir_attempt,
'id=%s' % self.udid, shards)
# TODO(crbug.com/914878): add heartbeat logging to xcodebuild_runner.
LOGGER.info('Start test attempt #%d for command [%s]' %
(attempt, ' '.join(cmd_list)))
output = self.launch_attempt(cmd_list)
if hasattr(self, 'use_clang_coverage') and self.use_clang_coverage:
# out_dir of LaunchCommand object is the TestRunner out_dir joined with
# UDID. Use os.path.dirname to retrieve the TestRunner out_dir.
file_util.move_raw_coverage_data(self.udid,
os.path.dirname(self.out_dir))
result = self._log_parser.collect_test_results(
outdir_attempt, output)
tests_selected_at_runtime = _tests_decided_at_runtime(
self.egtests_app.test_app_path)
# For most suites, only keep crash status from last attempt since retries
# will cover any missing tests. For these decided at runtime, retain
# crashes from all attempts and a dummy "crashed" result will be reported
# to indicate some tests might never ran.
# TODO(crbug.com/1235871): Switch back to excluded tests and set
# |overall_crash| to always True.
overall_launch_command_result.add_result_collection(
result, overwrite_crash=not tests_selected_at_runtime)
result.report_to_result_sink()
tests_to_include = set()
# |running_tests| are compiled tests in target intersecting with swarming
# sharding. For some suites, they are more than what's needed to run.
if not tests_selected_at_runtime:
tests_to_include = tests_to_include | (
running_tests -
overall_launch_command_result.expected_tests())
# Add failed tests from last rounds for runtime decided suites and device
# suites.
tests_to_include = (
tests_to_include
| overall_launch_command_result.never_expected_tests())
self.egtests_app.included_tests = list(tests_to_include)
# Nothing to run in retry.
if not self.egtests_app.included_tests:
break
# If tests are not completed(interrupted or did not start) and there are
# >= 20 remaining tests, run them with the same number of shards.
# otherwise re-run with shards=1.
if (not result.crashed
# If need to re-run less than 20 tests, 1 shard should be enough.
or (len(running_tests) -
len(overall_launch_command_result.expected_tests()) <=
MAXIMUM_TESTS_PER_SHARD_FOR_RERUN)):
shards = 1
return overall_launch_command_result
def launch(self):
"""Launches the test app."""
self.set_up()
# The overall ResultCorrection object holding all runs of all tests in the
# runner run. It will be updated with each test application launch.
overall_result = ResultCollection()
destination = 'id=%s' % self.udid
test_app = self.get_launch_test_app()
out_dir = os.path.join(self.out_dir, 'TestResults')
cmd = self.get_launch_command(test_app, out_dir, destination,
self.shards)
try:
result = self._run(cmd=cmd, shards=self.shards or 1)
if result.crashed and not result.crashed_tests():
# If the app crashed but not during any particular test case, assume
# it crashed on startup. Try one more time.
self.shutdown_and_restart()
LOGGER.warning('Crashed on startup, retrying...\n')
out_dir = os.path.join(self.out_dir,
'retry_after_crash_on_startup')
cmd = self.get_launch_command(test_app, out_dir, destination,
self.shards)
result = self._run(cmd)
result.report_to_result_sink()
if result.crashed and not result.crashed_tests():
raise AppLaunchError
overall_result.add_result_collection(result)
try:
while result.crashed and result.crashed_tests():
# If the app crashes during a specific test case, then resume at the
# next test case. This is achieved by filtering out every test case
# which has already run.
LOGGER.warning('Crashed during %s, resuming...\n',
list(result.crashed_tests()))
test_app.excluded_tests = list(
overall_result.all_test_names())
retry_out_dir = os.path.join(
self.out_dir,
'retry_after_crash_%d' % int(time.time()))
result = self._run(
self.get_launch_command(
test_app,
os.path.join(retry_out_dir, str(int(time.time()))),
destination))
result.report_to_result_sink()
# Only keep the last crash status in crash retries in overall crash
# status.
overall_result.add_result_collection(result,
overwrite_crash=True)
except OSError as e:
if e.errno == errno.E2BIG:
LOGGER.error('Too many test cases to resume.')
else:
raise
# Retry failed test cases.
test_app.excluded_tests = []
never_expected_tests = overall_result.never_expected_tests()
if self.retries and never_expected_tests:
LOGGER.warning('%s tests failed and will be retried.\n',
len(never_expected_tests))
for i in xrange(self.retries):
tests_to_retry = list(
overall_result.never_expected_tests())
for test in tests_to_retry:
LOGGER.info('Retry #%s for %s.\n', i + 1, test)
test_app.included_tests = [test]
retry_out_dir = os.path.join(self.out_dir,
test + '_failed',
'retry_%d' % i)
retry_result = self._run(
self.get_launch_command(test_app, retry_out_dir,
destination))
if not retry_result.all_test_names():
retry_result.add_test_result(
TestResult(
test,
TestStatus.SKIP,
test_log=
'In single test retry, result of this test '
'didn\'t appear in log.'))
retry_result.report_to_result_sink()
# No unknown tests might be skipped so do not change
# |overall_result|'s crash status.
overall_result.add_result_collection(retry_result,
ignore_crash=True)
interrupted = overall_result.crashed
if interrupted:
overall_result.add_and_report_crash(
crash_message_prefix_line=
'Test application crashed when running '
'tests which might have caused some tests never ran or finished.'
)
self.test_results = overall_result.standard_json_output()
self.logs.update(overall_result.test_runner_logs())
return not overall_result.never_expected_tests(
) and not interrupted
finally:
self.tear_down()