def collect_test_results(output_path, output):
"""Gets XCTest results, diagnostic data & artifacts from xcresult.
Args:
output_path: (str) An output path passed in --resultBundlePath when
running xcodebuild.
output: [str] An output of test run.
Returns:
test_result.ResultCollection: Test results.
"""
output_path = _sanitize_str(output_path)
output = _sanitize_str_list(output)
LOGGER.info('Reading %s' % output_path)
overall_collected_result = ResultCollection()
# Xcodebuild writes staging data to |output_path| folder during test
# execution. If |output_path| doesn't exist, it means tests didn't start at
# all.
if not os.path.exists(output_path):
overall_collected_result.crashed = True
overall_collected_result.crash_message = (
'%s with staging data does not exist.\n' % output_path +
'\n'.join(output))
return overall_collected_result
# During a run `xcodebuild .. -resultBundlePath %output_path%`
# that generates output_path folder,
# but Xcode 11+ generates `output_path.xcresult` and `output_path`
# where output_path.xcresult is a folder with results and `output_path`
# is symlink to the `output_path.xcresult` folder.
# `xcresulttool` with folder/symlink behaves in different way on laptop and
# on bots. This piece of code uses .xcresult folder.
xcresult = output_path + _XCRESULT_SUFFIX
# |output_path|.xcresult folder is created at the end of tests. If
# |output_path| folder exists but |output_path|.xcresult folder doesn't
# exist, it means xcodebuild exited or was killed half way during tests.
if not os.path.exists(xcresult):
overall_collected_result.crashed = True
overall_collected_result.crash_message = (
'%s with test results does not exist.\n' % xcresult +
'\n'.join(output))
overall_collected_result.add_result_collection(
parse_passed_failed_tests_for_interrupted_run(output))
return overall_collected_result
# See XCRESULT_ROOT in xcode_log_parser_test.py for an example of |root|.
root = json.loads(Xcode11LogParser._xcresulttool_get(xcresult))
metrics = root['metrics']
# In case of test crash both numbers of run and failed tests are equal to 0.
if (metrics.get('testsCount', {}).get('_value', 0) == 0
and metrics.get('testsFailedCount', {}).get('_value', 0) == 0):
overall_collected_result.crashed = True
overall_collected_result.crash_message = '0 tests executed!'
else:
overall_collected_result.add_result_collection(
Xcode11LogParser._get_test_statuses(xcresult))
# For some crashed tests info about error contained only in root node.
overall_collected_result.add_result_collection(
Xcode11LogParser._list_of_failed_tests(
root, excluded=overall_collected_result.all_test_names()))
Xcode11LogParser.export_diagnostic_data(output_path)
# Remove the symbol link file.
if os.path.islink(output_path):
os.unlink(output_path)
file_util.zip_and_remove_folder(xcresult)
return overall_collected_result
def launch(self):
"""Launches the test app."""
self.set_up()
# The overall ResultCorrection object holding all runs of all tests in the
# runner run. It will be updated with each test application launch.
overall_result = ResultCollection()
destination = 'id=%s' % self.udid
test_app = self.get_launch_test_app()
out_dir = os.path.join(self.out_dir, 'TestResults')
cmd = self.get_launch_command(test_app, out_dir, destination,
self.shards)
try:
result = self._run(cmd=cmd, shards=self.shards or 1)
if result.crashed and not result.crashed_tests():
# If the app crashed but not during any particular test case, assume
# it crashed on startup. Try one more time.
self.shutdown_and_restart()
LOGGER.warning('Crashed on startup, retrying...\n')
out_dir = os.path.join(self.out_dir,
'retry_after_crash_on_startup')
cmd = self.get_launch_command(test_app, out_dir, destination,
self.shards)
result = self._run(cmd)
result.report_to_result_sink()
if result.crashed and not result.crashed_tests():
raise AppLaunchError
overall_result.add_result_collection(result)
try:
while result.crashed and result.crashed_tests():
# If the app crashes during a specific test case, then resume at the
# next test case. This is achieved by filtering out every test case
# which has already run.
LOGGER.warning('Crashed during %s, resuming...\n',
list(result.crashed_tests()))
test_app.excluded_tests = list(
overall_result.all_test_names())
retry_out_dir = os.path.join(
self.out_dir,
'retry_after_crash_%d' % int(time.time()))
result = self._run(
self.get_launch_command(
test_app,
os.path.join(retry_out_dir, str(int(time.time()))),
destination))
result.report_to_result_sink()
# Only keep the last crash status in crash retries in overall crash
# status.
overall_result.add_result_collection(result,
overwrite_crash=True)
except OSError as e:
if e.errno == errno.E2BIG:
LOGGER.error('Too many test cases to resume.')
else:
raise
# Retry failed test cases.
test_app.excluded_tests = []
never_expected_tests = overall_result.never_expected_tests()
if self.retries and never_expected_tests:
LOGGER.warning('%s tests failed and will be retried.\n',
len(never_expected_tests))
for i in xrange(self.retries):
tests_to_retry = list(
overall_result.never_expected_tests())
for test in tests_to_retry:
LOGGER.info('Retry #%s for %s.\n', i + 1, test)
test_app.included_tests = [test]
retry_out_dir = os.path.join(self.out_dir,
test + '_failed',
'retry_%d' % i)
retry_result = self._run(
self.get_launch_command(test_app, retry_out_dir,
destination))
if not retry_result.all_test_names():
retry_result.add_test_result(
TestResult(
test,
TestStatus.SKIP,
test_log=
'In single test retry, result of this test '
'didn\'t appear in log.'))
retry_result.report_to_result_sink()
# No unknown tests might be skipped so do not change
# |overall_result|'s crash status.
overall_result.add_result_collection(retry_result,
ignore_crash=True)
interrupted = overall_result.crashed
if interrupted:
overall_result.add_and_report_crash(
crash_message_prefix_line=
'Test application crashed when running '
'tests which might have caused some tests never ran or finished.'
)
self.test_results = overall_result.standard_json_output()
self.logs.update(overall_result.test_runner_logs())
return not overall_result.never_expected_tests(
) and not interrupted
finally:
self.tear_down()