def collect_test_results(output_folder, output):
"""Gets XCtest result data from Info.plist and copies artifacts.
Args:
output_folder: (str) A path to output folder.
output: [str] An output of test run.
Returns:
test_result.ResultCollection representing all test results.
"""
output_folder = _sanitize_str(output_folder)
output = _sanitize_str_list(output)
overall_collected_result = ResultCollection()
plist_path = os.path.join(output_folder, 'Info.plist')
if not os.path.exists(plist_path):
overall_collected_result.crashed = True
overall_collected_result.crash_message += (
'%s with test results does not exist.\n' % plist_path +
'\n'.join(output))
overall_collected_result.add_result_collection(
parse_passed_failed_tests_for_interrupted_run(output))
return overall_collected_result
root = plistlib.readPlist(plist_path)
for action in root['Actions']:
action_result = action['ActionResult']
if ((root['TestsCount'] == 0 and root['TestsFailedCount'] == 0)
or 'TestSummaryPath' not in action_result):
overall_collected_result.crashed = True
if ('ErrorSummaries' in action_result
and action_result['ErrorSummaries']):
overall_collected_result.crash_message = '\n'.join(
_sanitize_str_list([
error_summary['Message'] for error_summary in
action_result['ErrorSummaries']
]))
else:
summary_plist = os.path.join(os.path.dirname(plist_path),
action_result['TestSummaryPath'])
overall_collected_result.add_result_collection(
XcodeLogParser._test_status_summary(summary_plist))
XcodeLogParser._copy_screenshots(output_folder)
return overall_collected_result
def test_unexpected_skipped_not_reported(self, mock_result):
"""Unexpected skip not reported for these selecting tests at runtime."""
crashed_collection = ResultCollection(
test_results=[TestResult('Class1/passedTest1', TestStatus.PASS)])
crashed_collection.crashed = True
mock_result.return_value = crashed_collection
tr = xcodebuild_runner.DeviceXcodeTestRunner(_FLAKY_EGTEST_APP_PATH,
"fake-host-app-path",
"fake-out-dir")
self.assertFalse(tr.launch())
self.assertEqual(len(tr.test_results['tests']), 2)
tests = tr.test_results['tests']
self.assertEqual(tests['BUILD_INTERRUPTED']['actual'], 'CRASH')
self.assertEqual(tests['Class1/passedTest1']['actual'], 'PASS')
def test_launch_command_not_restart_crashed_attempt(
self, mock_collect_results):
"""Crashed first attempt of runtime select test suite won't be retried."""
egtests = test_apps.EgtestsApp(_FLAKY_EGTEST_APP_PATH)
crashed_collection = ResultCollection()
crashed_collection.crashed = True
mock_collect_results.return_value = crashed_collection
launch_command = xcodebuild_runner.LaunchCommand(egtests,
_DESTINATION,
shards=1,
retries=3)
overall_result = launch_command.launch()
self.assertEqual(len(overall_result.all_test_names()), 0)
self.assertEqual(overall_result.expected_tests(), set([]))
self.assertTrue(overall_result.crashed)
def test_unexpected_skipped_crash_reported(self, mock_result):
"""Tests launch method in DeviceXcodeTestRunner"""
tr = xcodebuild_runner.DeviceXcodeTestRunner("fake-app-path",
"fake-host-app-path",
"fake-out-dir")
crashed_collection = ResultCollection(
test_results=[TestResult('Class1/passedTest1', TestStatus.PASS)])
crashed_collection.crashed = True
mock_result.return_value = crashed_collection
self.assertFalse(tr.launch())
self.assertEqual(len(tr.test_results['tests']), 3)
tests = tr.test_results['tests']
self.assertEqual(tests['BUILD_INTERRUPTED']['actual'], 'CRASH')
self.assertEqual(tests['Class1/passedTest1']['actual'], 'PASS')
self.assertEqual(tests['Class1/passedTest2']['actual'], 'SKIP')
self.assertEqual(tests['Class1/passedTest2']['expected'], 'PASS')
def testLaunchCommand_restartCrashed1stAttempt(self, mock_collect_results):
egtests = test_apps.EgtestsApp(_EGTESTS_APP_PATH)
crashed_collection = ResultCollection()
crashed_collection.crashed = True
mock_collect_results.side_effect = [
crashed_collection,
ResultCollection(test_results=[
TestResult('Class1/passedTest1', TestStatus.PASS),
TestResult('Class1/passedTest2', TestStatus.PASS)
])
]
launch_command = xcodebuild_runner.LaunchCommand(egtests,
_DESTINATION,
shards=1,
retries=3)
overall_result = launch_command.launch()
self.assertFalse(overall_result.crashed)
self.assertEqual(len(overall_result.all_test_names()), 2)
self.assertEqual(overall_result.expected_tests(),
set(['Class1/passedTest1', 'Class1/passedTest2']))
def collect_test_results(output_path, output):
"""Gets XCTest results, diagnostic data & artifacts from xcresult.
Args:
output_path: (str) An output path passed in --resultBundlePath when
running xcodebuild.
output: [str] An output of test run.
Returns:
test_result.ResultCollection: Test results.
"""
output_path = _sanitize_str(output_path)
output = _sanitize_str_list(output)
LOGGER.info('Reading %s' % output_path)
overall_collected_result = ResultCollection()
# Xcodebuild writes staging data to |output_path| folder during test
# execution. If |output_path| doesn't exist, it means tests didn't start at
# all.
if not os.path.exists(output_path):
overall_collected_result.crashed = True
overall_collected_result.crash_message = (
'%s with staging data does not exist.\n' % output_path +
'\n'.join(output))
return overall_collected_result
# During a run `xcodebuild .. -resultBundlePath %output_path%`
# that generates output_path folder,
# but Xcode 11+ generates `output_path.xcresult` and `output_path`
# where output_path.xcresult is a folder with results and `output_path`
# is symlink to the `output_path.xcresult` folder.
# `xcresulttool` with folder/symlink behaves in different way on laptop and
# on bots. This piece of code uses .xcresult folder.
xcresult = output_path + _XCRESULT_SUFFIX
# |output_path|.xcresult folder is created at the end of tests. If
# |output_path| folder exists but |output_path|.xcresult folder doesn't
# exist, it means xcodebuild exited or was killed half way during tests.
if not os.path.exists(xcresult):
overall_collected_result.crashed = True
overall_collected_result.crash_message = (
'%s with test results does not exist.\n' % xcresult +
'\n'.join(output))
overall_collected_result.add_result_collection(
parse_passed_failed_tests_for_interrupted_run(output))
return overall_collected_result
# See XCRESULT_ROOT in xcode_log_parser_test.py for an example of |root|.
root = json.loads(Xcode11LogParser._xcresulttool_get(xcresult))
metrics = root['metrics']
# In case of test crash both numbers of run and failed tests are equal to 0.
if (metrics.get('testsCount', {}).get('_value', 0) == 0
and metrics.get('testsFailedCount', {}).get('_value', 0) == 0):
overall_collected_result.crashed = True
overall_collected_result.crash_message = '0 tests executed!'
else:
overall_collected_result.add_result_collection(
Xcode11LogParser._get_test_statuses(xcresult))
# For some crashed tests info about error contained only in root node.
overall_collected_result.add_result_collection(
Xcode11LogParser._list_of_failed_tests(
root, excluded=overall_collected_result.all_test_names()))
Xcode11LogParser.export_diagnostic_data(output_path)
# Remove the symbol link file.
if os.path.islink(output_path):
os.unlink(output_path)
file_util.zip_and_remove_folder(xcresult)
return overall_collected_result
def _get_test_statuses(xcresult):
"""Returns test results from xcresult.
Also extracts and stores attachments for failed tests.
Args:
xcresult: (str) A path to xcresult.
Returns:
test_result.ResultCollection: Test results.
"""
result = ResultCollection()
# See TESTS_REF in xcode_log_parser_test.py for an example of |root|.
root = json.loads(
Xcode11LogParser._xcresulttool_get(xcresult, 'testsRef'))
for summary in root['summaries']['_values'][0]['testableSummaries'][
'_values']:
if not summary['tests']:
continue
for test_suite in summary['tests']['_values'][0]['subtests'][
'_values'][0]['subtests']['_values']:
if 'subtests' not in test_suite:
# Sometimes(if crash occurs) `subtests` node does not upload.
# It happens only for failed tests that and a list of failures
# can be parsed from root.
continue
for test in test_suite['subtests']['_values']:
test_name = _sanitize_str(test['identifier']['_value'])
if any(
test_name.endswith(suffix)
for suffix in SYSTEM_ERROR_TEST_NAME_SUFFIXES):
result.crashed = True
result.crash_message += 'System error in %s: %s\n' % (
xcresult, test_name)
continue
# If a test case was executed multiple times, there will be multiple
# |test| objects of it. Each |test| corresponds to an execution of the
# test case.
if test['testStatus']['_value'] == 'Success':
result.add_test_result(
TestResult(test_name, TestStatus.PASS))
else:
# Parse data for failed test by its id. See SINGLE_TEST_SUMMARY_REF
# in xcode_log_parser_test.py for an example of |summary_ref|.
summary_ref = json.loads(
Xcode11LogParser._xcresulttool_get(
xcresult, test['summaryRef']['id']['_value']))
failure_message = 'Logs from "failureSummaries" in .xcresult:\n'
# On rare occasions rootFailure doesn't have 'failureSummaries'.
for failure in summary_ref.get('failureSummaries',
{}).get('_values', []):
file_name = _sanitize_str(
failure.get('fileName', {}).get('_value', ''))
line_number = _sanitize_str(
failure.get('lineNumber',
{}).get('_value', ''))
failure_location = 'file: %s, line: %s' % (
file_name, line_number)
failure_message += failure_location + '\n'
failure_message += _sanitize_str(
failure['message']['_value']) + '\n'
attachments = Xcode11LogParser._extract_artifacts_for_test(
test_name, summary_ref, xcresult)
result.add_test_result(
TestResult(test_name,
TestStatus.FAIL,
test_log=failure_message,
attachments=attachments))
return result