def process_xcresult_dir(self):
"""Copies artifacts & diagnostic logs, zips and removes .xcresult dir."""
# .xcresult dir only exists when using Xcode 11+ and running as XCTest.
if not xcode_util.using_xcode_11_or_higher() or not self.xctest:
LOGGER.info('Skip processing xcresult directory.')
xcresult_paths = []
# Warning: This piece of code assumes .xcresult folder is directly under
# self.out_dir. This is true for TestRunner subclasses in this file.
# xcresult folder path is whatever passed in -resultBundlePath to xcodebuild
# command appended with '.xcresult' suffix.
for filename in os.listdir(self.out_dir):
full_path = os.path.join(self.out_dir, filename)
if full_path.endswith('.xcresult') and os.path.isdir(full_path):
xcresult_paths.append(full_path)
log_parser = xcode_log_parser.get_parser()
for xcresult in xcresult_paths:
# This is what was passed in -resultBundlePath to xcodebuild command.
result_bundle_path = os.path.splitext(xcresult)[0]
log_parser.copy_artifacts(result_bundle_path)
log_parser.export_diagnostic_data(result_bundle_path)
# result_bundle_path is a symlink to xcresult directory.
if os.path.islink(result_bundle_path):
os.unlink(result_bundle_path)
file_util.zip_and_remove_folder(xcresult)
def export_diagnostic_data(output_path):
"""Exports diagnostic data from xcresult to xcresult_diagnostic.zip.
Since Xcode 11 format of result bundles changed, to get diagnostic data
need to run command below:
xcresulttool export --type directory --id DIAGNOSTICS_REF --output-path
./export_folder --path ./RB.xcresult
Args:
output_path: (str) An output path passed in --resultBundlePath when
running xcodebuild.
"""
xcresult = output_path + _XCRESULT_SUFFIX
if not os.path.exists(xcresult):
LOGGER.warn('%s does not exist.' % xcresult)
return
root = json.loads(Xcode11LogParser._xcresulttool_get(xcresult))
try:
diagnostics_ref = root['actions']['_values'][0]['actionResult'][
'diagnosticsRef']['id']['_value']
diagnostic_folder = '%s_diagnostic' % xcresult
Xcode11LogParser._export_data(xcresult, diagnostics_ref,
'directory', diagnostic_folder)
file_util.zip_and_remove_folder(diagnostic_folder)
except KeyError:
LOGGER.warn('Did not parse diagnosticsRef from %s!' % xcresult)
def export_diagnostic_data(output_path):
"""Exports diagnostic data from xcresult to xcresult_diagnostic.zip.
Since Xcode 11 format of result bundles changed, to get diagnostic data
need to run command below:
xcresulttool export --type directory --id DIAGNOSTICS_REF --output-path
./export_folder --path ./RB.xcresult
Args:
output_path: (str) An output path passed in --resultBundlePath when
running xcodebuild.
"""
xcresult = output_path + _XCRESULT_SUFFIX
if not os.path.exists(xcresult):
LOGGER.warn('%s does not exist.' % xcresult)
return
root = json.loads(Xcode11LogParser._xcresulttool_get(xcresult))
try:
diagnostics_ref = root['actions']['_values'][0]['actionResult'][
'diagnosticsRef']['id']['_value']
diagnostic_folder = '%s_diagnostic' % xcresult
Xcode11LogParser._export_data(xcresult, diagnostics_ref,
'directory', diagnostic_folder)
# Copy log files out of diagnostic_folder if any. Use |name_count| to
# generate an index for same name files produced from Xcode parallel
# testing.
name_count = {}
for root, dirs, files in os.walk(diagnostic_folder):
for filename in files:
if 'StandardOutputAndStandardError' in filename:
file_index = name_count.get(filename, 0)
output_filename = ('%s_simulator#%d_%s' %
(os.path.basename(output_path),
file_index, filename))
output_filepath = os.path.join(output_path, os.pardir,
output_filename)
shutil.copy(os.path.join(root, filename),
output_filepath)
name_count[filename] = name_count.get(filename, 0) + 1
file_util.zip_and_remove_folder(diagnostic_folder)
except KeyError:
LOGGER.warn('Did not parse diagnosticsRef from %s!' % xcresult)
def collect_test_results(output_path, output):
"""Gets XCTest results, diagnostic data & artifacts from xcresult.
Args:
output_path: (str) An output path passed in --resultBundlePath when
running xcodebuild.
output: [str] An output of test run.
Returns:
Test result as a map:
{
'passed': [passed_tests],
'failed': {
'failed_test': ['StackTrace']
}
}
"""
LOGGER.info('Reading %s' % output_path)
test_results = {'passed': [], 'failed': {}}
# Xcodebuild writes staging data to |output_path| folder during test
# execution. If |output_path| doesn't exist, it means tests didn't start at
# all.
if not os.path.exists(output_path):
test_results['failed']['TESTS_DID_NOT_START'] = [
'%s with staging data does not exist.' % output_path
]
return test_results
# During a run `xcodebuild .. -resultBundlePath %output_path%`
# that generates output_path folder,
# but Xcode 11+ generates `output_path.xcresult` and `output_path`
# where output_path.xcresult is a folder with results and `output_path`
# is symlink to the `output_path.xcresult` folder.
# `xcresulttool` with folder/symlink behaves in different way on laptop and
# on bots. This piece of code uses .xcresult folder.
xcresult = output_path + _XCRESULT_SUFFIX
# |output_path|.xcresult folder is created at the end of tests. If
# |output_path| folder exists but |output_path|.xcresult folder doesn't
# exist, it means xcodebuild exited or was killed half way during tests.
if not os.path.exists(xcresult):
test_results['failed']['BUILD_INTERRUPTED'] = [
'%s with test results does not exist.' % xcresult
] + output
passed_tests, failed_tests_dict = parse_passed_failed_tests_for_interrupted_run(
output)
test_results['passed'] = passed_tests
test_results['failed'].update(failed_tests_dict)
return test_results
# See XCRESULT_ROOT in xcode_log_parser_test.py for an example of |root|.
root = json.loads(Xcode11LogParser._xcresulttool_get(xcresult))
metrics = root['metrics']
# In case of test crash both numbers of run and failed tests are equal to 0.
if (metrics.get('testsCount', {}).get('_value', 0) == 0
and metrics.get('testsFailedCount', {}).get('_value', 0) == 0):
test_results['failed']['TESTS_DID_NOT_START'] = [
'0 tests executed!'
]
else:
# For some crashed tests info about error contained only in root node.
test_results['failed'] = Xcode11LogParser._list_of_failed_tests(
root)
Xcode11LogParser._get_test_statuses(xcresult, test_results)
Xcode11LogParser.export_diagnostic_data(output_path)
Xcode11LogParser.copy_artifacts(output_path)
# Remove the symbol link file.
if os.path.islink(output_path):
os.unlink(output_path)
file_util.zip_and_remove_folder(xcresult)
return test_results
def collect_test_results(output_path, output):
"""Gets XCTest results, diagnostic data & artifacts from xcresult.
Args:
output_path: (str) An output path passed in --resultBundlePath when
running xcodebuild.
output: [str] An output of test run.
Returns:
test_result.ResultCollection: Test results.
"""
output_path = _sanitize_str(output_path)
output = _sanitize_str_list(output)
LOGGER.info('Reading %s' % output_path)
overall_collected_result = ResultCollection()
# Xcodebuild writes staging data to |output_path| folder during test
# execution. If |output_path| doesn't exist, it means tests didn't start at
# all.
if not os.path.exists(output_path):
overall_collected_result.crashed = True
overall_collected_result.crash_message = (
'%s with staging data does not exist.\n' % output_path +
'\n'.join(output))
return overall_collected_result
# During a run `xcodebuild .. -resultBundlePath %output_path%`
# that generates output_path folder,
# but Xcode 11+ generates `output_path.xcresult` and `output_path`
# where output_path.xcresult is a folder with results and `output_path`
# is symlink to the `output_path.xcresult` folder.
# `xcresulttool` with folder/symlink behaves in different way on laptop and
# on bots. This piece of code uses .xcresult folder.
xcresult = output_path + _XCRESULT_SUFFIX
# |output_path|.xcresult folder is created at the end of tests. If
# |output_path| folder exists but |output_path|.xcresult folder doesn't
# exist, it means xcodebuild exited or was killed half way during tests.
if not os.path.exists(xcresult):
overall_collected_result.crashed = True
overall_collected_result.crash_message = (
'%s with test results does not exist.\n' % xcresult +
'\n'.join(output))
overall_collected_result.add_result_collection(
parse_passed_failed_tests_for_interrupted_run(output))
return overall_collected_result
# See XCRESULT_ROOT in xcode_log_parser_test.py for an example of |root|.
root = json.loads(Xcode11LogParser._xcresulttool_get(xcresult))
metrics = root['metrics']
# In case of test crash both numbers of run and failed tests are equal to 0.
if (metrics.get('testsCount', {}).get('_value', 0) == 0
and metrics.get('testsFailedCount', {}).get('_value', 0) == 0):
overall_collected_result.crashed = True
overall_collected_result.crash_message = '0 tests executed!'
else:
overall_collected_result.add_result_collection(
Xcode11LogParser._get_test_statuses(xcresult))
# For some crashed tests info about error contained only in root node.
overall_collected_result.add_result_collection(
Xcode11LogParser._list_of_failed_tests(
root, excluded=overall_collected_result.all_test_names()))
Xcode11LogParser.export_diagnostic_data(output_path)
# Remove the symbol link file.
if os.path.islink(output_path):
os.unlink(output_path)
file_util.zip_and_remove_folder(xcresult)
return overall_collected_result