def collect_test_results(output_folder, output): """Gets XCtest result data from Info.plist and copies artifacts. Args: output_folder: (str) A path to output folder. output: [str] An output of test run. Returns: test_result.ResultCollection representing all test results. """ output_folder = _sanitize_str(output_folder) output = _sanitize_str_list(output) overall_collected_result = ResultCollection() plist_path = os.path.join(output_folder, 'Info.plist') if not os.path.exists(plist_path): overall_collected_result.crashed = True overall_collected_result.crash_message += ( '%s with test results does not exist.\n' % plist_path + '\n'.join(output)) overall_collected_result.add_result_collection( parse_passed_failed_tests_for_interrupted_run(output)) return overall_collected_result root = plistlib.readPlist(plist_path) for action in root['Actions']: action_result = action['ActionResult'] if ((root['TestsCount'] == 0 and root['TestsFailedCount'] == 0) or 'TestSummaryPath' not in action_result): overall_collected_result.crashed = True if ('ErrorSummaries' in action_result and action_result['ErrorSummaries']): overall_collected_result.crash_message = '\n'.join( _sanitize_str_list([ error_summary['Message'] for error_summary in action_result['ErrorSummaries'] ])) else: summary_plist = os.path.join(os.path.dirname(plist_path), action_result['TestSummaryPath']) overall_collected_result.add_result_collection( XcodeLogParser._test_status_summary(summary_plist)) XcodeLogParser._copy_screenshots(output_folder) return overall_collected_result
def collect_test_results(output_path, output): """Gets XCTest results, diagnostic data & artifacts from xcresult. Args: output_path: (str) An output path passed in --resultBundlePath when running xcodebuild. output: [str] An output of test run. Returns: test_result.ResultCollection: Test results. """ output_path = _sanitize_str(output_path) output = _sanitize_str_list(output) LOGGER.info('Reading %s' % output_path) overall_collected_result = ResultCollection() # Xcodebuild writes staging data to |output_path| folder during test # execution. If |output_path| doesn't exist, it means tests didn't start at # all. if not os.path.exists(output_path): overall_collected_result.crashed = True overall_collected_result.crash_message = ( '%s with staging data does not exist.\n' % output_path + '\n'.join(output)) return overall_collected_result # During a run `xcodebuild .. -resultBundlePath %output_path%` # that generates output_path folder, # but Xcode 11+ generates `output_path.xcresult` and `output_path` # where output_path.xcresult is a folder with results and `output_path` # is symlink to the `output_path.xcresult` folder. # `xcresulttool` with folder/symlink behaves in different way on laptop and # on bots. This piece of code uses .xcresult folder. xcresult = output_path + _XCRESULT_SUFFIX # |output_path|.xcresult folder is created at the end of tests. If # |output_path| folder exists but |output_path|.xcresult folder doesn't # exist, it means xcodebuild exited or was killed half way during tests. if not os.path.exists(xcresult): overall_collected_result.crashed = True overall_collected_result.crash_message = ( '%s with test results does not exist.\n' % xcresult + '\n'.join(output)) overall_collected_result.add_result_collection( parse_passed_failed_tests_for_interrupted_run(output)) return overall_collected_result # See XCRESULT_ROOT in xcode_log_parser_test.py for an example of |root|. root = json.loads(Xcode11LogParser._xcresulttool_get(xcresult)) metrics = root['metrics'] # In case of test crash both numbers of run and failed tests are equal to 0. if (metrics.get('testsCount', {}).get('_value', 0) == 0 and metrics.get('testsFailedCount', {}).get('_value', 0) == 0): overall_collected_result.crashed = True overall_collected_result.crash_message = '0 tests executed!' else: overall_collected_result.add_result_collection( Xcode11LogParser._get_test_statuses(xcresult)) # For some crashed tests info about error contained only in root node. overall_collected_result.add_result_collection( Xcode11LogParser._list_of_failed_tests( root, excluded=overall_collected_result.all_test_names())) Xcode11LogParser.export_diagnostic_data(output_path) # Remove the symbol link file. if os.path.islink(output_path): os.unlink(output_path) file_util.zip_and_remove_folder(xcresult) return overall_collected_result