def collect_test_results(output_folder, output): """Gets XCtest result data from Info.plist and copies artifacts. Args: output_folder: (str) A path to output folder. output: [str] An output of test run. Returns: test_result.ResultCollection representing all test results. """ output_folder = _sanitize_str(output_folder) output = _sanitize_str_list(output) overall_collected_result = ResultCollection() plist_path = os.path.join(output_folder, 'Info.plist') if not os.path.exists(plist_path): overall_collected_result.crashed = True overall_collected_result.crash_message += ( '%s with test results does not exist.\n' % plist_path + '\n'.join(output)) overall_collected_result.add_result_collection( parse_passed_failed_tests_for_interrupted_run(output)) return overall_collected_result root = plistlib.readPlist(plist_path) for action in root['Actions']: action_result = action['ActionResult'] if ((root['TestsCount'] == 0 and root['TestsFailedCount'] == 0) or 'TestSummaryPath' not in action_result): overall_collected_result.crashed = True if ('ErrorSummaries' in action_result and action_result['ErrorSummaries']): overall_collected_result.crash_message = '\n'.join( _sanitize_str_list([ error_summary['Message'] for error_summary in action_result['ErrorSummaries'] ])) else: summary_plist = os.path.join(os.path.dirname(plist_path), action_result['TestSummaryPath']) overall_collected_result.add_result_collection( XcodeLogParser._test_status_summary(summary_plist)) XcodeLogParser._copy_screenshots(output_folder) return overall_collected_result
def test_unexpected_skipped_not_reported(self, mock_result): """Unexpected skip not reported for these selecting tests at runtime.""" crashed_collection = ResultCollection( test_results=[TestResult('Class1/passedTest1', TestStatus.PASS)]) crashed_collection.crashed = True mock_result.return_value = crashed_collection tr = xcodebuild_runner.DeviceXcodeTestRunner(_FLAKY_EGTEST_APP_PATH, "fake-host-app-path", "fake-out-dir") self.assertFalse(tr.launch()) self.assertEqual(len(tr.test_results['tests']), 2) tests = tr.test_results['tests'] self.assertEqual(tests['BUILD_INTERRUPTED']['actual'], 'CRASH') self.assertEqual(tests['Class1/passedTest1']['actual'], 'PASS')
def test_launch_command_not_restart_crashed_attempt( self, mock_collect_results): """Crashed first attempt of runtime select test suite won't be retried.""" egtests = test_apps.EgtestsApp(_FLAKY_EGTEST_APP_PATH) crashed_collection = ResultCollection() crashed_collection.crashed = True mock_collect_results.return_value = crashed_collection launch_command = xcodebuild_runner.LaunchCommand(egtests, _DESTINATION, shards=1, retries=3) overall_result = launch_command.launch() self.assertEqual(len(overall_result.all_test_names()), 0) self.assertEqual(overall_result.expected_tests(), set([])) self.assertTrue(overall_result.crashed)
def test_unexpected_skipped_crash_reported(self, mock_result): """Tests launch method in DeviceXcodeTestRunner""" tr = xcodebuild_runner.DeviceXcodeTestRunner("fake-app-path", "fake-host-app-path", "fake-out-dir") crashed_collection = ResultCollection( test_results=[TestResult('Class1/passedTest1', TestStatus.PASS)]) crashed_collection.crashed = True mock_result.return_value = crashed_collection self.assertFalse(tr.launch()) self.assertEqual(len(tr.test_results['tests']), 3) tests = tr.test_results['tests'] self.assertEqual(tests['BUILD_INTERRUPTED']['actual'], 'CRASH') self.assertEqual(tests['Class1/passedTest1']['actual'], 'PASS') self.assertEqual(tests['Class1/passedTest2']['actual'], 'SKIP') self.assertEqual(tests['Class1/passedTest2']['expected'], 'PASS')
def testLaunchCommand_restartCrashed1stAttempt(self, mock_collect_results): egtests = test_apps.EgtestsApp(_EGTESTS_APP_PATH) crashed_collection = ResultCollection() crashed_collection.crashed = True mock_collect_results.side_effect = [ crashed_collection, ResultCollection(test_results=[ TestResult('Class1/passedTest1', TestStatus.PASS), TestResult('Class1/passedTest2', TestStatus.PASS) ]) ] launch_command = xcodebuild_runner.LaunchCommand(egtests, _DESTINATION, shards=1, retries=3) overall_result = launch_command.launch() self.assertFalse(overall_result.crashed) self.assertEqual(len(overall_result.all_test_names()), 2) self.assertEqual(overall_result.expected_tests(), set(['Class1/passedTest1', 'Class1/passedTest2']))
def collect_test_results(output_path, output): """Gets XCTest results, diagnostic data & artifacts from xcresult. Args: output_path: (str) An output path passed in --resultBundlePath when running xcodebuild. output: [str] An output of test run. Returns: test_result.ResultCollection: Test results. """ output_path = _sanitize_str(output_path) output = _sanitize_str_list(output) LOGGER.info('Reading %s' % output_path) overall_collected_result = ResultCollection() # Xcodebuild writes staging data to |output_path| folder during test # execution. If |output_path| doesn't exist, it means tests didn't start at # all. if not os.path.exists(output_path): overall_collected_result.crashed = True overall_collected_result.crash_message = ( '%s with staging data does not exist.\n' % output_path + '\n'.join(output)) return overall_collected_result # During a run `xcodebuild .. -resultBundlePath %output_path%` # that generates output_path folder, # but Xcode 11+ generates `output_path.xcresult` and `output_path` # where output_path.xcresult is a folder with results and `output_path` # is symlink to the `output_path.xcresult` folder. # `xcresulttool` with folder/symlink behaves in different way on laptop and # on bots. This piece of code uses .xcresult folder. xcresult = output_path + _XCRESULT_SUFFIX # |output_path|.xcresult folder is created at the end of tests. If # |output_path| folder exists but |output_path|.xcresult folder doesn't # exist, it means xcodebuild exited or was killed half way during tests. if not os.path.exists(xcresult): overall_collected_result.crashed = True overall_collected_result.crash_message = ( '%s with test results does not exist.\n' % xcresult + '\n'.join(output)) overall_collected_result.add_result_collection( parse_passed_failed_tests_for_interrupted_run(output)) return overall_collected_result # See XCRESULT_ROOT in xcode_log_parser_test.py for an example of |root|. root = json.loads(Xcode11LogParser._xcresulttool_get(xcresult)) metrics = root['metrics'] # In case of test crash both numbers of run and failed tests are equal to 0. if (metrics.get('testsCount', {}).get('_value', 0) == 0 and metrics.get('testsFailedCount', {}).get('_value', 0) == 0): overall_collected_result.crashed = True overall_collected_result.crash_message = '0 tests executed!' else: overall_collected_result.add_result_collection( Xcode11LogParser._get_test_statuses(xcresult)) # For some crashed tests info about error contained only in root node. overall_collected_result.add_result_collection( Xcode11LogParser._list_of_failed_tests( root, excluded=overall_collected_result.all_test_names())) Xcode11LogParser.export_diagnostic_data(output_path) # Remove the symbol link file. if os.path.islink(output_path): os.unlink(output_path) file_util.zip_and_remove_folder(xcresult) return overall_collected_result
def _get_test_statuses(xcresult): """Returns test results from xcresult. Also extracts and stores attachments for failed tests. Args: xcresult: (str) A path to xcresult. Returns: test_result.ResultCollection: Test results. """ result = ResultCollection() # See TESTS_REF in xcode_log_parser_test.py for an example of |root|. root = json.loads( Xcode11LogParser._xcresulttool_get(xcresult, 'testsRef')) for summary in root['summaries']['_values'][0]['testableSummaries'][ '_values']: if not summary['tests']: continue for test_suite in summary['tests']['_values'][0]['subtests'][ '_values'][0]['subtests']['_values']: if 'subtests' not in test_suite: # Sometimes(if crash occurs) `subtests` node does not upload. # It happens only for failed tests that and a list of failures # can be parsed from root. continue for test in test_suite['subtests']['_values']: test_name = _sanitize_str(test['identifier']['_value']) if any( test_name.endswith(suffix) for suffix in SYSTEM_ERROR_TEST_NAME_SUFFIXES): result.crashed = True result.crash_message += 'System error in %s: %s\n' % ( xcresult, test_name) continue # If a test case was executed multiple times, there will be multiple # |test| objects of it. Each |test| corresponds to an execution of the # test case. if test['testStatus']['_value'] == 'Success': result.add_test_result( TestResult(test_name, TestStatus.PASS)) else: # Parse data for failed test by its id. See SINGLE_TEST_SUMMARY_REF # in xcode_log_parser_test.py for an example of |summary_ref|. summary_ref = json.loads( Xcode11LogParser._xcresulttool_get( xcresult, test['summaryRef']['id']['_value'])) failure_message = 'Logs from "failureSummaries" in .xcresult:\n' # On rare occasions rootFailure doesn't have 'failureSummaries'. for failure in summary_ref.get('failureSummaries', {}).get('_values', []): file_name = _sanitize_str( failure.get('fileName', {}).get('_value', '')) line_number = _sanitize_str( failure.get('lineNumber', {}).get('_value', '')) failure_location = 'file: %s, line: %s' % ( file_name, line_number) failure_message += failure_location + '\n' failure_message += _sanitize_str( failure['message']['_value']) + '\n' attachments = Xcode11LogParser._extract_artifacts_for_test( test_name, summary_ref, xcresult) result.add_test_result( TestResult(test_name, TestStatus.FAIL, test_log=failure_message, attachments=attachments)) return result