def collect_test_results(output_path, output): """Gets XCTest results, diagnostic data & artifacts from xcresult. Args: output_path: (str) An output path passed in --resultBundlePath when running xcodebuild. output: [str] An output of test run. Returns: test_result.ResultCollection: Test results. """ output_path = _sanitize_str(output_path) output = _sanitize_str_list(output) LOGGER.info('Reading %s' % output_path) overall_collected_result = ResultCollection() # Xcodebuild writes staging data to |output_path| folder during test # execution. If |output_path| doesn't exist, it means tests didn't start at # all. if not os.path.exists(output_path): overall_collected_result.crashed = True overall_collected_result.crash_message = ( '%s with staging data does not exist.\n' % output_path + '\n'.join(output)) return overall_collected_result # During a run `xcodebuild .. -resultBundlePath %output_path%` # that generates output_path folder, # but Xcode 11+ generates `output_path.xcresult` and `output_path` # where output_path.xcresult is a folder with results and `output_path` # is symlink to the `output_path.xcresult` folder. # `xcresulttool` with folder/symlink behaves in different way on laptop and # on bots. This piece of code uses .xcresult folder. xcresult = output_path + _XCRESULT_SUFFIX # |output_path|.xcresult folder is created at the end of tests. If # |output_path| folder exists but |output_path|.xcresult folder doesn't # exist, it means xcodebuild exited or was killed half way during tests. if not os.path.exists(xcresult): overall_collected_result.crashed = True overall_collected_result.crash_message = ( '%s with test results does not exist.\n' % xcresult + '\n'.join(output)) overall_collected_result.add_result_collection( parse_passed_failed_tests_for_interrupted_run(output)) return overall_collected_result # See XCRESULT_ROOT in xcode_log_parser_test.py for an example of |root|. root = json.loads(Xcode11LogParser._xcresulttool_get(xcresult)) metrics = root['metrics'] # In case of test crash both numbers of run and failed tests are equal to 0. if (metrics.get('testsCount', {}).get('_value', 0) == 0 and metrics.get('testsFailedCount', {}).get('_value', 0) == 0): overall_collected_result.crashed = True overall_collected_result.crash_message = '0 tests executed!' else: overall_collected_result.add_result_collection( Xcode11LogParser._get_test_statuses(xcresult)) # For some crashed tests info about error contained only in root node. overall_collected_result.add_result_collection( Xcode11LogParser._list_of_failed_tests( root, excluded=overall_collected_result.all_test_names())) Xcode11LogParser.export_diagnostic_data(output_path) # Remove the symbol link file. if os.path.islink(output_path): os.unlink(output_path) file_util.zip_and_remove_folder(xcresult) return overall_collected_result
def launch(self): """Launches the test app.""" self.set_up() # The overall ResultCorrection object holding all runs of all tests in the # runner run. It will be updated with each test application launch. overall_result = ResultCollection() destination = 'id=%s' % self.udid test_app = self.get_launch_test_app() out_dir = os.path.join(self.out_dir, 'TestResults') cmd = self.get_launch_command(test_app, out_dir, destination, self.shards) try: result = self._run(cmd=cmd, shards=self.shards or 1) if result.crashed and not result.crashed_tests(): # If the app crashed but not during any particular test case, assume # it crashed on startup. Try one more time. self.shutdown_and_restart() LOGGER.warning('Crashed on startup, retrying...\n') out_dir = os.path.join(self.out_dir, 'retry_after_crash_on_startup') cmd = self.get_launch_command(test_app, out_dir, destination, self.shards) result = self._run(cmd) result.report_to_result_sink() if result.crashed and not result.crashed_tests(): raise AppLaunchError overall_result.add_result_collection(result) try: while result.crashed and result.crashed_tests(): # If the app crashes during a specific test case, then resume at the # next test case. This is achieved by filtering out every test case # which has already run. LOGGER.warning('Crashed during %s, resuming...\n', list(result.crashed_tests())) test_app.excluded_tests = list( overall_result.all_test_names()) retry_out_dir = os.path.join( self.out_dir, 'retry_after_crash_%d' % int(time.time())) result = self._run( self.get_launch_command( test_app, os.path.join(retry_out_dir, str(int(time.time()))), destination)) result.report_to_result_sink() # Only keep the last crash status in crash retries in overall crash # status. overall_result.add_result_collection(result, overwrite_crash=True) except OSError as e: if e.errno == errno.E2BIG: LOGGER.error('Too many test cases to resume.') else: raise # Retry failed test cases. test_app.excluded_tests = [] never_expected_tests = overall_result.never_expected_tests() if self.retries and never_expected_tests: LOGGER.warning('%s tests failed and will be retried.\n', len(never_expected_tests)) for i in xrange(self.retries): tests_to_retry = list( overall_result.never_expected_tests()) for test in tests_to_retry: LOGGER.info('Retry #%s for %s.\n', i + 1, test) test_app.included_tests = [test] retry_out_dir = os.path.join(self.out_dir, test + '_failed', 'retry_%d' % i) retry_result = self._run( self.get_launch_command(test_app, retry_out_dir, destination)) if not retry_result.all_test_names(): retry_result.add_test_result( TestResult( test, TestStatus.SKIP, test_log= 'In single test retry, result of this test ' 'didn\'t appear in log.')) retry_result.report_to_result_sink() # No unknown tests might be skipped so do not change # |overall_result|'s crash status. overall_result.add_result_collection(retry_result, ignore_crash=True) interrupted = overall_result.crashed if interrupted: overall_result.add_and_report_crash( crash_message_prefix_line= 'Test application crashed when running ' 'tests which might have caused some tests never ran or finished.' ) self.test_results = overall_result.standard_json_output() self.logs.update(overall_result.test_runner_logs()) return not overall_result.never_expected_tests( ) and not interrupted finally: self.tear_down()