def test_add_result_collection_ignore(self): """Tests add_result_collection overwrite.""" collection = ResultCollection(test_results=[FAILED_RESULT]) self.assertFalse(collection.crashed) crashed_collection = ResultCollection(test_results=[PASSED_RESULT], crashed=True) crashed_collection.append_crash_message('Crash2') collection.add_result_collection(crashed_collection, ignore_crash=True) self.assertFalse(collection.crashed) self.assertEqual(collection.crash_message, '') self.assertEqual(collection.test_results, [FAILED_RESULT, PASSED_RESULT])
def test_add_result_collection_default(self): """Tests add_result_collection default (merge crash info).""" collection = ResultCollection(test_results=[FAILED_RESULT]) self.assertFalse(collection.crashed) collection.append_crash_message('Crash1') crashed_collection = ResultCollection(test_results=[PASSED_RESULT], crashed=True) crashed_collection.append_crash_message('Crash2') collection.add_result_collection(crashed_collection) self.assertTrue(collection.crashed) self.assertEqual(collection.crash_message, 'Crash1\nCrash2') self.assertEqual(collection.test_results, [FAILED_RESULT, PASSED_RESULT])
def collect_test_results(output_folder, output): """Gets XCtest result data from Info.plist and copies artifacts. Args: output_folder: (str) A path to output folder. output: [str] An output of test run. Returns: test_result.ResultCollection representing all test results. """ output_folder = _sanitize_str(output_folder) output = _sanitize_str_list(output) overall_collected_result = ResultCollection() plist_path = os.path.join(output_folder, 'Info.plist') if not os.path.exists(plist_path): overall_collected_result.crashed = True overall_collected_result.crash_message += ( '%s with test results does not exist.\n' % plist_path + '\n'.join(output)) overall_collected_result.add_result_collection( parse_passed_failed_tests_for_interrupted_run(output)) return overall_collected_result root = plistlib.readPlist(plist_path) for action in root['Actions']: action_result = action['ActionResult'] if ((root['TestsCount'] == 0 and root['TestsFailedCount'] == 0) or 'TestSummaryPath' not in action_result): overall_collected_result.crashed = True if ('ErrorSummaries' in action_result and action_result['ErrorSummaries']): overall_collected_result.crash_message = '\n'.join( _sanitize_str_list([ error_summary['Message'] for error_summary in action_result['ErrorSummaries'] ])) else: summary_plist = os.path.join(os.path.dirname(plist_path), action_result['TestSummaryPath']) overall_collected_result.add_result_collection( XcodeLogParser._test_status_summary(summary_plist)) XcodeLogParser._copy_screenshots(output_folder) return overall_collected_result
def collect_test_results(output_path, output): """Gets XCTest results, diagnostic data & artifacts from xcresult. Args: output_path: (str) An output path passed in --resultBundlePath when running xcodebuild. output: [str] An output of test run. Returns: test_result.ResultCollection: Test results. """ output_path = _sanitize_str(output_path) output = _sanitize_str_list(output) LOGGER.info('Reading %s' % output_path) overall_collected_result = ResultCollection() # Xcodebuild writes staging data to |output_path| folder during test # execution. If |output_path| doesn't exist, it means tests didn't start at # all. if not os.path.exists(output_path): overall_collected_result.crashed = True overall_collected_result.crash_message = ( '%s with staging data does not exist.\n' % output_path + '\n'.join(output)) return overall_collected_result # During a run `xcodebuild .. -resultBundlePath %output_path%` # that generates output_path folder, # but Xcode 11+ generates `output_path.xcresult` and `output_path` # where output_path.xcresult is a folder with results and `output_path` # is symlink to the `output_path.xcresult` folder. # `xcresulttool` with folder/symlink behaves in different way on laptop and # on bots. This piece of code uses .xcresult folder. xcresult = output_path + _XCRESULT_SUFFIX # |output_path|.xcresult folder is created at the end of tests. If # |output_path| folder exists but |output_path|.xcresult folder doesn't # exist, it means xcodebuild exited or was killed half way during tests. if not os.path.exists(xcresult): overall_collected_result.crashed = True overall_collected_result.crash_message = ( '%s with test results does not exist.\n' % xcresult + '\n'.join(output)) overall_collected_result.add_result_collection( parse_passed_failed_tests_for_interrupted_run(output)) return overall_collected_result # See XCRESULT_ROOT in xcode_log_parser_test.py for an example of |root|. root = json.loads(Xcode11LogParser._xcresulttool_get(xcresult)) metrics = root['metrics'] # In case of test crash both numbers of run and failed tests are equal to 0. if (metrics.get('testsCount', {}).get('_value', 0) == 0 and metrics.get('testsFailedCount', {}).get('_value', 0) == 0): overall_collected_result.crashed = True overall_collected_result.crash_message = '0 tests executed!' else: overall_collected_result.add_result_collection( Xcode11LogParser._get_test_statuses(xcresult)) # For some crashed tests info about error contained only in root node. overall_collected_result.add_result_collection( Xcode11LogParser._list_of_failed_tests( root, excluded=overall_collected_result.all_test_names())) Xcode11LogParser.export_diagnostic_data(output_path) # Remove the symbol link file. if os.path.islink(output_path): os.unlink(output_path) file_util.zip_and_remove_folder(xcresult) return overall_collected_result
def launch(self): """Launches tests using xcodebuild.""" overall_launch_command_result = ResultCollection() shards = self.shards running_tests = set(self.egtests_app.get_all_tests()) # total number of attempts is self.retries+1 for attempt in range(self.retries + 1): # Erase all simulators per each attempt if iossim_util.is_device_with_udid_simulator(self.udid): # kill all running simulators to prevent possible memory leaks test_runner.SimulatorTestRunner.kill_simulators() shutdown_all_simulators() shutdown_all_simulators(XTDEVICE_FOLDER) erase_all_simulators() erase_all_simulators(XTDEVICE_FOLDER) outdir_attempt = os.path.join(self.out_dir, 'attempt_%d' % attempt) cmd_list = self.egtests_app.command(outdir_attempt, 'id=%s' % self.udid, shards) # TODO(crbug.com/914878): add heartbeat logging to xcodebuild_runner. LOGGER.info('Start test attempt #%d for command [%s]' % (attempt, ' '.join(cmd_list))) output = self.launch_attempt(cmd_list) if hasattr(self, 'use_clang_coverage') and self.use_clang_coverage: # out_dir of LaunchCommand object is the TestRunner out_dir joined with # UDID. Use os.path.dirname to retrieve the TestRunner out_dir. file_util.move_raw_coverage_data(self.udid, os.path.dirname(self.out_dir)) result = self._log_parser.collect_test_results( outdir_attempt, output) tests_selected_at_runtime = _tests_decided_at_runtime( self.egtests_app.test_app_path) # For most suites, only keep crash status from last attempt since retries # will cover any missing tests. For these decided at runtime, retain # crashes from all attempts and a dummy "crashed" result will be reported # to indicate some tests might never ran. # TODO(crbug.com/1235871): Switch back to excluded tests and set # |overall_crash| to always True. overall_launch_command_result.add_result_collection( result, overwrite_crash=not tests_selected_at_runtime) result.report_to_result_sink() tests_to_include = set() # |running_tests| are compiled tests in target intersecting with swarming # sharding. For some suites, they are more than what's needed to run. if not tests_selected_at_runtime: tests_to_include = tests_to_include | ( running_tests - overall_launch_command_result.expected_tests()) # Add failed tests from last rounds for runtime decided suites and device # suites. tests_to_include = ( tests_to_include | overall_launch_command_result.never_expected_tests()) self.egtests_app.included_tests = list(tests_to_include) # Nothing to run in retry. if not self.egtests_app.included_tests: break # If tests are not completed(interrupted or did not start) and there are # >= 20 remaining tests, run them with the same number of shards. # otherwise re-run with shards=1. if (not result.crashed # If need to re-run less than 20 tests, 1 shard should be enough. or (len(running_tests) - len(overall_launch_command_result.expected_tests()) <= MAXIMUM_TESTS_PER_SHARD_FOR_RERUN)): shards = 1 return overall_launch_command_result
def launch(self): """Launches the test app.""" self.set_up() # The overall ResultCorrection object holding all runs of all tests in the # runner run. It will be updated with each test application launch. overall_result = ResultCollection() destination = 'id=%s' % self.udid test_app = self.get_launch_test_app() out_dir = os.path.join(self.out_dir, 'TestResults') cmd = self.get_launch_command(test_app, out_dir, destination, self.shards) try: result = self._run(cmd=cmd, shards=self.shards or 1) if result.crashed and not result.crashed_tests(): # If the app crashed but not during any particular test case, assume # it crashed on startup. Try one more time. self.shutdown_and_restart() LOGGER.warning('Crashed on startup, retrying...\n') out_dir = os.path.join(self.out_dir, 'retry_after_crash_on_startup') cmd = self.get_launch_command(test_app, out_dir, destination, self.shards) result = self._run(cmd) result.report_to_result_sink() if result.crashed and not result.crashed_tests(): raise AppLaunchError overall_result.add_result_collection(result) try: while result.crashed and result.crashed_tests(): # If the app crashes during a specific test case, then resume at the # next test case. This is achieved by filtering out every test case # which has already run. LOGGER.warning('Crashed during %s, resuming...\n', list(result.crashed_tests())) test_app.excluded_tests = list( overall_result.all_test_names()) retry_out_dir = os.path.join( self.out_dir, 'retry_after_crash_%d' % int(time.time())) result = self._run( self.get_launch_command( test_app, os.path.join(retry_out_dir, str(int(time.time()))), destination)) result.report_to_result_sink() # Only keep the last crash status in crash retries in overall crash # status. overall_result.add_result_collection(result, overwrite_crash=True) except OSError as e: if e.errno == errno.E2BIG: LOGGER.error('Too many test cases to resume.') else: raise # Retry failed test cases. test_app.excluded_tests = [] never_expected_tests = overall_result.never_expected_tests() if self.retries and never_expected_tests: LOGGER.warning('%s tests failed and will be retried.\n', len(never_expected_tests)) for i in xrange(self.retries): tests_to_retry = list( overall_result.never_expected_tests()) for test in tests_to_retry: LOGGER.info('Retry #%s for %s.\n', i + 1, test) test_app.included_tests = [test] retry_out_dir = os.path.join(self.out_dir, test + '_failed', 'retry_%d' % i) retry_result = self._run( self.get_launch_command(test_app, retry_out_dir, destination)) if not retry_result.all_test_names(): retry_result.add_test_result( TestResult( test, TestStatus.SKIP, test_log= 'In single test retry, result of this test ' 'didn\'t appear in log.')) retry_result.report_to_result_sink() # No unknown tests might be skipped so do not change # |overall_result|'s crash status. overall_result.add_result_collection(retry_result, ignore_crash=True) interrupted = overall_result.crashed if interrupted: overall_result.add_and_report_crash( crash_message_prefix_line= 'Test application crashed when running ' 'tests which might have caused some tests never ran or finished.' ) self.test_results = overall_result.standard_json_output() self.logs.update(overall_result.test_runner_logs()) return not overall_result.never_expected_tests( ) and not interrupted finally: self.tear_down()
def _run(self, cmd, shards=1): """Runs the specified command, parsing GTest output. Args: cmd: List of strings forming the command to run. NOTE: in the case of WprProxySimulatorTestRunner, cmd is a dict forming the configuration for the test (including filter rules), and not indicative of the actual command we build and execute in _run. Returns: TestResult.ResultCollection() object. Raises: ShardingDisabledError: If shards > 1 as currently sharding is not supported. SystemAlertPresentError: If system alert is shown on the device. """ overall_result = ResultCollection() if shards > 1: # TODO(crbug.com/881096): reimplement sharding in the future raise test_runner.ShardingDisabledError() # TODO(crbug.com/812705): Implement test sharding for unit tests. # TODO(crbug.com/812712): Use thread pool for DeviceTestRunner as well. # Create a simulator for these tests, and prepare it with the # certificate needed for HTTPS proxying. udid = self.getSimulator() self.copy_trusted_certificate() for recipe_path in glob.glob('{}/*.test'.format(self.replay_path)): base_name = os.path.basename(recipe_path) test_name = os.path.splitext(base_name)[0] replay_path = '{}/{}'.format(self.replay_path, test_name) if self.should_run_wpr_test(test_name, cmd['test_filter'], cmd['invert']): parser, returncode = self.run_wpr_test(udid, test_name, recipe_path, replay_path) recipe_result = parser.GetResultCollection() # If this test fails, immediately rerun it to see if it deflakes. # We simply overwrite the first result with the second. if recipe_result.never_expected_tests(): parser, returncode = self.run_wpr_test( udid, test_name, recipe_path, replay_path) recipe_result = parser.GetResultCollection() # All test names will be the same since we re-run the same suite; # therefore, to differentiate the results, we append the recipe # name to the test suite. recipe_result.add_name_prefix_to_tests(base_name + '.') overall_result.add_result_collection(recipe_result) # Check for runtime errors. if self.xctest_path and parser.SystemAlertPresent(): raise test_runner.SystemAlertPresentError() LOGGER.info('%s test returned %s\n', recipe_path, returncode) self.deleteSimulator(udid) return overall_result