def test_launch(self): """Tests launch method in DeviceXcodeTestRunner""" self.mock(xcodebuild_runner.pool.ThreadPool, 'imap_unordered', lambda _1, _2, _3: []) self.mock(xcodebuild_runner, 'get_all_tests', lambda _1, _2: []) tr = xcodebuild_runner.DeviceXcodeTestRunner("fake-app-path", "fake-host-app-path", "fake-out-dir") self.assertTrue(tr.launch())
def test_launch(self, mock_result): """Tests launch method in DeviceXcodeTestRunner""" tr = xcodebuild_runner.DeviceXcodeTestRunner("fake-app-path", "fake-host-app-path", "fake-out-dir") mock_result.return_value = ResultCollection(test_results=[ TestResult('Class1/passedTest1', TestStatus.PASS), TestResult('Class1/passedTest2', TestStatus.PASS) ]) self.assertTrue(tr.launch()) self.assertEqual(len(tr.test_results['tests']), 2)
def test_unexpected_skipped_not_reported(self, mock_result): """Unexpected skip not reported for these selecting tests at runtime.""" crashed_collection = ResultCollection( test_results=[TestResult('Class1/passedTest1', TestStatus.PASS)]) crashed_collection.crashed = True mock_result.return_value = crashed_collection tr = xcodebuild_runner.DeviceXcodeTestRunner(_FLAKY_EGTEST_APP_PATH, "fake-host-app-path", "fake-out-dir") self.assertFalse(tr.launch()) self.assertEqual(len(tr.test_results['tests']), 2) tests = tr.test_results['tests'] self.assertEqual(tests['BUILD_INTERRUPTED']['actual'], 'CRASH') self.assertEqual(tests['Class1/passedTest1']['actual'], 'PASS')
def test_unexpected_skipped_crash_reported(self, mock_result): """Tests launch method in DeviceXcodeTestRunner""" tr = xcodebuild_runner.DeviceXcodeTestRunner("fake-app-path", "fake-host-app-path", "fake-out-dir") crashed_collection = ResultCollection( test_results=[TestResult('Class1/passedTest1', TestStatus.PASS)]) crashed_collection.crashed = True mock_result.return_value = crashed_collection self.assertFalse(tr.launch()) self.assertEqual(len(tr.test_results['tests']), 3) tests = tr.test_results['tests'] self.assertEqual(tests['BUILD_INTERRUPTED']['actual'], 'CRASH') self.assertEqual(tests['Class1/passedTest1']['actual'], 'PASS') self.assertEqual(tests['Class1/passedTest2']['actual'], 'SKIP') self.assertEqual(tests['Class1/passedTest2']['expected'], 'PASS')
def test_disabled_reported(self, mock_test_app, mock_result, _): """Tests launch method in DeviceXcodeTestRunner""" test_app = mock_test_app.return_value test_app.test_app_path = _EGTESTS_APP_PATH test_app.disabled_tests = ['Class2/disabled_test3'] test_app.get_all_tests.return_value = [ 'Class1/passedTest1', 'Class1/passedTest2' ] mock_result.return_value = ResultCollection(test_results=[ TestResult('Class1/passedTest1', TestStatus.PASS), TestResult('Class1/passedTest2', TestStatus.PASS) ]) tr = xcodebuild_runner.DeviceXcodeTestRunner("fake-app-path", "fake-host-app-path", "fake-out-dir") self.assertTrue(tr.launch()) self.assertEqual(len(tr.test_results['tests']), 3) tests = tr.test_results['tests'] self.assertEqual(tests['Class1/passedTest1']['actual'], 'PASS') self.assertEqual(tests['Class1/passedTest2']['actual'], 'PASS') self.assertEqual(tests['Class2/disabled_test3']['actual'], 'SKIP') self.assertEqual(tests['Class2/disabled_test3']['expected'], 'SKIP')
def test_tear_down(self): tr = xcodebuild_runner.DeviceXcodeTestRunner("fake-app-path", "fake-host-app-path", "fake-out-dir") tr.tear_down()
def run(self, args): """ Main coordinating function. """ self.parse_args(args) # This logic is run by default before the otool command is invoked such that # otool has the correct Xcode selected for command line dev tools. if not self.install_xcode(self.args.xcode_build_version, self.args.mac_toolchain_cmd, self.args.xcode_path): raise test_runner.XcodeVersionNotFoundError( self.args.xcode_build_version) # GTEST_SHARD_INDEX and GTEST_TOTAL_SHARDS are additional test environment # variables, set by Swarming, that are only set for a swarming task # shard count is > 1. # # For a given test on a given run, otool should return the same total # counts and thus, should generate the same sublists. With the shard index, # each shard would then know the exact test case to run. gtest_shard_index = os.getenv('GTEST_SHARD_INDEX', 0) gtest_total_shards = os.getenv('GTEST_TOTAL_SHARDS', 0) if gtest_shard_index and gtest_total_shards: self.args.test_cases = shard_util.shard_test_cases( self.args, gtest_shard_index, gtest_total_shards) summary = {} tr = None if not os.path.exists(self.args.out_dir): os.makedirs(self.args.out_dir) try: if self.args.xcode_parallelization: tr = xcodebuild_runner.SimulatorParallelTestRunner( self.args.app, self.args.host_app, self.args.iossim, self.args.version, self.args.platform, out_dir=self.args.out_dir, release=self.args.release, retries=self.args.retries, shards=self.args.shards, test_cases=self.args.test_cases, test_args=self.test_args, use_clang_coverage=self.args.use_clang_coverage, env_vars=self.args.env_var) elif self.args.replay_path != 'NO_PATH': tr = wpr_runner.WprProxySimulatorTestRunner( self.args.app, self.args.host_app, self.args.iossim, self.args.replay_path, self.args.platform, self.args.version, self.args.wpr_tools_path, self.args.out_dir, env_vars=self.args.env_var, retries=self.args.retries, shards=self.args.shards, test_args=self.test_args, test_cases=self.args.test_cases, xctest=self.args.xctest, ) elif self.args.iossim and self.args.platform and self.args.version: tr = test_runner.SimulatorTestRunner( self.args.app, self.args.iossim, self.args.platform, self.args.version, self.args.out_dir, env_vars=self.args.env_var, retries=self.args.retries, shards=self.args.shards, test_args=self.test_args, test_cases=self.args.test_cases, use_clang_coverage=self.args.use_clang_coverage, wpr_tools_path=self.args.wpr_tools_path, xctest=self.args.xctest, ) elif self.args.xcodebuild_device_runner and self.args.xctest: tr = xcodebuild_runner.DeviceXcodeTestRunner( app_path=self.args.app, host_app_path=self.args.host_app, out_dir=self.args.out_dir, release=self.args.release, retries=self.args.retries, test_cases=self.args.test_cases, test_args=self.test_args, env_vars=self.args.env_var) else: tr = test_runner.DeviceTestRunner( self.args.app, self.args.out_dir, env_vars=self.args.env_var, restart=self.args.restart, retries=self.args.retries, test_args=self.test_args, test_cases=self.args.test_cases, xctest=self.args.xctest, ) return 0 if tr.launch() else 1 except test_runner.TestRunnerError as e: sys.stderr.write(traceback.format_exc()) summary['step_text'] = '%s%s' % (e.__class__.__name__, ': %s' % e.args[0] if e.args else '') # test_runner.Launch returns 0 on success, 1 on failure, so return 2 # on exception to distinguish between a test failure, and a failure # to launch the test at all. return 2 finally: if tr: summary['logs'] = tr.logs with open(os.path.join(self.args.out_dir, 'summary.json'), 'w') as f: json.dump(summary, f) if tr: with open(os.path.join(self.args.out_dir, 'full_results.json'), 'w') as f: json.dump(tr.test_results, f) # The value of test-launcher-summary-output is set by the recipe # and passed here via swarming.py. This argument defaults to # ${ISOLATED_OUTDIR}/output.json. out-dir is set to ${ISOLATED_OUTDIR} # TODO(crbug.com/1031338) - the content of this output.json will # work with Chromium recipe because we use the noop_merge merge script, # but will require structural changes to support the default gtest # merge script (ref: //testing/merge_scripts/standard_gtest_merge.py) output_json_path = (self.args.test_launcher_summary_output or os.path.join(self.args.out_dir, 'output.json')) with open(output_json_path, 'w') as f: json.dump(tr.test_results, f) test_runner.defaults_delete('com.apple.CoreSimulator', 'FramebufferServerRendererPolicy')
def run(self, args): """ Main coordinating function. """ self.parse_args(args) # This logic is run by default before the otool command is invoked such that # otool has the correct Xcode selected for command line dev tools. install_success, is_legacy_xcode = self.install_xcode() if not install_success: raise test_runner.XcodeVersionNotFoundError( self.args.xcode_build_version) self.resolve_test_cases() summary = {} tr = None if not os.path.exists(self.args.out_dir): os.makedirs(self.args.out_dir) try: if self.args.xcode_parallelization: tr = xcodebuild_runner.SimulatorParallelTestRunner( self.args.app, self.args.host_app, self.args.iossim, self.args.version, self.args.platform, out_dir=self.args.out_dir, release=self.args.release, repeat_count=self.args.gtest_repeat, retries=self.args.retries, shards=self.args.shards, test_cases=self.args.test_cases, test_args=self.test_args, use_clang_coverage=self.args.use_clang_coverage, env_vars=self.args.env_var) elif self.args.variations_seed_path != 'NO_PATH': tr = variations_runner.VariationsSimulatorParallelTestRunner( self.args.app, self.args.host_app, self.args.iossim, self.args.version, self.args.platform, self.args.out_dir, self.args.variations_seed_path, release=self.args.release, test_cases=self.args.test_cases, test_args=self.test_args, env_vars=self.args.env_var) elif self.args.replay_path != 'NO_PATH': tr = wpr_runner.WprProxySimulatorTestRunner( self.args.app, self.args.host_app, self.args.iossim, self.args.replay_path, self.args.platform, self.args.version, self.args.wpr_tools_path, self.args.out_dir, env_vars=self.args.env_var, retries=self.args.retries, shards=self.args.shards, test_args=self.test_args, test_cases=self.args.test_cases, xctest=self.args.xctest, ) elif self.args.iossim and self.args.platform and self.args.version: tr = test_runner.SimulatorTestRunner( self.args.app, self.args.iossim, self.args.platform, self.args.version, self.args.out_dir, env_vars=self.args.env_var, repeat_count=self.args.gtest_repeat, retries=self.args.retries, shards=self.args.shards, test_args=self.test_args, test_cases=self.args.test_cases, use_clang_coverage=self.args.use_clang_coverage, wpr_tools_path=self.args.wpr_tools_path, xctest=self.args.xctest, ) elif self.args.xcodebuild_device_runner and self.args.xctest: tr = xcodebuild_runner.DeviceXcodeTestRunner( app_path=self.args.app, host_app_path=self.args.host_app, out_dir=self.args.out_dir, release=self.args.release, repeat_count=self.args.gtest_repeat, retries=self.args.retries, test_cases=self.args.test_cases, test_args=self.test_args, env_vars=self.args.env_var) else: tr = test_runner.DeviceTestRunner( self.args.app, self.args.out_dir, env_vars=self.args.env_var, repeat_count=self.args.gtest_repeat, restart=self.args.restart, retries=self.args.retries, test_args=self.test_args, test_cases=self.args.test_cases, xctest=self.args.xctest, ) logging.info("Using test runner %s" % type(tr).__name__) return 0 if tr.launch() else 1 except test_runner.DeviceError as e: sys.stderr.write(traceback.format_exc()) summary['step_text'] = '%s%s' % (e.__class__.__name__, ': %s' % e.args[0] if e.args else '') # Swarming infra marks device status unavailable for any device related # issue using this return code. return 3 except test_runner.TestRunnerError as e: sys.stderr.write(traceback.format_exc()) summary['step_text'] = '%s%s' % (e.__class__.__name__, ': %s' % e.args[0] if e.args else '') # test_runner.Launch returns 0 on success, 1 on failure, so return 2 # on exception to distinguish between a test failure, and a failure # to launch the test at all. return 2 finally: if tr: summary['logs'] = tr.logs with open(os.path.join(self.args.out_dir, 'summary.json'), 'w') as f: json.dump(summary, f) if tr: with open(os.path.join(self.args.out_dir, 'full_results.json'), 'w') as f: json.dump(tr.test_results, f) # The value of test-launcher-summary-output is set by the recipe # and passed here via swarming.py. This argument defaults to # ${ISOLATED_OUTDIR}/output.json. out-dir is set to ${ISOLATED_OUTDIR} # TODO(crbug.com/1031338) - the content of this output.json will # work with Chromium recipe because we use the noop_merge merge script, # but will require structural changes to support the default gtest # merge script (ref: //testing/merge_scripts/standard_gtest_merge.py) output_json_path = (self.args.test_launcher_summary_output or os.path.join(self.args.out_dir, 'output.json')) with open(output_json_path, 'w') as f: json.dump(tr.test_results, f) # Move the iOS runtime back to cache dir if the Xcode package is not # legacy (i.e. Xcode program & runtimes are in different CIPD packages.) # and it's a simulator task. if not is_legacy_xcode and self.args.version: runtime_cache_folder = xcode.construct_runtime_cache_folder( self.args.runtime_cache_prefix, self.args.version) xcode.move_runtime(runtime_cache_folder, self.args.xcode_path, False) test_runner.defaults_delete('com.apple.CoreSimulator', 'FramebufferServerRendererPolicy')
def main(): logging.basicConfig(format='[%(asctime)s:%(levelname)s] %(message)s', level=logging.DEBUG, datefmt='%I:%M:%S') test_runner.defaults_delete('com.apple.CoreSimulator', 'FramebufferServerRendererPolicy') args, test_args = parse_args() summary = {} tr = None if not os.path.exists(args.out_dir): os.makedirs(args.out_dir) try: if args.xcode_parallelization: tr = xcodebuild_runner.SimulatorParallelTestRunner( args.app, args.host_app, args.iossim, args.xcode_build_version, args.version, args.platform, out_dir=args.out_dir, mac_toolchain=args.mac_toolchain_cmd, retries=args.retries, shards=args.shards, xcode_path=args.xcode_path, test_cases=args.test_cases, test_args=test_args, env_vars=args.env_var ) elif args.replay_path != 'NO_PATH': tr = wpr_runner.WprProxySimulatorTestRunner( args.app, args.host_app, args.iossim, args.replay_path, args.platform, args.version, args.wpr_tools_path, args.xcode_build_version, args.out_dir, env_vars=args.env_var, mac_toolchain=args.mac_toolchain_cmd, retries=args.retries, shards=args.shards, test_args=test_args, test_cases=args.test_cases, xcode_path=args.xcode_path, xctest=args.xctest, ) elif args.iossim and args.platform and args.version: tr = test_runner.SimulatorTestRunner( args.app, args.iossim, args.platform, args.version, args.xcode_build_version, args.out_dir, env_vars=args.env_var, mac_toolchain=args.mac_toolchain_cmd, retries=args.retries, shards=args.shards, test_args=test_args, test_cases=args.test_cases, wpr_tools_path=args.wpr_tools_path, xcode_path=args.xcode_path, xctest=args.xctest, ) elif args.xcodebuild_device_runner and args.xctest: tr = xcodebuild_runner.DeviceXcodeTestRunner( app_path=args.app, host_app_path=args.host_app, xcode_build_version=args.xcode_build_version, out_dir=args.out_dir, mac_toolchain=args.mac_toolchain_cmd, retries=args.retries, xcode_path=args.xcode_path, test_cases=args.test_cases, test_args=test_args, env_vars=args.env_var) else: tr = test_runner.DeviceTestRunner( args.app, args.xcode_build_version, args.out_dir, env_vars=args.env_var, mac_toolchain=args.mac_toolchain_cmd, restart=args.restart, retries=args.retries, test_args=test_args, test_cases=args.test_cases, xcode_path=args.xcode_path, xctest=args.xctest, ) return 0 if tr.launch() else 1 except test_runner.TestRunnerError as e: sys.stderr.write(traceback.format_exc()) summary['step_text'] = '%s%s' % ( e.__class__.__name__, ': %s' % e.args[0] if e.args else '') # test_runner.Launch returns 0 on success, 1 on failure, so return 2 # on exception to distinguish between a test failure, and a failure # to launch the test at all. return 2 finally: if tr: summary['logs'] = tr.logs with open(os.path.join(args.out_dir, 'summary.json'), 'w') as f: json.dump(summary, f) if tr: with open(os.path.join(args.out_dir, 'full_results.json'), 'w') as f: json.dump(tr.test_results, f) test_runner.defaults_delete('com.apple.CoreSimulator', 'FramebufferServerRendererPolicy')
def run(self, args): """ Main coordinating function. """ self.parse_args(args) summary = {} tr = None if not os.path.exists(self.args.out_dir): os.makedirs(self.args.out_dir) try: if self.args.xcode_parallelization: tr = xcodebuild_runner.SimulatorParallelTestRunner( self.args.app, self.args.host_app, self.args.iossim, self.args.xcode_build_version, self.args.version, self.args.platform, out_dir=self.args.out_dir, mac_toolchain=self.args.mac_toolchain_cmd, retries=self.args.retries, shards=self.args.shards, xcode_path=self.args.xcode_path, test_cases=self.args.test_cases, test_args=self.test_args, env_vars=self.args.env_var) elif self.args.replay_path != 'NO_PATH': tr = wpr_runner.WprProxySimulatorTestRunner( self.args.app, self.args.host_app, self.args.iossim, self.args.replay_path, self.args.platform, self.args.version, self.args.wpr_tools_path, self.args.xcode_build_version, self.args.out_dir, env_vars=self.args.env_var, mac_toolchain=self.args.mac_toolchain_cmd, retries=self.args.retries, shards=self.args.shards, test_args=self.test_args, test_cases=self.args.test_cases, xcode_path=self.args.xcode_path, xctest=self.args.xctest, ) elif self.args.iossim and self.args.platform and self.args.version: tr = test_runner.SimulatorTestRunner( self.args.app, self.args.iossim, self.args.platform, self.args.version, self.args.xcode_build_version, self.args.out_dir, env_vars=self.args.env_var, mac_toolchain=self.args.mac_toolchain_cmd, retries=self.args.retries, shards=self.args.shards, test_args=self.test_args, test_cases=self.args.test_cases, wpr_tools_path=self.args.wpr_tools_path, xcode_path=self.args.xcode_path, xctest=self.args.xctest, ) elif self.args.xcodebuild_device_runner and self.args.xctest: tr = xcodebuild_runner.DeviceXcodeTestRunner( app_path=self.args.app, host_app_path=self.args.host_app, xcode_build_version=self.args.xcode_build_version, out_dir=self.args.out_dir, mac_toolchain=self.args.mac_toolchain_cmd, retries=self.args.retries, xcode_path=self.args.xcode_path, test_cases=self.args.test_cases, test_args=self.test_args, env_vars=self.args.env_var) else: tr = test_runner.DeviceTestRunner( self.args.app, self.args.xcode_build_version, self.args.out_dir, env_vars=self.args.env_var, mac_toolchain=self.args.mac_toolchain_cmd, restart=self.args.restart, retries=self.args.retries, test_args=self.test_args, test_cases=self.args.test_cases, xcode_path=self.args.xcode_path, xctest=self.args.xctest, ) return 0 if tr.launch() else 1 except test_runner.TestRunnerError as e: sys.stderr.write(traceback.format_exc()) summary['step_text'] = '%s%s' % (e.__class__.__name__, ': %s' % e.args[0] if e.args else '') # test_runner.Launch returns 0 on success, 1 on failure, so return 2 # on exception to distinguish between a test failure, and a failure # to launch the test at all. return 2 finally: if tr: summary['logs'] = tr.logs with open(os.path.join(self.args.out_dir, 'summary.json'), 'w') as f: json.dump(summary, f) if tr: with open(os.path.join(self.args.out_dir, 'full_results.json'), 'w') as f: json.dump(tr.test_results, f) # The value of test-launcher-summary-output is set by the recipe # and passed here via swarming.py. This argument defaults to # ${ISOLATED_OUTDIR}/output.json. out-dir is set to ${ISOLATED_OUTDIR} # TODO(crbug.com/1031338) - the content of this output.json will # work with Chromium recipe because we use the noop_merge merge script, # but will require structural changes to support the default gtest # merge script (ref: //testing/merge_scripts/standard_gtest_merge.py) output_json_path = (self.args.test_launcher_summary_output or os.path.join(self.args.out_dir, 'output.json')) with open(output_json_path, 'w') as f: json.dump(tr.test_results, f) test_runner.defaults_delete('com.apple.CoreSimulator', 'FramebufferServerRendererPolicy')