def test_startup_crash(self): """Ensures test is relaunched once on startup crash.""" def set_up(self): return @staticmethod def _run(cmd, shards=None): return collections.namedtuple( 'result', ['crashed', 'crashed_test'])(crashed=True, crashed_test=None) def tear_down(self): return self.mock(test_runner.SimulatorTestRunner, 'set_up', set_up) self.mock(test_runner.TestRunner, '_run', _run) self.mock(test_runner.SimulatorTestRunner, 'tear_down', tear_down) tr = test_runner.SimulatorTestRunner( 'fake-app', 'fake-iossim', 'platform', 'os', 'xcode-version', 'xcode-build', 'out-dir', xctest=True, ) with self.assertRaises(test_runner.AppLaunchError): tr.launch()
def test_init(self): """Ensures instance is created.""" def exists(path): return True def find_xcode(version): return {'found': True} def check_output(command): return 'fake-bundle-id' self.mock(test_runner.os.path, 'exists', exists) self.mock(test_runner.find_xcode, 'find_xcode', find_xcode) self.mock(test_runner.subprocess, 'check_output', check_output) tr = test_runner.SimulatorTestRunner( 'fake-app', 'fake-iossim', 'platform', 'os', 'xcode-version', 'out-dir', ) self.failUnless(tr)
def test_run(self): """Ensures the _run method is correct with test sharding.""" def shard_xctest(object_path, shards, test_cases=None): return [['a/1', 'b/2'], ['c/3', 'd/4'], ['e/5']] def run_tests(self, test_shard=None): out = [] for test in test_shard: testname = test.split('/') out.append('Test Case \'-[%s %s]\' started.' % (testname[0], testname[1])) out.append('Test Case \'-[%s %s]\' passed (0.1 seconds)' % (testname[0], testname[1])) return (out, 0, 0) tr = test_runner.SimulatorTestRunner( 'fake-app', 'fake-iossim', 'platform', 'os', 'xcode-version', 'xcode-build', 'out-dir', ) self.mock(test_runner, 'shard_xctest', shard_xctest) self.mock(test_runner.SimulatorTestRunner, 'run_tests', run_tests) tr.xctest_path = 'fake.xctest' cmd = tr.get_launch_command() result = tr._run(cmd=cmd, shards=3) self.assertIn('a/1', result.passed_tests) self.assertIn('b/2', result.passed_tests) self.assertIn('c/3', result.passed_tests) self.assertIn('d/4', result.passed_tests) self.assertIn('e/5', result.passed_tests)
def test_get_launch_command(self): """Ensures launch command is correct with test_filters, test sharding and test_cases.""" tr = test_runner.SimulatorTestRunner( 'fake-app', 'fake-iossim', 'platform', 'os', 'xcode-version', 'xcode-build', 'out-dir', ) tr.xctest_path = 'fake.xctest' # Cases test_filter is not empty, with empty/non-empty self.test_cases. tr.test_cases = [] cmd = tr.get_launch_command(['a'], invert=False) self.assertIn('-t', cmd) self.assertIn('a', cmd) tr.test_cases = ['a', 'b'] cmd = tr.get_launch_command(['a'], invert=False) self.assertIn('-t', cmd) self.assertIn('a', cmd) self.assertNotIn('b', cmd) # Cases test_filter is empty, with empty/non-empty self.test_cases. tr.test_cases = [] cmd = tr.get_launch_command(test_filter=None, invert=False) self.assertNotIn('-t', cmd) tr.test_cases = ['a', 'b'] cmd = tr.get_launch_command(test_filter=None, invert=False) self.assertIn('-t', cmd) self.assertIn('a', cmd) self.assertIn('b', cmd)
def main(args, test_args): summary = {} tr = None if not os.path.exists(args.out_dir): os.makedirs(args.out_dir) try: tr = test_runner.SimulatorTestRunner( args.app, args.iossim, args.platform, args.version, args.xcode_version, args.out_dir, test_args=test_args, ) return 0 if tr.launch() else 1 except test_runner.TestRunnerError as e: sys.stderr.write(traceback.format_exc()) summary['step_text'] = '%s%s' % ( e.__class__.__name__, ': %s' % e.args[0] if e.args else '') # test_runner.Launch returns 0 on success, 1 on failure, so return 2 # on exception to distinguish between a test failure, and a failure # to launch the test at all. return 2 finally: if tr: summary['logs'] = tr.logs with open(os.path.join(args.out_dir, 'summary.json'), 'w') as f: json.dump(summary, f)
def test_relaunch(self): """Ensures test is relaunched on test crash until tests complete.""" def set_up(self): return @staticmethod def _run(cmd, shards=None): result = collections.namedtuple( 'result', [ 'crashed', 'crashed_test', 'failed_tests', 'flaked_tests', 'passed_tests', ], ) if '-e' not in cmd: # First run, has no test filter supplied. Mock a crash. return result( crashed=True, crashed_test='c', failed_tests={ 'b': ['b-out'], 'c': ['Did not complete.'] }, flaked_tests={'d': ['d-out']}, passed_tests=['a'], ) else: return result( crashed=False, crashed_test=None, failed_tests={}, flaked_tests={}, passed_tests=[], ) def tear_down(self): return self.mock(test_runner.SimulatorTestRunner, 'set_up', set_up) self.mock(test_runner.TestRunner, '_run', _run) self.mock(test_runner.SimulatorTestRunner, 'tear_down', tear_down) tr = test_runner.SimulatorTestRunner( 'fake-app', 'fake-iossim', 'platform', 'os', 'xcode-version', 'xcode-build', 'out-dir', ) tr.launch() self.assertTrue(tr.logs)
def test_init(self): """Ensures instance is created.""" tr = test_runner.SimulatorTestRunner( 'fake-app', 'fake-iossim', 'platform', 'os', 'xcode-version', 'xcode-build', 'out-dir', ) self.assertTrue(tr)
def test_iossim_not_found(self): """Ensures SimulatorNotFoundError is raised.""" self.mock(os.path, 'exists', lambda p: not p.endswith('fake-iossim')) with self.assertRaises(test_runner.SimulatorNotFoundError): test_runner.SimulatorTestRunner( 'fake-app', 'fake-iossim', 'platform', 'os', 'xcode-version', 'xcode-build', 'out-dir', )
def test_run_with_system_alert(self): """Ensures SystemAlertPresentError is raised when warning 'System alert view is present, so skipping all tests' is in the output.""" with self.assertRaises(test_runner.SystemAlertPresentError): tr = test_runner.SimulatorTestRunner( 'fake-app', 'fake-iossim', 'platform', 'os', 'xcode-version', 'xcode-build', 'out-dir', ) tr.xctest_path = 'fake.xctest' cmd = ['echo', 'System alert view is present, so skipping all tests!'] result = tr._run(cmd=cmd)
def test_startup_crash(self): """Ensures test is relaunched once on startup crash.""" def exists(path): return True def find_xcode(version): return {'found': True} def check_output(command): return 'fake-bundle-id' def set_up(self): return @staticmethod def _run(command): return collections.namedtuple( 'result', ['crashed', 'crashed_test'])(crashed=True, crashed_test=None) def tear_down(self): return self.mock(test_runner.os.path, 'exists', exists) self.mock(test_runner.find_xcode, 'find_xcode', find_xcode) self.mock(test_runner.subprocess, 'check_output', check_output) self.mock(test_runner.SimulatorTestRunner, 'set_up', set_up) self.mock(test_runner.TestRunner, '_run', _run) self.mock(test_runner.SimulatorTestRunner, 'tear_down', tear_down) tr = test_runner.SimulatorTestRunner( 'fake-app', 'fake-iossim', 'platform', 'os', 'xcode-version', 'out-dir', ) self.assertRaises(test_runner.AppLaunchError, tr.launch)
def run(self, args): """ Main coordinating function. """ self.parse_args(args) # This logic is run by default before the otool command is invoked such that # otool has the correct Xcode selected for command line dev tools. if not self.install_xcode(self.args.xcode_build_version, self.args.mac_toolchain_cmd, self.args.xcode_path): raise test_runner.XcodeVersionNotFoundError( self.args.xcode_build_version) # GTEST_SHARD_INDEX and GTEST_TOTAL_SHARDS are additional test environment # variables, set by Swarming, that are only set for a swarming task # shard count is > 1. # # For a given test on a given run, otool should return the same total # counts and thus, should generate the same sublists. With the shard index, # each shard would then know the exact test case to run. gtest_shard_index = os.getenv('GTEST_SHARD_INDEX', 0) gtest_total_shards = os.getenv('GTEST_TOTAL_SHARDS', 0) if gtest_shard_index and gtest_total_shards: self.args.test_cases = shard_util.shard_test_cases( self.args, gtest_shard_index, gtest_total_shards) summary = {} tr = None if not os.path.exists(self.args.out_dir): os.makedirs(self.args.out_dir) try: if self.args.xcode_parallelization: tr = xcodebuild_runner.SimulatorParallelTestRunner( self.args.app, self.args.host_app, self.args.iossim, self.args.version, self.args.platform, out_dir=self.args.out_dir, release=self.args.release, retries=self.args.retries, shards=self.args.shards, test_cases=self.args.test_cases, test_args=self.test_args, use_clang_coverage=self.args.use_clang_coverage, env_vars=self.args.env_var) elif self.args.replay_path != 'NO_PATH': tr = wpr_runner.WprProxySimulatorTestRunner( self.args.app, self.args.host_app, self.args.iossim, self.args.replay_path, self.args.platform, self.args.version, self.args.wpr_tools_path, self.args.out_dir, env_vars=self.args.env_var, retries=self.args.retries, shards=self.args.shards, test_args=self.test_args, test_cases=self.args.test_cases, xctest=self.args.xctest, ) elif self.args.iossim and self.args.platform and self.args.version: tr = test_runner.SimulatorTestRunner( self.args.app, self.args.iossim, self.args.platform, self.args.version, self.args.out_dir, env_vars=self.args.env_var, retries=self.args.retries, shards=self.args.shards, test_args=self.test_args, test_cases=self.args.test_cases, use_clang_coverage=self.args.use_clang_coverage, wpr_tools_path=self.args.wpr_tools_path, xctest=self.args.xctest, ) elif self.args.xcodebuild_device_runner and self.args.xctest: tr = xcodebuild_runner.DeviceXcodeTestRunner( app_path=self.args.app, host_app_path=self.args.host_app, out_dir=self.args.out_dir, release=self.args.release, retries=self.args.retries, test_cases=self.args.test_cases, test_args=self.test_args, env_vars=self.args.env_var) else: tr = test_runner.DeviceTestRunner( self.args.app, self.args.out_dir, env_vars=self.args.env_var, restart=self.args.restart, retries=self.args.retries, test_args=self.test_args, test_cases=self.args.test_cases, xctest=self.args.xctest, ) return 0 if tr.launch() else 1 except test_runner.TestRunnerError as e: sys.stderr.write(traceback.format_exc()) summary['step_text'] = '%s%s' % (e.__class__.__name__, ': %s' % e.args[0] if e.args else '') # test_runner.Launch returns 0 on success, 1 on failure, so return 2 # on exception to distinguish between a test failure, and a failure # to launch the test at all. return 2 finally: if tr: summary['logs'] = tr.logs with open(os.path.join(self.args.out_dir, 'summary.json'), 'w') as f: json.dump(summary, f) if tr: with open(os.path.join(self.args.out_dir, 'full_results.json'), 'w') as f: json.dump(tr.test_results, f) # The value of test-launcher-summary-output is set by the recipe # and passed here via swarming.py. This argument defaults to # ${ISOLATED_OUTDIR}/output.json. out-dir is set to ${ISOLATED_OUTDIR} # TODO(crbug.com/1031338) - the content of this output.json will # work with Chromium recipe because we use the noop_merge merge script, # but will require structural changes to support the default gtest # merge script (ref: //testing/merge_scripts/standard_gtest_merge.py) output_json_path = (self.args.test_launcher_summary_output or os.path.join(self.args.out_dir, 'output.json')) with open(output_json_path, 'w') as f: json.dump(tr.test_results, f) test_runner.defaults_delete('com.apple.CoreSimulator', 'FramebufferServerRendererPolicy')
def run(self, args): """ Main coordinating function. """ self.parse_args(args) summary = {} tr = None if not os.path.exists(self.args.out_dir): os.makedirs(self.args.out_dir) try: if self.args.xcode_parallelization: tr = xcodebuild_runner.SimulatorParallelTestRunner( self.args.app, self.args.host_app, self.args.iossim, self.args.xcode_build_version, self.args.version, self.args.platform, out_dir=self.args.out_dir, mac_toolchain=self.args.mac_toolchain_cmd, retries=self.args.retries, shards=self.args.shards, xcode_path=self.args.xcode_path, test_cases=self.args.test_cases, test_args=self.test_args, env_vars=self.args.env_var) elif self.args.replay_path != 'NO_PATH': tr = wpr_runner.WprProxySimulatorTestRunner( self.args.app, self.args.host_app, self.args.iossim, self.args.replay_path, self.args.platform, self.args.version, self.args.wpr_tools_path, self.args.xcode_build_version, self.args.out_dir, env_vars=self.args.env_var, mac_toolchain=self.args.mac_toolchain_cmd, retries=self.args.retries, shards=self.args.shards, test_args=self.test_args, test_cases=self.args.test_cases, xcode_path=self.args.xcode_path, xctest=self.args.xctest, ) elif self.args.iossim and self.args.platform and self.args.version: tr = test_runner.SimulatorTestRunner( self.args.app, self.args.iossim, self.args.platform, self.args.version, self.args.xcode_build_version, self.args.out_dir, env_vars=self.args.env_var, mac_toolchain=self.args.mac_toolchain_cmd, retries=self.args.retries, shards=self.args.shards, test_args=self.test_args, test_cases=self.args.test_cases, wpr_tools_path=self.args.wpr_tools_path, xcode_path=self.args.xcode_path, xctest=self.args.xctest, ) elif self.args.xcodebuild_device_runner and self.args.xctest: tr = xcodebuild_runner.DeviceXcodeTestRunner( app_path=self.args.app, host_app_path=self.args.host_app, xcode_build_version=self.args.xcode_build_version, out_dir=self.args.out_dir, mac_toolchain=self.args.mac_toolchain_cmd, retries=self.args.retries, xcode_path=self.args.xcode_path, test_cases=self.args.test_cases, test_args=self.test_args, env_vars=self.args.env_var) else: tr = test_runner.DeviceTestRunner( self.args.app, self.args.xcode_build_version, self.args.out_dir, env_vars=self.args.env_var, mac_toolchain=self.args.mac_toolchain_cmd, restart=self.args.restart, retries=self.args.retries, test_args=self.test_args, test_cases=self.args.test_cases, xcode_path=self.args.xcode_path, xctest=self.args.xctest, ) return 0 if tr.launch() else 1 except test_runner.TestRunnerError as e: sys.stderr.write(traceback.format_exc()) summary['step_text'] = '%s%s' % (e.__class__.__name__, ': %s' % e.args[0] if e.args else '') # test_runner.Launch returns 0 on success, 1 on failure, so return 2 # on exception to distinguish between a test failure, and a failure # to launch the test at all. return 2 finally: if tr: summary['logs'] = tr.logs with open(os.path.join(self.args.out_dir, 'summary.json'), 'w') as f: json.dump(summary, f) if tr: with open(os.path.join(self.args.out_dir, 'full_results.json'), 'w') as f: json.dump(tr.test_results, f) # The value of test-launcher-summary-output is set by the recipe # and passed here via swarming.py. This argument defaults to # ${ISOLATED_OUTDIR}/output.json. out-dir is set to ${ISOLATED_OUTDIR} # TODO(crbug.com/1031338) - the content of this output.json will # work with Chromium recipe because we use the noop_merge merge script, # but will require structural changes to support the default gtest # merge script (ref: //testing/merge_scripts/standard_gtest_merge.py) output_json_path = (self.args.test_launcher_summary_output or os.path.join(self.args.out_dir, 'output.json')) with open(output_json_path, 'w') as f: json.dump(tr.test_results, f) test_runner.defaults_delete('com.apple.CoreSimulator', 'FramebufferServerRendererPolicy')
def test_relaunch(self): """Ensures test is relaunched on test crash until tests complete.""" def exists(path): return True def find_xcode(version): return {'found': True} def check_output(command): return 'fake-bundle-id' def set_up(self): return @staticmethod def _run(command): result = collections.namedtuple( 'result', [ 'crashed', 'crashed_test', 'failed_tests', 'flaked_tests', 'passed_tests', ], ) if '-e' not in command: # First run, has no test filter supplied. Mock a crash. return result( crashed=True, crashed_test='c', failed_tests={ 'b': ['b-out'], 'c': ['Did not complete.'] }, flaked_tests={'d': ['d-out']}, passed_tests=['a'], ) else: return result( crashed=False, crashed_test=None, failed_tests={}, flaked_tests={}, passed_tests=[], ) def tear_down(self): return self.mock(test_runner.os.path, 'exists', exists) self.mock(test_runner.find_xcode, 'find_xcode', find_xcode) self.mock(test_runner.subprocess, 'check_output', check_output) self.mock(test_runner.SimulatorTestRunner, 'set_up', set_up) self.mock(test_runner.TestRunner, '_run', _run) self.mock(test_runner.SimulatorTestRunner, 'tear_down', tear_down) tr = test_runner.SimulatorTestRunner( 'fake-app', 'fake-iossim', 'platform', 'os', 'xcode-version', 'out-dir', ) tr.launch() self.failUnless(tr.summary['logs'])
def main(): args, test_args = parse_args() summary = {} tr = None if not os.path.exists(args.out_dir): os.makedirs(args.out_dir) try: if args.iossim and args.platform and args.version: tr = test_runner.SimulatorTestRunner( args.app, args.iossim, args.platform, args.version, args.xcode_version, args.xcode_build_version, args.out_dir, env_vars=args.env_var, mac_toolchain=args.mac_toolchain_cmd, retries=args.retries, shards=args.shards, test_args=test_args, test_cases=args.test_cases, xcode_path=args.xcode_path, xctest=args.xctest, ) else: tr = test_runner.DeviceTestRunner( args.app, args.xcode_version, args.xcode_build_version, args.out_dir, env_vars=args.env_var, mac_toolchain=args.mac_toolchain_cmd, restart=args.restart, retries=args.retries, test_args=test_args, test_cases=args.test_cases, xcode_path=args.xcode_path, xctest=args.xctest, ) return 0 if tr.launch() else 1 except test_runner.TestRunnerError as e: sys.stderr.write(traceback.format_exc()) summary['step_text'] = '%s%s' % (e.__class__.__name__, ': %s' % e.args[0] if e.args else '') # test_runner.Launch returns 0 on success, 1 on failure, so return 2 # on exception to distinguish between a test failure, and a failure # to launch the test at all. return 2 finally: if tr: summary['logs'] = tr.logs with open(os.path.join(args.out_dir, 'summary.json'), 'w') as f: json.dump(summary, f) if tr: with open(os.path.join(args.out_dir, 'full_results.json'), 'w') as f: json.dump(tr.test_results, f)
def main(): logging.basicConfig(format='[%(asctime)s:%(levelname)s] %(message)s', level=logging.DEBUG, datefmt='%I:%M:%S') args, test_args = parse_args() summary = {} tr = None if not os.path.exists(args.out_dir): os.makedirs(args.out_dir) try: if args.xcode_parallelization: tr = xcodebuild_runner.SimulatorParallelTestRunner( args.app, args.iossim, args.xcode_build_version, args.version, args.platform, out_dir=args.out_dir, mac_toolchain=args.mac_toolchain_cmd, retries=args.retries, shards=args.shards, xcode_path=args.xcode_path, test_cases=args.test_cases, test_args=test_args, env_vars=args.env_var) elif args.replay_path != 'NO_PATH': tr = test_runner.WprProxySimulatorTestRunner( args.app, args.iossim, args.replay_path, args.platform, args.version, args.wpr_tools_path, args.xcode_build_version, args.out_dir, env_vars=args.env_var, mac_toolchain=args.mac_toolchain_cmd, retries=args.retries, shards=args.shards, test_args=test_args, test_cases=args.test_cases, xcode_path=args.xcode_path, xctest=args.xctest, ) elif args.iossim and args.platform and args.version: tr = test_runner.SimulatorTestRunner( args.app, args.iossim, args.platform, args.version, args.xcode_build_version, args.out_dir, env_vars=args.env_var, mac_toolchain=args.mac_toolchain_cmd, retries=args.retries, shards=args.shards, test_args=test_args, test_cases=args.test_cases, use_trusted_cert=args.use_trusted_cert, wpr_tools_path=args.wpr_tools_path, xcode_path=args.xcode_path, xctest=args.xctest, ) else: tr = test_runner.DeviceTestRunner( args.app, args.xcode_build_version, args.out_dir, env_vars=args.env_var, mac_toolchain=args.mac_toolchain_cmd, restart=args.restart, retries=args.retries, test_args=test_args, test_cases=args.test_cases, xcode_path=args.xcode_path, xctest=args.xctest, ) return 0 if tr.launch() else 1 except test_runner.TestRunnerError as e: sys.stderr.write(traceback.format_exc()) summary['step_text'] = '%s%s' % (e.__class__.__name__, ': %s' % e.args[0] if e.args else '') # test_runner.Launch returns 0 on success, 1 on failure, so return 2 # on exception to distinguish between a test failure, and a failure # to launch the test at all. return 2 finally: if tr: summary['logs'] = tr.logs with open(os.path.join(args.out_dir, 'summary.json'), 'w') as f: json.dump(summary, f) if tr: with open(os.path.join(args.out_dir, 'full_results.json'), 'w') as f: json.dump(tr.test_results, f)
def run(self, args): """ Main coordinating function. """ self.parse_args(args) summary = {} tr = None if not os.path.exists(self.args.out_dir): os.makedirs(self.args.out_dir) try: if self.args.xcode_parallelization: tr = xcodebuild_runner.SimulatorParallelTestRunner( self.args.app, self.args.host_app, self.args.iossim, self.args.xcode_build_version, self.args.version, self.args.platform, out_dir=self.args.out_dir, mac_toolchain=self.args.mac_toolchain_cmd, retries=self.args.retries, shards=self.args.shards, xcode_path=self.args.xcode_path, test_cases=self.args.test_cases, test_args=self.test_args, env_vars=self.args.env_var) elif self.args.replay_path != 'NO_PATH': tr = wpr_runner.WprProxySimulatorTestRunner( self.args.app, self.args.host_app, self.args.iossim, self.args.replay_path, self.args.platform, self.args.version, self.args.wpr_tools_path, self.args.xcode_build_version, self.args.out_dir, env_vars=self.args.env_var, mac_toolchain=self.args.mac_toolchain_cmd, retries=self.args.retries, shards=self.args.shards, test_args=self.test_args, test_cases=self.args.test_cases, xcode_path=self.args.xcode_path, xctest=self.args.xctest, ) elif self.args.iossim and self.args.platform and self.args.version: tr = test_runner.SimulatorTestRunner( self.args.app, self.args.iossim, self.args.platform, self.args.version, self.args.xcode_build_version, self.args.out_dir, env_vars=self.args.env_var, mac_toolchain=self.args.mac_toolchain_cmd, retries=self.args.retries, shards=self.args.shards, test_args=self.test_args, test_cases=self.args.test_cases, wpr_tools_path=self.args.wpr_tools_path, xcode_path=self.args.xcode_path, xctest=self.args.xctest, ) elif self.args.xcodebuild_device_runner and self.args.xctest: tr = xcodebuild_runner.DeviceXcodeTestRunner( app_path=self.args.app, host_app_path=self.args.host_app, xcode_build_version=self.args.xcode_build_version, out_dir=self.args.out_dir, mac_toolchain=self.args.mac_toolchain_cmd, retries=self.args.retries, xcode_path=self.args.xcode_path, test_cases=self.args.test_cases, test_args=self.test_args, env_vars=self.args.env_var) else: tr = test_runner.DeviceTestRunner( self.args.app, self.args.xcode_build_version, self.args.out_dir, env_vars=self.args.env_var, mac_toolchain=self.args.mac_toolchain_cmd, restart=self.args.restart, retries=self.args.retries, test_args=self.test_args, test_cases=self.args.test_cases, xcode_path=self.args.xcode_path, xctest=self.args.xctest, ) return 0 if tr.launch() else 1 except test_runner.TestRunnerError as e: sys.stderr.write(traceback.format_exc()) summary['step_text'] = '%s%s' % (e.__class__.__name__, ': %s' % e.args[0] if e.args else '') # test_runner.Launch returns 0 on success, 1 on failure, so return 2 # on exception to distinguish between a test failure, and a failure # to launch the test at all. return 2 finally: if tr: summary['logs'] = tr.logs with open(os.path.join(self.args.out_dir, 'summary.json'), 'w') as f: json.dump(summary, f) if tr: with open(os.path.join(self.args.out_dir, 'full_results.json'), 'w') as f: json.dump(tr.test_results, f) test_runner.defaults_delete('com.apple.CoreSimulator', 'FramebufferServerRendererPolicy')
def run(self, args): """ Main coordinating function. """ self.parse_args(args) # This logic is run by default before the otool command is invoked such that # otool has the correct Xcode selected for command line dev tools. install_success, is_legacy_xcode = self.install_xcode() if not install_success: raise test_runner.XcodeVersionNotFoundError( self.args.xcode_build_version) self.resolve_test_cases() summary = {} tr = None if not os.path.exists(self.args.out_dir): os.makedirs(self.args.out_dir) try: if self.args.xcode_parallelization: tr = xcodebuild_runner.SimulatorParallelTestRunner( self.args.app, self.args.host_app, self.args.iossim, self.args.version, self.args.platform, out_dir=self.args.out_dir, release=self.args.release, repeat_count=self.args.gtest_repeat, retries=self.args.retries, shards=self.args.shards, test_cases=self.args.test_cases, test_args=self.test_args, use_clang_coverage=self.args.use_clang_coverage, env_vars=self.args.env_var) elif self.args.variations_seed_path != 'NO_PATH': tr = variations_runner.VariationsSimulatorParallelTestRunner( self.args.app, self.args.host_app, self.args.iossim, self.args.version, self.args.platform, self.args.out_dir, self.args.variations_seed_path, release=self.args.release, test_cases=self.args.test_cases, test_args=self.test_args, env_vars=self.args.env_var) elif self.args.replay_path != 'NO_PATH': tr = wpr_runner.WprProxySimulatorTestRunner( self.args.app, self.args.host_app, self.args.iossim, self.args.replay_path, self.args.platform, self.args.version, self.args.wpr_tools_path, self.args.out_dir, env_vars=self.args.env_var, retries=self.args.retries, shards=self.args.shards, test_args=self.test_args, test_cases=self.args.test_cases, xctest=self.args.xctest, ) elif self.args.iossim and self.args.platform and self.args.version: tr = test_runner.SimulatorTestRunner( self.args.app, self.args.iossim, self.args.platform, self.args.version, self.args.out_dir, env_vars=self.args.env_var, repeat_count=self.args.gtest_repeat, retries=self.args.retries, shards=self.args.shards, test_args=self.test_args, test_cases=self.args.test_cases, use_clang_coverage=self.args.use_clang_coverage, wpr_tools_path=self.args.wpr_tools_path, xctest=self.args.xctest, ) elif self.args.xcodebuild_device_runner and self.args.xctest: tr = xcodebuild_runner.DeviceXcodeTestRunner( app_path=self.args.app, host_app_path=self.args.host_app, out_dir=self.args.out_dir, release=self.args.release, repeat_count=self.args.gtest_repeat, retries=self.args.retries, test_cases=self.args.test_cases, test_args=self.test_args, env_vars=self.args.env_var) else: tr = test_runner.DeviceTestRunner( self.args.app, self.args.out_dir, env_vars=self.args.env_var, repeat_count=self.args.gtest_repeat, restart=self.args.restart, retries=self.args.retries, test_args=self.test_args, test_cases=self.args.test_cases, xctest=self.args.xctest, ) logging.info("Using test runner %s" % type(tr).__name__) return 0 if tr.launch() else 1 except test_runner.DeviceError as e: sys.stderr.write(traceback.format_exc()) summary['step_text'] = '%s%s' % (e.__class__.__name__, ': %s' % e.args[0] if e.args else '') # Swarming infra marks device status unavailable for any device related # issue using this return code. return 3 except test_runner.TestRunnerError as e: sys.stderr.write(traceback.format_exc()) summary['step_text'] = '%s%s' % (e.__class__.__name__, ': %s' % e.args[0] if e.args else '') # test_runner.Launch returns 0 on success, 1 on failure, so return 2 # on exception to distinguish between a test failure, and a failure # to launch the test at all. return 2 finally: if tr: summary['logs'] = tr.logs with open(os.path.join(self.args.out_dir, 'summary.json'), 'w') as f: json.dump(summary, f) if tr: with open(os.path.join(self.args.out_dir, 'full_results.json'), 'w') as f: json.dump(tr.test_results, f) # The value of test-launcher-summary-output is set by the recipe # and passed here via swarming.py. This argument defaults to # ${ISOLATED_OUTDIR}/output.json. out-dir is set to ${ISOLATED_OUTDIR} # TODO(crbug.com/1031338) - the content of this output.json will # work with Chromium recipe because we use the noop_merge merge script, # but will require structural changes to support the default gtest # merge script (ref: //testing/merge_scripts/standard_gtest_merge.py) output_json_path = (self.args.test_launcher_summary_output or os.path.join(self.args.out_dir, 'output.json')) with open(output_json_path, 'w') as f: json.dump(tr.test_results, f) # Move the iOS runtime back to cache dir if the Xcode package is not # legacy (i.e. Xcode program & runtimes are in different CIPD packages.) # and it's a simulator task. if not is_legacy_xcode and self.args.version: runtime_cache_folder = xcode.construct_runtime_cache_folder( self.args.runtime_cache_prefix, self.args.version) xcode.move_runtime(runtime_cache_folder, self.args.xcode_path, False) test_runner.defaults_delete('com.apple.CoreSimulator', 'FramebufferServerRendererPolicy')