def test_startup_crash(self):
"""Ensures test is relaunched once on startup crash."""
def set_up(self):
return
@staticmethod
def _run(cmd, shards=None):
return collections.namedtuple(
'result', ['crashed', 'crashed_test'])(crashed=True,
crashed_test=None)
def tear_down(self):
return
self.mock(test_runner.SimulatorTestRunner, 'set_up', set_up)
self.mock(test_runner.TestRunner, '_run', _run)
self.mock(test_runner.SimulatorTestRunner, 'tear_down', tear_down)
tr = test_runner.SimulatorTestRunner(
'fake-app',
'fake-iossim',
'platform',
'os',
'xcode-version',
'xcode-build',
'out-dir',
xctest=True,
)
with self.assertRaises(test_runner.AppLaunchError):
tr.launch()
def test_init(self):
"""Ensures instance is created."""
def exists(path):
return True
def find_xcode(version):
return {'found': True}
def check_output(command):
return 'fake-bundle-id'
self.mock(test_runner.os.path, 'exists', exists)
self.mock(test_runner.find_xcode, 'find_xcode', find_xcode)
self.mock(test_runner.subprocess, 'check_output', check_output)
tr = test_runner.SimulatorTestRunner(
'fake-app',
'fake-iossim',
'platform',
'os',
'xcode-version',
'out-dir',
)
self.failUnless(tr)
def test_run(self):
"""Ensures the _run method is correct with test sharding."""
def shard_xctest(object_path, shards, test_cases=None):
return [['a/1', 'b/2'], ['c/3', 'd/4'], ['e/5']]
def run_tests(self, test_shard=None):
out = []
for test in test_shard:
testname = test.split('/')
out.append('Test Case \'-[%s %s]\' started.' %
(testname[0], testname[1]))
out.append('Test Case \'-[%s %s]\' passed (0.1 seconds)' %
(testname[0], testname[1]))
return (out, 0, 0)
tr = test_runner.SimulatorTestRunner(
'fake-app',
'fake-iossim',
'platform',
'os',
'xcode-version',
'xcode-build',
'out-dir',
)
self.mock(test_runner, 'shard_xctest', shard_xctest)
self.mock(test_runner.SimulatorTestRunner, 'run_tests', run_tests)
tr.xctest_path = 'fake.xctest'
cmd = tr.get_launch_command()
result = tr._run(cmd=cmd, shards=3)
self.assertIn('a/1', result.passed_tests)
self.assertIn('b/2', result.passed_tests)
self.assertIn('c/3', result.passed_tests)
self.assertIn('d/4', result.passed_tests)
self.assertIn('e/5', result.passed_tests)
def test_get_launch_command(self):
"""Ensures launch command is correct with test_filters, test sharding and
test_cases."""
tr = test_runner.SimulatorTestRunner(
'fake-app',
'fake-iossim',
'platform',
'os',
'xcode-version',
'xcode-build',
'out-dir',
)
tr.xctest_path = 'fake.xctest'
# Cases test_filter is not empty, with empty/non-empty self.test_cases.
tr.test_cases = []
cmd = tr.get_launch_command(['a'], invert=False)
self.assertIn('-t', cmd)
self.assertIn('a', cmd)
tr.test_cases = ['a', 'b']
cmd = tr.get_launch_command(['a'], invert=False)
self.assertIn('-t', cmd)
self.assertIn('a', cmd)
self.assertNotIn('b', cmd)
# Cases test_filter is empty, with empty/non-empty self.test_cases.
tr.test_cases = []
cmd = tr.get_launch_command(test_filter=None, invert=False)
self.assertNotIn('-t', cmd)
tr.test_cases = ['a', 'b']
cmd = tr.get_launch_command(test_filter=None, invert=False)
self.assertIn('-t', cmd)
self.assertIn('a', cmd)
self.assertIn('b', cmd)
def main(args, test_args):
summary = {}
tr = None
if not os.path.exists(args.out_dir):
os.makedirs(args.out_dir)
try:
tr = test_runner.SimulatorTestRunner(
args.app,
args.iossim,
args.platform,
args.version,
args.xcode_version,
args.out_dir,
test_args=test_args,
)
return 0 if tr.launch() else 1
except test_runner.TestRunnerError as e:
sys.stderr.write(traceback.format_exc())
summary['step_text'] = '%s%s' % (
e.__class__.__name__, ': %s' % e.args[0] if e.args else '')
# test_runner.Launch returns 0 on success, 1 on failure, so return 2
# on exception to distinguish between a test failure, and a failure
# to launch the test at all.
return 2
finally:
if tr:
summary['logs'] = tr.logs
with open(os.path.join(args.out_dir, 'summary.json'), 'w') as f:
json.dump(summary, f)
def test_relaunch(self):
"""Ensures test is relaunched on test crash until tests complete."""
def set_up(self):
return
@staticmethod
def _run(cmd, shards=None):
result = collections.namedtuple(
'result',
[
'crashed',
'crashed_test',
'failed_tests',
'flaked_tests',
'passed_tests',
],
)
if '-e' not in cmd:
# First run, has no test filter supplied. Mock a crash.
return result(
crashed=True,
crashed_test='c',
failed_tests={
'b': ['b-out'],
'c': ['Did not complete.']
},
flaked_tests={'d': ['d-out']},
passed_tests=['a'],
)
else:
return result(
crashed=False,
crashed_test=None,
failed_tests={},
flaked_tests={},
passed_tests=[],
)
def tear_down(self):
return
self.mock(test_runner.SimulatorTestRunner, 'set_up', set_up)
self.mock(test_runner.TestRunner, '_run', _run)
self.mock(test_runner.SimulatorTestRunner, 'tear_down', tear_down)
tr = test_runner.SimulatorTestRunner(
'fake-app',
'fake-iossim',
'platform',
'os',
'xcode-version',
'xcode-build',
'out-dir',
)
tr.launch()
self.assertTrue(tr.logs)
def test_init(self):
"""Ensures instance is created."""
tr = test_runner.SimulatorTestRunner(
'fake-app',
'fake-iossim',
'platform',
'os',
'xcode-version',
'xcode-build',
'out-dir',
)
self.assertTrue(tr)
def test_iossim_not_found(self):
"""Ensures SimulatorNotFoundError is raised."""
self.mock(os.path, 'exists', lambda p: not p.endswith('fake-iossim'))
with self.assertRaises(test_runner.SimulatorNotFoundError):
test_runner.SimulatorTestRunner(
'fake-app',
'fake-iossim',
'platform',
'os',
'xcode-version',
'xcode-build',
'out-dir',
)
def test_run_with_system_alert(self):
"""Ensures SystemAlertPresentError is raised when warning 'System alert
view is present, so skipping all tests' is in the output."""
with self.assertRaises(test_runner.SystemAlertPresentError):
tr = test_runner.SimulatorTestRunner(
'fake-app',
'fake-iossim',
'platform',
'os',
'xcode-version',
'xcode-build',
'out-dir',
)
tr.xctest_path = 'fake.xctest'
cmd = ['echo', 'System alert view is present, so skipping all tests!']
result = tr._run(cmd=cmd)
def test_startup_crash(self):
"""Ensures test is relaunched once on startup crash."""
def exists(path):
return True
def find_xcode(version):
return {'found': True}
def check_output(command):
return 'fake-bundle-id'
def set_up(self):
return
@staticmethod
def _run(command):
return collections.namedtuple(
'result', ['crashed', 'crashed_test'])(crashed=True,
crashed_test=None)
def tear_down(self):
return
self.mock(test_runner.os.path, 'exists', exists)
self.mock(test_runner.find_xcode, 'find_xcode', find_xcode)
self.mock(test_runner.subprocess, 'check_output', check_output)
self.mock(test_runner.SimulatorTestRunner, 'set_up', set_up)
self.mock(test_runner.TestRunner, '_run', _run)
self.mock(test_runner.SimulatorTestRunner, 'tear_down', tear_down)
tr = test_runner.SimulatorTestRunner(
'fake-app',
'fake-iossim',
'platform',
'os',
'xcode-version',
'out-dir',
)
self.assertRaises(test_runner.AppLaunchError, tr.launch)
def run(self, args):
"""
Main coordinating function.
"""
self.parse_args(args)
# This logic is run by default before the otool command is invoked such that
# otool has the correct Xcode selected for command line dev tools.
if not self.install_xcode(self.args.xcode_build_version,
self.args.mac_toolchain_cmd,
self.args.xcode_path):
raise test_runner.XcodeVersionNotFoundError(
self.args.xcode_build_version)
# GTEST_SHARD_INDEX and GTEST_TOTAL_SHARDS are additional test environment
# variables, set by Swarming, that are only set for a swarming task
# shard count is > 1.
#
# For a given test on a given run, otool should return the same total
# counts and thus, should generate the same sublists. With the shard index,
# each shard would then know the exact test case to run.
gtest_shard_index = os.getenv('GTEST_SHARD_INDEX', 0)
gtest_total_shards = os.getenv('GTEST_TOTAL_SHARDS', 0)
if gtest_shard_index and gtest_total_shards:
self.args.test_cases = shard_util.shard_test_cases(
self.args, gtest_shard_index, gtest_total_shards)
summary = {}
tr = None
if not os.path.exists(self.args.out_dir):
os.makedirs(self.args.out_dir)
try:
if self.args.xcode_parallelization:
tr = xcodebuild_runner.SimulatorParallelTestRunner(
self.args.app,
self.args.host_app,
self.args.iossim,
self.args.version,
self.args.platform,
out_dir=self.args.out_dir,
release=self.args.release,
retries=self.args.retries,
shards=self.args.shards,
test_cases=self.args.test_cases,
test_args=self.test_args,
use_clang_coverage=self.args.use_clang_coverage,
env_vars=self.args.env_var)
elif self.args.replay_path != 'NO_PATH':
tr = wpr_runner.WprProxySimulatorTestRunner(
self.args.app,
self.args.host_app,
self.args.iossim,
self.args.replay_path,
self.args.platform,
self.args.version,
self.args.wpr_tools_path,
self.args.out_dir,
env_vars=self.args.env_var,
retries=self.args.retries,
shards=self.args.shards,
test_args=self.test_args,
test_cases=self.args.test_cases,
xctest=self.args.xctest,
)
elif self.args.iossim and self.args.platform and self.args.version:
tr = test_runner.SimulatorTestRunner(
self.args.app,
self.args.iossim,
self.args.platform,
self.args.version,
self.args.out_dir,
env_vars=self.args.env_var,
retries=self.args.retries,
shards=self.args.shards,
test_args=self.test_args,
test_cases=self.args.test_cases,
use_clang_coverage=self.args.use_clang_coverage,
wpr_tools_path=self.args.wpr_tools_path,
xctest=self.args.xctest,
)
elif self.args.xcodebuild_device_runner and self.args.xctest:
tr = xcodebuild_runner.DeviceXcodeTestRunner(
app_path=self.args.app,
host_app_path=self.args.host_app,
out_dir=self.args.out_dir,
release=self.args.release,
retries=self.args.retries,
test_cases=self.args.test_cases,
test_args=self.test_args,
env_vars=self.args.env_var)
else:
tr = test_runner.DeviceTestRunner(
self.args.app,
self.args.out_dir,
env_vars=self.args.env_var,
restart=self.args.restart,
retries=self.args.retries,
test_args=self.test_args,
test_cases=self.args.test_cases,
xctest=self.args.xctest,
)
return 0 if tr.launch() else 1
except test_runner.TestRunnerError as e:
sys.stderr.write(traceback.format_exc())
summary['step_text'] = '%s%s' % (e.__class__.__name__, ': %s' %
e.args[0] if e.args else '')
# test_runner.Launch returns 0 on success, 1 on failure, so return 2
# on exception to distinguish between a test failure, and a failure
# to launch the test at all.
return 2
finally:
if tr:
summary['logs'] = tr.logs
with open(os.path.join(self.args.out_dir, 'summary.json'),
'w') as f:
json.dump(summary, f)
if tr:
with open(os.path.join(self.args.out_dir, 'full_results.json'),
'w') as f:
json.dump(tr.test_results, f)
# The value of test-launcher-summary-output is set by the recipe
# and passed here via swarming.py. This argument defaults to
# ${ISOLATED_OUTDIR}/output.json. out-dir is set to ${ISOLATED_OUTDIR}
# TODO(crbug.com/1031338) - the content of this output.json will
# work with Chromium recipe because we use the noop_merge merge script,
# but will require structural changes to support the default gtest
# merge script (ref: //testing/merge_scripts/standard_gtest_merge.py)
output_json_path = (self.args.test_launcher_summary_output
or os.path.join(self.args.out_dir,
'output.json'))
with open(output_json_path, 'w') as f:
json.dump(tr.test_results, f)
test_runner.defaults_delete('com.apple.CoreSimulator',
'FramebufferServerRendererPolicy')
def run(self, args):
"""
Main coordinating function.
"""
self.parse_args(args)
summary = {}
tr = None
if not os.path.exists(self.args.out_dir):
os.makedirs(self.args.out_dir)
try:
if self.args.xcode_parallelization:
tr = xcodebuild_runner.SimulatorParallelTestRunner(
self.args.app,
self.args.host_app,
self.args.iossim,
self.args.xcode_build_version,
self.args.version,
self.args.platform,
out_dir=self.args.out_dir,
mac_toolchain=self.args.mac_toolchain_cmd,
retries=self.args.retries,
shards=self.args.shards,
xcode_path=self.args.xcode_path,
test_cases=self.args.test_cases,
test_args=self.test_args,
env_vars=self.args.env_var)
elif self.args.replay_path != 'NO_PATH':
tr = wpr_runner.WprProxySimulatorTestRunner(
self.args.app,
self.args.host_app,
self.args.iossim,
self.args.replay_path,
self.args.platform,
self.args.version,
self.args.wpr_tools_path,
self.args.xcode_build_version,
self.args.out_dir,
env_vars=self.args.env_var,
mac_toolchain=self.args.mac_toolchain_cmd,
retries=self.args.retries,
shards=self.args.shards,
test_args=self.test_args,
test_cases=self.args.test_cases,
xcode_path=self.args.xcode_path,
xctest=self.args.xctest,
)
elif self.args.iossim and self.args.platform and self.args.version:
tr = test_runner.SimulatorTestRunner(
self.args.app,
self.args.iossim,
self.args.platform,
self.args.version,
self.args.xcode_build_version,
self.args.out_dir,
env_vars=self.args.env_var,
mac_toolchain=self.args.mac_toolchain_cmd,
retries=self.args.retries,
shards=self.args.shards,
test_args=self.test_args,
test_cases=self.args.test_cases,
wpr_tools_path=self.args.wpr_tools_path,
xcode_path=self.args.xcode_path,
xctest=self.args.xctest,
)
elif self.args.xcodebuild_device_runner and self.args.xctest:
tr = xcodebuild_runner.DeviceXcodeTestRunner(
app_path=self.args.app,
host_app_path=self.args.host_app,
xcode_build_version=self.args.xcode_build_version,
out_dir=self.args.out_dir,
mac_toolchain=self.args.mac_toolchain_cmd,
retries=self.args.retries,
xcode_path=self.args.xcode_path,
test_cases=self.args.test_cases,
test_args=self.test_args,
env_vars=self.args.env_var)
else:
tr = test_runner.DeviceTestRunner(
self.args.app,
self.args.xcode_build_version,
self.args.out_dir,
env_vars=self.args.env_var,
mac_toolchain=self.args.mac_toolchain_cmd,
restart=self.args.restart,
retries=self.args.retries,
test_args=self.test_args,
test_cases=self.args.test_cases,
xcode_path=self.args.xcode_path,
xctest=self.args.xctest,
)
return 0 if tr.launch() else 1
except test_runner.TestRunnerError as e:
sys.stderr.write(traceback.format_exc())
summary['step_text'] = '%s%s' % (e.__class__.__name__, ': %s' %
e.args[0] if e.args else '')
# test_runner.Launch returns 0 on success, 1 on failure, so return 2
# on exception to distinguish between a test failure, and a failure
# to launch the test at all.
return 2
finally:
if tr:
summary['logs'] = tr.logs
with open(os.path.join(self.args.out_dir, 'summary.json'),
'w') as f:
json.dump(summary, f)
if tr:
with open(os.path.join(self.args.out_dir, 'full_results.json'),
'w') as f:
json.dump(tr.test_results, f)
# The value of test-launcher-summary-output is set by the recipe
# and passed here via swarming.py. This argument defaults to
# ${ISOLATED_OUTDIR}/output.json. out-dir is set to ${ISOLATED_OUTDIR}
# TODO(crbug.com/1031338) - the content of this output.json will
# work with Chromium recipe because we use the noop_merge merge script,
# but will require structural changes to support the default gtest
# merge script (ref: //testing/merge_scripts/standard_gtest_merge.py)
output_json_path = (self.args.test_launcher_summary_output
or os.path.join(self.args.out_dir,
'output.json'))
with open(output_json_path, 'w') as f:
json.dump(tr.test_results, f)
test_runner.defaults_delete('com.apple.CoreSimulator',
'FramebufferServerRendererPolicy')
def test_relaunch(self):
"""Ensures test is relaunched on test crash until tests complete."""
def exists(path):
return True
def find_xcode(version):
return {'found': True}
def check_output(command):
return 'fake-bundle-id'
def set_up(self):
return
@staticmethod
def _run(command):
result = collections.namedtuple(
'result',
[
'crashed',
'crashed_test',
'failed_tests',
'flaked_tests',
'passed_tests',
],
)
if '-e' not in command:
# First run, has no test filter supplied. Mock a crash.
return result(
crashed=True,
crashed_test='c',
failed_tests={
'b': ['b-out'],
'c': ['Did not complete.']
},
flaked_tests={'d': ['d-out']},
passed_tests=['a'],
)
else:
return result(
crashed=False,
crashed_test=None,
failed_tests={},
flaked_tests={},
passed_tests=[],
)
def tear_down(self):
return
self.mock(test_runner.os.path, 'exists', exists)
self.mock(test_runner.find_xcode, 'find_xcode', find_xcode)
self.mock(test_runner.subprocess, 'check_output', check_output)
self.mock(test_runner.SimulatorTestRunner, 'set_up', set_up)
self.mock(test_runner.TestRunner, '_run', _run)
self.mock(test_runner.SimulatorTestRunner, 'tear_down', tear_down)
tr = test_runner.SimulatorTestRunner(
'fake-app',
'fake-iossim',
'platform',
'os',
'xcode-version',
'out-dir',
)
tr.launch()
self.failUnless(tr.summary['logs'])
def main():
args, test_args = parse_args()
summary = {}
tr = None
if not os.path.exists(args.out_dir):
os.makedirs(args.out_dir)
try:
if args.iossim and args.platform and args.version:
tr = test_runner.SimulatorTestRunner(
args.app,
args.iossim,
args.platform,
args.version,
args.xcode_version,
args.xcode_build_version,
args.out_dir,
env_vars=args.env_var,
mac_toolchain=args.mac_toolchain_cmd,
retries=args.retries,
shards=args.shards,
test_args=test_args,
test_cases=args.test_cases,
xcode_path=args.xcode_path,
xctest=args.xctest,
)
else:
tr = test_runner.DeviceTestRunner(
args.app,
args.xcode_version,
args.xcode_build_version,
args.out_dir,
env_vars=args.env_var,
mac_toolchain=args.mac_toolchain_cmd,
restart=args.restart,
retries=args.retries,
test_args=test_args,
test_cases=args.test_cases,
xcode_path=args.xcode_path,
xctest=args.xctest,
)
return 0 if tr.launch() else 1
except test_runner.TestRunnerError as e:
sys.stderr.write(traceback.format_exc())
summary['step_text'] = '%s%s' % (e.__class__.__name__,
': %s' % e.args[0] if e.args else '')
# test_runner.Launch returns 0 on success, 1 on failure, so return 2
# on exception to distinguish between a test failure, and a failure
# to launch the test at all.
return 2
finally:
if tr:
summary['logs'] = tr.logs
with open(os.path.join(args.out_dir, 'summary.json'), 'w') as f:
json.dump(summary, f)
if tr:
with open(os.path.join(args.out_dir, 'full_results.json'),
'w') as f:
json.dump(tr.test_results, f)
def main():
logging.basicConfig(format='[%(asctime)s:%(levelname)s] %(message)s',
level=logging.DEBUG,
datefmt='%I:%M:%S')
args, test_args = parse_args()
summary = {}
tr = None
if not os.path.exists(args.out_dir):
os.makedirs(args.out_dir)
try:
if args.xcode_parallelization:
tr = xcodebuild_runner.SimulatorParallelTestRunner(
args.app,
args.iossim,
args.xcode_build_version,
args.version,
args.platform,
out_dir=args.out_dir,
mac_toolchain=args.mac_toolchain_cmd,
retries=args.retries,
shards=args.shards,
xcode_path=args.xcode_path,
test_cases=args.test_cases,
test_args=test_args,
env_vars=args.env_var)
elif args.replay_path != 'NO_PATH':
tr = test_runner.WprProxySimulatorTestRunner(
args.app,
args.iossim,
args.replay_path,
args.platform,
args.version,
args.wpr_tools_path,
args.xcode_build_version,
args.out_dir,
env_vars=args.env_var,
mac_toolchain=args.mac_toolchain_cmd,
retries=args.retries,
shards=args.shards,
test_args=test_args,
test_cases=args.test_cases,
xcode_path=args.xcode_path,
xctest=args.xctest,
)
elif args.iossim and args.platform and args.version:
tr = test_runner.SimulatorTestRunner(
args.app,
args.iossim,
args.platform,
args.version,
args.xcode_build_version,
args.out_dir,
env_vars=args.env_var,
mac_toolchain=args.mac_toolchain_cmd,
retries=args.retries,
shards=args.shards,
test_args=test_args,
test_cases=args.test_cases,
use_trusted_cert=args.use_trusted_cert,
wpr_tools_path=args.wpr_tools_path,
xcode_path=args.xcode_path,
xctest=args.xctest,
)
else:
tr = test_runner.DeviceTestRunner(
args.app,
args.xcode_build_version,
args.out_dir,
env_vars=args.env_var,
mac_toolchain=args.mac_toolchain_cmd,
restart=args.restart,
retries=args.retries,
test_args=test_args,
test_cases=args.test_cases,
xcode_path=args.xcode_path,
xctest=args.xctest,
)
return 0 if tr.launch() else 1
except test_runner.TestRunnerError as e:
sys.stderr.write(traceback.format_exc())
summary['step_text'] = '%s%s' % (e.__class__.__name__,
': %s' % e.args[0] if e.args else '')
# test_runner.Launch returns 0 on success, 1 on failure, so return 2
# on exception to distinguish between a test failure, and a failure
# to launch the test at all.
return 2
finally:
if tr:
summary['logs'] = tr.logs
with open(os.path.join(args.out_dir, 'summary.json'), 'w') as f:
json.dump(summary, f)
if tr:
with open(os.path.join(args.out_dir, 'full_results.json'),
'w') as f:
json.dump(tr.test_results, f)
def run(self, args):
"""
Main coordinating function.
"""
self.parse_args(args)
summary = {}
tr = None
if not os.path.exists(self.args.out_dir):
os.makedirs(self.args.out_dir)
try:
if self.args.xcode_parallelization:
tr = xcodebuild_runner.SimulatorParallelTestRunner(
self.args.app,
self.args.host_app,
self.args.iossim,
self.args.xcode_build_version,
self.args.version,
self.args.platform,
out_dir=self.args.out_dir,
mac_toolchain=self.args.mac_toolchain_cmd,
retries=self.args.retries,
shards=self.args.shards,
xcode_path=self.args.xcode_path,
test_cases=self.args.test_cases,
test_args=self.test_args,
env_vars=self.args.env_var)
elif self.args.replay_path != 'NO_PATH':
tr = wpr_runner.WprProxySimulatorTestRunner(
self.args.app,
self.args.host_app,
self.args.iossim,
self.args.replay_path,
self.args.platform,
self.args.version,
self.args.wpr_tools_path,
self.args.xcode_build_version,
self.args.out_dir,
env_vars=self.args.env_var,
mac_toolchain=self.args.mac_toolchain_cmd,
retries=self.args.retries,
shards=self.args.shards,
test_args=self.test_args,
test_cases=self.args.test_cases,
xcode_path=self.args.xcode_path,
xctest=self.args.xctest,
)
elif self.args.iossim and self.args.platform and self.args.version:
tr = test_runner.SimulatorTestRunner(
self.args.app,
self.args.iossim,
self.args.platform,
self.args.version,
self.args.xcode_build_version,
self.args.out_dir,
env_vars=self.args.env_var,
mac_toolchain=self.args.mac_toolchain_cmd,
retries=self.args.retries,
shards=self.args.shards,
test_args=self.test_args,
test_cases=self.args.test_cases,
wpr_tools_path=self.args.wpr_tools_path,
xcode_path=self.args.xcode_path,
xctest=self.args.xctest,
)
elif self.args.xcodebuild_device_runner and self.args.xctest:
tr = xcodebuild_runner.DeviceXcodeTestRunner(
app_path=self.args.app,
host_app_path=self.args.host_app,
xcode_build_version=self.args.xcode_build_version,
out_dir=self.args.out_dir,
mac_toolchain=self.args.mac_toolchain_cmd,
retries=self.args.retries,
xcode_path=self.args.xcode_path,
test_cases=self.args.test_cases,
test_args=self.test_args,
env_vars=self.args.env_var)
else:
tr = test_runner.DeviceTestRunner(
self.args.app,
self.args.xcode_build_version,
self.args.out_dir,
env_vars=self.args.env_var,
mac_toolchain=self.args.mac_toolchain_cmd,
restart=self.args.restart,
retries=self.args.retries,
test_args=self.test_args,
test_cases=self.args.test_cases,
xcode_path=self.args.xcode_path,
xctest=self.args.xctest,
)
return 0 if tr.launch() else 1
except test_runner.TestRunnerError as e:
sys.stderr.write(traceback.format_exc())
summary['step_text'] = '%s%s' % (e.__class__.__name__,
': %s' % e.args[0] if e.args else '')
# test_runner.Launch returns 0 on success, 1 on failure, so return 2
# on exception to distinguish between a test failure, and a failure
# to launch the test at all.
return 2
finally:
if tr:
summary['logs'] = tr.logs
with open(os.path.join(self.args.out_dir, 'summary.json'), 'w') as f:
json.dump(summary, f)
if tr:
with open(os.path.join(self.args.out_dir, 'full_results.json'),
'w') as f:
json.dump(tr.test_results, f)
test_runner.defaults_delete('com.apple.CoreSimulator',
'FramebufferServerRendererPolicy')
def run(self, args):
"""
Main coordinating function.
"""
self.parse_args(args)
# This logic is run by default before the otool command is invoked such that
# otool has the correct Xcode selected for command line dev tools.
install_success, is_legacy_xcode = self.install_xcode()
if not install_success:
raise test_runner.XcodeVersionNotFoundError(
self.args.xcode_build_version)
self.resolve_test_cases()
summary = {}
tr = None
if not os.path.exists(self.args.out_dir):
os.makedirs(self.args.out_dir)
try:
if self.args.xcode_parallelization:
tr = xcodebuild_runner.SimulatorParallelTestRunner(
self.args.app,
self.args.host_app,
self.args.iossim,
self.args.version,
self.args.platform,
out_dir=self.args.out_dir,
release=self.args.release,
repeat_count=self.args.gtest_repeat,
retries=self.args.retries,
shards=self.args.shards,
test_cases=self.args.test_cases,
test_args=self.test_args,
use_clang_coverage=self.args.use_clang_coverage,
env_vars=self.args.env_var)
elif self.args.variations_seed_path != 'NO_PATH':
tr = variations_runner.VariationsSimulatorParallelTestRunner(
self.args.app,
self.args.host_app,
self.args.iossim,
self.args.version,
self.args.platform,
self.args.out_dir,
self.args.variations_seed_path,
release=self.args.release,
test_cases=self.args.test_cases,
test_args=self.test_args,
env_vars=self.args.env_var)
elif self.args.replay_path != 'NO_PATH':
tr = wpr_runner.WprProxySimulatorTestRunner(
self.args.app,
self.args.host_app,
self.args.iossim,
self.args.replay_path,
self.args.platform,
self.args.version,
self.args.wpr_tools_path,
self.args.out_dir,
env_vars=self.args.env_var,
retries=self.args.retries,
shards=self.args.shards,
test_args=self.test_args,
test_cases=self.args.test_cases,
xctest=self.args.xctest,
)
elif self.args.iossim and self.args.platform and self.args.version:
tr = test_runner.SimulatorTestRunner(
self.args.app,
self.args.iossim,
self.args.platform,
self.args.version,
self.args.out_dir,
env_vars=self.args.env_var,
repeat_count=self.args.gtest_repeat,
retries=self.args.retries,
shards=self.args.shards,
test_args=self.test_args,
test_cases=self.args.test_cases,
use_clang_coverage=self.args.use_clang_coverage,
wpr_tools_path=self.args.wpr_tools_path,
xctest=self.args.xctest,
)
elif self.args.xcodebuild_device_runner and self.args.xctest:
tr = xcodebuild_runner.DeviceXcodeTestRunner(
app_path=self.args.app,
host_app_path=self.args.host_app,
out_dir=self.args.out_dir,
release=self.args.release,
repeat_count=self.args.gtest_repeat,
retries=self.args.retries,
test_cases=self.args.test_cases,
test_args=self.test_args,
env_vars=self.args.env_var)
else:
tr = test_runner.DeviceTestRunner(
self.args.app,
self.args.out_dir,
env_vars=self.args.env_var,
repeat_count=self.args.gtest_repeat,
restart=self.args.restart,
retries=self.args.retries,
test_args=self.test_args,
test_cases=self.args.test_cases,
xctest=self.args.xctest,
)
logging.info("Using test runner %s" % type(tr).__name__)
return 0 if tr.launch() else 1
except test_runner.DeviceError as e:
sys.stderr.write(traceback.format_exc())
summary['step_text'] = '%s%s' % (e.__class__.__name__, ': %s' %
e.args[0] if e.args else '')
# Swarming infra marks device status unavailable for any device related
# issue using this return code.
return 3
except test_runner.TestRunnerError as e:
sys.stderr.write(traceback.format_exc())
summary['step_text'] = '%s%s' % (e.__class__.__name__, ': %s' %
e.args[0] if e.args else '')
# test_runner.Launch returns 0 on success, 1 on failure, so return 2
# on exception to distinguish between a test failure, and a failure
# to launch the test at all.
return 2
finally:
if tr:
summary['logs'] = tr.logs
with open(os.path.join(self.args.out_dir, 'summary.json'),
'w') as f:
json.dump(summary, f)
if tr:
with open(os.path.join(self.args.out_dir, 'full_results.json'),
'w') as f:
json.dump(tr.test_results, f)
# The value of test-launcher-summary-output is set by the recipe
# and passed here via swarming.py. This argument defaults to
# ${ISOLATED_OUTDIR}/output.json. out-dir is set to ${ISOLATED_OUTDIR}
# TODO(crbug.com/1031338) - the content of this output.json will
# work with Chromium recipe because we use the noop_merge merge script,
# but will require structural changes to support the default gtest
# merge script (ref: //testing/merge_scripts/standard_gtest_merge.py)
output_json_path = (self.args.test_launcher_summary_output
or os.path.join(self.args.out_dir,
'output.json'))
with open(output_json_path, 'w') as f:
json.dump(tr.test_results, f)
# Move the iOS runtime back to cache dir if the Xcode package is not
# legacy (i.e. Xcode program & runtimes are in different CIPD packages.)
# and it's a simulator task.
if not is_legacy_xcode and self.args.version:
runtime_cache_folder = xcode.construct_runtime_cache_folder(
self.args.runtime_cache_prefix, self.args.version)
xcode.move_runtime(runtime_cache_folder, self.args.xcode_path,
False)
test_runner.defaults_delete('com.apple.CoreSimulator',
'FramebufferServerRendererPolicy')
