def setUp(self):
super(DeviceTestRunnerTest, self).setUp()
def install_xcode(build, mac_toolchain_cmd, xcode_app_path):
return True
self.mock(
test_runner, 'get_current_xcode_info', lambda: {
'version': 'test version',
'build': 'test build',
'path': 'test/path'
})
self.mock(test_runner, 'install_xcode', install_xcode)
self.mock(test_runner.subprocess, 'check_output',
lambda _: 'fake-bundle-id')
self.mock(os.path, 'abspath', lambda path: '/abs/path/to/%s' % path)
self.mock(os.path, 'exists', lambda _: True)
self.mock(os, 'listdir', lambda _: [])
self.mock(tempfile, 'mkstemp', lambda: '/tmp/tmp_file')
self.tr = test_runner.DeviceTestRunner(
'fake-app',
'xcode-version',
'xcode-build',
'out-dir',
)
self.tr.xctestrun_data = {'TestTargetName': {}}
def main(args, test_args):
summary = {}
tr = None
if not os.path.exists(args.out_dir):
os.makedirs(args.out_dir)
try:
if args.iossim and args.platform and args.version:
tr = test_runner.SimulatorTestRunner(
args.app,
args.iossim,
args.platform,
args.version,
args.xcode_version,
args.out_dir,
env_vars=args.env_var,
retries=args.retries,
test_args=test_args,
xctest=args.xctest,
)
else:
tr = test_runner.DeviceTestRunner(
args.app,
args.xcode_version,
args.out_dir,
env_vars=args.env_var,
restart=args.restart,
retries=args.retries,
test_args=test_args,
xctest=args.xctest,
)
return 0 if tr.launch() else 1
except test_runner.TestRunnerError as e:
sys.stderr.write(traceback.format_exc())
summary['step_text'] = '%s%s' % (e.__class__.__name__,
': %s' % e.args[0] if e.args else '')
# test_runner.Launch returns 0 on success, 1 on failure, so return 2
# on exception to distinguish between a test failure, and a failure
# to launch the test at all.
return 2
finally:
if tr:
summary['logs'] = tr.logs
with open(os.path.join(args.out_dir, 'summary.json'), 'w') as f:
json.dump(summary, f)
if tr:
with open(os.path.join(args.out_dir, 'full_results.json'),
'w') as f:
json.dump(tr.test_results, f)
def main():
logging.basicConfig(format='[%(asctime)s:%(levelname)s] %(message)s',
level=logging.DEBUG,
datefmt='%I:%M:%S')
args, test_args = parse_args()
summary = {}
tr = None
if not os.path.exists(args.out_dir):
os.makedirs(args.out_dir)
try:
if args.xcode_parallelization:
tr = xcodebuild_runner.SimulatorParallelTestRunner(
args.app,
args.iossim,
args.xcode_build_version,
args.version,
args.platform,
out_dir=args.out_dir,
mac_toolchain=args.mac_toolchain_cmd,
retries=args.retries,
shards=args.shards,
xcode_path=args.xcode_path,
test_cases=args.test_cases,
test_args=test_args,
env_vars=args.env_var)
elif args.replay_path != 'NO_PATH':
tr = test_runner.WprProxySimulatorTestRunner(
args.app,
args.iossim,
args.replay_path,
args.platform,
args.version,
args.wpr_tools_path,
args.xcode_build_version,
args.out_dir,
env_vars=args.env_var,
mac_toolchain=args.mac_toolchain_cmd,
retries=args.retries,
shards=args.shards,
test_args=test_args,
test_cases=args.test_cases,
xcode_path=args.xcode_path,
xctest=args.xctest,
)
elif args.iossim and args.platform and args.version:
tr = test_runner.SimulatorTestRunner(
args.app,
args.iossim,
args.platform,
args.version,
args.xcode_build_version,
args.out_dir,
env_vars=args.env_var,
mac_toolchain=args.mac_toolchain_cmd,
retries=args.retries,
shards=args.shards,
test_args=test_args,
test_cases=args.test_cases,
use_trusted_cert=args.use_trusted_cert,
wpr_tools_path=args.wpr_tools_path,
xcode_path=args.xcode_path,
xctest=args.xctest,
)
else:
tr = test_runner.DeviceTestRunner(
args.app,
args.xcode_build_version,
args.out_dir,
env_vars=args.env_var,
mac_toolchain=args.mac_toolchain_cmd,
restart=args.restart,
retries=args.retries,
test_args=test_args,
test_cases=args.test_cases,
xcode_path=args.xcode_path,
xctest=args.xctest,
)
return 0 if tr.launch() else 1
except test_runner.TestRunnerError as e:
sys.stderr.write(traceback.format_exc())
summary['step_text'] = '%s%s' % (e.__class__.__name__,
': %s' % e.args[0] if e.args else '')
# test_runner.Launch returns 0 on success, 1 on failure, so return 2
# on exception to distinguish between a test failure, and a failure
# to launch the test at all.
return 2
finally:
if tr:
summary['logs'] = tr.logs
with open(os.path.join(args.out_dir, 'summary.json'), 'w') as f:
json.dump(summary, f)
if tr:
with open(os.path.join(args.out_dir, 'full_results.json'),
'w') as f:
json.dump(tr.test_results, f)
def run(self, args):
"""
Main coordinating function.
"""
self.parse_args(args)
# This logic is run by default before the otool command is invoked such that
# otool has the correct Xcode selected for command line dev tools.
if not self.install_xcode(self.args.xcode_build_version,
self.args.mac_toolchain_cmd,
self.args.xcode_path):
raise test_runner.XcodeVersionNotFoundError(
self.args.xcode_build_version)
# GTEST_SHARD_INDEX and GTEST_TOTAL_SHARDS are additional test environment
# variables, set by Swarming, that are only set for a swarming task
# shard count is > 1.
#
# For a given test on a given run, otool should return the same total
# counts and thus, should generate the same sublists. With the shard index,
# each shard would then know the exact test case to run.
gtest_shard_index = os.getenv('GTEST_SHARD_INDEX', 0)
gtest_total_shards = os.getenv('GTEST_TOTAL_SHARDS', 0)
if gtest_shard_index and gtest_total_shards:
self.args.test_cases = shard_util.shard_test_cases(
self.args, gtest_shard_index, gtest_total_shards)
summary = {}
tr = None
if not os.path.exists(self.args.out_dir):
os.makedirs(self.args.out_dir)
try:
if self.args.xcode_parallelization:
tr = xcodebuild_runner.SimulatorParallelTestRunner(
self.args.app,
self.args.host_app,
self.args.iossim,
self.args.version,
self.args.platform,
out_dir=self.args.out_dir,
release=self.args.release,
retries=self.args.retries,
shards=self.args.shards,
test_cases=self.args.test_cases,
test_args=self.test_args,
use_clang_coverage=self.args.use_clang_coverage,
env_vars=self.args.env_var)
elif self.args.replay_path != 'NO_PATH':
tr = wpr_runner.WprProxySimulatorTestRunner(
self.args.app,
self.args.host_app,
self.args.iossim,
self.args.replay_path,
self.args.platform,
self.args.version,
self.args.wpr_tools_path,
self.args.out_dir,
env_vars=self.args.env_var,
retries=self.args.retries,
shards=self.args.shards,
test_args=self.test_args,
test_cases=self.args.test_cases,
xctest=self.args.xctest,
)
elif self.args.iossim and self.args.platform and self.args.version:
tr = test_runner.SimulatorTestRunner(
self.args.app,
self.args.iossim,
self.args.platform,
self.args.version,
self.args.out_dir,
env_vars=self.args.env_var,
retries=self.args.retries,
shards=self.args.shards,
test_args=self.test_args,
test_cases=self.args.test_cases,
use_clang_coverage=self.args.use_clang_coverage,
wpr_tools_path=self.args.wpr_tools_path,
xctest=self.args.xctest,
)
elif self.args.xcodebuild_device_runner and self.args.xctest:
tr = xcodebuild_runner.DeviceXcodeTestRunner(
app_path=self.args.app,
host_app_path=self.args.host_app,
out_dir=self.args.out_dir,
release=self.args.release,
retries=self.args.retries,
test_cases=self.args.test_cases,
test_args=self.test_args,
env_vars=self.args.env_var)
else:
tr = test_runner.DeviceTestRunner(
self.args.app,
self.args.out_dir,
env_vars=self.args.env_var,
restart=self.args.restart,
retries=self.args.retries,
test_args=self.test_args,
test_cases=self.args.test_cases,
xctest=self.args.xctest,
)
return 0 if tr.launch() else 1
except test_runner.TestRunnerError as e:
sys.stderr.write(traceback.format_exc())
summary['step_text'] = '%s%s' % (e.__class__.__name__, ': %s' %
e.args[0] if e.args else '')
# test_runner.Launch returns 0 on success, 1 on failure, so return 2
# on exception to distinguish between a test failure, and a failure
# to launch the test at all.
return 2
finally:
if tr:
summary['logs'] = tr.logs
with open(os.path.join(self.args.out_dir, 'summary.json'),
'w') as f:
json.dump(summary, f)
if tr:
with open(os.path.join(self.args.out_dir, 'full_results.json'),
'w') as f:
json.dump(tr.test_results, f)
# The value of test-launcher-summary-output is set by the recipe
# and passed here via swarming.py. This argument defaults to
# ${ISOLATED_OUTDIR}/output.json. out-dir is set to ${ISOLATED_OUTDIR}
# TODO(crbug.com/1031338) - the content of this output.json will
# work with Chromium recipe because we use the noop_merge merge script,
# but will require structural changes to support the default gtest
# merge script (ref: //testing/merge_scripts/standard_gtest_merge.py)
output_json_path = (self.args.test_launcher_summary_output
or os.path.join(self.args.out_dir,
'output.json'))
with open(output_json_path, 'w') as f:
json.dump(tr.test_results, f)
test_runner.defaults_delete('com.apple.CoreSimulator',
'FramebufferServerRendererPolicy')
def run(self, args):
"""
Main coordinating function.
"""
self.parse_args(args)
# This logic is run by default before the otool command is invoked such that
# otool has the correct Xcode selected for command line dev tools.
install_success, is_legacy_xcode = self.install_xcode()
if not install_success:
raise test_runner.XcodeVersionNotFoundError(
self.args.xcode_build_version)
self.resolve_test_cases()
summary = {}
tr = None
if not os.path.exists(self.args.out_dir):
os.makedirs(self.args.out_dir)
try:
if self.args.xcode_parallelization:
tr = xcodebuild_runner.SimulatorParallelTestRunner(
self.args.app,
self.args.host_app,
self.args.iossim,
self.args.version,
self.args.platform,
out_dir=self.args.out_dir,
release=self.args.release,
repeat_count=self.args.gtest_repeat,
retries=self.args.retries,
shards=self.args.shards,
test_cases=self.args.test_cases,
test_args=self.test_args,
use_clang_coverage=self.args.use_clang_coverage,
env_vars=self.args.env_var)
elif self.args.variations_seed_path != 'NO_PATH':
tr = variations_runner.VariationsSimulatorParallelTestRunner(
self.args.app,
self.args.host_app,
self.args.iossim,
self.args.version,
self.args.platform,
self.args.out_dir,
self.args.variations_seed_path,
release=self.args.release,
test_cases=self.args.test_cases,
test_args=self.test_args,
env_vars=self.args.env_var)
elif self.args.replay_path != 'NO_PATH':
tr = wpr_runner.WprProxySimulatorTestRunner(
self.args.app,
self.args.host_app,
self.args.iossim,
self.args.replay_path,
self.args.platform,
self.args.version,
self.args.wpr_tools_path,
self.args.out_dir,
env_vars=self.args.env_var,
retries=self.args.retries,
shards=self.args.shards,
test_args=self.test_args,
test_cases=self.args.test_cases,
xctest=self.args.xctest,
)
elif self.args.iossim and self.args.platform and self.args.version:
tr = test_runner.SimulatorTestRunner(
self.args.app,
self.args.iossim,
self.args.platform,
self.args.version,
self.args.out_dir,
env_vars=self.args.env_var,
repeat_count=self.args.gtest_repeat,
retries=self.args.retries,
shards=self.args.shards,
test_args=self.test_args,
test_cases=self.args.test_cases,
use_clang_coverage=self.args.use_clang_coverage,
wpr_tools_path=self.args.wpr_tools_path,
xctest=self.args.xctest,
)
elif self.args.xcodebuild_device_runner and self.args.xctest:
tr = xcodebuild_runner.DeviceXcodeTestRunner(
app_path=self.args.app,
host_app_path=self.args.host_app,
out_dir=self.args.out_dir,
release=self.args.release,
repeat_count=self.args.gtest_repeat,
retries=self.args.retries,
test_cases=self.args.test_cases,
test_args=self.test_args,
env_vars=self.args.env_var)
else:
tr = test_runner.DeviceTestRunner(
self.args.app,
self.args.out_dir,
env_vars=self.args.env_var,
repeat_count=self.args.gtest_repeat,
restart=self.args.restart,
retries=self.args.retries,
test_args=self.test_args,
test_cases=self.args.test_cases,
xctest=self.args.xctest,
)
logging.info("Using test runner %s" % type(tr).__name__)
return 0 if tr.launch() else 1
except test_runner.DeviceError as e:
sys.stderr.write(traceback.format_exc())
summary['step_text'] = '%s%s' % (e.__class__.__name__, ': %s' %
e.args[0] if e.args else '')
# Swarming infra marks device status unavailable for any device related
# issue using this return code.
return 3
except test_runner.TestRunnerError as e:
sys.stderr.write(traceback.format_exc())
summary['step_text'] = '%s%s' % (e.__class__.__name__, ': %s' %
e.args[0] if e.args else '')
# test_runner.Launch returns 0 on success, 1 on failure, so return 2
# on exception to distinguish between a test failure, and a failure
# to launch the test at all.
return 2
finally:
if tr:
summary['logs'] = tr.logs
with open(os.path.join(self.args.out_dir, 'summary.json'),
'w') as f:
json.dump(summary, f)
if tr:
with open(os.path.join(self.args.out_dir, 'full_results.json'),
'w') as f:
json.dump(tr.test_results, f)
# The value of test-launcher-summary-output is set by the recipe
# and passed here via swarming.py. This argument defaults to
# ${ISOLATED_OUTDIR}/output.json. out-dir is set to ${ISOLATED_OUTDIR}
# TODO(crbug.com/1031338) - the content of this output.json will
# work with Chromium recipe because we use the noop_merge merge script,
# but will require structural changes to support the default gtest
# merge script (ref: //testing/merge_scripts/standard_gtest_merge.py)
output_json_path = (self.args.test_launcher_summary_output
or os.path.join(self.args.out_dir,
'output.json'))
with open(output_json_path, 'w') as f:
json.dump(tr.test_results, f)
# Move the iOS runtime back to cache dir if the Xcode package is not
# legacy (i.e. Xcode program & runtimes are in different CIPD packages.)
# and it's a simulator task.
if not is_legacy_xcode and self.args.version:
runtime_cache_folder = xcode.construct_runtime_cache_folder(
self.args.runtime_cache_prefix, self.args.version)
xcode.move_runtime(runtime_cache_folder, self.args.xcode_path,
False)
test_runner.defaults_delete('com.apple.CoreSimulator',
'FramebufferServerRendererPolicy')
def run(self, args):
"""
Main coordinating function.
"""
self.parse_args(args)
summary = {}
tr = None
if not os.path.exists(self.args.out_dir):
os.makedirs(self.args.out_dir)
try:
if self.args.xcode_parallelization:
tr = xcodebuild_runner.SimulatorParallelTestRunner(
self.args.app,
self.args.host_app,
self.args.iossim,
self.args.xcode_build_version,
self.args.version,
self.args.platform,
out_dir=self.args.out_dir,
mac_toolchain=self.args.mac_toolchain_cmd,
retries=self.args.retries,
shards=self.args.shards,
xcode_path=self.args.xcode_path,
test_cases=self.args.test_cases,
test_args=self.test_args,
env_vars=self.args.env_var)
elif self.args.replay_path != 'NO_PATH':
tr = wpr_runner.WprProxySimulatorTestRunner(
self.args.app,
self.args.host_app,
self.args.iossim,
self.args.replay_path,
self.args.platform,
self.args.version,
self.args.wpr_tools_path,
self.args.xcode_build_version,
self.args.out_dir,
env_vars=self.args.env_var,
mac_toolchain=self.args.mac_toolchain_cmd,
retries=self.args.retries,
shards=self.args.shards,
test_args=self.test_args,
test_cases=self.args.test_cases,
xcode_path=self.args.xcode_path,
xctest=self.args.xctest,
)
elif self.args.iossim and self.args.platform and self.args.version:
tr = test_runner.SimulatorTestRunner(
self.args.app,
self.args.iossim,
self.args.platform,
self.args.version,
self.args.xcode_build_version,
self.args.out_dir,
env_vars=self.args.env_var,
mac_toolchain=self.args.mac_toolchain_cmd,
retries=self.args.retries,
shards=self.args.shards,
test_args=self.test_args,
test_cases=self.args.test_cases,
wpr_tools_path=self.args.wpr_tools_path,
xcode_path=self.args.xcode_path,
xctest=self.args.xctest,
)
elif self.args.xcodebuild_device_runner and self.args.xctest:
tr = xcodebuild_runner.DeviceXcodeTestRunner(
app_path=self.args.app,
host_app_path=self.args.host_app,
xcode_build_version=self.args.xcode_build_version,
out_dir=self.args.out_dir,
mac_toolchain=self.args.mac_toolchain_cmd,
retries=self.args.retries,
xcode_path=self.args.xcode_path,
test_cases=self.args.test_cases,
test_args=self.test_args,
env_vars=self.args.env_var)
else:
tr = test_runner.DeviceTestRunner(
self.args.app,
self.args.xcode_build_version,
self.args.out_dir,
env_vars=self.args.env_var,
mac_toolchain=self.args.mac_toolchain_cmd,
restart=self.args.restart,
retries=self.args.retries,
test_args=self.test_args,
test_cases=self.args.test_cases,
xcode_path=self.args.xcode_path,
xctest=self.args.xctest,
)
return 0 if tr.launch() else 1
except test_runner.TestRunnerError as e:
sys.stderr.write(traceback.format_exc())
summary['step_text'] = '%s%s' % (e.__class__.__name__, ': %s' %
e.args[0] if e.args else '')
# test_runner.Launch returns 0 on success, 1 on failure, so return 2
# on exception to distinguish between a test failure, and a failure
# to launch the test at all.
return 2
finally:
if tr:
summary['logs'] = tr.logs
with open(os.path.join(self.args.out_dir, 'summary.json'),
'w') as f:
json.dump(summary, f)
if tr:
with open(os.path.join(self.args.out_dir, 'full_results.json'),
'w') as f:
json.dump(tr.test_results, f)
# The value of test-launcher-summary-output is set by the recipe
# and passed here via swarming.py. This argument defaults to
# ${ISOLATED_OUTDIR}/output.json. out-dir is set to ${ISOLATED_OUTDIR}
# TODO(crbug.com/1031338) - the content of this output.json will
# work with Chromium recipe because we use the noop_merge merge script,
# but will require structural changes to support the default gtest
# merge script (ref: //testing/merge_scripts/standard_gtest_merge.py)
output_json_path = (self.args.test_launcher_summary_output
or os.path.join(self.args.out_dir,
'output.json'))
with open(output_json_path, 'w') as f:
json.dump(tr.test_results, f)
test_runner.defaults_delete('com.apple.CoreSimulator',
'FramebufferServerRendererPolicy')
def run(self, args):
"""
Main coordinating function.
"""
self.parse_args(args)
summary = {}
tr = None
if not os.path.exists(self.args.out_dir):
os.makedirs(self.args.out_dir)
try:
if self.args.xcode_parallelization:
tr = xcodebuild_runner.SimulatorParallelTestRunner(
self.args.app,
self.args.host_app,
self.args.iossim,
self.args.xcode_build_version,
self.args.version,
self.args.platform,
out_dir=self.args.out_dir,
mac_toolchain=self.args.mac_toolchain_cmd,
retries=self.args.retries,
shards=self.args.shards,
xcode_path=self.args.xcode_path,
test_cases=self.args.test_cases,
test_args=self.test_args,
env_vars=self.args.env_var)
elif self.args.replay_path != 'NO_PATH':
tr = wpr_runner.WprProxySimulatorTestRunner(
self.args.app,
self.args.host_app,
self.args.iossim,
self.args.replay_path,
self.args.platform,
self.args.version,
self.args.wpr_tools_path,
self.args.xcode_build_version,
self.args.out_dir,
env_vars=self.args.env_var,
mac_toolchain=self.args.mac_toolchain_cmd,
retries=self.args.retries,
shards=self.args.shards,
test_args=self.test_args,
test_cases=self.args.test_cases,
xcode_path=self.args.xcode_path,
xctest=self.args.xctest,
)
elif self.args.iossim and self.args.platform and self.args.version:
tr = test_runner.SimulatorTestRunner(
self.args.app,
self.args.iossim,
self.args.platform,
self.args.version,
self.args.xcode_build_version,
self.args.out_dir,
env_vars=self.args.env_var,
mac_toolchain=self.args.mac_toolchain_cmd,
retries=self.args.retries,
shards=self.args.shards,
test_args=self.test_args,
test_cases=self.args.test_cases,
wpr_tools_path=self.args.wpr_tools_path,
xcode_path=self.args.xcode_path,
xctest=self.args.xctest,
)
elif self.args.xcodebuild_device_runner and self.args.xctest:
tr = xcodebuild_runner.DeviceXcodeTestRunner(
app_path=self.args.app,
host_app_path=self.args.host_app,
xcode_build_version=self.args.xcode_build_version,
out_dir=self.args.out_dir,
mac_toolchain=self.args.mac_toolchain_cmd,
retries=self.args.retries,
xcode_path=self.args.xcode_path,
test_cases=self.args.test_cases,
test_args=self.test_args,
env_vars=self.args.env_var)
else:
tr = test_runner.DeviceTestRunner(
self.args.app,
self.args.xcode_build_version,
self.args.out_dir,
env_vars=self.args.env_var,
mac_toolchain=self.args.mac_toolchain_cmd,
restart=self.args.restart,
retries=self.args.retries,
test_args=self.test_args,
test_cases=self.args.test_cases,
xcode_path=self.args.xcode_path,
xctest=self.args.xctest,
)
return 0 if tr.launch() else 1
except test_runner.TestRunnerError as e:
sys.stderr.write(traceback.format_exc())
summary['step_text'] = '%s%s' % (e.__class__.__name__,
': %s' % e.args[0] if e.args else '')
# test_runner.Launch returns 0 on success, 1 on failure, so return 2
# on exception to distinguish between a test failure, and a failure
# to launch the test at all.
return 2
finally:
if tr:
summary['logs'] = tr.logs
with open(os.path.join(self.args.out_dir, 'summary.json'), 'w') as f:
json.dump(summary, f)
if tr:
with open(os.path.join(self.args.out_dir, 'full_results.json'),
'w') as f:
json.dump(tr.test_results, f)
test_runner.defaults_delete('com.apple.CoreSimulator',
'FramebufferServerRendererPolicy')
