def test_init(self):
"""Ensures instance is created."""
tr = wpr_runner.WprProxySimulatorTestRunner(
'fake-app',
'fake-host-app',
'fake-iossim',
'replay-path',
'platform',
'os',
'wpr-tools-path',
'out-dir',
)
self.assertTrue(tr)
def test_wpr_tools_not_found(self):
"""Ensures WprToolsNotFoundError is raised."""
self.mock(os.path, 'exists', lambda p: not p.endswith('bad-tools-path'))
with self.assertRaises(wpr_runner.WprToolsNotFoundError):
wpr_runner.WprProxySimulatorTestRunner(
'fake-app',
'fake-host-app',
'fake-iossim',
'replay-path',
'platform',
'os',
'bad-tools-path',
'out-dir',
)
def test_replay_path_not_found(self):
"""Ensures ReplayPathNotFoundError is raised."""
self.mock(os.path, 'exists', lambda p: not p.endswith('bad-replay-path'))
with self.assertRaises(wpr_runner.ReplayPathNotFoundError):
wpr_runner.WprProxySimulatorTestRunner(
'fake-app',
'fake-host-app',
'fake-iossim',
'bad-replay-path',
'platform',
'os',
'wpr-tools-path',
'xcode-version',
'xcode-build',
'out-dir',
)
def run_wpr_test(self, test_filter=[], invert=False):
"""Wrapper that mocks the _run method and returns its result."""
class FakeStdout:
def __init__(self):
self.line_index = 0
self.lines = [
'Test Case \'-[a 1]\' started.',
'Test Case \'-[a 1]\' has uninteresting logs.',
'Test Case \'-[a 1]\' passed (0.1 seconds)',
'Test Case \'-[b 2]\' started.',
'Test Case \'-[b 2]\' passed (0.1 seconds)',
'Test Case \'-[c 3]\' started.',
'Test Case \'-[c 3]\' has interesting failure info.',
'Test Case \'-[c 3]\' failed (0.1 seconds)',
]
def readline(self):
if self.line_index < len(self.lines):
return_line = self.lines[self.line_index]
self.line_index += 1
return return_line
else:
return None
class FakeProcess:
def __init__(self):
self.stdout = FakeStdout()
self.returncode = 0
def stdout(self):
return self.stdout
def wait(self):
return
def popen(recipe_cmd, env, stdout, stderr):
return FakeProcess()
tr = wpr_runner.WprProxySimulatorTestRunner(
'fake-app',
'fake-host-app',
'fake-iossim',
'replay-path',
'platform',
'os',
'wpr-tools-path',
'out-dir',
)
self.mock(wpr_runner.WprProxySimulatorTestRunner, 'wprgo_start',
lambda a, b: None)
self.mock(wpr_runner.WprProxySimulatorTestRunner, 'wprgo_stop',
lambda _: None)
self.mock(wpr_runner.WprProxySimulatorTestRunner, 'get_wpr_test_command',
lambda a, b, c: ["command", "arg"])
self.mock(os.path, 'isfile', lambda _: True)
self.mock(glob, 'glob', lambda _: ["file1", "file2"])
self.mock(subprocess, 'Popen', popen)
tr.xctest_path = 'fake.xctest'
cmd = {'invert': invert, 'test_filter': test_filter}
return tr._run(cmd=cmd, shards=1)
def run(self, args):
"""
Main coordinating function.
"""
self.parse_args(args)
# This logic is run by default before the otool command is invoked such that
# otool has the correct Xcode selected for command line dev tools.
if not self.install_xcode(self.args.xcode_build_version,
self.args.mac_toolchain_cmd,
self.args.xcode_path):
raise test_runner.XcodeVersionNotFoundError(
self.args.xcode_build_version)
# GTEST_SHARD_INDEX and GTEST_TOTAL_SHARDS are additional test environment
# variables, set by Swarming, that are only set for a swarming task
# shard count is > 1.
#
# For a given test on a given run, otool should return the same total
# counts and thus, should generate the same sublists. With the shard index,
# each shard would then know the exact test case to run.
gtest_shard_index = os.getenv('GTEST_SHARD_INDEX', 0)
gtest_total_shards = os.getenv('GTEST_TOTAL_SHARDS', 0)
if gtest_shard_index and gtest_total_shards:
self.args.test_cases = shard_util.shard_test_cases(
self.args, gtest_shard_index, gtest_total_shards)
summary = {}
tr = None
if not os.path.exists(self.args.out_dir):
os.makedirs(self.args.out_dir)
try:
if self.args.xcode_parallelization:
tr = xcodebuild_runner.SimulatorParallelTestRunner(
self.args.app,
self.args.host_app,
self.args.iossim,
self.args.version,
self.args.platform,
out_dir=self.args.out_dir,
release=self.args.release,
retries=self.args.retries,
shards=self.args.shards,
test_cases=self.args.test_cases,
test_args=self.test_args,
use_clang_coverage=self.args.use_clang_coverage,
env_vars=self.args.env_var)
elif self.args.replay_path != 'NO_PATH':
tr = wpr_runner.WprProxySimulatorTestRunner(
self.args.app,
self.args.host_app,
self.args.iossim,
self.args.replay_path,
self.args.platform,
self.args.version,
self.args.wpr_tools_path,
self.args.out_dir,
env_vars=self.args.env_var,
retries=self.args.retries,
shards=self.args.shards,
test_args=self.test_args,
test_cases=self.args.test_cases,
xctest=self.args.xctest,
)
elif self.args.iossim and self.args.platform and self.args.version:
tr = test_runner.SimulatorTestRunner(
self.args.app,
self.args.iossim,
self.args.platform,
self.args.version,
self.args.out_dir,
env_vars=self.args.env_var,
retries=self.args.retries,
shards=self.args.shards,
test_args=self.test_args,
test_cases=self.args.test_cases,
use_clang_coverage=self.args.use_clang_coverage,
wpr_tools_path=self.args.wpr_tools_path,
xctest=self.args.xctest,
)
elif self.args.xcodebuild_device_runner and self.args.xctest:
tr = xcodebuild_runner.DeviceXcodeTestRunner(
app_path=self.args.app,
host_app_path=self.args.host_app,
out_dir=self.args.out_dir,
release=self.args.release,
retries=self.args.retries,
test_cases=self.args.test_cases,
test_args=self.test_args,
env_vars=self.args.env_var)
else:
tr = test_runner.DeviceTestRunner(
self.args.app,
self.args.out_dir,
env_vars=self.args.env_var,
restart=self.args.restart,
retries=self.args.retries,
test_args=self.test_args,
test_cases=self.args.test_cases,
xctest=self.args.xctest,
)
return 0 if tr.launch() else 1
except test_runner.TestRunnerError as e:
sys.stderr.write(traceback.format_exc())
summary['step_text'] = '%s%s' % (e.__class__.__name__, ': %s' %
e.args[0] if e.args else '')
# test_runner.Launch returns 0 on success, 1 on failure, so return 2
# on exception to distinguish between a test failure, and a failure
# to launch the test at all.
return 2
finally:
if tr:
summary['logs'] = tr.logs
with open(os.path.join(self.args.out_dir, 'summary.json'),
'w') as f:
json.dump(summary, f)
if tr:
with open(os.path.join(self.args.out_dir, 'full_results.json'),
'w') as f:
json.dump(tr.test_results, f)
# The value of test-launcher-summary-output is set by the recipe
# and passed here via swarming.py. This argument defaults to
# ${ISOLATED_OUTDIR}/output.json. out-dir is set to ${ISOLATED_OUTDIR}
# TODO(crbug.com/1031338) - the content of this output.json will
# work with Chromium recipe because we use the noop_merge merge script,
# but will require structural changes to support the default gtest
# merge script (ref: //testing/merge_scripts/standard_gtest_merge.py)
output_json_path = (self.args.test_launcher_summary_output
or os.path.join(self.args.out_dir,
'output.json'))
with open(output_json_path, 'w') as f:
json.dump(tr.test_results, f)
test_runner.defaults_delete('com.apple.CoreSimulator',
'FramebufferServerRendererPolicy')
def run(self, args):
"""
Main coordinating function.
"""
self.parse_args(args)
# This logic is run by default before the otool command is invoked such that
# otool has the correct Xcode selected for command line dev tools.
install_success, is_legacy_xcode = self.install_xcode()
if not install_success:
raise test_runner.XcodeVersionNotFoundError(
self.args.xcode_build_version)
self.resolve_test_cases()
summary = {}
tr = None
if not os.path.exists(self.args.out_dir):
os.makedirs(self.args.out_dir)
try:
if self.args.xcode_parallelization:
tr = xcodebuild_runner.SimulatorParallelTestRunner(
self.args.app,
self.args.host_app,
self.args.iossim,
self.args.version,
self.args.platform,
out_dir=self.args.out_dir,
release=self.args.release,
repeat_count=self.args.gtest_repeat,
retries=self.args.retries,
shards=self.args.shards,
test_cases=self.args.test_cases,
test_args=self.test_args,
use_clang_coverage=self.args.use_clang_coverage,
env_vars=self.args.env_var)
elif self.args.variations_seed_path != 'NO_PATH':
tr = variations_runner.VariationsSimulatorParallelTestRunner(
self.args.app,
self.args.host_app,
self.args.iossim,
self.args.version,
self.args.platform,
self.args.out_dir,
self.args.variations_seed_path,
release=self.args.release,
test_cases=self.args.test_cases,
test_args=self.test_args,
env_vars=self.args.env_var)
elif self.args.replay_path != 'NO_PATH':
tr = wpr_runner.WprProxySimulatorTestRunner(
self.args.app,
self.args.host_app,
self.args.iossim,
self.args.replay_path,
self.args.platform,
self.args.version,
self.args.wpr_tools_path,
self.args.out_dir,
env_vars=self.args.env_var,
retries=self.args.retries,
shards=self.args.shards,
test_args=self.test_args,
test_cases=self.args.test_cases,
xctest=self.args.xctest,
)
elif self.args.iossim and self.args.platform and self.args.version:
tr = test_runner.SimulatorTestRunner(
self.args.app,
self.args.iossim,
self.args.platform,
self.args.version,
self.args.out_dir,
env_vars=self.args.env_var,
repeat_count=self.args.gtest_repeat,
retries=self.args.retries,
shards=self.args.shards,
test_args=self.test_args,
test_cases=self.args.test_cases,
use_clang_coverage=self.args.use_clang_coverage,
wpr_tools_path=self.args.wpr_tools_path,
xctest=self.args.xctest,
)
elif self.args.xcodebuild_device_runner and self.args.xctest:
tr = xcodebuild_runner.DeviceXcodeTestRunner(
app_path=self.args.app,
host_app_path=self.args.host_app,
out_dir=self.args.out_dir,
release=self.args.release,
repeat_count=self.args.gtest_repeat,
retries=self.args.retries,
test_cases=self.args.test_cases,
test_args=self.test_args,
env_vars=self.args.env_var)
else:
tr = test_runner.DeviceTestRunner(
self.args.app,
self.args.out_dir,
env_vars=self.args.env_var,
repeat_count=self.args.gtest_repeat,
restart=self.args.restart,
retries=self.args.retries,
test_args=self.test_args,
test_cases=self.args.test_cases,
xctest=self.args.xctest,
)
logging.info("Using test runner %s" % type(tr).__name__)
return 0 if tr.launch() else 1
except test_runner.DeviceError as e:
sys.stderr.write(traceback.format_exc())
summary['step_text'] = '%s%s' % (e.__class__.__name__, ': %s' %
e.args[0] if e.args else '')
# Swarming infra marks device status unavailable for any device related
# issue using this return code.
return 3
except test_runner.TestRunnerError as e:
sys.stderr.write(traceback.format_exc())
summary['step_text'] = '%s%s' % (e.__class__.__name__, ': %s' %
e.args[0] if e.args else '')
# test_runner.Launch returns 0 on success, 1 on failure, so return 2
# on exception to distinguish between a test failure, and a failure
# to launch the test at all.
return 2
finally:
if tr:
summary['logs'] = tr.logs
with open(os.path.join(self.args.out_dir, 'summary.json'),
'w') as f:
json.dump(summary, f)
if tr:
with open(os.path.join(self.args.out_dir, 'full_results.json'),
'w') as f:
json.dump(tr.test_results, f)
# The value of test-launcher-summary-output is set by the recipe
# and passed here via swarming.py. This argument defaults to
# ${ISOLATED_OUTDIR}/output.json. out-dir is set to ${ISOLATED_OUTDIR}
# TODO(crbug.com/1031338) - the content of this output.json will
# work with Chromium recipe because we use the noop_merge merge script,
# but will require structural changes to support the default gtest
# merge script (ref: //testing/merge_scripts/standard_gtest_merge.py)
output_json_path = (self.args.test_launcher_summary_output
or os.path.join(self.args.out_dir,
'output.json'))
with open(output_json_path, 'w') as f:
json.dump(tr.test_results, f)
# Move the iOS runtime back to cache dir if the Xcode package is not
# legacy (i.e. Xcode program & runtimes are in different CIPD packages.)
# and it's a simulator task.
if not is_legacy_xcode and self.args.version:
runtime_cache_folder = xcode.construct_runtime_cache_folder(
self.args.runtime_cache_prefix, self.args.version)
xcode.move_runtime(runtime_cache_folder, self.args.xcode_path,
False)
test_runner.defaults_delete('com.apple.CoreSimulator',
'FramebufferServerRendererPolicy')
def main():
logging.basicConfig(format='[%(asctime)s:%(levelname)s] %(message)s',
level=logging.DEBUG, datefmt='%I:%M:%S')
test_runner.defaults_delete('com.apple.CoreSimulator',
'FramebufferServerRendererPolicy')
args, test_args = parse_args()
summary = {}
tr = None
if not os.path.exists(args.out_dir):
os.makedirs(args.out_dir)
try:
if args.xcode_parallelization:
tr = xcodebuild_runner.SimulatorParallelTestRunner(
args.app,
args.host_app,
args.iossim,
args.xcode_build_version,
args.version,
args.platform,
out_dir=args.out_dir,
mac_toolchain=args.mac_toolchain_cmd,
retries=args.retries,
shards=args.shards,
xcode_path=args.xcode_path,
test_cases=args.test_cases,
test_args=test_args,
env_vars=args.env_var
)
elif args.replay_path != 'NO_PATH':
tr = wpr_runner.WprProxySimulatorTestRunner(
args.app,
args.host_app,
args.iossim,
args.replay_path,
args.platform,
args.version,
args.wpr_tools_path,
args.xcode_build_version,
args.out_dir,
env_vars=args.env_var,
mac_toolchain=args.mac_toolchain_cmd,
retries=args.retries,
shards=args.shards,
test_args=test_args,
test_cases=args.test_cases,
xcode_path=args.xcode_path,
xctest=args.xctest,
)
elif args.iossim and args.platform and args.version:
tr = test_runner.SimulatorTestRunner(
args.app,
args.iossim,
args.platform,
args.version,
args.xcode_build_version,
args.out_dir,
env_vars=args.env_var,
mac_toolchain=args.mac_toolchain_cmd,
retries=args.retries,
shards=args.shards,
test_args=test_args,
test_cases=args.test_cases,
wpr_tools_path=args.wpr_tools_path,
xcode_path=args.xcode_path,
xctest=args.xctest,
)
elif args.xcodebuild_device_runner and args.xctest:
tr = xcodebuild_runner.DeviceXcodeTestRunner(
app_path=args.app,
host_app_path=args.host_app,
xcode_build_version=args.xcode_build_version,
out_dir=args.out_dir,
mac_toolchain=args.mac_toolchain_cmd,
retries=args.retries,
xcode_path=args.xcode_path,
test_cases=args.test_cases,
test_args=test_args,
env_vars=args.env_var)
else:
tr = test_runner.DeviceTestRunner(
args.app,
args.xcode_build_version,
args.out_dir,
env_vars=args.env_var,
mac_toolchain=args.mac_toolchain_cmd,
restart=args.restart,
retries=args.retries,
test_args=test_args,
test_cases=args.test_cases,
xcode_path=args.xcode_path,
xctest=args.xctest,
)
return 0 if tr.launch() else 1
except test_runner.TestRunnerError as e:
sys.stderr.write(traceback.format_exc())
summary['step_text'] = '%s%s' % (
e.__class__.__name__, ': %s' % e.args[0] if e.args else '')
# test_runner.Launch returns 0 on success, 1 on failure, so return 2
# on exception to distinguish between a test failure, and a failure
# to launch the test at all.
return 2
finally:
if tr:
summary['logs'] = tr.logs
with open(os.path.join(args.out_dir, 'summary.json'), 'w') as f:
json.dump(summary, f)
if tr:
with open(os.path.join(args.out_dir, 'full_results.json'), 'w') as f:
json.dump(tr.test_results, f)
test_runner.defaults_delete('com.apple.CoreSimulator',
'FramebufferServerRendererPolicy')
def run(self, args):
"""
Main coordinating function.
"""
self.parse_args(args)
summary = {}
tr = None
if not os.path.exists(self.args.out_dir):
os.makedirs(self.args.out_dir)
try:
if self.args.xcode_parallelization:
tr = xcodebuild_runner.SimulatorParallelTestRunner(
self.args.app,
self.args.host_app,
self.args.iossim,
self.args.xcode_build_version,
self.args.version,
self.args.platform,
out_dir=self.args.out_dir,
mac_toolchain=self.args.mac_toolchain_cmd,
retries=self.args.retries,
shards=self.args.shards,
xcode_path=self.args.xcode_path,
test_cases=self.args.test_cases,
test_args=self.test_args,
env_vars=self.args.env_var)
elif self.args.replay_path != 'NO_PATH':
tr = wpr_runner.WprProxySimulatorTestRunner(
self.args.app,
self.args.host_app,
self.args.iossim,
self.args.replay_path,
self.args.platform,
self.args.version,
self.args.wpr_tools_path,
self.args.xcode_build_version,
self.args.out_dir,
env_vars=self.args.env_var,
mac_toolchain=self.args.mac_toolchain_cmd,
retries=self.args.retries,
shards=self.args.shards,
test_args=self.test_args,
test_cases=self.args.test_cases,
xcode_path=self.args.xcode_path,
xctest=self.args.xctest,
)
elif self.args.iossim and self.args.platform and self.args.version:
tr = test_runner.SimulatorTestRunner(
self.args.app,
self.args.iossim,
self.args.platform,
self.args.version,
self.args.xcode_build_version,
self.args.out_dir,
env_vars=self.args.env_var,
mac_toolchain=self.args.mac_toolchain_cmd,
retries=self.args.retries,
shards=self.args.shards,
test_args=self.test_args,
test_cases=self.args.test_cases,
wpr_tools_path=self.args.wpr_tools_path,
xcode_path=self.args.xcode_path,
xctest=self.args.xctest,
)
elif self.args.xcodebuild_device_runner and self.args.xctest:
tr = xcodebuild_runner.DeviceXcodeTestRunner(
app_path=self.args.app,
host_app_path=self.args.host_app,
xcode_build_version=self.args.xcode_build_version,
out_dir=self.args.out_dir,
mac_toolchain=self.args.mac_toolchain_cmd,
retries=self.args.retries,
xcode_path=self.args.xcode_path,
test_cases=self.args.test_cases,
test_args=self.test_args,
env_vars=self.args.env_var)
else:
tr = test_runner.DeviceTestRunner(
self.args.app,
self.args.xcode_build_version,
self.args.out_dir,
env_vars=self.args.env_var,
mac_toolchain=self.args.mac_toolchain_cmd,
restart=self.args.restart,
retries=self.args.retries,
test_args=self.test_args,
test_cases=self.args.test_cases,
xcode_path=self.args.xcode_path,
xctest=self.args.xctest,
)
return 0 if tr.launch() else 1
except test_runner.TestRunnerError as e:
sys.stderr.write(traceback.format_exc())
summary['step_text'] = '%s%s' % (e.__class__.__name__, ': %s' %
e.args[0] if e.args else '')
# test_runner.Launch returns 0 on success, 1 on failure, so return 2
# on exception to distinguish between a test failure, and a failure
# to launch the test at all.
return 2
finally:
if tr:
summary['logs'] = tr.logs
with open(os.path.join(self.args.out_dir, 'summary.json'),
'w') as f:
json.dump(summary, f)
if tr:
with open(os.path.join(self.args.out_dir, 'full_results.json'),
'w') as f:
json.dump(tr.test_results, f)
# The value of test-launcher-summary-output is set by the recipe
# and passed here via swarming.py. This argument defaults to
# ${ISOLATED_OUTDIR}/output.json. out-dir is set to ${ISOLATED_OUTDIR}
# TODO(crbug.com/1031338) - the content of this output.json will
# work with Chromium recipe because we use the noop_merge merge script,
# but will require structural changes to support the default gtest
# merge script (ref: //testing/merge_scripts/standard_gtest_merge.py)
output_json_path = (self.args.test_launcher_summary_output
or os.path.join(self.args.out_dir,
'output.json'))
with open(output_json_path, 'w') as f:
json.dump(tr.test_results, f)
test_runner.defaults_delete('com.apple.CoreSimulator',
'FramebufferServerRendererPolicy')
