def testLaunchCommand_make_cmd_list_for_failed_tests(
self, fill_xctest_run_mock):
fill_xctest_run_mock.side_effect = ['/var/folders/tmpfile1']
egtest_app = 'module_1_egtests.app'
egtest_app_path = '%s/%s' % (_ROOT_FOLDER_PATH, egtest_app)
failed_tests = {
egtest_app: [
'TestCase1_1/TestMethod1',
'TestCase1_1/TestMethod2',
'TestCase1_2/TestMethod1',
]
}
expected_egtests = xcodebuild_runner.EgtestsApp(
egtest_app_path, filtered_tests=failed_tests[egtest_app])
mock_egtest = mock.MagicMock(spec=xcodebuild_runner.EgtestsApp)
type(mock_egtest).egtests_path = mock.PropertyMock(
return_value=egtest_app_path)
cmd = xcodebuild_runner.LaunchCommand(
egtests_app=mock_egtest,
destination=_DESTINATION,
out_dir='out/dir/attempt_2/iPhone X 12.0',
shards=1,
retries=1)
cmd._make_cmd_list_for_failed_tests(
failed_tests, os.path.join(_OUT_DIR, 'attempt_2'))
self.assertEqual(1, len(fill_xctest_run_mock.mock_calls))
self.assertItemsEqual(
expected_egtests.__dict__,
fill_xctest_run_mock.mock_calls[0][1][1].__dict__)
def get_wpr_test_command(self, recipe_path, test_name):
"""Creates xcodebuild commands for running a wpr test per recipe_path.
Args:
recipe_path: (str) Path to wpr recipe file.
test_name: (str) Test name(format: ios_website) of this wpr test.
Returns:
Xcodebuild command to run in the format of a list of str.
"""
wpr_test_args = [
'--enable-features=AutofillShowTypePredictions',
'-autofillautomation=%s' % recipe_path,
]
wpr_egtests_app = xcodebuild_runner.EgtestsApp(
self.app_path,
included_tests=["AutofillAutomationTestCase"],
env_vars=self.env_vars,
test_args=wpr_test_args,
host_app_path=self.host_app_path)
self.test_attempt_count[test_name] = self.test_attempt_count.get(
test_name, 0) + 1
destination = 'platform=iOS Simulator,OS=%s,name=%s' % (self.version,
self.platform)
destination_folder = '%s %s %s attempt#%s' % (
self.version, self.platform, test_name,
self.test_attempt_count[test_name])
out_dir = os.path.join(self.out_dir, destination_folder)
launch_command = xcodebuild_runner.LaunchCommand(
wpr_egtests_app, destination, self.shards, self.retries)
return launch_command.command(wpr_egtests_app, out_dir, destination,
self.shards)
def testLaunchCommand_notRestartPassedTest(self, mock_collect_results):
egtests = test_apps.EgtestsApp(_EGTESTS_APP_PATH)
collection = ResultCollection(test_results=[
TestResult('Class1/passedTest1', TestStatus.PASS),
TestResult('Class1/passedTest2', TestStatus.PASS)
])
mock_collect_results.side_effect = [collection]
launch_command = xcodebuild_runner.LaunchCommand(egtests,
_DESTINATION,
shards=1,
retries=3)
launch_command.launch()
xcodebuild_runner.LaunchCommand(egtests,
_DESTINATION,
shards=1,
retries=3)
self.assertEqual(1, len(mock_collect_results.mock_calls))
def testFill_xctest_run(self, xcode_version, mock_path_join, _):
mock_path_join.return_value = _XTEST_RUN
mock_egtest = mock.MagicMock(spec=xcodebuild_runner.EgtestsApp)
xcode_version.return_value = {'version': '10.2.1'}
launch_command = xcodebuild_runner.LaunchCommand(
mock_egtest, _DESTINATION, shards=1, retries=1, out_dir=_OUT_DIR)
self.assertEqual(_XTEST_RUN, launch_command.fill_xctest_run(mock_egtest))
self.assertEqual([mock.call.xctestrun_node()], mock_egtest.method_calls)
def testFill_xctest_run_exception(self):
with self.assertRaises(test_runner.AppNotFoundError):
xcodebuild_runner.LaunchCommand([],
'destination',
shards=1,
retries=1,
out_dir=_OUT_DIR).fill_xctest_run(
[])
def testLaunchCommand_restartFailed1stAttempt(self, mock_listdir):
mock_listdir.return_value = ['any_egtests.xctest']
egtests = xcodebuild_runner.EgtestsApp(_EGTESTS_APP_PATH)
launch_command = xcodebuild_runner.LaunchCommand(egtests,
_DESTINATION,
shards=1,
retries=3,
out_dir=self.tmpdir)
self.fake_launch_attempt(launch_command, ['not_started', 'pass'])
launch_command.launch()
self.assertEqual(2, len(launch_command.test_results))
def testFill_xctest_run(self, mock_path_join, _):
self._mocks[xcodebuild_runner.LaunchCommand].pop(
'fill_xctest_run', None)
mock_path_join.return_value = _XTEST_RUN
mock_egtest = mock.MagicMock(spec=xcodebuild_runner.EgtestsApp)
launch_command = xcodebuild_runner.LaunchCommand(mock_egtest,
_DESTINATION,
shards=1,
retries=1,
out_dir=_OUT_DIR)
self.assertEqual(_XTEST_RUN,
launch_command.fill_xctest_run(mock_egtest))
self.assertEqual([mock.call.xctestrun_node()],
mock_egtest.method_calls)
def testFill_xctest_run(self, mock_path_exists, _, mock_tmpfile):
mock_path_exists.return_value = True
mock_tmpfile.return_value = (1, _XTEST_RUN)
destination = 'platform=iOS Simulator,OS=12.0,name=iPhone X'
mock_egtest = mock.MagicMock(spec=xcodebuild_runner.EgtestsApp)
launch_command = xcodebuild_runner.LaunchCommand(mock_egtest,
destination,
shards=1,
retries=1,
out_dir=_OUT_DIR)
self.assertEqual(_XTEST_RUN,
launch_command.fill_xctest_run(mock_egtest))
self.assertEqual([mock.call.xctestrun_node()],
mock_egtest.method_calls)
def test_launch_command_not_restart_crashed_attempt(
self, mock_collect_results):
"""Crashed first attempt of runtime select test suite won't be retried."""
egtests = test_apps.EgtestsApp(_FLAKY_EGTEST_APP_PATH)
crashed_collection = ResultCollection()
crashed_collection.crashed = True
mock_collect_results.return_value = crashed_collection
launch_command = xcodebuild_runner.LaunchCommand(egtests,
_DESTINATION,
shards=1,
retries=3)
overall_result = launch_command.launch()
self.assertEqual(len(overall_result.all_test_names()), 0)
self.assertEqual(overall_result.expected_tests(), set([]))
self.assertTrue(overall_result.crashed)
def testLaunchCommand_notRestartPassedTest(self, mock_collect_results,
xcode_version):
egtests = xcodebuild_runner.EgtestsApp(_EGTESTS_APP_PATH)
xcode_version.return_value = {'version': '10.2.1'}
mock_collect_results.side_effect = [
{'failed': {'BUILD_INTERRUPTED': 'BUILD_INTERRUPTED: attempt # 0'},
'passed': ['Class1/passedTest1', 'Class1/passedTest2']}
]
launch_command = xcodebuild_runner.LaunchCommand(egtests,
_DESTINATION,
shards=1,
retries=3,
out_dir=self.tmpdir)
self.fake_launch_attempt(launch_command, ['pass'])
launch_command.launch()
self.assertEqual(2, len(launch_command.test_results))
def testLaunchCommand_restartFailed1stAttempt(self, mock_collect_results,
xcode_version):
egtests = xcodebuild_runner.EgtestsApp(_EGTESTS_APP_PATH)
xcode_version.return_value = {'version': '10.2.1'}
mock_collect_results.side_effect = [
{'failed': {'TESTS_DID_NOT_START': ['not started']}, 'passed': []},
{'failed': {}, 'passed': ['Class1/passedTest1', 'Class1/passedTest2']}
]
launch_command = xcodebuild_runner.LaunchCommand(egtests,
_DESTINATION,
shards=1,
retries=3,
out_dir=self.tmpdir)
self.fake_launch_attempt(launch_command, ['not_started', 'pass'])
launch_command.launch()
self.assertEqual(2, len(launch_command.test_results))
def testLaunchCommand_command(self, mock_fill_xctestrun):
mock_fill_xctestrun.return_value = _XTEST_RUN
mock_egtest = mock.MagicMock(spec=xcodebuild_runner.EgtestsApp)
type(mock_egtest).egtests_path = mock.PropertyMock(
return_value=_EGTESTS_APP_PATH)
cmd = xcodebuild_runner.LaunchCommand(
mock_egtest, _DESTINATION, shards=3, retries=1, out_dir=_OUT_DIR)
self.assertEqual(['xcodebuild', 'test-without-building',
'-xctestrun', '/tmp/temp_file.xctestrun',
'-destination',
'platform=iOS Simulator,OS=12.0,name=iPhone X',
'-resultBundlePath', 'out/dir',
'-parallel-testing-enabled', 'YES',
'-parallel-testing-worker-count', '3'],
cmd.command(egtests_app=mock_egtest,
out_dir=_OUT_DIR,
destination=_DESTINATION,
shards=3))
def testLaunchCommand_restartCrashed1stAttempt(self, mock_collect_results):
egtests = test_apps.EgtestsApp(_EGTESTS_APP_PATH)
crashed_collection = ResultCollection()
crashed_collection.crashed = True
mock_collect_results.side_effect = [
crashed_collection,
ResultCollection(test_results=[
TestResult('Class1/passedTest1', TestStatus.PASS),
TestResult('Class1/passedTest2', TestStatus.PASS)
])
]
launch_command = xcodebuild_runner.LaunchCommand(egtests,
_DESTINATION,
shards=1,
retries=3)
overall_result = launch_command.launch()
self.assertFalse(overall_result.crashed)
self.assertEqual(len(overall_result.all_test_names()), 2)
self.assertEqual(overall_result.expected_tests(),
set(['Class1/passedTest1', 'Class1/passedTest2']))
'ios_cwt_chromedriver_tests_module-Runner.app')
host_app = os.path.join(args.build_dir, 'ios_cwt_chromedriver_tests.app')
destination = iossim_util.get_simulator(args.device, args.os)
if not os.path.exists(args.out_dir):
os.mkdir(args.out_dir)
# Make sure each run produces a unique output directory, since reusing an
# existing directory will cause CWTChromeDriver's dummy test case to get
# skipped, meaning that CWTChromeDriver's http server won't get launched.
output_directory = os.path.join(args.out_dir, 'run%d' % int(time.time()))
inserted_libs = []
if args.asan_build:
inserted_libs = [
os.path.join(args.build_dir, 'libclang_rt.asan_iossim_dynamic.dylib')
]
egtests_app = test_apps.EgtestsApp(egtests_app=test_app,
test_args=['--port %s' % args.port],
host_app_path=host_app,
inserted_libs=inserted_libs)
launch_command = xcodebuild_runner.LaunchCommand(egtests_app,
destination,
shards=1,
retries=1,
out_dir=output_directory)
launch_command.launch()
