def testLaunchCommand_make_cmd_list_for_failed_tests( self, fill_xctest_run_mock): fill_xctest_run_mock.side_effect = ['/var/folders/tmpfile1'] egtest_app = 'module_1_egtests.app' egtest_app_path = '%s/%s' % (_ROOT_FOLDER_PATH, egtest_app) failed_tests = { egtest_app: [ 'TestCase1_1/TestMethod1', 'TestCase1_1/TestMethod2', 'TestCase1_2/TestMethod1', ] } expected_egtests = xcodebuild_runner.EgtestsApp( egtest_app_path, filtered_tests=failed_tests[egtest_app]) mock_egtest = mock.MagicMock(spec=xcodebuild_runner.EgtestsApp) type(mock_egtest).egtests_path = mock.PropertyMock( return_value=egtest_app_path) cmd = xcodebuild_runner.LaunchCommand( egtests_app=mock_egtest, destination=_DESTINATION, out_dir='out/dir/attempt_2/iPhone X 12.0', shards=1, retries=1) cmd._make_cmd_list_for_failed_tests( failed_tests, os.path.join(_OUT_DIR, 'attempt_2')) self.assertEqual(1, len(fill_xctest_run_mock.mock_calls)) self.assertItemsEqual( expected_egtests.__dict__, fill_xctest_run_mock.mock_calls[0][1][1].__dict__)
def get_wpr_test_command(self, recipe_path, test_name): """Creates xcodebuild commands for running a wpr test per recipe_path. Args: recipe_path: (str) Path to wpr recipe file. test_name: (str) Test name(format: ios_website) of this wpr test. Returns: Xcodebuild command to run in the format of a list of str. """ wpr_test_args = [ '--enable-features=AutofillShowTypePredictions', '-autofillautomation=%s' % recipe_path, ] wpr_egtests_app = xcodebuild_runner.EgtestsApp( self.app_path, included_tests=["AutofillAutomationTestCase"], env_vars=self.env_vars, test_args=wpr_test_args, host_app_path=self.host_app_path) self.test_attempt_count[test_name] = self.test_attempt_count.get( test_name, 0) + 1 destination = 'platform=iOS Simulator,OS=%s,name=%s' % (self.version, self.platform) destination_folder = '%s %s %s attempt#%s' % ( self.version, self.platform, test_name, self.test_attempt_count[test_name]) out_dir = os.path.join(self.out_dir, destination_folder) launch_command = xcodebuild_runner.LaunchCommand( wpr_egtests_app, destination, self.shards, self.retries) return launch_command.command(wpr_egtests_app, out_dir, destination, self.shards)
def testLaunchCommand_notRestartPassedTest(self, mock_collect_results): egtests = test_apps.EgtestsApp(_EGTESTS_APP_PATH) collection = ResultCollection(test_results=[ TestResult('Class1/passedTest1', TestStatus.PASS), TestResult('Class1/passedTest2', TestStatus.PASS) ]) mock_collect_results.side_effect = [collection] launch_command = xcodebuild_runner.LaunchCommand(egtests, _DESTINATION, shards=1, retries=3) launch_command.launch() xcodebuild_runner.LaunchCommand(egtests, _DESTINATION, shards=1, retries=3) self.assertEqual(1, len(mock_collect_results.mock_calls))
def testFill_xctest_run(self, xcode_version, mock_path_join, _): mock_path_join.return_value = _XTEST_RUN mock_egtest = mock.MagicMock(spec=xcodebuild_runner.EgtestsApp) xcode_version.return_value = {'version': '10.2.1'} launch_command = xcodebuild_runner.LaunchCommand( mock_egtest, _DESTINATION, shards=1, retries=1, out_dir=_OUT_DIR) self.assertEqual(_XTEST_RUN, launch_command.fill_xctest_run(mock_egtest)) self.assertEqual([mock.call.xctestrun_node()], mock_egtest.method_calls)
def testFill_xctest_run_exception(self): with self.assertRaises(test_runner.AppNotFoundError): xcodebuild_runner.LaunchCommand([], 'destination', shards=1, retries=1, out_dir=_OUT_DIR).fill_xctest_run( [])
def testLaunchCommand_restartFailed1stAttempt(self, mock_listdir): mock_listdir.return_value = ['any_egtests.xctest'] egtests = xcodebuild_runner.EgtestsApp(_EGTESTS_APP_PATH) launch_command = xcodebuild_runner.LaunchCommand(egtests, _DESTINATION, shards=1, retries=3, out_dir=self.tmpdir) self.fake_launch_attempt(launch_command, ['not_started', 'pass']) launch_command.launch() self.assertEqual(2, len(launch_command.test_results))
def testFill_xctest_run(self, mock_path_join, _): self._mocks[xcodebuild_runner.LaunchCommand].pop( 'fill_xctest_run', None) mock_path_join.return_value = _XTEST_RUN mock_egtest = mock.MagicMock(spec=xcodebuild_runner.EgtestsApp) launch_command = xcodebuild_runner.LaunchCommand(mock_egtest, _DESTINATION, shards=1, retries=1, out_dir=_OUT_DIR) self.assertEqual(_XTEST_RUN, launch_command.fill_xctest_run(mock_egtest)) self.assertEqual([mock.call.xctestrun_node()], mock_egtest.method_calls)
def testFill_xctest_run(self, mock_path_exists, _, mock_tmpfile): mock_path_exists.return_value = True mock_tmpfile.return_value = (1, _XTEST_RUN) destination = 'platform=iOS Simulator,OS=12.0,name=iPhone X' mock_egtest = mock.MagicMock(spec=xcodebuild_runner.EgtestsApp) launch_command = xcodebuild_runner.LaunchCommand(mock_egtest, destination, shards=1, retries=1, out_dir=_OUT_DIR) self.assertEqual(_XTEST_RUN, launch_command.fill_xctest_run(mock_egtest)) self.assertEqual([mock.call.xctestrun_node()], mock_egtest.method_calls)
def test_launch_command_not_restart_crashed_attempt( self, mock_collect_results): """Crashed first attempt of runtime select test suite won't be retried.""" egtests = test_apps.EgtestsApp(_FLAKY_EGTEST_APP_PATH) crashed_collection = ResultCollection() crashed_collection.crashed = True mock_collect_results.return_value = crashed_collection launch_command = xcodebuild_runner.LaunchCommand(egtests, _DESTINATION, shards=1, retries=3) overall_result = launch_command.launch() self.assertEqual(len(overall_result.all_test_names()), 0) self.assertEqual(overall_result.expected_tests(), set([])) self.assertTrue(overall_result.crashed)
def testLaunchCommand_notRestartPassedTest(self, mock_collect_results, xcode_version): egtests = xcodebuild_runner.EgtestsApp(_EGTESTS_APP_PATH) xcode_version.return_value = {'version': '10.2.1'} mock_collect_results.side_effect = [ {'failed': {'BUILD_INTERRUPTED': 'BUILD_INTERRUPTED: attempt # 0'}, 'passed': ['Class1/passedTest1', 'Class1/passedTest2']} ] launch_command = xcodebuild_runner.LaunchCommand(egtests, _DESTINATION, shards=1, retries=3, out_dir=self.tmpdir) self.fake_launch_attempt(launch_command, ['pass']) launch_command.launch() self.assertEqual(2, len(launch_command.test_results))
def testLaunchCommand_restartFailed1stAttempt(self, mock_collect_results, xcode_version): egtests = xcodebuild_runner.EgtestsApp(_EGTESTS_APP_PATH) xcode_version.return_value = {'version': '10.2.1'} mock_collect_results.side_effect = [ {'failed': {'TESTS_DID_NOT_START': ['not started']}, 'passed': []}, {'failed': {}, 'passed': ['Class1/passedTest1', 'Class1/passedTest2']} ] launch_command = xcodebuild_runner.LaunchCommand(egtests, _DESTINATION, shards=1, retries=3, out_dir=self.tmpdir) self.fake_launch_attempt(launch_command, ['not_started', 'pass']) launch_command.launch() self.assertEqual(2, len(launch_command.test_results))
def testLaunchCommand_command(self, mock_fill_xctestrun): mock_fill_xctestrun.return_value = _XTEST_RUN mock_egtest = mock.MagicMock(spec=xcodebuild_runner.EgtestsApp) type(mock_egtest).egtests_path = mock.PropertyMock( return_value=_EGTESTS_APP_PATH) cmd = xcodebuild_runner.LaunchCommand( mock_egtest, _DESTINATION, shards=3, retries=1, out_dir=_OUT_DIR) self.assertEqual(['xcodebuild', 'test-without-building', '-xctestrun', '/tmp/temp_file.xctestrun', '-destination', 'platform=iOS Simulator,OS=12.0,name=iPhone X', '-resultBundlePath', 'out/dir', '-parallel-testing-enabled', 'YES', '-parallel-testing-worker-count', '3'], cmd.command(egtests_app=mock_egtest, out_dir=_OUT_DIR, destination=_DESTINATION, shards=3))
def testLaunchCommand_restartCrashed1stAttempt(self, mock_collect_results): egtests = test_apps.EgtestsApp(_EGTESTS_APP_PATH) crashed_collection = ResultCollection() crashed_collection.crashed = True mock_collect_results.side_effect = [ crashed_collection, ResultCollection(test_results=[ TestResult('Class1/passedTest1', TestStatus.PASS), TestResult('Class1/passedTest2', TestStatus.PASS) ]) ] launch_command = xcodebuild_runner.LaunchCommand(egtests, _DESTINATION, shards=1, retries=3) overall_result = launch_command.launch() self.assertFalse(overall_result.crashed) self.assertEqual(len(overall_result.all_test_names()), 2) self.assertEqual(overall_result.expected_tests(), set(['Class1/passedTest1', 'Class1/passedTest2']))
'ios_cwt_chromedriver_tests_module-Runner.app') host_app = os.path.join(args.build_dir, 'ios_cwt_chromedriver_tests.app') destination = iossim_util.get_simulator(args.device, args.os) if not os.path.exists(args.out_dir): os.mkdir(args.out_dir) # Make sure each run produces a unique output directory, since reusing an # existing directory will cause CWTChromeDriver's dummy test case to get # skipped, meaning that CWTChromeDriver's http server won't get launched. output_directory = os.path.join(args.out_dir, 'run%d' % int(time.time())) inserted_libs = [] if args.asan_build: inserted_libs = [ os.path.join(args.build_dir, 'libclang_rt.asan_iossim_dynamic.dylib') ] egtests_app = test_apps.EgtestsApp(egtests_app=test_app, test_args=['--port %s' % args.port], host_app_path=host_app, inserted_libs=inserted_libs) launch_command = xcodebuild_runner.LaunchCommand(egtests_app, destination, shards=1, retries=1, out_dir=output_directory) launch_command.launch()