def get_wpr_test_command(self, recipe_path, test_name): """Creates xcodebuild commands for running a wpr test per recipe_path. Args: recipe_path: (str) Path to wpr recipe file. test_name: (str) Test name(format: ios_website) of this wpr test. Returns: Xcodebuild command to run in the format of a list of str. """ wpr_test_args = [ '--enable-features=AutofillShowTypePredictions', '-autofillautomation=%s' % recipe_path, ] wpr_egtests_app = test_apps.EgtestsApp( self.app_path, included_tests=["AutofillAutomationTestCase"], env_vars=self.env_vars, test_args=wpr_test_args, host_app_path=self.host_app_path) self.test_attempt_count[test_name] = self.test_attempt_count.get( test_name, 0) + 1 destination = 'platform=iOS Simulator,OS=%s,name=%s' % (self.version, self.platform) destination_folder = '%s %s %s attempt#%s' % ( self.version, self.platform, test_name, self.test_attempt_count[test_name]) out_dir = os.path.join(self.out_dir, destination_folder) return wpr_egtests_app.command(out_dir, destination, self.shards)
def test_xctestRunNode_without_filter(self): self.mock(test_apps.EgtestsApp, '_xctest_path', lambda _: 'xctest-path') egtest_node = test_apps.EgtestsApp( _TEST_APP_PATH).fill_xctestrun_node()['test_app_module'] self.assertNotIn('OnlyTestIdentifiers', egtest_node) self.assertNotIn('SkipTestIdentifiers', egtest_node)
def test_command_with_repeat_count_incorrect_xcode(self, _1, _2): """Tests |command| raises error with repeat_count in lower Xcode version.""" egtests_app = test_apps.EgtestsApp('app_path', host_app_path='host_app_path', repeat_count=2) with self.assertRaises( test_runner_errors.XcodeUnsupportedFeatureError): cmd = egtests_app.command('outdir', 'id=UUID', 1)
def testEgtests_xctestRunNode_with_filter_skip_identifiers(self): skipped_tests = [ 'TestCase1/testMethod1', 'TestCase1/testMethod2', 'TestCase2/testMethod1', 'TestCase1/testMethod2' ] egtest_node = test_apps.EgtestsApp( _EGTESTS_APP_PATH, excluded_tests=skipped_tests ).fill_xctestrun_node()['any_egtests_module'] self.assertEqual(skipped_tests, egtest_node['SkipTestIdentifiers']) self.assertNotIn('OnlyTestIdentifiers', egtest_node)
def testNonTestsFiltered(self, mock_fetch, _): mock_fetch.return_value = [ ('ATestCase', 'testB'), ('setUpForTestCase', 'testForStartup'), ('ChromeTestCase', 'testServer'), ('FindInPageTestCase', 'testURL'), ('CTestCase', 'testD'), ] test_app = test_apps.EgtestsApp(_TEST_APP_PATH) tests = test_app.get_all_tests() self.assertEqual(set(tests), set(['ATestCase/testB', 'CTestCase/testD']))
def test_xctestRunNode_with_filter_skip_identifiers(self): self.mock(test_apps.EgtestsApp, '_xctest_path', lambda _: 'xctest-path') skipped_tests = [ 'TestCase1/testMethod1', 'TestCase1/testMethod2', 'TestCase2/testMethod1', 'TestCase1/testMethod2' ] egtest_node = test_apps.EgtestsApp( _TEST_APP_PATH, excluded_tests=skipped_tests).fill_xctestrun_node( )['test_app_module'] self.assertEqual(skipped_tests, egtest_node['SkipTestIdentifiers']) self.assertNotIn('OnlyTestIdentifiers', egtest_node)
def get_launch_test_app(self): """Returns the proper test_app for the run. Returns: An implementation of EgtestsApp for the runner. """ return test_apps.EgtestsApp(self.app_path, included_tests=self.test_cases, env_vars=self.env_vars, test_args=self.test_args, release=self.release, repeat_count=self.repeat_count, host_app_path=self.host_app_path)
def test_command_with_repeat_count(self, _1, _2): """Tests command method can produce repeat_count arguments when available. """ egtests_app = test_apps.EgtestsApp('app_path', host_app_path='host_app_path', repeat_count=2) cmd = egtests_app.command('outdir', 'id=UUID', 1) expected_cmd = [ 'arch', '-arch', 'arm64', 'xcodebuild', 'test-without-building', '-xctestrun', 'xctestrun', '-destination', 'id=UUID', '-resultBundlePath', 'outdir', '-test-iterations', '2' ] self.assertEqual(cmd, expected_cmd)
def test_launch_command_not_restart_crashed_attempt( self, mock_collect_results): """Crashed first attempt of runtime select test suite won't be retried.""" egtests = test_apps.EgtestsApp(_FLAKY_EGTEST_APP_PATH) crashed_collection = ResultCollection() crashed_collection.crashed = True mock_collect_results.return_value = crashed_collection launch_command = xcodebuild_runner.LaunchCommand(egtests, _DESTINATION, shards=1, retries=3) overall_result = launch_command.launch() self.assertEqual(len(overall_result.all_test_names()), 0) self.assertEqual(overall_result.expected_tests(), set([])) self.assertTrue(overall_result.crashed)
def get_launch_test_app(self, params): """Returns the proper test_app for the run, requiring sharding data. Args: params: A collection of sharding_data params. Returns: An implementation of EgtestsApp included the sharding_data params """ return test_apps.EgtestsApp(params['app'], included_tests=params['test_cases'], env_vars=self.env_vars, test_args=self.test_args, release=self.release, host_app_path=params['host'])
def testLaunchCommand_notRestartPassedTest(self, mock_collect_results): egtests = test_apps.EgtestsApp(_EGTESTS_APP_PATH) collection = ResultCollection(test_results=[ TestResult('Class1/passedTest1', TestStatus.PASS), TestResult('Class1/passedTest2', TestStatus.PASS) ]) mock_collect_results.side_effect = [collection] launch_command = xcodebuild_runner.LaunchCommand(egtests, _DESTINATION, shards=1, retries=3) launch_command.launch() xcodebuild_runner.LaunchCommand(egtests, _DESTINATION, shards=1, retries=3) self.assertEqual(1, len(mock_collect_results.mock_calls))
def testLaunchCommand_notRestartPassedTest(self, mock_collect_results, xcode_version): egtests = test_apps.EgtestsApp(_EGTESTS_APP_PATH) xcode_version.return_value = {'version': '10.2.1'} mock_collect_results.side_effect = [{ 'failed': { 'BUILD_INTERRUPTED': 'BUILD_INTERRUPTED: attempt # 0' }, 'passed': ['Class1/passedTest1', 'Class1/passedTest2'] }] launch_command = xcodebuild_runner.LaunchCommand(egtests, _DESTINATION, shards=1, retries=3, out_dir=self.tmpdir) self.fake_launch_attempt(launch_command, ['pass']) launch_command.launch() self.assertEqual(1, len(launch_command.test_results['attempts']))
def testLaunchCommand_restartCrashed1stAttempt(self, mock_collect_results): egtests = test_apps.EgtestsApp(_EGTESTS_APP_PATH) crashed_collection = ResultCollection() crashed_collection.crashed = True mock_collect_results.side_effect = [ crashed_collection, ResultCollection(test_results=[ TestResult('Class1/passedTest1', TestStatus.PASS), TestResult('Class1/passedTest2', TestStatus.PASS) ]) ] launch_command = xcodebuild_runner.LaunchCommand(egtests, _DESTINATION, shards=1, retries=3) overall_result = launch_command.launch() self.assertFalse(overall_result.crashed) self.assertEqual(len(overall_result.all_test_names()), 2) self.assertEqual(overall_result.expected_tests(), set(['Class1/passedTest1', 'Class1/passedTest2']))
def testLaunchCommand_restartFailed1stAttempt(self, mock_collect_results, xcode_version): egtests = test_apps.EgtestsApp(_EGTESTS_APP_PATH) xcode_version.return_value = {'version': '10.2.1'} mock_collect_results.side_effect = [{ 'failed': { 'TESTS_DID_NOT_START': ['not started'] }, 'passed': [] }, { 'failed': {}, 'passed': ['Class1/passedTest1', 'Class1/passedTest2'] }] launch_command = xcodebuild_runner.LaunchCommand(egtests, _DESTINATION, shards=1, retries=3, out_dir=self.tmpdir) self.fake_launch_attempt(launch_command, ['not_started', 'pass']) launch_command.launch() self.assertEqual(1, len(launch_command.test_results))
def launch(self): """Launches the test app.""" self.set_up() destination = 'id=%s' % self.udid if self.xctest: test_app = test_apps.EgtestsApp(self.app_path, included_tests=self.test_cases, env_vars=self.env_vars, test_args=self.test_args) else: test_app = test_apps.GTestsApp(self.app_path, included_tests=self.test_cases, env_vars=self.env_vars, test_args=self.test_args) out_dir = os.path.join(self.out_dir, 'TestResults') cmd = self.get_launch_command(test_app, out_dir, destination, self.shards) try: result = self._run(cmd=cmd, shards=self.shards or 1) if result.crashed and not result.crashed_test: # If the app crashed but not during any particular test case, assume # it crashed on startup. Try one more time. self.shutdown_and_restart() LOGGER.warning('Crashed on startup, retrying...\n') out_dir = os.path.join(self.out_dir, 'retry_after_crash_on_startup') cmd = self.get_launch_command(test_app, out_dir, destination, self.shards) result = self._run(cmd) if result.crashed and not result.crashed_test: raise AppLaunchError passed = result.passed_tests failed = result.failed_tests flaked = result.flaked_tests try: while result.crashed and result.crashed_test: # If the app crashes during a specific test case, then resume at the # next test case. This is achieved by filtering out every test case # which has already run. LOGGER.warning('Crashed during %s, resuming...\n', result.crashed_test) test_app.excluded_tests = passed + failed.keys( ) + flaked.keys() retry_out_dir = os.path.join( self.out_dir, 'retry_after_crash_%d' % int(time.time())) result = self._run( self.get_launch_command( test_app, os.path.join(retry_out_dir, str(int(time.time()))), destination)) passed.extend(result.passed_tests) failed.update(result.failed_tests) flaked.update(result.flaked_tests) except OSError as e: if e.errno == errno.E2BIG: LOGGER.error('Too many test cases to resume.') else: raise # Retry failed test cases. retry_results = {} test_app.excluded_tests = [] if self.retries and failed: LOGGER.warning('%s tests failed and will be retried.\n', len(failed)) for i in xrange(self.retries): for test in failed.keys(): LOGGER.info('Retry #%s for %s.\n', i + 1, test) test_app.included_tests = [test] retry_out_dir = os.path.join(self.out_dir, test + '_failed', 'retry_%d' % i) retry_result = self._run( self.get_launch_command(test_app, retry_out_dir, destination)) # If the test passed on retry, consider it flake instead of failure. if test in retry_result.passed_tests: flaked[test] = failed.pop(test) # Save the result of the latest run for each test. retry_results[test] = retry_result # Build test_results.json. # Check if if any of the retries crashed in addition to the original run. interrupted = (result.crashed or any([r.crashed for r in retry_results.values()])) self.test_results['interrupted'] = interrupted self.test_results['num_failures_by_type'] = { 'FAIL': len(failed) + len(flaked), 'PASS': len(passed), } tests = collections.OrderedDict() for test in passed: tests[test] = {'expected': 'PASS', 'actual': 'PASS'} for test in failed: tests[test] = {'expected': 'PASS', 'actual': 'FAIL'} for test in flaked: tests[test] = {'expected': 'PASS', 'actual': 'FAIL'} self.test_results['tests'] = tests self.logs['passed tests'] = passed if flaked: self.logs['flaked tests'] = flaked if failed: self.logs['failed tests'] = failed for test, log_lines in failed.iteritems(): self.logs[test] = log_lines for test, log_lines in flaked.iteritems(): self.logs[test] = log_lines return not failed and not interrupted finally: self.tear_down()
def test_found_xctest(self, mock_listdir): mock_listdir.return_value = [ '/path/to/test_app.app/PlugIns/any_egtests.xctest' ] self.assertEqual('/PlugIns/any_egtests.xctest', test_apps.EgtestsApp(_TEST_APP_PATH)._xctest_path())
def testEgtests_not_found_egtests_app(self): self.mock(os.path, 'exists', lambda _: False) with self.assertRaises(test_runner.AppNotFoundError): test_apps.EgtestsApp(_EGTESTS_APP_PATH)
def testEgtests_not_found_plugins(self): egtests = test_apps.EgtestsApp(_EGTESTS_APP_PATH) self.mock(os.path, 'exists', lambda _: False) with self.assertRaises(test_runner.PlugInsNotFoundError): egtests._xctest_path()
'ios_cwt_chromedriver_tests_module-Runner.app') host_app = os.path.join(args.build_dir, 'ios_cwt_chromedriver_tests.app') destination = iossim_util.get_simulator(args.device, args.os) if not os.path.exists(args.out_dir): os.mkdir(args.out_dir) # Make sure each run produces a unique output directory, since reusing an # existing directory will cause CWTChromeDriver's dummy test case to get # skipped, meaning that CWTChromeDriver's http server won't get launched. output_directory = os.path.join(args.out_dir, 'run%d' % int(time.time())) inserted_libs = [] if args.asan_build: inserted_libs = [ os.path.join(args.build_dir, 'libclang_rt.asan_iossim_dynamic.dylib') ] egtests_app = test_apps.EgtestsApp(egtests_app=test_app, test_args=['--port %s' % args.port], host_app_path=host_app, inserted_libs=inserted_libs) launch_command = xcodebuild_runner.LaunchCommand(egtests_app, destination, shards=1, retries=1, out_dir=output_directory) launch_command.launch()
def launch(self): """Launches the test app.""" self.set_up() destination = 'id=%s' % self.udid if self.xctest: test_app = test_apps.EgtestsApp(self.app_path, included_tests=self.test_cases, env_vars=self.env_vars, test_args=self.test_args) elif self.xctest_path: if self.__class__.__name__ == 'DeviceTestRunner': # When self.xctest is False and (bool)self.xctest_path is True and it's # using a device runner, this is a XCTest hosted unit test, which is # currently running on real devices. # TODO(crbug.com/1006881): Separate "running style" from "parser style" # for XCtests and Gtests. test_app = test_apps.DeviceXCTestUnitTestsApp( self.app_path, included_tests=self.test_cases, env_vars=self.env_vars, test_args=self.test_args) else: raise XCTestConfigError( 'Trying to run a DeviceXCTestUnitTestsApp on a' 'non device runner!') else: test_app = test_apps.GTestsApp(self.app_path, included_tests=self.test_cases, env_vars=self.env_vars, test_args=self.test_args) out_dir = os.path.join(self.out_dir, 'TestResults') cmd = self.get_launch_command(test_app, out_dir, destination, self.shards) try: result = self._run(cmd=cmd, shards=self.shards or 1) if result.crashed and not result.crashed_test: # If the app crashed but not during any particular test case, assume # it crashed on startup. Try one more time. self.shutdown_and_restart() LOGGER.warning('Crashed on startup, retrying...\n') out_dir = os.path.join(self.out_dir, 'retry_after_crash_on_startup') cmd = self.get_launch_command(test_app, out_dir, destination, self.shards) result = self._run(cmd) if result.crashed and not result.crashed_test: raise AppLaunchError passed = result.passed_tests failed = result.failed_tests flaked = result.flaked_tests try: while result.crashed and result.crashed_test: # If the app crashes during a specific test case, then resume at the # next test case. This is achieved by filtering out every test case # which has already run. LOGGER.warning('Crashed during %s, resuming...\n', result.crashed_test) test_app.excluded_tests = passed + failed.keys( ) + flaked.keys() retry_out_dir = os.path.join( self.out_dir, 'retry_after_crash_%d' % int(time.time())) result = self._run( self.get_launch_command( test_app, os.path.join(retry_out_dir, str(int(time.time()))), destination)) passed.extend(result.passed_tests) failed.update(result.failed_tests) flaked.update(result.flaked_tests) except OSError as e: if e.errno == errno.E2BIG: LOGGER.error('Too many test cases to resume.') else: raise # Instantiate this after crash retries so that all tests have a first # pass before entering the retry block below. # For each retry that passes, we want to mark it separately as passed # (ie/ "FAIL PASS"), with is_flaky=True. output = sju.StdJson(passed=passed, failed=failed, flaked=flaked) # Retry failed test cases. retry_results = {} test_app.excluded_tests = [] if self.retries and failed: LOGGER.warning('%s tests failed and will be retried.\n', len(failed)) for i in xrange(self.retries): for test in failed.keys(): LOGGER.info('Retry #%s for %s.\n', i + 1, test) test_app.included_tests = [test] retry_out_dir = os.path.join(self.out_dir, test + '_failed', 'retry_%d' % i) retry_result = self._run( self.get_launch_command(test_app, retry_out_dir, destination)) # If the test passed on retry, consider it flake instead of failure. if test in retry_result.passed_tests: flaked[test] = failed.pop(test) output.mark_passed(test, flaky=True) # Save the result of the latest run for each test. retry_results[test] = retry_result # Build test_results.json. # Check if if any of the retries crashed in addition to the original run. interrupted = (result.crashed or any([r.crashed for r in retry_results.values()])) self.test_results['interrupted'] = interrupted self.test_results['num_failures_by_type'] = { 'FAIL': len(failed) + len(flaked), 'PASS': len(passed), } self.test_results['tests'] = output.tests self.logs['passed tests'] = passed if flaked: self.logs['flaked tests'] = flaked if failed: self.logs['failed tests'] = failed for test, log_lines in failed.iteritems(): self.logs[test] = log_lines for test, log_lines in flaked.iteritems(): self.logs[test] = log_lines return not failed and not interrupted finally: self.tear_down()
default='/tmp/cwt_chromedriver', help='Output directory for CWTChromeDriver\'s dummy test case') parser.add_argument('--os', default='14.3', help='iOS version') parser.add_argument('--device', default='iPhone 11 Pro', help='Device type') args = parser.parse_args() test_app = os.path.join(args.build_dir, 'ios_cwt_chromedriver_tests_module-Runner.app') host_app = os.path.join(args.build_dir, 'ios_cwt_chromedriver_tests.app') destination = iossim_util.get_simulator(args.device, args.os) if not os.path.exists(args.out_dir): os.mkdir(args.out_dir) # Make sure each run produces a unique output directory, since reusing an # existing directory will cause CWTChromeDriver's dummy test case to get # skipped, meaning that CWTChromeDriver's http server won't get launched. output_directory = os.path.join(args.out_dir, 'run%d' % int(time.time())) egtests_app = test_apps.EgtestsApp(egtests_app=test_app, test_args=['--port %s' % args.port], host_app_path=host_app) launch_command = xcodebuild_runner.LaunchCommand(egtests_app, destination, shards=1, retries=1, out_dir=output_directory) launch_command.launch()
def launch(self): """Launches tests using xcodebuild.""" launch_commands = [] for params in self.sharding_data: test_app = test_apps.EgtestsApp( params['app'], included_tests=params['test_cases'], env_vars=self.env_vars, test_args=self.test_args, host_app_path=params['host']) launch_commands.append( LaunchCommand(test_app, udid=params['udid'], shards=params['shards'], retries=self.retries, out_dir=os.path.join(self.out_dir, params['udid']), env=self.get_launch_env())) thread_pool = pool.ThreadPool(len(launch_commands)) attempts_results = [] for result in thread_pool.imap_unordered(LaunchCommand.launch, launch_commands): attempts_results.append(result['test_results']['attempts']) # Gets passed tests self.logs['passed tests'] = [] for shard_attempts in attempts_results: for attempt in shard_attempts: self.logs['passed tests'].extend(attempt['passed']) # If the last attempt does not have failures, mark failed as empty self.logs['failed tests'] = [] for shard_attempts in attempts_results: if shard_attempts[-1]['failed']: self.logs['failed tests'].extend( shard_attempts[-1]['failed'].keys()) # Gets all failures/flakes and lists them in bot summary all_failures = set() for shard_attempts in attempts_results: for attempt, attempt_results in enumerate(shard_attempts): for failure in attempt_results['failed']: if failure not in self.logs: self.logs[failure] = [] self.logs[failure].append('%s: attempt # %d' % (failure, attempt)) self.logs[failure].extend( attempt_results['failed'][failure]) all_failures.add(failure) # Gets only flaky(not failed) tests. self.logs['flaked tests'] = list(all_failures - set(self.logs['failed tests'])) # Gets not-started/interrupted tests all_tests_to_run = set(get_all_tests(self.app_path, self.test_cases)) aborted_tests = list(all_tests_to_run - set(self.logs['failed tests']) - set(self.logs['passed tests'])) aborted_tests.sort() self.logs['aborted tests'] = aborted_tests self.test_results['interrupted'] = bool(aborted_tests) self.test_results['num_failures_by_type'] = { 'FAIL': len(self.logs['failed tests'] + self.logs['aborted tests']), 'PASS': len(self.logs['passed tests']), } self.test_results['tests'] = collections.OrderedDict() for shard_attempts in attempts_results: for attempt, attempt_results in enumerate(shard_attempts): for test in attempt_results['failed'].keys( ) + self.logs['aborted tests']: if attempt == len(shard_attempts) - 1: test_result = 'FAIL' else: test_result = self.test_results['tests'].get( test, {}).get('actual', '') + ' FAIL' self.test_results['tests'][test] = { 'expected': 'PASS', 'actual': test_result.strip() } for test in attempt_results['passed']: test_result = self.test_results['tests'].get(test, {}).get( 'actual', '') + ' PASS' self.test_results['tests'][test] = { 'expected': 'PASS', 'actual': test_result.strip() } if 'FAIL' in test_result: self.test_results['tests'][test]['is_flaky'] = True # Test is failed if there are failures for the last run. return not self.logs['failed tests']
def testEgtests_found_xctest(self): self.assertEqual( '/PlugIns/any_egtests.xctest', test_apps.EgtestsApp(_EGTESTS_APP_PATH)._xctest_path())
def launch(self): """Launches tests using xcodebuild.""" launch_commands = [] for params in self.sharding_data: test_app = test_apps.EgtestsApp( params['app'], included_tests=params['test_cases'], env_vars=self.env_vars, test_args=self.test_args, release=self.release, host_app_path=params['host']) launch_commands.append( LaunchCommand( test_app, udid=params['udid'], shards=params['shards'], retries=self.retries, out_dir=os.path.join(self.out_dir, params['udid']), use_clang_coverage=(hasattr(self, 'use_clang_coverage') and self.use_clang_coverage), env=self.get_launch_env())) thread_pool = pool.ThreadPool(len(launch_commands)) attempts_results = [] for result in thread_pool.imap_unordered(LaunchCommand.launch, launch_commands): attempts_results.append(result['test_results']['attempts']) # Gets passed tests self.logs['passed tests'] = [] for shard_attempts in attempts_results: for attempt in shard_attempts: self.logs['passed tests'].extend(attempt['passed']) # If the last attempt does not have failures, mark failed as empty self.logs['failed tests'] = [] for shard_attempts in attempts_results: if shard_attempts[-1]['failed']: self.logs['failed tests'].extend( shard_attempts[-1]['failed'].keys()) # Gets all failures/flakes and lists them in bot summary all_failures = set() for shard_attempts in attempts_results: for attempt, attempt_results in enumerate(shard_attempts): for failure in attempt_results['failed']: if failure not in self.logs: self.logs[failure] = [] self.logs[failure].append('%s: attempt # %d' % (failure, attempt)) self.logs[failure].extend( attempt_results['failed'][failure]) all_failures.add(failure) # Gets only flaky(not failed) tests. self.logs['flaked tests'] = list(all_failures - set(self.logs['failed tests'])) # Gets not-started/interrupted tests. # all_tests_to_run takes into consideration that only a subset of tests may # have run due to the test sharding logic in run.py. all_tests_to_run = set([ test_name for launch_command in launch_commands for test_name in launch_command.egtests_app.get_all_tests() ]) aborted_tests = [] # TODO(crbug.com/1048758): For device targets, the list of test names parsed # from otool output is incorrect. For multitasking or any flaky test suite, # the list contains more tests than what actually runs. if (self.__class__.__name__ != 'DeviceXcodeTestRunner' and 'ios_chrome_multitasking_eg' not in self.app_path and '_flaky_eg' not in self.app_path): aborted_tests = list(all_tests_to_run - set(self.logs['failed tests']) - set(self.logs['passed tests'])) aborted_tests.sort() self.logs['aborted tests'] = aborted_tests self.test_results['interrupted'] = bool(aborted_tests) self.test_results['num_failures_by_type'] = { 'FAIL': len(self.logs['failed tests'] + self.logs['aborted tests']), 'PASS': len(self.logs['passed tests']), } output = sju.StdJson() for shard_attempts in attempts_results: for attempt, attempt_results in enumerate(shard_attempts): for test in attempt_results['failed'].keys(): output.mark_failed(test) # 'aborted tests' in logs is an array of strings, each string defined # as "{TestCase}/{testMethod}" for test in self.logs['aborted tests']: output.mark_timeout(test) for test in attempt_results['passed']: output.mark_passed(test) self.test_results['tests'] = output.tests # Test is failed if there are failures for the last run. # or if there are aborted tests. return not self.logs['failed tests'] and not self.logs['aborted tests']
def testEgtests_not_found_xctest(self, mock_listdir): mock_listdir.return_value = ['random_file'] egtest = test_apps.EgtestsApp(_EGTESTS_APP_PATH) with self.assertRaises(test_runner.XCTestPlugInNotFoundError): egtest._xctest_path()
def testEgtests_xctestRunNode_without_filter(self): egtest_node = test_apps.EgtestsApp( _EGTESTS_APP_PATH).fill_xctestrun_node()['any_egtests_module'] self.assertNotIn('OnlyTestIdentifiers', egtest_node) self.assertNotIn('SkipTestIdentifiers', egtest_node)