def test_timeout(self): """Test setting timeout""" test = 'e' output = sju.StdJson() output.mark_timeout(test) self.assertEqual(output.tests['e']['actual'], 'TIMEOUT') output = sju.StdJson() output.mark_failed(test) self.assertEqual(output.tests['e']['actual'], 'FAIL') output.mark_timeout(test) self.assertTrue(output.tests['e']['actual'], 'FAIL TIMEOUT')
def test_single_constructor(self): """Test one test passing, failing, flaking via constructor""" test = 'a' output = sju.StdJson(passed=test) self.assertEqual(output.tests['a']['actual'], 'PASS') output = sju.StdJson(failed=test) self.assertEqual(output.tests['a']['actual'], 'FAIL') output = sju.StdJson(passed=test, flaky=True) self.assertEqual(output.tests['a']['actual'], 'PASS') self.assertTrue(output.tests['a']['is_flaky'])
def test_skip(self): """Test setting expected skip.""" test = 'f' output = sju.StdJson() output.mark_skipped(test) self.assertEqual(output.tests['f']['actual'], 'SKIP') self.assertFalse(output.tests['f'].get('is_unexpected', False))
def test_base_cases(self): """Test invalid test names are skipped""" passed = ['', None] failed = ['', None] output = sju.StdJson(passed=passed, failed=failed) self.assertFalse(output.tests)
def test_shard(self): """Test shard into is written to test result.""" test = 'f' output = sju.StdJson() output.mark_passed(test) self.assertEqual(output.tests['f']['shard'], os.getenv('GTEST_SHARD_INDEX', 0))
def test_post_to_result_sink(self, mock_result_sink_class): result_sink = mock_result_sink_class.return_value passed_test = 'passed_test' failed_test = 'failed_test' skipped_test = 'skipped_test' timeout_test = 'timeout_test' output = sju.StdJson() output.mark_passed('passed_test') result_sink.post.assert_called_with('passed_test', 'PASS', True) output.mark_failed('failed_test', 'logs') result_sink.post.assert_called_with( 'failed_test', 'FAIL', False, test_log='logs') output.mark_disabled('skipped_test') result_sink.post.assert_called_with( 'skipped_test', 'SKIP', True, tags=[('disabled_test', 'true')]) output.mark_timeout('timeout_test') timeout_log = ( 'The test is compiled in test target but was unexpectedly not' ' run or not finished.') result_sink.post.assert_called_with( 'timeout_test', 'SKIP', False, test_log=timeout_log, tags=[('disabled_test', 'false')])
def test_flaky_without_explicit(self): """Test setting pass on an already failed test, w/o explicit flaky""" test = 'e' output = sju.StdJson() output.mark_failed(test) self.assertEqual(output.tests['e']['actual'], 'FAIL') output.mark_passed(test) self.assertEqual(output.tests['e']['actual'], 'FAIL PASS') self.assertTrue(output.tests['e']['is_flaky'])
def test_multi_run(self): """Test multiple executions of the same test""" test = 'a' output = sju.StdJson(passed=test) self.assertEqual(output.tests['a']['actual'], 'PASS') output.mark_failed(test) self.assertEqual(output.tests['a']['actual'], 'PASS FAIL') output.mark_pass(test, flaky=True) self.assertEqual(output.tests['a']['actual'], 'PASS FAIL PASS') self.assertTrue(output.tests['a']['is_flaky'])
def test_multi_scenario(self): """Test a scenario where some tests pass, fail and flake""" passed = ['a', 'b', 'c'] failed = ['d'] flaked = ['e'] output = sju.StdJson(passed=passed, failed=failed, flaked=flaked) self.assertEqual(len(output.tests), 5) # Ensure that the flaked is set as passed, with is_flaky=True self.assertTrue(output.tests['e']['is_flaky']) # A retry that re-runs failed fails again output.mark_failed('d') self.assertEqual(output.tests['d']['actual'], 'FAIL FAIL') # Another retry of 'd' passes, so we set it as a flaky pass output.mark_passed('d', flaky=True) self.assertEqual(output.tests['d']['actual'], 'FAIL FAIL PASS') self.assertTrue(output.tests['d']['is_flaky'])
def launch(self): """Launches tests using xcodebuild.""" launch_commands = [] for params in self.sharding_data: test_app = self.get_launch_test_app(params) launch_commands.append( LaunchCommand( test_app, udid=params['udid'], shards=params['shards'], retries=self.retries, out_dir=os.path.join(self.out_dir, params['udid']), use_clang_coverage=(hasattr(self, 'use_clang_coverage') and self.use_clang_coverage), env=self.get_launch_env())) thread_pool = pool.ThreadPool(len(launch_commands)) attempts_results = [] for result in thread_pool.imap_unordered(LaunchCommand.launch, launch_commands): attempts_results.append(result['test_results']['attempts']) # Deletes simulator used in the tests after tests end. if iossim_util.is_device_with_udid_simulator(self.udid): iossim_util.delete_simulator_by_udid(self.udid) # Gets passed tests self.logs['passed tests'] = [] for shard_attempts in attempts_results: for attempt in shard_attempts: self.logs['passed tests'].extend(attempt['passed']) # If the last attempt does not have failures, mark failed as empty self.logs['failed tests'] = [] for shard_attempts in attempts_results: if shard_attempts[-1]['failed']: self.logs['failed tests'].extend( shard_attempts[-1]['failed'].keys()) # Gets disabled tests from test app object if any. self.logs['disabled tests'] = [] for launch_command in launch_commands: self.logs['disabled tests'].extend( launch_command.egtests_app.disabled_tests) # Gets all failures/flakes and lists them in bot summary all_failures = set() for shard_attempts in attempts_results: for attempt, attempt_results in enumerate(shard_attempts): for failure in attempt_results['failed']: if failure not in self.logs: self.logs[failure] = [] self.logs[failure].append('%s: attempt # %d' % (failure, attempt)) self.logs[failure].extend( attempt_results['failed'][failure]) all_failures.add(failure) # Gets only flaky(not failed) tests. self.logs['flaked tests'] = list(all_failures - set(self.logs['failed tests'])) # Gets not-started/interrupted tests. # all_tests_to_run takes into consideration that only a subset of tests may # have run due to the test sharding logic in run.py. all_tests_to_run = set([ test_name for launch_command in launch_commands for test_name in launch_command.egtests_app.get_all_tests() ]) aborted_tests = [] # TODO(crbug.com/1048758): For device targets, the list of test names parsed # from otool output is incorrect. For multitasking or any flaky test suite, # the list contains more tests than what actually runs. if (self.__class__.__name__ != 'DeviceXcodeTestRunner' and 'ios_chrome_multitasking_eg' not in self.app_path and '_flaky_eg' not in self.app_path): aborted_tests = list(all_tests_to_run - set(self.logs['failed tests']) - set(self.logs['passed tests'])) aborted_tests.sort() self.logs['aborted tests'] = aborted_tests self.test_results['interrupted'] = bool(aborted_tests) self.test_results['num_failures_by_type'] = { 'FAIL': len(self.logs['failed tests'] + self.logs['aborted tests']), 'PASS': len(self.logs['passed tests']), } output = sju.StdJson() for shard_attempts in attempts_results: for attempt, attempt_results in enumerate(shard_attempts): for test in attempt_results['failed'].keys(): # TODO(crbug.com/1178923): Remove unicode check when it's figured out # where unicode is introduced. log_lines = [] for line in self.logs.get(test, []): if sys.version_info.major == 2: if isinstance(line, unicode): LOGGER.warning('Unicode string: %s' % line) line = line.encode('utf-8') log_lines.append(line) output.mark_failed(test, test_log='\n'.join(log_lines)) # 'aborted tests' in logs is an array of strings, each string defined # as "{TestCase}/{testMethod}" for test in self.logs['aborted tests']: output.mark_timeout(test) for test in attempt_results['passed']: output.mark_passed(test) output.mark_all_disabled(self.logs['disabled tests']) output.finalize() self.test_results['tests'] = output.tests # Test is failed if there are failures for the last run. # or if there are aborted tests. return not self.logs['failed tests'] and not self.logs['aborted tests']
def launch(self): """Launches the test app.""" self.set_up() destination = 'id=%s' % self.udid if self.xctest: test_app = test_apps.EgtestsApp(self.app_path, included_tests=self.test_cases, env_vars=self.env_vars, test_args=self.test_args) elif self.xctest_path: if self.__class__.__name__ == 'DeviceTestRunner': # When self.xctest is False and (bool)self.xctest_path is True and it's # using a device runner, this is a XCTest hosted unit test, which is # currently running on real devices. # TODO(crbug.com/1006881): Separate "running style" from "parser style" # for XCtests and Gtests. test_app = test_apps.DeviceXCTestUnitTestsApp( self.app_path, included_tests=self.test_cases, env_vars=self.env_vars, test_args=self.test_args) else: raise XCTestConfigError( 'Trying to run a DeviceXCTestUnitTestsApp on a' 'non device runner!') else: test_app = test_apps.GTestsApp(self.app_path, included_tests=self.test_cases, env_vars=self.env_vars, test_args=self.test_args) out_dir = os.path.join(self.out_dir, 'TestResults') cmd = self.get_launch_command(test_app, out_dir, destination, self.shards) try: result = self._run(cmd=cmd, shards=self.shards or 1) if result.crashed and not result.crashed_test: # If the app crashed but not during any particular test case, assume # it crashed on startup. Try one more time. self.shutdown_and_restart() LOGGER.warning('Crashed on startup, retrying...\n') out_dir = os.path.join(self.out_dir, 'retry_after_crash_on_startup') cmd = self.get_launch_command(test_app, out_dir, destination, self.shards) result = self._run(cmd) if result.crashed and not result.crashed_test: raise AppLaunchError passed = result.passed_tests failed = result.failed_tests flaked = result.flaked_tests try: while result.crashed and result.crashed_test: # If the app crashes during a specific test case, then resume at the # next test case. This is achieved by filtering out every test case # which has already run. LOGGER.warning('Crashed during %s, resuming...\n', result.crashed_test) test_app.excluded_tests = passed + failed.keys( ) + flaked.keys() retry_out_dir = os.path.join( self.out_dir, 'retry_after_crash_%d' % int(time.time())) result = self._run( self.get_launch_command( test_app, os.path.join(retry_out_dir, str(int(time.time()))), destination)) passed.extend(result.passed_tests) failed.update(result.failed_tests) flaked.update(result.flaked_tests) except OSError as e: if e.errno == errno.E2BIG: LOGGER.error('Too many test cases to resume.') else: raise # Instantiate this after crash retries so that all tests have a first # pass before entering the retry block below. # For each retry that passes, we want to mark it separately as passed # (ie/ "FAIL PASS"), with is_flaky=True. output = sju.StdJson(passed=passed, failed=failed, flaked=flaked) # Retry failed test cases. retry_results = {} test_app.excluded_tests = [] if self.retries and failed: LOGGER.warning('%s tests failed and will be retried.\n', len(failed)) for i in xrange(self.retries): for test in failed.keys(): LOGGER.info('Retry #%s for %s.\n', i + 1, test) test_app.included_tests = [test] retry_out_dir = os.path.join(self.out_dir, test + '_failed', 'retry_%d' % i) retry_result = self._run( self.get_launch_command(test_app, retry_out_dir, destination)) # If the test passed on retry, consider it flake instead of failure. if test in retry_result.passed_tests: flaked[test] = failed.pop(test) output.mark_passed(test, flaky=True) # Save the result of the latest run for each test. retry_results[test] = retry_result # Build test_results.json. # Check if if any of the retries crashed in addition to the original run. interrupted = (result.crashed or any([r.crashed for r in retry_results.values()])) self.test_results['interrupted'] = interrupted self.test_results['num_failures_by_type'] = { 'FAIL': len(failed) + len(flaked), 'PASS': len(passed), } self.test_results['tests'] = output.tests self.logs['passed tests'] = passed if flaked: self.logs['flaked tests'] = flaked if failed: self.logs['failed tests'] = failed for test, log_lines in failed.iteritems(): self.logs[test] = log_lines for test, log_lines in flaked.iteritems(): self.logs[test] = log_lines return not failed and not interrupted finally: self.tear_down()
def launch(self): """Launches the test app.""" self.set_up() destination = 'id=%s' % self.udid # When current |launch| method is invoked, this is running a unit test # target. For simulators, '--xctest' is passed to test runner scripts to # make it run XCTest based unit test. if self.xctest: # TODO(crbug.com/1085603): Pass in test runner an arg to determine if it's # device test or simulator test and test the arg here. if self.__class__.__name__ == 'SimulatorTestRunner': test_app = test_apps.SimulatorXCTestUnitTestsApp( self.app_path, included_tests=self.test_cases, env_vars=self.env_vars, test_args=self.test_args) elif self.__class__.__name__ == 'DeviceTestRunner': test_app = test_apps.DeviceXCTestUnitTestsApp( self.app_path, included_tests=self.test_cases, env_vars=self.env_vars, test_args=self.test_args) else: raise XCTestConfigError('Wrong config. TestRunner.launch() called from' ' an unexpected class.') else: test_app = test_apps.GTestsApp( self.app_path, included_tests=self.test_cases, env_vars=self.env_vars, test_args=self.test_args) out_dir = os.path.join(self.out_dir, 'TestResults') cmd = self.get_launch_command(test_app, out_dir, destination, self.shards) try: result = self._run(cmd=cmd, shards=self.shards or 1) if result.crashed and not result.crashed_test: # If the app crashed but not during any particular test case, assume # it crashed on startup. Try one more time. self.shutdown_and_restart() LOGGER.warning('Crashed on startup, retrying...\n') out_dir = os.path.join(self.out_dir, 'retry_after_crash_on_startup') cmd = self.get_launch_command(test_app, out_dir, destination, self.shards) result = self._run(cmd) if result.crashed and not result.crashed_test: raise AppLaunchError passed = result.passed_tests failed = result.failed_tests flaked = result.flaked_tests disabled = result.disabled_tests_from_compiled_tests_file try: while result.crashed and result.crashed_test: # If the app crashes during a specific test case, then resume at the # next test case. This is achieved by filtering out every test case # which has already run. LOGGER.warning('Crashed during %s, resuming...\n', result.crashed_test) test_app.excluded_tests = passed + failed.keys() + flaked.keys() retry_out_dir = os.path.join( self.out_dir, 'retry_after_crash_%d' % int(time.time())) result = self._run( self.get_launch_command( test_app, os.path.join(retry_out_dir, str(int(time.time()))), destination)) passed.extend(result.passed_tests) failed.update(result.failed_tests) flaked.update(result.flaked_tests) if not disabled: disabled = result.disabled_tests_from_compiled_tests_file except OSError as e: if e.errno == errno.E2BIG: LOGGER.error('Too many test cases to resume.') else: raise # Instantiate this after crash retries so that all tests have a first # pass before entering the retry block below. # For each retry that passes, we want to mark it separately as passed # (ie/ "FAIL PASS"), with is_flaky=True. # TODO(crbug.com/1132476): Report failed GTest logs to ResultSink. output = sju.StdJson(passed=passed, failed=failed, flaked=flaked) # Retry failed test cases. retry_results = {} test_app.excluded_tests = [] if self.retries and failed: LOGGER.warning('%s tests failed and will be retried.\n', len(failed)) for i in xrange(self.retries): for test in failed.keys(): LOGGER.info('Retry #%s for %s.\n', i + 1, test) test_app.included_tests = [test] retry_out_dir = os.path.join(self.out_dir, test + '_failed', 'retry_%d' % i) retry_result = self._run( self.get_launch_command(test_app, retry_out_dir, destination)) # If the test passed on retry, consider it flake instead of failure. if test in retry_result.passed_tests: flaked[test] = failed.pop(test) output.mark_passed(test, flaky=True) # Save the result of the latest run for each test. retry_results[test] = retry_result output.mark_all_skipped(disabled) # Build test_results.json. # Check if if any of the retries crashed in addition to the original run. interrupted = (result.crashed or any([r.crashed for r in retry_results.values()])) self.test_results['interrupted'] = interrupted self.test_results['num_failures_by_type'] = { 'FAIL': len(failed) + len(flaked), 'PASS': len(passed), } self.test_results['tests'] = output.tests self.logs['passed tests'] = passed if disabled: self.logs['disabled tests'] = disabled if flaked: self.logs['flaked tests'] = flaked if failed: self.logs['failed tests'] = failed for test, log_lines in failed.iteritems(): self.logs[test] = log_lines for test, log_lines in flaked.iteritems(): self.logs[test] = log_lines return not failed and not interrupted finally: self.tear_down()
def launch(self): """Launches tests using xcodebuild.""" launch_commands = [] for params in self.sharding_data: test_app = test_apps.EgtestsApp( params['app'], included_tests=params['test_cases'], env_vars=self.env_vars, test_args=self.test_args, release=self.release, host_app_path=params['host']) launch_commands.append( LaunchCommand( test_app, udid=params['udid'], shards=params['shards'], retries=self.retries, out_dir=os.path.join(self.out_dir, params['udid']), use_clang_coverage=(hasattr(self, 'use_clang_coverage') and self.use_clang_coverage), env=self.get_launch_env())) thread_pool = pool.ThreadPool(len(launch_commands)) attempts_results = [] for result in thread_pool.imap_unordered(LaunchCommand.launch, launch_commands): attempts_results.append(result['test_results']['attempts']) # Gets passed tests self.logs['passed tests'] = [] for shard_attempts in attempts_results: for attempt in shard_attempts: self.logs['passed tests'].extend(attempt['passed']) # If the last attempt does not have failures, mark failed as empty self.logs['failed tests'] = [] for shard_attempts in attempts_results: if shard_attempts[-1]['failed']: self.logs['failed tests'].extend( shard_attempts[-1]['failed'].keys()) # Gets all failures/flakes and lists them in bot summary all_failures = set() for shard_attempts in attempts_results: for attempt, attempt_results in enumerate(shard_attempts): for failure in attempt_results['failed']: if failure not in self.logs: self.logs[failure] = [] self.logs[failure].append('%s: attempt # %d' % (failure, attempt)) self.logs[failure].extend( attempt_results['failed'][failure]) all_failures.add(failure) # Gets only flaky(not failed) tests. self.logs['flaked tests'] = list(all_failures - set(self.logs['failed tests'])) # Gets not-started/interrupted tests. # all_tests_to_run takes into consideration that only a subset of tests may # have run due to the test sharding logic in run.py. all_tests_to_run = set([ test_name for launch_command in launch_commands for test_name in launch_command.egtests_app.get_all_tests() ]) aborted_tests = [] # TODO(crbug.com/1048758): For device targets, the list of test names parsed # from otool output is incorrect. For multitasking or any flaky test suite, # the list contains more tests than what actually runs. if (self.__class__.__name__ != 'DeviceXcodeTestRunner' and 'ios_chrome_multitasking_eg' not in self.app_path and '_flaky_eg' not in self.app_path): aborted_tests = list(all_tests_to_run - set(self.logs['failed tests']) - set(self.logs['passed tests'])) aborted_tests.sort() self.logs['aborted tests'] = aborted_tests self.test_results['interrupted'] = bool(aborted_tests) self.test_results['num_failures_by_type'] = { 'FAIL': len(self.logs['failed tests'] + self.logs['aborted tests']), 'PASS': len(self.logs['passed tests']), } output = sju.StdJson() for shard_attempts in attempts_results: for attempt, attempt_results in enumerate(shard_attempts): for test in attempt_results['failed'].keys(): output.mark_failed(test) # 'aborted tests' in logs is an array of strings, each string defined # as "{TestCase}/{testMethod}" for test in self.logs['aborted tests']: output.mark_timeout(test) for test in attempt_results['passed']: output.mark_passed(test) self.test_results['tests'] = output.tests # Test is failed if there are failures for the last run. # or if there are aborted tests. return not self.logs['failed tests'] and not self.logs['aborted tests']