def RunTests(device, test_suite, gtest_filter, test_arguments, rebaseline, timeout, performance_test, cleanup_test_files, tool, log_dump_name): """Runs the tests. Args: device: Device to run the tests. test_suite: A specific test suite to run, empty to run all. gtest_filter: A gtest_filter flag. test_arguments: Additional arguments to pass to the test binary. rebaseline: Whether or not to run tests in isolation and update the filter. timeout: Timeout for each test. performance_test: Whether or not performance test(s). cleanup_test_files: Whether or not to cleanup test files on device. tool: Name of the Valgrind tool. log_dump_name: Name of log dump file. Returns: A TestResults object. """ results = [] if test_suite: global _TEST_SUITES if not os.path.exists(test_suite): logging.critical('Unrecognized test suite, supported: %s' % _TEST_SUITES) if test_suite in _TEST_SUITES: logging.critical('(Remember to include the path: out/Release/%s)', test_suite) return TestResults.FromOkAndFailed([], [BaseTestResult(test_suite, '')]) _TEST_SUITES = [test_suite] else: # If not specified, assume the test suites are in out/Release test_suite_dir = os.path.abspath(os.path.join(run_tests_helper.CHROME_DIR, 'out', 'Release')) _TEST_SUITES = [os.path.join(test_suite_dir, t) for t in _TEST_SUITES] debug_info_list = [] for t in _TEST_SUITES: test = SingleTestRunner(device, t, gtest_filter, test_arguments, timeout, rebaseline, performance_test, cleanup_test_files, tool, not not log_dump_name) test.RunTests() results += [test.test_results] # Collect debug info. debug_info_list += [test.dump_debug_info] if rebaseline: test.UpdateFilter(test.test_results.failed) elif test.test_results.failed: # Stop running test if encountering failed test. test.test_results.LogFull() break # Zip all debug info outputs into a file named by log_dump_name. debug_info.GTestDebugInfo.ZipAndCleanResults( os.path.join(run_tests_helper.CHROME_DIR, 'out', 'Release', 'debug_info_dumps'), log_dump_name, [d for d in debug_info_list if d]) return TestResults.FromTestResults(results)
def RunShardedTests(self): """Runs the tests in all connected devices. Returns: A TestResults object. """ logging.warning('*' * 80) logging.warning('Sharding in ' + str(len(self.attached_devices)) + ' devices.') logging.warning('Note that the output is not synchronized.') logging.warning('Look for the "Final result" banner in the end.') logging.warning('*' * 80) final_results = TestResults() for retry in xrange(self.retries): logging.warning('Try %d of %d', retry + 1, self.retries) self.SetupSharding(self.tests) test_runners = [] for index, device in enumerate(self.attached_devices): logging.warning('*' * 80) logging.warning('Creating shard %d for %s', index, device) logging.warning('*' * 80) test_runner = self.CreateShardedTestRunner(device, index) test_runners += [test_runner] logging.warning('Starting...') pool = multiprocessing.Pool(len(self.attached_devices), SetTestsContainer, [BaseTestSharder.tests_container]) # map can't handle KeyboardInterrupt exception. It's a python bug. # So use map_async instead. async_results = pool.map_async(_ShardedTestRunnable, test_runners) results_lists = async_results.get(999999) test_results = TestResults.FromTestResults(results_lists) # Re-check the attached devices for some devices may # become offline retry_devices = set(android_commands.GetAttachedDevices()) # Remove devices that had exceptions. retry_devices -= TestResults.DeviceExceptions(results_lists) # Retry on devices that didn't have any exception. self.attached_devices = list(retry_devices) if (retry == self.retries - 1 or len(self.attached_devices) == 0): all_passed = final_results.ok + test_results.ok final_results = test_results final_results.ok = all_passed break else: final_results.ok += test_results.ok self.tests = [] for t in test_results.GetAllBroken(): self.tests += [t.name] if not self.tests: break self.OnTestsCompleted(test_runners, final_results) return final_results
def RunShardedTests(self): """Runs tests in parallel using a pool of workers. Returns: A list of test results aggregated from all test runs. """ logging.warning('*' * 80) logging.warning('Sharding in ' + str(len(self.attached_devices)) + ' devices.') logging.warning('Note that the output is not synchronized.') logging.warning('Look for the "Final result" banner in the end.') logging.warning('*' * 80) all_passed = [] test_results = TestResults() tests_to_run = self.tests for retry in xrange(self.retries): logging.warning('Try %d of %d', retry + 1, self.retries) self._SetupSharding(self.tests) test_runners = self._MakeTestRunners(self.attached_devices) logging.warning('Starting...') pool = multiprocessing.Pool(len(self.attached_devices), SetTestsContainer, [PythonTestSharder.tests_container]) # List of TestResults objects from each test execution. try: results_lists = pool.map(_DefaultRunnable, test_runners) except Exception: logging.exception('Unable to run tests. Something with the ' 'PythonTestRunners has gone wrong.') raise FatalTestException( 'PythonTestRunners were unable to run tests.') test_results = TestResults.FromTestResults(results_lists) # Accumulate passing results. all_passed += test_results.ok # If we have failed tests, map them to tests to retry. failed_tests = test_results.GetAllBroken() tests_to_run = self._GetTestsToRetry(self.tests, failed_tests) # Bail out early if we have no more tests. This can happen if all tests # pass before we're out of retries, for example. if not tests_to_run: break final_results = TestResults() # all_passed has accumulated all passing test results. # test_results will have the results from the most recent run, which could # include a variety of failure modes (unknown, crashed, failed, etc). final_results = test_results final_results.ok = all_passed return final_results
def RunTests(self): """Runs tests from the shared pool of tests, aggregating results. Returns: A list of test results for all of the tests which this runner executed. """ tests = PythonTestSharder.tests_container results = [] for t in tests: res = CallPythonTest(t, self.device_id, self.shard_index) results.append(res) return TestResults.FromTestResults(results)
def _RunPythonTests(tests_to_run, device_id): """Runs a list of Python tests serially on one device and returns results. Args: tests_to_run: a list of objects inheriting from PythonTestBase. device_id: ID of the device to run tests on. Returns: A list of test results, aggregated across all the tests run. """ # This is a list of TestResults objects. results = [CallPythonTest(t, device_id, 0) for t in tests_to_run] # Merge the list of TestResults into one TestResults. return TestResults.FromTestResults(results)
def RunShardedTests(self): """Runs the tests in all connected devices. Returns: A TestResults object. """ logging.warning('*' * 80) logging.warning('Sharding in ' + str(len(self.attached_devices)) + ' devices.') logging.warning('Note that the output is not synchronized.') logging.warning('Look for the "Final result" banner in the end.') logging.warning('*' * 80) final_results = TestResults() for retry in xrange(self.retries): logging.warning('Try %d of %d', retry + 1, self.retries) self.SetupSharding(self.tests) test_runners = [] for index, device in enumerate(self.attached_devices): logging.warning('*' * 80) logging.warning('Creating shard %d for %s', index, device) logging.warning('*' * 80) test_runner = self.CreateShardedTestRunner(device, index) test_runners += [test_runner] logging.warning('Starting...') pool = multiprocessing.Pool(len(self.attached_devices), SetTestsContainer, [BaseTestSharder.tests_container]) results_lists = pool.map(_ShardedTestRunnable, test_runners) test_results = TestResults.FromTestResults(results_lists) if retry == self.retries - 1: all_passed = final_results.ok + test_results.ok final_results = test_results final_results.ok = all_passed break else: final_results.ok += test_results.ok self.tests = [] for t in test_results.GetAllBroken(): self.tests += [t.name] if not self.tests: break self.OnTestsCompleted(test_runners, final_results) return final_results
def RunTests(device, test_suite, gtest_filter, test_arguments, rebaseline, timeout, performance_test, cleanup_test_files, tool, log_dump_name, apk, annotate=False): """Runs the tests. Args: device: Device to run the tests. test_suite: A specific test suite to run, empty to run all. gtest_filter: A gtest_filter flag. test_arguments: Additional arguments to pass to the test binary. rebaseline: Whether or not to run tests in isolation and update the filter. timeout: Timeout for each test. performance_test: Whether or not performance test(s). cleanup_test_files: Whether or not to cleanup test files on device. tool: Name of the Valgrind tool. log_dump_name: Name of log dump file. apk: boolean to state if we are using the apk based test runner annotate: should we print buildbot-style annotations? Returns: A TestResults object. """ results = [] if test_suite: global _TEST_SUITES if (not os.path.exists(test_suite) and not os.path.splitext(test_suite)[1] == '.apk'): logging.critical('Unrecognized test suite %s, supported: %s' % (test_suite, _TEST_SUITES)) if test_suite in _TEST_SUITES: logging.critical('(Remember to include the path: out/Release/%s)', test_suite) return TestResults.FromOkAndFailed([], [BaseTestResult(test_suite, '')], False, False) fully_qualified_test_suites = [test_suite] else: fully_qualified_test_suites = FullyQualifiedTestSuites(apk) debug_info_list = [] print 'Known suites: ' + str(_TEST_SUITES) print 'Running these: ' + str(fully_qualified_test_suites) for t in fully_qualified_test_suites: if annotate: print '@@@BUILD_STEP Test suite %s@@@' % os.path.basename(t) test = SingleTestRunner(device, t, gtest_filter, test_arguments, timeout, rebaseline, performance_test, cleanup_test_files, tool, 0, not not log_dump_name) test.Run() results += [test.test_results] # Collect debug info. debug_info_list += [test.dump_debug_info] if rebaseline: test.UpdateFilter(test.test_results.failed) test.test_results.LogFull() # Zip all debug info outputs into a file named by log_dump_name. debug_info.GTestDebugInfo.ZipAndCleanResults( os.path.join(run_tests_helper.CHROME_DIR, 'out', 'Release', 'debug_info_dumps'), log_dump_name, [d for d in debug_info_list if d]) if annotate: if test.test_results.timed_out: print '@@@STEP_WARNINGS@@@' elif test.test_results.failed: print '@@@STEP_FAILURE@@@' elif test.test_results.overall_fail: print '@@@STEP_FAILURE@@@' else: print 'Step success!' # No annotation needed return TestResults.FromTestResults(results)
def RunShardedTests(self): """Runs the tests in all connected devices. Returns: A TestResults object. """ logging.warning('*' * 80) logging.warning('Sharding in ' + str(len(self.attached_devices)) + ' devices.') logging.warning('Note that the output is not synchronized.') logging.warning('Look for the "Final result" banner in the end.') logging.warning('*' * 80) final_results = TestResults() self._KillHostForwarder() for retry in xrange(self.retries): logging.warning('Try %d of %d', retry + 1, self.retries) self.SetupSharding(self.tests) test_runners = [] # Try to create N shards, and retrying on failure. try: for index, device in enumerate(self.attached_devices): logging.warning('*' * 80) logging.warning('Creating shard %d for %s', index, device) logging.warning('*' * 80) test_runner = self.CreateShardedTestRunner(device, index) test_runners += [test_runner] except errors.DeviceUnresponsiveError as e: logging.critical('****Failed to create a shard: [%s]', e) self.attached_devices.remove(device) continue logging.warning('Starting...') pool = multiprocessing.Pool(len(self.attached_devices), SetTestsContainer, [BaseTestSharder.tests_container]) # map can't handle KeyboardInterrupt exception. It's a python bug. # So use map_async instead. async_results = pool.map_async(_ShardedTestRunnable, test_runners) try: results_lists = async_results.get(999999) except errors.DeviceUnresponsiveError as e: logging.critical('****Failed to run test: [%s]', e) self.attached_devices = android_commands.GetAttachedDevices() continue test_results = TestResults.FromTestResults(results_lists) # Re-check the attached devices for some devices may # become offline retry_devices = set(android_commands.GetAttachedDevices()) # Remove devices that had exceptions. retry_devices -= TestResults.DeviceExceptions(results_lists) # Retry on devices that didn't have any exception. self.attached_devices = list(retry_devices) if (retry == self.retries - 1 or len(self.attached_devices) == 0): all_passed = final_results.ok + test_results.ok final_results = test_results final_results.ok = all_passed break else: final_results.ok += test_results.ok self.tests = [] for t in test_results.GetAllBroken(): self.tests += [t.name] if not self.tests: break else: # We ran out retries, possibly out of healthy devices. # There's no recovery at this point. raise Exception('Unrecoverable error while retrying test runs.') self.OnTestsCompleted(test_runners, final_results) self._KillHostForwarder() return final_results