def RunShardedTests(self): """Runs tests in parallel using a pool of workers. Returns: A list of test results aggregated from all test runs. """ logging.warning('*' * 80) logging.warning('Sharding in ' + str(len(self.attached_devices)) + ' devices.') logging.warning('Note that the output is not synchronized.') logging.warning('Look for the "Final result" banner in the end.') logging.warning('*' * 80) all_passed = [] test_results = TestResults() tests_to_run = self.tests for retry in xrange(self.retries): logging.warning('Try %d of %d', retry + 1, self.retries) self._SetupSharding(self.tests) test_runners = self._MakeTestRunners(self.attached_devices) logging.warning('Starting...') pool = multiprocessing.Pool(len(self.attached_devices), SetTestsContainer, [PythonTestSharder.tests_container]) # List of TestResults objects from each test execution. try: results_lists = pool.map(_DefaultRunnable, test_runners) except Exception: logging.exception('Unable to run tests. Something with the ' 'PythonTestRunners has gone wrong.') raise FatalTestException( 'PythonTestRunners were unable to run tests.') test_results = TestResults.FromTestResults(results_lists) # Accumulate passing results. all_passed += test_results.ok # If we have failed tests, map them to tests to retry. failed_tests = test_results.GetAllBroken() tests_to_run = self._GetTestsToRetry(self.tests, failed_tests) # Bail out early if we have no more tests. This can happen if all tests # pass before we're out of retries, for example. if not tests_to_run: break final_results = TestResults() # all_passed has accumulated all passing test results. # test_results will have the results from the most recent run, which could # include a variety of failure modes (unknown, crashed, failed, etc). final_results = test_results final_results.ok = all_passed return final_results
def __init__(self, device, test_suite, gtest_filter, test_arguments, timeout, rebaseline, performance_test, cleanup_test_files, tool, shard_index, dump_debug_info=False, fast_and_loose=False): BaseTestRunner.__init__(self, device, shard_index) self._running_on_emulator = self.device.startswith('emulator') self._gtest_filter = gtest_filter self._test_arguments = test_arguments self.test_results = TestResults() if dump_debug_info: self.dump_debug_info = debug_info.GTestDebugInfo(self.adb, device, os.path.basename(test_suite), gtest_filter) else: self.dump_debug_info = None self.fast_and_loose = fast_and_loose if os.path.splitext(test_suite)[1] == '.apk': self.test_package = TestPackageApk( self.adb, device, test_suite, timeout, rebaseline, performance_test, cleanup_test_files, tool, self.dump_debug_info) else: self.test_package = TestPackageExecutable( self.adb, device, test_suite, timeout, rebaseline, performance_test, cleanup_test_files, tool, self.dump_debug_info)
def __init__(self, device, test_suite, gtest_filter, test_arguments, timeout, cleanup_test_files, tool_name, shard_index, dump_debug_info, fast_and_loose, build_type, in_webkit_checkout): BaseTestRunner.__init__(self, device, tool_name, shard_index, build_type) self._running_on_emulator = self.device.startswith('emulator') self._gtest_filter = gtest_filter self._test_arguments = test_arguments self.test_results = TestResults() if dump_debug_info: self.dump_debug_info = debug_info.GTestDebugInfo( self.adb, device, os.path.basename(test_suite), gtest_filter) else: self.dump_debug_info = None self.fast_and_loose = fast_and_loose self.in_webkit_checkout = in_webkit_checkout logging.warning('Test suite: ' + test_suite) if os.path.splitext(test_suite)[1] == '.apk': self.test_package = TestPackageApk(self.adb, device, test_suite, timeout, cleanup_test_files, self.tool, self.dump_debug_info) else: # Put a copy into the android out/target directory, to allow stack trace # generation. symbols_dir = os.path.join(constants.CHROME_DIR, 'out', build_type, 'lib.target') self.test_package = TestPackageExecutable( self.adb, device, test_suite, timeout, cleanup_test_files, self.tool, self.dump_debug_info, symbols_dir)
def __init__(self, device, test_suite, gtest_filter, test_arguments, timeout, rebaseline, performance_test, cleanup_test_files, tool_name, shard_index, dump_debug_info, fast_and_loose, build_type): BaseTestRunner.__init__(self, device, tool_name, shard_index, build_type) self._running_on_emulator = self.device.startswith('emulator') self._gtest_filter = gtest_filter self._test_arguments = test_arguments self.test_results = TestResults() if dump_debug_info: self.dump_debug_info = debug_info.GTestDebugInfo( self.adb, device, os.path.basename(test_suite), gtest_filter) else: self.dump_debug_info = None self.fast_and_loose = fast_and_loose logging.warning('Test suite: ' + test_suite) if os.path.splitext(test_suite)[1] == '.apk': self.test_package = TestPackageApk(self.adb, device, test_suite, timeout, rebaseline, performance_test, cleanup_test_files, self.tool, self.dump_debug_info) else: self.test_package = TestPackageExecutable( self.adb, device, test_suite, timeout, rebaseline, performance_test, cleanup_test_files, self.tool, self.dump_debug_info) self._performance_test_setup = None if performance_test: self._performance_test_setup = perf_tests_helper.PerfTestSetup( self.adb)
def RunShardedTests(self): """Runs the tests in all connected devices. Returns: A TestResults object. """ logging.warning('*' * 80) logging.warning('Sharding in ' + str(len(self.attached_devices)) + ' devices.') logging.warning('Note that the output is not synchronized.') logging.warning('Look for the "Final result" banner in the end.') logging.warning('*' * 80) final_results = TestResults() for retry in xrange(self.retries): logging.warning('Try %d of %d', retry + 1, self.retries) self.SetupSharding(self.tests) test_runners = [] for index, device in enumerate(self.attached_devices): logging.warning('*' * 80) logging.warning('Creating shard %d for %s', index, device) logging.warning('*' * 80) test_runner = self.CreateShardedTestRunner(device, index) test_runners += [test_runner] logging.warning('Starting...') pool = multiprocessing.Pool(len(self.attached_devices), SetTestsContainer, [BaseTestSharder.tests_container]) # map can't handle KeyboardInterrupt exception. It's a python bug. # So use map_async instead. async_results = pool.map_async(_ShardedTestRunnable, test_runners) results_lists = async_results.get(999999) test_results = TestResults.FromTestResults(results_lists) # Re-check the attached devices for some devices may # become offline retry_devices = set(android_commands.GetAttachedDevices()) # Remove devices that had exceptions. retry_devices -= TestResults.DeviceExceptions(results_lists) # Retry on devices that didn't have any exception. self.attached_devices = list(retry_devices) if (retry == self.retries - 1 or len(self.attached_devices) == 0): all_passed = final_results.ok + test_results.ok final_results = test_results final_results.ok = all_passed break else: final_results.ok += test_results.ok self.tests = [] for t in test_results.GetAllBroken(): self.tests += [t.name] if not self.tests: break self.OnTestsCompleted(test_runners, final_results) return final_results
def DispatchPythonTests(options): """Dispatches the Python tests. If there are multiple devices, use sharding. Args: options: command line options. Returns: A list of test results. """ attached_devices = android_commands.GetAttachedDevices() if not attached_devices: raise FatalTestException('You have no devices attached or visible!') if options.device: attached_devices = [options.device] test_collection = TestInfoCollection() all_tests = _GetAllTests(options.python_test_root, options.official_build) test_collection.AddTests(all_tests) test_names = [t.qualified_name for t in all_tests] logging.debug('All available tests: ' + str(test_names)) available_tests = test_collection.GetAvailableTests( options.annotation, options.test_filter) if not available_tests: logging.warning('No Python tests to run with current args.') return TestResults() available_tests *= options.number_of_runs test_names = [t.qualified_name for t in available_tests] logging.debug('Final list of tests to run: ' + str(test_names)) # Copy files to each device before running any tests. for device_id in attached_devices: logging.debug('Pushing files to device %s', device_id) apks = [ apk_info.ApkInfo(options.test_apk_path, options.test_apk_jar_path) ] test_files_copier = run_java_tests.TestRunner(options, device_id, None, False, 0, apks, []) test_files_copier.CopyTestFilesOnce() # Actually run the tests. if (len(attached_devices) > 1 and not options.wait_for_debugger): logging.debug('Sharding Python tests.') sharder = PythonTestSharder(attached_devices, options.shard_retries, available_tests) test_results = sharder.RunShardedTests() else: logging.debug('Running Python tests serially.') test_results = _RunPythonTests(available_tests, attached_devices[0]) return test_results
def _ProcessResults(self, result, start_ms, duration_ms): """Translates a Java test result into a Python result for this test. The TestRunner class that we use under the covers will return a test result for that specific Java test. However, to make reporting clearer, we have this method to abstract that detail and instead report that as a failure of this particular test case while still including the Java stack trace. Args: result: TestResults with a single Java test result start_ms: the time the test started duration_ms: the length of the test Returns: A TestResults object containing a result for this Python test. """ test_results = TestResults() # If our test is in broken, then it crashed/failed. broken = result.GetAllBroken() if broken: # Since we have run only one test, take the first and only item. single_result = broken[0] log = single_result.log if not log: log = 'No logging information.' short_error_msg = single_result.log.split('\n')[0] # err_info is ostensibly for Sponge to consume; it's a short error # message and a longer one. err_info = (short_error_msg, log) python_result = SingleTestResult(self.qualified_name, start_ms, duration_ms, PYTHON, log, err_info) # Figure out where the test belonged. There's probably a cleaner way of # doing this. if single_result in result.crashed: test_results.crashed = [python_result] elif single_result in result.failed: test_results.failed = [python_result] elif single_result in result.unknown: test_results.unknown = [python_result] else: python_result = SingleTestResult(self.qualified_name, start_ms, duration_ms, PYTHON) test_results.ok = [python_result] return test_results
def _ShardedTestRunnable(test): """Standalone function needed by multiprocessing.Pool.""" log_format = '[' + test.device + '] # %(asctime)-15s: %(message)s' if logging.getLogger().handlers: logging.getLogger().handlers[0].setFormatter( logging.Formatter(log_format)) else: logging.basicConfig(format=log_format) # Handle SystemExit here since python has a bug to exit current process try: return test.Run() except SystemExit: return TestResults()
def RunShardedTests(self): """Runs the tests in all connected devices. Returns: A TestResults object. """ logging.warning('*' * 80) logging.warning('Sharding in ' + str(len(self.attached_devices)) + ' devices.') logging.warning('Note that the output is not synchronized.') logging.warning('Look for the "Final result" banner in the end.') logging.warning('*' * 80) final_results = TestResults() for retry in xrange(self.retries): logging.warning('Try %d of %d', retry + 1, self.retries) self.SetupSharding(self.tests) test_runners = [] for index, device in enumerate(self.attached_devices): logging.warning('*' * 80) logging.warning('Creating shard %d for %s', index, device) logging.warning('*' * 80) test_runner = self.CreateShardedTestRunner(device, index) test_runners += [test_runner] logging.warning('Starting...') pool = multiprocessing.Pool(len(self.attached_devices), SetTestsContainer, [BaseTestSharder.tests_container]) results_lists = pool.map(_ShardedTestRunnable, test_runners) test_results = TestResults.FromTestResults(results_lists) if retry == self.retries - 1: all_passed = final_results.ok + test_results.ok final_results = test_results final_results.ok = all_passed break else: final_results.ok += test_results.ok self.tests = [] for t in test_results.GetAllBroken(): self.tests += [t.name] if not self.tests: break self.OnTestsCompleted(test_runners, final_results) return final_results
def __init__(self, options, device, tests_iter, coverage, shard_index, apks, ports_to_forward): """Create a new TestRunner. Args: options: An options object with the following required attributes: - build_type: 'Release' or 'Debug'. - install_apk: Re-installs the apk if opted. - save_perf_json: Whether or not to save the JSON file from UI perf tests. - screenshot_failures: Take a screenshot for a test failure - tool: Name of the Valgrind tool. - wait_for_debugger: blocks until the debugger is connected. - disable_assertions: Whether to disable java assertions on the device. device: Attached android device. tests_iter: A list of tests to be run. coverage: Collects coverage information if opted. shard_index: shard # for this TestRunner, used to create unique port numbers. apks: A list of ApkInfo objects need to be installed. The first element should be the tests apk, the rests could be the apks used in test. The default is ChromeTest.apk. ports_to_forward: A list of port numbers for which to set up forwarders. Can be optionally requested by a test case. Raises: FatalTestException: if coverage metadata is not available. """ BaseTestRunner.__init__(self, device, options.tool, shard_index, options.build_type) if not apks: apks = [ apk_info.ApkInfo(options.test_apk_path, options.test_apk_jar_path) ] self.build_type = options.build_type self.install_apk = options.install_apk self.test_data = options.test_data self.save_perf_json = options.save_perf_json self.screenshot_failures = options.screenshot_failures self.wait_for_debugger = options.wait_for_debugger self.disable_assertions = options.disable_assertions self.tests_iter = tests_iter self.coverage = coverage self.apks = apks self.test_apk = apks[0] self.instrumentation_class_path = self.test_apk.GetPackageName() self.ports_to_forward = ports_to_forward self.test_results = TestResults() self.forwarder = None if self.coverage: if os.path.exists(TestRunner._COVERAGE_MERGED_FILENAME): os.remove(TestRunner._COVERAGE_MERGED_FILENAME) if not os.path.exists(TestRunner._COVERAGE_META_INFO_PATH): raise FatalTestException('FATAL ERROR in ' + sys.argv[0] + ' : Coverage meta info [' + TestRunner._COVERAGE_META_INFO_PATH + '] does not exist.') if (not TestRunner._COVERAGE_WEB_ROOT_DIR or not os.path.exists(TestRunner._COVERAGE_WEB_ROOT_DIR)): raise FatalTestException( 'FATAL ERROR in ' + sys.argv[0] + ' : Path specified in $EMMA_WEB_ROOTDIR [' + TestRunner._COVERAGE_WEB_ROOT_DIR + '] does not exist.')
def DispatchJavaTests(options, apks): """Dispatches Java tests onto connected device(s). If possible, this method will attempt to shard the tests to all connected devices. Otherwise, dispatch and run tests on one device. Args: options: Command line options. apks: list of APKs to use. Returns: A TestResults object holding the results of the Java tests. Raises: FatalTestException: when there's no attached the devices. """ test_apk = apks[0] if options.annotation: available_tests = test_apk.GetAnnotatedTests(options.annotation) if len(options.annotation ) == 1 and options.annotation[0] == 'SmallTest': tests_without_annotation = [ m for m in test_apk.GetTestMethods() if not test_apk.GetTestAnnotations(m) and not apk_info.ApkInfo.IsPythonDrivenTest(m) ] if tests_without_annotation: tests_without_annotation.sort() logging.warning( 'The following tests do not contain any annotation. ' 'Assuming "SmallTest":\n%s', '\n'.join(tests_without_annotation)) available_tests += tests_without_annotation else: available_tests = [ m for m in test_apk.GetTestMethods() if not apk_info.ApkInfo.IsPythonDrivenTest(m) ] coverage = os.environ.get('EMMA_INSTRUMENT') == 'true' tests = [] if options.test_filter: # |available_tests| are in adb instrument format: package.path.class#test. filter_without_hash = options.test_filter.replace('#', '.') tests = [ t for t in available_tests if filter_without_hash in t.replace('#', '.') ] else: tests = available_tests if not tests: logging.warning('No Java tests to run with current args.') return TestResults() tests *= options.number_of_runs attached_devices = android_commands.GetAttachedDevices() test_results = TestResults() if not attached_devices: raise FatalTestException('You have no devices attached or visible!') if options.device: attached_devices = [options.device] logging.info('Will run: %s', str(tests)) if len(attached_devices) > 1 and (coverage or options.wait_for_debugger): logging.warning('Coverage / debugger can not be sharded, ' 'using first available device') attached_devices = attached_devices[:1] sharder = TestSharder(attached_devices, options, tests, apks) test_results = sharder.RunShardedTests() return test_results
def RunShardedTests(self): """Runs the tests in all connected devices. Returns: A TestResults object. """ logging.warning('*' * 80) logging.warning('Sharding in ' + str(len(self.attached_devices)) + ' devices.') logging.warning('Note that the output is not synchronized.') logging.warning('Look for the "Final result" banner in the end.') logging.warning('*' * 80) final_results = TestResults() self._KillHostForwarder() for retry in xrange(self.retries): logging.warning('Try %d of %d', retry + 1, self.retries) self.SetupSharding(self.tests) test_runners = [] # Try to create N shards, and retrying on failure. try: for index, device in enumerate(self.attached_devices): logging.warning('*' * 80) logging.warning('Creating shard %d for %s', index, device) logging.warning('*' * 80) test_runner = self.CreateShardedTestRunner(device, index) test_runners += [test_runner] except errors.DeviceUnresponsiveError as e: logging.critical('****Failed to create a shard: [%s]', e) self.attached_devices.remove(device) continue logging.warning('Starting...') pool = multiprocessing.Pool(len(self.attached_devices), SetTestsContainer, [BaseTestSharder.tests_container]) # map can't handle KeyboardInterrupt exception. It's a python bug. # So use map_async instead. async_results = pool.map_async(_ShardedTestRunnable, test_runners) try: results_lists = async_results.get(999999) except errors.DeviceUnresponsiveError as e: logging.critical('****Failed to run test: [%s]', e) self.attached_devices = android_commands.GetAttachedDevices() continue test_results = TestResults.FromTestResults(results_lists) # Re-check the attached devices for some devices may # become offline retry_devices = set(android_commands.GetAttachedDevices()) # Remove devices that had exceptions. retry_devices -= TestResults.DeviceExceptions(results_lists) # Retry on devices that didn't have any exception. self.attached_devices = list(retry_devices) if (retry == self.retries - 1 or len(self.attached_devices) == 0): all_passed = final_results.ok + test_results.ok final_results = test_results final_results.ok = all_passed break else: final_results.ok += test_results.ok self.tests = [] for t in test_results.GetAllBroken(): self.tests += [t.name] if not self.tests: break else: # We ran out retries, possibly out of healthy devices. # There's no recovery at this point. raise Exception('Unrecoverable error while retrying test runs.') self.OnTestsCompleted(test_runners, final_results) self._KillHostForwarder() return final_results