def RunJavaTest(fname, suite, test, ports_to_forward): device = android_commands.GetAttachedDevices()[0] package_name = _GetPackageName(fname) test = package_name + '.' + suite + '#' + test java_test_runner = TestRunner(False, device, [test], False, False, False, False, 0, ports_to_forward) return java_test_runner.Run()
def _RunATestSuite(options): """Run a single test suite. Helper for Dispatch() to allow stop/restart of the emulator across test bundles. If using the emulator, we start it on entry and stop it on exit. Args: options: options for running the tests. Returns: 0 if successful, number of failing tests otherwise. """ attached_devices = [] buildbot_emulator = None if options.use_emulator: t = TimeProfile('Emulator launch') buildbot_emulator = emulator.Emulator(options.fast_and_loose) buildbot_emulator.Launch() t.Stop() attached_devices.append(buildbot_emulator.device) else: attached_devices = android_commands.GetAttachedDevices() if not attached_devices: logging.critical('A device must be attached and online.') return 1 test_results = RunTests(attached_devices[0], options.test_suite, options.gtest_filter, options.test_arguments, options.rebaseline, options.timeout, options.performance_test, options.cleanup_test_files, options.tool, options.log_dump, fast_and_loose=options.fast_and_loose, annotate=options.annotate) if buildbot_emulator: buildbot_emulator.Shutdown() # Another chance if we timed out? At this point It is safe(r) to # run fast and loose since we just uploaded all the test data and # binary. if test_results.timed_out and options.repeat: logging.critical('Timed out; repeating in fast_and_loose mode.') options.fast_and_loose = True options.repeat = options.repeat - 1 logging.critical('Repeats left: ' + str(options.repeat)) return _RunATestSuite(options) return len(test_results.failed)
def RunShardedTests(self): """Runs the tests in all connected devices. Returns: A TestResults object. """ logging.warning('*' * 80) logging.warning('Sharding in ' + str(len(self.attached_devices)) + ' devices.') logging.warning('Note that the output is not synchronized.') logging.warning('Look for the "Final result" banner in the end.') logging.warning('*' * 80) final_results = TestResults() for retry in xrange(self.retries): logging.warning('Try %d of %d', retry + 1, self.retries) self.SetupSharding(self.tests) test_runners = [] for index, device in enumerate(self.attached_devices): logging.warning('*' * 80) logging.warning('Creating shard %d for %s', index, device) logging.warning('*' * 80) test_runner = self.CreateShardedTestRunner(device, index) test_runners += [test_runner] logging.warning('Starting...') pool = multiprocessing.Pool(len(self.attached_devices), SetTestsContainer, [BaseTestSharder.tests_container]) # map can't handle KeyboardInterrupt exception. It's a python bug. # So use map_async instead. async_results = pool.map_async(_ShardedTestRunnable, test_runners) results_lists = async_results.get(999999) test_results = TestResults.FromTestResults(results_lists) # Re-check the attached devices for some devices may # become offline retry_devices = set(android_commands.GetAttachedDevices()) # Remove devices that had exceptions. retry_devices -= TestResults.DeviceExceptions(results_lists) # Retry on devices that didn't have any exception. self.attached_devices = list(retry_devices) if (retry == self.retries - 1 or len(self.attached_devices) == 0): all_passed = final_results.ok + test_results.ok final_results = test_results final_results.ok = all_passed break else: final_results.ok += test_results.ok self.tests = [] for t in test_results.GetAllBroken(): self.tests += [t.name] if not self.tests: break self.OnTestsCompleted(test_runners, final_results) return final_results
def DispatchPythonTests(options): """Dispatches the Python tests. If there are multiple devices, use sharding. Args: options: command line options. Returns: A list of test results. """ attached_devices = android_commands.GetAttachedDevices() if not attached_devices: raise FatalTestException('You have no devices attached or visible!') if options.device: attached_devices = [options.device] test_collection = TestInfoCollection() all_tests = _GetAllTests(options.python_test_root, options.official_build) test_collection.AddTests(all_tests) test_names = [t.qualified_name for t in all_tests] logging.debug('All available tests: ' + str(test_names)) available_tests = test_collection.GetAvailableTests( options.annotation, options.test_filter) if not available_tests: logging.warning('No Python tests to run with current args.') return TestResults() available_tests *= options.number_of_runs test_names = [t.qualified_name for t in available_tests] logging.debug('Final list of tests to run: ' + str(test_names)) # Copy files to each device before running any tests. for device_id in attached_devices: logging.debug('Pushing files to device %s', device_id) apks = [ apk_info.ApkInfo(options.test_apk_path, options.test_apk_jar_path) ] test_files_copier = run_java_tests.TestRunner(options, device_id, None, False, 0, apks, []) test_files_copier.CopyTestFilesOnce() # Actually run the tests. if (len(attached_devices) > 1 and not options.wait_for_debugger): logging.debug('Sharding Python tests.') sharder = PythonTestSharder(attached_devices, options.shard_retries, available_tests) test_results = sharder.RunShardedTests() else: logging.debug('Running Python tests serially.') test_results = _RunPythonTests(available_tests, attached_devices[0]) return test_results
def Dispatch(options): """Dispatches the tests, sharding if possible. If options.use_emulator is True, all tests will be run in a new emulator instance. Args: options: options for running the tests. Returns: 0 if successful, number of failing tests otherwise. """ if options.test_suite == 'help': ListTestSuites() return 0 buildbot_emulator = None attached_devices = [] if options.use_xvfb: xvfb = Xvfb() xvfb.Start() if options.use_emulator: t = TimeProfile('Emulator launch') buildbot_emulator = emulator.Emulator() buildbot_emulator.Launch() t.Stop() attached_devices.append(buildbot_emulator.device) else: attached_devices = android_commands.GetAttachedDevices() if not attached_devices: logging.critical('A device must be attached and online.') return 1 test_results = RunTests(attached_devices[0], options.test_suite, options.gtest_filter, options.test_arguments, options.rebaseline, options.timeout, options.performance_test, options.cleanup_test_files, options.tool, options.log_dump) if buildbot_emulator: buildbot_emulator.Shutdown() if options.use_xvfb: xvfb.Stop() return len(test_results.failed)
def _RunATestSuite(options): """Run a single test suite. Helper for Dispatch() to allow stop/restart of the emulator across test bundles. If using the emulator, we start it on entry and stop it on exit. Args: options: options for running the tests. Returns: 0 if successful, number of failing tests otherwise. """ attached_devices = [] buildbot_emulators = [] if options.use_emulator: for n in range(options.use_emulator): t = TimeProfile('Emulator launch %d' % n) buildbot_emulator = emulator.Emulator(options.fast_and_loose) buildbot_emulator.Launch(kill_all_emulators=n == 0) t.Stop() buildbot_emulators.append(buildbot_emulator) attached_devices.append(buildbot_emulator.device) # Wait for all emulators to become available. map(lambda buildbot_emulator:buildbot_emulator.ConfirmLaunch(), buildbot_emulators) else: attached_devices = android_commands.GetAttachedDevices() if not attached_devices: logging.critical('A device must be attached and online.') return 1 if (len(attached_devices) > 1 and options.test_suite and not options.gtest_filter and not options.performance_test): sharder = TestSharder(attached_devices, options.test_suite, options.gtest_filter, options.test_arguments, options.timeout, options.rebaseline, options.performance_test, options.cleanup_test_files, options.tool) test_results = sharder.RunShardedTests() else: test_results = RunTests(attached_devices[0], options.test_suite, options.gtest_filter, options.test_arguments, options.rebaseline, options.timeout, options.performance_test, options.cleanup_test_files, options.tool, options.log_dump, options.apk, annotate=options.annotate) for buildbot_emulator in buildbot_emulators: buildbot_emulator.Shutdown() # Another chance if we timed out? At this point It is safe(r) to # run fast and loose since we just uploaded all the test data and # binary. if test_results.timed_out and options.repeat: logging.critical('Timed out; repeating in fast_and_loose mode.') options.fast_and_loose = True options.repeat = options.repeat - 1 logging.critical('Repeats left: ' + str(options.repeat)) return _RunATestSuite(options) return len(test_results.failed)
def DispatchJavaTests(options, apks): """Dispatches Java tests onto connected device(s). If possible, this method will attempt to shard the tests to all connected devices. Otherwise, dispatch and run tests on one device. Args: options: Command line options. apks: list of APKs to use. Returns: A TestResults object holding the results of the Java tests. Raises: FatalTestException: when there's no attached the devices. """ test_apk = apks[0] if options.annotation: available_tests = test_apk.GetAnnotatedTests(options.annotation) if len(options.annotation ) == 1 and options.annotation[0] == 'SmallTest': tests_without_annotation = [ m for m in test_apk.GetTestMethods() if not test_apk.GetTestAnnotations(m) and not apk_info.ApkInfo.IsPythonDrivenTest(m) ] if tests_without_annotation: tests_without_annotation.sort() logging.warning( 'The following tests do not contain any annotation. ' 'Assuming "SmallTest":\n%s', '\n'.join(tests_without_annotation)) available_tests += tests_without_annotation else: available_tests = [ m for m in test_apk.GetTestMethods() if not apk_info.ApkInfo.IsPythonDrivenTest(m) ] coverage = os.environ.get('EMMA_INSTRUMENT') == 'true' tests = [] if options.test_filter: # |available_tests| are in adb instrument format: package.path.class#test. filter_without_hash = options.test_filter.replace('#', '.') tests = [ t for t in available_tests if filter_without_hash in t.replace('#', '.') ] else: tests = available_tests if not tests: logging.warning('No Java tests to run with current args.') return TestResults() tests *= options.number_of_runs attached_devices = android_commands.GetAttachedDevices() test_results = TestResults() if not attached_devices: raise FatalTestException('You have no devices attached or visible!') if options.device: attached_devices = [options.device] logging.info('Will run: %s', str(tests)) if len(attached_devices) > 1 and (coverage or options.wait_for_debugger): logging.warning('Coverage / debugger can not be sharded, ' 'using first available device') attached_devices = attached_devices[:1] sharder = TestSharder(attached_devices, options, tests, apks) test_results = sharder.RunShardedTests() return test_results
def RunShardedTests(self): """Runs the tests in all connected devices. Returns: A TestResults object. """ logging.warning('*' * 80) logging.warning('Sharding in ' + str(len(self.attached_devices)) + ' devices.') logging.warning('Note that the output is not synchronized.') logging.warning('Look for the "Final result" banner in the end.') logging.warning('*' * 80) final_results = TestResults() self._KillHostForwarder() for retry in xrange(self.retries): logging.warning('Try %d of %d', retry + 1, self.retries) self.SetupSharding(self.tests) test_runners = [] # Try to create N shards, and retrying on failure. try: for index, device in enumerate(self.attached_devices): logging.warning('*' * 80) logging.warning('Creating shard %d for %s', index, device) logging.warning('*' * 80) test_runner = self.CreateShardedTestRunner(device, index) test_runners += [test_runner] except errors.DeviceUnresponsiveError as e: logging.critical('****Failed to create a shard: [%s]', e) self.attached_devices.remove(device) continue logging.warning('Starting...') pool = multiprocessing.Pool(len(self.attached_devices), SetTestsContainer, [BaseTestSharder.tests_container]) # map can't handle KeyboardInterrupt exception. It's a python bug. # So use map_async instead. async_results = pool.map_async(_ShardedTestRunnable, test_runners) try: results_lists = async_results.get(999999) except errors.DeviceUnresponsiveError as e: logging.critical('****Failed to run test: [%s]', e) self.attached_devices = android_commands.GetAttachedDevices() continue test_results = TestResults.FromTestResults(results_lists) # Re-check the attached devices for some devices may # become offline retry_devices = set(android_commands.GetAttachedDevices()) # Remove devices that had exceptions. retry_devices -= TestResults.DeviceExceptions(results_lists) # Retry on devices that didn't have any exception. self.attached_devices = list(retry_devices) if (retry == self.retries - 1 or len(self.attached_devices) == 0): all_passed = final_results.ok + test_results.ok final_results = test_results final_results.ok = all_passed break else: final_results.ok += test_results.ok self.tests = [] for t in test_results.GetAllBroken(): self.tests += [t.name] if not self.tests: break else: # We ran out retries, possibly out of healthy devices. # There's no recovery at this point. raise Exception('Unrecoverable error while retrying test runs.') self.OnTestsCompleted(test_runners, final_results) self._KillHostForwarder() return final_results