def wait_for_results(self, report, workqueue, test_filters): console = ndk.ansi.get_console() ui = ndk.test.ui.get_test_build_progress_ui(console, workqueue) with ndk.ansi.disable_terminal_echo(sys.stdin): with console.cursor_hide_context(): while not workqueue.finished(): suite, result, additional_tests = workqueue.get_result() # Filtered test. Skip them entirely to avoid polluting # --show-all results. if result is None: assert not additional_tests ui.draw() continue assert result.passed() or not additional_tests for test in additional_tests: workqueue.add_task(_run_test, suite, test, self.obj_dir, self.dist_dir, test_filters) if logger().isEnabledFor(logging.INFO): ui.clear() self.printer.print_result(result) elif result.failed(): ui.clear() self.printer.print_result(result) report.add_result(suite, result) ui.draw() ui.clear()
def clear_test_directories(workqueue, fleet): for group in fleet.get_unique_device_groups(): for device in group.devices: workqueue.add_task(clear_test_directory, device) while not workqueue.finished(): workqueue.get_result()
def do_build(self, test_filters): workqueue = ndk.test.builder.LoadRestrictingWorkQueue() try: for suite, tests in self.tests.items(): # Each test configuration was expanded when each test was # discovered, so the current order has all the largest tests # right next to each other. Spread them out to try to avoid # having too many heavy builds happening simultaneously. random.shuffle(tests) for test in tests: if not test_filters.filter(test.name): continue if test.name == 'libc++': workqueue.add_load_restricted_task( _run_test, suite, test, self.obj_dir, self.dist_dir, test_filters) else: workqueue.add_task(_run_test, suite, test, self.obj_dir, self.dist_dir, test_filters) report = ndk.test.report.Report() self.wait_for_results(report, workqueue, test_filters) return report finally: workqueue.terminate() workqueue.join()
def push_tests_to_devices(workqueue, test_dir, groups_for_config, use_sync): dest_dir = DEVICE_TEST_BASE_DIR for config, groups in groups_for_config.items(): src_dir = os.path.join(test_dir, str(config)) for group in groups: for device in group.devices: workqueue.add_task(push_tests_to_device, src_dir, dest_dir, config, device, use_sync) finish_workqueue_with_ui(workqueue) print('Finished pushing tests')
def test_subprocess_exception(self): """Tests that exceptions raised in the task are re-raised.""" workqueue = ndk.workqueue.DummyWorkQueue() try: workqueue.add_task(raise_error) with self.assertRaises(ndk.workqueue.TaskError): workqueue.get_result() finally: workqueue.terminate() workqueue.join()
def test_finished(self): """Tests that finished() returns the correct result.""" workqueue = ndk.workqueue.WorkQueue() self.assertTrue(workqueue.finished()) workqueue.add_task(put, 1) self.assertFalse(workqueue.finished()) workqueue.get_result() self.assertTrue(workqueue.finished()) workqueue.terminate() workqueue.join() self.assertTrue(workqueue.finished())
def restart_flaky_tests(report, workqueue): """Finds and restarts any failing flaky tests.""" rerun_tests = report.remove_all_failing_flaky(flake_filter) if len(rerun_tests) > 0: cooldown = 10 logger().warning( 'Found %d flaky failures. Sleeping for %d seconds to let ' 'devices recover.', len(rerun_tests), cooldown) time.sleep(cooldown) for flaky_report in rerun_tests: logger().warning('Flaky test failure: %s', flaky_report.result) group = flaky_report.result.test.device_group workqueue.add_task(group, run_test, flaky_report.result.test)
def test_put_func(self): """Test that we can pass a function to the queue and get results.""" workqueue = ndk.workqueue.WorkQueue(4) workqueue.add_task(put, 1) workqueue.add_task(put, 2) expected_results = [1, 2] while expected_results: i = workqueue.get_result() self.assertIn(i, expected_results) expected_results.remove(i) workqueue.terminate() workqueue.join()
def test_put_functor(self): """Test that we can pass a functor to the queue and get results.""" workqueue = ndk.workqueue.DummyWorkQueue() workqueue.add_task(Functor(1)) workqueue.add_task(Functor(2)) expected_results = [1, 2] while expected_results: i = workqueue.get_result() self.assertIn(i, expected_results) expected_results.remove(i) workqueue.terminate() workqueue.join()
def test_finished(self): """Tests that finished() returns the correct result.""" workqueue = ndk.workqueue.WorkQueue(4) self.assertTrue(workqueue.finished()) manager = multiprocessing.Manager() event = manager.Event() workqueue.add_task(block_on_event, event) self.assertFalse(workqueue.finished()) event.set() workqueue.get_result() self.assertTrue(workqueue.finished()) workqueue.terminate() workqueue.join() self.assertTrue(workqueue.finished())
def perform_asan_setup(workqueue, ndk_path, groups_for_config): # asan_device_setup is a shell script, so no asan there. if os.name == 'nt': return devices = [] for groups in groups_for_config.values(): for group in groups: devices.extend(group.devices) devices = sorted(list(set(devices))) for device in devices: if device.can_use_asan(): workqueue.add_task(setup_asan_for_device, ndk_path, device) finish_workqueue_with_ui(workqueue) print('Finished ASAN setup')
def test_status(self): """Tests that worker status can be accessed from the parent.""" workqueue = ndk.workqueue.WorkQueue(1) manager = multiprocessing.Manager() ready_event = manager.Event() finish_event = manager.Event() self.assertEqual(ndk.workqueue.Worker.IDLE_STATUS, workqueue.workers[0].status) workqueue.add_task(update_status, ready_event, finish_event, 'working') ready_event.wait() self.assertEqual('working', workqueue.workers[0].status) finish_event.set() workqueue.get_result() self.assertEqual(ndk.workqueue.Worker.IDLE_STATUS, workqueue.workers[0].status) workqueue.terminate() workqueue.join()
def test_subprocesses_killed(self): """Tests that terminate() kills descendents of worker processes.""" workqueue = ndk.workqueue.WorkQueue(4) manager = multiprocessing.Manager() queue = manager.Queue() workqueue.add_task(spawn_child, queue) pids = [] pids.append(queue.get()) pids.append(queue.get()) workqueue.terminate() workqueue.join() killed_pid = queue.get() self.assertIn(killed_pid, pids) pids.remove(killed_pid) killed_pid = queue.get() self.assertIn(killed_pid, pids) pids.remove(killed_pid)
def main(): logging.basicConfig() total_timer = build_support.Timer() total_timer.start() # It seems the build servers run us in our own session, in which case we # get EPERM from `setpgrp`. No need to call this in that case because we # will already be the process group leader. if os.getpid() != os.getsid(os.getpid()): os.setpgrp() args = parse_args() if args.modules is None: modules = get_all_module_names() else: modules = args.modules if args.host_only: modules = [ 'clang', 'gcc', 'host-tools', 'ndk-build', 'python-packages', 'renderscript-toolchain', 'shader-tools', 'simpleperf', ] required_package_modules = set(get_all_module_names()) have_required_modules = required_package_modules <= set(modules) if (args.package and have_required_modules) or args.force_package: do_package = True else: do_package = False # TODO(danalbert): wine? # We're building the Windows packages from Linux, so we can't actually run # any of the tests from here. if args.system.startswith('windows') or not do_package: args.test = False # Disable buffering on stdout so the build output doesn't hide all of our # "Building..." messages. sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0) os.chdir(os.path.dirname(os.path.realpath(__file__))) # Set ANDROID_BUILD_TOP. if 'ANDROID_BUILD_TOP' in os.environ: sys.exit(textwrap.dedent("""\ Error: ANDROID_BUILD_TOP is already set in your environment. This typically means you are running in a shell that has lunched a target in a platform build. The platform environment interferes with the NDK build environment, so the build cannot continue. Launch a new shell before building the NDK.""")) os.environ['ANDROID_BUILD_TOP'] = os.path.realpath('..') out_dir = build_support.get_out_dir() dist_dir = build_support.get_dist_dir(out_dir) tmp_dir = os.path.join(out_dir, 'build') if os.path.exists(tmp_dir): shutil.rmtree(tmp_dir) os.mkdir(tmp_dir) os.environ['TMPDIR'] = tmp_dir print('Cleaning up...') invoke_build('dev-cleanup.sh') print('Building modules: {}'.format(' '.join(modules))) print('Machine has {} CPUs'.format(multiprocessing.cpu_count())) log_dir = os.path.join(dist_dir, 'logs') if not os.path.exists(log_dir): os.makedirs(log_dir) build_timer = build_support.Timer() with build_timer: workqueue = ndk.workqueue.WorkQueue(args.jobs) try: for module in ALL_MODULES: if module.name in modules: workqueue.add_task( launch_build, module, out_dir, dist_dir, args, log_dir) while not workqueue.finished(): build_name, result, log_path = workqueue.get_result() if result: print('BUILD SUCCESSFUL: ' + build_name) else: # Kill all the children so the error we print appears last. workqueue.terminate() workqueue.join() print('BUILD FAILED: ' + build_name) with open(log_path, 'r') as log_file: contents = log_file.read() print(contents) # The build server has a build_error.log file that is # supposed to be the short log of the failure that # stopped the build. Append our failing log to that. build_error_log = os.path.join( dist_dir, 'logs/build_error.log') with open(build_error_log, 'a') as error_log: error_log.write('\n') error_log.write(contents) sys.exit(1) finally: workqueue.terminate() workqueue.join() ndk_dir = ndk.paths.get_install_path(out_dir) install_timer = build_support.Timer() with install_timer: if not os.path.exists(ndk_dir): os.makedirs(ndk_dir) for module in ALL_MODULES: if module.name in modules: module.install(out_dir, dist_dir, args) package_timer = build_support.Timer() with package_timer: if do_package: host_tag = build_support.host_to_tag(args.system) package_ndk(ndk_dir, dist_dir, host_tag, args.build_number) good = True test_timer = build_support.Timer() with test_timer: if args.test: good = test_ndk(out_dir, dist_dir, args) print() # Blank line between test results and timing data. total_timer.finish() print('Finished {}'.format('successfully' if good else 'unsuccessfully')) print('Build: {}'.format(build_timer.duration)) print('Install: {}'.format(install_timer.duration)) print('Packaging: {}'.format(package_timer.duration)) print('Testing: {}'.format(test_timer.duration)) print('Total: {}'.format(total_timer.duration)) subject = 'NDK Build {}!'.format('Passed' if good else 'Failed') body = 'Build finished in {}'.format(total_timer.duration) ndk.notify.toast(subject, body) sys.exit(not good)