def launch(loop: asyncio.AbstractEventLoop, workspace: Path, print_log: bool, print_metric_log: bool, verbose: bool): config_file = workspace / 'config.json' if not workspace.exists(): print(f'{workspace.resolve()} is not exist.', file=sys.stderr) return False elif not config_file.exists(): print(f'{config_file.resolve()} is not exist.', file=sys.stderr) return False with config_file.open() as local_config_fp, \ GLOBAL_CFG_PATH.open() as global_config_fp: local_cfg_source: Dict[str, Any] = json.load(local_config_fp) global_cfg_source: Dict[str, Any] = json.load(global_config_fp) bench_cfgs = parse_workload_cfg(local_cfg_source['workloads']) perf_cfg = parse_perf_cfg(global_cfg_source['perf'], local_cfg_source.get('perf', {'extra_events': []})) rabbit_cfg = parse_rabbit_mq_cfg(global_cfg_source['rabbitMQ']) launcher_cfg = parse_launcher_cfg(local_cfg_source.get('launcher')) task_map: Dict[asyncio.Task, Benchmark] = dict() was_successful = True result_file = workspace / 'result.json' if result_file.exists(): result_file.unlink() def stop_all(): print('force stop all tasks...', file=sys.stderr) for a_task in task_map: if not a_task.done(): a_task.cancel() nonlocal was_successful was_successful = False def store_runtime(a_task: asyncio.Task): a_bench = task_map[a_task] if result_file.exists(): with result_file.open('r+') as fp: try: original: Dict[str, Any] = json.load(fp) if 'runtime' in original: original['runtime'][a_bench.identifier] = a_bench.runtime else: original['runtime'] = {a_bench.identifier: a_bench.runtime} fp.seek(0) json.dump(original, fp, indent=4) except json.decoder.JSONDecodeError: fp.seek(0) fp.truncate() json.dump({'runtime': {a_bench.identifier: a_bench.runtime}}, fp, indent=4) else: with result_file.open('w') as fp: json.dump({'runtime': {a_bench.identifier: a_bench.runtime}}, fp, indent=4) a_task.remove_done_callback(store_runtime) benches = tuple(create_benchmarks(bench_cfgs, perf_cfg, rabbit_cfg, workspace, verbose)) loop.add_signal_handler(signal.SIGHUP, stop_all) loop.add_signal_handler(signal.SIGTERM, stop_all) loop.add_signal_handler(signal.SIGINT, stop_all) with power_monitor(workspace), hyper_threading_guard(loop, launcher_cfg.hyper_threading): # invoke benchmark loaders in parallel and wait for launching actual benchmarks loop.run_until_complete(asyncio.wait(tuple(bench.start_and_pause(print_log) for bench in benches))) for bench in benches: bench.resume() for bench in benches: task: asyncio.Task = loop.create_task(bench.monitor(print_metric_log)) task.add_done_callback(store_runtime) task_map[task] = bench # start monitoring return_when = asyncio.FIRST_COMPLETED if launcher_cfg.stops_with_the_first else asyncio.ALL_COMPLETED finished, unfinished = loop.run_until_complete(asyncio.wait(task_map.keys(), return_when=return_when)) for task in unfinished: task.cancel() loop.run_until_complete(asyncio.gather(*unfinished)) loop.remove_signal_handler(signal.SIGHUP) loop.remove_signal_handler(signal.SIGTERM) loop.remove_signal_handler(signal.SIGINT) # run post scripts for script in launcher_cfg.post_scripts: name = str(script) if name.endswith('.py'): name = name[:-3] script_module = importlib.import_module(name.replace('/', '.')) script_module.run(workspace, GLOBAL_CFG_PATH) return was_successful
def on_shutdown(loop: asyncio.AbstractEventLoop, server: VisibilityWriterServer) -> None: # in case the exit code below borks, we allow shutdown via traditional means loop.remove_signal_handler(signal.SIGINT) loop.remove_signal_handler(signal.SIGTERM) server.halt()
def run_single( app: Type[ASGIFramework], config: Config, *, loop: asyncio.AbstractEventLoop, sock: Optional[socket] = None, is_child: bool = False, ) -> None: """Create a server to run the app on given the options. Arguments: app: The ASGI Framework to run. config: The configuration that defines the server. loop: Asyncio loop to create the server in, if None, take default one. """ if loop is None: warnings.warn( 'Event loop is not specified, this can cause unexpected errors') loop = asyncio.get_event_loop() if config.pid_path is not None and not is_child: _write_pid_file(config.pid_path) loop.set_debug(config.debug) if hasattr(app, 'startup'): loop.run_until_complete(app.startup()) # type: ignore if sock is not None: create_server = loop.create_server( lambda: Server(app, loop, config), ssl=config.ssl, sock=sock, reuse_port=is_child, ) elif config.file_descriptor is not None: sock = socket_fromfd(config.file_descriptor, AF_UNIX, SOCK_STREAM) create_server = loop.create_server( lambda: Server(app, loop, config), ssl=config.ssl, sock=sock, ) elif config.unix_domain is not None: create_server = loop.create_unix_server( lambda: Server(app, loop, config), config.unix_domain, ssl=config.ssl, ) else: create_server = loop.create_server( lambda: Server(app, loop, config), host=config.host, port=config.port, ssl=config.ssl, reuse_port=is_child, ) server = loop.run_until_complete(create_server) if platform.system() == 'Windows': loop.create_task(_windows_signal_support()) try: loop.add_signal_handler(signal.SIGINT, _raise_shutdown) loop.add_signal_handler(signal.SIGTERM, _raise_shutdown) except NotImplementedError: pass # Unix only reload_ = False try: if config.use_reloader: loop.run_until_complete(_observe_changes()) reload_ = True else: loop.run_forever() except (SystemExit, KeyboardInterrupt): pass finally: server.close() loop.run_until_complete(server.wait_closed()) loop.run_until_complete(loop.shutdown_asyncgens()) try: loop.remove_signal_handler(signal.SIGINT) loop.remove_signal_handler(signal.SIGTERM) except NotImplementedError: pass # Unix only if hasattr(app, 'cleanup'): loop.run_until_complete(app.cleanup()) # type: ignore loop.close() if reload_: # Restart this process (only safe for dev/debug) os.execv(sys.executable, [sys.executable] + sys.argv)