class Runner(object): """Manages processes & workers and grabs results. """ def __init__(self, args, loop=None): self.args = args self.console = self.args.shared_console if loop is None: loop = asyncio.get_event_loop() self.loop = loop # the stastd client gets initialized after we fork # processes in case -p was used self.statsd = None self._tasks = [] self._procs = [] self._results = SharedCounters('WORKER', 'REACHED', 'RATIO', 'OK', 'FAILED', 'MINUTE_OK', 'MINUTE_FAILED') self.eventer = EventSender(self.console) def _set_statsd(self): if self.args.statsd: self.statsd = get_statsd_client(self.args.statsd_address, loop=self.loop) else: self.statsd = None def gather(self, *futures): return asyncio.gather(*futures, loop=self.loop, return_exceptions=True) def ensure_future(self, coro): return asyncio.ensure_future(coro, loop=self.loop) def __call__(self): global_setup = get_fixture('global_setup') if global_setup is not None: try: global_setup(self.args) except Exception as e: self.console.print("The global_setup() fixture failed") self.console.print_error(e) raise try: return self._launch_processes() finally: global_teardown = get_fixture('global_teardown') if global_teardown is not None: try: global_teardown() except Exception as e: # we can't stop the teardown process self.console.print_error(e) def _launch_processes(self): args = self.args signal.signal(signal.SIGINT, self._shutdown) signal.signal(signal.SIGTERM, self._shutdown) args.original_pid = os.getpid() if args.processes > 1: if not args.quiet: self.console.print('Forking %d processes' % args.processes) jobs = [] for i in range(args.processes): p = multiprocessing.Process(target=self._process) jobs.append(p) p.start() for job in jobs: self._procs.append(job) async def run(quiet, console): while len(self._procs) > 0: if not quiet: console.print(self.display_results(), end='\r') for job in jobs: if job.exitcode is not None and job in self._procs: self._procs.remove(job) await cancellable_sleep(args.console_update) await self.console.stop() await self.eventer.stop() tasks = [self.ensure_future(self.console.display()), self.ensure_future(self._send_workers_event(1)), self.ensure_future(run(args.quiet, self.console))] self.loop.run_until_complete(self.gather(*tasks)) else: self._process() return self._results def _shutdown(self, signal, frame): stop() self._kill_tasks() # send sigterms for proc in self._procs: proc.terminate() def _runner(self): args = self.args def _prepare(): tasks = [] delay = 0 if args.ramp_up > 0.: step = args.ramp_up / args.workers else: step = 0. for i in range(self.args.workers): worker = Worker(i, self._results, self.console, self.args, self.statsd, delay, self.loop) f = self.ensure_future(worker.run()) tasks.append(f) delay += step return tasks if self.args.quiet: return _prepare() else: msg = 'Preparing {} worker{}' msg = msg.format(args.workers, 's' if args.workers > 1 else '') return self.console.print_block(msg, _prepare) def _process(self): set_timer() if self.args.processes > 1: signal.signal(signal.SIGINT, self._shutdown) signal.signal(signal.SIGTERM, self._shutdown) self.loop = asyncio.new_event_loop() asyncio.set_event_loop(self.loop) if self.args.debug: self.console.print('**** RUNNING IN DEBUG MODE == SLOW ****') self.loop.set_debug(True) self._set_statsd() if self.args.original_pid == os.getpid(): self._tasks.append(self.ensure_future(self._send_workers_event(1))) if not self.args.quiet: fut = self._display_results(self.args.console_update) update = self.ensure_future(fut) display = self.ensure_future(self.console.display()) display = self.gather(update, display) self._tasks.append(display) workers = self.gather(*self._runner()) workers.add_done_callback(lambda fut: stop()) self._tasks.append(workers) try: self.loop.run_until_complete(self.gather(*self._tasks)) finally: self._kill_tasks() if self.statsd is not None: self.statsd.close() self.loop.close() def _kill_tasks(self): cancellable_sleep.cancel_all() for task in reversed(self._tasks): with suppress(asyncio.CancelledError): task.cancel() for task in self._tasks: del task self._tasks[:] = [] def display_results(self): ok, fail = self._results['OK'].value, self._results['FAILED'].value workers = self._results['WORKER'].value pat = 'SUCCESSES: %s | FAILURES: %s | WORKERS: %s' return pat % (ok, fail, workers) async def _display_results(self, update_interval): while not is_stopped(): self.console.print(self.display_results(), end='\r') await cancellable_sleep(update_interval) await self.console.stop() async def _send_workers_event(self, update_interval): while not self.eventer.stopped() and not is_stopped(): workers = self._results['WORKER'].value await self.eventer.send_event('current_workers', workers=workers) await cancellable_sleep(update_interval)
class Runner(object): """Manages processes & workers and grabs results. """ def __init__(self, args, loop=None): self.args = args self.console = self.args.shared_console if loop is None: loop = asyncio.get_event_loop() self.loop = loop # the stastd client gets initialized after we fork # processes in case -p was used self.statsd = None self._tasks = [] self._procs = [] self._results = SharedCounters( "WORKER", "REACHED", "RATIO", "OK", "FAILED", "MINUTE_OK", "MINUTE_FAILED" ) self.eventer = EventSender(self.console) def _set_statsd(self): if self.args.statsd: self.statsd = get_statsd_client(self.args.statsd_address) else: self.statsd = None def run_coro(self, coro): if not self.loop.is_running(): raise Exception("Loop is not running") future = asyncio.run_coroutine_threadsafe(coro, self.loop) return future.result() def gather(self, *futures): return asyncio.gather(*futures, loop=self.loop, return_exceptions=True) def ensure_future(self, coro): return asyncio.ensure_future(coro, loop=self.loop) def __call__(self): global_setup = get_fixture("global_setup") if global_setup is not None: try: global_setup(self.args) except Exception as e: self.console.print("The global_setup() fixture failed") self.console.print_error(e) raise try: return self._launch_processes() finally: global_teardown = get_fixture("global_teardown") if global_teardown is not None: try: global_teardown() except Exception as e: # we can't stop the teardown process self.console.print_error(e) def _launch_processes(self): args = self.args signal.signal(signal.SIGINT, self._shutdown) signal.signal(signal.SIGTERM, self._shutdown) args.original_pid = os.getpid() if args.processes > 1: if not args.quiet: self.console.print("Forking %d processes" % args.processes) jobs = [] for i in range(args.processes): p = multiprocessing.Process(target=self._process) jobs.append(p) p.start() for job in jobs: self._procs.append(job) async def run(quiet, console): while len(self._procs) > 0: if not quiet: console.print(self.display_results(), end="\r") for job in jobs: if job.exitcode is not None and job in self._procs: self._procs.remove(job) await cancellable_sleep(args.console_update) await self.console.stop() await self.eventer.stop() tasks = [ self.ensure_future(self.console.display()), self.ensure_future(self._send_workers_event(1)), self.ensure_future(run(args.quiet, self.console)), ] self.loop.run_until_complete(self.gather(*tasks)) else: self._process() return self._results def _shutdown(self, signal, frame): stop() self._kill_tasks() # send sigterms for proc in self._procs: proc.terminate() def _runner(self): args = self.args def _prepare(): tasks = [] delay = 0 if args.ramp_up > 0.0: step = args.ramp_up / args.workers else: step = 0.0 for i in range(self.args.workers): worker = Worker( i, self._results, self.console, self.args, self.statsd, delay, self.loop, ) f = self.ensure_future(worker.run()) tasks.append(f) delay += step return tasks if self.args.quiet: return _prepare() else: msg = "Preparing {} worker{}" msg = msg.format(args.workers, "s" if args.workers > 1 else "") return self.console.print_block(msg, _prepare) def _process(self): set_timer() # coroutine that will kill everything when duration is up if self.args.duration and self.args.force_shutdown: async def _duration_killer(): cancelled = object() res = await cancellable_sleep(self.args.duration, result=cancelled) if res is cancelled or (res and not res.canceled()): self._shutdown(None, None) await asyncio.sleep(0) _duration_killer = self.ensure_future(_duration_killer()) else: _duration_killer = None if self.args.processes > 1: signal.signal(signal.SIGINT, self._shutdown) signal.signal(signal.SIGTERM, self._shutdown) self.loop = asyncio.new_event_loop() asyncio.set_event_loop(self.loop) if self.args.debug: self.console.print("**** RUNNING IN DEBUG MODE == SLOW ****") self.loop.set_debug(True) self._set_statsd() if self.statsd is not None: self._tasks.append(self.ensure_future(self.statsd.connect())) if self.args.original_pid == os.getpid(): self._tasks.append(self.ensure_future(self._send_workers_event(1))) if not self.args.quiet: fut = self._display_results(self.args.console_update) update = self.ensure_future(fut) display = self.ensure_future(self.console.display()) display = self.gather(update, display) self._tasks.append(display) workers = self.gather(*self._runner()) def _stop(cb): if _duration_killer is not None: if not _duration_killer.done(): _duration_killer.cancel() stop() workers.add_done_callback(_stop) self._tasks.append(workers) try: self.loop.run_until_complete(self.gather(*self._tasks)) finally: if self.statsd is not None: self.loop.run_until_complete(self.ensure_future(self.statsd.close())) self._kill_tasks() self.loop.close() def _kill_tasks(self): cancellable_sleep.cancel_all() for task in reversed(self._tasks): with suppress(asyncio.CancelledError): task.cancel() for task in self._tasks: del task self._tasks[:] = [] def display_results(self): ok, fail = self._results["OK"].value, self._results["FAILED"].value workers = self._results["WORKER"].value pat = "SUCCESSES: %s | FAILURES: %s | WORKERS: %s" return pat % (ok, fail, workers) async def _display_results(self, update_interval): while not is_stopped(): self.console.print(self.display_results(), end="\r") await cancellable_sleep(update_interval) await self.console.stop() async def _send_workers_event(self, update_interval): while not self.eventer.stopped() and not is_stopped(): workers = self._results["WORKER"].value await self.eventer.send_event("current_workers", workers=workers) await cancellable_sleep(update_interval)
class Runner(object): """Manages processes & workers and grabs results. """ def __init__(self, args, loop=None): self.args = args self.console = self.args.shared_console if loop is None: loop = asyncio.get_event_loop() self.loop = loop # the stastd client gets initialized after we fork # processes in case -p was used self.statsd = None self._tasks = [] self._procs = [] self._results = SharedCounters('WORKER', 'REACHED', 'RATIO', 'OK', 'FAILED', 'MINUTE_OK', 'MINUTE_FAILED') self.eventer = EventSender(self.console) def _set_statsd(self): if self.args.statsd: self.statsd = get_statsd_client(self.args.statsd_address, loop=self.loop) else: self.statsd = None def gather(self, *futures): return asyncio.gather(*futures, loop=self.loop, return_exceptions=True) def ensure_future(self, coro): return asyncio.ensure_future(coro, loop=self.loop) def __call__(self): global_setup = get_fixture('global_setup') if global_setup is not None: try: global_setup(self.args) except Exception as e: self.console.print("The global_setup() fixture failed") self.console.print_error(e) raise try: return self._launch_processes() finally: global_teardown = get_fixture('global_teardown') if global_teardown is not None: try: global_teardown() except Exception as e: # we can't stop the teardown process self.console.print_error(e) def _launch_processes(self): args = self.args signal.signal(signal.SIGINT, self._shutdown) signal.signal(signal.SIGTERM, self._shutdown) args.original_pid = os.getpid() if args.processes > 1: if not args.quiet: self.console.print('Forking %d processes' % args.processes) jobs = [] for i in range(args.processes): p = multiprocessing.Process(target=self._process) jobs.append(p) p.start() for job in jobs: self._procs.append(job) async def run(quiet, console): while len(self._procs) > 0: if not quiet: console.print(self.display_results(), end='\r') for job in jobs: if job.exitcode is not None and job in self._procs: self._procs.remove(job) await cancellable_sleep(args.console_update) await self.console.stop() await self.eventer.stop() tasks = [self.ensure_future(self.console.display()), self.ensure_future(self._send_workers_event(1)), self.ensure_future(run(args.quiet, self.console))] self.loop.run_until_complete(self.gather(*tasks)) else: self._process() return self._results def _shutdown(self, signal, frame): stop() self._kill_tasks() # send sigterms for proc in self._procs: proc.terminate() def _runner(self): args = self.args def _prepare(): tasks = [] delay = 0 if args.ramp_up > 0.: step = args.ramp_up / args.workers else: step = 0. for i in range(self.args.workers): worker = Worker(i, self._results, self.console, self.args, self.statsd, delay, self.loop) f = self.ensure_future(worker.run()) tasks.append(f) delay += step return tasks if self.args.quiet: return _prepare() else: msg = 'Preparing {} worker{}' msg = msg.format(args.workers, 's' if args.workers > 1 else '') return self.console.print_block(msg, _prepare) def _process(self): set_timer() # coroutine that will kill everything when duration is up if self.args.duration and self.args.force_shutdown: async def _duration_killer(): cancelled = object() res = await cancellable_sleep(self.args.duration, result=cancelled) if res is cancelled or (res and not res.canceled()): self._shutdown(None, None) await asyncio.sleep(0) _duration_killer = self.ensure_future(_duration_killer()) else: _duration_killer = None if self.args.processes > 1: signal.signal(signal.SIGINT, self._shutdown) signal.signal(signal.SIGTERM, self._shutdown) self.loop = asyncio.new_event_loop() asyncio.set_event_loop(self.loop) if self.args.debug: self.console.print('**** RUNNING IN DEBUG MODE == SLOW ****') self.loop.set_debug(True) self._set_statsd() if self.args.original_pid == os.getpid(): self._tasks.append(self.ensure_future(self._send_workers_event(1))) if not self.args.quiet: fut = self._display_results(self.args.console_update) update = self.ensure_future(fut) display = self.ensure_future(self.console.display()) display = self.gather(update, display) self._tasks.append(display) workers = self.gather(*self._runner()) def _stop(cb): if _duration_killer is not None: if not _duration_killer.done(): _duration_killer.cancel() stop() workers.add_done_callback(_stop) self._tasks.append(workers) try: self.loop.run_until_complete(self.gather(*self._tasks)) finally: self._kill_tasks() if self.statsd is not None: self.statsd.close() self.loop.run_until_complete(self.ensure_future(asyncio.sleep(0))) self.loop.close() def _kill_tasks(self): cancellable_sleep.cancel_all() for task in reversed(self._tasks): with suppress(asyncio.CancelledError): task.cancel() for task in self._tasks: del task self._tasks[:] = [] def display_results(self): ok, fail = self._results['OK'].value, self._results['FAILED'].value workers = self._results['WORKER'].value pat = 'SUCCESSES: %s | FAILURES: %s | WORKERS: %s' return pat % (ok, fail, workers) async def _display_results(self, update_interval): while not is_stopped(): self.console.print(self.display_results(), end='\r') await cancellable_sleep(update_interval) await self.console.stop() async def _send_workers_event(self, update_interval): while not self.eventer.stopped() and not is_stopped(): workers = self._results['WORKER'].value await self.eventer.send_event('current_workers', workers=workers) await cancellable_sleep(update_interval)