def waitForEvent(emitter: EventEmitter, eventName: str, # noqa: C901 predicate: Callable[[Any], bool], timeout: float, loop: asyncio.AbstractEventLoop) -> Awaitable: """Wait for an event emitted from the emitter.""" promise = loop.create_future() def resolveCallback(target: Any) -> None: promise.set_result(target) def rejectCallback(exception: Exception) -> None: promise.set_exception(exception) async def timeoutTimer() -> None: await asyncio.sleep(timeout / 1000) rejectCallback( TimeoutError('Timeout exceeded while waiting for event')) def _listener(target: Any) -> None: if not predicate(target): return cleanup() resolveCallback(target) listener = addEventListener(emitter, eventName, _listener) if timeout: eventTimeout = loop.create_task(timeoutTimer()) def cleanup() -> None: removeEventListeners([listener]) if timeout: eventTimeout.cancel() return promise
def run_until_complete(coro: Awaitable, loop: AbstractEventLoop=None): """Run a task through to completion. The ``.run_until_complete()`` method on asyncio event loop objects does not finish tasks when it receives a SIGINT/CTRL-C. The method simply raises a ``KeyboardInterrupt`` exception and this usually results in warnings about unfinished tasks plus some "event loop closed" ``RuntimeError`` exceptions in pending tasks. This is a really annoying default behavior and this function aims at replacing that behavior with something that ensures the task actually runs through to completion. When the ``KeyboardInterrupt`` exception is caught, the task is canceled and resumed to give it a chance to clean up properly. .. versionadded:: 0.4 .. versionchanged:: 0.5 Can now be called with a ``asyncio.Task`` argument. """ loop = loop or asyncio.get_event_loop() if isinstance(coro, asyncio.Task): task = coro else: task = loop.create_task(coro) try: loop.run_until_complete(task) except KeyboardInterrupt: task.cancel() try: loop.run_until_complete(task) except asyncio.CancelledError: return None return task.result()
def start( loop: asyncio.AbstractEventLoop, framerate: int = DEFAULT_FRAMERATE ) -> asyncio.Future: return asyncio.gather( loop.create_task(_ui_refresh(loop, framerate)), loop.create_task(_process_inputs(loop)), loop=loop, )
def init_requirements(self, loop: asyncio.AbstractEventLoop): getattr(super(), "init_requirements")(loop) loop.run_until_complete( self.connect_to_stream() ) # Save to app object to be accessible in request handlers self._server_instance.app.stream = self._stream
def set_timeout(task: asyncio.Task, timeout: [float, int], loop: asyncio.AbstractEventLoop = None, timeout_cancel=True): assert isinstance(timeout, (float, int)) if loop is None: loop = get_running_loop() now_time = loop.time() out_time = now_time + timeout if timeout_cancel: if timeout <= 0: task.cancel() return unset_timeout(task) handle = loop.call_at(out_time, task.cancel) setattr(task, _MODULE_TIMEOUT_HANDLE, handle) setattr(task, _MODULE_TIMEOUT, out_time)
async def get_address(self, loop: asyncio.AbstractEventLoop) -> SocketAddress: server = ThriftServer(Handler(), port=0) serve_task = loop.create_task(server.serve()) addy = await server.get_address() server.stop() await serve_task return addy
def __init__(self, method: str, url: URL, *, writer: 'asyncio.Task[None]', continue100: Optional['asyncio.Future[bool]'], timer: BaseTimerContext, request_info: RequestInfo, traces: List['Trace'], loop: asyncio.AbstractEventLoop, session: 'ClientSession') -> None: assert isinstance(url, URL) self.method = method self.cookies = SimpleCookie() self._real_url = url self._url = url.with_fragment(None) self._body = None # type: Any self._writer = writer # type: Optional[asyncio.Task[None]] self._continue = continue100 # None by default self._closed = True self._history = () # type: Tuple[ClientResponse, ...] self._request_info = request_info self._timer = timer if timer is not None else TimerNoop() self._cache = {} # type: Dict[str, Any] self._traces = traces self._loop = loop # store a reference to session #1985 self._session = session # type: Optional[ClientSession] if loop.get_debug(): self._source_traceback = traceback.extract_stack(sys._getframe(1))
async def _scan(targets: Set[str], ports: Set[int], config: DockerScanModel, loop: asyncio.AbstractEventLoop): max_concurrency = asyncio.BoundedSemaphore(int(config.concurrency), loop=loop) results = [] tasks = [] for target in targets: for port in ports: await max_concurrency.acquire() tasks.append(loop.create_task(_check_ports( target, port, loop, max_concurrency, results, config ))) await asyncio.wait(tasks, loop=loop) return results
async def total(loop: asyncio.AbstractEventLoop): """ asdf asd f asd f asd f asdf .. note: asdfasdfas df as df >>> a = 1 >>> int(a) 1 @param loop: asdfasdf as df asdf :param loop: :type loop: :return: :rtype: """ t = [] sem = asyncio.Semaphore(10, loop=loop) for x in range(1000): await sem.acquire() t.append(loop.create_task(hello(x, sem))) return t
def __init__( self, name: str, offset: Position, size: Dimensions, loop: asyncio.AbstractEventLoop, actions: Dict[UIActionEnum, actions.Action] = None, ) -> None: self.name = name self.size = size self.offset = offset self.loop = loop self.busy = False self.children = set() # type: Set[UIComponent] self.actions = actions if actions is not None else {} for action in self.actions.values(): action.set_source(self) self.parent = None # type: UIComponent if loop is not None: qsize = self.ACTION_QUEUE_SIZE self._action_queue = asyncio.Queue(qsize) # type: asyncio.Queue self._action_runner = loop.create_task(self._process_actions()) self._action_queue_get = None # type: asyncio.Future self._current_action = None # type: UIActionEnum self._nw_corner = offset - Position(1, 1) self._se_corner = Position( x=self.offset.x + self.size.width - 1, y=self.offset.y + self.size.height - 1, )
def run(*tasks: Awaitable, loop: asyncio.AbstractEventLoop=asyncio.get_event_loop()): """Helper to run tasks in the event loop :param tasks: Tasks to run in the event loop. :param loop: The event loop. """ futures = [asyncio.ensure_future(task, loop=loop) for task in tasks] return loop.run_until_complete(asyncio.gather(*futures))
def __init__(self, *, loop: ALoop): self._subscription_waiters = [] # type: List[Future] self._assignment_waiters = [] # type: List[Future] self._loop = loop # type: ALoop # Fetch contexts self._fetch_count = 0 self._last_fetch_ended = loop.time()
async def start_lightsd_connection( loop: asyncio.AbstractEventLoop, lightsd_url: str, refresh_delay_s: float = DEFAULT_REFRESH_DELAY, ) -> None: global _refresh_task, lightsd lightsd = await lightsc.create_async_lightsd_connection(lightsd_url) _refresh_task = loop.create_task(_poll(loop, refresh_delay_s))
def current_task(loop: asyncio.AbstractEventLoop) -> asyncio.Task: if PY_37: task = asyncio.current_task(loop=loop) # type: ignore else: task = asyncio.Task.current_task(loop=loop) if task is None: # this should be removed, tokio must use register_task and family API if hasattr(loop, 'current_task'): task = loop.current_task() # type: ignore return task
def _cancel_all_tasks(loop: asyncio.AbstractEventLoop) -> None: to_cancel = all_tasks(loop) if not to_cancel: return for task in to_cancel: task.cancel() loop.run_until_complete( asyncio.gather(*to_cancel, loop=loop, return_exceptions=True)) for task in to_cancel: if task.cancelled(): continue if task.exception() is not None: loop.call_exception_handler({ 'message': 'unhandled exception during asyncio.run() shutdown', 'exception': task.exception(), 'task': task, })
def __init__(self, connector: 'BaseConnector', key: 'ConnectionKey', protocol: ResponseHandler, loop: asyncio.AbstractEventLoop) -> None: self._key = key self._connector = connector self._loop = loop self._protocol = protocol # type: Optional[ResponseHandler] self._callbacks = [] # type: List[Callable[[], None]] if loop.get_debug(): self._source_traceback = traceback.extract_stack(sys._getframe(1))
def get_left_time(task: asyncio.Task = None, loop: asyncio.AbstractEventLoop = None): if loop is None: loop = get_running_loop() if task is None: task = current_task() out_time = getattr(task, _MODULE_TIMEOUT, None) if not out_time: return sys.maxsize now_time = loop.time() left_time = out_time - now_time if left_time < 0: return 0 return left_time
def cli( loop: AbstractEventLoop, aiohttp_client: Callable[[web.Application], Awaitable[TestClient]], ) -> TestClient: app = init_app( db_dsn=settings.DB_DSN_TEST, secret_key="NQLSo4kyVKvWRDeo4tP_z25GPK4pN4vvrb14zv4SXI8=", session_cookie_name="sessionid", ) client = aiohttp_client(app) app.router.add_get('/__test__/identity', _get_user_identity) return loop.run_until_complete(client)
def test_get_cut_v2( self, caplog: pytest.LogCaptureFixture, temp_event_loop: asyncio.AbstractEventLoop, ): caplog.set_level(logging.DEBUG) mock_reader = self.MockAsyncReader(TEST_V2_DATA1_EXACT[0:20]) reslt = temp_event_loop.run_until_complete(get_proxy(mock_reader)) assert isinstance(reslt, ProxyData) assert not reslt.valid expect_msg = "PROXY exception: Connection lost while waiting for tail part" assert reslt.error == expect_msg expect = ("mail.debug", 30, expect_msg) assert expect in caplog.record_tuples
def test_asyncio_no_recycle_stopping_worker( asyncio_event_loop: asyncio.AbstractEventLoop, ) -> None: """Regression test for #323.""" async def taskfunc1() -> None: await anyio.to_thread.run_sync(time.sleep, 0) event1.set() await event2.wait() async def taskfunc2() -> None: await event1.wait() asyncio_event_loop.call_soon(event2.set) await anyio.to_thread.run_sync(time.sleep, 0) # At this point, the worker would be stopped but still in the idle workers pool, so the # following would hang prior to the fix await anyio.to_thread.run_sync(time.sleep, 0) event1 = asyncio.Event() event2 = asyncio.Event() task1 = asyncio_event_loop.create_task(taskfunc1()) task2 = asyncio_event_loop.create_task(taskfunc2()) asyncio_event_loop.run_until_complete(asyncio.gather(task1, task2))
def __init__( self, dtype: typing.Type[MessageClass], transport_session: pyuavcan.transport.InputSession, finalizer: PortFinalizer, loop: asyncio.AbstractEventLoop, ): self.dtype = dtype self.transport_session = transport_session self.deserialization_failure_count = 0 self._maybe_finalizer: typing.Optional[PortFinalizer] = finalizer self._loop = loop self._task = loop.create_task(self._task_function()) self._listeners: typing.List[_Listener[MessageClass]] = []
async def test_async_set_color_lights(event_loop: asyncio.AbstractEventLoop, MockConnectedGateway): mode = 7 result = event_loop.create_future() result.set_result(b"") with patch( "screenlogicpy.requests.lights.ScreenLogicProtocol.await_send_message", return_value=result, ) as mockRequest: gateway = MockConnectedGateway assert await gateway.async_set_color_lights(mode) await gateway.async_disconnect() assert mockRequest.call_args.args[0] == 12556 assert mockRequest.call_args.args[1] == struct.pack("<II", 0, mode)
def __init__(self, loop: AbstractEventLoop, idx: int, schedule_details: List[bytes]) -> None: """Initialize the schedule.""" self._loop = loop self._enabled = False self._recurring = False self._schedule_id = str(int(schedule_details[idx][0:2], 16)) self._days = [] # type: List[str] self._schedule_data = b"" self._start_time = WAITING_TEXT self._end_time = WAITING_TEXT self._duration = WAITING_TEXT self._init_future = loop.create_future() ensure_future(self.initialize(idx, schedule_details), loop=loop)
async def test_paginate_not_paginated( self, context: MockContext, fetcher: ListPageSource[int], event_loop: asyncio.AbstractEventLoop, advance_time: ClockAdvancer, ) -> None: p = InteractivePager.create(cast(Any, context), fetcher) assert not p.paginating event_loop.create_task(p.paginate()) await advance_time(125) assert not p.paginating page = await fetcher.get_page(1) assert p.embed.description == page['entry_text'] assert isinstance(p.message, MockMessage) assert context.message is p.message context.channel.send.assert_awaited_once_with(embed=p.embed) p.message.add_reaction.assert_not_awaited() p.message.clear_reactions.assert_not_awaited() p.message.clear_reactions.assert_not_awaited() p.message.remove_reaction.assert_not_awaited() assert p.match is None
def handle_exception(loop: asyncio.AbstractEventLoop, context: Dict[str, Any]) -> None: """Exception handler for event loop.""" # context["message"] will always be there; # but context["exception"] and context["future"] may not try: exc: Exception = context["exception"] future = context["future"] future_name = "unknown" try: # If future is a Task, get its name future_name = future.get_name() except AttributeError: pass _logger.error( "Caught exception `%s` in %s task: %s", exc.__class__.__name__, future_name, exc, ) except KeyError: _logger.error("Caught exception: %s", context["message"]) _logger.info("Shutting down...") loop.create_task(shutdown())
def request_schedules(addr: Tuple[str, int], print_results: bool = True, loop: asyncio.AbstractEventLoop = None): """Send a command to the server to Update Commands. Args: addr (tuple): Server IP address print_results (bool)[True]: If true print the schedules that were returned. loop (asyncio.AbstractEventLoop)[None]: Event loop to run the async command with. """ if loop is None: loop = get_loop() return loop.run_until_complete( request_schedules_async(addr, print_results=print_results))
def enable_async_loop_debugging( event_loop: AbstractEventLoop, slow_callback_duration: float = 0.1) -> AbstractEventLoop: """Enables debugging on an event loop. Args: event_loop: The event loop to enable debugging on slow_callback_duration: The threshold at which a callback should be alerted as slow. """ logging.info("Enabling coroutine debugging. Loop id {}.".format( id(asyncio.get_event_loop()))) # Enable debugging event_loop.set_debug(True) # Make the threshold for "slow" tasks very very small for # illustration. The default is 0.1 (= 100 milliseconds). event_loop.slow_callback_duration = slow_callback_duration # Report all mistakes managing asynchronous resources. warnings.simplefilter("always", ResourceWarning) return event_loop
async def test_race_timeout(event_loop: asyncio.AbstractEventLoop, advance_time: ClockAdvancer) -> None: async def one() -> int: await asyncio.sleep(100, loop=event_loop) return 1 async def two() -> int: await asyncio.sleep(50, loop=event_loop) return 2 task = event_loop.create_task( util.race([one(), two()], timeout=25, loop=event_loop)) await advance_time(30) assert isinstance(task.exception(), asyncio.TimeoutError)
def client( event_loop: asyncio.AbstractEventLoop, aiohttp_client: Callable, app_config: Dict, postgres_with_template_db: aiopg.sa.engine.Engine, mock_orphaned_services: mock.Mock, monkeypatch_setenv_from_app_config: Callable, ): # test config & env vars ---------------------- cfg = deepcopy(app_config) assert cfg["rest"]["version"] == API_VERSION assert cfg["rest"]["enabled"] cfg["projects"]["enabled"] = True cfg["director"]["enabled"] = True cfg["exporter"]["enabled"] = True monkeypatch_setenv_from_app_config(cfg) # app setup ---------------------------------- app = create_safe_application(cfg) # activates only security+restAPI sub-modules assert setup_settings(app) assert get_exporter_settings(app) is not None, "Should capture defaults" setup_db(app) setup_session(app) setup_security(app) setup_rest(app) setup_login(app) setup_users(app) setup_socketio(app) setup_projects(app) setup_director(app) setup_director_v2(app) setup_exporter(app) # <---- under test setup_storage(app) setup_products(app) setup_catalog(app) setup_scicrunch(app) assert setup_resource_manager(app) setup_garbage_collector(app) yield event_loop.run_until_complete( aiohttp_client( app, server_kwargs={"port": cfg["main"]["port"], "host": cfg["main"]["host"]}, ) )
def __init__(self, request: RequestType, deadline: Optional[float], metadata: Metadata, credentials: Optional[grpc.CallCredentials], wait_for_ready: Optional[bool], channel: cygrpc.AioChannel, method: bytes, request_serializer: SerializingFunction, response_deserializer: DeserializingFunction, loop: asyncio.AbstractEventLoop) -> None: super().__init__( channel.call(method, deadline, credentials, wait_for_ready), metadata, request_serializer, response_deserializer, loop) self._request = request self._send_unary_request_task = loop.create_task( self._send_unary_request()) self._init_stream_response_mixin(self._send_unary_request_task)
def get_client_session( loop: asyncio.AbstractEventLoop) -> aiohttp.ClientSession: """Get a shared ClientSession object that can be reused. If none exists yet it will create one using the passed-in loop. :param loop: an active (i.e. not closed) asyncio event loop :return: a ClientSession instance that can be used to do HTTP requests """ try: return loop._bravado_asyncio_client_session # type: ignore except AttributeError: client_session = aiohttp.ClientSession(loop=loop) loop._bravado_asyncio_client_session = client_session # type: ignore return client_session
async def start( self, loop: asyncio.AbstractEventLoop ) -> Tuple[HttpExchangeStream, asyncio.Task]: self.worker = faust.Worker(self.app, loop=loop, loglevel="info") async def start_worker(worker: faust.Worker) -> None: await worker.start() source = self.http_exchange_stream() worker_coro = start_worker(self.worker) self.worker_task = loop.create_task(worker_coro) return source, self.worker_task
def create_client(client_type, request, event_loop: asyncio.AbstractEventLoop, session, region, config, **kw): client = session.create_client(client_type, region_name=region, config=config, **kw) def fin(): event_loop.run_until_complete(client.__aexit__(None, None, None)) request.addfinalizer(fin) client = event_loop.run_until_complete(client.__aenter__()) return client
def test_create_user(client: TestClient, event_loop: asyncio.AbstractEventLoop): # nosec response = client.post("/users", json={"username": "******"}) assert response.status_code == 200, response.text data = response.json() assert data["username"] == "admin" assert "id" in data user_id = data["id"] async def get_user_by_db(): user = await Users.get(id=user_id) return user user_obj = event_loop.run_until_complete(get_user_by_db()) assert user_obj.id == user_id
def shutdown(loop: asyncio.AbstractEventLoop) -> None: """Cancel all pending tasks on `loop`, wait for them, and close the loop.""" try: if sys.version_info[:2] >= (3, 7): all_tasks = asyncio.all_tasks else: all_tasks = asyncio.Task.all_tasks # This part is borrowed from asyncio/runners.py in Python 3.7b2. to_cancel = [task for task in all_tasks(loop) if not task.done()] if not to_cancel: return for task in to_cancel: task.cancel() loop.run_until_complete( asyncio.gather(*to_cancel, loop=loop, return_exceptions=True)) finally: # `concurrent.futures.Future` objects cannot be cancelled once they # are already running. There might be some when the `shutdown()` happened. # Silence their logger's spew about the event loop being closed. cf_logger = logging.getLogger("concurrent.futures") cf_logger.setLevel(logging.CRITICAL) loop.close()
def __start_read_loop(self, *, loop: asyncio.AbstractEventLoop = None) -> None: """Start the web socket reader. If the reader is already running, this is a no-op. """ if self.__read_loop and not self.__read_loop.done(): return if loop is None: loop = asyncio.get_event_loop() self.__read_loop = loop.create_task(self.__web_socket_reader())
async def test_stream_buffer_push_and_pop( event_loop: asyncio.AbstractEventLoop) -> None: stream_buffer = StreamBuffer(EventWrapper) async def _push_over_limit() -> None: await stream_buffer.push(b"a" * (BUFFER_HIGH_WATER + 1)) return True task = event_loop.create_task(_push_over_limit()) assert not task.done() # Blocked as over high water await stream_buffer.pop(BUFFER_HIGH_WATER / 4) assert not task.done() # Blocked as over low water await stream_buffer.pop(BUFFER_HIGH_WATER / 4) assert (await task) is True
def handler(loop: asyncio.AbstractEventLoop, context: Dict[str, Any]) -> None: # pragma: no cover message = context["message"] exception = context.get("exception", Exception) if sys.version_info < (3, 7, 4) and message in ( "SSL error in data received", "Fatal error on transport", ): # Ignore aiohttp #3535 / cpython #13548 issue with SSL data after close # Adapted from https://github.com/aio-libs/aiohttp/issues/3535#issuecomment-483268542 import ssl from asyncio.sslproto import SSLProtocol protocol = context.get("protocol") if (isinstance(exception, ssl.SSLError) and exception.reason == "KRB5_S_INIT" and isinstance(protocol, SSLProtocol)): return # Our main interest here is minimising the other various errors and tracebacks that # drown out the politely formatted errors from cli.py when things go wrong if "exception was never retrieved" in message: from .request import MissingSession # While closing down, we remove the global session, causing other requests to fail. This # just causes noise, so ignore if that's the exception. if not isinstance(exception, MissingSession): print( f"ERROR (while closing down): {type(exception).__name__}: {exception}", file=sys.stderr, ) else: print( f"ERROR (from event loop): {type(exception).__name__}: {message}", file=sys.stderr) if loop.get_debug(): loop.default_exception_handler(context)
def aiohttp_client( loop: asyncio.AbstractEventLoop, aiohttp_client_cls: Type[TestClient] ) -> Generator[AiohttpClient, None, None]: """Factory to create a TestClient instance. aiohttp_client(app, **kwargs) aiohttp_client(server, **kwargs) aiohttp_client(raw_server, **kwargs) """ clients = [] async def go( __param: Union[Application, BaseTestServer], *, server_kwargs: Optional[Dict[str, Any]] = None, **kwargs: Any ) -> TestClient: if isinstance(__param, Application): server_kwargs = server_kwargs or {} server = TestServer(__param, **server_kwargs) client = aiohttp_client_cls(server, **kwargs) elif isinstance(__param, BaseTestServer): client = aiohttp_client_cls(__param, **kwargs) else: raise ValueError("Unknown argument type: %r" % type(__param)) await client.start_server() clients.append(client) return client yield go async def finalize() -> None: while clients: await clients.pop().close() loop.run_until_complete(finalize())
def _get_okta_group_users( self, group: Group, event_loop: asyncio.AbstractEventLoop ) -> Iterable[User]: logger.debug(f"Extracting users from Okta group named {group.profile.name}") # Note that this is not taking full advantage of Python AsyncIO; we are blocking on calls. query_parameters = {"limit": self.config.page_size} users = resp = err = None try: users, resp, err = event_loop.run_until_complete( self.okta_client.list_group_users(group.id, query_parameters) ) except OktaAPIException as api_err: self.report.report_failure( "okta_group_users", f"Failed to fetch Users of Group {group.profile.name} from Okta API: {api_err}", ) while True: if err: self.report.report_failure( "okta_group_users", f"Failed to fetch Users of Group {group.profile.name} from Okta API: {err}", ) if users: for user in users: yield user if resp and resp.has_next(): sleep(self.config.delay_seconds) try: users, err = event_loop.run_until_complete(resp.next()) except OktaAPIException as api_err: self.report.report_failure( "okta_group_users", f"Failed to fetch Users of Group {group.profile.name} from Okta API: {api_err}", ) else: break
def get_time_accelerator(loop: asyncio.AbstractEventLoop, now: typing.Optional[float] = None) -> typing.Callable[[float], typing.Awaitable[None]]: """ Returns an async advance() function This provides a way to advance() the BaseEventLoop.time for the scheduled TimerHandles made by call_later, call_at, and call_soon. """ _time = now or loop.time() loop.time = functools.wraps(loop.time)(lambda: _time) async def accelerate_time(seconds: float) -> None: nonlocal _time if seconds < 0: raise ValueError(f'Cannot go back in time ({seconds} seconds)') _time += seconds await past_events() await asyncio.sleep(0) async def past_events() -> None: while loop._scheduled: timer: asyncio.TimerHandle = loop._scheduled[0] if timer not in loop._ready and timer._when <= _time: loop._scheduled.remove(timer) loop._ready.append(timer) if timer._when > _time: break await asyncio.sleep(0) async def accelerator(seconds: float): steps = seconds * 10.0 for _ in range(max(int(steps), 1)): await accelerate_time(0.1) return accelerator
async def main(loop: asyncio.AbstractEventLoop, session: aiohttp.ClientSession, language_to_build: str, args: dict) -> None: """ Main method that starts the entire build process :param args: :param loop: :param session: :param language_to_build: :return: """ def get_next_batch_of_sets(queue: Iterator[List[str]]) -> List[List[str]]: """ To ensure better performance, we limit the number of sets built at a time to limit our memory impact. This will return the next group of sets to build. """ max_pops = int(args['max_sets_build'][0]) # User disabled this memory protection feature if max_pops == 0: return list(queue) return list(itertools.islice(queue, max_pops)) # Main Applied mtg_storage.ensure_set_dir_exists() sets_queue = iter(SETS_TO_BUILD) async with session: # Start asyncio tasks for building each set json_builder = mtg_builder.MTGJSON(SETS_TO_BUILD, session, loop) # We will only be building a few sets at a time, to allow for partial outputs sets_to_build_now = get_next_batch_of_sets(sets_queue) while sets_to_build_now: # Create our builders for the few sets futures = [ loop.create_task( json_builder.build_set(set_name, language_to_build)) for set_name in sets_to_build_now ] # Then wait until all of them are completed await asyncio.wait(futures) # Then queue up our next sets to build sets_to_build_now = get_next_batch_of_sets(sets_queue) # And we're done! :) return
def teardown_test_loop(loop: asyncio.AbstractEventLoop, fast: bool = False) -> None: """Teardown and cleanup an event_loop created by setup_test_loop.""" closed = loop.is_closed() if not closed: loop.call_soon(loop.stop) loop.run_forever() loop.close() if not fast: gc.collect() asyncio.set_event_loop(None)
def start_async_callback( event_loop: asyncio.AbstractEventLoop, coroutine_func, shutdown_event: Event, ): asyncio.set_event_loop(event_loop) try: event_loop.run_until_complete(coroutine_func()) event_loop.run_until_complete(event_loop.shutdown_asyncgens()) event_loop.close() finally: shutdown_event.set()
def client(loop: AbstractEventLoop, aiohttp_client: Callable, aiohttp_unused_port: Callable) -> TestClient: ports = [aiohttp_unused_port() for _ in range(2)] async def redirect(request: web.Request) -> web.Response: return web.HTTPFound(location="/return/200") async def return_response(request: web.Request) -> web.Response: code = int(request.match_info["code"]) return web.Response(status=code) async def raise_response(request: web.Request): status_code = int(request.match_info["code"]) status_to_http_exception = _collect_http_exceptions() http_exception_cls = status_to_http_exception[status_code] raise http_exception_cls( reason=f"raised from raised_error with code {status_code}") async def skip(request: web.Request): return web.HTTPServiceUnavailable(reason="should not happen") app = web.Application() app.add_routes([ web.get("/redirect", redirect), web.get("/return/{code}", return_response), web.get("/raise/{code}", raise_response), web.get("/skip", skip, name="skip"), ]) print("Resources:") for resource in app.router.resources(): print(resource) # UNDER TEST --- # SEE RoutesView to understand how resources can be iterated to get routes resource = app.router["skip"] routes_in_a_resource = list(resource) setup_tracing( app, service_name=f"{__name__}.client", host="127.0.0.1", port=ports[0], jaeger_base_url=DEFAULT_JAEGER_BASE_URL, skip_routes=routes_in_a_resource, ) return loop.run_until_complete( aiohttp_client(app, server_kwargs={"port": ports[0]}))
def __init__(self, interceptors: Sequence[UnaryUnaryClientInterceptor], request: RequestType, timeout: Optional[float], metadata: MetadataType, credentials: Optional[grpc.CallCredentials], wait_for_ready: Optional[bool], channel: cygrpc.AioChannel, method: bytes, request_serializer: SerializingFunction, response_deserializer: DeserializingFunction, loop: asyncio.AbstractEventLoop) -> None: self._loop = loop self._channel = channel interceptors_task = loop.create_task( self._invoke(interceptors, method, timeout, metadata, credentials, wait_for_ready, request, request_serializer, response_deserializer)) super().__init__(interceptors_task)
def teardown_test_loop(loop: asyncio.AbstractEventLoop, fast: bool=False) -> None: """Teardown and cleanup an event_loop created by setup_test_loop. """ closed = loop.is_closed() if not closed: loop.call_soon(loop.stop) loop.run_forever() loop.close() if not fast: gc.collect() asyncio.set_event_loop(None)
def redis_pool(event_loop: asyncio.AbstractEventLoop, redis_port: int): redis_pool = event_loop.run_until_complete( aioredis.create_pool(('localhost', redis_port))) yield redis_pool redis_pool.close() event_loop.run_until_complete(redis_pool.wait_closed())
def run(self, event_loop: AbstractEventLoop = None): # Configure the logging system if isinstance(self.logging_config, dict): logging.config.dictConfig(self.logging_config) elif self.logging_config: logging.basicConfig(level=logging.INFO) # Assign a new default executor with the given max worker thread limit event_loop = event_loop or asyncio.get_event_loop() event_loop.set_default_executor(ThreadPoolExecutor(self.max_threads)) # Create the application context context = self.create_context() try: # Start all the components and run the loop until they've finished self.logger.info("Starting components") coroutines = (component.start(context) for component in self.components) coroutines = [coro for coro in coroutines if coro is not None] event_loop.run_until_complete(asyncio.gather(*coroutines)) self.logger.info("All components started") # Run the application's custom startup code coro = self.start(context) if coro is not None: event_loop.run_until_complete(coro) # Run all the application context's start callbacks event_loop.run_until_complete(context.run_callbacks(ContextEventType.started)) self.logger.info("Application started") except Exception as exc: self.logger.exception("Error during application startup") context.exception = exc else: # Finally, run the event loop until the process is terminated or Ctrl+C is pressed try: event_loop.run_forever() except (KeyboardInterrupt, SystemExit): pass event_loop.run_until_complete(context.run_callbacks(ContextEventType.finished)) event_loop.close() self.logger.info("Application stopped")