async def test_cancel_from_shielded_scope() -> None: async with create_task_group() as tg: with CancelScope(shield=True) as inner_scope: assert inner_scope.shield tg.cancel_scope.cancel() with pytest.raises(get_cancelled_exc_class()): await sleep(0.01) with pytest.raises(get_cancelled_exc_class()): await sleep(0.01)
async def host_task() -> None: nonlocal done async with create_task_group() as tg: with CancelScope(shield=True) as inner_scope: assert inner_scope.shield tg.cancel_scope.cancel() with pytest.raises(get_cancelled_exc_class()): await sleep(0) with pytest.raises(get_cancelled_exc_class()): await sleep(0) done = True
async def _sender_loop(self, evt): keepalive_timeout = self.session.keep_alive if keepalive_timeout <= 0: keepalive_timeout = None try: async with anyio.open_cancel_scope() as scope: self._sender_task = scope await evt.set() while True: packet = None async with anyio.move_on_after(keepalive_timeout): packet = await self._send_q.get() if packet is None: # closing break if packet is None: # timeout await self.handle_write_timeout() continue # self.logger.debug("%s > %r",'B' if 'Broker' in type(self).__name__ else 'C', packet) await packet.to_stream(self.stream) await self.plugins_manager.fire_event( EVENT_MQTT_PACKET_SENT, packet=packet, session=self.session) except ConnectionResetError: await self.handle_connection_closed() except anyio.get_cancelled_exc_class(): raise except BaseException as e: self.logger.warning("Unhandled exception", exc_info=e) raise finally: async with anyio.fail_after(2, shield=True): await self._sender_stopped.set() self._sender_task = None
async def _run_reconnected(self, val: ValueEvent): try: async with anyio.open_cancel_scope() as scope: self._current_run = scope while True: try: await self._run_one(val) except anyio.get_cancelled_exc_class(): raise except ( BrokenPipeError, TimeoutError, EnvironmentError, anyio.IncompleteRead, ConnectionResetError, anyio.ClosedResourceError, StopAsyncIteration, ) as exc: if val is not None and not val.is_set(): await val.set_error(exc) return logger.error("Disconnected") val = None await anyio.sleep(self._backoff) if self._backoff < 10: self._backoff *= 1.5 else: pass finally: self._current_run = None
async def _run(self, client): state = self.state async with anyio.open_cancel_scope() as sc: self.scope = sc try: logger.debug("START %s", self.name) await self(client) except anyio.get_cancelled_exc_class(): state.exc = "Canceled" if self.scope is not None: state.n_fail += 1 state.fail_count += 1 state.fail_map.append(True) raise except Exception as exc: state.exc = traceback.format_exc().split('\n') state.n_fail += 1 state.fail_count += 1 state.fail_map.append(True) else: state.fail_count = 0 state.fail_map.append(False) finally: state.n_run += 1 if any(state.fail_map): del state.fail_map[:-20] else: state.fail_map = [] # zero out after 20 successes in sequence self.scope = None logger.debug("END %s", self.name)
async def _stop_site_on_cancel(aiohttp_tcp_site): """Stop the server after SIGINT.""" try: await anyio.sleep(float('inf')) except anyio.get_cancelled_exc_class(): logger.info(' Stopping the server '.center(50, '=')) await aiohttp_tcp_site.stop()
async def serve(self) -> None: await self._master.register_service(self.service_name) try: await anyio.sleep_forever() except anyio.get_cancelled_exc_class(): with anyio.CancelScope(shield=True), anyio.move_on_after(1): await self._master.unregister_service(self.service_name) raise
async def test_catch_cancellation(): finalizer_done = False async with move_on_after(0.1): try: await sleep(1) except get_cancelled_exc_class(): finalizer_done = True raise assert finalizer_done
async def _run_with_tg(self, *, evt: anyio.abc.Event=None): try: await super()._run_with_tg(evt=evt) except anyio.get_cancelled_exc_class(): if self._done.is_set(): await self._handle_prev(_ResultEvent(self._result)) else: await self._handle_prev(_ErrorEvent(CancelledError())) raise except Exception as exc: await self._handle_prev(_ErrorEvent(exc)) except BaseException: await self._handle_prev(_ErrorEvent(CancelledError())) raise else: await self._handle_prev(_ResultEvent(self._result))
async def handle_tcpros( self, protocol: str, header: abc.Header, client: SocketStream, ) -> None: """Handle incoming service client traffic.""" require_fields(header, "service", "md5sum", "callerid") check_md5sum(header, getattr(self.service_type, "_md5sum")) await client.send(encode_header(self.header)) if header.get("probe") == "1": return persistent = header.get("persistent", "").lower() in ("1", "true") serializer: Serializer[abc.ServiceRequestT] = Serializer() buffered_receiver = BufferedByteReceiveStream(client) while True: request = self.request_class() data = await read_data(buffered_receiver) await anyio.to_thread.run_sync(request.deserialize, data) setattr(request, "_connection_header", header) try: result = await self._handler(request) except anyio.get_cancelled_exc_class(): raise except Exception: # pylint: disable=broad-except logger.error("Exception occured in service callback", exc_info=True) await client.send( encode_byte(0) + encode_str("error processing request")) else: data = await anyio.to_thread.run_sync(serializer.serialize, result) await client.send(encode_byte(1) + data) if not persistent: return
async def release(self, partitions, suppress_cancelled=False): client = self.app.redis fn = self.app.scripts["lock_release"] keys = [self.redis_lock_key(p.number) for p in partitions] async with await client.pipeline(transaction=True) as pipe: for key in keys: await fn.execute(client=pipe, keys=[key], args=[self.id]) # Shield the execution from cancellation to ensure the locks are released. async with anyio.fail_after(4, shield=True): await pipe.execute() try: await self.set_owned(self.owned - partitions) except anyio.get_cancelled_exc_class(): if not suppress_cancelled: raise finally: logger.debug("released-partitions", partitions=[p.number for p in partitions])
async def route_github_event( *, github_event: GitHubEvent, github_app: GitHubApp, ) -> None: """Dispatch GitHub event to corresponsing handlers. Set up ``RUNTIME_CONTEXT`` before doing that. This is so the concrete event handlers have access to the API client and flags in runtime. """ is_gh_action = isinstance(github_app, GitHubAction) # pylint: disable=assigning-non-slot RUNTIME_CONTEXT.IS_GITHUB_ACTION = is_gh_action # pylint: disable=assigning-non-slot RUNTIME_CONTEXT.IS_GITHUB_APP = not is_gh_action # pylint: disable=assigning-non-slot RUNTIME_CONTEXT.github_app = github_app # pylint: disable=assigning-non-slot RUNTIME_CONTEXT.github_event = github_event # pylint: disable=assigning-non-slot RUNTIME_CONTEXT.app_installation = None if is_gh_action: # pylint: disable=assigning-non-slot RUNTIME_CONTEXT.app_installation_client = github_app.api_client else: with contextlib.suppress(LookupError): # pylint: disable=pointless-string-statement """Provision an installation API client if possible. Some events (like `ping`) are happening application/GitHub-wide and are not bound to a specific installation. The webhook payloads of such events don't contain any reference to an installaion. Some events don't even refer to a GitHub App (e.g. `security_advisory`). """ github_install = await github_app.get_installation(github_event) # pylint: disable=assigning-non-slot RUNTIME_CONTEXT.app_installation = github_install # pylint: disable=assigning-non-slot RUNTIME_CONTEXT.app_installation_client = github_install.api_client # Give GitHub a sec to deal w/ eventual consistency. # This is only needed for events that arrive over HTTP. # If the dispatcher is invoked from GitHub Actions, # by the time it's invoked the action must be already consistently # distributed within GitHub's systems because spawning VMs takes time # and actions are executed in workflows that rely on those VMs. await async_sleep(1) try: return await github_app.dispatch_event(github_event) except GitHubActionError: # Bypass GitHub Actions errors as they are supposed to be a # mechanism for communicating outcomes and are expected. raise except get_cancelled_exc_class(): raise except Exception as exc: # pylint: disable=broad-except # NOTE: It's probably better to wrap each event handler with # NOTE: try/except and call `capture_exception()` there instead. # NOTE: We'll also need to figure out the magic of associating # NOTE: breadcrumbs with event handlers. sentry_sdk.capture_exception(exc) # NOTE: Framework-wise, these exceptions are meaningless because they # NOTE: can be anything random that the webhook author (octomachinery # NOTE: end-user) forgot to handle. There's nothing we can do about # NOTE: them except put in the log so that the end-user would be able # NOTE: to properly debug their problem by inspecting the logs. # NOTE: P.S. This is also where we'd inject Sentry if isinstance(exc.__context__, get_cancelled_exc_class()): # The CancelledError context is irrelevant to the # user-defined webhook event handler workflow so we're # dropping it from the logs: exc.__context__ = None logger.exception( 'An unhandled exception happened while running webhook ' 'event handlers for "%s"...', github_event.name, ) delivery_id_msg = ('' if is_gh_action else f' (Delivery ID: {github_event.delivery_id!s})') logger.debug( 'The payload of "%s" event%s is: %r', github_event.name, delivery_id_msg, github_event.payload, ) if is_gh_action: # NOTE: In GitHub Actions env, the app is supposed to run as # NOTE: a foreground single event process rather than a # NOTE: server for multiple events. It's okay to propagate # NOTE: unhandled errors so that they are spit out to the # NOTE: console. raise except BaseException: # SystemExit + KeyboardInterrupt + GeneratorExit raise
def cancelledclass(): return anyio.get_cancelled_exc_class()
async def event_waiter(): nonlocal cancelled try: await sleep(3) except get_cancelled_exc_class(): cancelled = True
if raise_interrupt is not None: warnings.warn( 'raise_interrupt is deprecated, KeyboardInterrupt will cause this coroutine to be cancelled and then ' 'be raised by the top level asyncio.run call or equivalent, and should be caught there. See #136.', DeprecationWarning, ) if stop_event is None: stop_event_: 'AnyEvent' = anyio.Event() else: stop_event_ = stop_event force_polling = _default_force_pulling(force_polling) with RustNotify([str(p) for p in paths], debug, force_polling, poll_delay_ms) as watcher: timeout = _calc_async_timeout(rust_timeout) CancelledError = anyio.get_cancelled_exc_class() while True: async with anyio.create_task_group() as tg: try: raw_changes = await anyio.to_thread.run_sync(watcher.watch, debounce, step, timeout, stop_event_) except (CancelledError, KeyboardInterrupt): stop_event_.set() # suppressing KeyboardInterrupt wouldn't stop it getting raised by the top level asyncio.run call raise tg.cancel_scope.cancel() if raw_changes == 'timeout': if yield_on_timeout: yield set() else:
async def _reader_loop(self, evt): self.logger.debug("%s Starting reader coro", self.session.client_id) keepalive_timeout = self.session.keep_alive if keepalive_timeout <= 0: keepalive_timeout = None try: async with anyio.create_task_group() as tg: self._reader_task = tg await evt.set() while True: try: async with anyio.fail_after(keepalive_timeout): fixed_header = await MQTTFixedHeader.from_stream( self.stream) if fixed_header is None: self.logger.debug( "%s No more data (EOF received), stopping reader coro", self.session.client_id, ) break if (fixed_header.packet_type == RESERVED_0 or fixed_header.packet_type == RESERVED_15): self.logger.warning( "%s Received reserved packet, which is forbidden: closing connection", self.session.client_id, ) await self.handle_connection_closed() break cls = packet_class(fixed_header) packet = await cls.from_stream( self.stream, fixed_header=fixed_header) # self.logger.debug("< %s %r",'B' if 'Broker' in type(self).__name__ else 'C', packet) await self.plugins_manager.fire_event( EVENT_MQTT_PACKET_RECEIVED, packet=packet, session=self.session, ) try: pt, direct = PACKET_TYPES[ packet.fixed_header.packet_type] fn = getattr(self, "handle_" + pt) except (KeyError, AttributeError): self.logger.warning( "%s Unhandled packet type: %s", self.session.client_id, packet.fixed_header.packet_type, ) else: try: if direct: await fn(packet) else: await tg.spawn(fn, packet) except StopAsyncIteration: break except MQTTException: self.logger.debug("Message discarded") except TimeoutError: self.logger.debug( "%s Input stream read timeout", self.session.client_id if self.session else "?", ) await self.handle_read_timeout() except NoDataException: self.logger.debug("%s No data available", self.session.client_id) break # XXX except anyio.get_cancelled_exc_class(): self.logger.warning("%s CANCEL", type(self).__name__) raise except BaseException as e: self.logger.warning( "%s Unhandled exception in reader coro", type(self).__name__, exc_info=e, ) raise await tg.cancel_scope.cancel() finally: async with anyio.fail_after(2, shield=True): self.logger.debug( "%s %s coro stopped", "Broker" if "Broker" in type(self).__name__ else "Client", self.session.client_id if self.session else "?", ) await self._reader_stopped.set() if self._reader_task is not None: self._reader_task = None await self.handle_connection_closed()
async def run(self, start_event: Optional[Event] = None) -> None: self._stop_event = create_event() self._running = True if start_event: await start_event.set() while self._running: async with open_cancel_scope() as self._acquire_cancel_scope: try: schedules = await self.data_store.acquire_schedules( self.identity, 100) except get_cancelled_exc_class(): break finally: del self._acquire_cancel_scope now = datetime.now(timezone.utc) for schedule in schedules: # Look up the task definition try: taskdef = self._get_taskdef(schedule.task_id) except LookupError: self.logger.error( 'Cannot locate task definition %r for schedule %r – ' 'removing schedule', schedule.task_id, schedule.id) schedule.next_fire_time = None continue # Calculate a next fire time for the schedule, if possible fire_times = [schedule.next_fire_time] calculate_next = schedule.trigger.next while True: try: fire_time = calculate_next() except Exception: self.logger.exception( 'Error computing next fire time for schedule %r of task %r – ' 'removing schedule', schedule.id, taskdef.id) break # Stop if the calculated fire time is in the future if fire_time is None or fire_time > now: schedule.next_fire_time = fire_time break # Only keep all the fire times if coalesce policy = "all" if schedule.coalesce is CoalescePolicy.all: fire_times.append(fire_time) elif schedule.coalesce is CoalescePolicy.latest: fire_times[0] = fire_time # Add one or more jobs to the job queue for fire_time in fire_times: schedule.last_fire_time = fire_time job = Job(taskdef.id, taskdef.func, schedule.args, schedule.kwargs, schedule.id, fire_time, schedule.next_deadline, schedule.tags) await self.data_store.add_job(job) await self.data_store.release_schedules(self.identity, schedules) await self._stop_event.set() del self._stop_event