async def test_cancel_from_shielded_scope() -> None:
async with create_task_group() as tg:
with CancelScope(shield=True) as inner_scope:
assert inner_scope.shield
tg.cancel_scope.cancel()
with pytest.raises(get_cancelled_exc_class()):
await sleep(0.01)
with pytest.raises(get_cancelled_exc_class()):
await sleep(0.01)
async def host_task() -> None:
nonlocal done
async with create_task_group() as tg:
with CancelScope(shield=True) as inner_scope:
assert inner_scope.shield
tg.cancel_scope.cancel()
with pytest.raises(get_cancelled_exc_class()):
await sleep(0)
with pytest.raises(get_cancelled_exc_class()):
await sleep(0)
done = True
async def _sender_loop(self, evt):
keepalive_timeout = self.session.keep_alive
if keepalive_timeout <= 0:
keepalive_timeout = None
try:
async with anyio.open_cancel_scope() as scope:
self._sender_task = scope
await evt.set()
while True:
packet = None
async with anyio.move_on_after(keepalive_timeout):
packet = await self._send_q.get()
if packet is None: # closing
break
if packet is None: # timeout
await self.handle_write_timeout()
continue
# self.logger.debug("%s > %r",'B' if 'Broker' in type(self).__name__ else 'C', packet)
await packet.to_stream(self.stream)
await self.plugins_manager.fire_event(
EVENT_MQTT_PACKET_SENT,
packet=packet,
session=self.session)
except ConnectionResetError:
await self.handle_connection_closed()
except anyio.get_cancelled_exc_class():
raise
except BaseException as e:
self.logger.warning("Unhandled exception", exc_info=e)
raise
finally:
async with anyio.fail_after(2, shield=True):
await self._sender_stopped.set()
self._sender_task = None
async def _run_reconnected(self, val: ValueEvent):
try:
async with anyio.open_cancel_scope() as scope:
self._current_run = scope
while True:
try:
await self._run_one(val)
except anyio.get_cancelled_exc_class():
raise
except (
BrokenPipeError,
TimeoutError,
EnvironmentError,
anyio.IncompleteRead,
ConnectionResetError,
anyio.ClosedResourceError,
StopAsyncIteration,
) as exc:
if val is not None and not val.is_set():
await val.set_error(exc)
return
logger.error("Disconnected")
val = None
await anyio.sleep(self._backoff)
if self._backoff < 10:
self._backoff *= 1.5
else:
pass
finally:
self._current_run = None
async def _run(self, client):
state = self.state
async with anyio.open_cancel_scope() as sc:
self.scope = sc
try:
logger.debug("START %s", self.name)
await self(client)
except anyio.get_cancelled_exc_class():
state.exc = "Canceled"
if self.scope is not None:
state.n_fail += 1
state.fail_count += 1
state.fail_map.append(True)
raise
except Exception as exc:
state.exc = traceback.format_exc().split('\n')
state.n_fail += 1
state.fail_count += 1
state.fail_map.append(True)
else:
state.fail_count = 0
state.fail_map.append(False)
finally:
state.n_run += 1
if any(state.fail_map):
del state.fail_map[:-20]
else:
state.fail_map = []
# zero out after 20 successes in sequence
self.scope = None
logger.debug("END %s", self.name)
async def _stop_site_on_cancel(aiohttp_tcp_site):
"""Stop the server after SIGINT."""
try:
await anyio.sleep(float('inf'))
except anyio.get_cancelled_exc_class():
logger.info(' Stopping the server '.center(50, '='))
await aiohttp_tcp_site.stop()
async def serve(self) -> None:
await self._master.register_service(self.service_name)
try:
await anyio.sleep_forever()
except anyio.get_cancelled_exc_class():
with anyio.CancelScope(shield=True), anyio.move_on_after(1):
await self._master.unregister_service(self.service_name)
raise
async def test_catch_cancellation():
finalizer_done = False
async with move_on_after(0.1):
try:
await sleep(1)
except get_cancelled_exc_class():
finalizer_done = True
raise
assert finalizer_done
async def _run_with_tg(self, *, evt: anyio.abc.Event=None):
try:
await super()._run_with_tg(evt=evt)
except anyio.get_cancelled_exc_class():
if self._done.is_set():
await self._handle_prev(_ResultEvent(self._result))
else:
await self._handle_prev(_ErrorEvent(CancelledError()))
raise
except Exception as exc:
await self._handle_prev(_ErrorEvent(exc))
except BaseException:
await self._handle_prev(_ErrorEvent(CancelledError()))
raise
else:
await self._handle_prev(_ResultEvent(self._result))
async def handle_tcpros(
self,
protocol: str,
header: abc.Header,
client: SocketStream,
) -> None:
"""Handle incoming service client traffic."""
require_fields(header, "service", "md5sum", "callerid")
check_md5sum(header, getattr(self.service_type, "_md5sum"))
await client.send(encode_header(self.header))
if header.get("probe") == "1":
return
persistent = header.get("persistent", "").lower() in ("1", "true")
serializer: Serializer[abc.ServiceRequestT] = Serializer()
buffered_receiver = BufferedByteReceiveStream(client)
while True:
request = self.request_class()
data = await read_data(buffered_receiver)
await anyio.to_thread.run_sync(request.deserialize, data)
setattr(request, "_connection_header", header)
try:
result = await self._handler(request)
except anyio.get_cancelled_exc_class():
raise
except Exception: # pylint: disable=broad-except
logger.error("Exception occured in service callback",
exc_info=True)
await client.send(
encode_byte(0) + encode_str("error processing request"))
else:
data = await anyio.to_thread.run_sync(serializer.serialize,
result)
await client.send(encode_byte(1) + data)
if not persistent:
return
async def release(self, partitions, suppress_cancelled=False):
client = self.app.redis
fn = self.app.scripts["lock_release"]
keys = [self.redis_lock_key(p.number) for p in partitions]
async with await client.pipeline(transaction=True) as pipe:
for key in keys:
await fn.execute(client=pipe, keys=[key], args=[self.id])
# Shield the execution from cancellation to ensure the locks are released.
async with anyio.fail_after(4, shield=True):
await pipe.execute()
try:
await self.set_owned(self.owned - partitions)
except anyio.get_cancelled_exc_class():
if not suppress_cancelled:
raise
finally:
logger.debug("released-partitions",
partitions=[p.number for p in partitions])
async def route_github_event(
*,
github_event: GitHubEvent,
github_app: GitHubApp,
) -> None:
"""Dispatch GitHub event to corresponsing handlers.
Set up ``RUNTIME_CONTEXT`` before doing that. This is so
the concrete event handlers have access to the API client
and flags in runtime.
"""
is_gh_action = isinstance(github_app, GitHubAction)
# pylint: disable=assigning-non-slot
RUNTIME_CONTEXT.IS_GITHUB_ACTION = is_gh_action
# pylint: disable=assigning-non-slot
RUNTIME_CONTEXT.IS_GITHUB_APP = not is_gh_action
# pylint: disable=assigning-non-slot
RUNTIME_CONTEXT.github_app = github_app
# pylint: disable=assigning-non-slot
RUNTIME_CONTEXT.github_event = github_event
# pylint: disable=assigning-non-slot
RUNTIME_CONTEXT.app_installation = None
if is_gh_action:
# pylint: disable=assigning-non-slot
RUNTIME_CONTEXT.app_installation_client = github_app.api_client
else:
with contextlib.suppress(LookupError):
# pylint: disable=pointless-string-statement
"""Provision an installation API client if possible.
Some events (like `ping`) are
happening application/GitHub-wide and are not bound to
a specific installation. The webhook payloads of such events
don't contain any reference to an installaion.
Some events don't even refer to a GitHub App
(e.g. `security_advisory`).
"""
github_install = await github_app.get_installation(github_event)
# pylint: disable=assigning-non-slot
RUNTIME_CONTEXT.app_installation = github_install
# pylint: disable=assigning-non-slot
RUNTIME_CONTEXT.app_installation_client = github_install.api_client
# Give GitHub a sec to deal w/ eventual consistency.
# This is only needed for events that arrive over HTTP.
# If the dispatcher is invoked from GitHub Actions,
# by the time it's invoked the action must be already consistently
# distributed within GitHub's systems because spawning VMs takes time
# and actions are executed in workflows that rely on those VMs.
await async_sleep(1)
try:
return await github_app.dispatch_event(github_event)
except GitHubActionError:
# Bypass GitHub Actions errors as they are supposed to be a
# mechanism for communicating outcomes and are expected.
raise
except get_cancelled_exc_class():
raise
except Exception as exc: # pylint: disable=broad-except
# NOTE: It's probably better to wrap each event handler with
# NOTE: try/except and call `capture_exception()` there instead.
# NOTE: We'll also need to figure out the magic of associating
# NOTE: breadcrumbs with event handlers.
sentry_sdk.capture_exception(exc)
# NOTE: Framework-wise, these exceptions are meaningless because they
# NOTE: can be anything random that the webhook author (octomachinery
# NOTE: end-user) forgot to handle. There's nothing we can do about
# NOTE: them except put in the log so that the end-user would be able
# NOTE: to properly debug their problem by inspecting the logs.
# NOTE: P.S. This is also where we'd inject Sentry
if isinstance(exc.__context__, get_cancelled_exc_class()):
# The CancelledError context is irrelevant to the
# user-defined webhook event handler workflow so we're
# dropping it from the logs:
exc.__context__ = None
logger.exception(
'An unhandled exception happened while running webhook '
'event handlers for "%s"...',
github_event.name,
)
delivery_id_msg = ('' if is_gh_action else
f' (Delivery ID: {github_event.delivery_id!s})')
logger.debug(
'The payload of "%s" event%s is: %r',
github_event.name,
delivery_id_msg,
github_event.payload,
)
if is_gh_action:
# NOTE: In GitHub Actions env, the app is supposed to run as
# NOTE: a foreground single event process rather than a
# NOTE: server for multiple events. It's okay to propagate
# NOTE: unhandled errors so that they are spit out to the
# NOTE: console.
raise
except BaseException: # SystemExit + KeyboardInterrupt + GeneratorExit
raise
def cancelledclass():
return anyio.get_cancelled_exc_class()
async def event_waiter():
nonlocal cancelled
try:
await sleep(3)
except get_cancelled_exc_class():
cancelled = True
if raise_interrupt is not None:
warnings.warn(
'raise_interrupt is deprecated, KeyboardInterrupt will cause this coroutine to be cancelled and then '
'be raised by the top level asyncio.run call or equivalent, and should be caught there. See #136.',
DeprecationWarning,
)
if stop_event is None:
stop_event_: 'AnyEvent' = anyio.Event()
else:
stop_event_ = stop_event
force_polling = _default_force_pulling(force_polling)
with RustNotify([str(p) for p in paths], debug, force_polling, poll_delay_ms) as watcher:
timeout = _calc_async_timeout(rust_timeout)
CancelledError = anyio.get_cancelled_exc_class()
while True:
async with anyio.create_task_group() as tg:
try:
raw_changes = await anyio.to_thread.run_sync(watcher.watch, debounce, step, timeout, stop_event_)
except (CancelledError, KeyboardInterrupt):
stop_event_.set()
# suppressing KeyboardInterrupt wouldn't stop it getting raised by the top level asyncio.run call
raise
tg.cancel_scope.cancel()
if raw_changes == 'timeout':
if yield_on_timeout:
yield set()
else:
async def _reader_loop(self, evt):
self.logger.debug("%s Starting reader coro", self.session.client_id)
keepalive_timeout = self.session.keep_alive
if keepalive_timeout <= 0:
keepalive_timeout = None
try:
async with anyio.create_task_group() as tg:
self._reader_task = tg
await evt.set()
while True:
try:
async with anyio.fail_after(keepalive_timeout):
fixed_header = await MQTTFixedHeader.from_stream(
self.stream)
if fixed_header is None:
self.logger.debug(
"%s No more data (EOF received), stopping reader coro",
self.session.client_id,
)
break
if (fixed_header.packet_type == RESERVED_0
or fixed_header.packet_type == RESERVED_15):
self.logger.warning(
"%s Received reserved packet, which is forbidden: closing connection",
self.session.client_id,
)
await self.handle_connection_closed()
break
cls = packet_class(fixed_header)
packet = await cls.from_stream(
self.stream, fixed_header=fixed_header)
# self.logger.debug("< %s %r",'B' if 'Broker' in type(self).__name__ else 'C', packet)
await self.plugins_manager.fire_event(
EVENT_MQTT_PACKET_RECEIVED,
packet=packet,
session=self.session,
)
try:
pt, direct = PACKET_TYPES[
packet.fixed_header.packet_type]
fn = getattr(self, "handle_" + pt)
except (KeyError, AttributeError):
self.logger.warning(
"%s Unhandled packet type: %s",
self.session.client_id,
packet.fixed_header.packet_type,
)
else:
try:
if direct:
await fn(packet)
else:
await tg.spawn(fn, packet)
except StopAsyncIteration:
break
except MQTTException:
self.logger.debug("Message discarded")
except TimeoutError:
self.logger.debug(
"%s Input stream read timeout",
self.session.client_id if self.session else "?",
)
await self.handle_read_timeout()
except NoDataException:
self.logger.debug("%s No data available",
self.session.client_id)
break # XXX
except anyio.get_cancelled_exc_class():
self.logger.warning("%s CANCEL", type(self).__name__)
raise
except BaseException as e:
self.logger.warning(
"%s Unhandled exception in reader coro",
type(self).__name__,
exc_info=e,
)
raise
await tg.cancel_scope.cancel()
finally:
async with anyio.fail_after(2, shield=True):
self.logger.debug(
"%s %s coro stopped",
"Broker" if "Broker" in type(self).__name__ else "Client",
self.session.client_id if self.session else "?",
)
await self._reader_stopped.set()
if self._reader_task is not None:
self._reader_task = None
await self.handle_connection_closed()
async def run(self, start_event: Optional[Event] = None) -> None:
self._stop_event = create_event()
self._running = True
if start_event:
await start_event.set()
while self._running:
async with open_cancel_scope() as self._acquire_cancel_scope:
try:
schedules = await self.data_store.acquire_schedules(
self.identity, 100)
except get_cancelled_exc_class():
break
finally:
del self._acquire_cancel_scope
now = datetime.now(timezone.utc)
for schedule in schedules:
# Look up the task definition
try:
taskdef = self._get_taskdef(schedule.task_id)
except LookupError:
self.logger.error(
'Cannot locate task definition %r for schedule %r – '
'removing schedule', schedule.task_id, schedule.id)
schedule.next_fire_time = None
continue
# Calculate a next fire time for the schedule, if possible
fire_times = [schedule.next_fire_time]
calculate_next = schedule.trigger.next
while True:
try:
fire_time = calculate_next()
except Exception:
self.logger.exception(
'Error computing next fire time for schedule %r of task %r – '
'removing schedule', schedule.id, taskdef.id)
break
# Stop if the calculated fire time is in the future
if fire_time is None or fire_time > now:
schedule.next_fire_time = fire_time
break
# Only keep all the fire times if coalesce policy = "all"
if schedule.coalesce is CoalescePolicy.all:
fire_times.append(fire_time)
elif schedule.coalesce is CoalescePolicy.latest:
fire_times[0] = fire_time
# Add one or more jobs to the job queue
for fire_time in fire_times:
schedule.last_fire_time = fire_time
job = Job(taskdef.id, taskdef.func, schedule.args,
schedule.kwargs, schedule.id, fire_time,
schedule.next_deadline, schedule.tags)
await self.data_store.add_job(job)
await self.data_store.release_schedules(self.identity, schedules)
await self._stop_event.set()
del self._stop_event
