async def do_run(self, event_bus: EndpointAPI) -> None:
boot_info = self._boot_info
if boot_info.args.enable_metrics:
metrics_service = metrics_service_from_args(
boot_info.args, AsyncioMetricsService)
else:
# Use a NoopMetricsService so that no code branches need to be taken if metrics
# are disabled
metrics_service = NOOP_METRICS_SERVICE
trinity_config = boot_info.trinity_config
NodeClass = trinity_config.get_app_config(Eth1AppConfig).node_class
node = NodeClass(event_bus, metrics_service, trinity_config)
strategy = self.get_active_strategy(boot_info)
async with background_asyncio_service(node) as node_manager:
sync_task = create_task(
self.launch_sync(node, strategy, boot_info, event_bus),
self.name)
# The Node service is our responsibility, so we must exit if either that or the syncer
# returns.
node_manager_task = create_task(
node_manager.wait_finished(),
f'{NodeClass.__name__} wait_finished() task')
tasks = [sync_task, node_manager_task]
try:
await wait_first(tasks, max_wait_after_cancellation=2)
except asyncio.TimeoutError:
self.logger.warning(
"Timed out waiting for tasks to terminate after cancellation: %s",
tasks)
async def apply(
self,
connection: ConnectionAPI) -> AsyncIterator[asyncio.Task[Any]]:
"""
See LogicAPI.apply()
The future returned here will be done when the first of the futures obtained from applying
all behaviors of this application is done.
"""
self.connection = connection
async with contextlib.AsyncExitStack() as stack:
futures: List[asyncio.Task[Any]] = []
# First apply all the child behaviors
for behavior in self._behaviors:
if behavior.should_apply_to(connection):
fut = await stack.enter_async_context(
behavior.apply(connection))
futures.append(fut)
# If none of our behaviors were applied, use a never-ending task so that callsites
# can wait on it like when behaviors are applied.
if not futures:
futures.append(
create_task(_never_ending_coro(),
f'Application/{self.name}/no-behaviors-fut'))
# Now register ourselves with the connection.
with connection.add_logic(self.name, self):
name = f'Application/{self.name}/apply/{connection.remote}'
yield create_task(wait_first(futures), name=name)
async def _do_run(self) -> None:
with child_process_logging(self._boot_info):
endpoint_name = self.get_endpoint_name()
event_bus_service = AsyncioEventBusService(
self._boot_info.trinity_config,
endpoint_name,
)
async with background_asyncio_service(
event_bus_service) as eventbus_manager:
event_bus = await event_bus_service.get_event_bus()
loop_monitoring_task = create_task(
self._loop_monitoring_task(event_bus),
f'AsyncioIsolatedComponent/{self.name}/loop_monitoring_task'
)
do_run_task = create_task(
self.do_run(event_bus),
f'AsyncioIsolatedComponent/{self.name}/do_run')
eventbus_task = create_task(
eventbus_manager.wait_finished(),
f'AsyncioIsolatedComponent/{self.name}/eventbus/wait_finished'
)
try:
max_wait_after_cancellation = 2
tasks = [do_run_task, eventbus_task, loop_monitoring_task]
if self._boot_info.profile:
with profiler(f'profile_{self.get_endpoint_name()}'):
await wait_first(tasks,
max_wait_after_cancellation)
else:
# XXX: When open_in_process() injects a KeyboardInterrupt into us (via
# coro.throw()), we hang forever here, until open_in_process() times
# out and sends us a SIGTERM, at which point we exit without executing
# either the except or the finally blocks below.
# See https://github.com/ethereum/trinity/issues/1711 for more.
try:
await wait_first(
tasks,
max_wait_after_cancellation,
)
except asyncio.TimeoutError:
self.logger.warning(
"Timed out waiting for tasks to terminate after cancellation: %s",
tasks)
except KeyboardInterrupt:
self.logger.debug("%s: KeyboardInterrupt", self)
# Currently we never reach this code path, but when we fix the issue above
# it will be needed.
return
finally:
# Once we start seeing this in the logs after a Ctrl-C, we'll likely have
# figured out the issue above.
self.logger.debug("%s: do_run() finished", self)
async def run_behaviors(self, behaviors: Tuple[BehaviorAPI, ...]) -> None:
async with contextlib.AsyncExitStack() as stack:
futures: List[asyncio.Task[Any]] = [
create_task(self.manager.wait_finished(),
'Connection/run_behaviors/wait_finished')
]
for behavior in behaviors:
if behavior.should_apply_to(self):
behavior_exit = await stack.enter_async_context(
behavior.apply(self))
futures.append(behavior_exit)
self.behaviors_applied.set()
# If wait_first() is called, cleanup_tasks() will be a no-op, but if any post_apply()
# calls raise an exception, it will ensure we don't leak pending tasks that would
# cause asyncio to complain.
async with cleanup_tasks(*futures):
try:
for behavior in behaviors:
behavior.post_apply()
await wait_first(futures, max_wait_after_cancellation=2)
except PeerConnectionLost:
# Any of our behaviors may propagate a PeerConnectionLost, which is to be
# expected as many Connection APIs used by them can raise that. To avoid a
# DaemonTaskExit since we're returning silently, ensure we're cancelled.
self.manager.cancel()
async def apply(
self,
connection: ConnectionAPI) -> AsyncIterator[asyncio.Future[None]]:
service = PingAndDisconnectIfIdle(connection, self.idle_timeout)
async with background_asyncio_service(service) as manager:
task_name = f'PingAndDisconnectIfIdleService/{connection.remote}'
yield create_task(manager.wait_finished(), name=task_name)
def create_raising_after_cancellation_task():
async def _raise_after_cancellation():
try:
await asyncio.Future()
except asyncio.CancelledError:
raise AfterCancellationException()
return create_task(_raise_after_cancellation(), 'raising-after-cancellation')
def create_rogue_task(sleep_after_cancellation):
async def _rogue(sleep_after_cancellation):
try:
await asyncio.Future()
except asyncio.CancelledError:
await asyncio.sleep(sleep_after_cancellation)
return create_task(_rogue(sleep_after_cancellation), 'rogue-task')
async def _find_urgent_nodes(
self,
queen: ETHPeer,
urgent_hashes: Tuple[Hash32, ...],
batch_id: int) -> None:
# Generate and schedule the tasks to request the urgent node(s) from multiple peers
knights = tuple(self._queen_tracker.pop_knights())
urgent_requests = [
create_task(
self._get_nodes(peer, urgent_hashes, urgent=True),
name=f"BeamDownloader._get_nodes({peer.remote}, ...)",
)
for peer in (queen,) + knights
]
# Process the returned nodes, in the order they complete
urgent_timer = Timer()
async with cleanup_tasks(*urgent_requests):
for result_coro in asyncio.as_completed(urgent_requests):
nodes_returned, new_nodes, peer = await result_coro
time_on_urgent = urgent_timer.elapsed
# After the first peer returns something, cancel all other pending tasks
if len(nodes_returned) > 0:
# Stop waiting for other peer responses
break
elif peer == queen:
self.logger.debug("queen %s returned 0 urgent nodes of %r", peer, urgent_hashes)
# Wait for the next peer response
# Log the received urgent nodes
if peer == queen:
log_header = "beam-queen-urgent-rtt"
else:
log_header = "spread-beam-urgent-rtt"
self.logger.debug(
"%s: got %d/%d +%d nodes in %.3fs from %s (%s)",
log_header,
len(nodes_returned),
len(urgent_hashes),
len(new_nodes),
time_on_urgent,
peer.remote,
urgent_hashes[0][:2].hex(),
)
# Stat updates
self._total_processed_nodes += len(new_nodes)
self._urgent_processed_nodes += len(new_nodes)
self._time_on_urgent += time_on_urgent
# Complete the task in the TaskQueue
self._node_tasks.complete(batch_id, tuple(node_hash for node_hash, _ in nodes_returned))
# Re-insert the peers for the next request
for knight in knights:
self._queen_tracker.insert_peer(knight)
async def run(self) -> None:
connection_config = ConnectionConfig.from_name(
MAIN_EVENTBUS_ENDPOINT, self._boot_info.trinity_config.ipc_dir)
async with AsyncioEndpoint.serve(connection_config) as endpoint:
self._endpoint = endpoint
# start the background process that tracks and propagates available
# endpoints to the other connected endpoints
self.manager.run_daemon_task(
self._track_and_propagate_available_endpoints)
await endpoint.wait_until_any_endpoint_subscribed_to(
EventBusConnected)
# signal the endpoint is up and running and available
self._endpoint_available.set()
# instantiate all of the components
all_components = tuple(
component_cls(self._boot_info)
for component_cls in self._component_types)
# filter out any components that should not be enabled.
enabled_components = tuple(component
for component in all_components
if component.is_enabled)
# a little bit of extra try/finally structure here to produce good
# logging messages about the component lifecycle.
from p2p.asyncio_utils import create_task, wait_first
try:
self.logger.info(
"Starting components: %s",
'/'.join(component.name
for component in enabled_components),
)
tasks: List[asyncio.Task[Any]] = []
for component in enabled_components:
tasks.append(
create_task(
component.run_in_process(),
f'IsolatedComponent/{component.name}/run_in_process'
))
tasks.append(
asyncio.create_task(self._trigger_component_exit.wait()))
self.logger.info("Components started")
try:
# The timeout is long as our component tasks can do a lot of stuff during
# their cleanup.
await wait_first(tasks, max_wait_after_cancellation=10)
finally:
self.logger.info("Stopping components")
finally:
self.logger.info("Components stopped.")
self.manager.cancel()
async def connect(self, remote: NodeAPI) -> BasePeer:
"""
Connect to the given remote and return a Peer instance when successful.
Returns None if the remote is unreachable, times out or is useless.
"""
if not self._handshake_locks.is_locked(remote):
self.logger.warning("Tried to connect to %s without acquiring lock first!", remote)
if any(peer.remote == remote for peer in self.connected_nodes.values()):
self.logger.warning(
"Attempted to connect to peer we are already connected to: %s", remote)
raise IneligiblePeer(f"Already connected to {remote}")
try:
self.logger.debug("Connecting to %s...", remote)
task_name = f'PeerPool/Handshake/{remote}'
task = create_task(self.get_peer_factory().handshake(remote), task_name)
return await asyncio.wait_for(task, timeout=HANDSHAKE_TIMEOUT)
except BadAckMessage:
# This is kept separate from the
# `COMMON_PEER_CONNECTION_EXCEPTIONS` to be sure that we aren't
# silencing an error in our authentication code.
self.logger.error('Got bad auth ack from %r', remote)
# dump the full stacktrace in the debug logs
self.logger.debug('Got bad auth ack from %r', remote, exc_info=True)
raise
except MalformedMessage as e:
# This is kept separate from the
# `COMMON_PEER_CONNECTION_EXCEPTIONS` to be sure that we aren't
# silencing an error in how we decode messages during handshake.
self.logger.error('Got malformed response from %r during handshake', remote)
# dump the full stacktrace in the debug logs
self.logger.debug('Got malformed response from %r', remote, exc_info=True)
self.connection_tracker.record_failure(remote, e)
raise
except HandshakeFailure as e:
self.logger.debug("Could not complete handshake with %r: %s", remote, repr(e))
self.connection_tracker.record_failure(remote, e)
raise
except COMMON_PEER_CONNECTION_EXCEPTIONS as e:
self.logger.debug("Could not complete handshake with %r: %s", remote, repr(e))
self.connection_tracker.record_failure(remote, e)
raise
finally:
# XXX: We sometimes get an exception here but the task is finished and with
# an exception as well. No idea how that happens but if we don't consume the
# task's exception, asyncio complains.
if not task.done():
task.cancel()
try:
await task
except (Exception, asyncio.CancelledError):
pass
async def __aenter__(self) -> Tuple[Any, ...]:
futures = [
create_task(cm.__aenter__(), f'AsyncContextGroup/{repr(cm)}')
for cm in self.cms
]
await asyncio.wait(futures)
# Exclude futures not successfully entered from the list so that we don't attempt to exit
# them.
self.cms_to_exit = tuple(
cm for cm, future in zip(self.cms, futures)
if not future.cancelled() and not future.exception())
try:
return tuple(future.result() for future in futures)
except: # noqa: E722
await self._exit(*sys.exc_info())
raise
async def apply(
self,
connection: ConnectionAPI) -> AsyncIterator[asyncio.Task[Any]]:
"""
See LogicAPI.apply()
The future returned here will never be done as a CommandHandler doesn't run a background
task.
"""
self.connection = connection
with connection.add_command_handler(self.command_type,
cast(HandlerFn, self.handle)):
yield create_task(
_never_ending_coro(),
f'CommandHandler/{self.__class__.__name__}/apply')
async def run_exchange(
self,
connection: ConnectionAPI) -> AsyncIterator[asyncio.Task[Any]]:
protocol = connection.get_protocol_for_command_type(
self.get_request_cmd_type())
response_stream: ResponseCandidateStream[
TRequestCommand,
TResponseCommand] = ResponseCandidateStream( # noqa: E501
connection,
protocol,
self.get_response_cmd_type(),
)
async with background_asyncio_service(
response_stream) as response_stream_manager:
self._manager = ExchangeManager(
connection,
response_stream,
)
name = f'{self.__class__.__name__}/{connection.remote}'
yield create_task(response_stream_manager.wait_finished(),
name=name)
async def _do_run(self) -> None:
with child_process_logging(self._boot_info):
endpoint_name = self.get_endpoint_name()
event_bus_service = AsyncioEventBusService(
self._boot_info.trinity_config,
endpoint_name,
)
# FIXME: Must terminate component if event_bus_service terminates.
async with background_asyncio_service(event_bus_service):
event_bus = await event_bus_service.get_event_bus()
loop_monitoring_task = create_task(
self._loop_monitoring_task(event_bus),
f'AsyncioIsolatedComponent/{self.name}/loop_monitoring_task'
)
# FIXME: Must terminate component if loop_monitoring_task terminates.
async with cleanup_tasks(loop_monitoring_task):
try:
if self._boot_info.profile:
with profiler(
f'profile_{self.get_endpoint_name()}'):
await self.do_run(event_bus)
else:
# XXX: When open_in_process() injects a KeyboardInterrupt into us (via
# coro.throw()), we hang forever here, until open_in_process() times
# out and sends us a SIGTERM, at which point we exit without executing
# either the except or the finally blocks below.
# See https://github.com/ethereum/trinity/issues/1711 for more.
await self.do_run(event_bus)
except KeyboardInterrupt:
# Currently we never reach this code path, but when we fix the issue above
# it will be needed.
return
finally:
# Once we start seeing this in the logs after a Ctrl-C, we'll likely have
# figured out the issue above.
self.logger.debug("%s: do_run() finished", self)
async def _find_urgent_nodes(self, queen: ETHPeer,
urgent_hashes: Tuple[Hash32, ...],
batch_id: int) -> None:
# Generate and schedule the tasks to request the urgent node(s) from multiple peers
knights = tuple(self._queen_tracker.pop_knights())
urgent_requests = [
create_task(
self._get_nodes(peer, urgent_hashes, urgent=True),
name=f"BeamDownloader._get_nodes({peer.remote}, ...)",
) for peer in (queen, ) + knights
]
# Process the returned nodes, in the order they complete
urgent_timer = Timer()
async with cleanup_tasks(*urgent_requests):
for result_coro in asyncio.as_completed(urgent_requests):
nodes_returned, new_nodes, peer = await result_coro
time_on_urgent = urgent_timer.elapsed
# After the first peer returns something, cancel all other pending tasks
if len(nodes_returned) > 0:
# Stop waiting for other peer responses
break
elif peer == queen:
self.logger.debug("queen %s returned 0 urgent nodes of %r",
peer, urgent_hashes)
# Wait for the next peer response
# Log the received urgent nodes
if peer == queen:
log_header = "beam-queen-urgent-rtt"
else:
log_header = "spread-beam-urgent-rtt"
self.logger.debug(
"%s: got %d/%d +%d nodes in %.3fs from %s (%s)",
log_header,
len(nodes_returned),
len(urgent_hashes),
len(new_nodes),
time_on_urgent,
peer.remote,
urgent_hashes[0][:2].hex(),
)
# Stat updates
self._total_processed_nodes += len(new_nodes)
self._urgent_processed_nodes += len(new_nodes)
self._time_on_urgent += time_on_urgent
# If it took to long to get a single urgent node, then increase "spread" factor
if len(urgent_hashes
) == 1 and time_on_urgent > MAX_ACCEPTABLE_WAIT_FOR_URGENT_NODE:
new_spread_factor = clamp(
0,
self._max_spread_beam_factor(),
self._spread_factor + 1,
)
if new_spread_factor != self._spread_factor:
self.logger.debug(
"spread-beam-update: Urgent node latency=%.3fs, update factor %d to %d",
time_on_urgent,
self._spread_factor,
new_spread_factor,
)
self._queen_tracker.set_desired_knight_count(new_spread_factor)
self._spread_factor = new_spread_factor
# Complete the task in the TaskQueue
task_hashes = tuple(node_hash for node_hash, _ in nodes_returned)
await self._node_tasks.complete(batch_id, task_hashes)
# Re-insert the peers for the next request
for knight in knights:
self._queen_tracker.insert_peer(knight)
async def run(self) -> AsyncIterator[asyncio.Task[Any]]:
from p2p.asyncio_utils import create_task
yield create_task(self._run_in_process(), f'IsolatedComponent/{self.name}/run_in_process')
def create_sleep0_task():
return create_task(asyncio.sleep(0), 'sleep-0')
def create_raising_task():
async def _raise():
raise TaskException()
return create_task(_raise(), 'raising-task')
def create_sleep_forever_task():
async def _sleep_forever():
await asyncio.Future()
return create_task(_sleep_forever(), 'sleep-forever')
