class QueeningQueue(Service, PeerSubscriber, QueenTrackerAPI):
# The best peer gets skipped for backfill, because we prefer to use it for
# urgent beam sync nodes. _queen_peer should only ever be set to a new peer
# in _insert_peer(). It may be set to None anywhere.
_queen_peer: ETHPeer = None
_queen_updated: asyncio.Event
_waiting_peers: WaitingPeers[ETHPeer]
# We are only interested in peers entering or leaving the pool
subscription_msg_types: FrozenSet[Type[CommandAPI[Any]]] = frozenset()
# This is a rather arbitrary value, but when the sync is operating normally we never see
# the msg queue grow past a few hundred items, so this should be a reasonable limit for
# now.
msg_queue_maxsize: int = 2000
def __init__(self, peer_pool: ETHPeerPool) -> None:
self.logger = get_logger('trinity.sync.beam.queen.QueeningQueue')
self._peer_pool = peer_pool
self._waiting_peers = WaitingPeers(NodeDataV65)
self._queen_updated = asyncio.Event()
async def run(self) -> None:
with self.subscribe(self._peer_pool):
await self.manager.wait_finished()
def register_peer(self, peer: BasePeer) -> None:
super().register_peer(peer)
self._insert_peer(peer) # type: ignore
def deregister_peer(self, peer: BasePeer) -> None:
super().deregister_peer(peer)
if self._queen_peer == peer:
self._queen_peer = None
# If it's not the queen, we will catch the peer as cancelled when we try to draw it
# as a peasant. (We can't drop an element from the middle of a Queue)
async def get_queen_peer(self) -> ETHPeer:
"""
Wait until a queen peer is designated, then return it.
"""
t = Timer()
while self._queen_peer is None:
await self._queen_updated.wait()
self._queen_updated.clear()
queen_starve_time = t.elapsed
if queen_starve_time > WARN_AFTER_QUEEN_STARVED:
self.logger.info(
"Blocked for %.2fs waiting for queen=%s",
queen_starve_time,
self._queen_peer,
)
return self._queen_peer
@property
def queen(self) -> Optional[ETHPeer]:
"""
Might be None. If None is unacceptable, use :meth:`get_queen_peer`
"""
return self._queen_peer
async def pop_fastest_peasant(self) -> ETHPeer:
"""
Get the fastest peer that is not the queen.
"""
while self.manager.is_running:
peer = await self._waiting_peers.get_fastest()
if not peer.is_alive:
# drop any peers that aren't alive anymore
self.logger.info(
"Dropping %s from beam peers, as no longer active", peer)
if peer == self._queen_peer:
self._queen_peer = None
continue
if self._should_be_queen(peer):
self.logger.debug(
"About to draw peasant %s, but realized it should be queen",
peer)
self._insert_peer(peer)
continue
# FIXME: This should not be hard-coded to get_node_data as we could, in theory, be
# used to request other types of data.
peer_is_requesting = peer.eth_api.get_node_data.is_requesting
if peer_is_requesting:
# skip the peer if there's an active request
self.logger.debug("QueenQueuer is skipping active peer %s",
peer)
loop = asyncio.get_event_loop()
loop.call_later(10, functools.partial(self._insert_peer, peer))
continue
return peer
# This should never happen as we run as a daemon and if we return before our caller it'd
# raise a DaemonTaskExit, but just in case we raise a CancelledError() to ensure our
# caller realizes we've stopped.
self.logger.error("Service ended before a queen peer could be elected")
raise asyncio.CancelledError()
def readd_peasant(self, peer: ETHPeer, delay: float = 0) -> None:
if delay > 0:
loop = asyncio.get_event_loop()
loop.call_later(delay, functools.partial(self._insert_peer, peer))
else:
self._insert_peer(peer)
def penalize_queen(self,
peer: ETHPeer,
delay: float = NON_IDEAL_RESPONSE_PENALTY) -> None:
if peer == self._queen_peer:
self._queen_peer = None
self.logger.debug(
"Penalizing %s for %.2fs, for minor infraction",
peer,
delay,
)
loop = asyncio.get_event_loop()
loop.call_later(delay, functools.partial(self._insert_peer, peer))
def _should_be_queen(self, peer: ETHPeer) -> bool:
if not peer.is_alive:
raise ValueError(f"{peer} is no longer alive")
if self._queen_peer is None:
return True
elif not self._queen_peer.is_alive:
return True
elif peer == self._queen_peer:
return True
else:
new_peer_quality = _peer_sort_key(peer)
current_queen_quality = _peer_sort_key(self._queen_peer)
# Quality is designed so that an ascending sort puts the best quality at the front
# of a sequence. So, a lower value means a better quality.
return new_peer_quality < current_queen_quality
def _insert_peer(self, peer: ETHPeer) -> None:
"""
Add peer as ready to receive requests. Check if it should be queen, and promote if
appropriate. Otherwise, insert to be drawn as a peasant.
If the given peer is no longer running, do nothing. This is needed because we're sometimes
called via loop.call_later().
"""
if not peer.is_alive:
self.logger.debug(
"Peer %s is no longer alive, not adding to queue", peer)
return
if self._should_be_queen(peer):
old_queen, self._queen_peer = self._queen_peer, peer
if peer != old_queen:
# We only need to log the change if there was an actual change in queen
self.logger.debug("Switching queen peer from %s to %s",
old_queen, peer)
self._queen_updated.set()
if old_queen is not None:
self._waiting_peers.put_nowait(old_queen)
else:
self._waiting_peers.put_nowait(peer)
class QueeningQueue(Service, PeerSubscriber, QueenTrackerAPI):
# The best peer gets skipped for backfill, because we prefer to use it for
# urgent beam sync nodes. _queen_peer should only ever be set to a new peer
# in _insert_peer(). It may be set to None anywhere.
_queen_peer: ETHPeer = None
_queen_updated: asyncio.Event
_knights: WaitingPeers[ETHPeer]
_peasants: WaitingPeers[ETHPeer]
# We are only interested in peers entering or leaving the pool
subscription_msg_types: FrozenSet[Type[CommandAPI[Any]]] = frozenset()
# This is a rather arbitrary value, but when the sync is operating normally we never see
# the msg queue grow past a few hundred items, so this should be a reasonable limit for
# now.
msg_queue_maxsize: int = 2000
_report_interval = 30
def __init__(self, peer_pool: ETHPeerPool) -> None:
self.logger = get_logger('trinity.sync.beam.queen.QueeningQueue')
self._peer_pool = peer_pool
self._knights = WaitingPeers(NodeDataV65)
self._peasants = WaitingPeers(NodeDataV65)
self._queen_updated = asyncio.Event()
self._desired_knights = 0
self._num_peers = 0
async def run(self) -> None:
with self.subscribe(self._peer_pool):
self.manager.run_daemon_task(self._report_statistics)
await self.manager.wait_finished()
async def _report_statistics(self) -> None:
while self.manager.is_running:
await asyncio.sleep(self._report_interval)
self.logger.debug(
"queen-stats: free_knights=%d/%d free_peasants=%d/%d queen=%s",
len(self._knights),
self._desired_knights,
len(self._peasants),
self._num_peers - self._desired_knights - 1,
self._queen_peer,
)
def register_peer(self, peer: BasePeer) -> None:
super().register_peer(peer)
self._insert_peer(peer) # type: ignore
self._num_peers += 1
def deregister_peer(self, peer: BasePeer) -> None:
super().deregister_peer(peer)
if self._queen_peer == peer:
self._queen_peer = None
# If it's not the queen, we will catch the peer as cancelled when we try to draw it
# as a peasant. (We can't drop an element from the middle of a Queue)
self._num_peers -= 1
async def get_queen_peer(self) -> ETHPeer:
"""
Wait until a queen peer is designated, then return it.
"""
t = Timer()
while self._queen_peer is None:
try:
promote_knight = self._knights.pop_nowait()
except asyncio.QueueEmpty:
# There are no knights available. Wait for a new queen to appear.
await self._queen_updated.wait()
self._queen_updated.clear()
else:
# There is a knight who can be promoted to queen immediately.
self._insert_peer(promote_knight)
queen_starve_time = t.elapsed
if queen_starve_time > WARN_AFTER_QUEEN_STARVED:
self.logger.debug(
"Blocked for %.2fs waiting for queen=%s",
queen_starve_time,
self._queen_peer,
)
return self._queen_peer
@property
def queen(self) -> Optional[ETHPeer]:
"""
Might be None. If None is unacceptable, use :meth:`get_queen_peer`
"""
return self._queen_peer
def set_desired_knight_count(self, desired_knights: int) -> None:
self._desired_knights = desired_knights
# Promote knights if there are not enough
while len(self._knights) < self._desired_knights:
try:
promoted_knight = self._peasants.pop_nowait()
except asyncio.QueueEmpty:
# no peasants available to promote
break
else:
self._knights.put_nowait(promoted_knight)
def pop_knights(self) -> Iterable[ETHPeer]:
for _ in range(self._desired_knights):
try:
yield self._knights.pop_nowait()
except asyncio.QueueEmpty:
try:
yield self._peasants.pop_nowait()
except asyncio.QueueEmpty:
break
# Push all remaining knights down to peasants
while len(self._knights):
try:
demoted_knight = self._knights.pop_nowait()
except asyncio.QueueEmpty:
break
else:
self._peasants.put_nowait(demoted_knight)
async def pop_fastest_peasant(self) -> ETHPeer:
"""
Get the fastest peer that is not the queen.
"""
# NOTE: We don't use the common `while self.is_running` idiom here because this method
# runs in a task belonging to the service that runs this service as a child, so that task
# has to monitor the child service (us) and stop calling this method when we're no longer
# running.
while True:
peer = await self._peasants.get_fastest()
if not peer.is_alive:
# drop any peers that aren't alive anymore
self.logger.info(
"Dropping %s from beam peers, as no longer active", peer)
if peer == self._queen_peer:
self._queen_peer = None
continue
if self._should_be_queen(peer):
self.logger.debug(
"About to draw peasant %s, but realized it should be queen",
peer)
self._insert_peer(peer)
continue
# FIXME: This should not be hard-coded to get_node_data as we could, in theory, be
# used to request other types of data.
peer_is_requesting = peer.eth_api.get_node_data.is_requesting
if peer_is_requesting:
# skip the peer if there's an active request
self.logger.debug("QueenQueuer is skipping active peer %s",
peer)
self.insert_peer(peer, delay=NON_IDEAL_RESPONSE_PENALTY)
continue
return peer
def insert_peer(self, peer: ETHPeer, delay: float = 0) -> None:
if not peer.is_alive:
# Peer exited, dropping it...
return
elif self._should_be_queen(peer):
self.logger.debug("Fast-tracking peasant to promote to queen: %s",
peer)
self._insert_peer(peer)
elif delay > 0:
loop = asyncio.get_event_loop()
loop.call_later(delay, functools.partial(self._insert_peer, peer))
else:
self._insert_peer(peer)
def penalize_queen(self,
peer: ETHPeer,
delay: float = NON_IDEAL_RESPONSE_PENALTY) -> None:
if peer == self._queen_peer:
self._queen_peer = None
self.logger.debug(
"Penalizing %s for %.2fs, for minor infraction",
peer,
delay,
)
loop = asyncio.get_event_loop()
loop.call_later(delay, functools.partial(self._insert_peer, peer))
def _should_be_queen(self, peer: ETHPeer) -> bool:
if not peer.is_alive:
raise ValueError(f"{peer} is no longer alive")
if self._queen_peer is None:
return True
elif not self._queen_peer.is_alive:
return True
elif peer == self._queen_peer:
return True
else:
new_peer_quality = _peer_sort_key(peer)
current_queen_quality = _peer_sort_key(self._queen_peer)
# Quality is designed so that an ascending sort puts the best quality at the front
# of a sequence. So, a lower value means a better quality.
return new_peer_quality < current_queen_quality
def _insert_peer(self, peer: ETHPeer) -> None:
"""
Add peer as ready to receive requests. Check if it should be queen, and promote if
appropriate. Otherwise, insert to be drawn as a peasant.
If the given peer is no longer running, do nothing. This is needed because we're sometimes
called via loop.call_later().
"""
if not peer.is_alive:
self.logger.debug(
"Peer %s is no longer alive, not adding to queue", peer)
return
if self._should_be_queen(peer):
old_queen, self._queen_peer = self._queen_peer, peer
if peer != old_queen:
# We only need to log the change if there was an actual change in queen
self.logger.debug("Switching queen peer from %s to %s",
old_queen, peer)
self._queen_updated.set()
if old_queen is not None:
self._insert_peer(old_queen)
else:
self._peasants.put_nowait(peer)
# If the number of knights is too low, add one.
# Insert the peasant before promoting a knight, in case a former peasant is
# now faster than the peer being inserted now.
if len(self._knights) < self._desired_knights:
try:
promoted_knight = self._peasants.pop_nowait()
except asyncio.QueueEmpty:
# no peasants available to promote, exit cleanly
return
else:
self._knights.put_nowait(promoted_knight)
class QueeningQueue(BaseService, PeerSubscriber, QueenTrackerAPI):
# The best peer gets skipped for backfill, because we prefer to use it for
# urgent beam sync nodes
_queen_peer: ETHPeer = None
_waiting_peers: WaitingPeers[ETHPeer]
# We are only interested in peers entering or leaving the pool
subscription_msg_types: FrozenSet[Type[CommandAPI[Any]]] = frozenset()
# This is a rather arbitrary value, but when the sync is operating normally we never see
# the msg queue grow past a few hundred items, so this should be a reasonable limit for
# now.
msg_queue_maxsize: int = 2000
def __init__(self, peer_pool: ETHPeerPool, token: CancelToken = None) -> None:
super().__init__(token=token)
self._peer_pool = peer_pool
self._waiting_peers = WaitingPeers(NodeData)
async def _run(self) -> None:
with self.subscribe(self._peer_pool):
await self.cancellation()
def register_peer(self, peer: BasePeer) -> None:
super().register_peer(peer)
# when a new peer is added to the pool, add it to the idle peer list
self._waiting_peers.put_nowait(peer) # type: ignore
def deregister_peer(self, peer: BasePeer) -> None:
super().deregister_peer(peer)
if self._queen_peer == peer:
self._queen_peer = None
async def get_queen_peer(self) -> ETHPeer:
"""
Wait until a queen peer is designated, then return it.
"""
while self._queen_peer is None:
peer = await self._waiting_peers.get_fastest()
self._update_queen(peer)
return self._queen_peer
@property
def queen(self) -> Optional[ETHPeer]:
"""
Might be None. If None is unacceptable, use :meth:`get_queen_peer`
"""
return self._queen_peer
async def pop_fastest_peasant(self) -> ETHPeer:
"""
Get the fastest peer that is not the queen.
"""
while self.is_operational:
peer = await self.wait(self._waiting_peers.get_fastest())
if not peer.is_operational:
# drop any peers that aren't alive anymore
self.logger.warning("Dropping %s from beam queue as no longer operational", peer)
if peer == self._queen_peer:
self._queen_peer = None
continue
old_queen = self._update_queen(peer)
if peer == self._queen_peer:
self.logger.debug("Switching queen peer from %s to %s", old_queen, peer)
continue
if peer.eth_api.get_node_data.is_requesting:
# skip the peer if there's an active request
self.logger.debug("Queen Queuer is skipping active peer %s", peer)
self.call_later(10, self._waiting_peers.put_nowait, peer)
continue
return peer
raise OperationCancelled("Service ended before a queen peer could be elected")
def readd_peasant(self, peer: ETHPeer, delay: float = 0) -> None:
if delay > 0:
self.call_later(delay, self._waiting_peers.put_nowait, peer)
else:
self._waiting_peers.put_nowait(peer)
def penalize_queen(self, peer: ETHPeer) -> None:
if peer == self._queen_peer:
self._queen_peer = None
delay = NON_IDEAL_RESPONSE_PENALTY
self.logger.debug(
"Penalizing %s for %.2fs, for minor infraction",
peer,
delay,
)
self.call_later(delay, self._waiting_peers.put_nowait, peer)
def _update_queen(self, peer: ETHPeer) -> ETHPeer:
'''
@return peer that is no longer queen
'''
if self._queen_peer is None:
self._queen_peer = peer
return None
elif peer == self._queen_peer:
# nothing to do, peer is already the queen
return None
elif _peer_sort_key(peer) < _peer_sort_key(self._queen_peer):
old_queen, self._queen_peer = self._queen_peer, peer
self._waiting_peers.put_nowait(old_queen)
return old_queen
else:
# nothing to do, peer is slower than the queen
return None
class QueeningQueue(Service, PeerSubscriber, QueenTrackerAPI):
# The best peer gets skipped for backfill, because we prefer to use it for
# urgent beam sync nodes
_queen_peer: ETHPeer = None
_waiting_peers: WaitingPeers[ETHPeer]
# We are only interested in peers entering or leaving the pool
subscription_msg_types: FrozenSet[Type[CommandAPI[Any]]] = frozenset()
# This is a rather arbitrary value, but when the sync is operating normally we never see
# the msg queue grow past a few hundred items, so this should be a reasonable limit for
# now.
msg_queue_maxsize: int = 2000
def __init__(self, peer_pool: ETHPeerPool) -> None:
self.logger = get_logger('trinity.sync.beam.queen.QueeningQueue')
self._peer_pool = peer_pool
self._waiting_peers = WaitingPeers(NodeDataV65)
async def run(self) -> None:
with self.subscribe(self._peer_pool):
await self.manager.wait_finished()
def register_peer(self, peer: BasePeer) -> None:
super().register_peer(peer)
# when a new peer is added to the pool, add it to the idle peer list
self._waiting_peers.put_nowait(peer) # type: ignore
def deregister_peer(self, peer: BasePeer) -> None:
super().deregister_peer(peer)
if self._queen_peer == peer:
self._queen_peer = None
async def get_queen_peer(self) -> ETHPeer:
"""
Wait until a queen peer is designated, then return it.
"""
while self._queen_peer is None:
peer = await self._waiting_peers.get_fastest()
self._update_queen(peer)
return self._queen_peer
@property
def queen(self) -> Optional[ETHPeer]:
"""
Might be None. If None is unacceptable, use :meth:`get_queen_peer`
"""
return self._queen_peer
async def pop_fastest_peasant(self) -> ETHPeer:
"""
Get the fastest peer that is not the queen.
"""
while self.manager.is_running:
peer = await self._waiting_peers.get_fastest()
if not peer.manager.is_running:
# drop any peers that aren't alive anymore
self.logger.info("Dropping %s from beam peers, as no longer active", peer)
if peer == self._queen_peer:
self._queen_peer = None
continue
old_queen = self._queen_peer
self._update_queen(peer)
if peer == self._queen_peer:
self.logger.debug("Switching queen peer from %s to %s", old_queen, peer)
continue
try:
peer_is_requesting_nodes = peer.eth_api.get_node_data.is_requesting
except PeerConnectionLost:
self.logger.debug("QueenQueuer is skipping disconnecting peer %s", peer)
# Don't bother re-adding to _waiting_peers, since the peer is disconnected
else:
if peer_is_requesting_nodes:
# skip the peer if there's an active request
self.logger.debug("QueenQueuer is skipping active peer %s", peer)
loop = asyncio.get_event_loop()
loop.call_later(10, functools.partial(self._waiting_peers.put_nowait, peer))
continue
return peer
# This should never happen as we run as a daemon and if we return before our caller it'd
# raise a DaemonTaskExit, but just in case we raise a CancelledError() to ensure our
# caller realizes we've stopped.
self.logger.error("Service ended before a queen peer could be elected")
raise asyncio.CancelledError()
def readd_peasant(self, peer: ETHPeer, delay: float = 0) -> None:
if delay > 0:
loop = asyncio.get_event_loop()
loop.call_later(delay, functools.partial(self._waiting_peers.put_nowait, peer))
else:
self._waiting_peers.put_nowait(peer)
def penalize_queen(self, peer: ETHPeer) -> None:
if peer == self._queen_peer:
self._queen_peer = None
delay = NON_IDEAL_RESPONSE_PENALTY
self.logger.debug(
"Penalizing %s for %.2fs, for minor infraction",
peer,
delay,
)
loop = asyncio.get_event_loop()
loop.call_later(delay, functools.partial(self._waiting_peers.put_nowait, peer))
def _update_queen(self, peer: ETHPeer) -> None:
'''
@return peer that is no longer queen
'''
if self._queen_peer is None:
self._queen_peer = peer
return
elif peer == self._queen_peer:
# nothing to do, peer is already the queen
return
else:
try:
new_peer_quality = _peer_sort_key(peer)
except (UnknownAPI, PeerConnectionLost) as exc:
self.logger.debug("Ignoring %s, because we can't get speed stats: %r", peer, exc)
return
try:
old_queen_quality = _peer_sort_key(self._queen_peer)
force_drop_queen = False
except (UnknownAPI, PeerConnectionLost) as exc:
self.logger.debug(
"Dropping queen %s, because we can't get speed stats: %r",
self._queen_peer,
exc,
)
force_drop_queen = True
if force_drop_queen or new_peer_quality < old_queen_quality:
old_queen, self._queen_peer = self._queen_peer, peer
self._waiting_peers.put_nowait(old_queen)
return
else:
# nothing to do, peer is slower than the queen
return
raise ValidationError(
"Unreachable: every queen peer check should have finished and returned. "
f"Was checking {peer} against queen {self._queen_peer}."
)