async def test_get_nowait(tasks, get_size, expected_tasks):
q = TaskQueue()
await q.add(tasks)
batch, tasks = q.get_nowait(get_size)
assert tasks == expected_tasks
q.complete(batch, tasks)
assert all(task not in q for task in tasks)
class BaseHeaderChainSyncer(BaseService, PeerSubscriber):
"""
Sync with the Ethereum network by fetching/storing block headers.
Here, the run() method will execute the sync loop until our local head is the same as the one
with the highest TD announced by any of our peers.
"""
# We'll only sync if we are connected to at least min_peers_to_sync.
min_peers_to_sync = 1
# Post-processing steps can exit out of sync (for example, fast sync) by triggering this token:
complete_token = None
# TODO: Instead of a fixed timeout, we should use a variable one that gets adjusted based on
# the round-trip times from our download requests.
_reply_timeout = 60
_seal_check_random_sample_rate = SEAL_CHECK_RANDOM_SAMPLE_RATE
# the latest header hash of the peer on the current sync
_target_header_hash = None
header_queue: TaskQueue[BlockHeader]
def __init__(self,
chain: AsyncChain,
db: AsyncHeaderDB,
peer_pool: PeerPool,
token: CancelToken = None) -> None:
self.complete_token = CancelToken(
'trinity.sync.common.BaseHeaderChainSyncer.SyncCompleted')
if token is None:
master_service_token = self.complete_token
else:
master_service_token = token.chain(self.complete_token)
super().__init__(master_service_token)
self.chain = chain
self.db = db
self.peer_pool = peer_pool
self._handler = PeerRequestHandler(self.db, self.logger,
self.cancel_token)
self._syncing = False
self._sync_complete = asyncio.Event()
self._sync_requests: asyncio.Queue[
HeaderRequestingPeer] = asyncio.Queue()
# pending queue size should be big enough to avoid starving the processing consumers, but
# small enough to avoid wasteful over-requests before post-processing can happen
max_pending_headers = ETHPeer.max_headers_fetch * 8
self.header_queue = TaskQueue(max_pending_headers,
attrgetter('block_number'))
@property
def msg_queue_maxsize(self) -> int:
# This is a rather arbitrary value, but when the sync is operating normally we never see
# the msg queue grow past a few hundred items, so this should be a reasonable limit for
# now.
return 2000
def get_target_header_hash(self) -> Hash32:
if self._target_header_hash is None:
raise ValidationError(
"Cannot check the target hash when there is no active sync")
else:
return self._target_header_hash
def register_peer(self, peer: BasePeer) -> None:
self._sync_requests.put_nowait(
cast(HeaderRequestingPeer, self.peer_pool.highest_td_peer))
async def _handle_msg_loop(self) -> None:
while self.is_operational:
peer, cmd, msg = await self.wait(self.msg_queue.get())
# Our handle_msg() method runs cpu-intensive tasks in sub-processes so that the main
# loop can keep processing msgs, and that's why we use self.run_task() instead of
# awaiting for it to finish here.
self.run_task(
self.handle_msg(cast(HeaderRequestingPeer, peer), cmd, msg))
async def handle_msg(self, peer: HeaderRequestingPeer,
cmd: protocol.Command,
msg: protocol._DecodedMsgType) -> None:
try:
await self._handle_msg(peer, cmd, msg)
except OperationCancelled:
# Silently swallow OperationCancelled exceptions because otherwise they'll be caught
# by the except below and treated as unexpected.
pass
except Exception:
self.logger.exception(
"Unexpected error when processing msg from %s", peer)
async def _run(self) -> None:
self.run_task(self._handle_msg_loop())
with self.subscribe(self.peer_pool):
while self.is_operational:
try:
peer = await self.wait(self._sync_requests.get())
except OperationCancelled:
# In the case of a fast sync, we return once the sync is completed, and our
# caller must then run the StateDownloader.
return
else:
self.run_task(self.sync(peer))
async def sync(self, peer: HeaderRequestingPeer) -> None:
if self._syncing:
self.logger.debug(
"Got a NewBlock or a new peer, but already syncing so doing nothing"
)
return
elif len(self.peer_pool) < self.min_peers_to_sync:
self.logger.info(
"Connected to less peers (%d) than the minimum (%d) required to sync, "
"doing nothing", len(self.peer_pool), self.min_peers_to_sync)
return
self._syncing = True
try:
await self._sync(peer)
except OperationCancelled as e:
self.logger.info("Sync with %s aborted: %s", peer, e)
finally:
self._syncing = False
async def _sync(self, peer: HeaderRequestingPeer) -> None:
"""Try to fetch/process blocks until the given peer's head_hash.
Returns when the peer's head_hash is available in our ChainDB, or if any error occurs
during the sync.
If in fast-sync mode, the _sync_completed event will be set upon successful completion of
a sync.
"""
head = await self.wait(self.db.coro_get_canonical_head())
head_td = await self.wait(self.db.coro_get_score(head.hash))
if peer.head_td <= head_td:
self.logger.info(
"Head TD (%d) announced by %s not higher than ours (%d), not syncing",
peer.head_td, peer, head_td)
return
self.logger.info("Starting sync with %s", peer)
last_received_header: BlockHeader = None
# When we start the sync with a peer, we always request up to MAX_REORG_DEPTH extra
# headers before our current head's number, in case there were chain reorgs since the last
# time _sync() was called. All of the extra headers that are already present in our DB
# will be discarded by _fetch_missing_headers() so we don't unnecessarily process them
# again.
start_at = max(GENESIS_BLOCK_NUMBER + 1,
head.block_number - MAX_REORG_DEPTH)
while self.is_operational:
if not peer.is_operational:
self.logger.info("%s disconnected, aborting sync", peer)
break
try:
fetch_headers_coro = self._fetch_missing_headers(
peer, start_at)
headers = await self.wait(fetch_headers_coro)
except OperationCancelled:
self.logger.info("Sync with %s completed", peer)
break
except TimeoutError:
self.logger.warn(
"Timeout waiting for header batch from %s, aborting sync",
peer)
await peer.disconnect(DisconnectReason.timeout)
break
except ValidationError as err:
self.logger.warn(
"Invalid header response sent by peer %s disconnecting: %s",
peer,
err,
)
await peer.disconnect(DisconnectReason.useless_peer)
break
if not headers:
self.logger.info("Got no new headers from %s, aborting sync",
peer)
break
first = headers[0]
first_parent = None
if last_received_header is None:
# on the first request, make sure that the earliest ancestor has a parent in our db
try:
first_parent = await self.wait(
self.db.coro_get_block_header_by_hash(
first.parent_hash))
except HeaderNotFound:
self.logger.warn(
"Unable to find common ancestor betwen our chain and %s",
peer)
break
elif last_received_header.hash != first.parent_hash:
# on follow-ups, require the first header in this batch to be next in succession
self.logger.warn(
"Header batch starts with %r, with parent %s, but last header was %r",
first,
encode_hex(first.parent_hash[:4]),
last_received_header,
)
break
self.logger.debug("Got new header chain starting at #%d",
first.block_number)
try:
await self.chain.coro_validate_chain(
last_received_header or first_parent,
headers,
self._seal_check_random_sample_rate,
)
except ValidationError as e:
self.logger.warn(
"Received invalid headers from %s, disconnecting: %s",
peer, e)
await peer.disconnect(DisconnectReason.subprotocol_error)
break
# Setting the latest header hash for the peer, before queuing header processing tasks
self._target_header_hash = peer.head_hash
unrequested_headers = tuple(h for h in headers
if h not in self.header_queue)
await self.header_queue.add(unrequested_headers)
last_received_header = headers[-1]
start_at = last_received_header.block_number + 1
# erase any pending tasks, to restart on next _sync() run
try:
batch_id, pending_tasks = self.header_queue.get_nowait()
except asyncio.QueueFull:
# nothing pending, continue
pass
else:
# fully remove pending tasks from queue
self.header_queue.complete(batch_id, pending_tasks)
async def _fetch_missing_headers(self, peer: HeaderRequestingPeer,
start_at: int) -> Tuple[BlockHeader, ...]:
"""Fetch a batch of headers starting at start_at and return the ones we're missing."""
self.logger.debug("Fetching chain segment starting at #%d", start_at)
headers = await peer.requests.get_block_headers(
start_at,
peer.max_headers_fetch,
skip=0,
reverse=False,
)
# We only want headers that are missing, so we iterate over the list
# until we find the first missing header, after which we return all of
# the remaining headers.
async def get_missing_tail(
self: 'BaseHeaderChainSyncer',
headers: Tuple[BlockHeader,
...]) -> AsyncGenerator[BlockHeader, None]:
iter_headers = iter(headers)
for header in iter_headers:
is_missing = not await self.wait(
self.db.coro_header_exists(header.hash))
if is_missing:
yield header
break
else:
self.logger.debug(
"Discarding header that we already have: %s", header)
for header in iter_headers:
yield header
# The inner list comprehension is needed because async_generators
# cannot be cast to a tuple.
tail_headers = tuple(
[header async for header in get_missing_tail(self, headers)])
return tail_headers
@abstractmethod
async def _handle_msg(self, peer: HeaderRequestingPeer,
cmd: protocol.Command,
msg: protocol._DecodedMsgType) -> None:
raise NotImplementedError("Must be implemented by subclasses")
def test_get_nowait_queuefull(get_size):
q = TaskQueue()
with pytest.raises(asyncio.QueueFull):
q.get_nowait(get_size)
