async def _raise_for_empty_code(
self,
block_hash: Hash32,
address: Address,
code_hash: Hash32,
peer: LESPeer) -> None:
"""
A peer might return b'' if it doesn't have the block at the requested header,
or it might maliciously return b'' when the code is non-empty. This method tries to tell the
difference.
This method MUST raise an exception, it's trying to determine the appropriate one.
:raise BadLESResponse: if peer seems to be maliciously responding with invalid empty code
:raise NoEligiblePeers: if peer might simply not have the code available
"""
try:
header = await self._get_block_header_by_hash(block_hash, peer)
except HeaderNotFound:
# We presume that the current peer is the best peer. Because
# our best peer doesn't have the header we want, there are no eligible peers.
raise NoEligiblePeers("Our best peer does not have the header %s" % block_hash)
head_number = peer.head_info.block_number
if head_number - header.block_number > MAX_REORG_DEPTH:
# The peer claims to be far ahead of the header we requested
if self.headerdb.get_canonical_block_hash(header.block_number) == block_hash:
# Our node believes that the header at the reference hash is canonical,
# so treat the peer as malicious
raise BadLESResponse(
"Peer %s sent empty code that did not match hash %s in account %s" % (
peer,
encode_hex(code_hash),
encode_hex(address),
)
)
else:
# our header isn't canonical, so treat the empty response as missing data
raise NoEligiblePeers(
"Our best peer does not have the non-canonical header %s" % block_hash
)
elif head_number - header.block_number < 0:
# The peer claims to be behind the header we requested, but somehow served it to us.
# Odd, it might be a race condition. Treat as if there are no eligible peers for now.
raise NoEligiblePeers("Our best peer's head does include header %s" % block_hash)
else:
# The peer is ahead of the current block header, but only by a bit. It might be on
# an uncle, or we might be. So we can't tell the difference between missing and
# malicious. We don't want to aggressively drop this peer, so treat the code as missing.
raise NoEligiblePeers(
"Peer %s claims to be ahead of %s, but returned empty code with hash %s. "
"It is on number %d, maybe an uncle. Retry with an older block hash." % (
peer,
header,
code_hash,
head_number,
)
)
async def _retry_on_bad_response(
self, make_request_to_peer: Callable[[LESPeer], Any]) -> Any:
"""
Make a call to a peer. If it behaves badly, drop it and retry with a different peer.
:param make_request_to_peer: an abstract call to a peer that may raise a BadLESResponse
:raise NoEligiblePeers: if no peers are available to fulfill the request
:raise TimeoutError: if an individual request or the overall process times out
"""
for _ in range(MAX_REQUEST_ATTEMPTS):
try:
peer = cast(LESPeer, self.peer_pool.highest_td_peer)
except NoConnectedPeers as exc:
raise NoEligiblePeers() from exc
try:
return await make_request_to_peer(peer)
except BadLESResponse as exc:
self.logger.warn("Disconnecting from peer, because: %s", exc)
await peer.disconnect(DisconnectReason.subprotocol_error)
# reattempt after removing this peer from our pool
raise TimeoutError("Could not complete peer request in %d attempts" %
MAX_REQUEST_ATTEMPTS)
async def coro_get_contract_code(self, block_hash: Hash32,
address: ETHAddress) -> bytes:
"""
:param block_hash: find code as of the block with block_hash
:param address: which contract to look up
:return: bytecode of the contract, ``b''`` if no code is set
:raise NoEligiblePeers: if no peers are available to fulfill the request
:raise asyncio.TimeoutError: if an individual request or the overall process times out
"""
# get account for later verification, and
# to confirm that our highest total difficulty peer has the info
try:
account = await self.coro_get_account(block_hash, address)
except HeaderNotFound as exc:
raise NoEligiblePeers(
f"Our best peer does not have header {block_hash.hex()}"
) from exc
code_hash = account.code_hash
return await self._retry_on_bad_response(
partial(self._get_contract_code_from_peer, block_hash, address,
code_hash))
def _request_block_parts(
self, target_td: int, headers: List[BlockHeader],
request_func: Callable[[ETHPeer, List[BlockHeader]], None]) -> int:
peers = self.peer_pool.get_peers(target_td)
if not peers:
raise NoEligiblePeers()
length = math.ceil(len(headers) / len(peers))
batches = list(partition_all(length, headers))
for peer, batch in zip(peers, batches):
request_func(cast(ETHPeer, peer), batch)
return len(batches)
async def _download_receipts(self,
target_td: int,
all_headers: Tuple[BlockHeader, ...]) -> None:
"""
Downloads and persists the receipts for the given set of block headers.
Receipts are requested from all peers in equal sized batches.
"""
# Post-Byzantium blocks may have identical receipt roots (e.g. when they have the same
# number of transactions and all succeed/failed: ropsten blocks 2503212 and 2503284),
# so we do this to avoid requesting the same receipts multiple times.
headers = tuple(unique(
(header for header in all_headers if not _is_receipts_empty(header)),
key=operator.attrgetter('receipt_root'),
))
while headers:
# split the remaining headers into equal sized batches for each peer.
peers = cast(Tuple[ETHPeer, ...], self.peer_pool.get_peers(target_td))
if not peers:
raise NoEligiblePeers(
"No connected peers have the receipts we need for td={0}".format(target_td)
)
batch_size = math.ceil(len(headers) / len(peers))
batches = tuple(partition_all(batch_size, headers))
# issue requests to all of the peers and wait for all of them to respond.
requests = tuple(
self._get_receipts(peer, batch)
for peer, batch
in zip(peers, batches)
)
responses = await self.wait(asyncio.gather(
*requests,
loop=self.get_event_loop(),
))
# extract the returned receipt data and the headers for which we
# are still missing receipts.
all_receipt_bundles, all_missing_headers = zip(*responses)
receipt_bundles = tuple(concat(all_receipt_bundles))
headers = tuple(concat(all_missing_headers))
if len(receipt_bundles) == 0:
continue
# process all of the returned receipts, storing their trie data
# dicts in the database
receipts, trie_roots_and_data_dicts = zip(*receipt_bundles)
trie_roots, trie_data_dicts = zip(*trie_roots_and_data_dicts)
for trie_data in trie_data_dicts:
await self.wait(self.db.coro_persist_trie_data_dict(trie_data))
self.logger.debug("Got receipts batch for %d headers", len(all_headers))
async def _download_block_bodies(
self, target_td: int, all_headers: Tuple[BlockHeader, ...]
) -> Dict[Tuple[Hash32, Hash32], BlockBody]:
"""
Downloads and persists the block bodies for the given set of block headers.
Block bodies are requested from all peers in equal sized batches.
"""
headers = tuple(header for header in all_headers
if not _is_body_empty(header))
block_bodies_by_key: Dict[Tuple[Hash32, Hash32], BlockBody] = {}
while headers:
# split the remaining headers into equal sized batches for each peer.
peers = cast(Tuple[ETHPeer, ...],
self.peer_pool.get_peers(target_td))
if not peers:
raise NoEligiblePeers(
"No connected peers have the block bodies we need for td={0}"
.format(target_td))
batch_size = math.ceil(len(headers) / len(peers))
batches = tuple(partition_all(batch_size, headers))
# issue requests to all of the peers and wait for all of them to respond.
requests = tuple(
self._get_block_bodies(peer, batch)
for peer, batch in zip(peers, batches))
responses = await self.wait(
asyncio.gather(
*requests,
loop=self.get_event_loop(),
))
# extract the returned block body data and the headers for which we
# are still missing block bodies.
all_block_body_bundles, all_missing_headers = zip(*responses)
for (body, (tx_root, trie_data_dict),
uncles_hash) in concat(all_block_body_bundles):
await self.wait(
self.db.coro_persist_trie_data_dict(trie_data_dict))
block_bodies_by_key = merge(
block_bodies_by_key,
{(transaction_root, uncles_hash): block_body
for block_body, (transaction_root, trie_dict_data),
uncles_hash in concat(all_block_body_bundles)})
headers = tuple(concat(all_missing_headers))
self.logger.debug("Got block bodies batch for %d headers",
len(all_headers))
return block_bodies_by_key
def _request_block_parts(
self,
target_td: int,
headers: List[BlockHeader],
request_func: Callable[[ETHPeer, List[BlockHeader]], None],
block_part: BlockPartEnum,
) -> int:
"""
:return: how many requests were made (one request per peer)
"""
peers = self.peer_pool.get_peers(target_td)
if not peers:
raise NoEligiblePeers()
# request headers proportionally, so faster peers are asked for more parts than slow peers
speeds = {peer: self._get_peer_stats(peer, block_part).get_throughput() for peer in peers}
peer_batches = get_scaled_batches(speeds, headers)
for peer, batch in peer_batches.items():
self._get_peer_stats(peer, block_part).begin_work()
request_func(cast(ETHPeer, peer), batch)
return len(peer_batches)
async def get_peer_for_request(self, node_keys: Set[Hash32]) -> ETHPeer:
"""Return an idle peer that may have any of the trie nodes in node_keys.
If none of our peers have any of the given node keys, raise NoEligiblePeers. If none of
the peers which may have at least one of the given node keys is idle, raise NoIdlePeers.
"""
has_eligible_peers = False
async for peer in self.peer_pool:
peer = cast(ETHPeer, peer)
if self._peer_missing_nodes[peer].issuperset(node_keys):
self.logger.trace("%s doesn't have any of the nodes we want, skipping it", peer)
continue
has_eligible_peers = True
if peer in self.request_tracker.active_requests:
self.logger.trace("%s is not idle, skipping it", peer)
continue
return peer
if not has_eligible_peers:
raise NoEligiblePeers()
else:
raise NoIdlePeers()