async def connect_to_nodes(self, nodes: Iterator[NodeAPI]) -> None:
# create an generator for the nodes
nodes_iter = iter(nodes)
while True:
if self.is_full or not self.is_operational:
return
# only attempt to connect to up to the maximum number of available
# peer slots that are open.
available_peer_slots = self.max_peers - len(self)
batch_size = clamp(1, 10, available_peer_slots)
batch = tuple(take(batch_size, nodes_iter))
# There are no more *known* nodes to connect to.
if not batch:
return
self.logger.debug(
'Initiating %d peer connection attempts with %d open peer slots',
len(batch),
available_peer_slots,
)
# Try to connect to the peers concurrently.
await asyncio.gather(
*(self.connect_to_node(node) for node in batch),
loop=self.get_event_loop(),
)
async def ensure_nodes_present(self,
node_hashes: Collection[Hash32],
urgent: bool = True) -> int:
"""
Wait until the nodes that are the preimages of `node_hashes` are available in the database.
If one is not available in the first check, request it from peers.
:param urgent: Should this node be downloaded urgently? If False, download as backfill
Note that if your ultimate goal is an account or storage data, it's probably better to use
download_account or download_storage. This method is useful for other
scenarios, like bytecode lookups or intermediate node lookups.
:return: how many nodes had to be downloaded
"""
if urgent:
t = Timer()
num_nodes_found = await self._wait_for_nodes(
node_hashes,
self._node_tasks,
BLOCK_IMPORT_MISSING_STATE_TIMEOUT,
)
# If it took to long to get a single urgent node, then increase "spread" factor
if len(node_hashes
) == 1 and t.elapsed > MAX_ACCEPTABLE_WAIT_FOR_URGENT_NODE:
new_spread_factor = clamp(
0,
self._max_spread_beam_factor(),
self._spread_factor + 1,
)
if new_spread_factor != self._spread_factor:
self.logger.debug(
"spread-beam-update: Urgent node latency=%.3fs, update factor %d to %d",
t.elapsed,
self._spread_factor,
new_spread_factor,
)
self._queen_tracker.set_desired_knight_count(
new_spread_factor)
self._spread_factor = new_spread_factor
else:
num_nodes_found = await self._wait_for_nodes(
node_hashes,
self._maybe_useful_nodes,
BLOCK_IMPORT_MISSING_STATE_TIMEOUT,
)
return num_nodes_found
async def _find_urgent_nodes(self, queen: ETHPeer,
urgent_hashes: Tuple[Hash32, ...],
batch_id: int) -> None:
# Generate and schedule the tasks to request the urgent node(s) from multiple peers
knights = tuple(self._queen_tracker.pop_knights())
urgent_requests = [
create_task(
self._get_nodes(peer, urgent_hashes, urgent=True),
name=f"BeamDownloader._get_nodes({peer.remote}, ...)",
) for peer in (queen, ) + knights
]
# Process the returned nodes, in the order they complete
urgent_timer = Timer()
async with cleanup_tasks(*urgent_requests):
for result_coro in asyncio.as_completed(urgent_requests):
nodes_returned, new_nodes, peer = await result_coro
time_on_urgent = urgent_timer.elapsed
# After the first peer returns something, cancel all other pending tasks
if len(nodes_returned) > 0:
# Stop waiting for other peer responses
break
elif peer == queen:
self.logger.debug("queen %s returned 0 urgent nodes of %r",
peer, urgent_hashes)
# Wait for the next peer response
# Log the received urgent nodes
if peer == queen:
log_header = "beam-queen-urgent-rtt"
else:
log_header = "spread-beam-urgent-rtt"
self.logger.debug(
"%s: got %d/%d +%d nodes in %.3fs from %s (%s)",
log_header,
len(nodes_returned),
len(urgent_hashes),
len(new_nodes),
time_on_urgent,
peer.remote,
urgent_hashes[0][:2].hex(),
)
# Stat updates
self._total_processed_nodes += len(new_nodes)
self._urgent_processed_nodes += len(new_nodes)
self._time_on_urgent += time_on_urgent
# If it took to long to get a single urgent node, then increase "spread" factor
if len(urgent_hashes
) == 1 and time_on_urgent > MAX_ACCEPTABLE_WAIT_FOR_URGENT_NODE:
new_spread_factor = clamp(
0,
self._max_spread_beam_factor(),
self._spread_factor + 1,
)
if new_spread_factor != self._spread_factor:
self.logger.debug(
"spread-beam-update: Urgent node latency=%.3fs, update factor %d to %d",
time_on_urgent,
self._spread_factor,
new_spread_factor,
)
self._queen_tracker.set_desired_knight_count(new_spread_factor)
self._spread_factor = new_spread_factor
# Complete the task in the TaskQueue
task_hashes = tuple(node_hash for node_hash, _ in nodes_returned)
await self._node_tasks.complete(batch_id, task_hashes)
# Re-insert the peers for the next request
for knight in knights:
self._queen_tracker.insert_peer(knight)
def return_value_decimal(lower: decimal.Decimal, upper: decimal.Decimal,
value: decimal.Decimal) -> decimal.Decimal:
return clamp(lower, upper, value)
def return_value_float(lower: float, upper: float, value: float) -> float:
return clamp(lower, upper, value)
def return_value_int(lower: int, upper: int, value: int) -> int:
return clamp(lower, upper, value)
