async def test_request_header(self, two_nodes):
full_node_1, full_node_2, server_1, server_2 = two_nodes
num_blocks = 2
blocks = bt.get_consecutive_blocks(
test_constants, num_blocks, [], 10, seed=b"test_request_header"
)
for block in blocks[:2]:
async for _ in full_node_1.respond_block(fnp.RespondBlock(block)):
pass
msgs = [
_
async for _ in full_node_1.request_header(
wallet_protocol.RequestHeader(uint32(1), blocks[1].header_hash)
)
]
assert len(msgs) == 1
assert isinstance(msgs[0].message.data, wallet_protocol.RespondHeader)
assert msgs[0].message.data.header_block.header == blocks[1].header
assert msgs[0].message.data.transactions_filter == blocks[1].transactions_filter
# Don't have
msgs = [
_
async for _ in full_node_1.request_header(
wallet_protocol.RequestHeader(uint32(2), blocks[2].header_hash)
)
]
assert len(msgs) == 1
assert isinstance(msgs[0].message.data, wallet_protocol.RejectHeaderRequest)
assert msgs[0].message.data.height == 2
assert msgs[0].message.data.header_hash == blocks[2].header_hash
async def new_lca(self, request: wallet_protocol.NewLCA):
"""
Notification from full node that a new LCA (Least common ancestor of the three blockchain
tips) has been added to the full node.
"""
if self.wallet_state_manager is None:
return
if self._shut_down:
return
if self.wallet_state_manager.sync_mode:
return
# If already seen LCA, ignore.
if request.lca_hash in self.wallet_state_manager.block_records:
return
lca = self.wallet_state_manager.block_records[
self.wallet_state_manager.lca]
# If it's not the heaviest chain, ignore.
if request.weight < lca.weight:
return
if int(request.height) - int(lca.height) > self.short_sync_threshold:
try:
# Performs sync, and catch exceptions so we don't close the connection
self.wallet_state_manager.set_sync_mode(True)
self.sync_generator_task = self._sync()
assert self.sync_generator_task is not None
async for ret_msg in self.sync_generator_task:
yield ret_msg
except Exception as e:
tb = traceback.format_exc()
self.log.error(f"Error with syncing. {type(e)} {tb}")
self.wallet_state_manager.set_sync_mode(False)
else:
header_request = wallet_protocol.RequestHeader(
uint32(request.height), request.lca_hash)
yield OutboundMessage(
NodeType.FULL_NODE,
Message("request_header", header_request),
Delivery.RESPOND,
)
# Try sending queued up transaction when new LCA arrives
await self._resend_queue()
async def respond_header(self, response: wallet_protocol.RespondHeader):
"""
The full node responds to our RequestHeader call. We cannot finish this block
until we have the required additions / removals for our wallets.
"""
while True:
if self._shut_down:
return
# We loop, to avoid infinite recursion. At the end of each iteration, we might want to
# process the next block, if it exists.
block = response.header_block
# If we already have, return
if block.header_hash in self.wallet_state_manager.block_records:
return
if block.height < 1:
return
block_record = BlockRecord(
block.header_hash,
block.prev_header_hash,
block.height,
block.weight,
None,
None,
response.header_block.header.data.total_iters,
response.header_block.challenge.get_hash(),
)
if self.wallet_state_manager.sync_mode:
self.potential_blocks_received[uint32(block.height)].set()
self.potential_header_hashes[block.height] = block.header_hash
# Caches the block so we can finalize it when additions and removals arrive
self.cached_blocks[block_record.header_hash] = (
block_record,
block,
response.transactions_filter,
)
if block.prev_header_hash not in self.wallet_state_manager.block_records:
# We do not have the previous block record, so wait for that. When the previous gets added to chain,
# this method will get called again and we can continue. During sync, the previous blocks are already
# requested. During normal operation, this might not be the case.
self.future_block_hashes[
block.prev_header_hash] = block.header_hash
lca = self.wallet_state_manager.block_records[
self.wallet_state_manager.lca]
if (block_record.height - lca.height <
self.short_sync_threshold
and not self.wallet_state_manager.sync_mode):
# Only requests the previous block if we are not in sync mode, close to the new block,
# and don't have prev
header_request = wallet_protocol.RequestHeader(
uint32(block_record.height - 1),
block_record.prev_header_hash,
)
yield OutboundMessage(
NodeType.FULL_NODE,
Message("request_header", header_request),
Delivery.RESPOND,
)
return
# If the block has transactions that we are interested in, fetch adds/deletes
if response.transactions_filter is not None:
(
additions,
removals,
) = await self.wallet_state_manager.get_filter_additions_removals(
block_record, response.transactions_filter)
if len(additions) > 0 or len(removals) > 0:
request_a = wallet_protocol.RequestAdditions(
block.height, block.header_hash, additions)
yield OutboundMessage(
NodeType.FULL_NODE,
Message("request_additions", request_a),
Delivery.RESPOND,
)
return
# If we don't have any transactions in filter, don't fetch, and finish the block
block_record = BlockRecord(
block_record.header_hash,
block_record.prev_header_hash,
block_record.height,
block_record.weight,
[],
[],
block_record.total_iters,
block_record.new_challenge_hash,
)
respond_header_msg: Optional[
wallet_protocol.RespondHeader] = await self._block_finished(
block_record, block, response.transactions_filter)
if respond_header_msg is None:
return
else:
response = respond_header_msg
async def _sync(self):
"""
Wallet has fallen far behind (or is starting up for the first time), and must be synced
up to the LCA of the blockchain.
"""
# 1. Get all header hashes
self.header_hashes = []
self.header_hashes_error = False
self.proof_hashes = []
self.potential_header_hashes = {}
genesis = FullBlock.from_bytes(self.constants["GENESIS_BLOCK"])
genesis_challenge = genesis.proof_of_space.challenge_hash
request_header_hashes = wallet_protocol.RequestAllHeaderHashesAfter(
uint32(0), genesis_challenge)
yield OutboundMessage(
NodeType.FULL_NODE,
Message("request_all_header_hashes_after", request_header_hashes),
Delivery.RESPOND,
)
timeout = 100
sleep_interval = 10
sleep_interval_short = 1
start_wait = time.time()
while time.time() - start_wait < timeout:
if self._shut_down:
return
if self.header_hashes_error:
raise ValueError(
f"Received error from full node while fetching hashes from {request_header_hashes}."
)
if len(self.header_hashes) > 0:
break
await asyncio.sleep(0.5)
if len(self.header_hashes) == 0:
raise TimeoutError("Took too long to fetch header hashes.")
# 2. Find fork point
fork_point_height: uint32 = self.wallet_state_manager.find_fork_point_alternate_chain(
self.header_hashes)
fork_point_hash: bytes32 = self.header_hashes[fork_point_height]
# Sync a little behind, in case there is a short reorg
tip_height = (len(self.header_hashes) - 5 if
len(self.header_hashes) > 5 else len(self.header_hashes))
self.log.info(
f"Fork point: {fork_point_hash} at height {fork_point_height}. Will sync up to {tip_height}"
)
for height in range(0, tip_height + 1):
self.potential_blocks_received[uint32(height)] = asyncio.Event()
header_validate_start_height: uint32
if self.config["starting_height"] == 0:
header_validate_start_height = fork_point_height
else:
# Request all proof hashes
request_proof_hashes = wallet_protocol.RequestAllProofHashes()
yield OutboundMessage(
NodeType.FULL_NODE,
Message("request_all_proof_hashes", request_proof_hashes),
Delivery.RESPOND,
)
start_wait = time.time()
while time.time() - start_wait < timeout:
if self._shut_down:
return
if len(self.proof_hashes) > 0:
break
await asyncio.sleep(0.5)
if len(self.proof_hashes) == 0:
raise TimeoutError("Took too long to fetch proof hashes.")
if len(self.proof_hashes) < tip_height:
raise ValueError("Not enough proof hashes fetched.")
# Creates map from height to difficulty
heights: List[uint32] = []
difficulty_weights: List[uint64] = []
difficulty: uint64
for i in range(tip_height):
if self.proof_hashes[i][1] is not None:
difficulty = self.proof_hashes[i][1]
if i > (fork_point_height +
1) and i % 2 == 1: # Only add odd heights
heights.append(uint32(i))
difficulty_weights.append(difficulty)
# Randomly sample based on difficulty
query_heights_odd = sorted(
list(
set(
random.choices(heights,
difficulty_weights,
k=min(100, len(heights))))))
query_heights: List[uint32] = []
for odd_height in query_heights_odd:
query_heights += [uint32(odd_height - 1), odd_height]
# Send requests for these heights
# Verify these proofs
last_request_time = float(0)
highest_height_requested = uint32(0)
request_made = False
for height_index in range(len(query_heights)):
total_time_slept = 0
while True:
if self._shut_down:
return
if total_time_slept > timeout:
raise TimeoutError("Took too long to fetch blocks")
# Request batches that we don't have yet
for batch_start_index in range(
height_index,
min(
height_index + self.config["num_sync_batches"],
len(query_heights),
),
):
blocks_missing = not self.potential_blocks_received[
uint32(query_heights[batch_start_index])].is_set()
if ((time.time() - last_request_time > sleep_interval
and blocks_missing)
or (query_heights[batch_start_index]) >
highest_height_requested):
self.log.info(
f"Requesting sync header {query_heights[batch_start_index]}"
)
if (query_heights[batch_start_index] >
highest_height_requested):
highest_height_requested = uint32(
query_heights[batch_start_index])
request_made = True
request_header = wallet_protocol.RequestHeader(
uint32(query_heights[batch_start_index]),
self.header_hashes[
query_heights[batch_start_index]],
)
yield OutboundMessage(
NodeType.FULL_NODE,
Message("request_header", request_header),
Delivery.RANDOM,
)
if request_made:
last_request_time = time.time()
request_made = False
try:
aw = self.potential_blocks_received[uint32(
query_heights[height_index])].wait()
await asyncio.wait_for(aw, timeout=sleep_interval)
break
except concurrent.futures.TimeoutError:
total_time_slept += sleep_interval
self.log.info("Did not receive desired headers")
self.log.info(
f"Finished downloading sample of headers at heights: {query_heights}, validating."
)
# Validates the downloaded proofs
assert self.wallet_state_manager.validate_select_proofs(
self.proof_hashes,
query_heights_odd,
self.cached_blocks,
self.potential_header_hashes,
)
self.log.info("All proofs validated successfuly.")
# Add blockrecords one at a time, to catch up to starting height
weight = self.wallet_state_manager.block_records[
fork_point_hash].weight
header_validate_start_height = min(
max(fork_point_height, self.config["starting_height"] - 1),
tip_height + 1,
)
if fork_point_height == 0:
difficulty = self.constants["DIFFICULTY_STARTING"]
else:
fork_point_parent_hash = self.wallet_state_manager.block_records[
fork_point_hash].prev_header_hash
fork_point_parent_weight = self.wallet_state_manager.block_records[
fork_point_parent_hash]
difficulty = uint64(weight - fork_point_parent_weight)
for height in range(fork_point_height + 1,
header_validate_start_height):
_, difficulty_change, total_iters = self.proof_hashes[height]
weight += difficulty
block_record = BlockRecord(
self.header_hashes[height],
self.header_hashes[height - 1],
uint32(height),
weight,
[],
[],
total_iters,
None,
)
res = await self.wallet_state_manager.receive_block(
block_record, None)
assert (res == ReceiveBlockResult.ADDED_TO_HEAD
or res == ReceiveBlockResult.ADDED_AS_ORPHAN)
self.log.info(
f"Fast sync successful up to height {header_validate_start_height - 1}"
)
# Download headers in batches, and verify them as they come in. We download a few batches ahead,
# in case there are delays. TODO(mariano): optimize sync by pipelining
last_request_time = float(0)
highest_height_requested = uint32(0)
request_made = False
for height_checkpoint in range(header_validate_start_height + 1,
tip_height + 1):
total_time_slept = 0
while True:
if self._shut_down:
return
if total_time_slept > timeout:
raise TimeoutError("Took too long to fetch blocks")
# Request batches that we don't have yet
for batch_start in range(
height_checkpoint,
min(
height_checkpoint +
self.config["num_sync_batches"],
tip_height + 1,
),
):
batch_end = min(batch_start + 1, tip_height + 1)
blocks_missing = any([
not (self.potential_blocks_received[uint32(h)]
).is_set() for h in range(batch_start, batch_end)
])
if (time.time() - last_request_time > sleep_interval
and blocks_missing
) or (batch_end - 1) > highest_height_requested:
self.log.info(f"Requesting sync header {batch_start}")
if batch_end - 1 > highest_height_requested:
highest_height_requested = uint32(batch_end - 1)
request_made = True
request_header = wallet_protocol.RequestHeader(
uint32(batch_start),
self.header_hashes[batch_start],
)
yield OutboundMessage(
NodeType.FULL_NODE,
Message("request_header", request_header),
Delivery.RANDOM,
)
if request_made:
last_request_time = time.time()
request_made = False
awaitables = [
self.potential_blocks_received[uint32(
height_checkpoint)].wait()
]
future = asyncio.gather(*awaitables, return_exceptions=True)
try:
await asyncio.wait_for(future, timeout=sleep_interval)
except concurrent.futures.TimeoutError:
try:
await future
except asyncio.CancelledError:
pass
total_time_slept += sleep_interval
self.log.info("Did not receive desired headers")
continue
# Succesfully downloaded header. Now confirm it's added to chain.
hh = self.potential_header_hashes[height_checkpoint]
if hh in self.wallet_state_manager.block_records:
# Successfully added the block to chain
break
else:
# Not added to chain yet. Try again soon.
await asyncio.sleep(sleep_interval_short)
total_time_slept += sleep_interval_short
if hh in self.wallet_state_manager.block_records:
break
else:
self.log.warning(
"Received header, but it has not been added to chain. Retrying."
)
_, hb, tfilter = self.cached_blocks[hh]
respond_header_msg = wallet_protocol.RespondHeader(
hb, tfilter)
async for msg in self.respond_header(
respond_header_msg):
yield msg
self.log.info(
f"Finished sync process up to height {max(self.wallet_state_manager.height_to_hash.keys())}"
)