def validate_additions(
self,
coins: List[Tuple[bytes32, List[Coin]]],
proofs: Optional[List[Tuple[bytes32, bytes, Optional[bytes]]]],
root,
):
if proofs is None:
# Verify root
additions_merkle_set = MerkleSet()
# Addition Merkle set contains puzzlehash and hash of all coins with that puzzlehash
for puzzle_hash, coins_l in coins:
additions_merkle_set.add_already_hashed(puzzle_hash)
additions_merkle_set.add_already_hashed(
hash_coin_list(coins_l))
additions_root = additions_merkle_set.get_root()
if root != additions_root:
return False
else:
for i in range(len(coins)):
assert coins[i][0] == proofs[i][0]
coin_list_1: List[Coin] = coins[i][1]
puzzle_hash_proof: bytes32 = proofs[i][1]
coin_list_proof: Optional[bytes32] = proofs[i][2]
if len(coin_list_1) == 0:
# Verify exclusion proof for puzzle hash
not_included = confirm_not_included_already_hashed(
root,
coins[i][0],
puzzle_hash_proof,
)
if not_included is False:
return False
else:
try:
# Verify inclusion proof for coin list
included = confirm_included_already_hashed(
root,
hash_coin_list(coin_list_1),
coin_list_proof,
)
if included is False:
return False
except AssertionError:
return False
try:
# Verify inclusion proof for puzzle hash
included = confirm_included_already_hashed(
root,
coins[i][0],
puzzle_hash_proof,
)
if included is False:
return False
except AssertionError:
return False
return True
async def test_basics(self):
wallet_tool = WalletTool()
num_blocks = 10
blocks = bt.get_consecutive_blocks(
test_constants,
num_blocks,
[],
10,
reward_puzzlehash=wallet_tool.get_new_puzzlehash(),
)
merkle_set = MerkleSet()
merkle_set_reverse = MerkleSet()
for block in reversed(blocks):
merkle_set_reverse.add_already_hashed(
block.header.data.coinbase.name())
for block in blocks:
merkle_set.add_already_hashed(block.header.data.coinbase.name())
for block in blocks:
result, proof = merkle_set.is_included_already_hashed(
block.header.data.coinbase.name())
assert result is True
result_fee, proof_fee = merkle_set.is_included_already_hashed(
block.header.data.fees_coin.name())
assert result_fee is False
validate_proof = confirm_included_already_hashed(
merkle_set.get_root(), block.header.data.coinbase.name(),
proof)
validate_proof_fee = confirm_included_already_hashed(
merkle_set.get_root(), block.header.data.fees_coin.name(),
proof_fee)
assert validate_proof is True
assert validate_proof_fee is False
# Test if order of adding items change the outcome
assert merkle_set.get_root() == merkle_set_reverse.get_root()
def validate_removals(self, coins, proofs, root):
if proofs is None:
# If there are no proofs, it means all removals were returned in the response.
# we must find the ones relevant to our wallets.
# Verify removals root
removals_merkle_set = MerkleSet()
for name_coin in coins:
# TODO review all verification
name, coin = name_coin
if coin is not None:
removals_merkle_set.add_already_hashed(coin.name())
removals_root = removals_merkle_set.get_root()
if root != removals_root:
return False
else:
# This means the full node has responded only with the relevant removals
# for our wallet. Each merkle proof must be verified.
if len(coins) != len(proofs):
return False
for i in range(len(coins)):
# Coins are in the same order as proofs
if coins[i][0] != proofs[i][0]:
return False
coin = coins[i][1]
if coin is None:
# Verifies merkle proof of exclusion
not_included = confirm_not_included_already_hashed(
root,
coins[i][0],
proofs[i][1],
)
if not_included is False:
return False
else:
# Verifies merkle proof of inclusion of coin name
if coins[i][0] != coin.name():
return False
included = confirm_included_already_hashed(
root,
coin.name(),
proofs[i][1],
)
if included is False:
return False
return True
async def respond_additions(self,
response: wallet_protocol.RespondAdditions):
"""
The full node has responded with the additions for a block. We will use this
to try to finish the block, and add it to the state.
"""
if self._shut_down:
return
if response.header_hash not in self.cached_blocks:
self.log.warning("Do not have header for additions")
return
block_record, header_block, transaction_filter = self.cached_blocks[
response.header_hash]
assert response.height == block_record.height
additions: List[Coin]
if response.proofs is None:
# If there are no proofs, it means all additions were returned in the response.
# we must find the ones relevant to our wallets.
all_coins: List[Coin] = []
for puzzle_hash, coin_list_0 in response.coins:
all_coins += coin_list_0
additions = await self.wallet_state_manager.get_relevant_additions(
all_coins)
# Verify root
additions_merkle_set = MerkleSet()
# Addition Merkle set contains puzzlehash and hash of all coins with that puzzlehash
for puzzle_hash, coins in response.coins:
additions_merkle_set.add_already_hashed(puzzle_hash)
additions_merkle_set.add_already_hashed(hash_coin_list(coins))
additions_root = additions_merkle_set.get_root()
if header_block.header.data.additions_root != additions_root:
return
else:
# This means the full node has responded only with the relevant additions
# for our wallet. Each merkle proof must be verified.
additions = []
assert len(response.coins) == len(response.proofs)
for i in range(len(response.coins)):
assert response.coins[i][0] == response.proofs[i][0]
coin_list_1: List[Coin] = response.coins[i][1]
puzzle_hash_proof: bytes32 = response.proofs[i][1]
coin_list_proof: Optional[bytes32] = response.proofs[i][2]
if len(coin_list_1) == 0:
# Verify exclusion proof for puzzle hash
assert confirm_not_included_already_hashed(
header_block.header.data.additions_root,
response.coins[i][0],
puzzle_hash_proof,
)
else:
# Verify inclusion proof for puzzle hash
assert confirm_included_already_hashed(
header_block.header.data.additions_root,
response.coins[i][0],
puzzle_hash_proof,
)
# Verify inclusion proof for coin list
assert confirm_included_already_hashed(
header_block.header.data.additions_root,
hash_coin_list(coin_list_1),
coin_list_proof,
)
for coin in coin_list_1:
assert coin.puzzle_hash == response.coins[i][0]
additions += coin_list_1
new_br = BlockRecord(
block_record.header_hash,
block_record.prev_header_hash,
block_record.height,
block_record.weight,
additions,
None,
block_record.total_iters,
header_block.challenge.get_hash(),
)
self.cached_blocks[response.header_hash] = (
new_br,
header_block,
transaction_filter,
)
if transaction_filter is None:
raise RuntimeError("Got additions for block with no transactions.")
(
_,
removals,
) = await self.wallet_state_manager.get_filter_additions_removals(
new_br, transaction_filter)
request_all_removals = False
for coin in additions:
puzzle_store = self.wallet_state_manager.puzzle_store
record_info: Optional[
DerivationRecord] = await puzzle_store.get_derivation_record_for_puzzle_hash(
coin.puzzle_hash.hex())
if (record_info is not None
and record_info.wallet_type == WalletType.COLOURED_COIN):
request_all_removals = True
break
if len(removals) > 0 or request_all_removals:
if request_all_removals:
request_r = wallet_protocol.RequestRemovals(
header_block.height, header_block.header_hash, None)
else:
request_r = wallet_protocol.RequestRemovals(
header_block.height, header_block.header_hash, removals)
yield OutboundMessage(
NodeType.FULL_NODE,
Message("request_removals", request_r),
Delivery.RESPOND,
)
else:
# We have collected all three things: header, additions, and removals (since there are no
# relevant removals for us). Can proceed. Otherwise, we wait for the removals to arrive.
new_br = BlockRecord(
new_br.header_hash,
new_br.prev_header_hash,
new_br.height,
new_br.weight,
new_br.additions,
[],
new_br.total_iters,
new_br.new_challenge_hash,
)
respond_header_msg: Optional[
wallet_protocol.RespondHeader] = await self._block_finished(
new_br, header_block, transaction_filter)
if respond_header_msg is not None:
async for msg in self.respond_header(respond_header_msg):
yield msg
async def respond_removals(self,
response: wallet_protocol.RespondRemovals):
"""
The full node has responded with the removals for a block. We will use this
to try to finish the block, and add it to the state.
"""
if self._shut_down:
return
if (response.header_hash not in self.cached_blocks or
self.cached_blocks[response.header_hash][0].additions is None):
self.log.warning(
"Do not have header for removals, or do not have additions")
return
block_record, header_block, transaction_filter = self.cached_blocks[
response.header_hash]
assert response.height == block_record.height
all_coins: List[Coin] = []
for coin_name, coin in response.coins:
if coin is not None:
all_coins.append(coin)
if response.proofs is None:
# If there are no proofs, it means all removals were returned in the response.
# we must find the ones relevant to our wallets.
# Verify removals root
removals_merkle_set = MerkleSet()
for coin in all_coins:
if coin is not None:
removals_merkle_set.add_already_hashed(coin.name())
removals_root = removals_merkle_set.get_root()
assert header_block.header.data.removals_root == removals_root
else:
# This means the full node has responded only with the relevant removals
# for our wallet. Each merkle proof must be verified.
assert len(response.coins) == len(response.proofs)
for i in range(len(response.coins)):
# Coins are in the same order as proofs
assert response.coins[i][0] == response.proofs[i][0]
coin = response.coins[i][1]
if coin is None:
# Verifies merkle proof of exclusion
assert confirm_not_included_already_hashed(
header_block.header.data.removals_root,
response.coins[i][0],
response.proofs[i][1],
)
else:
# Verifies merkle proof of inclusion of coin name
assert response.coins[i][0] == coin.name()
assert confirm_included_already_hashed(
header_block.header.data.removals_root,
coin.name(),
response.proofs[i][1],
)
new_br = BlockRecord(
block_record.header_hash,
block_record.prev_header_hash,
block_record.height,
block_record.weight,
block_record.additions,
all_coins,
block_record.total_iters,
header_block.challenge.get_hash(),
)
self.cached_blocks[response.header_hash] = (
new_br,
header_block,
transaction_filter,
)
# We have collected all three things: header, additions, and removals. Can proceed.
respond_header_msg: Optional[
wallet_protocol.RespondHeader] = await self._block_finished(
new_br, header_block, transaction_filter)
if respond_header_msg is not None:
async for msg in self.respond_header(respond_header_msg):
yield msg
async def test_request_removals(self, two_nodes, wallet_blocks):
full_node_1, full_node_2, server_1, server_2 = two_nodes
wallet_a, wallet_receiver, blocks = wallet_blocks
await server_2.start_client(PeerInfo("localhost", uint16(server_1._port)), None)
blocks_list = await get_block_path(full_node_1)
blocks_new = bt.get_consecutive_blocks(
test_constants, 5, seed=b"test_request_removals"
)
# Request removals for nonexisting block fails
msgs = [
_
async for _ in full_node_1.request_removals(
wallet_protocol.RequestRemovals(
blocks_new[-1].height, blocks_new[-1].header_hash, None
)
)
]
assert len(msgs) == 1
assert isinstance(msgs[0].message.data, wallet_protocol.RejectRemovalsRequest)
# Request removals for orphaned block fails
for block in blocks_new:
async for _ in full_node_1.respond_block(fnp.RespondBlock(block)):
pass
msgs = [
_
async for _ in full_node_1.request_removals(
wallet_protocol.RequestRemovals(
blocks_new[-1].height, blocks_new[-1].header_hash, None
)
)
]
assert len(msgs) == 1
assert isinstance(msgs[0].message.data, wallet_protocol.RejectRemovalsRequest)
# If there are no transactions, empty proof and coins
blocks_new = bt.get_consecutive_blocks(
test_constants, 10, block_list=blocks_list,
)
for block in blocks_new:
[_ async for _ in full_node_1.respond_block(fnp.RespondBlock(block))]
msgs = [
_
async for _ in full_node_1.request_removals(
wallet_protocol.RequestRemovals(
blocks_new[-4].height, blocks_new[-4].header_hash, None
)
)
]
assert len(msgs) == 1
assert isinstance(msgs[0].message.data, wallet_protocol.RespondRemovals)
assert len(msgs[0].message.data.coins) == 0
assert msgs[0].message.data.proofs is None
# Add a block with transactions
spend_bundles = []
for i in range(5):
spend_bundles.append(
wallet_a.generate_signed_transaction(
100,
wallet_a.get_new_puzzlehash(),
blocks_new[i - 8].get_coinbase(),
)
)
height_with_transactions = len(blocks_new) + 1
agg = SpendBundle.aggregate(spend_bundles)
dic_h = {
height_with_transactions: (
best_solution_program(agg),
agg.aggregated_signature,
)
}
blocks_new = bt.get_consecutive_blocks(
test_constants, 5, block_list=blocks_new, transaction_data_at_height=dic_h
)
for block in blocks_new:
[_ async for _ in full_node_1.respond_block(fnp.RespondBlock(block))]
# If no coins requested, respond all coins and NO proof
msgs = [
_
async for _ in full_node_1.request_removals(
wallet_protocol.RequestRemovals(
blocks_new[height_with_transactions].height,
blocks_new[height_with_transactions].header_hash,
None,
)
)
]
assert len(msgs) == 1
assert isinstance(msgs[0].message.data, wallet_protocol.RespondRemovals)
assert len(msgs[0].message.data.coins) == 5
assert msgs[0].message.data.proofs is None
removals_merkle_set = MerkleSet()
for sb in spend_bundles:
for coin in sb.removals():
if coin is not None:
removals_merkle_set.add_already_hashed(coin.name())
# Ask for one coin and check PoI
coin_list = [spend_bundles[0].removals()[0].name()]
msgs = [
_
async for _ in full_node_1.request_removals(
wallet_protocol.RequestRemovals(
blocks_new[height_with_transactions].height,
blocks_new[height_with_transactions].header_hash,
coin_list,
)
)
]
assert len(msgs) == 1
assert isinstance(msgs[0].message.data, wallet_protocol.RespondRemovals)
assert len(msgs[0].message.data.coins) == 1
assert msgs[0].message.data.proofs is not None
assert len(msgs[0].message.data.proofs) == 1
assert confirm_included_already_hashed(
blocks_new[height_with_transactions].header.data.removals_root,
coin_list[0],
msgs[0].message.data.proofs[0][1],
)
# Ask for one coin and check PoE
coin_list = [token_bytes(32)]
msgs = [
_
async for _ in full_node_1.request_removals(
wallet_protocol.RequestRemovals(
blocks_new[height_with_transactions].height,
blocks_new[height_with_transactions].header_hash,
coin_list,
)
)
]
assert len(msgs) == 1
assert isinstance(msgs[0].message.data, wallet_protocol.RespondRemovals)
assert len(msgs[0].message.data.coins) == 1
assert msgs[0].message.data.coins[0][1] is None
assert msgs[0].message.data.proofs is not None
assert len(msgs[0].message.data.proofs) == 1
assert confirm_not_included_already_hashed(
blocks_new[height_with_transactions].header.data.removals_root,
coin_list[0],
msgs[0].message.data.proofs[0][1],
)
# Ask for two coins
coin_list = [spend_bundles[0].removals()[0].name(), token_bytes(32)]
msgs = [
_
async for _ in full_node_1.request_removals(
wallet_protocol.RequestRemovals(
blocks_new[height_with_transactions].height,
blocks_new[height_with_transactions].header_hash,
coin_list,
)
)
]
assert len(msgs) == 1
assert isinstance(msgs[0].message.data, wallet_protocol.RespondRemovals)
assert len(msgs[0].message.data.coins) == 2
assert msgs[0].message.data.coins[0][1] is not None
assert msgs[0].message.data.coins[1][1] is None
assert msgs[0].message.data.proofs is not None
assert len(msgs[0].message.data.proofs) == 2
assert confirm_included_already_hashed(
blocks_new[height_with_transactions].header.data.removals_root,
coin_list[0],
msgs[0].message.data.proofs[0][1],
)
assert confirm_not_included_already_hashed(
blocks_new[height_with_transactions].header.data.removals_root,
coin_list[1],
msgs[0].message.data.proofs[1][1],
)
async def test_request_additions(self, two_nodes, wallet_blocks):
full_node_1, full_node_2, server_1, server_2 = two_nodes
wallet_a, wallet_receiver, blocks = wallet_blocks
await server_2.start_client(PeerInfo("localhost", uint16(server_1._port)), None)
blocks_list = await get_block_path(full_node_1)
blocks_new = bt.get_consecutive_blocks(
test_constants, 5, seed=b"test_request_additions"
)
# Request additinos for nonexisting block fails
msgs = [
_
async for _ in full_node_1.request_additions(
wallet_protocol.RequestAdditions(
blocks_new[-1].height, blocks_new[-1].header_hash, None
)
)
]
assert len(msgs) == 1
assert isinstance(msgs[0].message.data, wallet_protocol.RejectAdditionsRequest)
# Request additions for orphaned block fails
for block in blocks_new:
async for _ in full_node_1.respond_block(fnp.RespondBlock(block)):
pass
msgs = [
_
async for _ in full_node_1.request_additions(
wallet_protocol.RequestAdditions(
blocks_new[-1].height, blocks_new[-1].header_hash, None
)
)
]
assert len(msgs) == 1
assert isinstance(msgs[0].message.data, wallet_protocol.RejectAdditionsRequest)
# If there are no transactions, only cb and fees additions
blocks_new = bt.get_consecutive_blocks(
test_constants, 10, block_list=blocks_list,
)
for block in blocks_new:
[_ async for _ in full_node_1.respond_block(fnp.RespondBlock(block))]
msgs = [
_
async for _ in full_node_1.request_additions(
wallet_protocol.RequestAdditions(
blocks_new[-4].height, blocks_new[-4].header_hash, None
)
)
]
assert len(msgs) == 1
assert isinstance(msgs[0].message.data, wallet_protocol.RespondAdditions)
assert len(msgs[0].message.data.coins) == 2
assert msgs[0].message.data.proofs is None
# Add a block with transactions
spend_bundles = []
puzzle_hashes = [wallet_a.get_new_puzzlehash(), wallet_a.get_new_puzzlehash()]
for i in range(5):
spend_bundles.append(
wallet_a.generate_signed_transaction(
100, puzzle_hashes[i % 2], blocks_new[i - 8].get_coinbase(),
)
)
height_with_transactions = len(blocks_new) + 1
agg = SpendBundle.aggregate(spend_bundles)
dic_h = {
height_with_transactions: (
best_solution_program(agg),
agg.aggregated_signature,
)
}
blocks_new = bt.get_consecutive_blocks(
test_constants, 5, block_list=blocks_new, transaction_data_at_height=dic_h
)
for block in blocks_new:
[_ async for _ in full_node_1.respond_block(fnp.RespondBlock(block))]
# If no puzzle hashes requested, respond all coins and NO proof
msgs = [
_
async for _ in full_node_1.request_additions(
wallet_protocol.RequestAdditions(
blocks_new[height_with_transactions].height,
blocks_new[height_with_transactions].header_hash,
None,
)
)
]
assert len(msgs) == 1
assert isinstance(msgs[0].message.data, wallet_protocol.RespondAdditions)
# One puzzle hash with change and fee (x3) = 9, minus two repeated ph = 7 + coinbase and fees = 9
assert len(msgs[0].message.data.coins) == 9
assert msgs[0].message.data.proofs is None
additions_merkle_set = MerkleSet()
for sb in spend_bundles:
for coin in sb.additions():
if coin is not None:
additions_merkle_set.add_already_hashed(coin.name())
# Ask for one coin and check both PoI
ph_list = [puzzle_hashes[0]]
msgs = [
_
async for _ in full_node_1.request_additions(
wallet_protocol.RequestAdditions(
blocks_new[height_with_transactions].height,
blocks_new[height_with_transactions].header_hash,
ph_list,
)
)
]
assert len(msgs) == 1
assert isinstance(msgs[0].message.data, wallet_protocol.RespondAdditions)
assert len(msgs[0].message.data.coins) == 1
assert len(msgs[0].message.data.coins[0][1]) == 3
assert msgs[0].message.data.proofs is not None
assert len(msgs[0].message.data.proofs) == 1
assert confirm_included_already_hashed(
blocks_new[height_with_transactions].header.data.additions_root,
ph_list[0],
msgs[0].message.data.proofs[0][1],
)
coin_list_for_ph = [
coin
for coin in blocks_new[height_with_transactions].additions()
if coin.puzzle_hash == ph_list[0]
]
assert confirm_included_already_hashed(
blocks_new[height_with_transactions].header.data.additions_root,
hash_coin_list(coin_list_for_ph),
msgs[0].message.data.proofs[0][2],
)
# Ask for one ph and check PoE
ph_list = [token_bytes(32)]
msgs = [
_
async for _ in full_node_1.request_additions(
wallet_protocol.RequestAdditions(
blocks_new[height_with_transactions].height,
blocks_new[height_with_transactions].header_hash,
ph_list,
)
)
]
assert len(msgs) == 1
assert isinstance(msgs[0].message.data, wallet_protocol.RespondAdditions)
assert len(msgs[0].message.data.coins) == 1
assert len(msgs[0].message.data.coins[0][1]) == 0
assert msgs[0].message.data.proofs is not None
assert len(msgs[0].message.data.proofs) == 1
assert confirm_not_included_already_hashed(
blocks_new[height_with_transactions].header.data.additions_root,
ph_list[0],
msgs[0].message.data.proofs[0][1],
)
assert msgs[0].message.data.proofs[0][2] is None
# Ask for two puzzle_hashes
ph_list = [puzzle_hashes[0], token_bytes(32)]
msgs = [
_
async for _ in full_node_1.request_additions(
wallet_protocol.RequestAdditions(
blocks_new[height_with_transactions].height,
blocks_new[height_with_transactions].header_hash,
ph_list,
)
)
]
assert len(msgs) == 1
assert isinstance(msgs[0].message.data, wallet_protocol.RespondAdditions)
assert len(msgs[0].message.data.coins) == 2
assert len(msgs[0].message.data.coins[0][1]) == 3
assert msgs[0].message.data.proofs is not None
assert len(msgs[0].message.data.proofs) == 2
assert confirm_included_already_hashed(
blocks_new[height_with_transactions].header.data.additions_root,
ph_list[0],
msgs[0].message.data.proofs[0][1],
)
assert confirm_included_already_hashed(
blocks_new[height_with_transactions].header.data.additions_root,
hash_coin_list(coin_list_for_ph),
msgs[0].message.data.proofs[0][2],
)
assert confirm_not_included_already_hashed(
blocks_new[height_with_transactions].header.data.additions_root,
ph_list[1],
msgs[0].message.data.proofs[1][1],
)
assert msgs[0].message.data.proofs[1][2] is None
