def validate_additions(
self,
coins: List[Tuple[bytes32, List[Coin]]],
proofs: Optional[List[Tuple[bytes32, bytes, Optional[bytes]]]],
root,
):
if proofs is None:
# Verify root
additions_merkle_set = MerkleSet()
# Addition Merkle set contains puzzlehash and hash of all coins with that puzzlehash
for puzzle_hash, coins_l in coins:
additions_merkle_set.add_already_hashed(puzzle_hash)
additions_merkle_set.add_already_hashed(
hash_coin_list(coins_l))
additions_root = additions_merkle_set.get_root()
if root != additions_root:
return False
else:
for i in range(len(coins)):
assert coins[i][0] == proofs[i][0]
coin_list_1: List[Coin] = coins[i][1]
puzzle_hash_proof: bytes32 = proofs[i][1]
coin_list_proof: Optional[bytes32] = proofs[i][2]
if len(coin_list_1) == 0:
# Verify exclusion proof for puzzle hash
not_included = confirm_not_included_already_hashed(
root,
coins[i][0],
puzzle_hash_proof,
)
if not_included is False:
return False
else:
try:
# Verify inclusion proof for coin list
included = confirm_included_already_hashed(
root,
hash_coin_list(coin_list_1),
coin_list_proof,
)
if included is False:
return False
except AssertionError:
return False
try:
# Verify inclusion proof for puzzle hash
included = confirm_included_already_hashed(
root,
coins[i][0],
puzzle_hash_proof,
)
if included is False:
return False
except AssertionError:
return False
return True
def validate_removals(self, coins, proofs, root):
if proofs is None:
# If there are no proofs, it means all removals were returned in the response.
# we must find the ones relevant to our wallets.
# Verify removals root
removals_merkle_set = MerkleSet()
for name_coin in coins:
# TODO review all verification
name, coin = name_coin
if coin is not None:
removals_merkle_set.add_already_hashed(coin.name())
removals_root = removals_merkle_set.get_root()
if root != removals_root:
return False
else:
# This means the full node has responded only with the relevant removals
# for our wallet. Each merkle proof must be verified.
if len(coins) != len(proofs):
return False
for i in range(len(coins)):
# Coins are in the same order as proofs
if coins[i][0] != proofs[i][0]:
return False
coin = coins[i][1]
if coin is None:
# Verifies merkle proof of exclusion
not_included = confirm_not_included_already_hashed(
root,
coins[i][0],
proofs[i][1],
)
if not_included is False:
return False
else:
# Verifies merkle proof of inclusion of coin name
if coins[i][0] != coin.name():
return False
included = confirm_included_already_hashed(
root,
coin.name(),
proofs[i][1],
)
if included is False:
return False
return True
async def test_basics(self):
wallet_tool = WalletTool()
num_blocks = 10
blocks = bt.get_consecutive_blocks(
test_constants,
num_blocks,
[],
10,
reward_puzzlehash=wallet_tool.get_new_puzzlehash(),
)
merkle_set = MerkleSet()
merkle_set_reverse = MerkleSet()
for block in reversed(blocks):
merkle_set_reverse.add_already_hashed(
block.header.data.coinbase.name())
for block in blocks:
merkle_set.add_already_hashed(block.header.data.coinbase.name())
for block in blocks:
result, proof = merkle_set.is_included_already_hashed(
block.header.data.coinbase.name())
assert result is True
result_fee, proof_fee = merkle_set.is_included_already_hashed(
block.header.data.fees_coin.name())
assert result_fee is False
validate_proof = confirm_included_already_hashed(
merkle_set.get_root(), block.header.data.coinbase.name(),
proof)
validate_proof_fee = confirm_included_already_hashed(
merkle_set.get_root(), block.header.data.fees_coin.name(),
proof_fee)
assert validate_proof is True
assert validate_proof_fee is False
# Test if order of adding items change the outcome
assert merkle_set.get_root() == merkle_set_reverse.get_root()
def _validate_merkle_root(
self,
block: FullBlock,
tx_additions: List[Coin] = None,
tx_removals: List[bytes32] = None,
) -> Optional[Err]:
additions = []
removals = []
if tx_additions:
additions.extend(tx_additions)
if tx_removals:
removals.extend(tx_removals)
removal_merkle_set = MerkleSet()
addition_merkle_set = MerkleSet()
# Create removal Merkle set
for coin_name in removals:
removal_merkle_set.add_already_hashed(coin_name)
# Create addition Merkle set
puzzlehash_coins_map: Dict[bytes32, List[Coin]] = {}
for coin in additions + [block.get_coinbase(), block.get_fees_coin()]:
if coin.puzzle_hash in puzzlehash_coins_map:
puzzlehash_coins_map[coin.puzzle_hash].append(coin)
else:
puzzlehash_coins_map[coin.puzzle_hash] = [coin]
# Addition Merkle set contains puzzlehash and hash of all coins with that puzzlehash
for puzzle, coins in puzzlehash_coins_map.items():
addition_merkle_set.add_already_hashed(puzzle)
addition_merkle_set.add_already_hashed(hash_coin_list(coins))
additions_root = addition_merkle_set.get_root()
removals_root = removal_merkle_set.get_root()
if block.header.data.additions_root != additions_root:
return Err.BAD_ADDITION_ROOT
if block.header.data.removals_root != removals_root:
return Err.BAD_REMOVAL_ROOT
return None
def validate_block_merkle_roots(
block_additions_root: bytes32,
block_removals_root: bytes32,
tx_additions: List[Coin] = None,
tx_removals: List[bytes32] = None,
) -> Optional[Err]:
if tx_removals is None:
tx_removals = []
if tx_additions is None:
tx_additions = []
removal_merkle_set = MerkleSet()
addition_merkle_set = MerkleSet()
# Create removal Merkle set
for coin_name in tx_removals:
removal_merkle_set.add_already_hashed(coin_name)
# Create addition Merkle set
puzzlehash_coins_map: Dict[bytes32, List[Coin]] = {}
for coin in tx_additions:
if coin.puzzle_hash in puzzlehash_coins_map:
puzzlehash_coins_map[coin.puzzle_hash].append(coin)
else:
puzzlehash_coins_map[coin.puzzle_hash] = [coin]
# Addition Merkle set contains puzzlehash and hash of all coins with that puzzlehash
for puzzle, coins in puzzlehash_coins_map.items():
addition_merkle_set.add_already_hashed(puzzle)
addition_merkle_set.add_already_hashed(hash_coin_list(coins))
additions_root = addition_merkle_set.get_root()
removals_root = removal_merkle_set.get_root()
if block_additions_root != additions_root:
return Err.BAD_ADDITION_ROOT
if block_removals_root != removals_root:
return Err.BAD_REMOVAL_ROOT
return None
def _create_block(
self,
test_constants: ConsensusConstants,
challenge_hash: bytes32,
height: uint32,
prev_header_hash: bytes32,
prev_iters: uint64,
prev_weight: uint128,
timestamp: uint64,
difficulty: int,
min_iters: int,
seed: bytes,
genesis: bool = False,
reward_puzzlehash: bytes32 = None,
transactions: Program = None,
aggsig: G2Element = None,
fees: uint64 = uint64(0),
) -> FullBlock:
"""
Creates a block with the specified details. Uses the stored plots to create a proof of space,
and also evaluates the VDF for the proof of time.
"""
selected_plot_info = None
selected_proof_index = 0
selected_quality: Optional[bytes] = None
best_quality = 0
plots = [
pinfo for _, pinfo in sorted(list(self.plots.items()),
key=lambda x: str(x[0]))
]
if self.use_any_pos:
random.seed(seed)
for i in range(len(plots) * 3):
# Allow passing in seed, to create reorgs and different chains
seeded_pn = random.randint(0, len(plots) - 1)
plot_info = plots[seeded_pn]
plot_id = plot_info.prover.get_id()
ccp = ProofOfSpace.can_create_proof(
plot_id,
challenge_hash,
test_constants.NUMBER_ZERO_BITS_CHALLENGE_SIG,
)
if not ccp:
continue
qualities = plot_info.prover.get_qualities_for_challenge(
challenge_hash)
if len(qualities) > 0:
selected_plot_info = plot_info
selected_quality = qualities[0]
break
else:
for i in range(len(plots)):
plot_info = plots[i]
j = 0
plot_id = plot_info.prover.get_id()
ccp = ProofOfSpace.can_create_proof(
plot_id,
challenge_hash,
test_constants.NUMBER_ZERO_BITS_CHALLENGE_SIG,
)
if not ccp:
continue
qualities = plot_info.prover.get_qualities_for_challenge(
challenge_hash)
for quality in qualities:
qual_int = int.from_bytes(quality, "big", signed=False)
if qual_int > best_quality:
best_quality = qual_int
selected_quality = quality
selected_plot_info = plot_info
selected_proof_index = j
j += 1
assert selected_plot_info is not None
if selected_quality is None:
raise RuntimeError("No proofs for this challenge")
proof_xs: bytes = selected_plot_info.prover.get_full_proof(
challenge_hash, selected_proof_index)
plot_pk = ProofOfSpace.generate_plot_public_key(
selected_plot_info.local_sk.get_g1(),
selected_plot_info.farmer_public_key,
)
proof_of_space: ProofOfSpace = ProofOfSpace(
challenge_hash,
selected_plot_info.pool_public_key,
plot_pk,
selected_plot_info.prover.get_size(),
proof_xs,
)
number_iters: uint64 = pot_iterations.calculate_iterations(
proof_of_space,
difficulty,
min_iters,
test_constants.NUMBER_ZERO_BITS_CHALLENGE_SIG,
)
if self.real_plots:
print(f"Performing {number_iters} VDF iterations")
int_size = (test_constants.DISCRIMINANT_SIZE_BITS + 16) >> 4
result = prove(challenge_hash, test_constants.DISCRIMINANT_SIZE_BITS,
number_iters)
output = ClassgroupElement(
int512(int.from_bytes(
result[0:int_size],
"big",
signed=True,
)),
int512(
int.from_bytes(
result[int_size:2 * int_size],
"big",
signed=True,
)),
)
proof_bytes = result[2 * int_size:4 * int_size]
proof_of_time = ProofOfTime(
challenge_hash,
number_iters,
output,
uint8(0),
proof_bytes,
)
# Use the extension data to create different blocks based on header hash
extension_data: bytes32 = bytes32(
[random.randint(0, 255) for _ in range(32)])
cost = uint64(0)
fee_reward = uint64(block_rewards.calculate_base_fee(height) + fees)
std_hash(std_hash(height))
# Create filter
byte_array_tx: List[bytes32] = []
tx_additions: List[Coin] = []
tx_removals: List[bytes32] = []
if transactions:
error, npc_list, _ = get_name_puzzle_conditions(transactions)
additions: List[Coin] = additions_for_npc(npc_list)
for coin in additions:
tx_additions.append(coin)
byte_array_tx.append(bytearray(coin.puzzle_hash))
for npc in npc_list:
tx_removals.append(npc.coin_name)
byte_array_tx.append(bytearray(npc.coin_name))
farmer_ph = self.farmer_ph
pool_ph = self.pool_ph
if reward_puzzlehash is not None:
farmer_ph = reward_puzzlehash
pool_ph = reward_puzzlehash
byte_array_tx.append(bytearray(farmer_ph))
byte_array_tx.append(bytearray(pool_ph))
bip158: PyBIP158 = PyBIP158(byte_array_tx)
encoded = bytes(bip158.GetEncoded())
removal_merkle_set = MerkleSet()
addition_merkle_set = MerkleSet()
# Create removal Merkle set
for coin_name in tx_removals:
removal_merkle_set.add_already_hashed(coin_name)
# Create addition Merkle set
puzzlehash_coin_map: Dict[bytes32, List[Coin]] = {}
cb_reward = calculate_block_reward(height)
cb_coin = create_coinbase_coin(height, pool_ph, cb_reward)
fees_coin = create_fees_coin(height, farmer_ph, fee_reward)
for coin in tx_additions + [cb_coin, fees_coin]:
if coin.puzzle_hash in puzzlehash_coin_map:
puzzlehash_coin_map[coin.puzzle_hash].append(coin)
else:
puzzlehash_coin_map[coin.puzzle_hash] = [coin]
# Addition Merkle set contains puzzlehash and hash of all coins with that puzzlehash
for puzzle, coins in puzzlehash_coin_map.items():
addition_merkle_set.add_already_hashed(puzzle)
addition_merkle_set.add_already_hashed(hash_coin_list(coins))
additions_root = addition_merkle_set.get_root()
removal_root = removal_merkle_set.get_root()
generator_hash = (transactions.get_tree_hash()
if transactions is not None else bytes32([0] * 32))
filter_hash = std_hash(encoded)
pool_target = PoolTarget(pool_ph, uint32(height))
pool_target_signature = self.get_pool_key_signature(
pool_target, proof_of_space.pool_public_key)
assert pool_target_signature is not None
final_aggsig: G2Element = pool_target_signature
if aggsig is not None:
final_aggsig = AugSchemeMPL.aggregate([final_aggsig, aggsig])
header_data: HeaderData = HeaderData(
height,
prev_header_hash,
timestamp,
filter_hash,
proof_of_space.get_hash(),
uint128(prev_weight + difficulty),
uint64(prev_iters + number_iters),
additions_root,
removal_root,
farmer_ph,
fee_reward,
pool_target,
final_aggsig,
cost,
extension_data,
generator_hash,
)
header_hash_sig: G2Element = self.get_plot_signature(
header_data, plot_pk)
header: Header = Header(header_data, header_hash_sig)
full_block: FullBlock = FullBlock(proof_of_space, proof_of_time,
header, transactions, encoded)
return full_block
def create_foliage(
constants: ConsensusConstants,
reward_block_unfinished: RewardChainBlockUnfinished,
spend_bundle: Optional[SpendBundle],
additions: List[Coin],
removals: List[Coin],
prev_block: Optional[BlockRecord],
blocks: BlockchainInterface,
total_iters_sp: uint128,
timestamp: uint64,
farmer_reward_puzzlehash: bytes32,
pool_target: PoolTarget,
get_plot_signature: Callable[[bytes32, G1Element], G2Element],
get_pool_signature: Callable[[PoolTarget, Optional[G1Element]],
Optional[G2Element]],
seed: bytes32 = b"",
) -> Tuple[Foliage, Optional[FoliageTransactionBlock],
Optional[TransactionsInfo], Optional[SerializedProgram]]:
"""
Creates a foliage for a given reward chain block. This may or may not be a tx block. In the case of a tx block,
the return values are not None. This is called at the signage point, so some of this information may be
tweaked at the infusion point.
Args:
constants: consensus constants being used for this chain
reward_block_unfinished: the reward block to look at, potentially at the signage point
spend_bundle: the spend bundle including all transactions
prev_block: the previous block at the signage point
blocks: dict from header hash to blocks, of all ancestor blocks
total_iters_sp: total iters at the signage point
timestamp: timestamp to put into the foliage block
farmer_reward_puzzlehash: where to pay out farming reward
pool_target: where to pay out pool reward
get_plot_signature: retrieve the signature corresponding to the plot public key
get_pool_signature: retrieve the signature corresponding to the pool public key
seed: seed to randomize block
"""
if prev_block is not None:
res = get_prev_transaction_block(prev_block, blocks, total_iters_sp)
is_transaction_block: bool = res[0]
prev_transaction_block: Optional[BlockRecord] = res[1]
else:
# Genesis is a transaction block
prev_transaction_block = None
is_transaction_block = True
random.seed(seed)
# Use the extension data to create different blocks based on header hash
extension_data: bytes32 = random.randint(0, 100000000).to_bytes(32, "big")
if prev_block is None:
height: uint32 = uint32(0)
else:
height = uint32(prev_block.height + 1)
# Create filter
byte_array_tx: List[bytes32] = []
tx_additions: List[Coin] = []
tx_removals: List[bytes32] = []
pool_target_signature: Optional[G2Element] = get_pool_signature(
pool_target, reward_block_unfinished.proof_of_space.pool_public_key)
foliage_data = FoliageBlockData(
reward_block_unfinished.get_hash(),
pool_target,
pool_target_signature,
farmer_reward_puzzlehash,
extension_data,
)
foliage_block_data_signature: G2Element = get_plot_signature(
foliage_data.get_hash(),
reward_block_unfinished.proof_of_space.plot_public_key,
)
prev_block_hash: bytes32 = constants.GENESIS_CHALLENGE
if height != 0:
assert prev_block is not None
prev_block_hash = prev_block.header_hash
solution_program: Optional[SerializedProgram] = None
if is_transaction_block:
spend_bundle_fees: int = 0
aggregate_sig: G2Element = G2Element.infinity()
cost = uint64(0)
if spend_bundle is not None:
solution_program = best_solution_program(spend_bundle)
aggregate_sig = spend_bundle.aggregated_signature
# Calculate the cost of transactions
if solution_program is not None:
result: CostResult = calculate_cost_of_program(
solution_program, constants.CLVM_COST_RATIO_CONSTANT)
cost = result.cost
removal_amount = 0
addition_amount = 0
for coin in removals:
removal_amount += coin.amount
for coin in additions:
addition_amount += coin.amount
spend_bundle_fees = removal_amount - addition_amount
else:
spend_bundle_fees = 0
# TODO: prev generators root
reward_claims_incorporated = []
if height > 0:
assert prev_transaction_block is not None
assert prev_block is not None
curr: BlockRecord = prev_block
while not curr.is_transaction_block:
curr = blocks.block_record(curr.prev_hash)
assert curr.fees is not None
pool_coin = create_pool_coin(
curr.height,
curr.pool_puzzle_hash,
calculate_pool_reward(curr.height),
)
farmer_coin = create_farmer_coin(
curr.height,
curr.farmer_puzzle_hash,
uint64(calculate_base_farmer_reward(curr.height) + curr.fees),
)
assert curr.header_hash == prev_transaction_block.header_hash
reward_claims_incorporated += [pool_coin, farmer_coin]
if curr.height > 0:
curr = blocks.block_record(curr.prev_hash)
# Prev block is not genesis
while not curr.is_transaction_block:
pool_coin = create_pool_coin(
curr.height,
curr.pool_puzzle_hash,
calculate_pool_reward(curr.height),
)
farmer_coin = create_farmer_coin(
curr.height,
curr.farmer_puzzle_hash,
calculate_base_farmer_reward(curr.height),
)
reward_claims_incorporated += [pool_coin, farmer_coin]
curr = blocks.block_record(curr.prev_hash)
additions.extend(reward_claims_incorporated.copy())
for coin in additions:
tx_additions.append(coin)
byte_array_tx.append(bytearray(coin.puzzle_hash))
for coin in removals:
tx_removals.append(coin.name())
byte_array_tx.append(bytearray(coin.name()))
bip158: PyBIP158 = PyBIP158(byte_array_tx)
encoded = bytes(bip158.GetEncoded())
removal_merkle_set = MerkleSet()
addition_merkle_set = MerkleSet()
# Create removal Merkle set
for coin_name in tx_removals:
removal_merkle_set.add_already_hashed(coin_name)
# Create addition Merkle set
puzzlehash_coin_map: Dict[bytes32, List[Coin]] = {}
for coin in tx_additions:
if coin.puzzle_hash in puzzlehash_coin_map:
puzzlehash_coin_map[coin.puzzle_hash].append(coin)
else:
puzzlehash_coin_map[coin.puzzle_hash] = [coin]
# Addition Merkle set contains puzzlehash and hash of all coins with that puzzlehash
for puzzle, coins in puzzlehash_coin_map.items():
addition_merkle_set.add_already_hashed(puzzle)
addition_merkle_set.add_already_hashed(hash_coin_list(coins))
additions_root = addition_merkle_set.get_root()
removals_root = removal_merkle_set.get_root()
generator_hash = solution_program.get_tree_hash(
) if solution_program is not None else bytes32([0] * 32)
filter_hash: bytes32 = std_hash(encoded)
transactions_info: Optional[TransactionsInfo] = TransactionsInfo(
bytes([0] * 32),
generator_hash,
aggregate_sig,
uint64(spend_bundle_fees),
cost,
reward_claims_incorporated,
)
if prev_transaction_block is None:
prev_transaction_block_hash: bytes32 = constants.GENESIS_CHALLENGE
else:
prev_transaction_block_hash = prev_transaction_block.header_hash
assert transactions_info is not None
foliage_transaction_block: Optional[
FoliageTransactionBlock] = FoliageTransactionBlock(
prev_transaction_block_hash,
timestamp,
filter_hash,
additions_root,
removals_root,
transactions_info.get_hash(),
)
assert foliage_transaction_block is not None
foliage_transaction_block_hash: Optional[
bytes32] = foliage_transaction_block.get_hash()
foliage_transaction_block_signature: Optional[
G2Element] = get_plot_signature(
foliage_transaction_block_hash,
reward_block_unfinished.proof_of_space.plot_public_key)
assert foliage_transaction_block_signature is not None
else:
foliage_transaction_block_hash = None
foliage_transaction_block_signature = None
foliage_transaction_block = None
transactions_info = None
assert (foliage_transaction_block_hash is
None) == (foliage_transaction_block_signature is None)
foliage = Foliage(
prev_block_hash,
reward_block_unfinished.get_hash(),
foliage_data,
foliage_block_data_signature,
foliage_transaction_block_hash,
foliage_transaction_block_signature,
)
return foliage, foliage_transaction_block, transactions_info, solution_program
async def respond_additions(self,
response: wallet_protocol.RespondAdditions):
"""
The full node has responded with the additions for a block. We will use this
to try to finish the block, and add it to the state.
"""
if self._shut_down:
return
if response.header_hash not in self.cached_blocks:
self.log.warning("Do not have header for additions")
return
block_record, header_block, transaction_filter = self.cached_blocks[
response.header_hash]
assert response.height == block_record.height
additions: List[Coin]
if response.proofs is None:
# If there are no proofs, it means all additions were returned in the response.
# we must find the ones relevant to our wallets.
all_coins: List[Coin] = []
for puzzle_hash, coin_list_0 in response.coins:
all_coins += coin_list_0
additions = await self.wallet_state_manager.get_relevant_additions(
all_coins)
# Verify root
additions_merkle_set = MerkleSet()
# Addition Merkle set contains puzzlehash and hash of all coins with that puzzlehash
for puzzle_hash, coins in response.coins:
additions_merkle_set.add_already_hashed(puzzle_hash)
additions_merkle_set.add_already_hashed(hash_coin_list(coins))
additions_root = additions_merkle_set.get_root()
if header_block.header.data.additions_root != additions_root:
return
else:
# This means the full node has responded only with the relevant additions
# for our wallet. Each merkle proof must be verified.
additions = []
assert len(response.coins) == len(response.proofs)
for i in range(len(response.coins)):
assert response.coins[i][0] == response.proofs[i][0]
coin_list_1: List[Coin] = response.coins[i][1]
puzzle_hash_proof: bytes32 = response.proofs[i][1]
coin_list_proof: Optional[bytes32] = response.proofs[i][2]
if len(coin_list_1) == 0:
# Verify exclusion proof for puzzle hash
assert confirm_not_included_already_hashed(
header_block.header.data.additions_root,
response.coins[i][0],
puzzle_hash_proof,
)
else:
# Verify inclusion proof for puzzle hash
assert confirm_included_already_hashed(
header_block.header.data.additions_root,
response.coins[i][0],
puzzle_hash_proof,
)
# Verify inclusion proof for coin list
assert confirm_included_already_hashed(
header_block.header.data.additions_root,
hash_coin_list(coin_list_1),
coin_list_proof,
)
for coin in coin_list_1:
assert coin.puzzle_hash == response.coins[i][0]
additions += coin_list_1
new_br = BlockRecord(
block_record.header_hash,
block_record.prev_header_hash,
block_record.height,
block_record.weight,
additions,
None,
block_record.total_iters,
header_block.challenge.get_hash(),
)
self.cached_blocks[response.header_hash] = (
new_br,
header_block,
transaction_filter,
)
if transaction_filter is None:
raise RuntimeError("Got additions for block with no transactions.")
(
_,
removals,
) = await self.wallet_state_manager.get_filter_additions_removals(
new_br, transaction_filter)
request_all_removals = False
for coin in additions:
puzzle_store = self.wallet_state_manager.puzzle_store
record_info: Optional[
DerivationRecord] = await puzzle_store.get_derivation_record_for_puzzle_hash(
coin.puzzle_hash.hex())
if (record_info is not None
and record_info.wallet_type == WalletType.COLOURED_COIN):
request_all_removals = True
break
if len(removals) > 0 or request_all_removals:
if request_all_removals:
request_r = wallet_protocol.RequestRemovals(
header_block.height, header_block.header_hash, None)
else:
request_r = wallet_protocol.RequestRemovals(
header_block.height, header_block.header_hash, removals)
yield OutboundMessage(
NodeType.FULL_NODE,
Message("request_removals", request_r),
Delivery.RESPOND,
)
else:
# We have collected all three things: header, additions, and removals (since there are no
# relevant removals for us). Can proceed. Otherwise, we wait for the removals to arrive.
new_br = BlockRecord(
new_br.header_hash,
new_br.prev_header_hash,
new_br.height,
new_br.weight,
new_br.additions,
[],
new_br.total_iters,
new_br.new_challenge_hash,
)
respond_header_msg: Optional[
wallet_protocol.RespondHeader] = await self._block_finished(
new_br, header_block, transaction_filter)
if respond_header_msg is not None:
async for msg in self.respond_header(respond_header_msg):
yield msg
async def respond_removals(self,
response: wallet_protocol.RespondRemovals):
"""
The full node has responded with the removals for a block. We will use this
to try to finish the block, and add it to the state.
"""
if self._shut_down:
return
if (response.header_hash not in self.cached_blocks or
self.cached_blocks[response.header_hash][0].additions is None):
self.log.warning(
"Do not have header for removals, or do not have additions")
return
block_record, header_block, transaction_filter = self.cached_blocks[
response.header_hash]
assert response.height == block_record.height
all_coins: List[Coin] = []
for coin_name, coin in response.coins:
if coin is not None:
all_coins.append(coin)
if response.proofs is None:
# If there are no proofs, it means all removals were returned in the response.
# we must find the ones relevant to our wallets.
# Verify removals root
removals_merkle_set = MerkleSet()
for coin in all_coins:
if coin is not None:
removals_merkle_set.add_already_hashed(coin.name())
removals_root = removals_merkle_set.get_root()
assert header_block.header.data.removals_root == removals_root
else:
# This means the full node has responded only with the relevant removals
# for our wallet. Each merkle proof must be verified.
assert len(response.coins) == len(response.proofs)
for i in range(len(response.coins)):
# Coins are in the same order as proofs
assert response.coins[i][0] == response.proofs[i][0]
coin = response.coins[i][1]
if coin is None:
# Verifies merkle proof of exclusion
assert confirm_not_included_already_hashed(
header_block.header.data.removals_root,
response.coins[i][0],
response.proofs[i][1],
)
else:
# Verifies merkle proof of inclusion of coin name
assert response.coins[i][0] == coin.name()
assert confirm_included_already_hashed(
header_block.header.data.removals_root,
coin.name(),
response.proofs[i][1],
)
new_br = BlockRecord(
block_record.header_hash,
block_record.prev_header_hash,
block_record.height,
block_record.weight,
block_record.additions,
all_coins,
block_record.total_iters,
header_block.challenge.get_hash(),
)
self.cached_blocks[response.header_hash] = (
new_br,
header_block,
transaction_filter,
)
# We have collected all three things: header, additions, and removals. Can proceed.
respond_header_msg: Optional[
wallet_protocol.RespondHeader] = await self._block_finished(
new_br, header_block, transaction_filter)
if respond_header_msg is not None:
async for msg in self.respond_header(respond_header_msg):
yield msg
def _create_block(
self,
test_constants: Dict,
challenge_hash: bytes32,
height: uint32,
prev_header_hash: bytes32,
prev_iters: uint64,
prev_weight: uint128,
timestamp: uint64,
difficulty: uint64,
min_iters: uint64,
seed: bytes,
genesis: bool = False,
reward_puzzlehash: bytes32 = None,
transactions: Program = None,
aggsig: BLSSignature = None,
fees: uint64 = uint64(0),
) -> FullBlock:
"""
Creates a block with the specified details. Uses the stored plots to create a proof of space,
and also evaluates the VDF for the proof of time.
"""
selected_prover = None
selected_plot_sk = None
selected_pool_sk = None
selected_proof_index = 0
plots = list(self.plot_config["plots"].items())
selected_quality: Optional[bytes] = None
best_quality = 0
if self.use_any_pos:
for i in range(len(plots) * 3):
# Allow passing in seed, to create reorgs and different chains
random.seed(seed + i.to_bytes(4, "big"))
seeded_pn = random.randint(0, len(plots) - 1)
pool_sk = PrivateKey.from_bytes(
bytes.fromhex(plots[seeded_pn][1]["pool_sk"])
)
plot_sk = PrivateKey.from_bytes(
bytes.fromhex(plots[seeded_pn][1]["sk"])
)
prover = DiskProver(plots[seeded_pn][0])
qualities = prover.get_qualities_for_challenge(challenge_hash)
if len(qualities) > 0:
if self.use_any_pos:
selected_quality = qualities[0]
selected_prover = prover
selected_pool_sk = pool_sk
selected_plot_sk = plot_sk
break
else:
for i in range(len(plots)):
pool_sk = PrivateKey.from_bytes(bytes.fromhex(plots[i][1]["pool_sk"]))
plot_sk = PrivateKey.from_bytes(bytes.fromhex(plots[i][1]["sk"]))
prover = DiskProver(plots[i][0])
qualities = prover.get_qualities_for_challenge(challenge_hash)
j = 0
for quality in qualities:
qual_int = int.from_bytes(quality, "big", signed=False)
if qual_int > best_quality:
best_quality = qual_int
selected_quality = quality
selected_prover = prover
selected_pool_sk = pool_sk
selected_plot_sk = plot_sk
selected_proof_index = j
j += 1
assert selected_prover
assert selected_pool_sk
assert selected_plot_sk
pool_pk = selected_pool_sk.get_public_key()
plot_pk = selected_plot_sk.get_public_key()
if selected_quality is None:
raise RuntimeError("No proofs for this challenge")
proof_xs: bytes = selected_prover.get_full_proof(
challenge_hash, selected_proof_index
)
proof_of_space: ProofOfSpace = ProofOfSpace(
challenge_hash, pool_pk, plot_pk, selected_prover.get_size(), proof_xs
)
number_iters: uint64 = pot_iterations.calculate_iterations(
proof_of_space, difficulty, min_iters
)
# print("Doing iters", number_iters)
int_size = (test_constants["DISCRIMINANT_SIZE_BITS"] + 16) >> 4
result = prove(
challenge_hash, test_constants["DISCRIMINANT_SIZE_BITS"], number_iters
)
output = ClassgroupElement(
int512(int.from_bytes(result[0:int_size], "big", signed=True,)),
int512(
int.from_bytes(result[int_size : 2 * int_size], "big", signed=True,)
),
)
proof_bytes = result[2 * int_size : 4 * int_size]
proof_of_time = ProofOfTime(
challenge_hash, number_iters, output, self.n_wesolowski, proof_bytes,
)
if not reward_puzzlehash:
reward_puzzlehash = self.fee_target
# Use the extension data to create different blocks based on header hash
extension_data: bytes32 = bytes32([random.randint(0, 255) for _ in range(32)])
cost = uint64(0)
coinbase_reward = block_rewards.calculate_block_reward(height)
fee_reward = uint64(block_rewards.calculate_base_fee(height) + fees)
coinbase_coin, coinbase_signature = create_coinbase_coin_and_signature(
height, reward_puzzlehash, coinbase_reward, selected_pool_sk
)
parent_coin_name = std_hash(std_hash(height))
fees_coin = Coin(parent_coin_name, reward_puzzlehash, uint64(fee_reward))
# Create filter
byte_array_tx: List[bytes32] = []
tx_additions: List[Coin] = []
tx_removals: List[bytes32] = []
encoded = None
if transactions:
error, npc_list, _ = get_name_puzzle_conditions(transactions)
additions: List[Coin] = additions_for_npc(npc_list)
for coin in additions:
tx_additions.append(coin)
byte_array_tx.append(bytearray(coin.puzzle_hash))
for npc in npc_list:
tx_removals.append(npc.coin_name)
byte_array_tx.append(bytearray(npc.coin_name))
bip158: PyBIP158 = PyBIP158(byte_array_tx)
encoded = bytes(bip158.GetEncoded())
removal_merkle_set = MerkleSet()
addition_merkle_set = MerkleSet()
# Create removal Merkle set
for coin_name in tx_removals:
removal_merkle_set.add_already_hashed(coin_name)
# Create addition Merkle set
puzzlehash_coin_map: Dict[bytes32, List[Coin]] = {}
for coin in tx_additions:
if coin.puzzle_hash in puzzlehash_coin_map:
puzzlehash_coin_map[coin.puzzle_hash].append(coin)
else:
puzzlehash_coin_map[coin.puzzle_hash] = [coin]
# Addition Merkle set contains puzzlehash and hash of all coins with that puzzlehash
for puzzle, coins in puzzlehash_coin_map.items():
addition_merkle_set.add_already_hashed(puzzle)
addition_merkle_set.add_already_hashed(hash_coin_list(coins))
additions_root = addition_merkle_set.get_root()
removal_root = removal_merkle_set.get_root()
generator_hash = (
transactions.get_tree_hash()
if transactions is not None
else bytes32([0] * 32)
)
filter_hash = std_hash(encoded) if encoded is not None else bytes32([0] * 32)
header_data: HeaderData = HeaderData(
height,
prev_header_hash,
timestamp,
filter_hash,
proof_of_space.get_hash(),
uint128(prev_weight + difficulty),
uint64(prev_iters + number_iters),
additions_root,
removal_root,
coinbase_coin,
coinbase_signature,
fees_coin,
aggsig,
cost,
extension_data,
generator_hash,
)
header_hash_sig: PrependSignature = selected_plot_sk.sign_prepend(
header_data.get_hash()
)
header: Header = Header(header_data, header_hash_sig)
full_block: FullBlock = FullBlock(
proof_of_space, proof_of_time, header, transactions, encoded
)
return full_block
async def test_request_removals(self, two_nodes, wallet_blocks):
full_node_1, full_node_2, server_1, server_2 = two_nodes
wallet_a, wallet_receiver, blocks = wallet_blocks
await server_2.start_client(PeerInfo("localhost", uint16(server_1._port)), None)
blocks_list = await get_block_path(full_node_1)
blocks_new = bt.get_consecutive_blocks(
test_constants, 5, seed=b"test_request_removals"
)
# Request removals for nonexisting block fails
msgs = [
_
async for _ in full_node_1.request_removals(
wallet_protocol.RequestRemovals(
blocks_new[-1].height, blocks_new[-1].header_hash, None
)
)
]
assert len(msgs) == 1
assert isinstance(msgs[0].message.data, wallet_protocol.RejectRemovalsRequest)
# Request removals for orphaned block fails
for block in blocks_new:
async for _ in full_node_1.respond_block(fnp.RespondBlock(block)):
pass
msgs = [
_
async for _ in full_node_1.request_removals(
wallet_protocol.RequestRemovals(
blocks_new[-1].height, blocks_new[-1].header_hash, None
)
)
]
assert len(msgs) == 1
assert isinstance(msgs[0].message.data, wallet_protocol.RejectRemovalsRequest)
# If there are no transactions, empty proof and coins
blocks_new = bt.get_consecutive_blocks(
test_constants, 10, block_list=blocks_list,
)
for block in blocks_new:
[_ async for _ in full_node_1.respond_block(fnp.RespondBlock(block))]
msgs = [
_
async for _ in full_node_1.request_removals(
wallet_protocol.RequestRemovals(
blocks_new[-4].height, blocks_new[-4].header_hash, None
)
)
]
assert len(msgs) == 1
assert isinstance(msgs[0].message.data, wallet_protocol.RespondRemovals)
assert len(msgs[0].message.data.coins) == 0
assert msgs[0].message.data.proofs is None
# Add a block with transactions
spend_bundles = []
for i in range(5):
spend_bundles.append(
wallet_a.generate_signed_transaction(
100,
wallet_a.get_new_puzzlehash(),
blocks_new[i - 8].get_coinbase(),
)
)
height_with_transactions = len(blocks_new) + 1
agg = SpendBundle.aggregate(spend_bundles)
dic_h = {
height_with_transactions: (
best_solution_program(agg),
agg.aggregated_signature,
)
}
blocks_new = bt.get_consecutive_blocks(
test_constants, 5, block_list=blocks_new, transaction_data_at_height=dic_h
)
for block in blocks_new:
[_ async for _ in full_node_1.respond_block(fnp.RespondBlock(block))]
# If no coins requested, respond all coins and NO proof
msgs = [
_
async for _ in full_node_1.request_removals(
wallet_protocol.RequestRemovals(
blocks_new[height_with_transactions].height,
blocks_new[height_with_transactions].header_hash,
None,
)
)
]
assert len(msgs) == 1
assert isinstance(msgs[0].message.data, wallet_protocol.RespondRemovals)
assert len(msgs[0].message.data.coins) == 5
assert msgs[0].message.data.proofs is None
removals_merkle_set = MerkleSet()
for sb in spend_bundles:
for coin in sb.removals():
if coin is not None:
removals_merkle_set.add_already_hashed(coin.name())
# Ask for one coin and check PoI
coin_list = [spend_bundles[0].removals()[0].name()]
msgs = [
_
async for _ in full_node_1.request_removals(
wallet_protocol.RequestRemovals(
blocks_new[height_with_transactions].height,
blocks_new[height_with_transactions].header_hash,
coin_list,
)
)
]
assert len(msgs) == 1
assert isinstance(msgs[0].message.data, wallet_protocol.RespondRemovals)
assert len(msgs[0].message.data.coins) == 1
assert msgs[0].message.data.proofs is not None
assert len(msgs[0].message.data.proofs) == 1
assert confirm_included_already_hashed(
blocks_new[height_with_transactions].header.data.removals_root,
coin_list[0],
msgs[0].message.data.proofs[0][1],
)
# Ask for one coin and check PoE
coin_list = [token_bytes(32)]
msgs = [
_
async for _ in full_node_1.request_removals(
wallet_protocol.RequestRemovals(
blocks_new[height_with_transactions].height,
blocks_new[height_with_transactions].header_hash,
coin_list,
)
)
]
assert len(msgs) == 1
assert isinstance(msgs[0].message.data, wallet_protocol.RespondRemovals)
assert len(msgs[0].message.data.coins) == 1
assert msgs[0].message.data.coins[0][1] is None
assert msgs[0].message.data.proofs is not None
assert len(msgs[0].message.data.proofs) == 1
assert confirm_not_included_already_hashed(
blocks_new[height_with_transactions].header.data.removals_root,
coin_list[0],
msgs[0].message.data.proofs[0][1],
)
# Ask for two coins
coin_list = [spend_bundles[0].removals()[0].name(), token_bytes(32)]
msgs = [
_
async for _ in full_node_1.request_removals(
wallet_protocol.RequestRemovals(
blocks_new[height_with_transactions].height,
blocks_new[height_with_transactions].header_hash,
coin_list,
)
)
]
assert len(msgs) == 1
assert isinstance(msgs[0].message.data, wallet_protocol.RespondRemovals)
assert len(msgs[0].message.data.coins) == 2
assert msgs[0].message.data.coins[0][1] is not None
assert msgs[0].message.data.coins[1][1] is None
assert msgs[0].message.data.proofs is not None
assert len(msgs[0].message.data.proofs) == 2
assert confirm_included_already_hashed(
blocks_new[height_with_transactions].header.data.removals_root,
coin_list[0],
msgs[0].message.data.proofs[0][1],
)
assert confirm_not_included_already_hashed(
blocks_new[height_with_transactions].header.data.removals_root,
coin_list[1],
msgs[0].message.data.proofs[1][1],
)
async def test_request_additions(self, two_nodes, wallet_blocks):
full_node_1, full_node_2, server_1, server_2 = two_nodes
wallet_a, wallet_receiver, blocks = wallet_blocks
await server_2.start_client(PeerInfo("localhost", uint16(server_1._port)), None)
blocks_list = await get_block_path(full_node_1)
blocks_new = bt.get_consecutive_blocks(
test_constants, 5, seed=b"test_request_additions"
)
# Request additinos for nonexisting block fails
msgs = [
_
async for _ in full_node_1.request_additions(
wallet_protocol.RequestAdditions(
blocks_new[-1].height, blocks_new[-1].header_hash, None
)
)
]
assert len(msgs) == 1
assert isinstance(msgs[0].message.data, wallet_protocol.RejectAdditionsRequest)
# Request additions for orphaned block fails
for block in blocks_new:
async for _ in full_node_1.respond_block(fnp.RespondBlock(block)):
pass
msgs = [
_
async for _ in full_node_1.request_additions(
wallet_protocol.RequestAdditions(
blocks_new[-1].height, blocks_new[-1].header_hash, None
)
)
]
assert len(msgs) == 1
assert isinstance(msgs[0].message.data, wallet_protocol.RejectAdditionsRequest)
# If there are no transactions, only cb and fees additions
blocks_new = bt.get_consecutive_blocks(
test_constants, 10, block_list=blocks_list,
)
for block in blocks_new:
[_ async for _ in full_node_1.respond_block(fnp.RespondBlock(block))]
msgs = [
_
async for _ in full_node_1.request_additions(
wallet_protocol.RequestAdditions(
blocks_new[-4].height, blocks_new[-4].header_hash, None
)
)
]
assert len(msgs) == 1
assert isinstance(msgs[0].message.data, wallet_protocol.RespondAdditions)
assert len(msgs[0].message.data.coins) == 2
assert msgs[0].message.data.proofs is None
# Add a block with transactions
spend_bundles = []
puzzle_hashes = [wallet_a.get_new_puzzlehash(), wallet_a.get_new_puzzlehash()]
for i in range(5):
spend_bundles.append(
wallet_a.generate_signed_transaction(
100, puzzle_hashes[i % 2], blocks_new[i - 8].get_coinbase(),
)
)
height_with_transactions = len(blocks_new) + 1
agg = SpendBundle.aggregate(spend_bundles)
dic_h = {
height_with_transactions: (
best_solution_program(agg),
agg.aggregated_signature,
)
}
blocks_new = bt.get_consecutive_blocks(
test_constants, 5, block_list=blocks_new, transaction_data_at_height=dic_h
)
for block in blocks_new:
[_ async for _ in full_node_1.respond_block(fnp.RespondBlock(block))]
# If no puzzle hashes requested, respond all coins and NO proof
msgs = [
_
async for _ in full_node_1.request_additions(
wallet_protocol.RequestAdditions(
blocks_new[height_with_transactions].height,
blocks_new[height_with_transactions].header_hash,
None,
)
)
]
assert len(msgs) == 1
assert isinstance(msgs[0].message.data, wallet_protocol.RespondAdditions)
# One puzzle hash with change and fee (x3) = 9, minus two repeated ph = 7 + coinbase and fees = 9
assert len(msgs[0].message.data.coins) == 9
assert msgs[0].message.data.proofs is None
additions_merkle_set = MerkleSet()
for sb in spend_bundles:
for coin in sb.additions():
if coin is not None:
additions_merkle_set.add_already_hashed(coin.name())
# Ask for one coin and check both PoI
ph_list = [puzzle_hashes[0]]
msgs = [
_
async for _ in full_node_1.request_additions(
wallet_protocol.RequestAdditions(
blocks_new[height_with_transactions].height,
blocks_new[height_with_transactions].header_hash,
ph_list,
)
)
]
assert len(msgs) == 1
assert isinstance(msgs[0].message.data, wallet_protocol.RespondAdditions)
assert len(msgs[0].message.data.coins) == 1
assert len(msgs[0].message.data.coins[0][1]) == 3
assert msgs[0].message.data.proofs is not None
assert len(msgs[0].message.data.proofs) == 1
assert confirm_included_already_hashed(
blocks_new[height_with_transactions].header.data.additions_root,
ph_list[0],
msgs[0].message.data.proofs[0][1],
)
coin_list_for_ph = [
coin
for coin in blocks_new[height_with_transactions].additions()
if coin.puzzle_hash == ph_list[0]
]
assert confirm_included_already_hashed(
blocks_new[height_with_transactions].header.data.additions_root,
hash_coin_list(coin_list_for_ph),
msgs[0].message.data.proofs[0][2],
)
# Ask for one ph and check PoE
ph_list = [token_bytes(32)]
msgs = [
_
async for _ in full_node_1.request_additions(
wallet_protocol.RequestAdditions(
blocks_new[height_with_transactions].height,
blocks_new[height_with_transactions].header_hash,
ph_list,
)
)
]
assert len(msgs) == 1
assert isinstance(msgs[0].message.data, wallet_protocol.RespondAdditions)
assert len(msgs[0].message.data.coins) == 1
assert len(msgs[0].message.data.coins[0][1]) == 0
assert msgs[0].message.data.proofs is not None
assert len(msgs[0].message.data.proofs) == 1
assert confirm_not_included_already_hashed(
blocks_new[height_with_transactions].header.data.additions_root,
ph_list[0],
msgs[0].message.data.proofs[0][1],
)
assert msgs[0].message.data.proofs[0][2] is None
# Ask for two puzzle_hashes
ph_list = [puzzle_hashes[0], token_bytes(32)]
msgs = [
_
async for _ in full_node_1.request_additions(
wallet_protocol.RequestAdditions(
blocks_new[height_with_transactions].height,
blocks_new[height_with_transactions].header_hash,
ph_list,
)
)
]
assert len(msgs) == 1
assert isinstance(msgs[0].message.data, wallet_protocol.RespondAdditions)
assert len(msgs[0].message.data.coins) == 2
assert len(msgs[0].message.data.coins[0][1]) == 3
assert msgs[0].message.data.proofs is not None
assert len(msgs[0].message.data.proofs) == 2
assert confirm_included_already_hashed(
blocks_new[height_with_transactions].header.data.additions_root,
ph_list[0],
msgs[0].message.data.proofs[0][1],
)
assert confirm_included_already_hashed(
blocks_new[height_with_transactions].header.data.additions_root,
hash_coin_list(coin_list_for_ph),
msgs[0].message.data.proofs[0][2],
)
assert confirm_not_included_already_hashed(
blocks_new[height_with_transactions].header.data.additions_root,
ph_list[1],
msgs[0].message.data.proofs[1][1],
)
assert msgs[0].message.data.proofs[1][2] is None