async def rollback_to_block(self, block_index: int) -> List[CoinRecord]: """ Note that block_index can be negative, in which case everything is rolled back Returns the list of coin records that have been modified """ # Update memory cache delete_queue: List[bytes32] = [] for coin_name, coin_record in list( self.coin_record_cache.cache.items()): if int(coin_record.spent_block_index) > block_index: new_record = CoinRecord( coin_record.coin, coin_record.confirmed_block_index, uint32(0), coin_record.coinbase, coin_record.timestamp, ) self.coin_record_cache.put(coin_record.coin.name(), new_record) if int(coin_record.confirmed_block_index) > block_index: delete_queue.append(coin_name) for coin_name in delete_queue: self.coin_record_cache.remove(coin_name) coin_changes: Dict[bytes32, CoinRecord] = {} async with self.coin_record_db.execute( "SELECT confirmed_index, spent_index, coinbase, puzzle_hash, " "coin_parent, amount, timestamp FROM coin_record WHERE confirmed_index>?", (block_index, ), ) as cursor: for row in await cursor.fetchall(): coin = self.row_to_coin(row) record = CoinRecord(coin, uint32(0), row[1], row[2], uint64(0)) coin_changes[record.name] = record # Delete from storage await self.coin_record_db.execute( "DELETE FROM coin_record WHERE confirmed_index>?", (block_index, )) async with self.coin_record_db.execute( "SELECT confirmed_index, spent_index, coinbase, puzzle_hash, " "coin_parent, amount, timestamp FROM coin_record WHERE confirmed_index>?", (block_index, ), ) as cursor: for row in await cursor.fetchall(): coin = self.row_to_coin(row) record = CoinRecord(coin, row[0], uint32(0), row[2], row[6]) if record.name not in coin_changes: coin_changes[record.name] = record if self.db_wrapper.db_version == 2: await self.coin_record_db.execute( "UPDATE coin_record SET spent_index=0 WHERE spent_index>?", (block_index, )) else: await self.coin_record_db.execute( "UPDATE coin_record SET spent_index = 0, spent = 0 WHERE spent_index>?", (block_index, )) return list(coin_changes.values())
async def new_block( self, height: uint32, timestamp: uint64, included_reward_coins: Set[Coin], tx_additions: List[Coin], tx_removals: List[bytes32], ) -> List[CoinRecord]: """ Only called for blocks which are blocks (and thus have rewards and transactions) Returns a list of the CoinRecords that were added by this block """ start = time() additions = [] for coin in tx_additions: record: CoinRecord = CoinRecord( coin, height, uint32(0), False, timestamp, ) additions.append(record) if height == 0: assert len(included_reward_coins) == 0 else: assert len(included_reward_coins) >= 2 for coin in included_reward_coins: reward_coin_r: CoinRecord = CoinRecord( coin, height, uint32(0), True, timestamp, ) additions.append(reward_coin_r) await self._add_coin_records(additions) await self._set_spent(tx_removals, height) end = time() log.log( logging.WARNING if end - start > 10 else logging.DEBUG, f"It took {end - start:0.2f}s to apply {len(tx_additions)} additions and " + f"{len(tx_removals)} removals to the coin store. Make sure " + "blockchain database is on a fast drive", ) return additions
async def get_additions_and_removals(self, header_hash: bytes32) -> Tuple[List[CoinRecord], List[CoinRecord]]: try: response = await self.fetch("get_additions_and_removals", {"header_hash": header_hash.hex()}) except Exception: return [], [] removals = [] additions = [] for coin_record in response["removals"]: removals.append(CoinRecord.from_json_dict(coin_record)) for coin_record in response["additions"]: additions.append(CoinRecord.from_json_dict(coin_record)) return additions, removals
async def rollback_to_block(self, block_index: int): """ Note that block_index can be negative, in which case everything is rolled back """ # Update memory cache delete_queue: bytes32 = [] for coin_name, coin_record in self.coin_record_cache.items(): if int(coin_record.spent_block_index) > block_index: new_record = CoinRecord( coin_record.coin, coin_record.confirmed_block_index, coin_record.spent_block_index, False, coin_record.coinbase, coin_record.timestamp, ) self.coin_record_cache[ coin_record.coin.name().hex()] = new_record if int(coin_record.confirmed_block_index) > block_index: delete_queue.append(coin_name) for coin_name in delete_queue: del self.coin_record_cache[coin_name] # Delete from storage c1 = await self.coin_record_db.execute( "DELETE FROM coin_record WHERE confirmed_index>?", (block_index, )) await c1.close() c2 = await self.coin_record_db.execute( "UPDATE coin_record SET spent_index = 0, spent = 0 WHERE spent_index>?", (block_index, ), ) await c2.close()
async def get_coin_records_by_parent_ids( self, include_spent_coins: bool, parent_ids: List[bytes32], start_height: uint32 = uint32(0), end_height: uint32 = uint32((2 ** 32) - 1), ) -> List[CoinRecord]: if len(parent_ids) == 0: return [] coins = set() parent_ids_db = tuple([pid.hex() for pid in parent_ids]) cursor = await self.coin_record_db.execute( f'SELECT * from coin_record WHERE coin_parent in ({"?," * (len(parent_ids_db) - 1)}?) ' f"AND confirmed_index>=? AND confirmed_index<? " f"{'' if include_spent_coins else 'AND spent=0'}", parent_ids_db + (start_height, end_height), ) rows = await cursor.fetchall() await cursor.close() for row in rows: coin = Coin(bytes32(bytes.fromhex(row[6])), bytes32(bytes.fromhex(row[5])), uint64.from_bytes(row[7])) coins.add(CoinRecord(coin, row[1], row[2], row[3], row[4], row[8])) return list(coins)
async def get_coin_records_by_parent_ids( self, include_spent_coins: bool, parent_ids: List[bytes32], start_height: uint32 = uint32(0), end_height: uint32 = uint32((2**32) - 1), ) -> List[CoinRecord]: if len(parent_ids) == 0: return [] coins = set() parent_ids_db: Tuple[Any, ...] if self.db_wrapper.db_version == 2: parent_ids_db = tuple(parent_ids) else: parent_ids_db = tuple([pid.hex() for pid in parent_ids]) async with self.coin_record_db.execute( f"SELECT confirmed_index, spent_index, coinbase, puzzle_hash, " f'coin_parent, amount, timestamp FROM coin_record WHERE coin_parent in ({"?," * (len(parent_ids) - 1)}?) ' f"AND confirmed_index>=? AND confirmed_index<? " f"{'' if include_spent_coins else 'AND spent_index=0'}", parent_ids_db + (start_height, end_height), ) as cursor: for row in await cursor.fetchall(): coin = self.row_to_coin(row) coins.add(CoinRecord(coin, row[0], row[1], row[2], row[6])) return list(coins)
def new_coin_record(self, coin: Coin, coinbase=False) -> CoinRecord: return CoinRecord( coin, uint32(self.block_height + 1), uint32(0), coinbase, self.timestamp, )
async def get_coin_record_by_name( self, coin_id: bytes32) -> Optional[CoinRecord]: try: response = await self.fetch("get_coin_record_by_name", {"name": coin_id.hex()}) except Exception: return None return CoinRecord.from_json_dict(response["coin_record"])
async def get_coins_added_at_height(self, height: uint32) -> List[CoinRecord]: cursor = await self.coin_record_db.execute("SELECT * from coin_record WHERE confirmed_index=?", (height,)) rows = await cursor.fetchall() await cursor.close() coins = [] for row in rows: coin = Coin(bytes32(bytes.fromhex(row[6])), bytes32(bytes.fromhex(row[5])), uint64.from_bytes(row[7])) coins.append(CoinRecord(coin, row[1], row[2], row[3], row[4], row[8])) return coins
async def new_block(self, block: FullBlock, tx_additions: List[Coin], tx_removals: List[bytes32]): """ Only called for blocks which are blocks (and thus have rewards and transactions) """ if block.is_transaction_block() is False: return assert block.foliage_transaction_block is not None for coin in tx_additions: record: CoinRecord = CoinRecord( coin, block.height, uint32(0), False, False, block.foliage_transaction_block.timestamp, ) await self._add_coin_record(record, False) included_reward_coins = block.get_included_reward_coins() if block.height == 0: assert len(included_reward_coins) == 0 else: assert len(included_reward_coins) >= 2 for coin in included_reward_coins: reward_coin_r: CoinRecord = CoinRecord( coin, block.height, uint32(0), False, True, block.foliage_transaction_block.timestamp, ) await self._add_coin_record(reward_coin_r, False) total_amount_spent: int = 0 for coin_name in tx_removals: total_amount_spent += await self._set_spent( coin_name, block.height) # Sanity check, already checked in block_body_validation assert sum([a.amount for a in tx_additions]) <= total_amount_spent
async def get_coin_record(self, coin_name: bytes32) -> Optional[CoinRecord]: if coin_name.hex() in self.coin_record_cache: return self.coin_record_cache[coin_name.hex()] cursor = await self.coin_record_db.execute("SELECT * from coin_record WHERE coin_name=?", (coin_name.hex(),)) row = await cursor.fetchone() await cursor.close() if row is not None: coin = Coin(bytes32(bytes.fromhex(row[6])), bytes32(bytes.fromhex(row[5])), uint64.from_bytes(row[7])) return CoinRecord(coin, row[1], row[2], row[3], row[4], row[8]) return None
def _add_coin_entry(self, coin: Coin, birthday: CoinTimestamp) -> None: name = coin.name() # assert name not in self._db self._db[name] = CoinRecord( coin, uint32(birthday.height), uint32(0), False, uint64(birthday.seconds), ) self._ph_index[coin.puzzle_hash].append(name)
async def get_coin_record(self, coin_name: bytes32) -> Optional[CoinRecord]: cached = self.coin_record_cache.get(coin_name) if cached is not None: return cached cursor = await self.coin_record_db.execute("SELECT * from coin_record WHERE coin_name=?", (coin_name.hex(),)) row = await cursor.fetchone() await cursor.close() if row is not None: coin = Coin(bytes32(bytes.fromhex(row[6])), bytes32(bytes.fromhex(row[5])), uint64.from_bytes(row[7])) record = CoinRecord(coin, row[1], row[2], row[3], row[4], row[8]) self.coin_record_cache.put(record.coin.name(), record) return record return None
async def get_coins_added_at_height(self, height: uint32) -> List[CoinRecord]: async with self.coin_record_db.execute( "SELECT confirmed_index, spent_index, coinbase, puzzle_hash, " "coin_parent, amount, timestamp FROM coin_record WHERE confirmed_index=?", (height, ), ) as cursor: rows = await cursor.fetchall() coins = [] for row in rows: coin = self.row_to_coin(row) coins.append(CoinRecord(coin, row[0], row[1], row[2], row[6])) return coins
async def _set_spent(self, coin_name: bytes32, index: uint32): current: Optional[CoinRecord] = await self.get_coin_record(coin_name) if current is None: return spent: CoinRecord = CoinRecord( current.coin, current.confirmed_block_index, index, True, current.coinbase, current.timestamp, ) # type: ignore # noqa await self._add_coin_record(spent)
async def new_block(self, block: FullBlock, additions: List[Coin], removals: List[bytes32]): """ Only called for blocks which are blocks (and thus have rewards and transactions) """ if block.is_transaction_block() is False: return assert block.foliage_transaction_block is not None for coin in additions: record: CoinRecord = CoinRecord( coin, block.height, uint32(0), False, False, block.foliage_transaction_block.timestamp, ) await self._add_coin_record(record) included_reward_coins = block.get_included_reward_coins() if block.height == 0: assert len(included_reward_coins) == 0 else: assert len(included_reward_coins) >= 2 for coin in included_reward_coins: reward_coin_r: CoinRecord = CoinRecord( coin, block.height, uint32(0), False, True, block.foliage_transaction_block.timestamp, ) await self._add_coin_record(reward_coin_r) for coin_name in removals: await self._set_spent(coin_name, block.height)
async def _set_spent(self, coin_name: bytes32, index: uint32): current: Optional[CoinRecord] = await self.get_coin_record(coin_name) if current is None: raise ValueError( f"Cannot spend a coin that does not exist in db: {coin_name}") spent: CoinRecord = CoinRecord( current.coin, current.confirmed_block_index, index, True, current.coinbase, current.timestamp, ) # type: ignore # noqa await self._add_coin_record(spent)
async def get_coin_records_by_puzzle_hash( self, puzzle_hash: bytes32, include_spent_coins: bool = True, start_height: Optional[int] = None, end_height: Optional[int] = None, ) -> List: d = {"puzzle_hash": puzzle_hash.hex(), "include_spent_coins": include_spent_coins} if start_height is not None: d["start_height"] = start_height if end_height is not None: d["end_height"] = end_height return [ CoinRecord.from_json_dict(coin) for coin in (await self.fetch("get_coin_records_by_puzzle_hash", d))["coin_records"] ]
async def _set_spent(self, coin_name: bytes32, index: uint32) -> uint64: current: Optional[CoinRecord] = await self.get_coin_record(coin_name) if current is None: raise ValueError(f"Cannot spend a coin that does not exist in db: {coin_name}") assert not current.spent # Redundant sanity check, already checked in block_body_validation spent: CoinRecord = CoinRecord( current.coin, current.confirmed_block_index, index, True, current.coinbase, current.timestamp, ) # type: ignore # noqa await self._add_coin_record(spent, True) return current.coin.amount
async def get_coin_records_by_parent_ids( self, parent_ids: List[bytes32], include_spent_coins: bool = True, start_height: Optional[int] = None, end_height: Optional[int] = None, ) -> List: parent_ids_hex = [pid.hex() for pid in parent_ids] d = {"parent_ids": parent_ids_hex, "include_spent_coins": include_spent_coins} if start_height is not None: d["start_height"] = start_height if end_height is not None: d["end_height"] = end_height return [ CoinRecord.from_json_dict(coin) for coin in (await self.fetch("get_coin_records_by_parent_ids", d))["coin_records"] ]
async def get_coin_record(self, coin_name: bytes32) -> Optional[CoinRecord]: cached = self.coin_record_cache.get(coin_name) if cached is not None: return cached async with self.coin_record_db.execute( "SELECT confirmed_index, spent_index, coinbase, puzzle_hash, " "coin_parent, amount, timestamp FROM coin_record WHERE coin_name=?", (self.maybe_to_hex(coin_name), ), ) as cursor: row = await cursor.fetchone() if row is not None: coin = self.row_to_coin(row) record = CoinRecord(coin, row[0], row[1], row[2], row[6]) self.coin_record_cache.put(record.coin.name(), record) return record return None
async def get_coins_removed_at_height(self, height: uint32) -> List[CoinRecord]: # Special case to avoid querying all unspent coins (spent_index=0) if height == 0: return [] async with self.coin_record_db.execute( "SELECT confirmed_index, spent_index, coinbase, puzzle_hash, " "coin_parent, amount, timestamp FROM coin_record WHERE spent_index=?", (height, ), ) as cursor: coins = [] for row in await cursor.fetchall(): if row[1] != 0: coin = self.row_to_coin(row) coin_record = CoinRecord(coin, row[0], row[1], row[2], row[6]) coins.append(coin_record) return coins
def validate_spend_bundle( self, spend_bundle: SpendBundle, now: CoinTimestamp, max_cost: int, cost_per_byte: int, ) -> int: # this should use blockchain consensus code program = simple_solution_generator(spend_bundle) result: NPCResult = get_name_puzzle_conditions( program, max_cost, cost_per_byte=cost_per_byte, mempool_mode=True ) if result.error is not None: raise BadSpendBundleError(f"condition validation failure {Err(result.error)}") ephemeral_db = dict(self._db) for npc in result.npc_list: for coin in created_outputs_for_conditions_dict(npc.condition_dict, npc.coin_name): name = coin.name() ephemeral_db[name] = CoinRecord( coin, uint32(now.height), uint32(0), False, uint64(now.seconds), ) for npc in result.npc_list: prev_transaction_block_height = uint32(now.height) timestamp = uint64(now.seconds) coin_record = ephemeral_db.get(npc.coin_name) if coin_record is None: raise BadSpendBundleError(f"coin not found for id 0x{npc.coin_name.hex()}") # noqa err = mempool_check_conditions_dict( coin_record, npc.condition_dict, prev_transaction_block_height, timestamp, ) if err is not None: raise BadSpendBundleError(f"condition validation failure {Err(err)}") return 0
async def get_coin_records_by_names( self, names: List[bytes32], include_spent_coins: bool = True, start_height: Optional[int] = None, end_height: Optional[int] = None, ) -> List: names_hex = [name.hex() for name in names] d = {"names": names_hex, "include_spent_coins": include_spent_coins} if start_height is not None: d["start_height"] = start_height if end_height is not None: d["end_height"] = end_height response = await self.fetch("get_coin_records_by_names", d) return [ CoinRecord.from_json_dict(coin_record_dict_backwards_compat(coin)) for coin in response["coin_records"] ]
async def get_coin_records_by_puzzle_hash( self, include_spent_coins: bool, puzzle_hash: bytes32, start_height: uint32 = uint32(0), end_height: uint32 = uint32((2 ** 32) - 1), ) -> List[CoinRecord]: coins = set() cursor = await self.coin_record_db.execute( f"SELECT * from coin_record WHERE puzzle_hash=? AND confirmed_index>=? AND confirmed_index<? " f"{'' if include_spent_coins else 'AND spent=0'}", (puzzle_hash.hex(), start_height, end_height), ) rows = await cursor.fetchall() await cursor.close() for row in rows: coin = Coin(bytes32(bytes.fromhex(row[6])), bytes32(bytes.fromhex(row[5])), uint64.from_bytes(row[7])) coins.add(CoinRecord(coin, row[1], row[2], row[3], row[4], row[8])) return list(coins)
async def get_coin_records_by_puzzle_hash( self, include_spent_coins: bool, puzzle_hash: bytes32, start_height: uint32 = uint32(0), end_height: uint32 = uint32((2**32) - 1), ) -> List[CoinRecord]: coins = set() async with self.coin_record_db.execute( f"SELECT confirmed_index, spent_index, coinbase, puzzle_hash, " f"coin_parent, amount, timestamp FROM coin_record INDEXED BY coin_puzzle_hash WHERE puzzle_hash=? " f"AND confirmed_index>=? AND confirmed_index<? " f"{'' if include_spent_coins else 'AND spent_index=0'}", (self.maybe_to_hex(puzzle_hash), start_height, end_height), ) as cursor: for row in await cursor.fetchall(): coin = self.row_to_coin(row) coins.add(CoinRecord(coin, row[0], row[1], row[2], row[6])) return list(coins)
async def _set_spent(self, coin_names: List[bytes32], index: uint32): assert len(coin_names) == 0 or index > 0 # if this coin is in the cache, mark it as spent in there updates = [] for coin_name in coin_names: r = self.coin_record_cache.get(coin_name) if r is not None: self.coin_record_cache.put( r.name, CoinRecord(r.coin, r.confirmed_block_index, index, r.coinbase, r.timestamp)) updates.append((index, self.maybe_to_hex(coin_name))) if self.db_wrapper.db_version == 2: await self.coin_record_db.executemany( "UPDATE OR FAIL coin_record SET spent_index=? WHERE coin_name=?", updates) else: await self.coin_record_db.executemany( "UPDATE OR FAIL coin_record SET spent=1,spent_index=? WHERE coin_name=?", updates)
async def validate_block_body( constants: ConsensusConstants, blocks: BlockchainInterface, block_store: BlockStore, coin_store: CoinStore, peak: Optional[BlockRecord], block: Union[FullBlock, UnfinishedBlock], height: uint32, npc_result: Optional[NPCResult], fork_point_with_peak: Optional[uint32], get_block_generator: Callable, ) -> Tuple[Optional[Err], Optional[NPCResult]]: """ This assumes the header block has been completely validated. Validates the transactions and body of the block. Returns None for the first value if everything validates correctly, or an Err if something does not validate. For the second value, returns a CostResult if validation succeeded, and there are transactions """ if isinstance(block, FullBlock): assert height == block.height prev_transaction_block_height: uint32 = uint32(0) # 1. For non block blocks, foliage block, transaction filter, transactions info, and generator must be empty # If it is a block but not a transaction block, there is no body to validate. Check that all fields are None if block.foliage.foliage_transaction_block_hash is None: if ( block.foliage_transaction_block is not None or block.transactions_info is not None or block.transactions_generator is not None ): return Err.NOT_BLOCK_BUT_HAS_DATA, None return None, None # This means the block is valid # 2. For blocks, foliage block, transaction filter, transactions info must not be empty if ( block.foliage_transaction_block is None or block.foliage_transaction_block.filter_hash is None or block.transactions_info is None ): return Err.IS_TRANSACTION_BLOCK_BUT_NO_DATA, None # keeps track of the reward coins that need to be incorporated expected_reward_coins: Set[Coin] = set() # 3. The transaction info hash in the Foliage block must match the transaction info if block.foliage_transaction_block.transactions_info_hash != std_hash(block.transactions_info): return Err.INVALID_TRANSACTIONS_INFO_HASH, None # 4. The foliage block hash in the foliage block must match the foliage block if block.foliage.foliage_transaction_block_hash != std_hash(block.foliage_transaction_block): return Err.INVALID_FOLIAGE_BLOCK_HASH, None # 7. The reward claims must be valid for the previous blocks, and current block fees if height > 0: # Add reward claims for all blocks from the prev prev block, until the prev block (including the latter) prev_transaction_block = blocks.block_record(block.foliage_transaction_block.prev_transaction_block_hash) prev_transaction_block_height = prev_transaction_block.height assert prev_transaction_block.fees is not None pool_coin = create_pool_coin( prev_transaction_block_height, prev_transaction_block.pool_puzzle_hash, calculate_pool_reward(prev_transaction_block.height), constants.GENESIS_CHALLENGE, ) farmer_coin = create_farmer_coin( prev_transaction_block_height, prev_transaction_block.farmer_puzzle_hash, uint64(calculate_base_farmer_reward(prev_transaction_block.height) + prev_transaction_block.fees), constants.GENESIS_CHALLENGE, ) # Adds the previous block expected_reward_coins.add(pool_coin) expected_reward_coins.add(farmer_coin) # For the second block in the chain, don't go back further if prev_transaction_block.height > 0: curr_b = blocks.block_record(prev_transaction_block.prev_hash) while not curr_b.is_transaction_block: expected_reward_coins.add( create_pool_coin( curr_b.height, curr_b.pool_puzzle_hash, calculate_pool_reward(curr_b.height), constants.GENESIS_CHALLENGE, ) ) expected_reward_coins.add( create_farmer_coin( curr_b.height, curr_b.farmer_puzzle_hash, calculate_base_farmer_reward(curr_b.height), constants.GENESIS_CHALLENGE, ) ) curr_b = blocks.block_record(curr_b.prev_hash) if set(block.transactions_info.reward_claims_incorporated) != expected_reward_coins: return Err.INVALID_REWARD_COINS, None removals: List[bytes32] = [] coinbase_additions: List[Coin] = list(expected_reward_coins) additions: List[Coin] = [] coin_announcement_names: Set[bytes32] = set() puzzle_announcement_names: Set[bytes32] = set() npc_list: List[NPC] = [] removals_puzzle_dic: Dict[bytes32, bytes32] = {} cost: uint64 = uint64(0) if height <= constants.INITIAL_FREEZE_PERIOD and block.transactions_generator is not None: return Err.INITIAL_TRANSACTION_FREEZE, None if height > constants.INITIAL_FREEZE_PERIOD and constants.NETWORK_TYPE == NetworkType.MAINNET: return Err.INITIAL_TRANSACTION_FREEZE, None else: # 6a. The generator root must be the hash of the serialized bytes of # the generator for this block (or zeroes if no generator) if block.transactions_generator is not None: if std_hash(bytes(block.transactions_generator)) != block.transactions_info.generator_root: return Err.INVALID_TRANSACTIONS_GENERATOR_ROOT, None else: if block.transactions_info.generator_root != bytes([0] * 32): return Err.INVALID_TRANSACTIONS_GENERATOR_ROOT, None # 6b. The generator_ref_list must be the hash of the serialized bytes of # the generator ref list for this block (or 'one' bytes [0x01] if no generator) # 6c. The generator ref list length must be less than or equal to MAX_GENERATOR_REF_LIST_SIZE entries if block.transactions_generator_ref_list in (None, []): if block.transactions_info.generator_refs_root != bytes([1] * 32): return Err.INVALID_TRANSACTIONS_GENERATOR_REFS_ROOT, None else: # If we have a generator reference list, we must have a generator if block.transactions_generator is None: return Err.INVALID_TRANSACTIONS_GENERATOR_REFS_ROOT, None # The generator_refs_root must be the hash of the concatenation of the List[uint32] generator_refs_hash = std_hash(b"".join([bytes(i) for i in block.transactions_generator_ref_list])) if block.transactions_info.generator_refs_root != generator_refs_hash: return Err.INVALID_TRANSACTIONS_GENERATOR_REFS_ROOT, None if len(block.transactions_generator_ref_list) > constants.MAX_GENERATOR_REF_LIST_SIZE: return Err.PRE_SOFT_FORK_TOO_MANY_GENERATOR_REFS, None if block.transactions_generator is not None: # Get List of names removed, puzzles hashes for removed coins and conditions created assert npc_result is not None cost = calculate_cost_of_program(block.transactions_generator, npc_result, constants.COST_PER_BYTE) npc_list = npc_result.npc_list # 8. Check that cost <= MAX_BLOCK_COST_CLVM log.warning(f"Cost: {cost} max: {constants.MAX_BLOCK_COST_CLVM}") if cost > constants.MAX_BLOCK_COST_CLVM: return Err.BLOCK_COST_EXCEEDS_MAX, None if npc_result.error is not None: return Err.GENERATOR_RUNTIME_ERROR, None for npc in npc_list: removals.append(npc.coin_name) removals_puzzle_dic[npc.coin_name] = npc.puzzle_hash additions = additions_for_npc(npc_list) coin_announcement_names = coin_announcements_names_for_npc(npc_list) puzzle_announcement_names = puzzle_announcements_names_for_npc(npc_list) else: assert npc_result is None # 9. Check that the correct cost is in the transactions info if block.transactions_info.cost != cost: return Err.INVALID_BLOCK_COST, None additions_dic: Dict[bytes32, Coin] = {} # 10. Check additions for max coin amount # Be careful to check for 64 bit overflows in other languages. This is the max 64 bit unsigned integer for coin in additions + coinbase_additions: additions_dic[coin.name()] = coin if coin.amount > constants.MAX_COIN_AMOUNT: return Err.COIN_AMOUNT_EXCEEDS_MAXIMUM, None # 11. Validate addition and removal roots root_error = validate_block_merkle_roots( block.foliage_transaction_block.additions_root, block.foliage_transaction_block.removals_root, additions + coinbase_additions, removals, ) if root_error: return root_error, None # 12. The additions and removals must result in the correct filter byte_array_tx: List[bytes32] = [] for coin in additions + coinbase_additions: byte_array_tx.append(bytearray(coin.puzzle_hash)) for coin_name in removals: byte_array_tx.append(bytearray(coin_name)) bip158: PyBIP158 = PyBIP158(byte_array_tx) encoded_filter = bytes(bip158.GetEncoded()) filter_hash = std_hash(encoded_filter) if filter_hash != block.foliage_transaction_block.filter_hash: return Err.INVALID_TRANSACTIONS_FILTER_HASH, None # 13. Check for duplicate outputs in additions addition_counter = collections.Counter(_.name() for _ in additions + coinbase_additions) for k, v in addition_counter.items(): if v > 1: return Err.DUPLICATE_OUTPUT, None # 14. Check for duplicate spends inside block removal_counter = collections.Counter(removals) for k, v in removal_counter.items(): if v > 1: return Err.DOUBLE_SPEND, None # 15. Check if removals exist and were not previously spent. (unspent_db + diff_store + this_block) if peak is None or height == 0: fork_h: int = -1 elif fork_point_with_peak is not None: fork_h = fork_point_with_peak else: fork_h = find_fork_point_in_chain(blocks, peak, blocks.block_record(block.prev_header_hash)) if fork_h == -1: coin_store_reorg_height = -1 else: last_block_in_common = await blocks.get_block_record_from_db(blocks.height_to_hash(uint32(fork_h))) assert last_block_in_common is not None coin_store_reorg_height = last_block_in_common.height # Get additions and removals since (after) fork_h but not including this block additions_since_fork: Dict[bytes32, Tuple[Coin, uint32]] = {} removals_since_fork: Set[bytes32] = set() coinbases_since_fork: Dict[bytes32, uint32] = {} if height > 0: prev_block: Optional[FullBlock] = await block_store.get_full_block(block.prev_header_hash) reorg_blocks: Dict[int, FullBlock] = {} curr: Optional[FullBlock] = prev_block assert curr is not None reorg_blocks[curr.height] = curr while curr.height > fork_h: if curr.height == 0: break curr = await block_store.get_full_block(curr.prev_header_hash) assert curr is not None reorg_blocks[curr.height] = curr curr = prev_block assert curr is not None while curr.height > fork_h: # Coin store doesn't contain coins from fork, we have to run generator for each block in fork if curr.transactions_generator is not None: curr_block_generator: Optional[BlockGenerator] = await get_block_generator(curr) assert curr_block_generator is not None npc_result = get_name_puzzle_conditions(curr_block_generator, False) removals_in_curr, additions_in_curr = tx_removals_and_additions(npc_result.npc_list) else: removals_in_curr = [] additions_in_curr = [] for c_name in removals_in_curr: removals_since_fork.add(c_name) for c in additions_in_curr: additions_since_fork[c.name()] = (c, curr.height) for coinbase_coin in curr.get_included_reward_coins(): additions_since_fork[coinbase_coin.name()] = (coinbase_coin, curr.height) coinbases_since_fork[coinbase_coin.name()] = curr.height if curr.height == 0: break curr = reorg_blocks[curr.height - 1] assert curr is not None removal_coin_records: Dict[bytes32, CoinRecord] = {} for rem in removals: if rem in additions_dic: # Ephemeral coin rem_coin: Coin = additions_dic[rem] new_unspent: CoinRecord = CoinRecord( rem_coin, height, uint32(0), False, (rem in coinbases_since_fork), block.foliage_transaction_block.timestamp, ) removal_coin_records[new_unspent.name] = new_unspent else: unspent = await coin_store.get_coin_record(rem) if unspent is not None and unspent.confirmed_block_index <= coin_store_reorg_height: # Spending something in the current chain, confirmed before fork # (We ignore all coins confirmed after fork) if unspent.spent == 1 and unspent.spent_block_index <= coin_store_reorg_height: # Check for coins spent in an ancestor block return Err.DOUBLE_SPEND, None removal_coin_records[unspent.name] = unspent else: # This coin is not in the current heaviest chain, so it must be in the fork if rem not in additions_since_fork: # Check for spending a coin that does not exist in this fork # TODO: fix this, there is a consensus bug here return Err.UNKNOWN_UNSPENT, None new_coin, confirmed_height = additions_since_fork[rem] new_coin_record: CoinRecord = CoinRecord( new_coin, confirmed_height, uint32(0), False, (rem in coinbases_since_fork), block.foliage_transaction_block.timestamp, ) removal_coin_records[new_coin_record.name] = new_coin_record # This check applies to both coins created before fork (pulled from coin_store), # and coins created after fork (additions_since_fork)> if rem in removals_since_fork: # This coin was spent in the fork return Err.DOUBLE_SPEND, None removed = 0 for unspent in removal_coin_records.values(): removed += unspent.coin.amount added = 0 for coin in additions: added += coin.amount # 16. Check that the total coin amount for added is <= removed if removed < added: return Err.MINTING_COIN, None fees = removed - added assert_fee_sum: uint64 = uint64(0) for npc in npc_list: if ConditionOpcode.RESERVE_FEE in npc.condition_dict: fee_list: List[ConditionWithArgs] = npc.condition_dict[ConditionOpcode.RESERVE_FEE] for cvp in fee_list: fee = int_from_bytes(cvp.vars[0]) assert_fee_sum = assert_fee_sum + fee # 17. Check that the assert fee sum <= fees if fees < assert_fee_sum: return Err.RESERVE_FEE_CONDITION_FAILED, None # 18. Check that the assert fee amount < maximum coin amount if fees > constants.MAX_COIN_AMOUNT: return Err.COIN_AMOUNT_EXCEEDS_MAXIMUM, None # 19. Check that the computed fees are equal to the fees in the block header if block.transactions_info.fees != fees: return Err.INVALID_BLOCK_FEE_AMOUNT, None # 20. Verify that removed coin puzzle_hashes match with calculated puzzle_hashes for unspent in removal_coin_records.values(): if unspent.coin.puzzle_hash != removals_puzzle_dic[unspent.name]: return Err.WRONG_PUZZLE_HASH, None # 21. Verify conditions # create hash_key list for aggsig check pairs_pks = [] pairs_msgs = [] for npc in npc_list: assert height is not None unspent = removal_coin_records[npc.coin_name] error = mempool_check_conditions_dict( unspent, coin_announcement_names, puzzle_announcement_names, npc.condition_dict, prev_transaction_block_height, block.foliage_transaction_block.timestamp, ) if error: return error, None for pk, m in pkm_pairs_for_conditions_dict( npc.condition_dict, npc.coin_name, constants.AGG_SIG_ME_ADDITIONAL_DATA ): pairs_pks.append(pk) pairs_msgs.append(m) # 22. Verify aggregated signature # TODO: move this to pre_validate_blocks_multiprocessing so we can sync faster if not block.transactions_info.aggregated_signature: return Err.BAD_AGGREGATE_SIGNATURE, None # noinspection PyTypeChecker if not AugSchemeMPL.aggregate_verify(pairs_pks, pairs_msgs, block.transactions_info.aggregated_signature): return Err.BAD_AGGREGATE_SIGNATURE, None return None, npc_result
async def add_spendbundle( self, new_spend: SpendBundle, cost_result: CostResult, spend_name: bytes32, validate_signature=True, ) -> Tuple[Optional[uint64], MempoolInclusionStatus, Optional[Err]]: """ Tries to add spendbundle to either self.mempools or to_pool if it's specified. Returns true if it's added in any of pools, Returns error if it fails. """ start_time = time.time() if self.peak is None: return None, MempoolInclusionStatus.FAILED, Err.MEMPOOL_NOT_INITIALIZED npc_list = cost_result.npc_list cost = cost_result.cost log.debug(f"Cost: {cost}") if cost > self.constants.MAX_BLOCK_COST_CLVM: return None, MempoolInclusionStatus.FAILED, Err.BLOCK_COST_EXCEEDS_MAX if cost_result.error is not None: return None, MempoolInclusionStatus.FAILED, Err(cost_result.error) # build removal list removal_names: List[bytes32] = new_spend.removal_names() additions = additions_for_npc(npc_list) additions_dict: Dict[bytes32, Coin] = {} for add in additions: additions_dict[add.name()] = add addition_amount = uint64(0) # Check additions for max coin amount for coin in additions: if coin.amount > self.constants.MAX_COIN_AMOUNT: return ( None, MempoolInclusionStatus.FAILED, Err.COIN_AMOUNT_EXCEEDS_MAXIMUM, ) addition_amount = uint64(addition_amount + coin.amount) # Check for duplicate outputs addition_counter = collections.Counter(_.name() for _ in additions) for k, v in addition_counter.items(): if v > 1: return None, MempoolInclusionStatus.FAILED, Err.DUPLICATE_OUTPUT # Check for duplicate inputs removal_counter = collections.Counter(name for name in removal_names) for k, v in removal_counter.items(): if v > 1: return None, MempoolInclusionStatus.FAILED, Err.DOUBLE_SPEND # Skip if already added if spend_name in self.mempool.spends: return uint64(cost), MempoolInclusionStatus.SUCCESS, None removal_record_dict: Dict[bytes32, CoinRecord] = {} removal_coin_dict: Dict[bytes32, Coin] = {} unknown_unspent_error: bool = False removal_amount = uint64(0) for name in removal_names: removal_record = await self.coin_store.get_coin_record(name) if removal_record is None and name not in additions_dict: unknown_unspent_error = True break elif name in additions_dict: removal_coin = additions_dict[name] # TODO(straya): what timestamp to use here? removal_record = CoinRecord( removal_coin, uint32( self.peak.height + 1), # In mempool, so will be included in next height uint32(0), False, False, uint64(int(time.time())), ) assert removal_record is not None removal_amount = uint64(removal_amount + removal_record.coin.amount) removal_record_dict[name] = removal_record removal_coin_dict[name] = removal_record.coin if unknown_unspent_error: return None, MempoolInclusionStatus.FAILED, Err.UNKNOWN_UNSPENT if addition_amount > removal_amount: print(addition_amount, removal_amount) return None, MempoolInclusionStatus.FAILED, Err.MINTING_COIN fees = removal_amount - addition_amount assert_fee_sum: uint64 = uint64(0) for npc in npc_list: if ConditionOpcode.RESERVE_FEE in npc.condition_dict: fee_list: List[ConditionWithArgs] = npc.condition_dict[ ConditionOpcode.RESERVE_FEE] for cvp in fee_list: fee = int_from_bytes(cvp.vars[0]) assert_fee_sum = assert_fee_sum + fee if fees < assert_fee_sum: return ( None, MempoolInclusionStatus.FAILED, Err.RESERVE_FEE_CONDITION_FAILED, ) if cost == 0: return None, MempoolInclusionStatus.FAILED, Err.UNKNOWN fees_per_cost: float = fees / cost # If pool is at capacity check the fee, if not then accept even without the fee if self.mempool.at_full_capacity(cost): if fees == 0: return None, MempoolInclusionStatus.FAILED, Err.INVALID_FEE_LOW_FEE if fees_per_cost <= self.mempool.get_min_fee_rate(cost): return None, MempoolInclusionStatus.FAILED, Err.INVALID_FEE_LOW_FEE # Check removals against UnspentDB + DiffStore + Mempool + SpendBundle # Use this information later when constructing a block fail_reason, conflicts = await self.check_removals(removal_record_dict) # If there is a mempool conflict check if this spendbundle has a higher fee per cost than all others tmp_error: Optional[Err] = None conflicting_pool_items: Dict[bytes32, MempoolItem] = {} if fail_reason is Err.MEMPOOL_CONFLICT: for conflicting in conflicts: sb: MempoolItem = self.mempool.removals[conflicting.name()] conflicting_pool_items[sb.name] = sb if not self.can_replace(conflicting_pool_items, removal_record_dict, fees, fees_per_cost): self.add_to_potential_tx_set(new_spend, spend_name, cost_result) return ( uint64(cost), MempoolInclusionStatus.PENDING, Err.MEMPOOL_CONFLICT, ) elif fail_reason: return None, MempoolInclusionStatus.FAILED, fail_reason if tmp_error: return None, MempoolInclusionStatus.FAILED, tmp_error # Verify conditions, create hash_key list for aggsig check pks: List[G1Element] = [] msgs: List[bytes32] = [] error: Optional[Err] = None coin_announcements_in_spend: Set[ bytes32] = coin_announcements_names_for_npc(npc_list) puzzle_announcements_in_spend: Set[ bytes32] = puzzle_announcements_names_for_npc(npc_list) for npc in npc_list: coin_record: CoinRecord = removal_record_dict[npc.coin_name] # Check that the revealed removal puzzles actually match the puzzle hash if npc.puzzle_hash != coin_record.coin.puzzle_hash: log.warning( "Mempool rejecting transaction because of wrong puzzle_hash" ) log.warning( f"{npc.puzzle_hash} != {coin_record.coin.puzzle_hash}") return None, MempoolInclusionStatus.FAILED, Err.WRONG_PUZZLE_HASH chialisp_height = (self.peak.prev_transaction_block_height if not self.peak.is_transaction_block else self.peak.height) error = mempool_check_conditions_dict( coin_record, coin_announcements_in_spend, puzzle_announcements_in_spend, npc.condition_dict, uint32(chialisp_height), ) if error: if error is Err.ASSERT_HEIGHT_ABSOLUTE_FAILED or error is Err.ASSERT_HEIGHT_RELATIVE_FAILED: self.add_to_potential_tx_set(new_spend, spend_name, cost_result) return uint64(cost), MempoolInclusionStatus.PENDING, error break if validate_signature: for pk, message in pkm_pairs_for_conditions_dict( npc.condition_dict, npc.coin_name, self.constants.AGG_SIG_ME_ADDITIONAL_DATA): pks.append(pk) msgs.append(message) if error: return None, MempoolInclusionStatus.FAILED, error if validate_signature: # Verify aggregated signature if not AugSchemeMPL.aggregate_verify( pks, msgs, new_spend.aggregated_signature): log.warning( f"Aggsig validation error {pks} {msgs} {new_spend}") return None, MempoolInclusionStatus.FAILED, Err.BAD_AGGREGATE_SIGNATURE # Remove all conflicting Coins and SpendBundles if fail_reason: mempool_item: MempoolItem for mempool_item in conflicting_pool_items.values(): self.mempool.remove_from_pool(mempool_item) removals: List[Coin] = [coin for coin in removal_coin_dict.values()] new_item = MempoolItem(new_spend, uint64(fees), cost_result, spend_name, additions, removals) self.mempool.add_to_pool(new_item, additions, removal_coin_dict) log.info(f"add_spendbundle took {time.time() - start_time} seconds") return uint64(cost), MempoolInclusionStatus.SUCCESS, None
async def add_spendbundle( self, new_spend: SpendBundle, npc_result: NPCResult, spend_name: bytes32, program: Optional[SerializedProgram] = None, ) -> Tuple[Optional[uint64], MempoolInclusionStatus, Optional[Err]]: """ Tries to add spend bundle to the mempool Returns the cost (if SUCCESS), the result (MempoolInclusion status), and an optional error """ start_time = time.time() if self.peak is None: return None, MempoolInclusionStatus.FAILED, Err.MEMPOOL_NOT_INITIALIZED npc_list = npc_result.npc_list assert npc_result.error is None if program is None: program = simple_solution_generator(new_spend).program cost = calculate_cost_of_program(program, npc_result, self.constants.COST_PER_BYTE) log.debug(f"Cost: {cost}") if cost > int(self.limit_factor * self.constants.MAX_BLOCK_COST_CLVM): # we shouldn't ever end up here, since the cost is limited when we # execute the CLVM program. return None, MempoolInclusionStatus.FAILED, Err.BLOCK_COST_EXCEEDS_MAX # build removal list removal_names: List[bytes32] = [npc.coin_name for npc in npc_list] if set(removal_names) != set([s.name() for s in new_spend.removals()]): return None, MempoolInclusionStatus.FAILED, Err.INVALID_SPEND_BUNDLE additions = additions_for_npc(npc_list) additions_dict: Dict[bytes32, Coin] = {} for add in additions: additions_dict[add.name()] = add addition_amount = uint64(0) # Check additions for max coin amount for coin in additions: if coin.amount < 0: return ( None, MempoolInclusionStatus.FAILED, Err.COIN_AMOUNT_NEGATIVE, ) if coin.amount > self.constants.MAX_COIN_AMOUNT: return ( None, MempoolInclusionStatus.FAILED, Err.COIN_AMOUNT_EXCEEDS_MAXIMUM, ) addition_amount = uint64(addition_amount + coin.amount) # Check for duplicate outputs addition_counter = collections.Counter(_.name() for _ in additions) for k, v in addition_counter.items(): if v > 1: return None, MempoolInclusionStatus.FAILED, Err.DUPLICATE_OUTPUT # Check for duplicate inputs removal_counter = collections.Counter(name for name in removal_names) for k, v in removal_counter.items(): if v > 1: return None, MempoolInclusionStatus.FAILED, Err.DOUBLE_SPEND # Skip if already added if spend_name in self.mempool.spends: return uint64(cost), MempoolInclusionStatus.SUCCESS, None removal_record_dict: Dict[bytes32, CoinRecord] = {} removal_coin_dict: Dict[bytes32, Coin] = {} removal_amount = uint64(0) for name in removal_names: removal_record = await self.coin_store.get_coin_record(name) if removal_record is None and name not in additions_dict: return None, MempoolInclusionStatus.FAILED, Err.UNKNOWN_UNSPENT elif name in additions_dict: removal_coin = additions_dict[name] # TODO(straya): what timestamp to use here? assert self.peak.timestamp is not None removal_record = CoinRecord( removal_coin, uint32( self.peak.height + 1), # In mempool, so will be included in next height uint32(0), False, uint64(self.peak.timestamp + 1), ) assert removal_record is not None removal_amount = uint64(removal_amount + removal_record.coin.amount) removal_record_dict[name] = removal_record removal_coin_dict[name] = removal_record.coin removals: List[Coin] = [coin for coin in removal_coin_dict.values()] if addition_amount > removal_amount: print(addition_amount, removal_amount) return None, MempoolInclusionStatus.FAILED, Err.MINTING_COIN fees = uint64(removal_amount - addition_amount) assert_fee_sum: uint64 = uint64(0) for npc in npc_list: if ConditionOpcode.RESERVE_FEE in npc.condition_dict: fee_list: List[ConditionWithArgs] = npc.condition_dict[ ConditionOpcode.RESERVE_FEE] for cvp in fee_list: fee = int_from_bytes(cvp.vars[0]) if fee < 0: return None, MempoolInclusionStatus.FAILED, Err.RESERVE_FEE_CONDITION_FAILED assert_fee_sum = assert_fee_sum + fee if fees < assert_fee_sum: return ( None, MempoolInclusionStatus.FAILED, Err.RESERVE_FEE_CONDITION_FAILED, ) if cost == 0: return None, MempoolInclusionStatus.FAILED, Err.UNKNOWN fees_per_cost: float = fees / cost # If pool is at capacity check the fee, if not then accept even without the fee if self.mempool.at_full_capacity(cost): if fees_per_cost < self.nonzero_fee_minimum_fpc: return None, MempoolInclusionStatus.FAILED, Err.INVALID_FEE_TOO_CLOSE_TO_ZERO if fees_per_cost <= self.mempool.get_min_fee_rate(cost): return None, MempoolInclusionStatus.FAILED, Err.INVALID_FEE_LOW_FEE # Check removals against UnspentDB + DiffStore + Mempool + SpendBundle # Use this information later when constructing a block fail_reason, conflicts = await self.check_removals(removal_record_dict) # If there is a mempool conflict check if this spendbundle has a higher fee per cost than all others tmp_error: Optional[Err] = None conflicting_pool_items: Dict[bytes32, MempoolItem] = {} if fail_reason is Err.MEMPOOL_CONFLICT: for conflicting in conflicts: sb: MempoolItem = self.mempool.removals[conflicting.name()] conflicting_pool_items[sb.name] = sb if not self.can_replace(conflicting_pool_items, removal_record_dict, fees, fees_per_cost): potential = MempoolItem(new_spend, uint64(fees), npc_result, cost, spend_name, additions, removals, program) self.potential_cache.add(potential) return ( uint64(cost), MempoolInclusionStatus.PENDING, Err.MEMPOOL_CONFLICT, ) elif fail_reason: return None, MempoolInclusionStatus.FAILED, fail_reason if tmp_error: return None, MempoolInclusionStatus.FAILED, tmp_error # Verify conditions, create hash_key list for aggsig check error: Optional[Err] = None for npc in npc_list: coin_record: CoinRecord = removal_record_dict[npc.coin_name] # Check that the revealed removal puzzles actually match the puzzle hash if npc.puzzle_hash != coin_record.coin.puzzle_hash: log.warning( "Mempool rejecting transaction because of wrong puzzle_hash" ) log.warning( f"{npc.puzzle_hash} != {coin_record.coin.puzzle_hash}") return None, MempoolInclusionStatus.FAILED, Err.WRONG_PUZZLE_HASH chialisp_height = (self.peak.prev_transaction_block_height if not self.peak.is_transaction_block else self.peak.height) assert self.peak.timestamp is not None error = mempool_check_conditions_dict( coin_record, npc.condition_dict, uint32(chialisp_height), self.peak.timestamp, ) if error: if error is Err.ASSERT_HEIGHT_ABSOLUTE_FAILED or error is Err.ASSERT_HEIGHT_RELATIVE_FAILED: potential = MempoolItem(new_spend, uint64(fees), npc_result, cost, spend_name, additions, removals, program) self.potential_cache.add(potential) return uint64(cost), MempoolInclusionStatus.PENDING, error break if error: return None, MempoolInclusionStatus.FAILED, error # Remove all conflicting Coins and SpendBundles if fail_reason: mempool_item: MempoolItem for mempool_item in conflicting_pool_items.values(): self.mempool.remove_from_pool(mempool_item) new_item = MempoolItem(new_spend, uint64(fees), npc_result, cost, spend_name, additions, removals, program) self.mempool.add_to_pool(new_item) now = time.time() log.log( logging.DEBUG, f"add_spendbundle {spend_name} took {now - start_time:0.2f} seconds. " f"Cost: {cost} ({round(100.0 * cost/self.constants.MAX_BLOCK_COST_CLVM, 3)}% of max block cost)", ) return uint64(cost), MempoolInclusionStatus.SUCCESS, None