async def add_full_block(self, block: FullBlock, sub_block: SubBlockRecord) -> None: cursor_1 = await self.db.execute( "INSERT OR REPLACE INTO full_blocks VALUES(?, ?, ?, ?, ?)", ( block.header_hash.hex(), sub_block.height, block.sub_block_height, int(block.is_block()), bytes(block), ), ) await cursor_1.close() cursor_2 = await self.db.execute( "INSERT OR REPLACE INTO sub_block_records VALUES(?, ?, ?, ?, ?, ?)", ( block.header_hash.hex(), block.prev_header_hash.hex(), block.sub_block_height, bytes(sub_block), False, block.is_block(), ), ) await cursor_2.close() await self.db.commit()
async def add_full_block(self, block: FullBlock, block_record: BlockRecord) -> None: self.block_cache.put(block.header_hash, block) cursor_1 = await self.db.execute( "INSERT OR REPLACE INTO full_blocks VALUES(?, ?, ?, ?, ?)", ( block.header_hash.hex(), block.height, int(block.is_transaction_block()), int(block.is_fully_compactified()), bytes(block), ), ) await cursor_1.close() cursor_2 = await self.db.execute( "INSERT OR REPLACE INTO block_records VALUES(?, ?, ?, ?,?, ?, ?)", ( block.header_hash.hex(), block.prev_header_hash.hex(), block.height, bytes(block_record), None if block_record.sub_epoch_summary_included is None else bytes(block_record.sub_epoch_summary_included), False, block.is_transaction_block(), ), ) await cursor_2.close() await self.db.commit()
async def setup_two_nodes(): """ Setup and teardown of two full nodes, with blockchains and separate DBs. """ # SETUP store_1 = await FullNodeStore.create("blockchain_test") store_2 = await FullNodeStore.create("blockchain_test_2") await store_1._clear_database() await store_2._clear_database() b_1: Blockchain = await Blockchain.create({}, test_constants) b_2: Blockchain = await Blockchain.create({}, test_constants) await store_1.add_block(FullBlock.from_bytes(test_constants["GENESIS_BLOCK"])) await store_2.add_block(FullBlock.from_bytes(test_constants["GENESIS_BLOCK"])) full_node_1 = FullNode(store_1, b_1) server_1 = ChiaServer(21234, full_node_1, NodeType.FULL_NODE) _ = await server_1.start_server("127.0.0.1", full_node_1._on_connect) full_node_1._set_server(server_1) full_node_2 = FullNode(store_2, b_2) server_2 = ChiaServer(21235, full_node_2, NodeType.FULL_NODE) full_node_2._set_server(server_2) yield (full_node_1, full_node_2, server_1, server_2) # TEARDOWN full_node_1._shutdown() full_node_2._shutdown() server_1.close_all() server_2.close_all() await server_1.await_closed() await server_2.await_closed() await store_1.close() await store_2.close()
async def test_timestamp(self, initial_blockchain): blocks, b = initial_blockchain # Time too far in the past block_bad = FullBlock( HeaderBlock( blocks[9].header_block.proof_of_space, blocks[9].header_block.proof_of_time, blocks[9].header_block.challenge, Header( HeaderData( blocks[9].header_block.header.data.prev_header_hash, blocks[9].header_block.header.data.timestamp - 1000, blocks[9].header_block.header.data.filter_hash, blocks[9].header_block.header.data.proof_of_space_hash, blocks[9].header_block.header.data.body_hash, blocks[9].header_block.header.data.extension_data, ), blocks[9].header_block.header.harvester_signature, ), ), blocks[9].body, ) assert (await b.receive_block( block_bad, blocks[8].header_block)) == ReceiveBlockResult.INVALID_BLOCK # Time too far in the future block_bad = FullBlock( HeaderBlock( blocks[9].header_block.proof_of_space, blocks[9].header_block.proof_of_time, blocks[9].header_block.challenge, Header( HeaderData( blocks[9].header_block.header.data.prev_header_hash, uint64(int(time.time() + 3600 * 3)), blocks[9].header_block.header.data.filter_hash, blocks[9].header_block.header.data.proof_of_space_hash, blocks[9].header_block.header.data.body_hash, blocks[9].header_block.header.data.extension_data, ), blocks[9].header_block.header.harvester_signature, ), ), blocks[9].body, ) assert (await b.receive_block( block_bad, blocks[8].header_block)) == ReceiveBlockResult.INVALID_BLOCK
async def get_header_blocks_by_hash( self, header_hashes: List[bytes32]) -> List[HeaderBlock]: if len(header_hashes) == 0: return [] header_hashes_db = tuple(h.hex() for h in header_hashes) formatted_str = f'SELECT * from blocks WHERE header_hash in ({"?," * (len(header_hashes_db) - 1)}?)' cursor = await self.db.execute(formatted_str, header_hashes_db) rows = await cursor.fetchall() await cursor.close() header_blocks: List[HeaderBlock] = [] for row in rows: header_blocks.append(FullBlock.from_bytes(row[2]).header_block) # Sorts the passed in header hashes by hash, with original index header_hashes_sorted = sorted(enumerate(header_hashes), key=lambda pair: pair[1]) # Sorts the fetched header blocks by hash header_blocks_sorted = sorted(header_blocks, key=lambda hb: hb.header_hash) # Combine both and sort by the original indeces combined = sorted(zip(header_hashes_sorted, header_blocks_sorted), key=lambda pair: pair[0][0]) # Return only the header blocks in the original order return [pair[1] for pair in combined]
async def create( coin_store: CoinStore, block_store: BlockStore, consensus_constants: ConsensusConstants, ): """ Initializes a blockchain with the header blocks from disk, assuming they have all been validated. Uses the genesis block given in override_constants, or as a fallback, in the consensus constants config. """ self = Blockchain() self.lock = asyncio.Lock() # External lock handled by full node cpu_count = multiprocessing.cpu_count() self.pool = concurrent.futures.ProcessPoolExecutor( max_workers=max(cpu_count - 1, 1)) self.constants = consensus_constants self.tips = [] self.height_to_hash = {} self.headers = {} self.coin_store = coin_store self.block_store = block_store self._shut_down = False self.genesis = FullBlock.from_bytes(self.constants.GENESIS_BLOCK) self.coinbase_freeze = self.constants.COINBASE_FREEZE_PERIOD await self._load_chain_from_store() return self
async def header_signature( self, header_signature: farmer_protocol.HeaderSignature ) -> OutboundMessageGenerator: """ Signature of header hash, by the harvester. This is enough to create an unfinished block, which only needs a Proof of Time to be finished. If the signature is valid, we call the unfinished_block routine. """ async with self.store.lock: candidate: Optional[ Tuple[Body, HeaderData, ProofOfSpace]] = await self.store.get_candidate_block( header_signature.pos_hash) if candidate is None: log.warning( f"PoS hash {header_signature.pos_hash} not found in database" ) return # Verifies that we have the correct header and body self.stored block_body, block_header_data, pos = candidate assert block_header_data.get_hash() == header_signature.header_hash block_header: Header = Header(block_header_data, header_signature.header_signature) header: HeaderBlock = HeaderBlock(pos, None, None, block_header) unfinished_block_obj: FullBlock = FullBlock(header, block_body) # Propagate to ourselves (which validates and does further propagations) request = peer_protocol.UnfinishedBlock(unfinished_block_obj) async for m in self.unfinished_block(request): # Yield all new messages (propagation to peers) yield m
def pre_validate_finished_block_header(constants: Dict, data: bytes): """ Validates all parts of block that don't need to be serially checked """ block = FullBlock.from_bytes(data) if not block.proof_of_time: return False, None # 4. Check PoT if not block.proof_of_time.is_valid(constants["DISCRIMINANT_SIZE_BITS"]): return False, None # 9. Check harvester signature of header data is valid based on harvester key if not block.header.harvester_signature.verify( [blspy.Util.hash256(block.header.data.get_hash())], [block.proof_of_space.plot_pubkey], ): return False, None # 10. Check proof of space based on challenge pos_quality_string = block.proof_of_space.verify_and_get_quality_string() if not pos_quality_string: return False, None return True, bytes(pos_quality_string)
async def get_block(self, header_hash) -> Optional[FullBlock]: try: response = await self.fetch("get_block", {"header_hash": header_hash.hex()}) except Exception: return None return FullBlock.from_json_dict(response["block"])
async def get_blocks_by_hash( self, header_hashes: List[bytes32]) -> List[FullBlock]: """ Returns a list of Full Blocks blocks, ordered by the same order in which header_hashes are passed in. Throws an exception if the blocks are not present """ if len(header_hashes) == 0: return [] header_hashes_db = tuple([hh.hex() for hh in header_hashes]) formatted_str = f'SELECT block from full_blocks WHERE header_hash in ({"?," * (len(header_hashes_db) - 1)}?)' cursor = await self.db.execute(formatted_str, header_hashes_db) rows = await cursor.fetchall() await cursor.close() all_blocks: Dict[bytes32, FullBlock] = {} for row in rows: full_block: FullBlock = FullBlock.from_bytes(row[0]) all_blocks[full_block.header_hash] = full_block ret: List[FullBlock] = [] for hh in header_hashes: if hh not in all_blocks: raise ValueError(f"Header hash {hh} not in the blockchain") ret.append(all_blocks[hh]) return ret
async def test_body_hash(self, initial_blockchain): blocks, b = initial_blockchain block_bad = FullBlock( HeaderBlock( blocks[9].header_block.proof_of_space, blocks[9].header_block.proof_of_time, blocks[9].header_block.challenge, Header( HeaderData( blocks[9].header_block.header.data.prev_header_hash, blocks[9].header_block.header.data.timestamp, blocks[9].header_block.header.data.filter_hash, blocks[9].header_block.header.data.proof_of_space_hash, bytes([1] * 32), blocks[9].header_block.header.data.extension_data, ), blocks[9].header_block.header.harvester_signature, ), ), blocks[9].body, ) assert (await b.receive_block( block_bad, blocks[8].header_block)) == ReceiveBlockResult.INVALID_BLOCK
async def initialize(self): seen_blocks = {} async for block in self.store.get_blocks(): if not self.tips or block.weight > self.tips[0].weight: self.tips = [block] seen_blocks[block.header_hash] = block if len(self.tips) > 0: curr = self.tips[0] reverse_blocks = [curr] while curr.height > 0: curr = seen_blocks[curr.prev_header_hash] reverse_blocks.append(curr) for block in reversed(reverse_blocks): self.height_to_hash[block.height] = block.header_hash self.header_blocks[block.header_hash] = block.header_block self.lca_block = self.tips[0] else: self.genesis = FullBlock.from_bytes( self.constants["GENESIS_BLOCK"]) result = await self.receive_block(self.genesis) if result != ReceiveBlockResult.ADDED_TO_HEAD: raise InvalidGenesisBlock() assert self.lca_block
async def get_full_block(self, header_hash: bytes32) -> Optional[FullBlock]: cursor = await self.db.execute("SELECT block from full_blocks WHERE header_hash=?", (header_hash.hex(),)) row = await cursor.fetchone() await cursor.close() if row is not None: return FullBlock.from_bytes(row[0]) return None
async def test_generator_hash(self, initial_blockchain): blocks, b = initial_blockchain new_header_data = HeaderData( blocks[9].header.data.height, blocks[9].header.data.prev_header_hash, blocks[9].header.data.timestamp, blocks[9].header.data.filter_hash, blocks[9].header.data.proof_of_space_hash, blocks[9].header.data.weight, blocks[9].header.data.total_iters, blocks[9].header.data.additions_root, blocks[9].header.data.removals_root, blocks[9].header.data.farmer_rewards_puzzle_hash, blocks[9].header.data.total_transaction_fees, blocks[9].header.data.pool_target, blocks[9].header.data.aggregated_signature, blocks[9].header.data.cost, blocks[9].header.data.extension_data, bytes([1] * 32), ) block_bad = FullBlock( blocks[9].proof_of_space, blocks[9].proof_of_time, Header( new_header_data, bt.get_plot_signature( new_header_data, blocks[9].proof_of_space.plot_public_key), ), blocks[9].transactions_generator, blocks[9].transactions_filter, ) result, removed, error_code = await b.receive_block(block_bad) assert result == ReceiveBlockResult.INVALID_BLOCK assert error_code == Err.INVALID_TRANSACTIONS_GENERATOR_HASH
async def get_unfinished_blocks(self) -> Dict[Tuple[bytes32, uint64], FullBlock]: cursor = await self.db.execute( "SELECT challenge_hash, iterations, block from unfinished_blocks" ) rows = await cursor.fetchall() await cursor.close() return {(bytes.fromhex(a), b): FullBlock.from_bytes(c) for a, b, c in rows}
def persistent_blocks( num_of_blocks: int, db_name: str, seed: bytes = b"", empty_sub_slots=0, normalized_to_identity: bool = False, ): # try loading from disc, if not create new blocks.db file # TODO hash fixtures.py and blocktool.py, add to path, delete if the files changed block_path_dir = Path("~/.chia/blocks").expanduser() file_path = Path(f"~/.chia/blocks/{db_name}").expanduser() if not path.exists(block_path_dir): mkdir(block_path_dir.parent) mkdir(block_path_dir) if file_path.exists(): try: bytes_list = file_path.read_bytes() block_bytes_list: List[bytes] = pickle.loads(bytes_list) blocks: List[FullBlock] = [] for block_bytes in block_bytes_list: blocks.append(FullBlock.from_bytes(block_bytes)) if len(blocks) == num_of_blocks: print(f"\n loaded {file_path} with {len(blocks)} blocks") return blocks except EOFError: print("\n error reading db file") return new_test_db(file_path, num_of_blocks, seed, empty_sub_slots, normalized_to_identity)
async def test_prev_pointer(self, initial_blockchain): blocks, b = initial_blockchain block_bad = FullBlock( blocks[9].proof_of_space, blocks[9].proof_of_time, Header( HeaderData( blocks[9].header.data.height, bytes([1] * 32), blocks[9].header.data.timestamp, blocks[9].header.data.filter_hash, blocks[9].header.data.proof_of_space_hash, blocks[9].header.data.weight, blocks[9].header.data.total_iters, blocks[9].header.data.additions_root, blocks[9].header.data.removals_root, blocks[9].header.data.farmer_rewards_puzzle_hash, blocks[9].header.data.total_transaction_fees, blocks[9].header.data.pool_target, blocks[9].header.data.aggregated_signature, blocks[9].header.data.cost, blocks[9].header.data.extension_data, blocks[9].header.data.generator_hash, ), blocks[9].header.plot_signature, ), blocks[9].transactions_generator, blocks[9].transactions_filter, ) result, removed, error_code = await b.receive_block(block_bad) assert (result) == ReceiveBlockResult.DISCONNECTED_BLOCK assert error_code is None
def pre_validate_finished_block_header(constants: ConsensusConstants, data: bytes): """ Validates all parts of block that don't need to be serially checked """ block = FullBlock.from_bytes(data) if not block.proof_of_time: return False, None # 4. Check PoT if not block.proof_of_time.is_valid(constants.DISCRIMINANT_SIZE_BITS): return False, None # 9. Check harvester signature of header data is valid based on harvester key validates = blspy.AugSchemeMPL.verify( block.proof_of_space.plot_public_key, block.header.data.get_hash(), block.header.plot_signature, ) if not validates: return False, None # 10. Check proof of space based on challenge pos_quality_string = block.proof_of_space.verify_and_get_quality_string( constants.NUMBER_ZERO_BITS_CHALLENGE_SIG) if not pos_quality_string: return False, None return True, bytes(pos_quality_string)
async def setup_full_node(db_name, port, introducer_port=None, dic={}): # SETUP test_constants_copy = test_constants.copy() for k in dic.keys(): test_constants_copy[k] = dic[k] db_path = Path(db_name) connection = await aiosqlite.connect(db_path) store_1 = await FullNodeStore.create(connection) await store_1._clear_database() unspent_store_1 = await CoinStore.create(connection) await unspent_store_1._clear_database() mempool_1 = MempoolManager(unspent_store_1, test_constants_copy) b_1: Blockchain = await Blockchain.create(unspent_store_1, store_1, test_constants_copy) await mempool_1.new_tips(await b_1.get_full_tips()) await store_1.add_block( FullBlock.from_bytes(test_constants_copy["GENESIS_BLOCK"])) net_config = load_config(root_path, "config.yaml") ping_interval = net_config.get("ping_interval") network_id = net_config.get("network_id") config = load_config(root_path, "config.yaml", "full_node") if introducer_port is not None: config["introducer_peer"]["host"] = "127.0.0.1" config["introducer_peer"]["port"] = introducer_port full_node_1 = FullNode( store_1, b_1, config, mempool_1, unspent_store_1, f"full_node_{port}", test_constants_copy, ) assert ping_interval is not None assert network_id is not None server_1 = ChiaServer( port, full_node_1, NodeType.FULL_NODE, ping_interval, network_id, root_path, config, ) _ = await server_1.start_server(full_node_1._on_connect) full_node_1._set_server(server_1) yield (full_node_1, server_1) # TEARDOWN full_node_1._shutdown() server_1.close_all() await connection.close() Path(db_name).unlink()
async def get_potential_block(self, height: uint32) -> Optional[FullBlock]: cursor = await self.db.execute( "SELECT * from potential_blocks WHERE height=?", (height, )) row = await cursor.fetchone() await cursor.close() if row is not None: return FullBlock.from_bytes(row[1]) return None
async def get_block(self, header_hash) -> Optional[FullBlock]: try: response = await self.fetch("get_block", {"header_hash": header_hash.hex()}) except aiohttp.client_exceptions.ClientResponseError as e: if e.message == "Not Found": return None raise return FullBlock.from_json_dict(response["block"])
async def get_all_block(self, start: uint32, end: uint32) -> List[FullBlock]: response = await self.fetch("get_blocks", { "start": start, "end": end, "exclude_header_hash": True }) return [FullBlock.from_json_dict(r) for r in response["blocks"]]
def get_cand(index: int): unf_block = FullBlock( candidates[index].proof_of_space, None, candidates[index].header, candidates[index].transactions_generator, candidates[index].transactions_filter, ) return fnp.RespondUnfinishedBlock(unf_block)
async def proof_of_time_finished( self, request: timelord_protocol.ProofOfTimeFinished ) -> OutboundMessageGenerator: """ A proof of time, received by a peer timelord. We can use this to complete a block, and call the block routine (which handles propagation and verification of blocks). """ async with self.store.lock: dict_key = ( request.proof.challenge_hash, request.proof.number_of_iterations, ) unfinished_block_obj: Optional[ FullBlock] = await self.store.get_unfinished_block(dict_key) if not unfinished_block_obj: log.warning( f"Received a proof of time that we cannot use to complete a block {dict_key}" ) return prev_full_block = await self.store.get_block( unfinished_block_obj.prev_header_hash) assert prev_full_block prev_block: HeaderBlock = prev_full_block.header_block difficulty: uint64 = self.blockchain.get_next_difficulty( unfinished_block_obj.prev_header_hash) assert prev_block.challenge challenge: Challenge = Challenge( request.proof.challenge_hash, unfinished_block_obj.header_block.proof_of_space.get_hash(), request.proof.output.get_hash(), uint32(prev_block.challenge.height + 1), uint64(prev_block.challenge.total_weight + difficulty), uint64(prev_block.challenge.total_iters + request.proof.number_of_iterations), ) new_header_block = HeaderBlock( unfinished_block_obj.header_block.proof_of_space, request.proof, challenge, unfinished_block_obj.header_block.header, ) new_full_block: FullBlock = FullBlock(new_header_block, unfinished_block_obj.body) async with self.store.lock: sync_mode = await self.store.get_sync_mode() if sync_mode: async with self.store.lock: await self.store.add_potential_future_block(new_full_block) else: async for msg in self.block(peer_protocol.Block(new_full_block)): yield msg
async def get_full_blocks_at(self, sub_heights: List[uint32]) -> List[FullBlock]: if len(sub_heights) == 0: return [] heights_db = tuple(sub_heights) formatted_str = f'SELECT block from full_blocks WHERE sub_height in ({"?," * (len(heights_db) - 1)}?)' cursor = await self.db.execute(formatted_str, heights_db) rows = await cursor.fetchall() await cursor.close() return [FullBlock.from_bytes(row[0]) for row in rows]
def _validate_merkle_root( self, block: FullBlock, tx_additions: List[Coin] = None, tx_removals: List[bytes32] = None, ) -> Optional[Err]: additions = [] removals = [] if tx_additions: additions.extend(tx_additions) if tx_removals: removals.extend(tx_removals) removal_merkle_set = MerkleSet() addition_merkle_set = MerkleSet() # Create removal Merkle set for coin_name in removals: removal_merkle_set.add_already_hashed(coin_name) # Create addition Merkle set puzzlehash_coins_map: Dict[bytes32, List[Coin]] = {} for coin in additions + [block.get_coinbase(), block.get_fees_coin()]: if coin.puzzle_hash in puzzlehash_coins_map: puzzlehash_coins_map[coin.puzzle_hash].append(coin) else: puzzlehash_coins_map[coin.puzzle_hash] = [coin] # Addition Merkle set contains puzzlehash and hash of all coins with that puzzlehash for puzzle, coins in puzzlehash_coins_map.items(): addition_merkle_set.add_already_hashed(puzzle) addition_merkle_set.add_already_hashed(hash_coin_list(coins)) additions_root = addition_merkle_set.get_root() removals_root = removal_merkle_set.get_root() if block.header.data.additions_root != additions_root: return Err.BAD_ADDITION_ROOT if block.header.data.removals_root != removals_root: return Err.BAD_REMOVAL_ROOT return None
async def get_unfinished_block( self, key: Tuple[bytes32, uint64]) -> Optional[FullBlock]: cursor = await self.db.execute( "SELECT block from unfinished_blocks WHERE challenge_hash=? AND iterations=?", (key[0].hex(), key[1]), ) row = await cursor.fetchone() await cursor.close() if row is not None: return FullBlock.from_bytes(row[0]) return None
async def test_basic_store(self): assert sqlite3.threadsafety == 1 blocks = bt.get_consecutive_blocks(test_constants, 9, [], 9, b"0") # blocks_alt = bt.get_consecutive_blocks(test_constants, 3, [], 9, b"1") bt.get_consecutive_blocks(test_constants, 3, [], 9, b"1") db = await SyncStore.create() # db_2 = await SyncStore.create() await SyncStore.create() # Save/get sync for sync_mode in (False, True): db.set_sync_mode(sync_mode) assert sync_mode == db.get_sync_mode() FullBlock.from_bytes(test_constants.GENESIS_BLOCK) # clear sync info await db.clear_sync_info() # add/get potential tip, get potential tips num db.add_potential_tip(blocks[6]) assert blocks[6] == db.get_potential_tip(blocks[6].header_hash)
async def new_lca(self, block: FullBlock): removals, additions = await block.tx_removals_and_additions() for coin in additions: record: CoinRecord = CoinRecord(coin, block.height, uint32(0), False, False) await self.add_coin_record(record) for coin_name in removals: await self.set_spent(coin_name, block.height) coinbase_coin = block.get_coinbase() fees_coin = block.get_fees_coin() coinbase_r: CoinRecord = CoinRecord(coinbase_coin, block.height, uint32(0), False, True) fees_r: CoinRecord = CoinRecord(fees_coin, block.height, uint32(0), False, True) await self.add_coin_record(coinbase_r) await self.add_coin_record(fees_r)
async def new_block(self, block: FullBlock): """ Only called for blocks which are blocks (and thus have rewards and transactions) """ if block.is_transaction_block() is False: return assert block.foliage_transaction_block is not None removals, additions = block.tx_removals_and_additions() for coin in additions: record: CoinRecord = CoinRecord( coin, block.height, uint32(0), False, False, block.foliage_transaction_block.timestamp, ) await self._add_coin_record(record) for coin_name in removals: await self._set_spent(coin_name, block.height) included_reward_coins = block.get_included_reward_coins() if block.height == 0: assert len(included_reward_coins) == 0 else: assert len(included_reward_coins) >= 2 for coin in included_reward_coins: reward_coin_r: CoinRecord = CoinRecord( coin, block.height, uint32(0), False, True, block.foliage_transaction_block.timestamp, ) await self._add_coin_record(reward_coin_r)