async def add_full_block(self, block: FullBlock, block_record: BlockRecord) -> None: cached = self.block_cache.get(block.header_hash) if cached is not None: # Since write to db can fail, we remove from cache here to avoid potential inconsistency # Adding to cache only from reading self.block_cache.put(block.header_hash, None) cursor_1 = await self.db.execute( "INSERT OR REPLACE INTO full_blocks VALUES(?, ?, ?, ?, ?)", ( block.header_hash.hex(), block.height, int(block.is_transaction_block()), int(block.is_fully_compactified()), bytes(block), ), ) await cursor_1.close() cursor_2 = await self.db.execute( "INSERT OR REPLACE INTO block_records VALUES(?, ?, ?, ?,?, ?, ?)", ( block.header_hash.hex(), block.prev_header_hash.hex(), block.height, bytes(block_record), None if block_record.sub_epoch_summary_included is None else bytes(block_record.sub_epoch_summary_included), False, block.is_transaction_block(), ), ) await cursor_2.close()
def get_block_header(block: FullBlock, tx_addition_coins: List[Coin], removals_names: List[bytes32]) -> HeaderBlock: # Create filter byte_array_tx: List[bytes32] = [] addition_coins = tx_addition_coins + list( block.get_included_reward_coins()) if block.is_transaction_block(): for coin in addition_coins: byte_array_tx.append(bytearray(coin.puzzle_hash)) for name in removals_names: byte_array_tx.append(bytearray(name)) bip158: PyBIP158 = PyBIP158(byte_array_tx) encoded_filter: bytes = bytes(bip158.GetEncoded()) return HeaderBlock( block.finished_sub_slots, block.reward_chain_block, block.challenge_chain_sp_proof, block.challenge_chain_ip_proof, block.reward_chain_sp_proof, block.reward_chain_ip_proof, block.infused_challenge_chain_ip_proof, block.foliage, block.foliage_transaction_block, encoded_filter, block.transactions_info, )
def run_and_get_removals_and_additions( block: FullBlock, max_cost: int, cost_per_byte: int, rust_checker: bool, safe_mode=False) -> Tuple[List[bytes32], List[Coin]]: removals: List[bytes32] = [] additions: List[Coin] = [] assert len(block.transactions_generator_ref_list) == 0 if not block.is_transaction_block(): return [], [] if block.transactions_generator is not None: npc_result = get_name_puzzle_conditions( BlockGenerator(block.transactions_generator, []), max_cost, cost_per_byte=cost_per_byte, safe_mode=safe_mode, rust_checker=rust_checker, ) # build removals list for npc in npc_result.npc_list: removals.append(npc.coin_name) additions.extend(additions_for_npc(npc_result.npc_list)) rewards = block.get_included_reward_coins() additions.extend(rewards) return removals, additions
def get_block_header(block: FullBlock, tx_addition_coins: List[Coin], removals_names: List[bytes32]) -> HeaderBlock: # Create filter byte_array_tx: List[bytes32] = [] addition_coins = tx_addition_coins + list( block.get_included_reward_coins()) if block.is_transaction_block(): for coin in addition_coins: # TODO: address hint error and remove ignore # error: Argument 1 to "append" of "list" has incompatible type "bytearray"; expected "bytes32" # [arg-type] byte_array_tx.append(bytearray( coin.puzzle_hash)) # type: ignore[arg-type] for name in removals_names: # TODO: address hint error and remove ignore # error: Argument 1 to "append" of "list" has incompatible type "bytearray"; expected "bytes32" # [arg-type] byte_array_tx.append(bytearray(name)) # type: ignore[arg-type] bip158: PyBIP158 = PyBIP158(byte_array_tx) encoded_filter: bytes = bytes(bip158.GetEncoded()) return HeaderBlock( block.finished_sub_slots, block.reward_chain_block, block.challenge_chain_sp_proof, block.challenge_chain_ip_proof, block.reward_chain_sp_proof, block.reward_chain_ip_proof, block.infused_challenge_chain_ip_proof, block.foliage, block.foliage_transaction_block, encoded_filter, block.transactions_info, )
async def add_full_block(self, block: FullBlock, block_record: BlockRecord) -> None: self.block_cache.put(block.header_hash, block) cursor_1 = await self.db.execute( "INSERT OR REPLACE INTO full_blocks VALUES(?, ?, ?, ?, ?)", ( block.header_hash.hex(), block.height, int(block.is_transaction_block()), int(block.is_fully_compactified()), bytes(block), ), ) await cursor_1.close() cursor_2 = await self.db.execute( "INSERT OR REPLACE INTO block_records VALUES(?, ?, ?, ?,?, ?, ?)", ( block.header_hash.hex(), block.prev_header_hash.hex(), block.height, bytes(block_record), None if block_record.sub_epoch_summary_included is None else bytes(block_record.sub_epoch_summary_included), False, block.is_transaction_block(), ), ) await cursor_2.close() await self.db.commit()
async def get_removals_and_additions(self, block: FullBlock) -> Tuple[List[bytes32], List[Coin]]: if block.is_transaction_block(): if block.transactions_generator is not None: block_generator: Optional[BlockGenerator] = await self.get_block_generator(block) assert block_generator is not None npc_result = get_name_puzzle_conditions(block_generator, self.constants.MAX_BLOCK_COST_CLVM, False) removals, additions = block_removals_and_additions(block, npc_result.npc_list) return removals, additions else: return [], list(block.get_included_reward_coins()) else: return [], []
async def add_full_block( self, header_hash: bytes32, block: FullBlock, block_record: BlockRecord, in_main_chain: bool ) -> None: self.block_cache.put(header_hash, block) if self.db_wrapper.db_version == 2: ses: Optional[bytes] = ( None if block_record.sub_epoch_summary_included is None else bytes(block_record.sub_epoch_summary_included) ) await self.db.execute( "INSERT OR REPLACE INTO full_blocks VALUES(?, ?, ?, ?, ?, ?, ?, ?)", ( header_hash, block.prev_header_hash, block.height, ses, int(block.is_fully_compactified()), in_main_chain, # in_main_chain self.compress(block), bytes(block_record), ), ) else: await self.db.execute( "INSERT OR REPLACE INTO full_blocks VALUES(?, ?, ?, ?, ?)", ( header_hash.hex(), block.height, int(block.is_transaction_block()), int(block.is_fully_compactified()), bytes(block), ), ) await self.db.execute( "INSERT OR REPLACE INTO block_records VALUES(?, ?, ?, ?,?, ?, ?)", ( header_hash.hex(), block.prev_header_hash.hex(), block.height, bytes(block_record), None if block_record.sub_epoch_summary_included is None else bytes(block_record.sub_epoch_summary_included), False, block.is_transaction_block(), ), )
async def get_block(self, header_hash) -> Optional[FullBlock]: try: response = await self.fetch("get_block", {"header_hash": header_hash.hex()}) except Exception: return None return FullBlock.from_json_dict(response["block"])
async def get_blocks_by_hash( self, header_hashes: List[bytes32]) -> List[FullBlock]: """ Returns a list of Full Blocks blocks, ordered by the same order in which header_hashes are passed in. Throws an exception if the blocks are not present """ if len(header_hashes) == 0: return [] header_hashes_db = tuple([hh.hex() for hh in header_hashes]) formatted_str = f'SELECT block from full_blocks WHERE header_hash in ({"?," * (len(header_hashes_db) - 1)}?)' cursor = await self.db.execute(formatted_str, header_hashes_db) rows = await cursor.fetchall() await cursor.close() all_blocks: Dict[bytes32, FullBlock] = {} for row in rows: full_block: FullBlock = FullBlock.from_bytes(row[0]) all_blocks[full_block.header_hash] = full_block ret: List[FullBlock] = [] for hh in header_hashes: if hh not in all_blocks: raise ValueError(f"Header hash {hh} not in the blockchain") ret.append(all_blocks[hh]) return ret
async def get_tx_removals_and_additions( self, block: FullBlock, npc_result: Optional[NPCResult] = None ) -> Tuple[List[bytes32], List[Coin]]: if block.is_transaction_block(): if block.transactions_generator is not None: if npc_result is None: block_generator: Optional[ BlockGenerator] = await self.get_block_generator(block) assert block_generator is not None npc_result = get_name_puzzle_conditions( block_generator, self.constants.MAX_BLOCK_COST_CLVM, cost_per_byte=self.constants.COST_PER_BYTE, safe_mode=False, rust_checker=block.height > self.constants.RUST_CONDITION_CHECKER, ) tx_removals, tx_additions = tx_removals_and_additions( npc_result.npc_list) return tx_removals, tx_additions else: return [], [] else: return [], []
async def get_all_block(self, start: uint32, end: uint32) -> List[FullBlock]: response = await self.fetch("get_blocks", { "start": start, "end": end, "exclude_header_hash": True }) return [FullBlock.from_json_dict(r) for r in response["blocks"]]
async def new_block(self, block: FullBlock, tx_additions: List[Coin], tx_removals: List[bytes32]): """ Only called for blocks which are blocks (and thus have rewards and transactions) """ if block.is_transaction_block() is False: return assert block.foliage_transaction_block is not None for coin in tx_additions: record: CoinRecord = CoinRecord( coin, block.height, uint32(0), False, False, block.foliage_transaction_block.timestamp, ) await self._add_coin_record(record, False) included_reward_coins = block.get_included_reward_coins() if block.height == 0: assert len(included_reward_coins) == 0 else: assert len(included_reward_coins) >= 2 for coin in included_reward_coins: reward_coin_r: CoinRecord = CoinRecord( coin, block.height, uint32(0), False, True, block.foliage_transaction_block.timestamp, ) await self._add_coin_record(reward_coin_r, False) total_amount_spent: int = 0 for coin_name in tx_removals: total_amount_spent += await self._set_spent( coin_name, block.height) # Sanity check, already checked in block_body_validation assert sum([a.amount for a in tx_additions]) <= total_amount_spent
async def get_full_blocks_at(self, heights: List[uint32]) -> List[FullBlock]: if len(heights) == 0: return [] heights_db = tuple(heights) formatted_str = f'SELECT block from full_blocks WHERE height in ({"?," * (len(heights_db) - 1)}?)' cursor = await self.db.execute(formatted_str, heights_db) rows = await cursor.fetchall() await cursor.close() return [FullBlock.from_bytes(row[0]) for row in rows]
async def get_full_block(self, header_hash: bytes32) -> Optional[FullBlock]: cached = self.block_cache.get(header_hash) if cached is not None: return cached cursor = await self.db.execute("SELECT block from full_blocks WHERE header_hash=?", (header_hash.hex(),)) row = await cursor.fetchone() await cursor.close() if row is not None: return FullBlock.from_bytes(row[0]) return None
async def new_block(self, block: FullBlock, additions: List[Coin], removals: List[bytes32]): """ Only called for blocks which are blocks (and thus have rewards and transactions) """ if block.is_transaction_block() is False: return assert block.foliage_transaction_block is not None for coin in additions: record: CoinRecord = CoinRecord( coin, block.height, uint32(0), False, False, block.foliage_transaction_block.timestamp, ) await self._add_coin_record(record) included_reward_coins = block.get_included_reward_coins() if block.height == 0: assert len(included_reward_coins) == 0 else: assert len(included_reward_coins) >= 2 for coin in included_reward_coins: reward_coin_r: CoinRecord = CoinRecord( coin, block.height, uint32(0), False, True, block.foliage_transaction_block.timestamp, ) await self._add_coin_record(reward_coin_r) for coin_name in removals: await self._set_spent(coin_name, block.height)
async def get_blocks(self, start: int, end: int, exclude_reorged: bool = False) -> List[FullBlock]: response = await self.fetch( "get_blocks", { "start": start, "end": end, "exclude_header_hash": True, "exclude_reorged": exclude_reorged }) return [ FullBlock.from_json_dict(block) for block in response["blocks"] ]
async def get_full_block(self, header_hash: bytes32) -> Optional[FullBlock]: cached = self.block_cache.get(header_hash) if cached is not None: log.debug(f"cache hit for block {header_hash.hex()}") return cached log.debug(f"cache miss for block {header_hash.hex()}") cursor = await self.db.execute("SELECT block from full_blocks WHERE header_hash=?", (header_hash.hex(),)) row = await cursor.fetchone() await cursor.close() if row is not None: block = FullBlock.from_bytes(row[0]) self.block_cache.put(header_hash, block) return block return None
async def get_tx_removals_and_additions( self, block: FullBlock, npc_result: Optional[NPCResult] = None ) -> Tuple[List[bytes32], List[Coin]]: if block.is_transaction_block(): if block.transactions_generator is not None: if npc_result is None: block_generator: Optional[BlockGenerator] = await self.get_block_generator(block) assert block_generator is not None npc_result = get_name_puzzle_conditions(block_generator, self.constants.MAX_BLOCK_COST_CLVM, False) tx_removals, tx_additions = tx_removals_and_additions(npc_result.npc_list) return tx_removals, tx_additions else: return [], [] else: return [], []
def get_future_reward_coins(block: FullBlock) -> Tuple[Coin, Coin]: pool_amount = calculate_pool_reward(block.height) farmer_amount = calculate_base_farmer_reward(block.height) if block.is_transaction_block(): assert block.transactions_info is not None farmer_amount = uint64(farmer_amount + block.transactions_info.fees) pool_coin: Coin = create_pool_coin( block.height, block.foliage.foliage_block_data.pool_target.puzzle_hash, pool_amount, constants.GENESIS_CHALLENGE) farmer_coin: Coin = create_farmer_coin( block.height, block.foliage.foliage_block_data.farmer_reward_puzzle_hash, farmer_amount, constants.GENESIS_CHALLENGE, ) return pool_coin, farmer_coin
async def get_header_blocks_in_range( self, start: int, stop: int, ) -> Dict[bytes32, HeaderBlock]: formatted_str = f"SELECT header_hash,block from full_blocks WHERE height >= {start} and height <= {stop}" cursor = await self.db.execute(formatted_str) rows = await cursor.fetchall() await cursor.close() ret: Dict[bytes32, HeaderBlock] = {} for row in rows: # Ugly hack, until full_block.get_block_header is rewritten as part of generator runner change await asyncio.sleep(0.001) header_hash = bytes.fromhex(row[0]) full_block: FullBlock = FullBlock.from_bytes(row[1]) ret[header_hash] = full_block.get_block_header() return ret
def persistent_blocks( num_of_blocks: int, db_name: str, seed: bytes = b"", empty_sub_slots=0, normalized_to_identity_cc_eos: bool = False, normalized_to_identity_icc_eos: bool = False, normalized_to_identity_cc_sp: bool = False, normalized_to_identity_cc_ip: bool = False, ): # try loading from disc, if not create new blocks.db file # TODO hash fixtures.py and blocktool.py, add to path, delete if the files changed block_path_dir = Path("~/.chia/blocks").expanduser() file_path = Path(f"~/.chia/blocks/{db_name}").expanduser() if not path.exists(block_path_dir): mkdir(block_path_dir.parent) mkdir(block_path_dir) if file_path.exists(): try: bytes_list = file_path.read_bytes() block_bytes_list: List[bytes] = pickle.loads(bytes_list) blocks: List[FullBlock] = [] for block_bytes in block_bytes_list: blocks.append(FullBlock.from_bytes(block_bytes)) if len(blocks) == num_of_blocks: print(f"\n loaded {file_path} with {len(blocks)} blocks") return blocks except EOFError: print("\n error reading db file") return new_test_db( file_path, num_of_blocks, seed, empty_sub_slots, normalized_to_identity_cc_eos, normalized_to_identity_icc_eos, normalized_to_identity_cc_sp, normalized_to_identity_cc_ip, )
def block_removals_and_additions( block: FullBlock, npc_list: List[NPC]) -> Tuple[List[bytes32], List[Coin]]: """ Returns all coins added and removed in block, including farmer and pool reward. """ removals: List[bytes32] = [] additions: List[Coin] = [] # build removals list if npc_list is None: return [], [] for npc in npc_list: removals.append(npc.coin_name) additions.extend(additions_for_npc(npc_list)) rewards = block.get_included_reward_coins() additions.extend(rewards) return removals, additions
def batch_pre_validate_blocks( constants_dict: Dict, blocks_pickled: Dict[bytes, bytes], full_blocks_pickled: Optional[List[bytes]], header_blocks_pickled: Optional[List[bytes]], prev_transaction_generators: List[Optional[bytes]], npc_results: Dict[uint32, bytes], check_filter: bool, expected_difficulty: List[uint64], expected_sub_slot_iters: List[uint64], ) -> List[bytes]: blocks = {} for k, v in blocks_pickled.items(): blocks[k] = BlockRecord.from_bytes(v) results: List[PreValidationResult] = [] constants: ConsensusConstants = dataclass_from_dict( ConsensusConstants, constants_dict) if full_blocks_pickled is not None and header_blocks_pickled is not None: assert ValueError("Only one should be passed here") if full_blocks_pickled is not None: for i in range(len(full_blocks_pickled)): try: block: FullBlock = FullBlock.from_bytes(full_blocks_pickled[i]) tx_additions: List[Coin] = [] removals: List[bytes32] = [] npc_result: Optional[NPCResult] = None if block.height in npc_results: npc_result = NPCResult.from_bytes( npc_results[block.height]) assert npc_result is not None if npc_result.npc_list is not None: removals, tx_additions = tx_removals_and_additions( npc_result.npc_list) else: removals, tx_additions = [], [] if block.transactions_generator is not None and npc_result is None: prev_generator_bytes = prev_transaction_generators[i] assert prev_generator_bytes is not None assert block.transactions_info is not None block_generator: BlockGenerator = BlockGenerator.from_bytes( prev_generator_bytes) assert block_generator.program == block.transactions_generator npc_result = get_name_puzzle_conditions( block_generator, min(constants.MAX_BLOCK_COST_CLVM, block.transactions_info.cost), True) removals, tx_additions = tx_removals_and_additions( npc_result.npc_list) header_block = get_block_header(block, tx_additions, removals) required_iters, error = validate_finished_header_block( constants, BlockCache(blocks), header_block, check_filter, expected_difficulty[i], expected_sub_slot_iters[i], ) error_int: Optional[uint16] = None if error is not None: error_int = uint16(error.code.value) results.append( PreValidationResult(error_int, required_iters, npc_result)) except Exception: error_stack = traceback.format_exc() log.error(f"Exception: {error_stack}") results.append( PreValidationResult(uint16(Err.UNKNOWN.value), None, None)) elif header_blocks_pickled is not None: for i in range(len(header_blocks_pickled)): try: header_block = HeaderBlock.from_bytes(header_blocks_pickled[i]) required_iters, error = validate_finished_header_block( constants, BlockCache(blocks), header_block, check_filter, expected_difficulty[i], expected_sub_slot_iters[i], ) error_int = None if error is not None: error_int = uint16(error.code.value) results.append( PreValidationResult(error_int, required_iters, None)) except Exception: error_stack = traceback.format_exc() log.error(f"Exception: {error_stack}") results.append( PreValidationResult(uint16(Err.UNKNOWN.value), None, None)) return [bytes(r) for r in results]
def maybe_decompress(self, block_bytes: bytes) -> FullBlock: if self.db_wrapper.db_version == 2: return FullBlock.from_bytes(zstd.decompress(block_bytes)) else: return FullBlock.from_bytes(block_bytes)
def test_json(self): block = bt.create_genesis_block(test_constants, bytes([0] * 32), b"0") dict_block = block.to_json_dict() assert FullBlock.from_json_dict(dict_block) == block
async def run_add_block_benchmark(version: int): verbose: bool = "--verbose" in sys.argv db_wrapper: DBWrapper = await setup_db("block-store-benchmark.db", version) # keep track of benchmark total time all_test_time = 0.0 prev_block = bytes32([0] * 32) prev_ses_hash = bytes32([0] * 32) header_hashes = [] try: block_store = await BlockStore.create(db_wrapper) block_height = 1 timestamp = uint64(1631794488) weight = uint128(10) iters = uint128(123456) sp_index = uint8(0) deficit = uint8(0) sub_slot_iters = uint64(10) required_iters = uint64(100) transaction_block_counter = 0 prev_transaction_block = bytes32([0] * 32) prev_transaction_height = uint32(0) total_time = 0.0 ses_counter = 0 if verbose: print("profiling add_full_block", end="") for height in range(block_height, block_height + NUM_ITERS): is_transaction = transaction_block_counter == 0 fees = uint64(random.randint(0, 150000)) farmer_coin, pool_coin = rewards(uint32(height)) reward_claims_incorporated = [farmer_coin, pool_coin] # TODO: increase fidelity by setting these as well finished_challenge_slot_hashes = None finished_infused_challenge_slot_hashes = None finished_reward_slot_hashes = None sub_epoch_summary_included = None if ses_counter == 0: sub_epoch_summary_included = SubEpochSummary( prev_ses_hash, rand_hash(), uint8(random.randint(0, 255)), # num_blocks_overflow: uint8 None, # new_difficulty: Optional[uint64] None, # new_sub_slot_iters: Optional[uint64] ) has_pool_pk = random.randint(0, 1) proof_of_space = ProofOfSpace( rand_hash(), # challenge rand_g1() if has_pool_pk else None, rand_hash() if not has_pool_pk else None, rand_g1(), # plot_public_key uint8(32), rand_bytes(8 * 32), ) reward_chain_block = RewardChainBlock( weight, uint32(height), iters, sp_index, rand_hash(), # pos_ss_cc_challenge_hash proof_of_space, None if sp_index == 0 else rand_vdf(), rand_g2(), # challenge_chain_sp_signature rand_vdf(), # challenge_chain_ip_vdf rand_vdf() if sp_index != 0 else None, # reward_chain_sp_vdf rand_g2(), # reward_chain_sp_signature rand_vdf(), # reward_chain_ip_vdf rand_vdf() if deficit < 16 else None, is_transaction, ) pool_target = PoolTarget( rand_hash(), # puzzle_hash uint32(0), # max_height ) foliage_block_data = FoliageBlockData( rand_hash(), # unfinished_reward_block_hash pool_target, rand_g2() if has_pool_pk else None, # pool_signature rand_hash(), # farmer_reward_puzzle_hash bytes32([0] * 32), # extension_data ) foliage = Foliage( prev_block, rand_hash(), # reward_block_hash foliage_block_data, rand_g2(), # foliage_block_data_signature rand_hash() if is_transaction else None, # foliage_transaction_block_hash rand_g2() if is_transaction else None, # foliage_transaction_block_signature ) foliage_transaction_block = ( None if not is_transaction else FoliageTransactionBlock( prev_transaction_block, timestamp, rand_hash(), # filter_hash rand_hash(), # additions_root rand_hash(), # removals_root rand_hash(), # transactions_info_hash )) transactions_info = ( None if not is_transaction else TransactionsInfo( rand_hash(), # generator_root rand_hash(), # generator_refs_root rand_g2(), # aggregated_signature fees, uint64(random.randint(0, 12000000000)), # cost reward_claims_incorporated, )) full_block = FullBlock( [], # finished_sub_slots reward_chain_block, rand_vdf_proof() if sp_index > 0 else None, # challenge_chain_sp_proof rand_vdf_proof(), # challenge_chain_ip_proof rand_vdf_proof() if sp_index > 0 else None, # reward_chain_sp_proof rand_vdf_proof(), # reward_chain_ip_proof rand_vdf_proof() if deficit < 4 else None, # infused_challenge_chain_ip_proof foliage, foliage_transaction_block, transactions_info, None if is_transaction else SerializedProgram.from_bytes( clvm_generator), # transactions_generator [], # transactions_generator_ref_list ) header_hash = full_block.header_hash record = BlockRecord( header_hash, prev_block, uint32(height), weight, iters, sp_index, rand_class_group_element(), None if deficit > 3 else rand_class_group_element(), rand_hash(), # reward_infusion_new_challenge rand_hash(), # challenge_block_info_hash sub_slot_iters, rand_hash(), # pool_puzzle_hash rand_hash(), # farmer_puzzle_hash required_iters, deficit, deficit == 16, prev_transaction_height, timestamp if is_transaction else None, prev_transaction_block if prev_transaction_block != bytes32([0] * 32) else None, None if fees == 0 else fees, reward_claims_incorporated, finished_challenge_slot_hashes, finished_infused_challenge_slot_hashes, finished_reward_slot_hashes, sub_epoch_summary_included, ) start = time() await block_store.add_full_block(header_hash, full_block, record, False) await block_store.set_in_chain([(header_hash, )]) header_hashes.append(header_hash) await block_store.set_peak(header_hash) await db_wrapper.db.commit() stop = time() total_time += stop - start # 19 seconds per block timestamp = uint64(timestamp + 19) weight = uint128(weight + 10) iters = uint128(iters + 123456) sp_index = uint8((sp_index + 1) % 64) deficit = uint8((deficit + 3) % 17) ses_counter = (ses_counter + 1) % 384 prev_block = header_hash # every 33 blocks is a transaction block transaction_block_counter = (transaction_block_counter + 1) % 33 if is_transaction: prev_transaction_block = header_hash prev_transaction_height = uint32(height) if ses_counter == 0: prev_ses_hash = header_hash if verbose: print(".", end="") sys.stdout.flush() block_height += NUM_ITERS if verbose: print("") print(f"{total_time:0.4f}s, add_full_block") all_test_time += total_time total_time = 0.0 if verbose: print("profiling get_full_block") random.shuffle(header_hashes) start = time() for h in header_hashes: block = await block_store.get_full_block(h) assert block.header_hash == h stop = time() total_time += stop - start print(f"{total_time:0.4f}s, get_full_block") all_test_time += total_time total_time = 0.0 if verbose: print("profiling get_full_block_bytes") start = time() for h in header_hashes: block = await block_store.get_full_block_bytes(h) assert len(block) > 0 stop = time() total_time += stop - start print(f"{total_time:0.4f}s, get_full_block_bytes") all_test_time += total_time total_time = 0.0 if verbose: print("profiling get_full_blocks_at") start = time() for h in range(1, block_height): blocks = await block_store.get_full_blocks_at([h]) assert len(blocks) == 1 assert blocks[0].height == h stop = time() total_time += stop - start print(f"{total_time:0.4f}s, get_full_blocks_at") all_test_time += total_time total_time = 0.0 if verbose: print("profiling get_block_records_by_hash") start = time() for h in header_hashes: blocks = await block_store.get_block_records_by_hash([h]) assert len(blocks) == 1 assert blocks[0].header_hash == h stop = time() total_time += stop - start print(f"{total_time:0.4f}s, get_block_records_by_hash") all_test_time += total_time total_time = 0.0 if verbose: print("profiling get_blocks_by_hash") start = time() for h in header_hashes: blocks = await block_store.get_blocks_by_hash([h]) assert len(blocks) == 1 assert blocks[0].header_hash == h stop = time() total_time += stop - start print(f"{total_time:0.4f}s, get_blocks_by_hash") all_test_time += total_time total_time = 0.0 if verbose: print("profiling get_block_record") start = time() for h in header_hashes: blocks = await block_store.get_block_record(h) assert blocks.header_hash == h stop = time() total_time += stop - start print(f"{total_time:0.4f}s, get_block_record") all_test_time += total_time total_time = 0.0 if verbose: print("profiling get_block_records_in_range") start = time() for i in range(100): h = random.randint(1, block_height - 100) blocks = await block_store.get_block_records_in_range(h, h + 99) assert len(blocks) == 100 stop = time() total_time += stop - start print(f"{total_time:0.4f}s, get_block_records_in_range") all_test_time += total_time total_time = 0.0 if verbose: print("profiling get_block_records_close_to_peak") start = time() blocks, peak = await block_store.get_block_records_close_to_peak(99) assert len(blocks) == 100 stop = time() total_time += stop - start print(f"{total_time:0.4f}s, get_block_records_close_to_peak") all_test_time += total_time total_time = 0.0 if verbose: print("profiling is_fully_compactified") start = time() for h in header_hashes: compactified = await block_store.is_fully_compactified(h) assert compactified is False stop = time() total_time += stop - start print(f"{total_time:0.4f}s, get_block_record") all_test_time += total_time total_time = 0.0 if verbose: print("profiling get_random_not_compactified") start = time() for i in range(1, 5000): blocks = await block_store.get_random_not_compactified(100) assert len(blocks) == 100 stop = time() total_time += stop - start print(f"{total_time:0.4f}s, get_random_not_compactified") all_test_time += total_time print(f"all tests completed in {all_test_time:0.4f}s") db_size = os.path.getsize(Path("block-store-benchmark.db")) print(f"database size: {db_size/1000000:.3f} MB") finally: await db_wrapper.db.close()
async def receive_block( self, block: FullBlock, pre_validation_result: Optional[PreValidationResult] = None, fork_point_with_peak: Optional[uint32] = None, summaries_to_check: List[SubEpochSummary] = None, # passed only on long sync ) -> Tuple[ReceiveBlockResult, Optional[Err], Optional[uint32]]: """ This method must be called under the blockchain lock Adds a new block into the blockchain, if it's valid and connected to the current blockchain, regardless of whether it is the child of a head, or another block. Returns a header if block is added to head. Returns an error if the block is invalid. Also returns the fork height, in the case of a new peak. """ genesis: bool = block.height == 0 if self.contains_block(block.header_hash): return ReceiveBlockResult.ALREADY_HAVE_BLOCK, None, None if not self.contains_block(block.prev_header_hash) and not genesis: return ( ReceiveBlockResult.DISCONNECTED_BLOCK, Err.INVALID_PREV_BLOCK_HASH, None, ) if not genesis and (self.block_record(block.prev_header_hash).height + 1) != block.height: return ReceiveBlockResult.INVALID_BLOCK, Err.INVALID_HEIGHT, None npc_result: Optional[NPCResult] = None if pre_validation_result is None: if block.height == 0: prev_b: Optional[BlockRecord] = None else: prev_b = self.block_record(block.prev_header_hash) sub_slot_iters, difficulty = get_next_sub_slot_iters_and_difficulty( self.constants, len(block.finished_sub_slots) > 0, prev_b, self ) if block.is_transaction_block(): if block.transactions_generator is not None: try: block_generator: Optional[BlockGenerator] = await self.get_block_generator(block) except ValueError: return ReceiveBlockResult.INVALID_BLOCK, Err.GENERATOR_REF_HAS_NO_GENERATOR, None assert block_generator is not None and block.transactions_info is not None npc_result = get_name_puzzle_conditions( block_generator, min(self.constants.MAX_BLOCK_COST_CLVM, block.transactions_info.cost), False ) removals, tx_additions = tx_removals_and_additions(npc_result.npc_list) else: removals, tx_additions = [], [] header_block = get_block_header(block, tx_additions, removals) else: npc_result = None header_block = get_block_header(block, [], []) required_iters, error = validate_finished_header_block( self.constants, self, header_block, False, difficulty, sub_slot_iters, ) if error is not None: return ReceiveBlockResult.INVALID_BLOCK, error.code, None else: npc_result = pre_validation_result.npc_result required_iters = pre_validation_result.required_iters assert pre_validation_result.error is None assert required_iters is not None error_code, _ = await validate_block_body( self.constants, self, self.block_store, self.coin_store, self.get_peak(), block, block.height, npc_result, fork_point_with_peak, self.get_block_generator, ) if error_code is not None: return ReceiveBlockResult.INVALID_BLOCK, error_code, None block_record = block_to_block_record( self.constants, self, required_iters, block, None, ) # Always add the block to the database async with self.block_store.db_wrapper.lock: try: # Perform the DB operations to update the state, and rollback if something goes wrong await self.block_store.db_wrapper.begin_transaction() await self.block_store.add_full_block(block, block_record) fork_height, peak_height, records = await self._reconsider_peak( block_record, genesis, fork_point_with_peak, npc_result ) await self.block_store.db_wrapper.commit_transaction() # Then update the memory cache. It is important that this task is not cancelled and does not throw self.add_block_record(block_record) for fetched_block_record in records: self.__height_to_hash[fetched_block_record.height] = fetched_block_record.header_hash if fetched_block_record.sub_epoch_summary_included is not None: self.__sub_epoch_summaries[ fetched_block_record.height ] = fetched_block_record.sub_epoch_summary_included if peak_height is not None: self._peak_height = peak_height self.block_store.cache_block(block) except BaseException: await self.block_store.db_wrapper.rollback_transaction() raise if fork_height is not None: return ReceiveBlockResult.NEW_PEAK, None, fork_height else: return ReceiveBlockResult.ADDED_AS_ORPHAN, None, None
async def receive_block( self, block: FullBlock, pre_validation_result: Optional[PreValidationResult] = None, fork_point_with_peak: Optional[uint32] = None, ) -> Tuple[ReceiveBlockResult, Optional[Err], Optional[uint32]]: """ This method must be called under the blockchain lock Adds a new block into the blockchain, if it's valid and connected to the current blockchain, regardless of whether it is the child of a head, or another block. Returns a header if block is added to head. Returns an error if the block is invalid. Also returns the fork height, in the case of a new peak. """ genesis: bool = block.height == 0 if self.contains_block(block.header_hash): return ReceiveBlockResult.ALREADY_HAVE_BLOCK, None, None if not self.contains_block(block.prev_header_hash) and not genesis: return ( ReceiveBlockResult.DISCONNECTED_BLOCK, Err.INVALID_PREV_BLOCK_HASH, None, ) if not genesis and (self.block_record(block.prev_header_hash).height + 1) != block.height: return ReceiveBlockResult.INVALID_BLOCK, Err.INVALID_HEIGHT, None if pre_validation_result is None: if block.height == 0: prev_b: Optional[BlockRecord] = None else: prev_b = self.block_record(block.prev_header_hash) sub_slot_iters, difficulty = get_next_sub_slot_iters_and_difficulty( self.constants, len(block.finished_sub_slots) > 0, prev_b, self ) required_iters, error = validate_finished_header_block( self.constants, self, block.get_block_header(), False, difficulty, sub_slot_iters, ) if error is not None: return ReceiveBlockResult.INVALID_BLOCK, error.code, None else: required_iters = pre_validation_result.required_iters assert pre_validation_result.error is None assert required_iters is not None error_code, _ = await validate_block_body( self.constants, self, self.block_store, self.coin_store, self.get_peak(), block, block.height, pre_validation_result.cost_result if pre_validation_result is not None else None, fork_point_with_peak, ) if error_code is not None: return ReceiveBlockResult.INVALID_BLOCK, error_code, None block_record = block_to_block_record( self.constants, self, required_iters, block, None, ) # Always add the block to the database async with self.block_store.db_wrapper.lock: try: await self.block_store.db_wrapper.begin_transaction() await self.block_store.add_full_block(block, block_record) fork_height, peak_height, records = await self._reconsider_peak( block_record, genesis, fork_point_with_peak ) await self.block_store.db_wrapper.commit_transaction() self.add_block_record(block_record) for fetched_block_record in records: self.__height_to_hash[fetched_block_record.height] = fetched_block_record.header_hash if fetched_block_record.sub_epoch_summary_included is not None: self.__sub_epoch_summaries[ fetched_block_record.height ] = fetched_block_record.sub_epoch_summary_included if peak_height is not None: self._peak_height = peak_height except BaseException: await self.block_store.db_wrapper.rollback_transaction() raise if fork_height is not None: return ReceiveBlockResult.NEW_PEAK, None, fork_height else: return ReceiveBlockResult.ADDED_AS_ORPHAN, None, None
ConditionOpcode.ASSERT_SECONDS_ABSOLUTE[0]: 13, ConditionOpcode.ASSERT_HEIGHT_RELATIVE[0]: 14, ConditionOpcode.ASSERT_HEIGHT_ABSOLUTE[0]: 15, } c = sqlite3.connect(sys.argv[1]) rows = c.execute( "SELECT header_hash, height, block FROM full_blocks ORDER BY height") height_to_hash: List[bytes] = [] for r in rows: hh = bytes.fromhex(r[0]) height = r[1] block = FullBlock.from_bytes(r[2]) if len(height_to_hash) <= height: assert len(height_to_hash) == height height_to_hash.append(hh) else: height_to_hash[height] = hh if height > 0: prev_hh = block.prev_header_hash h = height - 1 while height_to_hash[h] != prev_hh: height_to_hash[h] = prev_hh ref = c.execute( "SELECT block FROM full_blocks WHERE header_hash=?", (prev_hh.hex(), ))
def unfinished_block_to_full_block( unfinished_block: UnfinishedBlock, cc_ip_vdf: VDFInfo, cc_ip_proof: VDFProof, rc_ip_vdf: VDFInfo, rc_ip_proof: VDFProof, icc_ip_vdf: Optional[VDFInfo], icc_ip_proof: Optional[VDFProof], finished_sub_slots: List[EndOfSubSlotBundle], prev_block: Optional[BlockRecord], blocks: BlockchainInterface, total_iters_sp: uint128, difficulty: uint64, ) -> FullBlock: """ Converts an unfinished block to a finished block. Includes all the infusion point VDFs as well as tweaking other properties (height, weight, sub-slots, etc) Args: unfinished_block: the unfinished block to finish cc_ip_vdf: the challenge chain vdf info at the infusion point cc_ip_proof: the challenge chain proof rc_ip_vdf: the reward chain vdf info at the infusion point rc_ip_proof: the reward chain proof icc_ip_vdf: the infused challenge chain vdf info at the infusion point icc_ip_proof: the infused challenge chain proof finished_sub_slots: finished sub slots from the prev block to the infusion point prev_block: prev block from the infusion point blocks: dictionary from header hash to SBR of all included SBR total_iters_sp: total iters at the signage point difficulty: difficulty at the infusion point """ # Replace things that need to be replaced, since foliage blocks did not necessarily have the latest information if prev_block is None: is_transaction_block = True new_weight = uint128(difficulty) new_height = uint32(0) new_foliage = unfinished_block.foliage new_foliage_transaction_block = unfinished_block.foliage_transaction_block new_tx_info = unfinished_block.transactions_info new_generator = unfinished_block.transactions_generator new_generator_ref_list = unfinished_block.transactions_generator_ref_list else: is_transaction_block, _ = get_prev_transaction_block(prev_block, blocks, total_iters_sp) new_weight = uint128(prev_block.weight + difficulty) new_height = uint32(prev_block.height + 1) if is_transaction_block: new_fbh = unfinished_block.foliage.foliage_transaction_block_hash new_fbs = unfinished_block.foliage.foliage_transaction_block_signature new_foliage_transaction_block = unfinished_block.foliage_transaction_block new_tx_info = unfinished_block.transactions_info new_generator = unfinished_block.transactions_generator new_generator_ref_list = unfinished_block.transactions_generator_ref_list else: new_fbh = None new_fbs = None new_foliage_transaction_block = None new_tx_info = None new_generator = None new_generator_ref_list = [] assert (new_fbh is None) == (new_fbs is None) new_foliage = replace( unfinished_block.foliage, prev_block_hash=prev_block.header_hash, foliage_transaction_block_hash=new_fbh, foliage_transaction_block_signature=new_fbs, ) ret = FullBlock( finished_sub_slots, RewardChainBlock( new_weight, new_height, unfinished_block.reward_chain_block.total_iters, unfinished_block.reward_chain_block.signage_point_index, unfinished_block.reward_chain_block.pos_ss_cc_challenge_hash, unfinished_block.reward_chain_block.proof_of_space, unfinished_block.reward_chain_block.challenge_chain_sp_vdf, unfinished_block.reward_chain_block.challenge_chain_sp_signature, cc_ip_vdf, unfinished_block.reward_chain_block.reward_chain_sp_vdf, unfinished_block.reward_chain_block.reward_chain_sp_signature, rc_ip_vdf, icc_ip_vdf, is_transaction_block, ), unfinished_block.challenge_chain_sp_proof, cc_ip_proof, unfinished_block.reward_chain_sp_proof, rc_ip_proof, icc_ip_proof, new_foliage, new_foliage_transaction_block, new_tx_info, new_generator, new_generator_ref_list, ) return recursive_replace( ret, "foliage.reward_block_hash", ret.reward_chain_block.get_hash(), )