async def get_block_records_close_to_peak( self, blocks_n: int ) -> Tuple[Dict[bytes32, BlockRecord], Optional[bytes32]]: """ Returns a dictionary with all blocks that have height >= peak height - blocks_n, as well as the peak header hash. """ peak = await self.get_peak() if peak is None: return {}, None ret: Dict[bytes32, BlockRecord] = {} if self.db_wrapper.db_version == 2: async with self.db.execute( "SELECT header_hash, block_record FROM full_blocks WHERE height >= ?", (peak[1] - blocks_n,), ) as cursor: for row in await cursor.fetchall(): header_hash = bytes32(row[0]) ret[header_hash] = BlockRecord.from_bytes(row[1]) else: formatted_str = f"SELECT header_hash, block from block_records WHERE height >= {peak[1] - blocks_n}" async with self.db.execute(formatted_str) as cursor: for row in await cursor.fetchall(): header_hash = bytes32(self.maybe_from_hex(row[0])) ret[header_hash] = BlockRecord.from_bytes(row[1]) return ret, peak[0]
async def get_block_records_by_hash(self, header_hashes: List[bytes32]): """ Returns a list of Block Records, ordered by the same order in which header_hashes are passed in. Throws an exception if the blocks are not present """ if len(header_hashes) == 0: return [] all_blocks: Dict[bytes32, BlockRecord] = {} if self.db_wrapper.db_version == 2: async with self.db.execute( "SELECT header_hash,block_record FROM full_blocks " f'WHERE header_hash in ({"?," * (len(header_hashes) - 1)}?)', tuple(header_hashes), ) as cursor: for row in await cursor.fetchall(): header_hash = bytes32(row[0]) all_blocks[header_hash] = BlockRecord.from_bytes(row[1]) else: formatted_str = f'SELECT block from block_records WHERE header_hash in ({"?," * (len(header_hashes) - 1)}?)' async with self.db.execute(formatted_str, tuple([hh.hex() for hh in header_hashes])) as cursor: for row in await cursor.fetchall(): block_rec: BlockRecord = BlockRecord.from_bytes(row[0]) all_blocks[block_rec.header_hash] = block_rec ret: List[BlockRecord] = [] for hh in header_hashes: if hh not in all_blocks: raise ValueError(f"Header hash {hh} not in the blockchain") ret.append(all_blocks[hh]) return ret
async def get_block_records_in_range( self, start: int, stop: int, ) -> Dict[bytes32, BlockRecord]: """ Returns a dictionary with all blocks in range between start and stop if present. """ ret: Dict[bytes32, BlockRecord] = {} if self.db_wrapper.db_version == 2: async with self.db.execute( "SELECT header_hash, block_record FROM full_blocks WHERE height >= ? AND height <= ?", (start, stop), ) as cursor: for row in await cursor.fetchall(): header_hash = bytes32(row[0]) ret[header_hash] = BlockRecord.from_bytes(row[1]) else: formatted_str = f"SELECT header_hash, block from block_records WHERE height >= {start} and height <= {stop}" async with await self.db.execute(formatted_str) as cursor: for row in await cursor.fetchall(): header_hash = bytes32(self.maybe_from_hex(row[0])) ret[header_hash] = BlockRecord.from_bytes(row[1]) return ret
async def get_block_records_close_to_peak( self, blocks_n: int ) -> Tuple[Dict[bytes32, BlockRecord], Optional[bytes32]]: """ Returns a dictionary with all blocks, as well as the header hash of the peak, if present. """ res = await self.db.execute( "SELECT header_hash, height from block_records WHERE is_peak = 1") row = await res.fetchone() await res.close() if row is None: return {}, None header_hash_bytes, peak_height = row peak: bytes32 = bytes32(bytes.fromhex(header_hash_bytes)) formatted_str = f"SELECT header_hash, block from block_records WHERE height >= {peak_height - blocks_n}" cursor = await self.db.execute(formatted_str) rows = await cursor.fetchall() await cursor.close() ret: Dict[bytes32, BlockRecord] = {} for row in rows: header_hash_bytes, block_record_bytes = row header_hash = bytes.fromhex(header_hash_bytes) ret[header_hash] = BlockRecord.from_bytes(block_record_bytes) return ret, peak
async def get_block_record_by_height(self, height) -> Optional[BlockRecord]: try: response = await self.fetch("get_block_record_by_height", {"height": height}) except Exception: return None return BlockRecord.from_json_dict(response["block_record"])
async def get_block_record(self, header_hash) -> Optional[BlockRecord]: try: response = await self.fetch("get_block_record", {"header_hash": header_hash.hex()}) if response["block_record"] is None: return None except Exception: return None return BlockRecord.from_json_dict(response["block_record"])
async def get_block_record(self, header_hash: bytes32) -> Optional[BlockRecord]: cursor = await self.db.execute( "SELECT block from block_records WHERE header_hash=?", (header_hash.hex(),), ) row = await cursor.fetchone() await cursor.close() if row is not None: return BlockRecord.from_bytes(row[0]) return None
async def get_block_records_at(self, heights: List[uint32]) -> List[BlockRecord]: if len(heights) == 0: return [] heights_db = tuple(heights) formatted_str = ( f'SELECT block from block_records WHERE height in ({"?," * (len(heights_db) - 1)}?) ORDER BY height ASC;' ) cursor = await self.db.execute(formatted_str, heights_db) rows = await cursor.fetchall() await cursor.close() return [BlockRecord.from_bytes(row[0]) for row in rows]
async def get_block_record(self, header_hash: bytes32) -> Optional[BlockRecord]: if self.db_wrapper.db_version == 2: async with self.db.execute( "SELECT block_record FROM full_blocks WHERE header_hash=?", (header_hash,), ) as cursor: row = await cursor.fetchone() if row is not None: return BlockRecord.from_bytes(row[0]) else: async with self.db.execute( "SELECT block from block_records WHERE header_hash=?", (header_hash.hex(),), ) as cursor: row = await cursor.fetchone() if row is not None: return BlockRecord.from_bytes(row[0]) return None
def batch_pre_validate_blocks( constants_dict: Dict, blocks_pickled: Dict[bytes, bytes], header_blocks_pickled: List[bytes], transaction_generators: List[Optional[bytes]], check_filter: bool, expected_difficulty: List[uint64], expected_sub_slot_iters: List[uint64], validate_transactions: bool, ) -> List[bytes]: assert len(header_blocks_pickled) == len(transaction_generators) blocks = {} for k, v in blocks_pickled.items(): blocks[k] = BlockRecord.from_bytes(v) results: List[PreValidationResult] = [] constants: ConsensusConstants = dataclass_from_dict( ConsensusConstants, constants_dict) for i in range(len(header_blocks_pickled)): try: header_block: HeaderBlock = HeaderBlock.from_bytes( header_blocks_pickled[i]) generator: Optional[bytes] = transaction_generators[i] required_iters, error = validate_finished_header_block( constants, BlockCache(blocks), header_block, check_filter, expected_difficulty[i], expected_sub_slot_iters[i], ) cost_result: Optional[CostResult] = None error_int: Optional[uint16] = None if error is not None: error_int = uint16(error.code.value) if constants_dict["NETWORK_TYPE"] == NetworkType.MAINNET.value: cost_result = None else: if not error and generator is not None and validate_transactions: cost_result = calculate_cost_of_program( SerializedProgram.from_bytes(generator), constants.CLVM_COST_RATIO_CONSTANT) results.append( PreValidationResult(error_int, required_iters, cost_result)) except Exception: error_stack = traceback.format_exc() log.error(f"Exception: {error_stack}") results.append( PreValidationResult(uint16(Err.UNKNOWN.value), None, None)) return [bytes(r) for r in results]
async def get_block_records( self, ) -> Tuple[Dict[bytes32, BlockRecord], Optional[bytes32]]: """ Returns a dictionary with all blocks, as well as the header hash of the peak, if present. """ cursor = await self.db.execute("SELECT * from block_records") rows = await cursor.fetchall() await cursor.close() ret: Dict[bytes32, BlockRecord] = {} peak: Optional[bytes32] = None for row in rows: header_hash = bytes.fromhex(row[0]) ret[header_hash] = BlockRecord.from_bytes(row[3]) if row[5]: assert peak is None # Sanity check, only one peak peak = header_hash return ret, peak
async def get_block_records_in_range( self, start: int, stop: int, ) -> Dict[bytes32, BlockRecord]: """ Returns a dictionary with all blocks in range between start and stop if present. """ formatted_str = f"SELECT header_hash, block from block_records WHERE height >= {start} and height <= {stop}" cursor = await self.db.execute(formatted_str) rows = await cursor.fetchall() await cursor.close() ret: Dict[bytes32, BlockRecord] = {} for row in rows: header_hash = bytes.fromhex(row[0]) ret[header_hash] = BlockRecord.from_bytes(row[1]) return ret
async def get_block_records_in_range( self, start: int, stop: int, ) -> Dict[bytes32, BlockRecord]: """ Returns a dictionary with all blocks, as well as the header hash of the peak, if present. """ formatted_str = f"SELECT header_hash, block from block_records WHERE height >= {start} and height <= {stop}" cursor = await self.db.execute(formatted_str) rows = await cursor.fetchall() await cursor.close() ret: Dict[bytes32, BlockRecord] = {} for row in rows: header_hash_bytes, block_record_bytes = row header_hash = bytes.fromhex(header_hash_bytes) ret[header_hash] = BlockRecord.from_bytes(block_record_bytes) return ret
async def get_block_records_by_hash(self, header_hashes: List[bytes32]): """ Returns a list of Block Records, ordered by the same order in which header_hashes are passed in. Throws an exception if the blocks are not present """ if len(header_hashes) == 0: return [] header_hashes_db = tuple([hh.hex() for hh in header_hashes]) formatted_str = f'SELECT block from block_records WHERE header_hash in ({"?," * (len(header_hashes_db) - 1)}?)' cursor = await self.db.execute(formatted_str, header_hashes_db) rows = await cursor.fetchall() await cursor.close() all_blocks: Dict[bytes32, BlockRecord] = {} for row in rows: block_rec: BlockRecord = BlockRecord.from_bytes(row[0]) all_blocks[block_rec.header_hash] = block_rec ret: List[BlockRecord] = [] for hh in header_hashes: if hh not in all_blocks: raise ValueError(f"Header hash {hh} not in the blockchain") ret.append(all_blocks[hh]) return ret
async def get_block_records_close_to_peak( self, blocks_n: int ) -> Tuple[Dict[bytes32, BlockRecord], Optional[bytes32]]: """ Returns a dictionary with all blocks that have height >= peak height - blocks_n, as well as the peak header hash. """ res = await self.db.execute("SELECT * from block_records WHERE is_peak = 1") peak_row = await res.fetchone() await res.close() if peak_row is None: return {}, None formatted_str = f"SELECT header_hash, block from block_records WHERE height >= {peak_row[2] - blocks_n}" cursor = await self.db.execute(formatted_str) rows = await cursor.fetchall() await cursor.close() ret: Dict[bytes32, BlockRecord] = {} for row in rows: header_hash = bytes.fromhex(row[0]) ret[header_hash] = BlockRecord.from_bytes(row[1]) return ret, bytes.fromhex(peak_row[0])
async def run_add_block_benchmark(version: int): verbose: bool = "--verbose" in sys.argv db_wrapper: DBWrapper = await setup_db("block-store-benchmark.db", version) # keep track of benchmark total time all_test_time = 0.0 prev_block = bytes32([0] * 32) prev_ses_hash = bytes32([0] * 32) header_hashes = [] try: block_store = await BlockStore.create(db_wrapper) block_height = 1 timestamp = uint64(1631794488) weight = uint128(10) iters = uint128(123456) sp_index = uint8(0) deficit = uint8(0) sub_slot_iters = uint64(10) required_iters = uint64(100) transaction_block_counter = 0 prev_transaction_block = bytes32([0] * 32) prev_transaction_height = uint32(0) total_time = 0.0 ses_counter = 0 if verbose: print("profiling add_full_block", end="") for height in range(block_height, block_height + NUM_ITERS): is_transaction = transaction_block_counter == 0 fees = uint64(random.randint(0, 150000)) farmer_coin, pool_coin = rewards(uint32(height)) reward_claims_incorporated = [farmer_coin, pool_coin] # TODO: increase fidelity by setting these as well finished_challenge_slot_hashes = None finished_infused_challenge_slot_hashes = None finished_reward_slot_hashes = None sub_epoch_summary_included = None if ses_counter == 0: sub_epoch_summary_included = SubEpochSummary( prev_ses_hash, rand_hash(), uint8(random.randint(0, 255)), # num_blocks_overflow: uint8 None, # new_difficulty: Optional[uint64] None, # new_sub_slot_iters: Optional[uint64] ) has_pool_pk = random.randint(0, 1) proof_of_space = ProofOfSpace( rand_hash(), # challenge rand_g1() if has_pool_pk else None, rand_hash() if not has_pool_pk else None, rand_g1(), # plot_public_key uint8(32), rand_bytes(8 * 32), ) reward_chain_block = RewardChainBlock( weight, uint32(height), iters, sp_index, rand_hash(), # pos_ss_cc_challenge_hash proof_of_space, None if sp_index == 0 else rand_vdf(), rand_g2(), # challenge_chain_sp_signature rand_vdf(), # challenge_chain_ip_vdf rand_vdf() if sp_index != 0 else None, # reward_chain_sp_vdf rand_g2(), # reward_chain_sp_signature rand_vdf(), # reward_chain_ip_vdf rand_vdf() if deficit < 16 else None, is_transaction, ) pool_target = PoolTarget( rand_hash(), # puzzle_hash uint32(0), # max_height ) foliage_block_data = FoliageBlockData( rand_hash(), # unfinished_reward_block_hash pool_target, rand_g2() if has_pool_pk else None, # pool_signature rand_hash(), # farmer_reward_puzzle_hash bytes32([0] * 32), # extension_data ) foliage = Foliage( prev_block, rand_hash(), # reward_block_hash foliage_block_data, rand_g2(), # foliage_block_data_signature rand_hash() if is_transaction else None, # foliage_transaction_block_hash rand_g2() if is_transaction else None, # foliage_transaction_block_signature ) foliage_transaction_block = ( None if not is_transaction else FoliageTransactionBlock( prev_transaction_block, timestamp, rand_hash(), # filter_hash rand_hash(), # additions_root rand_hash(), # removals_root rand_hash(), # transactions_info_hash )) transactions_info = ( None if not is_transaction else TransactionsInfo( rand_hash(), # generator_root rand_hash(), # generator_refs_root rand_g2(), # aggregated_signature fees, uint64(random.randint(0, 12000000000)), # cost reward_claims_incorporated, )) full_block = FullBlock( [], # finished_sub_slots reward_chain_block, rand_vdf_proof() if sp_index > 0 else None, # challenge_chain_sp_proof rand_vdf_proof(), # challenge_chain_ip_proof rand_vdf_proof() if sp_index > 0 else None, # reward_chain_sp_proof rand_vdf_proof(), # reward_chain_ip_proof rand_vdf_proof() if deficit < 4 else None, # infused_challenge_chain_ip_proof foliage, foliage_transaction_block, transactions_info, None if is_transaction else SerializedProgram.from_bytes( clvm_generator), # transactions_generator [], # transactions_generator_ref_list ) header_hash = full_block.header_hash record = BlockRecord( header_hash, prev_block, uint32(height), weight, iters, sp_index, rand_class_group_element(), None if deficit > 3 else rand_class_group_element(), rand_hash(), # reward_infusion_new_challenge rand_hash(), # challenge_block_info_hash sub_slot_iters, rand_hash(), # pool_puzzle_hash rand_hash(), # farmer_puzzle_hash required_iters, deficit, deficit == 16, prev_transaction_height, timestamp if is_transaction else None, prev_transaction_block if prev_transaction_block != bytes32([0] * 32) else None, None if fees == 0 else fees, reward_claims_incorporated, finished_challenge_slot_hashes, finished_infused_challenge_slot_hashes, finished_reward_slot_hashes, sub_epoch_summary_included, ) start = time() await block_store.add_full_block(header_hash, full_block, record, False) await block_store.set_in_chain([(header_hash, )]) header_hashes.append(header_hash) await block_store.set_peak(header_hash) await db_wrapper.db.commit() stop = time() total_time += stop - start # 19 seconds per block timestamp = uint64(timestamp + 19) weight = uint128(weight + 10) iters = uint128(iters + 123456) sp_index = uint8((sp_index + 1) % 64) deficit = uint8((deficit + 3) % 17) ses_counter = (ses_counter + 1) % 384 prev_block = header_hash # every 33 blocks is a transaction block transaction_block_counter = (transaction_block_counter + 1) % 33 if is_transaction: prev_transaction_block = header_hash prev_transaction_height = uint32(height) if ses_counter == 0: prev_ses_hash = header_hash if verbose: print(".", end="") sys.stdout.flush() block_height += NUM_ITERS if verbose: print("") print(f"{total_time:0.4f}s, add_full_block") all_test_time += total_time total_time = 0.0 if verbose: print("profiling get_full_block") random.shuffle(header_hashes) start = time() for h in header_hashes: block = await block_store.get_full_block(h) assert block.header_hash == h stop = time() total_time += stop - start print(f"{total_time:0.4f}s, get_full_block") all_test_time += total_time total_time = 0.0 if verbose: print("profiling get_full_block_bytes") start = time() for h in header_hashes: block = await block_store.get_full_block_bytes(h) assert len(block) > 0 stop = time() total_time += stop - start print(f"{total_time:0.4f}s, get_full_block_bytes") all_test_time += total_time total_time = 0.0 if verbose: print("profiling get_full_blocks_at") start = time() for h in range(1, block_height): blocks = await block_store.get_full_blocks_at([h]) assert len(blocks) == 1 assert blocks[0].height == h stop = time() total_time += stop - start print(f"{total_time:0.4f}s, get_full_blocks_at") all_test_time += total_time total_time = 0.0 if verbose: print("profiling get_block_records_by_hash") start = time() for h in header_hashes: blocks = await block_store.get_block_records_by_hash([h]) assert len(blocks) == 1 assert blocks[0].header_hash == h stop = time() total_time += stop - start print(f"{total_time:0.4f}s, get_block_records_by_hash") all_test_time += total_time total_time = 0.0 if verbose: print("profiling get_blocks_by_hash") start = time() for h in header_hashes: blocks = await block_store.get_blocks_by_hash([h]) assert len(blocks) == 1 assert blocks[0].header_hash == h stop = time() total_time += stop - start print(f"{total_time:0.4f}s, get_blocks_by_hash") all_test_time += total_time total_time = 0.0 if verbose: print("profiling get_block_record") start = time() for h in header_hashes: blocks = await block_store.get_block_record(h) assert blocks.header_hash == h stop = time() total_time += stop - start print(f"{total_time:0.4f}s, get_block_record") all_test_time += total_time total_time = 0.0 if verbose: print("profiling get_block_records_in_range") start = time() for i in range(100): h = random.randint(1, block_height - 100) blocks = await block_store.get_block_records_in_range(h, h + 99) assert len(blocks) == 100 stop = time() total_time += stop - start print(f"{total_time:0.4f}s, get_block_records_in_range") all_test_time += total_time total_time = 0.0 if verbose: print("profiling get_block_records_close_to_peak") start = time() blocks, peak = await block_store.get_block_records_close_to_peak(99) assert len(blocks) == 100 stop = time() total_time += stop - start print(f"{total_time:0.4f}s, get_block_records_close_to_peak") all_test_time += total_time total_time = 0.0 if verbose: print("profiling is_fully_compactified") start = time() for h in header_hashes: compactified = await block_store.is_fully_compactified(h) assert compactified is False stop = time() total_time += stop - start print(f"{total_time:0.4f}s, get_block_record") all_test_time += total_time total_time = 0.0 if verbose: print("profiling get_random_not_compactified") start = time() for i in range(1, 5000): blocks = await block_store.get_random_not_compactified(100) assert len(blocks) == 100 stop = time() total_time += stop - start print(f"{total_time:0.4f}s, get_random_not_compactified") all_test_time += total_time print(f"all tests completed in {all_test_time:0.4f}s") db_size = os.path.getsize(Path("block-store-benchmark.db")) print(f"database size: {db_size/1000000:.3f} MB") finally: await db_wrapper.db.close()
def header_block_to_sub_block_record( constants: ConsensusConstants, required_iters: uint64, block: Union[FullBlock, HeaderBlock], sub_slot_iters: uint64, overflow: bool, deficit: uint8, prev_transaction_block_height: uint32, ses: Optional[SubEpochSummary], ): reward_claims_incorporated = ( block.transactions_info.reward_claims_incorporated if block.transactions_info is not None else None ) cbi = ChallengeBlockInfo( block.reward_chain_block.proof_of_space, block.reward_chain_block.challenge_chain_sp_vdf, block.reward_chain_block.challenge_chain_sp_signature, block.reward_chain_block.challenge_chain_ip_vdf, ) if block.reward_chain_block.infused_challenge_chain_ip_vdf is not None: icc_output: Optional[ClassgroupElement] = block.reward_chain_block.infused_challenge_chain_ip_vdf.output else: icc_output = None if len(block.finished_sub_slots) > 0: finished_challenge_slot_hashes: Optional[List[bytes32]] = [ sub_slot.challenge_chain.get_hash() for sub_slot in block.finished_sub_slots ] finished_reward_slot_hashes: Optional[List[bytes32]] = [ sub_slot.reward_chain.get_hash() for sub_slot in block.finished_sub_slots ] finished_infused_challenge_slot_hashes: Optional[List[bytes32]] = [ sub_slot.infused_challenge_chain.get_hash() for sub_slot in block.finished_sub_slots if sub_slot.infused_challenge_chain is not None ] elif block.height == 0: finished_challenge_slot_hashes = [constants.GENESIS_CHALLENGE] finished_reward_slot_hashes = [constants.GENESIS_CHALLENGE] finished_infused_challenge_slot_hashes = None else: finished_challenge_slot_hashes = None finished_reward_slot_hashes = None finished_infused_challenge_slot_hashes = None prev_transaction_block_hash = ( block.foliage_transaction_block.prev_transaction_block_hash if block.foliage_transaction_block is not None else None ) timestamp = block.foliage_transaction_block.timestamp if block.foliage_transaction_block is not None else None fees = block.transactions_info.fees if block.transactions_info is not None else None return BlockRecord( block.header_hash, block.prev_header_hash, block.height, block.weight, block.total_iters, block.reward_chain_block.signage_point_index, block.reward_chain_block.challenge_chain_ip_vdf.output, icc_output, block.reward_chain_block.get_hash(), cbi.get_hash(), sub_slot_iters, block.foliage.foliage_block_data.pool_target.puzzle_hash, block.foliage.foliage_block_data.farmer_reward_puzzle_hash, required_iters, deficit, overflow, prev_transaction_block_height, timestamp, prev_transaction_block_hash, fees, reward_claims_incorporated, finished_challenge_slot_hashes, finished_infused_challenge_slot_hashes, finished_reward_slot_hashes, ses, )
async def get_blockchain_state(self) -> Dict: response = await self.fetch("get_blockchain_state", {}) if response["blockchain_state"]["peak"] is not None: response["blockchain_state"]["peak"] = BlockRecord.from_json_dict( response["blockchain_state"]["peak"]) return response["blockchain_state"]
def new_peak( self, peak: BlockRecord, peak_full_block: FullBlock, sp_sub_slot: Optional[ EndOfSubSlotBundle], # None if not overflow, or in first/second slot ip_sub_slot: Optional[EndOfSubSlotBundle], # None if in first slot reorg: bool, blocks: BlockchainInterface, ) -> Tuple[Optional[EndOfSubSlotBundle], List[Tuple[uint8, SignagePoint]], List[timelord_protocol.NewInfusionPointVDF]]: """ If the peak is an overflow block, must provide two sub-slots: one for the current sub-slot and one for the prev sub-slot (since we still might get more blocks with an sp in the previous sub-slot) Results in either one or two sub-slots in finished_sub_slots. """ assert len(self.finished_sub_slots) >= 1 if ip_sub_slot is None: # We are still in the first sub-slot, no new sub slots ey self.initialize_genesis_sub_slot() else: # This is not the first sub-slot in the chain sp_sub_slot_sps: List[Optional[SignagePoint]] = [ None ] * self.constants.NUM_SPS_SUB_SLOT ip_sub_slot_sps: List[Optional[SignagePoint]] = [ None ] * self.constants.NUM_SPS_SUB_SLOT if not reorg: # If it's not a reorg, we can keep signage points that we had before, in the cache for index, (sub_slot, sps, total_iters) in enumerate(self.finished_sub_slots): if sub_slot is None: continue if sub_slot == sp_sub_slot: sp_sub_slot_sps = sps if sub_slot == ip_sub_slot: ip_sub_slot_sps = sps self.clear_slots() prev_sub_slot_total_iters = peak.sp_sub_slot_total_iters( self.constants) if sp_sub_slot is not None or prev_sub_slot_total_iters == 0: assert peak.overflow or prev_sub_slot_total_iters self.finished_sub_slots.append( (sp_sub_slot, sp_sub_slot_sps, prev_sub_slot_total_iters)) ip_sub_slot_total_iters = peak.ip_sub_slot_total_iters( self.constants) self.finished_sub_slots.append( (ip_sub_slot, ip_sub_slot_sps, ip_sub_slot_total_iters)) new_eos: Optional[EndOfSubSlotBundle] = None new_sps: List[Tuple[uint8, SignagePoint]] = [] new_ips: List[timelord_protocol.NewInfusionPointVDF] = [] future_eos: List[EndOfSubSlotBundle] = self.future_eos_cache.get( peak.reward_infusion_new_challenge, []).copy() for eos in future_eos: if self.new_finished_sub_slot(eos, blocks, peak, peak_full_block) is not None: new_eos = eos break future_sps: List[Tuple[uint8, SignagePoint]] = self.future_sp_cache.get( peak.reward_infusion_new_challenge, []).copy() for index, sp in future_sps: assert sp.cc_vdf is not None if self.new_signage_point(index, blocks, peak, peak.sub_slot_iters, sp): new_sps.append((index, sp)) for ip in self.future_ip_cache.get(peak.reward_infusion_new_challenge, []): new_ips.append(ip) self.future_eos_cache.pop(peak.reward_infusion_new_challenge, []) self.future_sp_cache.pop(peak.reward_infusion_new_challenge, []) self.future_ip_cache.pop(peak.reward_infusion_new_challenge, []) return new_eos, new_sps, new_ips
async def _validate_weight_proof_inner( self, weight_proof: WeightProof, skip_segment_validation: bool ) -> Tuple[bool, uint32, List[SubEpochSummary], List[BlockRecord]]: assert len(weight_proof.sub_epochs) > 0 if len(weight_proof.sub_epochs) == 0: return False, uint32(0), [], [] peak_height = weight_proof.recent_chain_data[ -1].reward_chain_block.height log.info(f"validate weight proof peak height {peak_height}") summaries, sub_epoch_weight_list = _validate_sub_epoch_summaries( self._constants, weight_proof) if summaries is None: log.error("weight proof failed sub epoch data validation") return False, uint32(0), [], [] seed = summaries[-2].get_hash() rng = random.Random(seed) if not validate_sub_epoch_sampling(rng, sub_epoch_weight_list, weight_proof): log.error("failed weight proof sub epoch sample validation") return False, uint32(0), [], [] constants, summary_bytes, wp_segment_bytes, wp_recent_chain_bytes = vars_to_bytes( self._constants, summaries, weight_proof) vdf_tasks: List[asyncio.Future] = [] recent_blocks_validation_task: asyncio.Future = asyncio.get_running_loop( ).run_in_executor( self._executor, _validate_recent_blocks_and_get_records, constants, wp_recent_chain_bytes, summary_bytes, pathlib.Path(self._executor_shutdown_tempfile.name), ) try: if not skip_segment_validation: segments_validated, vdfs_to_validate = _validate_sub_epoch_segments( constants, rng, wp_segment_bytes, summary_bytes) if not segments_validated: return False, uint32(0), [], [] vdf_chunks = chunks(vdfs_to_validate, self._num_processes) for chunk in vdf_chunks: byte_chunks = [] for vdf_proof, classgroup, vdf_info in chunk: byte_chunks.append( (bytes(vdf_proof), bytes(classgroup), bytes(vdf_info))) vdf_task: asyncio.Future = asyncio.get_running_loop( ).run_in_executor( self._executor, _validate_vdf_batch, constants, byte_chunks, pathlib.Path(self._executor_shutdown_tempfile.name), ) vdf_tasks.append(vdf_task) for vdf_task in vdf_tasks: validated = await vdf_task if not validated: return False, uint32(0), [], [] valid_recent_blocks, records_bytes = await recent_blocks_validation_task finally: recent_blocks_validation_task.cancel() for vdf_task in vdf_tasks: vdf_task.cancel() if not valid_recent_blocks: log.error("failed validating weight proof recent blocks") # Verify the data return False, uint32(0), [], [] records = [BlockRecord.from_bytes(b) for b in records_bytes] # TODO fix find fork point return True, uint32(0), summaries, records
def new_peak( self, peak: BlockRecord, peak_full_block: FullBlock, sp_sub_slot: Optional[ EndOfSubSlotBundle], # None if not overflow, or in first/second slot ip_sub_slot: Optional[EndOfSubSlotBundle], # None if in first slot fork_block: Optional[BlockRecord], blocks: BlockchainInterface, ) -> Tuple[Optional[EndOfSubSlotBundle], List[Tuple[uint8, SignagePoint]], List[timelord_protocol.NewInfusionPointVDF]]: """ If the peak is an overflow block, must provide two sub-slots: one for the current sub-slot and one for the prev sub-slot (since we still might get more blocks with an sp in the previous sub-slot) Results in either one or two sub-slots in finished_sub_slots. """ assert len(self.finished_sub_slots) >= 1 if ip_sub_slot is None: # We are still in the first sub-slot, no new sub slots ey self.initialize_genesis_sub_slot() else: # This is not the first sub-slot in the chain sp_sub_slot_sps: List[Optional[SignagePoint]] = [ None ] * self.constants.NUM_SPS_SUB_SLOT ip_sub_slot_sps: List[Optional[SignagePoint]] = [ None ] * self.constants.NUM_SPS_SUB_SLOT if fork_block is not None and fork_block.sub_slot_iters != peak.sub_slot_iters: # If there was a reorg and a difficulty adjustment, just clear all the slots self.clear_slots() else: interval_iters = calculate_sp_interval_iters( self.constants, peak.sub_slot_iters) # If it's not a reorg, or there is a reorg on the same difficulty, we can keep signage points # that we had before, in the cache for index, (sub_slot, sps, total_iters) in enumerate(self.finished_sub_slots): if sub_slot is None: continue if fork_block is None: # If this is not a reorg, we still want to remove signage points after the new peak fork_block = peak replaced_sps: List[Optional[SignagePoint]] = [ ] # index 0 is the end of sub slot for i, sp in enumerate(sps): if (total_iters + i * interval_iters) < fork_block.total_iters: # Sps before the fork point as still valid replaced_sps.append(sp) else: if sp is not None: log.debug( f"Reverting {i} {(total_iters + i * interval_iters)} {fork_block.total_iters}" ) # Sps after the fork point should be removed replaced_sps.append(None) assert len(sps) == len(replaced_sps) if sub_slot == sp_sub_slot: sp_sub_slot_sps = replaced_sps if sub_slot == ip_sub_slot: ip_sub_slot_sps = replaced_sps self.clear_slots() prev_sub_slot_total_iters = peak.sp_sub_slot_total_iters( self.constants) if sp_sub_slot is not None or prev_sub_slot_total_iters == 0: assert peak.overflow or prev_sub_slot_total_iters self.finished_sub_slots.append( (sp_sub_slot, sp_sub_slot_sps, prev_sub_slot_total_iters)) ip_sub_slot_total_iters = peak.ip_sub_slot_total_iters( self.constants) self.finished_sub_slots.append( (ip_sub_slot, ip_sub_slot_sps, ip_sub_slot_total_iters)) new_eos: Optional[EndOfSubSlotBundle] = None new_sps: List[Tuple[uint8, SignagePoint]] = [] new_ips: List[timelord_protocol.NewInfusionPointVDF] = [] future_eos: List[EndOfSubSlotBundle] = self.future_eos_cache.get( peak.reward_infusion_new_challenge, []).copy() for eos in future_eos: if self.new_finished_sub_slot(eos, blocks, peak, peak_full_block) is not None: new_eos = eos break future_sps: List[Tuple[uint8, SignagePoint]] = self.future_sp_cache.get( peak.reward_infusion_new_challenge, []).copy() for index, sp in future_sps: assert sp.cc_vdf is not None if self.new_signage_point(index, blocks, peak, peak.sub_slot_iters, sp): new_sps.append((index, sp)) for ip in self.future_ip_cache.get(peak.reward_infusion_new_challenge, []): new_ips.append(ip) self.future_eos_cache.pop(peak.reward_infusion_new_challenge, []) self.future_sp_cache.pop(peak.reward_infusion_new_challenge, []) self.future_ip_cache.pop(peak.reward_infusion_new_challenge, []) for eos_op, _, _ in self.finished_sub_slots: if eos_op is not None: self.recent_eos.put(eos_op.challenge_chain.get_hash(), (eos_op, time.time())) return new_eos, new_sps, new_ips
def batch_pre_validate_blocks( constants_dict: Dict, blocks_pickled: Dict[bytes, bytes], full_blocks_pickled: Optional[List[bytes]], header_blocks_pickled: Optional[List[bytes]], prev_transaction_generators: List[Optional[bytes]], npc_results: Dict[uint32, bytes], check_filter: bool, expected_difficulty: List[uint64], expected_sub_slot_iters: List[uint64], ) -> List[bytes]: blocks = {} for k, v in blocks_pickled.items(): blocks[k] = BlockRecord.from_bytes(v) results: List[PreValidationResult] = [] constants: ConsensusConstants = dataclass_from_dict( ConsensusConstants, constants_dict) if full_blocks_pickled is not None and header_blocks_pickled is not None: assert ValueError("Only one should be passed here") if full_blocks_pickled is not None: for i in range(len(full_blocks_pickled)): try: block: FullBlock = FullBlock.from_bytes(full_blocks_pickled[i]) tx_additions: List[Coin] = [] removals: List[bytes32] = [] npc_result: Optional[NPCResult] = None if block.height in npc_results: npc_result = NPCResult.from_bytes( npc_results[block.height]) assert npc_result is not None if npc_result.npc_list is not None: removals, tx_additions = tx_removals_and_additions( npc_result.npc_list) else: removals, tx_additions = [], [] if block.transactions_generator is not None and npc_result is None: prev_generator_bytes = prev_transaction_generators[i] assert prev_generator_bytes is not None assert block.transactions_info is not None block_generator: BlockGenerator = BlockGenerator.from_bytes( prev_generator_bytes) assert block_generator.program == block.transactions_generator npc_result = get_name_puzzle_conditions( block_generator, min(constants.MAX_BLOCK_COST_CLVM, block.transactions_info.cost), True) removals, tx_additions = tx_removals_and_additions( npc_result.npc_list) header_block = get_block_header(block, tx_additions, removals) required_iters, error = validate_finished_header_block( constants, BlockCache(blocks), header_block, check_filter, expected_difficulty[i], expected_sub_slot_iters[i], ) error_int: Optional[uint16] = None if error is not None: error_int = uint16(error.code.value) results.append( PreValidationResult(error_int, required_iters, npc_result)) except Exception: error_stack = traceback.format_exc() log.error(f"Exception: {error_stack}") results.append( PreValidationResult(uint16(Err.UNKNOWN.value), None, None)) elif header_blocks_pickled is not None: for i in range(len(header_blocks_pickled)): try: header_block = HeaderBlock.from_bytes(header_blocks_pickled[i]) required_iters, error = validate_finished_header_block( constants, BlockCache(blocks), header_block, check_filter, expected_difficulty[i], expected_sub_slot_iters[i], ) error_int = None if error is not None: error_int = uint16(error.code.value) results.append( PreValidationResult(error_int, required_iters, None)) except Exception: error_stack = traceback.format_exc() log.error(f"Exception: {error_stack}") results.append( PreValidationResult(uint16(Err.UNKNOWN.value), None, None)) return [bytes(r) for r in results]
def batch_pre_validate_blocks( constants_dict: Dict, blocks_pickled: Dict[bytes, bytes], full_blocks_pickled: Optional[List[bytes]], header_blocks_pickled: Optional[List[bytes]], prev_transaction_generators: List[Optional[bytes]], npc_results: Dict[uint32, bytes], check_filter: bool, expected_difficulty: List[uint64], expected_sub_slot_iters: List[uint64], validate_signatures: bool, ) -> List[bytes]: blocks: Dict[bytes, BlockRecord] = {} for k, v in blocks_pickled.items(): blocks[k] = BlockRecord.from_bytes(v) results: List[PreValidationResult] = [] constants: ConsensusConstants = dataclass_from_dict( ConsensusConstants, constants_dict) if full_blocks_pickled is not None and header_blocks_pickled is not None: assert ValueError("Only one should be passed here") # In this case, we are validating full blocks, not headers if full_blocks_pickled is not None: for i in range(len(full_blocks_pickled)): try: block: FullBlock = FullBlock.from_bytes(full_blocks_pickled[i]) tx_additions: List[Coin] = [] removals: List[bytes32] = [] npc_result: Optional[NPCResult] = None if block.height in npc_results: npc_result = NPCResult.from_bytes( npc_results[block.height]) assert npc_result is not None if npc_result.npc_list is not None: removals, tx_additions = tx_removals_and_additions( npc_result.npc_list) else: removals, tx_additions = [], [] if block.transactions_generator is not None and npc_result is None: prev_generator_bytes = prev_transaction_generators[i] assert prev_generator_bytes is not None assert block.transactions_info is not None block_generator: BlockGenerator = BlockGenerator.from_bytes( prev_generator_bytes) assert block_generator.program == block.transactions_generator npc_result = get_name_puzzle_conditions( block_generator, min(constants.MAX_BLOCK_COST_CLVM, block.transactions_info.cost), cost_per_byte=constants.COST_PER_BYTE, mempool_mode=False, height=block.height, ) removals, tx_additions = tx_removals_and_additions( npc_result.npc_list) if npc_result is not None and npc_result.error is not None: results.append( PreValidationResult(uint16(npc_result.error), None, npc_result, False)) continue header_block = get_block_header(block, tx_additions, removals) # TODO: address hint error and remove ignore # error: Argument 1 to "BlockCache" has incompatible type "Dict[bytes, BlockRecord]"; expected # "Dict[bytes32, BlockRecord]" [arg-type] required_iters, error = validate_finished_header_block( constants, BlockCache(blocks), # type: ignore[arg-type] header_block, check_filter, expected_difficulty[i], expected_sub_slot_iters[i], ) error_int: Optional[uint16] = None if error is not None: error_int = uint16(error.code.value) successfully_validated_signatures = False # If we failed CLVM, no need to validate signature, the block is already invalid if error_int is None: # If this is False, it means either we don't have a signature (not a tx block) or we have an invalid # signature (which also puts in an error) or we didn't validate the signature because we want to # validate it later. receive_block will attempt to validate the signature later. if validate_signatures: if npc_result is not None and block.transactions_info is not None: pairs_pks, pairs_msgs = pkm_pairs( npc_result.npc_list, constants.AGG_SIG_ME_ADDITIONAL_DATA) pks_objects: List[G1Element] = [ G1Element.from_bytes(pk) for pk in pairs_pks ] if not AugSchemeMPL.aggregate_verify( pks_objects, pairs_msgs, block. transactions_info.aggregated_signature): error_int = uint16( Err.BAD_AGGREGATE_SIGNATURE.value) else: successfully_validated_signatures = True results.append( PreValidationResult(error_int, required_iters, npc_result, successfully_validated_signatures)) except Exception: error_stack = traceback.format_exc() log.error(f"Exception: {error_stack}") results.append( PreValidationResult(uint16(Err.UNKNOWN.value), None, None, False)) # In this case, we are validating header blocks elif header_blocks_pickled is not None: for i in range(len(header_blocks_pickled)): try: header_block = HeaderBlock.from_bytes(header_blocks_pickled[i]) # TODO: address hint error and remove ignore # error: Argument 1 to "BlockCache" has incompatible type "Dict[bytes, BlockRecord]"; expected # "Dict[bytes32, BlockRecord]" [arg-type] required_iters, error = validate_finished_header_block( constants, BlockCache(blocks), # type: ignore[arg-type] header_block, check_filter, expected_difficulty[i], expected_sub_slot_iters[i], ) error_int = None if error is not None: error_int = uint16(error.code.value) results.append( PreValidationResult(error_int, required_iters, None, False)) except Exception: error_stack = traceback.format_exc() log.error(f"Exception: {error_stack}") results.append( PreValidationResult(uint16(Err.UNKNOWN.value), None, None, False)) return [bytes(r) for r in results]