async def load_blocks_dont_validate( blocks, ) -> Tuple[Dict[bytes32, HeaderBlock], Dict[uint32, bytes32], Dict[ bytes32, BlockRecord], Dict[bytes32, SubEpochSummary]]: header_cache: Dict[bytes32, HeaderBlock] = {} height_to_hash: Dict[uint32, bytes32] = {} sub_blocks: Dict[bytes32, BlockRecord] = {} sub_epoch_summaries: Dict[bytes32, SubEpochSummary] = {} prev_block = None difficulty = test_constants.DIFFICULTY_STARTING block: FullBlock for block in blocks: if block.height > 0: assert prev_block is not None difficulty = block.reward_chain_block.weight - prev_block.weight if block.reward_chain_block.challenge_chain_sp_vdf is None: assert block.reward_chain_block.signage_point_index == 0 cc_sp: bytes32 = block.reward_chain_block.pos_ss_cc_challenge_hash else: cc_sp = block.reward_chain_block.challenge_chain_sp_vdf.output.get_hash( ) quality_string: Optional[ bytes32] = block.reward_chain_block.proof_of_space.verify_and_get_quality_string( test_constants, block.reward_chain_block.pos_ss_cc_challenge_hash, cc_sp, ) assert quality_string is not None required_iters: uint64 = calculate_iterations_quality( test_constants.DIFFICULTY_CONSTANT_FACTOR, quality_string, block.reward_chain_block.proof_of_space.size, difficulty, cc_sp, ) sub_block = block_to_block_record( test_constants, BlockCache(sub_blocks, height_to_hash), required_iters, block, None) sub_blocks[block.header_hash] = sub_block height_to_hash[block.height] = block.header_hash header_cache[block.header_hash] = block.get_block_header() if sub_block.sub_epoch_summary_included is not None: sub_epoch_summaries[ block.height] = sub_block.sub_epoch_summary_included prev_block = block return header_cache, height_to_hash, sub_blocks, sub_epoch_summaries
async def pre_validate_blocks_multiprocessing( constants: ConsensusConstants, constants_json: Dict, block_records: BlockchainInterface, blocks: Sequence[Union[FullBlock, HeaderBlock]], pool: ProcessPoolExecutor, validate_transactions: bool, check_filter: bool, ) -> Optional[List[PreValidationResult]]: """ This method must be called under the blockchain lock If all the full blocks pass pre-validation, (only validates header), returns the list of required iters. if any validation issue occurs, returns False. Args: check_filter: validate_transactions: constants_json: pool: constants: block_records: blocks: list of full blocks to validate (must be connected to current chain) """ batch_size = 4 prev_b: Optional[BlockRecord] = None # Collects all the recent blocks (up to the previous sub-epoch) recent_blocks: Dict[bytes32, BlockRecord] = {} recent_blocks_compressed: Dict[bytes32, BlockRecord] = {} num_sub_slots_found = 0 num_blocks_seen = 0 if blocks[0].height > 0: if not block_records.contains_block(blocks[0].prev_header_hash): return [ PreValidationResult(uint16(Err.INVALID_PREV_BLOCK_HASH.value), None, None) ] curr = block_records.block_record(blocks[0].prev_header_hash) num_sub_slots_to_look_for = 3 if curr.overflow else 2 while (curr.sub_epoch_summary_included is None or num_blocks_seen < constants.NUMBER_OF_TIMESTAMPS or num_sub_slots_found < num_sub_slots_to_look_for ) and curr.height > 0: if num_blocks_seen < constants.NUMBER_OF_TIMESTAMPS or num_sub_slots_found < num_sub_slots_to_look_for: recent_blocks_compressed[curr.header_hash] = curr if curr.first_in_sub_slot: assert curr.finished_challenge_slot_hashes is not None num_sub_slots_found += len(curr.finished_challenge_slot_hashes) recent_blocks[curr.header_hash] = curr if curr.is_transaction_block: num_blocks_seen += 1 curr = block_records.block_record(curr.prev_hash) recent_blocks[curr.header_hash] = curr recent_blocks_compressed[curr.header_hash] = curr block_record_was_present = [] for block in blocks: block_record_was_present.append( block_records.contains_block(block.header_hash)) diff_ssis: List[Tuple[uint64, uint64]] = [] for block in blocks: if block.height != 0 and prev_b is None: prev_b = block_records.block_record(block.prev_header_hash) sub_slot_iters, difficulty = get_next_sub_slot_iters_and_difficulty( constants, len(block.finished_sub_slots) > 0, prev_b, block_records) if block.reward_chain_block.signage_point_index >= constants.NUM_SPS_SUB_SLOT: log.warning(f"Block: {block.reward_chain_block}") overflow = is_overflow_block( constants, block.reward_chain_block.signage_point_index) challenge = get_block_challenge(constants, block, BlockCache(recent_blocks), prev_b is None, overflow, False) if block.reward_chain_block.challenge_chain_sp_vdf is None: cc_sp_hash: bytes32 = challenge else: cc_sp_hash = block.reward_chain_block.challenge_chain_sp_vdf.output.get_hash( ) q_str: Optional[ bytes32] = block.reward_chain_block.proof_of_space.verify_and_get_quality_string( constants, challenge, cc_sp_hash) if q_str is None: for i, block_i in enumerate(blocks): if not block_record_was_present[ i] and block_records.contains_block( block_i.header_hash): block_records.remove_block_record(block_i.header_hash) return None required_iters: uint64 = calculate_iterations_quality( constants.DIFFICULTY_CONSTANT_FACTOR, q_str, block.reward_chain_block.proof_of_space.size, difficulty, cc_sp_hash, ) block_rec = block_to_block_record( constants, block_records, required_iters, block, None, ) recent_blocks[block_rec.header_hash] = block_rec recent_blocks_compressed[block_rec.header_hash] = block_rec block_records.add_block_record( block_rec) # Temporarily add block to dict prev_b = block_rec diff_ssis.append((difficulty, sub_slot_iters)) for i, block in enumerate(blocks): if not block_record_was_present[i]: block_records.remove_block_record(block.header_hash) recent_sb_compressed_pickled = { bytes(k): bytes(v) for k, v in recent_blocks_compressed.items() } futures = [] # Pool of workers to validate blocks concurrently for i in range(0, len(blocks), batch_size): end_i = min(i + batch_size, len(blocks)) blocks_to_validate = blocks[i:end_i] if any([ len(block.finished_sub_slots) > 0 for block in blocks_to_validate ]): final_pickled = { bytes(k): bytes(v) for k, v in recent_blocks.items() } else: final_pickled = recent_sb_compressed_pickled hb_pickled: List[bytes] = [] generators: List[Optional[bytes]] = [] for block in blocks_to_validate: if isinstance(block, FullBlock): hb_pickled.append(bytes(block.get_block_header())) generators.append( bytes(block.transactions_generator) if block. transactions_generator is not None else None) else: hb_pickled.append(bytes(block)) generators.append(None) futures.append(asyncio.get_running_loop().run_in_executor( pool, batch_pre_validate_blocks, constants_json, final_pickled, hb_pickled, generators, check_filter, [diff_ssis[j][0] for j in range(i, end_i)], [diff_ssis[j][1] for j in range(i, end_i)], validate_transactions, )) # Collect all results into one flat list return [ PreValidationResult.from_bytes(result) for batch_result in (await asyncio.gather(*futures)) for result in batch_result ]
async def receive_block( self, block: FullBlock, pre_validation_result: Optional[PreValidationResult] = None, fork_point_with_peak: Optional[uint32] = None, ) -> Tuple[ReceiveBlockResult, Optional[Err], Optional[uint32]]: """ This method must be called under the blockchain lock Adds a new block into the blockchain, if it's valid and connected to the current blockchain, regardless of whether it is the child of a head, or another block. Returns a header if block is added to head. Returns an error if the block is invalid. Also returns the fork height, in the case of a new peak. """ genesis: bool = block.height == 0 if self.contains_block(block.header_hash): return ReceiveBlockResult.ALREADY_HAVE_BLOCK, None, None if not self.contains_block(block.prev_header_hash) and not genesis: return ( ReceiveBlockResult.DISCONNECTED_BLOCK, Err.INVALID_PREV_BLOCK_HASH, None, ) if pre_validation_result is None: if block.height == 0: prev_b: Optional[BlockRecord] = None else: prev_b = self.block_record(block.prev_header_hash) sub_slot_iters, difficulty = get_sub_slot_iters_and_difficulty( self.constants, block, prev_b, self) required_iters, error = validate_finished_header_block( self.constants, self, block.get_block_header(), False, difficulty, sub_slot_iters, ) if error is not None: return ReceiveBlockResult.INVALID_BLOCK, error.code, None else: required_iters = pre_validation_result.required_iters assert pre_validation_result.error is None assert required_iters is not None error_code, _ = await validate_block_body( self.constants, self, self.block_store, self.coin_store, self.get_peak(), block, block.height, pre_validation_result.cost_result if pre_validation_result is not None else None, fork_point_with_peak, ) if error_code is not None: return ReceiveBlockResult.INVALID_BLOCK, error_code, None block_record = block_to_block_record( self.constants, self, required_iters, block, None, ) # Always add the block to the database await self.block_store.add_full_block(block, block_record) self.add_block_record(block_record) fork_height: Optional[uint32] = await self._reconsider_peak( block_record, genesis, fork_point_with_peak) if fork_height is not None: return ReceiveBlockResult.NEW_PEAK, None, fork_height else: return ReceiveBlockResult.ADDED_AS_ORPHAN, None, None
async def receive_block( self, header_block_record: HeaderBlockRecord, pre_validation_result: Optional[PreValidationResult] = None, trusted: bool = False, fork_point_with_peak: Optional[uint32] = None, ) -> Tuple[ReceiveBlockResult, Optional[Err], Optional[uint32]]: """ Adds a new block into the blockchain, if it's valid and connected to the current blockchain, regardless of whether it is the child of a head, or another block. Returns a header if block is added to head. Returns an error if the block is invalid. Also returns the fork height, in the case of a new peak. """ block = header_block_record.header genesis: bool = block.height == 0 if self.contains_block(block.header_hash): return ReceiveBlockResult.ALREADY_HAVE_BLOCK, None, None if not self.contains_block(block.prev_header_hash) and not genesis: return ( ReceiveBlockResult.DISCONNECTED_BLOCK, Err.INVALID_PREV_BLOCK_HASH, None, ) if block.height == 0: prev_b: Optional[BlockRecord] = None else: prev_b = self.block_record(block.prev_header_hash) sub_slot_iters, difficulty = get_sub_slot_iters_and_difficulty( self.constants, block, prev_b, self) if trusted is False and pre_validation_result is None: required_iters, error = validate_finished_header_block( self.constants, self, block, False, difficulty, sub_slot_iters) elif trusted: unfinished_header_block = UnfinishedHeaderBlock( block.finished_sub_slots, block.reward_chain_block.get_unfinished(), block.challenge_chain_sp_proof, block.reward_chain_sp_proof, block.foliage, block.foliage_transaction_block, block.transactions_filter, ) required_iters, val_error = validate_unfinished_header_block( self.constants, self, unfinished_header_block, False, difficulty, sub_slot_iters, False, True) error = ValidationError( Err(val_error)) if val_error is not None else None else: assert pre_validation_result is not None required_iters = pre_validation_result.required_iters error = (ValidationError(Err(pre_validation_result.error)) if pre_validation_result.error is not None else None) if error is not None: return ReceiveBlockResult.INVALID_BLOCK, error.code, None assert required_iters is not None block_record = block_to_block_record( self.constants, self, required_iters, None, block, ) # Always add the block to the database await self.block_store.add_block_record(header_block_record, block_record) self.add_block_record(block_record) self.clean_block_record(block_record.height - self.constants.BLOCKS_CACHE_SIZE) fork_height: Optional[uint32] = await self._reconsider_peak( block_record, genesis, fork_point_with_peak) if fork_height is not None: self.log.info( f"💰 Updated wallet peak to height {block_record.height}, weight {block_record.weight}, " ) return ReceiveBlockResult.NEW_PEAK, None, fork_height else: return ReceiveBlockResult.ADDED_AS_ORPHAN, None, None