async def pre_validate_blocks_multiprocessing( constants: ConsensusConstants, constants_json: Dict, block_records: BlockchainInterface, blocks: Sequence[Union[FullBlock, HeaderBlock]], pool: ProcessPoolExecutor, validate_transactions: bool, check_filter: bool, ) -> Optional[List[PreValidationResult]]: """ This method must be called under the blockchain lock If all the full blocks pass pre-validation, (only validates header), returns the list of required iters. if any validation issue occurs, returns False. Args: check_filter: validate_transactions: constants_json: pool: constants: block_records: blocks: list of full blocks to validate (must be connected to current chain) """ batch_size = 4 prev_b: Optional[BlockRecord] = None # Collects all the recent blocks (up to the previous sub-epoch) recent_blocks: Dict[bytes32, BlockRecord] = {} recent_blocks_compressed: Dict[bytes32, BlockRecord] = {} num_sub_slots_found = 0 num_blocks_seen = 0 if blocks[0].height > 0: if not block_records.contains_block(blocks[0].prev_header_hash): return [ PreValidationResult(uint16(Err.INVALID_PREV_BLOCK_HASH.value), None, None) ] curr = block_records.block_record(blocks[0].prev_header_hash) num_sub_slots_to_look_for = 3 if curr.overflow else 2 while (curr.sub_epoch_summary_included is None or num_blocks_seen < constants.NUMBER_OF_TIMESTAMPS or num_sub_slots_found < num_sub_slots_to_look_for ) and curr.height > 0: if num_blocks_seen < constants.NUMBER_OF_TIMESTAMPS or num_sub_slots_found < num_sub_slots_to_look_for: recent_blocks_compressed[curr.header_hash] = curr if curr.first_in_sub_slot: assert curr.finished_challenge_slot_hashes is not None num_sub_slots_found += len(curr.finished_challenge_slot_hashes) recent_blocks[curr.header_hash] = curr if curr.is_transaction_block: num_blocks_seen += 1 curr = block_records.block_record(curr.prev_hash) recent_blocks[curr.header_hash] = curr recent_blocks_compressed[curr.header_hash] = curr block_record_was_present = [] for block in blocks: block_record_was_present.append( block_records.contains_block(block.header_hash)) diff_ssis: List[Tuple[uint64, uint64]] = [] for block in blocks: if block.height != 0 and prev_b is None: prev_b = block_records.block_record(block.prev_header_hash) sub_slot_iters, difficulty = get_next_sub_slot_iters_and_difficulty( constants, len(block.finished_sub_slots) > 0, prev_b, block_records) if block.reward_chain_block.signage_point_index >= constants.NUM_SPS_SUB_SLOT: log.warning(f"Block: {block.reward_chain_block}") overflow = is_overflow_block( constants, block.reward_chain_block.signage_point_index) challenge = get_block_challenge(constants, block, BlockCache(recent_blocks), prev_b is None, overflow, False) if block.reward_chain_block.challenge_chain_sp_vdf is None: cc_sp_hash: bytes32 = challenge else: cc_sp_hash = block.reward_chain_block.challenge_chain_sp_vdf.output.get_hash( ) q_str: Optional[ bytes32] = block.reward_chain_block.proof_of_space.verify_and_get_quality_string( constants, challenge, cc_sp_hash) if q_str is None: for i, block_i in enumerate(blocks): if not block_record_was_present[ i] and block_records.contains_block( block_i.header_hash): block_records.remove_block_record(block_i.header_hash) return None required_iters: uint64 = calculate_iterations_quality( constants.DIFFICULTY_CONSTANT_FACTOR, q_str, block.reward_chain_block.proof_of_space.size, difficulty, cc_sp_hash, ) block_rec = block_to_block_record( constants, block_records, required_iters, block, None, ) recent_blocks[block_rec.header_hash] = block_rec recent_blocks_compressed[block_rec.header_hash] = block_rec block_records.add_block_record( block_rec) # Temporarily add block to dict prev_b = block_rec diff_ssis.append((difficulty, sub_slot_iters)) for i, block in enumerate(blocks): if not block_record_was_present[i]: block_records.remove_block_record(block.header_hash) recent_sb_compressed_pickled = { bytes(k): bytes(v) for k, v in recent_blocks_compressed.items() } futures = [] # Pool of workers to validate blocks concurrently for i in range(0, len(blocks), batch_size): end_i = min(i + batch_size, len(blocks)) blocks_to_validate = blocks[i:end_i] if any([ len(block.finished_sub_slots) > 0 for block in blocks_to_validate ]): final_pickled = { bytes(k): bytes(v) for k, v in recent_blocks.items() } else: final_pickled = recent_sb_compressed_pickled hb_pickled: List[bytes] = [] generators: List[Optional[bytes]] = [] for block in blocks_to_validate: if isinstance(block, FullBlock): hb_pickled.append(bytes(block.get_block_header())) generators.append( bytes(block.transactions_generator) if block. transactions_generator is not None else None) else: hb_pickled.append(bytes(block)) generators.append(None) futures.append(asyncio.get_running_loop().run_in_executor( pool, batch_pre_validate_blocks, constants_json, final_pickled, hb_pickled, generators, check_filter, [diff_ssis[j][0] for j in range(i, end_i)], [diff_ssis[j][1] for j in range(i, end_i)], validate_transactions, )) # Collect all results into one flat list return [ PreValidationResult.from_bytes(result) for batch_result in (await asyncio.gather(*futures)) for result in batch_result ]
def _get_next_sub_slot_iters( constants: ConsensusConstants, blocks: BlockchainInterface, prev_header_hash: bytes32, height: uint32, curr_sub_slot_iters: uint64, deficit: uint8, block_at_height_included_ses: bool, new_slot: bool, signage_point_total_iters: uint128, skip_epoch_check=False, ) -> uint64: """ Returns the slot iterations required for the next block after the one at height, where new_slot is true iff the next block will be in the next slot. WARNING: assumes that the block at height is not the first block in a sub-epoch. Args: constants: consensus constants being used for this chain blocks: dictionary from header hash to SBR of all included SBR prev_header_hash: header hash of the previous block height: the block height of the block to look at curr_sub_slot_iters: sub-slot iters at the infusion point of the block at height deficit: deficit of block at height height new_slot: whether or not there is a new slot after height signage_point_total_iters: signage point iters of the block at height skip_epoch_check: don't check correct epoch """ next_height: uint32 = uint32(height + 1) if next_height < constants.EPOCH_BLOCKS: return uint64(constants.SUB_SLOT_ITERS_STARTING) if not blocks.contains_block(prev_header_hash): raise ValueError(f"Header hash {prev_header_hash} not in blocks") prev_b: BlockRecord = blocks.block_record(prev_header_hash) # If we are in the same epoch, return same ssi if not skip_epoch_check: _, can_finish_epoch = can_finish_sub_and_full_epoch( constants, blocks, height, prev_header_hash, deficit, block_at_height_included_ses) if not new_slot or not can_finish_epoch: return curr_sub_slot_iters last_block_prev: BlockRecord = _get_second_to_last_transaction_block_in_previous_epoch( constants, blocks, prev_b) # This gets the last transaction block before this block's signage point. Assuming the block at height height # is the last block infused in the epoch: If this block ends up being a # transaction block, then last_block_curr will be the second to last tx block in the epoch. If this block # is not a transaction block, that means there was exactly one other tx block included in between our signage # point and infusion point, and therefore last_block_curr is the second to last as well. last_block_curr = prev_b while last_block_curr.total_iters > signage_point_total_iters or not last_block_curr.is_transaction_block: last_block_curr = blocks.block_record(last_block_curr.prev_hash) assert last_block_curr.timestamp is not None and last_block_prev.timestamp is not None # This is computed as the iterations per second in last epoch, times the target number of seconds per slot new_ssi_precise: uint64 = uint64( constants.SUB_SLOT_TIME_TARGET * (last_block_curr.total_iters - last_block_prev.total_iters) // (last_block_curr.timestamp - last_block_prev.timestamp)) # Only change by a max factor as a sanity check max_ssi = uint64(constants.DIFFICULTY_CHANGE_MAX_FACTOR * last_block_curr.sub_slot_iters) min_ssi = uint64(last_block_curr.sub_slot_iters // constants.DIFFICULTY_CHANGE_MAX_FACTOR) if new_ssi_precise >= last_block_curr.sub_slot_iters: new_ssi_precise = uint64(min(new_ssi_precise, max_ssi)) else: new_ssi_precise = uint64( max([constants.NUM_SPS_SUB_SLOT, new_ssi_precise, min_ssi])) new_ssi = truncate_to_significant_bits(new_ssi_precise, constants.SIGNIFICANT_BITS) new_ssi = uint64( new_ssi - new_ssi % constants.NUM_SPS_SUB_SLOT) # Must divide the sub slot assert count_significant_bits(new_ssi) <= constants.SIGNIFICANT_BITS return new_ssi
def _get_next_difficulty( constants: ConsensusConstants, blocks: BlockchainInterface, prev_header_hash: bytes32, height: uint32, current_difficulty: uint64, deficit: uint8, block_at_height_included_ses: bool, new_slot: bool, signage_point_total_iters: uint128, skip_epoch_check=False, ) -> uint64: """ Returns the difficulty of the next block that extends onto block. Used to calculate the number of iterations. WARNING: assumes that the block at height is not the first block in a sub-epoch. Args: constants: consensus constants being used for this chain blocks: dictionary from header hash to SBR of all included SBR prev_header_hash: header hash of the previous block height: the block height of the block to look at deficit: deficit of block at height height current_difficulty: difficulty at the infusion point of the block at height new_slot: whether or not there is a new slot after height signage_point_total_iters: signage point iters of the block at height skip_epoch_check: don't check correct epoch """ next_height: uint32 = uint32(height + 1) if next_height < constants.EPOCH_BLOCKS: # We are in the first epoch return uint64(constants.DIFFICULTY_STARTING) if not blocks.contains_block(prev_header_hash): raise ValueError(f"Header hash {prev_header_hash} not in blocks") prev_b: BlockRecord = blocks.block_record(prev_header_hash) # If we are in the same slot as previous block, return same difficulty if not skip_epoch_check: _, can_finish_epoch = can_finish_sub_and_full_epoch( constants, blocks, height, prev_header_hash, deficit, block_at_height_included_ses) if not new_slot or not can_finish_epoch: return current_difficulty last_block_prev: BlockRecord = _get_second_to_last_transaction_block_in_previous_epoch( constants, blocks, prev_b) # This gets the last transaction block before this block's signage point. Assuming the block at height height # is the last block infused in the epoch: If this block ends up being a # transaction block, then last_block_curr will be the second to last tx block in the epoch. If this block # is not a transaction block, that means there was exactly one other tx block included in between our signage # point and infusion point, and therefore last_block_curr is the second to last as well. last_block_curr = prev_b while last_block_curr.total_iters > signage_point_total_iters or not last_block_curr.is_transaction_block: last_block_curr = blocks.block_record(last_block_curr.prev_hash) assert last_block_curr.timestamp is not None assert last_block_prev.timestamp is not None actual_epoch_time: uint64 = uint64(last_block_curr.timestamp - last_block_prev.timestamp) old_difficulty = uint64(prev_b.weight - blocks.block_record(prev_b.prev_hash).weight) # Terms are rearranged so there is only one division. new_difficulty_precise = uint64( (last_block_curr.weight - last_block_prev.weight) * constants.SUB_SLOT_TIME_TARGET // (constants.SLOT_BLOCKS_TARGET * actual_epoch_time)) # Only change by a max factor, to prevent attacks, as in greenpaper, and must be at least 1 max_diff = uint64(constants.DIFFICULTY_CHANGE_MAX_FACTOR * old_difficulty) min_diff = uint64(old_difficulty // constants.DIFFICULTY_CHANGE_MAX_FACTOR) if new_difficulty_precise >= old_difficulty: new_difficulty_precise = uint64(min(new_difficulty_precise, max_diff)) else: new_difficulty_precise = uint64( max([uint64(1), new_difficulty_precise, min_diff])) new_difficulty = truncate_to_significant_bits(new_difficulty_precise, constants.SIGNIFICANT_BITS) assert count_significant_bits(new_difficulty) <= constants.SIGNIFICANT_BITS return uint64(new_difficulty)
def get_next_difficulty( constants: ConsensusConstants, blocks: BlockchainInterface, prev_header_hash: bytes32, height: uint32, current_difficulty: uint64, deficit: uint8, new_slot: bool, signage_point_total_iters: uint128, skip_epoch_check=False, ) -> uint64: """ Returns the difficulty of the next block that extends onto block. Used to calculate the number of iterations. When changing this, also change the implementation in wallet_state_manager.py. Args: constants: consensus constants being used for this chain blocks: dictionary from header hash to SBR of all included SBR prev_header_hash: header hash of the previous block height: the block height of the block to look at current_difficulty: difficulty at the infusion point of the block at height deficit: deficit of the block at height new_slot: whether or not there is a new slot after height signage_point_total_iters: signage point iters of the block at height skip_epoch_check: don't check correct epoch """ next_height: uint32 = uint32(height + 1) if next_height < (constants.EPOCH_BLOCKS - constants.MAX_SUB_SLOT_BLOCKS): # We are in the first epoch return uint64(constants.DIFFICULTY_STARTING) if not blocks.contains_block(prev_header_hash): raise ValueError(f"Header hash {prev_header_hash} not in blocks") prev_b: BlockRecord = blocks.block_record(prev_header_hash) # If we are in the same slot as previous block, return same difficulty if not skip_epoch_check: _, can_finish_epoch = can_finish_sub_and_full_epoch( constants, height, deficit, blocks, prev_header_hash, False) if not new_slot or not can_finish_epoch: return current_difficulty last_block_prev: BlockRecord = _get_last_transaction_block_in_previous_epoch( constants, blocks, prev_b) # Ensure we get a tx block for the last block as well, and that it is before the signage point last_block_curr = prev_b while last_block_curr.total_iters > signage_point_total_iters or not last_block_curr.is_transaction_block: last_block_curr = blocks.block_record(last_block_curr.prev_hash) assert last_block_curr.timestamp is not None assert last_block_prev.timestamp is not None actual_epoch_time: uint64 = uint64(last_block_curr.timestamp - last_block_prev.timestamp) old_difficulty = uint64(prev_b.weight - blocks.block_record(prev_b.prev_hash).weight) # Terms are rearranged so there is only one division. new_difficulty_precise = ( (last_block_curr.weight - last_block_prev.weight) * constants.SUB_SLOT_TIME_TARGET // (constants.SLOT_BLOCKS_TARGET * actual_epoch_time)) # Take only DIFFICULTY_SIGNIFICANT_BITS significant bits new_difficulty = uint64( truncate_to_significant_bits(new_difficulty_precise, constants.SIGNIFICANT_BITS)) assert count_significant_bits(new_difficulty) <= constants.SIGNIFICANT_BITS # Only change by a max factor, to prevent attacks, as in greenpaper, and must be at least 1 max_diff = uint64( truncate_to_significant_bits( constants.DIFFICULTY_CHANGE_MAX_FACTOR * old_difficulty, constants.SIGNIFICANT_BITS, )) min_diff = uint64( truncate_to_significant_bits( old_difficulty // constants.DIFFICULTY_CHANGE_MAX_FACTOR, constants.SIGNIFICANT_BITS, )) if new_difficulty >= old_difficulty: return min(new_difficulty, max_diff) else: return max([uint64(1), new_difficulty, min_diff])