async def _fetch_state_and_process_block(self, signed_block: Optional[ spec.SignedBeaconBlock], dest: trio.MemorySendChannel, is_canon: bool) -> bool: block = signed_block.message block_root = spec.Root(block.hash_tree_root()) prev_slot = spec.Slot(block.slot - 1) pre_state_cached: CachedState try: pre_state_cached = await self.get_state_by_block_and_slot( block.parent_root, prev_slot) except Exception as e: print(f"Failed to fetch state: {e}") return False state = pre_state_cached.state pre_state = state.copy() epc = pre_state_cached.epc.copy() print( f"processing state transition of block {block_root} (slot {block.slot}) " f"on state {state.hash_tree_root().hex()} (slot {state.slot}) " f"(asked for state of parent {block.parent_root.hex()}, at slot {prev_slot})" ) try: unchecked_state_transition(epc, state, signed_block) except Exception as e: print( f"WARNING: {block_root.hex()} (slot {block.slot}) failed state transition: {e}" ) traceback.print_exc() await self._on_bad_transition(signed_block, state) return False if block.state_root != state.hash_tree_root(): print( f"WARNING: {block.hash_tree_root().hex()} (slot {block.slot}) state root ({block.state_root.hex()})" f" does not match computed state root ({state.hash_tree_root().hex()})" ) await self._on_bad_transition(signed_block, state) return False await self.cache_state(block_root, state, epc) await dest.send((pre_state, state.copy(), signed_block, pre_state_cached.epc, epc, is_canon)) return True
async def get_state_by_state_root(self, state_root: spec.Root) -> CachedState: if state_root not in self.state_by_state_root_cache_dict: print(f"fetching state by state root {state_root.hex()}") api_state = await self.api.beacon.state(root=state_root) state = api_state.beacon_state print(f"retrieved state: {state.hash_tree_root().hex()}") temp_header = state.latest_block_header.copy() if temp_header.state_root == spec.Bytes32(): temp_header.state_root = state.hash_tree_root() block_root = spec.Root(temp_header.hash_tree_root()) print(f"last block of retrieved state: {block_root.hex()}") epc = eth2fastspec.EpochsContext() epc.load_state(state) await self.cache_state(block_root, state, epc) return CachedState(state, epc) out: CachedState = self.state_by_state_root_cache_dict[state_root] print( f"state (by state root) getter out: {out.state.hash_tree_root().hex()} key: ({state_root.hex()})" ) return out.copy()
async def _fetch_state_empty_slots(self, epc: eth2fastspec.EpochsContext, state: spec.BeaconState, delta_slots: int, dest: trio.MemorySendChannel, deltas_canon: int) -> spec.BeaconState: state = state.copy() epc = epc.copy() start_slot = state.slot + 1 to_slot = start_slot + delta_slots # If deltas_canon == 0, then none of the slots are canon canon_to_slot = start_slot + deltas_canon for slot in range(start_slot, to_slot): pre_state = state.copy() pre_epc = epc.copy() print( f"processing state transition of empty slot {slot} on pre-state {pre_state.hash_tree_root().hex()})" ) eth2fastspec.process_slots(epc, state, spec.Slot(slot)) block_root = spec.Root(state.latest_block_header.hash_tree_root()) await self.cache_state(block_root, state, epc) is_canon = slot < canon_to_slot await dest.send( (pre_state, state.copy(), None, pre_epc, epc, is_canon)) return state
async def backfill_cold_chain(self, from_slot: spec.Slot, to_slot: spec.Slot, dest: trio.MemorySendChannel, step_slowdown: float = 0.0): genesis = False if from_slot == 0: genesis = True from_slot = 1 # Don't try to catch an exception here, if the first thing fails, then we restart the process as a whole. api_block = await self.api.beacon.block(slot=spec.Slot(from_slot - 1)) if genesis: api_state = await self.api.beacon.state(slot=spec.Slot(0)) epc = eth2fastspec.EpochsContext() epc.load_state(api_state.beacon_state) await self.cache_state(api_block.root, api_state.beacon_state, epc) await dest.send( (None, api_state.beacon_state, None, epc, epc, True)) prev_block_root = api_block.root print(f"starting block root: {prev_block_root}") slot = from_slot while slot < to_slot: try: signed_block = (await self.api.beacon.block(slot=slot)).beacon_block except Exception as e: print(f"Failed to fetch block for slot {slot}: {e}") await trio.sleep(step_slowdown) continue if signed_block.message.slot < slot: print(f"empty slot {slot}") # We got the same block again, it's an empty slot. try: pre_state_cached = await self.get_state_by_block_and_slot( prev_block_root, spec.Slot(slot - 1)) except Exception as e: print( f"Failed to fetch state for slot {slot} after block {prev_block_root.hex()}: {e}" ) await trio.sleep(step_slowdown) continue await self._fetch_state_empty_slots(pre_state_cached.epc, pre_state_cached.state, 1, dest, 1) print(f"completed processing empty slot {slot}") else: print( f"block {signed_block.message.hash_tree_root().hex()} state_root: {signed_block.message.state_root.hex()} parent_root: {signed_block.message.parent_root.hex()} slot: {signed_block.message.slot}" ) ok = await self._fetch_state_and_process_block( signed_block=signed_block, dest=dest, is_canon=True) if not ok: print( f"state transition/fetch error! slot {slot}, block: {signed_block.message.hash_tree_root().hex()}" ) continue print(f"completed processing filled slot {slot}") prev_block_root = spec.Root( signed_block.message.hash_tree_root()) print( f"backfill: new prev block root: {prev_block_root.hex()}") slot += 1 print(f"backfill: new slot: {slot}") if step_slowdown > 0.0: # Don't spam the serving side with requests too much, pause a little await trio.sleep(step_slowdown)
async def cache_signed_block(self, signed_block: spec.SignedBeaconBlock): block = signed_block.message block_root = spec.Root(block.hash_tree_root()) print(f"caching block (root: {block_root.hex()}, slot: {block.slot})") cached = signed_block.copy() self.block_cache[block_root] = cached
def store_block(session: Session, post_state: spec.BeaconState, signed_block: spec.SignedBeaconBlock): block = signed_block.message block_root = block.hash_tree_root() body = block.body # Eth1 eth1_data = body.eth1_data eth1_data_root = eth1_data.hash_tree_root() upsert( session, Eth1Data( data_root=eth1_data_root, deposit_root=eth1_data.deposit_root, deposit_count=eth1_data.deposit_count, block_hash=eth1_data.block_hash, )) upsert( session, Eth1BlockVote( beacon_block_root=block_root, slot=block.slot, eth1_data_root=eth1_data_root, proposer_index=block.proposer_index, )) def handle_header(block: spec.BeaconBlockHeader): upsert( session, BeaconBlock( block_root=block.hash_tree_root(), slot=block.slot, proposer_index=block.proposer_index, parent_root=block.parent_root, state_root=block.state_root, body_root=block.body_root, )) def handle_signed_header(signed_block: spec.SignedBeaconBlockHeader): upsert( session, SignedBeaconBlock( root=signed_block.hash_tree_root(), signature=signed_block.signature, block_root=signed_block.message.hash_tree_root(), )) # Ugly but effective: collect operations, ensuring they are unique first, and then upsert as batch. # Proposer slashings proposer_slashing: spec.ProposerSlashing result_prop_slashing = [] result_prop_slashing_inc = [] for i, proposer_slashing in enumerate( body.proposer_slashings.readonly_iter()): handle_header(proposer_slashing.signed_header_1.message) handle_header(proposer_slashing.signed_header_2.message) handle_signed_header(proposer_slashing.signed_header_1) handle_signed_header(proposer_slashing.signed_header_2) result_prop_slashing.append( ProposerSlashing( root=proposer_slashing.hash_tree_root(), signed_header_1=proposer_slashing.signed_header_1. hash_tree_root(), signed_header_2=proposer_slashing.signed_header_2. hash_tree_root(), )) result_prop_slashing_inc.append( ProposerSlashingInclusion( intro_block_root=block_root, intro_index=i, root=proposer_slashing.hash_tree_root(), )) if len(result_prop_slashing) > 0: upsert_all(session, ProposerSlashing, result_prop_slashing) if len(result_prop_slashing_inc) > 0: upsert_all(session, ProposerSlashingInclusion, result_prop_slashing_inc) result_checkpoints: Set[spec.Checkpoint] = set() result_att_datas: Set[spec.AttestationData] = set() def handle_att_data(data: spec.AttestationData): result_checkpoints.add(data.source) result_checkpoints.add(data.target) result_att_datas.add(data) result_indexed_atts: Set[spec.IndexedAttestation] = set() bits_to_indexed: Dict[spec.Root, spec.Root] = dict() def handle_indexed_att(indexed: spec.IndexedAttestation): result_indexed_atts.add(indexed) # Attester slashings attester_slashing: spec.AttesterSlashing result_att_slashing = [] result_att_slashing_inc = [] for i, attester_slashing in enumerate( body.attester_slashings.readonly_iter()): handle_att_data(attester_slashing.attestation_1.data) handle_att_data(attester_slashing.attestation_2.data) handle_indexed_att(attester_slashing.attestation_1) handle_indexed_att(attester_slashing.attestation_2) result_att_slashing.append( AttesterSlashing( root=attester_slashing.hash_tree_root(), attestation_1=attester_slashing.attestation_1.hash_tree_root(), attestation_2=attester_slashing.attestation_2.hash_tree_root(), )) result_att_slashing_inc.append( AttesterSlashingInclusion( intro_block_root=block_root, intro_index=i, root=attester_slashing.hash_tree_root(), )) # Attestations attestation: spec.Attestation result_pending_atts: PyList[spec.IndexedAttestation] = [] for i, attestation in enumerate(body.attestations.readonly_iter()): data = attestation.data handle_att_data(data) indexed = spec.get_indexed_attestation(post_state, attestation) bits_to_indexed[spec.Root(attestation.hash_tree_root())] = spec.Root( indexed.hash_tree_root()) handle_indexed_att(indexed) result_pending_atts.append(indexed) if len(result_checkpoints) > 0: upsert_all(session, Checkpoint, [ Checkpoint( checkpoint_root=ch.hash_tree_root(), epoch=ch.epoch, block_root=ch.root, ) for ch in result_checkpoints ]) if len(result_att_datas) > 0: upsert_all(session, AttestationData, [ AttestationData( att_data_root=data.hash_tree_root(), slot=data.slot, index=data.index, beacon_block_root=data.beacon_block_root, source=data.source.hash_tree_root(), target=data.target.hash_tree_root(), ) for data in result_att_datas ]) if len(bits_to_indexed) > 0: upsert_all(session, BitsAttestation, [ BitsAttestation( bits_attestation_root=attestation_root, indexed_attestation_root=indexed_root, ) for attestation_root, indexed_root in bits_to_indexed.items() ]) if len(result_indexed_atts) > 0: upsert_all(session, IndexedAttestation, [ IndexedAttestation( indexed_attestation_root=indexed.hash_tree_root(), attesting_indices=', '.join( map(str, indexed.attesting_indices.readonly_iter())), data=indexed.data.hash_tree_root(), signature=indexed.signature, ) for indexed in result_indexed_atts ]) if len(result_pending_atts) > 0: upsert_all(session, PendingAttestation, [ PendingAttestation( intro_block_root=block_root, intro_index=i, indexed_att=indexed.hash_tree_root(), inclusion_delay=block.slot - indexed.data.slot, proposer_index=block.proposer_index, ) for i, indexed in enumerate(result_pending_atts) ]) # After inserting the attestations, do the attester slashings (attestations may be foreign key) if len(result_att_slashing) > 0: upsert_all(session, AttesterSlashing, result_att_slashing) if len(result_att_slashing_inc) > 0: upsert_all(session, AttesterSlashingInclusion, result_att_slashing_inc) # Deposits deposit: spec.Deposit pre_dep_count = post_state.eth1_deposit_index - len(body.deposits) result_dep_datas: Set[spec.DepositData] = set() result_deps: PyList[Deposit] = [] result_dep_incl: PyList[DepositInclusion] = [] for i, deposit in enumerate(body.deposits.readonly_iter()): data = deposit.data dep_data_root = data.hash_tree_root() result_dep_datas.add(data) result_deps.append( Deposit( root=deposit.hash_tree_root(), deposit_index=pre_dep_count + i, dep_tree_root=post_state.eth1_data.deposit_root, data=dep_data_root, )) result_dep_incl.append( DepositInclusion( intro_block_root=block_root, intro_index=i, root=deposit.hash_tree_root(), )) if len(result_dep_datas) > 0: upsert_all(session, DepositData, [ DepositData( data_root=data.hash_tree_root(), pubkey=data.pubkey, withdrawal_credentials=data.withdrawal_credentials, amount=data.amount, signature=data.signature, ) for data in result_dep_datas ]) if len(result_deps) > 0: upsert_all(session, Deposit, result_deps) if len(result_dep_incl) > 0: upsert_all(session, DepositInclusion, result_dep_incl) # Voluntary Exits sig_vol_exit: spec.SignedVoluntaryExit vol_exits = list(body.voluntary_exits.readonly_iter()) if len(vol_exits) > 0: upsert_all(session, SignedVoluntaryExit, [ SignedVoluntaryExit( root=sig_vol_exit.hash_tree_root(), epoch=sig_vol_exit.message.epoch, validator_index=sig_vol_exit.message.validator_index, signature=sig_vol_exit.signature, ) for sig_vol_exit in vol_exits ]) upsert_all(session, SignedVoluntaryExitInclusion, [ SignedVoluntaryExitInclusion( intro_block_root=block_root, intro_index=i, root=sig_vol_exit.hash_tree_root(), ) for i, sig_vol_exit in enumerate(vol_exits) ]) # The body upsert( session, BeaconBlockBody( body_root=body.hash_tree_root(), randao_reveal=body.randao_reveal, eth1_data_root=body.eth1_data.hash_tree_root(), graffiti=body.graffiti, # Operations proposer_slashings_count=len(body.proposer_slashings), attester_slashings_count=len(body.attester_slashings), attestations_count=len(body.attestations), deposits_count=len(body.deposits), voluntary_exits_count=len(body.voluntary_exits), )) # The block itself upsert( session, BeaconBlock( block_root=block_root, slot=block.slot, proposer_index=block.proposer_index, parent_root=block.parent_root, state_root=block.state_root, body_root=body.hash_tree_root(), )) # Block signature upsert( session, SignedBeaconBlock( root=signed_block.hash_tree_root(), signature=signed_block.signature, block_root=block_root, ))