async def _fetch_state_and_process_block(self, signed_block: Optional[
spec.SignedBeaconBlock], dest: trio.MemorySendChannel,
is_canon: bool) -> bool:
block = signed_block.message
block_root = spec.Root(block.hash_tree_root())
prev_slot = spec.Slot(block.slot - 1)
pre_state_cached: CachedState
try:
pre_state_cached = await self.get_state_by_block_and_slot(
block.parent_root, prev_slot)
except Exception as e:
print(f"Failed to fetch state: {e}")
return False
state = pre_state_cached.state
pre_state = state.copy()
epc = pre_state_cached.epc.copy()
print(
f"processing state transition of block {block_root} (slot {block.slot}) "
f"on state {state.hash_tree_root().hex()} (slot {state.slot}) "
f"(asked for state of parent {block.parent_root.hex()}, at slot {prev_slot})"
)
try:
unchecked_state_transition(epc, state, signed_block)
except Exception as e:
print(
f"WARNING: {block_root.hex()} (slot {block.slot}) failed state transition: {e}"
)
traceback.print_exc()
await self._on_bad_transition(signed_block, state)
return False
if block.state_root != state.hash_tree_root():
print(
f"WARNING: {block.hash_tree_root().hex()} (slot {block.slot}) state root ({block.state_root.hex()})"
f" does not match computed state root ({state.hash_tree_root().hex()})"
)
await self._on_bad_transition(signed_block, state)
return False
await self.cache_state(block_root, state, epc)
await dest.send((pre_state, state.copy(), signed_block,
pre_state_cached.epc, epc, is_canon))
return True
async def get_state_by_state_root(self,
state_root: spec.Root) -> CachedState:
if state_root not in self.state_by_state_root_cache_dict:
print(f"fetching state by state root {state_root.hex()}")
api_state = await self.api.beacon.state(root=state_root)
state = api_state.beacon_state
print(f"retrieved state: {state.hash_tree_root().hex()}")
temp_header = state.latest_block_header.copy()
if temp_header.state_root == spec.Bytes32():
temp_header.state_root = state.hash_tree_root()
block_root = spec.Root(temp_header.hash_tree_root())
print(f"last block of retrieved state: {block_root.hex()}")
epc = eth2fastspec.EpochsContext()
epc.load_state(state)
await self.cache_state(block_root, state, epc)
return CachedState(state, epc)
out: CachedState = self.state_by_state_root_cache_dict[state_root]
print(
f"state (by state root) getter out: {out.state.hash_tree_root().hex()} key: ({state_root.hex()})"
)
return out.copy()
async def _fetch_state_empty_slots(self, epc: eth2fastspec.EpochsContext,
state: spec.BeaconState,
delta_slots: int,
dest: trio.MemorySendChannel,
deltas_canon: int) -> spec.BeaconState:
state = state.copy()
epc = epc.copy()
start_slot = state.slot + 1
to_slot = start_slot + delta_slots
# If deltas_canon == 0, then none of the slots are canon
canon_to_slot = start_slot + deltas_canon
for slot in range(start_slot, to_slot):
pre_state = state.copy()
pre_epc = epc.copy()
print(
f"processing state transition of empty slot {slot} on pre-state {pre_state.hash_tree_root().hex()})"
)
eth2fastspec.process_slots(epc, state, spec.Slot(slot))
block_root = spec.Root(state.latest_block_header.hash_tree_root())
await self.cache_state(block_root, state, epc)
is_canon = slot < canon_to_slot
await dest.send(
(pre_state, state.copy(), None, pre_epc, epc, is_canon))
return state
async def backfill_cold_chain(self,
from_slot: spec.Slot,
to_slot: spec.Slot,
dest: trio.MemorySendChannel,
step_slowdown: float = 0.0):
genesis = False
if from_slot == 0:
genesis = True
from_slot = 1
# Don't try to catch an exception here, if the first thing fails, then we restart the process as a whole.
api_block = await self.api.beacon.block(slot=spec.Slot(from_slot - 1))
if genesis:
api_state = await self.api.beacon.state(slot=spec.Slot(0))
epc = eth2fastspec.EpochsContext()
epc.load_state(api_state.beacon_state)
await self.cache_state(api_block.root, api_state.beacon_state, epc)
await dest.send(
(None, api_state.beacon_state, None, epc, epc, True))
prev_block_root = api_block.root
print(f"starting block root: {prev_block_root}")
slot = from_slot
while slot < to_slot:
try:
signed_block = (await
self.api.beacon.block(slot=slot)).beacon_block
except Exception as e:
print(f"Failed to fetch block for slot {slot}: {e}")
await trio.sleep(step_slowdown)
continue
if signed_block.message.slot < slot:
print(f"empty slot {slot}")
# We got the same block again, it's an empty slot.
try:
pre_state_cached = await self.get_state_by_block_and_slot(
prev_block_root, spec.Slot(slot - 1))
except Exception as e:
print(
f"Failed to fetch state for slot {slot} after block {prev_block_root.hex()}: {e}"
)
await trio.sleep(step_slowdown)
continue
await self._fetch_state_empty_slots(pre_state_cached.epc,
pre_state_cached.state, 1,
dest, 1)
print(f"completed processing empty slot {slot}")
else:
print(
f"block {signed_block.message.hash_tree_root().hex()} state_root: {signed_block.message.state_root.hex()} parent_root: {signed_block.message.parent_root.hex()} slot: {signed_block.message.slot}"
)
ok = await self._fetch_state_and_process_block(
signed_block=signed_block, dest=dest, is_canon=True)
if not ok:
print(
f"state transition/fetch error! slot {slot}, block: {signed_block.message.hash_tree_root().hex()}"
)
continue
print(f"completed processing filled slot {slot}")
prev_block_root = spec.Root(
signed_block.message.hash_tree_root())
print(
f"backfill: new prev block root: {prev_block_root.hex()}")
slot += 1
print(f"backfill: new slot: {slot}")
if step_slowdown > 0.0:
# Don't spam the serving side with requests too much, pause a little
await trio.sleep(step_slowdown)
async def cache_signed_block(self, signed_block: spec.SignedBeaconBlock):
block = signed_block.message
block_root = spec.Root(block.hash_tree_root())
print(f"caching block (root: {block_root.hex()}, slot: {block.slot})")
cached = signed_block.copy()
self.block_cache[block_root] = cached
def store_block(session: Session, post_state: spec.BeaconState,
signed_block: spec.SignedBeaconBlock):
block = signed_block.message
block_root = block.hash_tree_root()
body = block.body
# Eth1
eth1_data = body.eth1_data
eth1_data_root = eth1_data.hash_tree_root()
upsert(
session,
Eth1Data(
data_root=eth1_data_root,
deposit_root=eth1_data.deposit_root,
deposit_count=eth1_data.deposit_count,
block_hash=eth1_data.block_hash,
))
upsert(
session,
Eth1BlockVote(
beacon_block_root=block_root,
slot=block.slot,
eth1_data_root=eth1_data_root,
proposer_index=block.proposer_index,
))
def handle_header(block: spec.BeaconBlockHeader):
upsert(
session,
BeaconBlock(
block_root=block.hash_tree_root(),
slot=block.slot,
proposer_index=block.proposer_index,
parent_root=block.parent_root,
state_root=block.state_root,
body_root=block.body_root,
))
def handle_signed_header(signed_block: spec.SignedBeaconBlockHeader):
upsert(
session,
SignedBeaconBlock(
root=signed_block.hash_tree_root(),
signature=signed_block.signature,
block_root=signed_block.message.hash_tree_root(),
))
# Ugly but effective: collect operations, ensuring they are unique first, and then upsert as batch.
# Proposer slashings
proposer_slashing: spec.ProposerSlashing
result_prop_slashing = []
result_prop_slashing_inc = []
for i, proposer_slashing in enumerate(
body.proposer_slashings.readonly_iter()):
handle_header(proposer_slashing.signed_header_1.message)
handle_header(proposer_slashing.signed_header_2.message)
handle_signed_header(proposer_slashing.signed_header_1)
handle_signed_header(proposer_slashing.signed_header_2)
result_prop_slashing.append(
ProposerSlashing(
root=proposer_slashing.hash_tree_root(),
signed_header_1=proposer_slashing.signed_header_1.
hash_tree_root(),
signed_header_2=proposer_slashing.signed_header_2.
hash_tree_root(),
))
result_prop_slashing_inc.append(
ProposerSlashingInclusion(
intro_block_root=block_root,
intro_index=i,
root=proposer_slashing.hash_tree_root(),
))
if len(result_prop_slashing) > 0:
upsert_all(session, ProposerSlashing, result_prop_slashing)
if len(result_prop_slashing_inc) > 0:
upsert_all(session, ProposerSlashingInclusion,
result_prop_slashing_inc)
result_checkpoints: Set[spec.Checkpoint] = set()
result_att_datas: Set[spec.AttestationData] = set()
def handle_att_data(data: spec.AttestationData):
result_checkpoints.add(data.source)
result_checkpoints.add(data.target)
result_att_datas.add(data)
result_indexed_atts: Set[spec.IndexedAttestation] = set()
bits_to_indexed: Dict[spec.Root, spec.Root] = dict()
def handle_indexed_att(indexed: spec.IndexedAttestation):
result_indexed_atts.add(indexed)
# Attester slashings
attester_slashing: spec.AttesterSlashing
result_att_slashing = []
result_att_slashing_inc = []
for i, attester_slashing in enumerate(
body.attester_slashings.readonly_iter()):
handle_att_data(attester_slashing.attestation_1.data)
handle_att_data(attester_slashing.attestation_2.data)
handle_indexed_att(attester_slashing.attestation_1)
handle_indexed_att(attester_slashing.attestation_2)
result_att_slashing.append(
AttesterSlashing(
root=attester_slashing.hash_tree_root(),
attestation_1=attester_slashing.attestation_1.hash_tree_root(),
attestation_2=attester_slashing.attestation_2.hash_tree_root(),
))
result_att_slashing_inc.append(
AttesterSlashingInclusion(
intro_block_root=block_root,
intro_index=i,
root=attester_slashing.hash_tree_root(),
))
# Attestations
attestation: spec.Attestation
result_pending_atts: PyList[spec.IndexedAttestation] = []
for i, attestation in enumerate(body.attestations.readonly_iter()):
data = attestation.data
handle_att_data(data)
indexed = spec.get_indexed_attestation(post_state, attestation)
bits_to_indexed[spec.Root(attestation.hash_tree_root())] = spec.Root(
indexed.hash_tree_root())
handle_indexed_att(indexed)
result_pending_atts.append(indexed)
if len(result_checkpoints) > 0:
upsert_all(session, Checkpoint, [
Checkpoint(
checkpoint_root=ch.hash_tree_root(),
epoch=ch.epoch,
block_root=ch.root,
) for ch in result_checkpoints
])
if len(result_att_datas) > 0:
upsert_all(session, AttestationData, [
AttestationData(
att_data_root=data.hash_tree_root(),
slot=data.slot,
index=data.index,
beacon_block_root=data.beacon_block_root,
source=data.source.hash_tree_root(),
target=data.target.hash_tree_root(),
) for data in result_att_datas
])
if len(bits_to_indexed) > 0:
upsert_all(session, BitsAttestation, [
BitsAttestation(
bits_attestation_root=attestation_root,
indexed_attestation_root=indexed_root,
) for attestation_root, indexed_root in bits_to_indexed.items()
])
if len(result_indexed_atts) > 0:
upsert_all(session, IndexedAttestation, [
IndexedAttestation(
indexed_attestation_root=indexed.hash_tree_root(),
attesting_indices=', '.join(
map(str, indexed.attesting_indices.readonly_iter())),
data=indexed.data.hash_tree_root(),
signature=indexed.signature,
) for indexed in result_indexed_atts
])
if len(result_pending_atts) > 0:
upsert_all(session, PendingAttestation, [
PendingAttestation(
intro_block_root=block_root,
intro_index=i,
indexed_att=indexed.hash_tree_root(),
inclusion_delay=block.slot - indexed.data.slot,
proposer_index=block.proposer_index,
) for i, indexed in enumerate(result_pending_atts)
])
# After inserting the attestations, do the attester slashings (attestations may be foreign key)
if len(result_att_slashing) > 0:
upsert_all(session, AttesterSlashing, result_att_slashing)
if len(result_att_slashing_inc) > 0:
upsert_all(session, AttesterSlashingInclusion, result_att_slashing_inc)
# Deposits
deposit: spec.Deposit
pre_dep_count = post_state.eth1_deposit_index - len(body.deposits)
result_dep_datas: Set[spec.DepositData] = set()
result_deps: PyList[Deposit] = []
result_dep_incl: PyList[DepositInclusion] = []
for i, deposit in enumerate(body.deposits.readonly_iter()):
data = deposit.data
dep_data_root = data.hash_tree_root()
result_dep_datas.add(data)
result_deps.append(
Deposit(
root=deposit.hash_tree_root(),
deposit_index=pre_dep_count + i,
dep_tree_root=post_state.eth1_data.deposit_root,
data=dep_data_root,
))
result_dep_incl.append(
DepositInclusion(
intro_block_root=block_root,
intro_index=i,
root=deposit.hash_tree_root(),
))
if len(result_dep_datas) > 0:
upsert_all(session, DepositData, [
DepositData(
data_root=data.hash_tree_root(),
pubkey=data.pubkey,
withdrawal_credentials=data.withdrawal_credentials,
amount=data.amount,
signature=data.signature,
) for data in result_dep_datas
])
if len(result_deps) > 0:
upsert_all(session, Deposit, result_deps)
if len(result_dep_incl) > 0:
upsert_all(session, DepositInclusion, result_dep_incl)
# Voluntary Exits
sig_vol_exit: spec.SignedVoluntaryExit
vol_exits = list(body.voluntary_exits.readonly_iter())
if len(vol_exits) > 0:
upsert_all(session, SignedVoluntaryExit, [
SignedVoluntaryExit(
root=sig_vol_exit.hash_tree_root(),
epoch=sig_vol_exit.message.epoch,
validator_index=sig_vol_exit.message.validator_index,
signature=sig_vol_exit.signature,
) for sig_vol_exit in vol_exits
])
upsert_all(session, SignedVoluntaryExitInclusion, [
SignedVoluntaryExitInclusion(
intro_block_root=block_root,
intro_index=i,
root=sig_vol_exit.hash_tree_root(),
) for i, sig_vol_exit in enumerate(vol_exits)
])
# The body
upsert(
session,
BeaconBlockBody(
body_root=body.hash_tree_root(),
randao_reveal=body.randao_reveal,
eth1_data_root=body.eth1_data.hash_tree_root(),
graffiti=body.graffiti,
# Operations
proposer_slashings_count=len(body.proposer_slashings),
attester_slashings_count=len(body.attester_slashings),
attestations_count=len(body.attestations),
deposits_count=len(body.deposits),
voluntary_exits_count=len(body.voluntary_exits),
))
# The block itself
upsert(
session,
BeaconBlock(
block_root=block_root,
slot=block.slot,
proposer_index=block.proposer_index,
parent_root=block.parent_root,
state_root=block.state_root,
body_root=body.hash_tree_root(),
))
# Block signature
upsert(
session,
SignedBeaconBlock(
root=signed_block.hash_tree_root(),
signature=signed_block.signature,
block_root=block_root,
))