def _persist_uncles(db: DatabaseAPI, uncles: Tuple[BlockHeaderAPI,
...]) -> Hash32:
uncles_hash = keccak(rlp.encode(uncles))
db.set(uncles_hash,
rlp.encode(uncles, sedes=rlp.sedes.CountableList(HeaderSedes)))
return cast(Hash32, uncles_hash)
def _add_block_hash_tree_root_to_signing_root_lookup(
db: DatabaseAPI, block: BaseBeaconBlock
) -> None:
key = SchemaV1.make_block_hash_tree_root_to_signing_root_lookup_key(
block.hash_tree_root
)
db.set(key, block.signing_root)
def _update_header_chain_gaps(cls,
db: DatabaseAPI,
persisting_header: BlockHeaderAPI,
base_gaps: ChainGaps = None) -> GapInfo:
# The only reason we overwrite this here is to be able to detect when the HeaderDB
# de-canonicalizes an uncle that should cause us to re-open a block gap.
gap_change, gaps = super()._update_header_chain_gaps(
db, persisting_header, base_gaps)
if gap_change is not GapChange.NoChange or persisting_header.block_number == 0:
return gap_change, gaps
# We have written a header for which block number we've already had a header.
# This might be a sign of a de-canonicalized uncle.
current_gaps = cls._get_chain_gaps(db)
if not is_block_number_in_gap(persisting_header.block_number,
current_gaps):
# ChainDB believes we have that block. If the header has changed, we need to re-open
# a gap for the corresponding block.
old_canonical_header = cls._get_canonical_block_header_by_number(
db, persisting_header.block_number)
if old_canonical_header != persisting_header:
updated_gaps = reopen_gap(persisting_header.block_number,
current_gaps)
db.set(SchemaV1.make_chain_gaps_lookup_key(),
rlp.encode(updated_gaps, sedes=chain_gaps))
return gap_change, gaps
def _add_block_slot_to_root_lookup(db: DatabaseAPI, block: BaseBeaconBlock) -> None:
"""
Set a record in the database to allow looking up this block by its
block slot.
"""
block_slot_to_root_key = SchemaV1.make_block_slot_to_root_lookup_key(block.slot)
db.set(block_slot_to_root_key, block.message.hash_tree_root)
def _persist_canonical_epoch_info(db: DatabaseAPI, state: BeaconState) -> None:
epoch_info = EpochInfo(
previous_justified_checkpoint=state.previous_justified_checkpoint,
current_justified_checkpoint=state.current_justified_checkpoint,
finalized_checkpoint=state.finalized_checkpoint,
)
db.set(SchemaV1.make_canonical_epoch_info_lookup_key(), ssz.encode(epoch_info))
def _persist_header_chain(
cls, db: DatabaseAPI, headers: Iterable[BlockHeaderAPI]
) -> Tuple[Tuple[BlockHeaderAPI, ...], Tuple[BlockHeaderAPI, ...]]:
headers_iterator = iter(headers)
try:
first_header = first(headers_iterator)
except StopIteration:
return tuple(), tuple()
is_genesis = first_header.parent_hash == GENESIS_PARENT_HASH
if not is_genesis and not cls._header_exists(db,
first_header.parent_hash):
raise ParentNotFound(
"Cannot persist block header ({}) with unknown parent ({})".
format(encode_hex(first_header.hash),
encode_hex(first_header.parent_hash)))
if is_genesis:
score = 0
else:
score = cls._get_score(db, first_header.parent_hash)
curr_chain_head = first_header
db.set(
curr_chain_head.hash,
rlp.encode(curr_chain_head),
)
score = cls._set_hash_scores_to_db(db, curr_chain_head, score)
orig_headers_seq = concat([(first_header, ), headers_iterator])
for parent, child in sliding_window(2, orig_headers_seq):
if parent.hash != child.parent_hash:
raise ValidationError(
"Non-contiguous chain. Expected {} to have {} as parent but was {}"
.format(
encode_hex(child.hash),
encode_hex(parent.hash),
encode_hex(child.parent_hash),
))
curr_chain_head = child
db.set(
curr_chain_head.hash,
rlp.encode(curr_chain_head),
)
score = cls._set_hash_scores_to_db(db, curr_chain_head, score)
try:
previous_canonical_head = cls._get_canonical_head(db).hash
head_score = cls._get_score(db, previous_canonical_head)
except CanonicalHeadNotFound:
return cls._set_as_canonical_chain_head(db, curr_chain_head.hash)
if score > head_score:
return cls._set_as_canonical_chain_head(db, curr_chain_head.hash)
return tuple(), tuple()
def _add_block_root_to_slot_lookup(db: DatabaseAPI, block: BaseBeaconBlock) -> None:
"""
Set a record in the database to allow looking up the slot number by its
block root.
"""
block_root_to_slot_key = SchemaV1.make_block_root_to_slot_lookup_key(
block.message.hash_tree_root
)
db.set(block_root_to_slot_key, ssz.encode(block.slot, sedes=ssz.sedes.uint64))
def _persist_checkpoint_header(cls, db: DatabaseAPI,
header: BlockHeaderAPI, score: int) -> None:
db.set(
header.hash,
rlp.encode(header),
)
previous_score = score - header.difficulty
cls._set_hash_scores_to_db(db, header, previous_score)
cls._set_as_canonical_chain_head(db, header.hash, header.parent_hash)
def _remove_transaction_from_canonical_chain(
db: DatabaseAPI, transaction_hash: Hash32) -> None:
"""
Removes the transaction specified by the given hash from the canonical
chain.
"""
db.delete(
SchemaV1.make_transaction_hash_to_block_lookup_key(
transaction_hash))
def test_database_api_delete(self, db: DatabaseAPI) -> None:
db[b'key-1'] = b'value-1'
assert b'key-1' in db
db.delete(b'key-1')
assert not db.exists(b'key-1')
assert b'key-1' not in db
def _set_hash_scores_to_db(cls, db: DatabaseAPI, header: BlockHeaderAPI,
score: int) -> int:
new_score = score + header.difficulty
db.set(
SchemaV1.make_block_hash_to_score_lookup_key(header.hash),
rlp.encode(new_score, sedes=rlp.sedes.big_endian_int),
)
return new_score
def _add_attestations_root_to_block_lookup(db: DatabaseAPI,
block: BaseBeaconBlock) -> None:
root = block.signing_root
for index, attestation in enumerate(block.body.attestations):
attestation_key = AttestationKey(root, index)
db.set(
SchemaV1.make_attestation_root_to_block_lookup_key(
attestation.hash_tree_root),
ssz.encode(attestation_key),
)
def _set_block_score_to_db(db: DatabaseAPI, block: BaseBeaconBlock,
score: int) -> int:
# NOTE if we change the score serialization, we will likely need to
# patch up the fork choice logic.
# We will decide the score serialization is fixed for now.
db.set(
SchemaV1.make_block_root_to_score_lookup_key(block.signing_root),
ssz.encode(score, sedes=ssz.sedes.uint64),
)
return score
def _set_block_score_to_db(
db: DatabaseAPI, block: BaseBeaconBlock, score: BaseScore
) -> BaseScore:
# NOTE if we change the score serialization, we will likely need to
# patch up the fork choice logic.
# We will decide the score serialization is fixed for now.
db.set(
SchemaV1.make_block_root_to_score_lookup_key(block.message.hash_tree_root),
score.serialize(),
)
return score
def _add_block_number_to_hash_lookup(db: DatabaseAPI,
header: BlockHeaderAPI) -> None:
"""
Sets a record in the database to allow looking up this header by its
block number.
"""
block_number_to_hash_key = SchemaV1.make_block_number_to_hash_lookup_key(
header.block_number)
db.set(
block_number_to_hash_key,
rlp.encode(header.hash, sedes=rlp.sedes.binary),
)
def _add_block_slot_to_root_lookup(db: DatabaseAPI,
block: BaseBeaconBlock) -> None:
"""
Set a record in the database to allow looking up this block by its
block slot.
"""
block_slot_to_root_key = SchemaV1.make_block_slot_to_root_lookup_key(
block.slot)
db.set(
block_slot_to_root_key,
ssz.encode(block.signing_root, sedes=ssz.sedes.bytes32),
)
def _decanonicalize_descendant_orphans(
cls,
db: DatabaseAPI,
header: BlockHeaderAPI,
checkpoints: Tuple[Hash32, ...]) -> None:
# Determine if any children need to be de-canonicalized because they are not children of
# the new chain head
new_gaps = starting_gaps = cls._get_header_chain_gaps(db)
child_number = BlockNumber(header.block_number + 1)
try:
child = cls._get_canonical_block_header_by_number(db, child_number)
except HeaderNotFound:
# There is no canonical block here
next_invalid_child = None
else:
if child.parent_hash != header.hash:
if child.hash in checkpoints:
raise CheckpointsMustBeCanonical(
f"Trying to decanonicalize {child} while making {header} the chain tip"
)
else:
next_invalid_child = child
else:
next_invalid_child = None
while next_invalid_child:
# decanonicalize, and add gap for tracking
db.delete(SchemaV1.make_block_number_to_hash_lookup_key(child_number))
new_gaps = reopen_gap(child_number, new_gaps)
# find next child
child_number = BlockNumber(child_number + 1)
try:
# All contiguous children must now be made invalid
next_invalid_child = cls._get_canonical_block_header_by_number(db, child_number)
except HeaderNotFound:
# Found the end of this streak of canonical blocks
break
else:
if next_invalid_child.hash in checkpoints:
raise CheckpointsMustBeCanonical(
f"Trying to decanonicalize {next_invalid_child} while making {header} the"
" chain tip"
)
if new_gaps != starting_gaps:
db.set(
SchemaV1.make_header_chain_gaps_lookup_key(),
rlp.encode(new_gaps, sedes=chain_gaps)
)
def _persist_checkpoint_header(
cls,
db: DatabaseAPI,
header: BlockHeaderAPI,
score: int
) -> None:
db.set(
header.hash,
rlp.encode(header),
)
# Add new checkpoint header
previous_checkpoints = cls._get_checkpoints(db)
new_checkpoints = previous_checkpoints + (header.hash,)
db.set(
SchemaV1.make_checkpoint_headers_key(),
b''.join(new_checkpoints),
)
previous_score = score - header.difficulty
cls._set_hash_scores_to_db(db, header, previous_score)
cls._set_as_canonical_chain_head(db, header, GENESIS_PARENT_HASH)
_, gaps = cls._update_header_chain_gaps(db, header)
# check if the parent block number exists, and is not a match for checkpoint.parent_hash
parent_block_num = BlockNumber(header.block_number - 1)
try:
parent_hash = cls._get_canonical_block_hash(db, parent_block_num)
except HeaderNotFound:
# no parent to check
pass
else:
# User is asserting that the checkpoint must be canonical, so if the parent doesn't
# match, then the parent must not be canonical, and should be de-canonicalized.
if parent_hash != header.parent_hash:
# does the correct header exist in the database?
try:
true_parent = cls._get_block_header_by_hash(db, header.parent_hash)
except HeaderNotFound:
# True parent unavailable, just delete the now non-canonical one
cls._decanonicalize_single(db, parent_block_num, gaps)
else:
# True parent should have already been canonicalized during
# _set_as_canonical_chain_head()
raise ValidationError(
f"Why was a non-matching parent header {parent_hash!r} left as canonical "
f"after _set_as_canonical_chain_head() and {true_parent} is available?"
)
cls._decanonicalize_descendant_orphans(db, header, new_checkpoints)
def _add_transaction_to_canonical_chain(db: DatabaseAPI,
transaction_hash: Hash32,
block_header: BlockHeaderAPI,
index: int) -> None:
"""
:param bytes transaction_hash: the hash of the transaction to add the lookup for
:param block_header: The header of the block with the txn that is in the canonical chain
:param int index: the position of the transaction in the block
- add lookup from transaction hash to the block number and index that the body is stored at
- remove transaction hash to body lookup in the pending pool
"""
transaction_key = TransactionKey(block_header.block_number, index)
db.set(
SchemaV1.make_transaction_hash_to_block_lookup_key(transaction_hash),
rlp.encode(transaction_key),
)
def _decanonicalize_single(cls, db: DatabaseAPI, block_num: BlockNumber,
base_gaps: ChainGaps) -> ChainGaps:
"""
A single block number was found to no longer be canonical. At doc-time,
this only happens because it does not link up with a checkpoint header.
So de-canonicalize this block number and insert a gap in the tracked
chain gaps.
"""
db.delete(SchemaV1.make_block_number_to_hash_lookup_key(block_num))
new_gaps = reopen_gap(block_num, base_gaps)
if new_gaps != base_gaps:
db.set(SchemaV1.make_header_chain_gaps_lookup_key(),
rlp.encode(new_gaps, sedes=chain_gaps))
return new_gaps
def _update_chain_gaps(cls,
db: DatabaseAPI,
persisted_block: BlockAPI,
base_gaps: ChainGaps = None) -> GapInfo:
# If we make many updates in a row, we can avoid reloading the integrity info by
# continuously caching it and providing it as a parameter to this API
if base_gaps is None:
base_gaps = cls._get_chain_gaps(db)
gap_change, gaps = fill_gap(persisted_block.number, base_gaps)
if gap_change is not GapChange.NoChange:
db.set(SchemaV1.make_chain_gaps_lookup_key(),
rlp.encode(gaps, sedes=chain_gaps))
return gap_change, gaps
def _get_checkpoints(cls, db: DatabaseAPI) -> Tuple[Hash32, ...]:
concatenated_checkpoints = db.get(
SchemaV1.make_checkpoint_headers_key())
if concatenated_checkpoints is None:
return ()
else:
return tuple(
Hash32(concatenated_checkpoints[index:index + 32])
for index in range(0, len(concatenated_checkpoints), 32))
def _set_as_canonical_chain_head(
cls,
db: DatabaseAPI,
block_hash: Hash32,
) -> Tuple[Tuple[BlockHeaderAPI, ...], Tuple[BlockHeaderAPI,
...]]: # noqa: E501
try:
header = cls._get_block_header_by_hash(db, block_hash)
except HeaderNotFound:
raise ValueError(
"Cannot use unknown block hash as canonical head: {}".format(
header.hash))
new_canonical_headers = tuple(
reversed(cls._find_new_ancestors(db, header)))
old_canonical_headers = []
# remove transaction lookups for blocks that are no longer canonical
for h in new_canonical_headers:
try:
old_hash = cls._get_canonical_block_hash(db, h.block_number)
except HeaderNotFound:
# no old block, and no more possible
break
else:
old_header = cls._get_block_header_by_hash(db, old_hash)
old_canonical_headers.append(old_header)
try:
for transaction_hash in cls._get_block_transaction_hashes(
db, old_header):
cls._remove_transaction_from_canonical_chain(
db, transaction_hash)
except MissingTrieNode:
# If the transactions were never stored for the (now) non-canonical chain,
# then you don't need to remove them from the canonical chain lookup.
pass
for h in new_canonical_headers:
cls._add_block_number_to_hash_lookup(db, h)
db.set(SchemaV1.make_canonical_head_hash_lookup_key(), header.hash)
return new_canonical_headers, tuple(old_canonical_headers)
def _set_as_canonical_chain_head(
cls,
db: DatabaseAPI,
block_root: SigningRoot,
block_class: Type[BaseBeaconBlock],
) -> Tuple[Tuple[BaseBeaconBlock, ...], Tuple[BaseBeaconBlock, ...]]:
"""
Set the canonical chain HEAD to the block as specified by the
given block root.
:return: a tuple of the blocks that are newly in the canonical chain, and the blocks that
are no longer in the canonical chain
"""
try:
block = cls._get_block_by_root(db, block_root, block_class)
except BlockNotFound:
raise ValueError(
"Cannot use unknown block root as canonical head: {block_root.hex()}"
)
new_canonical_blocks = tuple(
reversed(cls._find_new_ancestors(db, block, block_class)))
old_canonical_blocks = []
for block in new_canonical_blocks:
try:
old_canonical_root = cls._get_canonical_block_root(
db, block.slot)
except BlockNotFound:
# no old_canonical block, and no more possible
break
else:
old_canonical_block = cls._get_block_by_root(
db, old_canonical_root, block_class)
old_canonical_blocks.append(old_canonical_block)
for block in new_canonical_blocks:
cls._add_block_slot_to_root_lookup(db, block)
db.set(SchemaV1.make_canonical_head_root_lookup_key(),
block.signing_root)
return new_canonical_blocks, tuple(old_canonical_blocks)
def _set_as_canonical_chain_head(
cls,
db: DatabaseAPI,
block_hash: Hash32,
genesis_parent_hash: Hash32,
) -> Tuple[Tuple[BlockHeaderAPI, ...], Tuple[BlockHeaderAPI, ...]]:
"""
Sets the canonical chain HEAD to the block header as specified by the
given block hash.
:return: a tuple of the headers that are newly in the canonical chain, and the headers that
are no longer in the canonical chain
"""
try:
header = cls._get_block_header_by_hash(db, block_hash)
except HeaderNotFound:
raise ValueError(
"Cannot use unknown block hash as canonical head: {}".format(
block_hash))
new_canonical_headers = tuple(
reversed(cls._find_new_ancestors(db, header, genesis_parent_hash)))
old_canonical_headers = []
for h in new_canonical_headers:
try:
old_canonical_hash = cls._get_canonical_block_hash(
db, h.block_number)
except HeaderNotFound:
# no old_canonical block, and no more possible
break
else:
old_canonical_header = cls._get_block_header_by_hash(
db, old_canonical_hash)
old_canonical_headers.append(old_canonical_header)
for h in new_canonical_headers:
cls._add_block_number_to_hash_lookup(db, h)
db.set(SchemaV1.make_canonical_head_hash_lookup_key(), header.hash)
return new_canonical_headers, tuple(old_canonical_headers)
def _set_as_canonical_chain_head(
cls,
db: DatabaseAPI,
header: BlockHeaderAPI,
genesis_parent_hash: Hash32,
) -> Tuple[Tuple[BlockHeaderAPI, ...], Tuple[BlockHeaderAPI, ...]]:
"""
Sets the canonical chain HEAD to the block header as specified by the
given block hash.
:return: a tuple of the headers that are newly in the canonical chain, and the headers that
are no longer in the canonical chain
:raises CheckpointsMustBeCanonical: if trying to set a head that would
de-canonicalize a checkpoint
"""
try:
current_canonical_head = cls._get_canonical_head_hash(db)
except CanonicalHeadNotFound:
current_canonical_head = None
new_canonical_headers: Tuple[BlockHeaderAPI, ...]
old_canonical_headers: Tuple[BlockHeaderAPI, ...]
if current_canonical_head and header.parent_hash == current_canonical_head:
# the calls to _find_new_ancestors and _find_headers_to_decanonicalize are
# relatively expensive, it's better to skip them in this case, where we're
# extending the canonical chain by a header
new_canonical_headers = (header, )
old_canonical_headers = ()
cls._add_block_number_to_hash_lookup(db, header)
else:
(
new_canonical_headers,
old_canonical_headers,
) = cls._canonicalize_header(db, header, genesis_parent_hash)
db.set(SchemaV1.make_canonical_head_hash_lookup_key(), header.hash)
return new_canonical_headers, old_canonical_headers
def _persist_block_chain(
cls,
db: DatabaseAPI,
blocks: Iterable[BaseBeaconBlock],
block_class: Type[BaseBeaconBlock],
fork_choice_scorings: Iterable[ForkChoiceScoringFn],
) -> Tuple[Tuple[BaseBeaconBlock, ...], Tuple[BaseBeaconBlock, ...]]:
blocks_iterator = iter(blocks)
scorings_iterator = iter(fork_choice_scorings)
try:
first_block = first(blocks_iterator)
first_scoring = first(scorings_iterator)
except StopIteration:
return tuple(), tuple()
try:
previous_canonical_head = cls._get_canonical_head(
db, block_class).signing_root
head_score = cls._get_score(db, previous_canonical_head)
except CanonicalHeadNotFound:
no_canonical_head = True
else:
no_canonical_head = False
is_genesis = first_block.is_genesis
if not is_genesis and not cls._block_exists(db,
first_block.parent_root):
raise ParentNotFound(
"Cannot persist block ({}) with unknown parent ({})".format(
encode_hex(first_block.signing_root),
encode_hex(first_block.parent_root),
))
score = first_scoring(first_block)
curr_block_head = first_block
db.set(curr_block_head.signing_root, ssz.encode(curr_block_head))
cls._add_block_root_to_slot_lookup(db, curr_block_head)
cls._set_block_score_to_db(db, curr_block_head, score)
cls._add_attestations_root_to_block_lookup(db, curr_block_head)
orig_blocks_seq = concat([(first_block, ), blocks_iterator])
for parent, child in sliding_window(2, orig_blocks_seq):
if parent.signing_root != child.parent_root:
raise ValidationError(
"Non-contiguous chain. Expected {} to have {} as parent but was {}"
.format(
encode_hex(child.signing_root),
encode_hex(parent.signing_root),
encode_hex(child.parent_root),
))
curr_block_head = child
db.set(curr_block_head.signing_root, ssz.encode(curr_block_head))
cls._add_block_root_to_slot_lookup(db, curr_block_head)
cls._add_attestations_root_to_block_lookup(db, curr_block_head)
# NOTE: len(scorings_iterator) should equal len(blocks_iterator)
try:
next_scoring = next(scorings_iterator)
except StopIteration:
raise MissingForkChoiceScoringFns
score = next_scoring(curr_block_head)
cls._set_block_score_to_db(db, curr_block_head, score)
if no_canonical_head:
return cls._set_as_canonical_chain_head(
db, curr_block_head.signing_root, block_class)
if score > head_score:
return cls._set_as_canonical_chain_head(
db, curr_block_head.signing_root, block_class)
else:
return tuple(), tuple()
def test_database_api_delete_missing_key(self, db: DatabaseAPI) -> None:
assert b'key-1' not in db
db.delete(b'key-1')
def _persist_header_chain(
cls,
db: DatabaseAPI,
headers: Iterable[BlockHeaderAPI],
genesis_parent_hash: Hash32,
) -> Tuple[Tuple[BlockHeaderAPI, ...], Tuple[BlockHeaderAPI, ...]]:
headers_iterator = iter(headers)
try:
first_header = first(headers_iterator)
except StopIteration:
return tuple(), tuple()
is_genesis = first_header.parent_hash == genesis_parent_hash
if not is_genesis and not cls._header_exists(db,
first_header.parent_hash):
raise ParentNotFound(
f"Cannot persist block header ({encode_hex(first_header.hash)}) "
f"with unknown parent ({encode_hex(first_header.parent_hash)})"
)
if is_genesis:
score = 0
else:
score = cls._get_score(db, first_header.parent_hash)
curr_chain_head = first_header
db.set(
curr_chain_head.hash,
rlp.encode(curr_chain_head),
)
score = cls._set_hash_scores_to_db(db, curr_chain_head, score)
base_gaps = cls._get_header_chain_gaps(db)
gap_info = cls._update_header_chain_gaps(db, curr_chain_head,
base_gaps)
gaps = cls._handle_gap_change(db, gap_info, curr_chain_head,
genesis_parent_hash)
orig_headers_seq = concat([(first_header, ), headers_iterator])
for parent, child in sliding_window(2, orig_headers_seq):
if parent.hash != child.parent_hash:
raise ValidationError(
f"Non-contiguous chain. Expected {encode_hex(child.hash)} "
f"to have {encode_hex(parent.hash)} as parent "
f"but was {encode_hex(child.parent_hash)}")
curr_chain_head = child
db.set(
curr_chain_head.hash,
rlp.encode(curr_chain_head),
)
score = cls._set_hash_scores_to_db(db, curr_chain_head, score)
gap_info = cls._update_header_chain_gaps(db, curr_chain_head, gaps)
gaps = cls._handle_gap_change(db, gap_info, curr_chain_head,
genesis_parent_hash)
try:
previous_canonical_head = cls._get_canonical_head_hash(db)
head_score = cls._get_score(db, previous_canonical_head)
except CanonicalHeadNotFound:
return cls._set_as_canonical_chain_head(db, curr_chain_head,
genesis_parent_hash)
if score > head_score:
return cls._set_as_canonical_chain_head(db, curr_chain_head,
genesis_parent_hash)
return tuple(), tuple()
def _write_signed_block(cls, db: DatabaseAPI,
block: BaseSignedBeaconBlock) -> None:
# TODO key by block root or signed block root?
db.set(block.message.hash_tree_root, ssz.encode(block))
cls._add_block_root_to_slot_lookup(db, block)
cls._add_attestations_root_to_block_lookup(db, block.message)
