class AccountStorageDB(AccountStorageDatabaseAPI):
logger = get_extended_debug_logger("eth.db.storage.AccountStorageDB")
def __init__(self, db: AtomicDatabaseAPI, storage_root: Hash32, address: Address) -> None:
"""
Database entries go through several pipes, like so...
.. code::
db -> _storage_lookup -> _storage_cache -> _locked_changes -> _journal_storage
db is the raw database, we can assume it hits disk when written to.
Keys are stored as node hashes and rlp-encoded node values.
_storage_lookup is itself a pair of databases: (BatchDB -> HexaryTrie),
writes to storage lookup *are* immeditaely applied to a trie, generating
the appropriate trie nodes and and root hash (via the HexaryTrie). The
writes are *not* persisted to db, until _storage_lookup is explicitly instructed to,
via :meth:`StorageLookup.commit_to`
_storage_cache is a cache tied to the state root of the trie. It
is important that this cache is checked *after* looking for
the key in _journal_storage, because the cache is only invalidated
after a state root change. Otherwise, you will see data since the last
storage root was calculated.
_locked_changes is a batch database that includes only those values that are
un-revertable in the EVM. Currently, that means changes that completed in a
previous transaction.
Journaling batches writes at the _journal_storage layer, until persist is called.
It manages all the checkpointing and rollbacks that happen during EVM execution.
In both _storage_cache and _journal_storage, Keys are set/retrieved as the
big_endian encoding of the slot integer, and the rlp-encoded value.
"""
self._address = address
self._storage_lookup = StorageLookup(db, storage_root, address)
self._storage_cache = CacheDB(self._storage_lookup)
self._locked_changes = BatchDB(self._storage_cache)
self._journal_storage = JournalDB(self._locked_changes)
def get(self, slot: int, from_journal: bool=True) -> int:
key = int_to_big_endian(slot)
lookup_db = self._journal_storage if from_journal else self._locked_changes
try:
encoded_value = lookup_db[key]
except MissingStorageTrieNode:
raise
except KeyError:
return 0
if encoded_value == b'':
return 0
else:
return rlp.decode(encoded_value, sedes=rlp.sedes.big_endian_int)
def set(self, slot: int, value: int) -> None:
key = int_to_big_endian(slot)
if value:
self._journal_storage[key] = rlp.encode(value)
else:
del self._journal_storage[key]
def delete(self) -> None:
self.logger.debug2(
"Deleting all storage in account 0x%s, hashed 0x%s",
self._address.hex(),
keccak(self._address).hex(),
)
self._journal_storage.clear()
self._storage_cache.reset_cache()
def record(self, checkpoint: JournalDBCheckpoint) -> None:
self._journal_storage.record(checkpoint)
def discard(self, checkpoint: JournalDBCheckpoint) -> None:
self.logger.debug2('discard checkpoint %r', checkpoint)
if self._journal_storage.has_checkpoint(checkpoint):
self._journal_storage.discard(checkpoint)
else:
# if the checkpoint comes before this account started tracking,
# then simply reset to the beginning
self._journal_storage.reset()
self._storage_cache.reset_cache()
def commit(self, checkpoint: JournalDBCheckpoint) -> None:
if self._journal_storage.has_checkpoint(checkpoint):
self._journal_storage.commit(checkpoint)
else:
# if the checkpoint comes before this account started tracking,
# then flatten all changes, without persisting
self._journal_storage.flatten()
def lock_changes(self) -> None:
self._journal_storage.persist()
def make_storage_root(self) -> None:
self.lock_changes()
self._locked_changes.commit(apply_deletes=True)
def _validate_flushed(self) -> None:
"""
Will raise an exception if there are some changes made since the last persist.
"""
journal_diff = self._journal_storage.diff()
if len(journal_diff) > 0:
raise ValidationError(
f"StorageDB had a dirty journal when it needed to be clean: {journal_diff!r}"
)
@property
def has_changed_root(self) -> bool:
return self._storage_lookup.has_changed_root
def get_changed_root(self) -> Hash32:
return self._storage_lookup.get_changed_root()
def persist(self, db: DatabaseAPI) -> None:
self._validate_flushed()
if self._storage_lookup.has_changed_root:
self._storage_lookup.commit_to(db)
class AccountStorageDB(AccountStorageDatabaseAPI):
logger = get_extended_debug_logger("eth.db.storage.AccountStorageDB")
def __init__(self, db: AtomicDatabaseAPI, storage_root: Hash32,
address: Address) -> None:
"""
Database entries go through several pipes, like so...
.. code::
db -> _storage_lookup -> _storage_cache -> _locked_changes -> _journal_storage
db is the raw database, we can assume it hits disk when written to.
Keys are stored as node hashes and rlp-encoded node values.
_storage_lookup is itself a pair of databases: (BatchDB -> HexaryTrie),
writes to storage lookup *are* immeditaely applied to a trie, generating
the appropriate trie nodes and and root hash (via the HexaryTrie). The
writes are *not* persisted to db, until _storage_lookup is explicitly instructed to,
via :meth:`StorageLookup.commit_to`
_storage_cache is a cache tied to the state root of the trie. It
is important that this cache is checked *after* looking for
the key in _journal_storage, because the cache is only invalidated
after a state root change. Otherwise, you will see data since the last
storage root was calculated.
_locked_changes is a batch database that includes only those values that are
un-revertable in the EVM. Currently, that means changes that completed in a
previous transaction.
Journaling batches writes at the _journal_storage layer, until persist is called.
It manages all the checkpointing and rollbacks that happen during EVM execution.
In both _storage_cache and _journal_storage, Keys are set/retrieved as the
big_endian encoding of the slot integer, and the rlp-encoded value.
"""
self._address = address
self._storage_lookup = StorageLookup(db, storage_root, address)
self._storage_cache = CacheDB(self._storage_lookup)
self._locked_changes = JournalDB(self._storage_cache)
self._journal_storage = JournalDB(self._locked_changes)
# Track how many times we have cleared the storage. This is journaled
# in lockstep with other storage changes. That way, we can detect if a revert
# causes use to revert past the previous storage deletion. The clear count is used
# as an index to find the base trie from before the revert.
self._clear_count = JournalDB(
MemoryDB({CLEAR_COUNT_KEY_NAME: to_bytes(0)}))
def get(self, slot: int, from_journal: bool = True) -> int:
key = int_to_big_endian(slot)
lookup_db = self._journal_storage if from_journal else self._locked_changes
try:
encoded_value = lookup_db[key]
except MissingStorageTrieNode:
raise
except KeyError:
return 0
if encoded_value == b'':
return 0
else:
return rlp.decode(encoded_value, sedes=rlp.sedes.big_endian_int)
def set(self, slot: int, value: int) -> None:
key = int_to_big_endian(slot)
if value:
self._journal_storage[key] = rlp.encode(value)
else:
try:
current_val = self._journal_storage[key]
except KeyError:
# deleting an empty key has no effect
return
else:
if current_val != b'':
# only try to delete the value if it's present
del self._journal_storage[key]
def delete(self) -> None:
self.logger.debug2(
"Deleting all storage in account 0x%s",
self._address.hex(),
)
self._journal_storage.clear()
self._storage_cache.reset_cache()
# Empty out the storage lookup trie (keeping history, in case of a revert)
new_clear_count = self._storage_lookup.new_trie()
# Look up the previous count of how many times the account has been deleted.
# This can happen multiple times in one block, via CREATE2.
old_clear_count = to_int(self._clear_count[CLEAR_COUNT_KEY_NAME])
# Gut check that we have incremented correctly
if new_clear_count != old_clear_count + 1:
raise ValidationError(
f"Must increase clear count by one on each delete. Instead, went from"
f" {old_clear_count} -> {new_clear_count} in account 0x{self._address.hex()}"
)
# Save the new count, ie~ the index used for a future revert.
self._clear_count[CLEAR_COUNT_KEY_NAME] = to_bytes(new_clear_count)
def record(self, checkpoint: JournalDBCheckpoint) -> None:
self._journal_storage.record(checkpoint)
self._clear_count.record(checkpoint)
def discard(self, checkpoint: JournalDBCheckpoint) -> None:
self.logger.debug2('discard checkpoint %r', checkpoint)
latest_clear_count = to_int(self._clear_count[CLEAR_COUNT_KEY_NAME])
if self._journal_storage.has_checkpoint(checkpoint):
self._journal_storage.discard(checkpoint)
self._clear_count.discard(checkpoint)
else:
# if the checkpoint comes before this account started tracking,
# then simply reset to the beginning
self._journal_storage.reset()
self._clear_count.reset()
self._storage_cache.reset_cache()
reverted_clear_count = to_int(self._clear_count[CLEAR_COUNT_KEY_NAME])
if reverted_clear_count == latest_clear_count - 1:
# This revert rewinds past a trie deletion, so roll back to the trie at
# that point. We use the clear count as an index to get back to the
# old base trie.
self._storage_lookup.rollback_trie(reverted_clear_count)
elif reverted_clear_count == latest_clear_count:
# No change in the base trie, take no action
pass
else:
# Although CREATE2 permits multiple creates and deletes in a single block,
# you can still only revert across a single delete. That's because delete
# is only triggered at the end of the transaction.
raise ValidationError(
f"This revert has changed the clear count in an invalid way, from"
f" {latest_clear_count} to {reverted_clear_count}, in 0x{self._address.hex()}"
)
def commit(self, checkpoint: JournalDBCheckpoint) -> None:
if self._journal_storage.has_checkpoint(checkpoint):
self._journal_storage.commit(checkpoint)
self._clear_count.commit(checkpoint)
else:
# if the checkpoint comes before this account started tracking,
# then flatten all changes, without persisting
self._journal_storage.flatten()
self._clear_count.flatten()
def lock_changes(self) -> None:
if self._journal_storage.has_clear():
self._locked_changes.clear()
self._journal_storage.persist()
def make_storage_root(self) -> None:
self.lock_changes()
self._locked_changes.persist()
def _validate_flushed(self) -> None:
"""
Will raise an exception if there are some changes made since the last persist.
"""
journal_diff = self._journal_storage.diff()
if len(journal_diff) > 0:
raise ValidationError(
f"StorageDB had a dirty journal when it needed to be clean: {journal_diff!r}"
)
@property
def has_changed_root(self) -> bool:
return self._storage_lookup.has_changed_root
def get_changed_root(self) -> Hash32:
return self._storage_lookup.get_changed_root()
def persist(self, db: DatabaseAPI) -> None:
self._validate_flushed()
if self._storage_lookup.has_changed_root:
self._storage_lookup.commit_to(db)
class JournalComparison(RuleBasedStateMachine):
"""
Compare an older version of JournalDB against a newer, optimized one.
"""
def __init__(self):
super().__init__()
self.slow_wrapped = {}
self.slow_journal = SlowJournalDB(self.slow_wrapped)
self.fast_wrapped = {}
self.fast_journal = JournalDB(self.fast_wrapped)
keys = Bundle('keys')
values = Bundle('values')
checkpoints = Bundle('checkpoints')
@rule(target=keys, k=st.binary())
def add_key(self, k):
return k
@rule(target=values, v=st.binary())
def add_value(self, v):
return v
@rule(target=checkpoints)
def record(self):
slow_checkpoint = self.slow_journal.record()
fast_checkpoint = self.fast_journal.record()
assert self.slow_journal.diff() == self.fast_journal.diff()
return slow_checkpoint, fast_checkpoint
@rule(k=keys, v=values)
def set(self, k, v):
self.slow_journal[k] = v
self.fast_journal[k] = v
assert self.slow_journal.diff() == self.fast_journal.diff()
@rule(k=keys)
def delete(self, k):
if k not in self.slow_journal:
assert k not in self.fast_journal
return
else:
del self.slow_journal[k]
del self.fast_journal[k]
assert self.slow_journal.diff() == self.fast_journal.diff()
@rule(c=checkpoints)
def commit(self, c):
slow_checkpoint, fast_checkpoint = c
if not self.slow_journal.has_changeset(slow_checkpoint):
assert not self.fast_journal.has_checkpoint(fast_checkpoint)
return
else:
self.slow_journal.commit(slow_checkpoint)
self.fast_journal.commit(fast_checkpoint)
assert self.slow_journal.diff() == self.fast_journal.diff()
@rule(c=checkpoints)
def discard(self, c):
slow_checkpoint, fast_checkpoint = c
if not self.slow_journal.has_changeset(slow_checkpoint):
assert not self.fast_journal.has_checkpoint(fast_checkpoint)
else:
self.slow_journal.discard(slow_checkpoint)
self.fast_journal.discard(fast_checkpoint)
assert self.slow_journal.diff() == self.fast_journal.diff()
@rule()
def flatten(self):
self.slow_journal.flatten()
self.fast_journal.flatten()
assert self.slow_journal.diff() == self.fast_journal.diff()
@rule()
def persist(self):
self.slow_journal.persist()
self.fast_journal.persist()
assert self.slow_wrapped == self.fast_wrapped
