def test_journal_db_diff_respects_clear():
memory_db = MemoryDB({})
journal_db = JournalDB(memory_db)
journal_db[b'first'] = b'val'
journal_db.clear()
pending = journal_db.diff().pending_items()
assert len(pending) == 0
class AccountStorageDB(AccountStorageDatabaseAPI):
logger = get_extended_debug_logger("eth.db.storage.AccountStorageDB")
def __init__(self, db: AtomicDatabaseAPI, storage_root: Hash32,
address: Address) -> None:
"""
Database entries go through several pipes, like so...
.. code::
db -> _storage_lookup -> _storage_cache -> _locked_changes -> _journal_storage
db is the raw database, we can assume it hits disk when written to.
Keys are stored as node hashes and rlp-encoded node values.
_storage_lookup is itself a pair of databases: (BatchDB -> HexaryTrie),
writes to storage lookup *are* immeditaely applied to a trie, generating
the appropriate trie nodes and and root hash (via the HexaryTrie). The
writes are *not* persisted to db, until _storage_lookup is explicitly instructed to,
via :meth:`StorageLookup.commit_to`
_storage_cache is a cache tied to the state root of the trie. It
is important that this cache is checked *after* looking for
the key in _journal_storage, because the cache is only invalidated
after a state root change. Otherwise, you will see data since the last
storage root was calculated.
_locked_changes is a batch database that includes only those values that are
un-revertable in the EVM. Currently, that means changes that completed in a
previous transaction.
Journaling batches writes at the _journal_storage layer, until persist is called.
It manages all the checkpointing and rollbacks that happen during EVM execution.
In both _storage_cache and _journal_storage, Keys are set/retrieved as the
big_endian encoding of the slot integer, and the rlp-encoded value.
"""
self._address = address
self._storage_lookup = StorageLookup(db, storage_root, address)
self._storage_cache = CacheDB(self._storage_lookup)
self._locked_changes = JournalDB(self._storage_cache)
self._journal_storage = JournalDB(self._locked_changes)
# Track how many times we have cleared the storage. This is journaled
# in lockstep with other storage changes. That way, we can detect if a revert
# causes use to revert past the previous storage deletion. The clear count is used
# as an index to find the base trie from before the revert.
self._clear_count = JournalDB(
MemoryDB({CLEAR_COUNT_KEY_NAME: to_bytes(0)}))
def get(self, slot: int, from_journal: bool = True) -> int:
key = int_to_big_endian(slot)
lookup_db = self._journal_storage if from_journal else self._locked_changes
try:
encoded_value = lookup_db[key]
except MissingStorageTrieNode:
raise
except KeyError:
return 0
if encoded_value == b'':
return 0
else:
return rlp.decode(encoded_value, sedes=rlp.sedes.big_endian_int)
def set(self, slot: int, value: int) -> None:
key = int_to_big_endian(slot)
if value:
self._journal_storage[key] = rlp.encode(value)
else:
try:
current_val = self._journal_storage[key]
except KeyError:
# deleting an empty key has no effect
return
else:
if current_val != b'':
# only try to delete the value if it's present
del self._journal_storage[key]
def delete(self) -> None:
self.logger.debug2(
"Deleting all storage in account 0x%s",
self._address.hex(),
)
self._journal_storage.clear()
self._storage_cache.reset_cache()
# Empty out the storage lookup trie (keeping history, in case of a revert)
new_clear_count = self._storage_lookup.new_trie()
# Look up the previous count of how many times the account has been deleted.
# This can happen multiple times in one block, via CREATE2.
old_clear_count = to_int(self._clear_count[CLEAR_COUNT_KEY_NAME])
# Gut check that we have incremented correctly
if new_clear_count != old_clear_count + 1:
raise ValidationError(
f"Must increase clear count by one on each delete. Instead, went from"
f" {old_clear_count} -> {new_clear_count} in account 0x{self._address.hex()}"
)
# Save the new count, ie~ the index used for a future revert.
self._clear_count[CLEAR_COUNT_KEY_NAME] = to_bytes(new_clear_count)
def record(self, checkpoint: JournalDBCheckpoint) -> None:
self._journal_storage.record(checkpoint)
self._clear_count.record(checkpoint)
def discard(self, checkpoint: JournalDBCheckpoint) -> None:
self.logger.debug2('discard checkpoint %r', checkpoint)
latest_clear_count = to_int(self._clear_count[CLEAR_COUNT_KEY_NAME])
if self._journal_storage.has_checkpoint(checkpoint):
self._journal_storage.discard(checkpoint)
self._clear_count.discard(checkpoint)
else:
# if the checkpoint comes before this account started tracking,
# then simply reset to the beginning
self._journal_storage.reset()
self._clear_count.reset()
self._storage_cache.reset_cache()
reverted_clear_count = to_int(self._clear_count[CLEAR_COUNT_KEY_NAME])
if reverted_clear_count == latest_clear_count - 1:
# This revert rewinds past a trie deletion, so roll back to the trie at
# that point. We use the clear count as an index to get back to the
# old base trie.
self._storage_lookup.rollback_trie(reverted_clear_count)
elif reverted_clear_count == latest_clear_count:
# No change in the base trie, take no action
pass
else:
# Although CREATE2 permits multiple creates and deletes in a single block,
# you can still only revert across a single delete. That's because delete
# is only triggered at the end of the transaction.
raise ValidationError(
f"This revert has changed the clear count in an invalid way, from"
f" {latest_clear_count} to {reverted_clear_count}, in 0x{self._address.hex()}"
)
def commit(self, checkpoint: JournalDBCheckpoint) -> None:
if self._journal_storage.has_checkpoint(checkpoint):
self._journal_storage.commit(checkpoint)
self._clear_count.commit(checkpoint)
else:
# if the checkpoint comes before this account started tracking,
# then flatten all changes, without persisting
self._journal_storage.flatten()
self._clear_count.flatten()
def lock_changes(self) -> None:
if self._journal_storage.has_clear():
self._locked_changes.clear()
self._journal_storage.persist()
def make_storage_root(self) -> None:
self.lock_changes()
self._locked_changes.persist()
def _validate_flushed(self) -> None:
"""
Will raise an exception if there are some changes made since the last persist.
"""
journal_diff = self._journal_storage.diff()
if len(journal_diff) > 0:
raise ValidationError(
f"StorageDB had a dirty journal when it needed to be clean: {journal_diff!r}"
)
@property
def has_changed_root(self) -> bool:
return self._storage_lookup.has_changed_root
def get_changed_root(self) -> Hash32:
return self._storage_lookup.get_changed_root()
def persist(self, db: DatabaseAPI) -> None:
self._validate_flushed()
if self._storage_lookup.has_changed_root:
self._storage_lookup.commit_to(db)
class AccountStorageDB(AccountStorageDatabaseAPI):
logger = get_extended_debug_logger("eth.db.storage.AccountStorageDB")
def __init__(self, db: AtomicDatabaseAPI, storage_root: Hash32, address: Address) -> None:
"""
Database entries go through several pipes, like so...
.. code::
db -> _storage_lookup -> _storage_cache -> _locked_changes -> _journal_storage
db is the raw database, we can assume it hits disk when written to.
Keys are stored as node hashes and rlp-encoded node values.
_storage_lookup is itself a pair of databases: (BatchDB -> HexaryTrie),
writes to storage lookup *are* immeditaely applied to a trie, generating
the appropriate trie nodes and and root hash (via the HexaryTrie). The
writes are *not* persisted to db, until _storage_lookup is explicitly instructed to,
via :meth:`StorageLookup.commit_to`
_storage_cache is a cache tied to the state root of the trie. It
is important that this cache is checked *after* looking for
the key in _journal_storage, because the cache is only invalidated
after a state root change. Otherwise, you will see data since the last
storage root was calculated.
_locked_changes is a batch database that includes only those values that are
un-revertable in the EVM. Currently, that means changes that completed in a
previous transaction.
Journaling batches writes at the _journal_storage layer, until persist is called.
It manages all the checkpointing and rollbacks that happen during EVM execution.
In both _storage_cache and _journal_storage, Keys are set/retrieved as the
big_endian encoding of the slot integer, and the rlp-encoded value.
"""
self._address = address
self._storage_lookup = StorageLookup(db, storage_root, address)
self._storage_cache = CacheDB(self._storage_lookup)
self._locked_changes = BatchDB(self._storage_cache)
self._journal_storage = JournalDB(self._locked_changes)
def get(self, slot: int, from_journal: bool=True) -> int:
key = int_to_big_endian(slot)
lookup_db = self._journal_storage if from_journal else self._locked_changes
try:
encoded_value = lookup_db[key]
except MissingStorageTrieNode:
raise
except KeyError:
return 0
if encoded_value == b'':
return 0
else:
return rlp.decode(encoded_value, sedes=rlp.sedes.big_endian_int)
def set(self, slot: int, value: int) -> None:
key = int_to_big_endian(slot)
if value:
self._journal_storage[key] = rlp.encode(value)
else:
del self._journal_storage[key]
def delete(self) -> None:
self.logger.debug2(
"Deleting all storage in account 0x%s, hashed 0x%s",
self._address.hex(),
keccak(self._address).hex(),
)
self._journal_storage.clear()
self._storage_cache.reset_cache()
def record(self, checkpoint: JournalDBCheckpoint) -> None:
self._journal_storage.record(checkpoint)
def discard(self, checkpoint: JournalDBCheckpoint) -> None:
self.logger.debug2('discard checkpoint %r', checkpoint)
if self._journal_storage.has_checkpoint(checkpoint):
self._journal_storage.discard(checkpoint)
else:
# if the checkpoint comes before this account started tracking,
# then simply reset to the beginning
self._journal_storage.reset()
self._storage_cache.reset_cache()
def commit(self, checkpoint: JournalDBCheckpoint) -> None:
if self._journal_storage.has_checkpoint(checkpoint):
self._journal_storage.commit(checkpoint)
else:
# if the checkpoint comes before this account started tracking,
# then flatten all changes, without persisting
self._journal_storage.flatten()
def lock_changes(self) -> None:
self._journal_storage.persist()
def make_storage_root(self) -> None:
self.lock_changes()
self._locked_changes.commit(apply_deletes=True)
def _validate_flushed(self) -> None:
"""
Will raise an exception if there are some changes made since the last persist.
"""
journal_diff = self._journal_storage.diff()
if len(journal_diff) > 0:
raise ValidationError(
f"StorageDB had a dirty journal when it needed to be clean: {journal_diff!r}"
)
@property
def has_changed_root(self) -> bool:
return self._storage_lookup.has_changed_root
def get_changed_root(self) -> Hash32:
return self._storage_lookup.get_changed_root()
def persist(self, db: DatabaseAPI) -> None:
self._validate_flushed()
if self._storage_lookup.has_changed_root:
self._storage_lookup.commit_to(db)