def test_db_diff_inspection(series_of_diffs, expected_updates,
expected_deletions):
diffs = []
for changes in series_of_diffs:
tracker = DBDiffTracker()
for key, val in changes.items():
if val is None:
del tracker[key]
else:
tracker[key] = val
diffs.append(tracker.diff())
actual_diff = DBDiff.join(diffs)
if expected_updates:
expected_keys, _ = zip(*expected_updates.items())
else:
expected_keys = tuple()
assert actual_diff.pending_keys() == expected_keys
assert actual_diff.pending_items() == tuple(expected_updates.items())
assert actual_diff.deleted_keys() == tuple(expected_deletions)
class AtomicBatch(BaseDB):
"""
This is returned by a DBClient during an atomic_batch, to provide a temporary view
of the database, before commit.
"""
logger = logging.getLogger("trinity.db.manager.AtomicBatch")
_write_target_db: BaseDB = None
_diff: DBDiffTracker = None
def __init__(self, db: DatabaseAPI) -> None:
self._db = db
self._track_diff = DBDiffTracker()
def __getitem__(self, key: bytes) -> bytes:
if self._track_diff is None:
raise ValidationError(
"Cannot get data from a write batch, out of context")
try:
value = self._track_diff[key]
except DiffMissingError as missing:
if missing.is_deleted:
raise KeyError(key)
else:
return self._db[key]
else:
return value
def __setitem__(self, key: bytes, value: bytes) -> None:
if self._track_diff is None:
raise ValidationError(
"Cannot set data from a write batch, out of context")
self._track_diff[key] = value
def __delitem__(self, key: bytes) -> None:
if key not in self:
raise KeyError(key)
del self._track_diff[key]
def _exists(self, key: bytes) -> bool:
try:
self[key]
except KeyError:
return False
else:
return True
def finalize(self) -> DBDiff:
diff = self._track_diff.diff()
self._track_diff = None
self._db = None
return diff
def diff(self) -> DBDiff:
"""
Generate a DBDiff of all pending changes.
These are the changes that would occur if :meth:`persist()` were called.
"""
tracker = DBDiffTracker()
visited_keys = set() # type: Set[bytes]
# Iterate in reverse, so you can skip over any keys from old checkpoints.
# This is purely for performance, not correctness.
for changeset in reversed(self.journal.journal_data.values()):
for key, value in changeset.items():
if key in visited_keys:
# this old change has already been tracked
continue
elif value is DELETED_ENTRY:
del tracker[key]
else:
tracker[
key] = value # type: ignore # This is always bytes, but mypy can't tell
visited_keys.add(key)
return tracker.diff()
def __init__(self, write_target_db: BaseDB) -> None:
self._write_target_db = write_target_db
self._track_diff = DBDiffTracker()
class AtomicDBWriteBatch(BaseDB):
"""
This is returned by a BaseAtomicDB during an atomic_batch, to provide a temporary view
of the database, before commit.
"""
logger = logging.getLogger("eth.db.AtomicDBWriteBatch")
_write_target_db = None # type: BaseDB
_track_diff = None # type: DBDiffTracker
def __init__(self, write_target_db: BaseDB) -> None:
self._write_target_db = write_target_db
self._track_diff = DBDiffTracker()
def __getitem__(self, key: bytes) -> bytes:
if self._track_diff is None:
raise ValidationError(
"Cannot get data from a write batch, out of context")
try:
value = self._track_diff[key]
except DiffMissingError as missing:
if missing.is_deleted:
raise KeyError(key)
else:
return self._write_target_db[key]
else:
return value
def __setitem__(self, key: bytes, value: bytes) -> None:
if self._track_diff is None:
raise ValidationError(
"Cannot set data from a write batch, out of context")
self._track_diff[key] = value
def __delitem__(self, key: bytes) -> None:
if self._track_diff is None:
raise ValidationError(
"Cannot delete data from a write batch, out of context")
if key not in self:
raise KeyError(key)
del self._track_diff[key]
def _diff(self) -> DBDiff:
return self._track_diff.diff()
def _commit(self) -> None:
self._diff().apply_to(self._write_target_db, apply_deletes=True)
def _exists(self, key: bytes) -> bool:
if self._track_diff is None:
raise ValidationError(
"Cannot test data existance from a write batch, out of context"
)
try:
self[key]
except KeyError:
return False
else:
return True
@classmethod
@contextmanager
def _commit_unless_raises(
cls, write_target_db: BaseDB) -> Iterator['AtomicDBWriteBatch']:
"""
Commit all writes inside the context, unless an exception was raised.
Although this is technically an external API, it (and this whole class) is only intended
to be used by AtomicDB.
"""
readable_write_batch = cls(write_target_db) # type: AtomicDBWriteBatch
try:
yield readable_write_batch
except Exception:
cls.logger.exception(
"Unexpected error in atomic db write, dropped partial writes: %r",
readable_write_batch._diff(),
)
raise
else:
readable_write_batch._commit()
finally:
# force a shutdown of this batch, to prevent out-of-context usage
readable_write_batch._track_diff = None
readable_write_batch._write_target_db = None
def __init__(self, db: DatabaseAPI) -> None:
self._db = db
self._track_diff = DBDiffTracker()
def __init__(self, original_read_db: BaseDB,
write_batch: 'plyvel.WriteBatch') -> None:
self._original_read_db = original_read_db
self._write_batch = write_batch
# keep track of the temporary changes made
self._track_diff = DBDiffTracker()
def clear(self):
self._track_diff = DBDiffTracker()
def __init__(self, wrapped_db: BaseDB) -> None:
self.wrapped_db = wrapped_db
self._track_diff = DBDiffTracker()
class BatchDB(BaseDB):
"""
A wrapper of basic DB objects with uncommitted DB changes stored in local cache,
which represents as a dictionary of database keys and values.
This class should be usable as a context manager, the changes either all fail or all succeed.
Upon exiting the context, it writes all of the key value pairs from the cache into
the underlying database. If any error occurred before committing phase,
we would not apply commits at all.
"""
logger = logging.getLogger("eth.db.BatchDB")
wrapped_db = None # type: BaseDB
_track_diff = None # type: DBDiffTracker
def __init__(self, wrapped_db: BaseDB) -> None:
self.wrapped_db = wrapped_db
self._track_diff = DBDiffTracker()
def __enter__(self) -> 'BatchDB':
return self
def __exit__(self, exc_type: None, exc_value: None,
traceback: None) -> None:
# commit all the changes from local cache to underlying db
if exc_type is None:
self.commit()
else:
self.clear()
self.logger.exception(
"Unexpected error occurred during batch update")
def clear(self):
self._track_diff = DBDiffTracker()
def commit(self, apply_deletes: bool = True) -> None:
self.diff().apply_to(self.wrapped_db, apply_deletes)
self.clear()
def _exists(self, key: bytes) -> bool:
try:
self[key]
except KeyError:
return False
else:
return True
def __getitem__(self, key: bytes) -> bytes:
try:
value = self._track_diff[key]
except DiffMissingError as missing:
if missing.is_deleted:
raise KeyError(key)
else:
return self.wrapped_db[key]
else:
return value
def __setitem__(self, key: bytes, value: bytes) -> None:
self._track_diff[key] = value
def __delitem__(self, key: bytes) -> None:
if key not in self:
raise KeyError(key)
del self._track_diff[key]
def diff(self) -> DBDiff:
return self._track_diff.diff()
def clear(self) -> None:
self._track_diff = DBDiffTracker()
def __init__(self, wrapped_db: BaseDB, read_through_deletes: bool = False) -> None:
self.wrapped_db = wrapped_db
self._track_diff = DBDiffTracker()
self._read_through_deletes = read_through_deletes
def db():
return DBDiffTracker()
def __init__(self, write_target_db: DatabaseAPI) -> None:
self._write_target_db = write_target_db
self._track_diff = DBDiffTracker()
