def diff(self) -> DBDiff: tracker = DBDiffTracker() visited_keys = set() # type: Set[bytes] # Iterate in reverse, so you can skip over any keys from old checkpoints. # This is required so that when a key is created and then deleted in the journal, # we don't add the delete to the diff. (We simply omit the change altogether) for changeset_id, changeset in reversed(self.journal_data.items()): if changeset_id in self._clears_at: break for key, value in changeset.items(): if key in visited_keys: # this old change has already been tracked continue elif value is DELETED_ENTRY: del tracker[key] elif value is ERASE_CREATED_ENTRY: pass else: tracker[key] = cast(bytes, value) visited_keys.add(key) return tracker.diff()
class AtomicBatch(BaseDB): """ This is returned by a DBClient during an atomic_batch, to provide a temporary view of the database, before commit. """ logger = logging.getLogger("trinity.db.manager.AtomicBatch") _write_target_db: BaseDB = None _diff: DBDiffTracker = None def __init__(self, db: DatabaseAPI) -> None: self._db = db self._track_diff = DBDiffTracker() def __getitem__(self, key: bytes) -> bytes: if self._track_diff is None: raise ValidationError( "Cannot get data from a write batch, out of context") try: value = self._track_diff[key] except DiffMissingError as missing: if missing.is_deleted: raise KeyError(key) else: return self._db[key] else: return value def __setitem__(self, key: bytes, value: bytes) -> None: if self._track_diff is None: raise ValidationError( "Cannot set data from a write batch, out of context") self._track_diff[key] = value def __delitem__(self, key: bytes) -> None: if key not in self: raise KeyError(key) del self._track_diff[key] def _exists(self, key: bytes) -> bool: try: self[key] except KeyError: return False else: return True def finalize(self) -> DBDiff: diff = self._track_diff.diff() self._track_diff = None self._db = None return diff
def test_join_diffs(db, series_of_diffs, expected): diffs = [] for changes in series_of_diffs: tracker = DBDiffTracker() for key, val in changes.items(): if val is None: del tracker[key] else: tracker[key] = val diffs.append(tracker.diff()) DBDiff.join(diffs).apply_to(db) assert db == expected
def test_db_diff_inspection(series_of_diffs, expected_updates, expected_deletions): diffs = [] for changes in series_of_diffs: tracker = DBDiffTracker() for key, val in changes.items(): if val is None: del tracker[key] else: tracker[key] = val diffs.append(tracker.diff()) actual_diff = DBDiff.join(diffs) if expected_updates: expected_keys, _ = zip(*expected_updates.items()) else: expected_keys = tuple() assert actual_diff.pending_keys() == expected_keys assert actual_diff.pending_items() == tuple(expected_updates.items()) assert actual_diff.deleted_keys() == tuple(expected_deletions)
def diff(self) -> DBDiff: """ Generate a DBDiff of all pending changes. These are the changes that would occur if :meth:`persist()` were called. """ tracker = DBDiffTracker() visited_keys = set() # type: Set[bytes] # Iterate in reverse, so you can skip over any keys from old checkpoints. # This is purely for performance, not correctness. for changeset in reversed(self.journal.journal_data.values()): for key, value in changeset.items(): if key in visited_keys: # this old change has already been tracked continue elif value is DELETED_ENTRY: del tracker[key] else: tracker[ key] = value # type: ignore # This is always bytes, but mypy can't tell visited_keys.add(key) return tracker.diff()
class AtomicDBWriteBatch(BaseDB): """ This is returned by a BaseAtomicDB during an atomic_batch, to provide a temporary view of the database, before commit. """ logger = logging.getLogger("eth.db.AtomicDBWriteBatch") _write_target_db = None # type: BaseDB _track_diff = None # type: DBDiffTracker def __init__(self, write_target_db: BaseDB) -> None: self._write_target_db = write_target_db self._track_diff = DBDiffTracker() def __getitem__(self, key: bytes) -> bytes: if self._track_diff is None: raise ValidationError( "Cannot get data from a write batch, out of context") try: value = self._track_diff[key] except DiffMissingError as missing: if missing.is_deleted: raise KeyError(key) else: return self._write_target_db[key] else: return value def __setitem__(self, key: bytes, value: bytes) -> None: if self._track_diff is None: raise ValidationError( "Cannot set data from a write batch, out of context") self._track_diff[key] = value def __delitem__(self, key: bytes) -> None: if self._track_diff is None: raise ValidationError( "Cannot delete data from a write batch, out of context") if key not in self: raise KeyError(key) del self._track_diff[key] def _diff(self) -> DBDiff: return self._track_diff.diff() def _commit(self) -> None: self._diff().apply_to(self._write_target_db, apply_deletes=True) def _exists(self, key: bytes) -> bool: if self._track_diff is None: raise ValidationError( "Cannot test data existance from a write batch, out of context" ) try: self[key] except KeyError: return False else: return True @classmethod @contextmanager def _commit_unless_raises( cls, write_target_db: BaseDB) -> Iterator['AtomicDBWriteBatch']: """ Commit all writes inside the context, unless an exception was raised. Although this is technically an external API, it (and this whole class) is only intended to be used by AtomicDB. """ readable_write_batch = cls(write_target_db) # type: AtomicDBWriteBatch try: yield readable_write_batch except Exception: cls.logger.exception( "Unexpected error in atomic db write, dropped partial writes: %r", readable_write_batch._diff(), ) raise else: readable_write_batch._commit() finally: # force a shutdown of this batch, to prevent out-of-context usage readable_write_batch._track_diff = None readable_write_batch._write_target_db = None
class BatchDB(BaseDB): """ A wrapper of basic DB objects with uncommitted DB changes stored in local cache, which represents as a dictionary of database keys and values. This class should be usable as a context manager, the changes either all fail or all succeed. Upon exiting the context, it writes all of the key value pairs from the cache into the underlying database. If any error occurred before committing phase, we would not apply commits at all. """ logger = logging.getLogger("eth.db.BatchDB") wrapped_db = None # type: BaseDB _track_diff = None # type: DBDiffTracker def __init__(self, wrapped_db: BaseDB) -> None: self.wrapped_db = wrapped_db self._track_diff = DBDiffTracker() def __enter__(self) -> 'BatchDB': return self def __exit__(self, exc_type: None, exc_value: None, traceback: None) -> None: # commit all the changes from local cache to underlying db if exc_type is None: self.commit() else: self.clear() self.logger.exception( "Unexpected error occurred during batch update") def clear(self): self._track_diff = DBDiffTracker() def commit(self, apply_deletes: bool = True) -> None: self.diff().apply_to(self.wrapped_db, apply_deletes) self.clear() def _exists(self, key: bytes) -> bool: try: self[key] except KeyError: return False else: return True def __getitem__(self, key: bytes) -> bytes: try: value = self._track_diff[key] except DiffMissingError as missing: if missing.is_deleted: raise KeyError(key) else: return self.wrapped_db[key] else: return value def __setitem__(self, key: bytes, value: bytes) -> None: self._track_diff[key] = value def __delitem__(self, key: bytes) -> None: if key not in self: raise KeyError(key) del self._track_diff[key] def diff(self) -> DBDiff: return self._track_diff.diff()