def db_with_no_comparator(request, tempdir) -> KeyValueStorageLeveldb: global db_no if request.param == 'leveldb': db = KeyValueStorageLeveldb(tempdir, 'kv{}'.format(db_no)) else: db = KeyValueStorageRocksdb(tempdir, 'kv{}'.format(db_no)) db_no += 1 yield db db.close()
def storage(request, tdir) -> KeyValueStorage: global db_no if request.param == 'leveldb': db = KeyValueStorageLeveldb(tdir, 'metrics_ldb_{}'.format(db_no)) else: db = KeyValueStorageRocksdb(tdir, 'metrics_rdb_{}'.format(db_no)) db_no += 1 yield db db.close()
def __init__(self, dataLocation): HasFileStorage.__init__(self, dataLocation) self.clientDataLocation = self.dataLocation if not os.path.exists(self.clientDataLocation): os.makedirs(self.clientDataLocation) # self.transactionLog = TextFileStore(self.clientDataLocation, # "transactions") self.transactionLog = KeyValueStorageLeveldb(self.clientDataLocation, "transactions") self.serializer = ledger_txn_serializer
def kv(request, tempdir) -> KeyValueStorage: global i if request.param == 'leveldb': kv = KeyValueStorageLeveldb(tempdir, 'kv{}'.format(i)) elif request.param == 'rocksdb': kv = KeyValueStorageRocksdb(tempdir, 'kv{}'.format(i)) else: kv = KeyValueStorageInMemory() i += 1 yield kv kv.close()
def __init__(self, name, baseDir=None): self.dataDir = "data/clients" self.name = name HasFileStorage.__init__(self, name, baseDir=baseDir, dataDir=self.dataDir) self.clientDataLocation = self.dataLocation if not os.path.exists(self.clientDataLocation): os.makedirs(self.clientDataLocation) # self.transactionLog = TextFileStore(self.clientDataLocation, # "transactions") self.transactionLog = KeyValueStorageLeveldb( self.clientDataLocation, "transactions") self.serializer = ledger_txn_serializer
def initKeyValueStorage(keyValueType, dataLocation, keyValueStorageName) -> KeyValueStorage: if keyValueType == KeyValueStorageType.Leveldb: return KeyValueStorageLeveldb(dataLocation, keyValueStorageName) elif keyValueType == KeyValueStorageType.Memory: return KeyValueStorageInMemory() else: raise KeyValueStorageConfigNotFound
def state(request, tmpdir_factory) -> State: if request.param == 'memory': db = KeyValueStorageInMemory() if request.param == 'leveldb': db = KeyValueStorageLeveldb(tmpdir_factory.mktemp('').strpath, 'some_db') state = PruningState(db) yield state state.close()
def parametrised_storage(request, tmpdir_factory) -> KeyValueStorage: if request.param == 'memory': db = KeyValueStorageInMemory() elif request.param == 'leveldb': db = KeyValueStorageLeveldb( tmpdir_factory.mktemp('').strpath, 'some_db') else: raise ValueError('Unsupported storage') yield db db.close()
def test_leveldb_and_text_store_equality(tmpdir): """ This test verifies that TextFileStore and LeveldbStore behave equally """ isLineNoKey = True storeContentHash = False ensureDurability = True dbDir = str(tmpdir) text_store = TextFileStore(dbDir=dbDir, dbName="text_data", isLineNoKey=isLineNoKey, storeContentHash=storeContentHash, ensureDurability=ensureDurability) leveldb_store = KeyValueStorageLeveldb(dbDir, "leveldb_data") for i in range(1, 10): value = "Some data {}".format(str(i)) text_store.put(None, value) leveldb_store.put(str(i), value) assert text_store.get(str(i)) assert leveldb_store.get(str(i)).decode() == text_store.get(str(i)) assert list(v.decode() for k, v in leveldb_store.iterator()) == \ list(v for k, v in text_store.iterator())
def initKeyValueStorage(keyValueType, dataLocation, keyValueStorageName, open=True, read_only=False, db_config=None) -> KeyValueStorage: from storage.kv_store_leveldb import KeyValueStorageLeveldb from storage.kv_store_rocksdb import KeyValueStorageRocksdb if keyValueType == KeyValueStorageType.Leveldb: return KeyValueStorageLeveldb(dataLocation, keyValueStorageName, open, read_only) if keyValueType == KeyValueStorageType.Rocksdb: return KeyValueStorageRocksdb(dataLocation, keyValueStorageName, open, read_only, db_config) elif keyValueType == KeyValueStorageType.Memory: return KeyValueStorageInMemory() else: raise KeyValueStorageConfigNotFound
def initKeyValueStorage(keyValueType, dataLocation, keyValueStorageName, open=True, read_only=False) -> KeyValueStorage: if keyValueType == KeyValueStorageType.Leveldb: return KeyValueStorageLeveldb(dataLocation, keyValueStorageName, open, read_only) if keyValueType == KeyValueStorageType.Rocksdb: return KeyValueStorageRocksdb(dataLocation, keyValueStorageName, open, read_only) elif keyValueType == KeyValueStorageType.Memory: return KeyValueStorageInMemory() else: raise KeyValueStorageConfigNotFound
def initKeyValueStorage(keyValueType, dataLocation, keyValueStorageName, open=True, read_only=False, db_config=None, txn_serializer=None) -> KeyValueStorage: from storage.kv_store_leveldb import KeyValueStorageLeveldb from storage.kv_store_rocksdb import KeyValueStorageRocksdb if keyValueType == KeyValueStorageType.Leveldb: return KeyValueStorageLeveldb(dataLocation, keyValueStorageName, open, read_only) if keyValueType == KeyValueStorageType.Rocksdb: return KeyValueStorageRocksdb(dataLocation, keyValueStorageName, open, read_only, db_config) if keyValueType == KeyValueStorageType.Memory: return KeyValueStorageInMemory() if keyValueType == KeyValueStorageType.ChunkedBinaryFile: def chunk_creator(name): return BinarySerializerBasedFileStore(txn_serializer, os.path.join(dataLocation, keyValueStorageName), name, isLineNoKey=True, storeContentHash=False, ensureDurability=False) return ChunkedFileStore(dataLocation, keyValueStorageName, isLineNoKey=True, chunkSize=5, chunk_creator=chunk_creator, storeContentHash=False, ensureDurability=False) if keyValueType == KeyValueStorageType.BinaryFile: return BinaryFileStore(dataLocation, keyValueStorageName, delimiter=b'\0x8b\0xad\0xf0\0x0d\0x8b\0xad\0xf0\0x0d', lineSep=b'\0xde\0xad\0xbe\0xef\0xde\0xad\0xbe\0xef', storeContentHash=False) raise KeyValueStorageConfigNotFound
class ClientTxnLog(HasFileStorage): """ An immutable log of transactions made by the client. """ def __init__(self, name, baseDir=None): self.dataDir = "data/clients" self.name = name HasFileStorage.__init__(self, name, baseDir=baseDir, dataDir=self.dataDir) self.clientDataLocation = self.dataLocation if not os.path.exists(self.clientDataLocation): os.makedirs(self.clientDataLocation) # self.transactionLog = TextFileStore(self.clientDataLocation, # "transactions") self.transactionLog = KeyValueStorageLeveldb( self.clientDataLocation, "transactions") self.serializer = ledger_txn_serializer def close(self): self.transactionLog.close() @property def txnFieldOrdering(self): fields = getTxnOrderedFields() return updateFieldsWithSeqNo(fields) def append(self, identifier: str, reqId, txn): key = '{}{}'.format(identifier, reqId) self.transactionLog.put( key=key, value=self.serializer.serialize( txn, fields=self.txnFieldOrdering, toBytes=False)) def hasTxn(self, identifier, reqId) -> bool: key = '{}{}'.format(identifier, reqId) return key in self.transactionLog def reset(self): self.transactionLog.reset()
def kv(request, tempdir) -> KeyValueStorage: global i if request.param == 'leveldb': kv = KeyValueStorageLeveldb(tempdir, 'kv{}'.format(i)) else: kv = KeyValueStorageRocksdb(tempdir, 'kv{}'.format(i)) assert kv.read_only is False kv.put('k1', 'v1') kv.put('k2', 'v2') kv.put('k3', 'v3') kv.close() if request.param == 'leveldb': kv = KeyValueStorageLeveldb(tempdir, 'kv{}'.format(i), read_only=True) else: kv = KeyValueStorageRocksdb(tempdir, 'kv{}'.format(i), read_only=True) i += 1 yield kv kv.close()
def kv(tempdir) -> KeyValueStorageLeveldb: global i kv = KeyValueStorageLeveldb(tempdir, 'kv{}'.format(i)) i += 1 yield kv kv.close()
def state2(tempdir) -> State: global i state = PruningState(KeyValueStorageLeveldb(tempdir, 'kv2{}'.format(i))) yield state state.close()
def db(request, tempdir) -> KeyValueStorage: if request.param == 'leveldb': return KeyValueStorageLeveldb(tempdir, 'kv{}'.format(i)) if request.param == 'rocksdb': return KeyValueStorageRocksdb(tempdir, 'kv{}'.format(i)) return KeyValueStorageInMemory()
class LevelDbHashStore(HashStore): def __init__(self, dataDir, fileNamePrefix=""): self.dataDir = dataDir self.nodesDb = None self.leavesDb = None self._leafCount = 0 self.nodes_db_name = fileNamePrefix + '_merkleNodes' self.leaves_db_name = fileNamePrefix + '_merkleLeaves' self.open() @property def is_persistent(self) -> bool: return True def writeLeaf(self, leafHash): self.leavesDb.put(str(self.leafCount + 1), leafHash) self.leafCount += 1 def writeNode(self, node): start, height, nodeHash = node seqNo = self.getNodePosition(start, height) self.nodesDb.put(str(seqNo), nodeHash) def readLeaf(self, seqNo): return self._readOne(seqNo, self.leavesDb) def readNode(self, seqNo): return self._readOne(seqNo, self.nodesDb) def _readOne(self, pos, db): self._validatePos(pos) try: # Converting any bytearray to bytes return bytes(db.get(str(pos))) except KeyError: logger.error("{} does not have position {}".format(db, pos)) def readLeafs(self, start, end): return self._readMultiple(start, end, self.leavesDb) def readNodes(self, start, end): return self._readMultiple(start, end, self.nodesDb) def _readMultiple(self, start, end, db): """ Returns a list of hashes with serial numbers between start and end, both inclusive. """ self._validatePos(start, end) # Converting any bytearray to bytes return [bytes(db.get(str(pos))) for pos in range(start, end + 1)] @property def leafCount(self) -> int: return self._leafCount @property def nodeCount(self) -> int: return self.nodesDb.size @leafCount.setter def leafCount(self, count: int) -> None: self._leafCount = count @property def closed(self): return (self.nodesDb is None and self.leavesDb is None) \ or \ (self.nodesDb.closed and self.leavesDb.closed) def open(self): self.nodesDb = KeyValueStorageLeveldb(self.dataDir, self.nodes_db_name) self.leavesDb = KeyValueStorageLeveldb( self.dataDir, self.leaves_db_name) self._leafCount = self.leavesDb.size def close(self): self.nodesDb.close() self.leavesDb.close() def reset(self) -> bool: self.nodesDb.reset() self.leavesDb.reset() self.leafCount = 0 return True
def open(self): self.nodesDb = KeyValueStorageLeveldb(self.dataDir, self.nodes_db_name) self.leavesDb = KeyValueStorageLeveldb( self.dataDir, self.leaves_db_name) self._leafCount = self.leavesDb.size
def db(request, tempdir) -> State: if request == 'leveldb': return KeyValueStorageLeveldb(tempdir, 'kv{}'.format(i)) return KeyValueStorageInMemory()