def db_with_no_comparator(request, tempdir) -> KeyValueStorageLeveldb:
global db_no
if request.param == 'leveldb':
db = KeyValueStorageLeveldb(tempdir, 'kv{}'.format(db_no))
else:
db = KeyValueStorageRocksdb(tempdir, 'kv{}'.format(db_no))
db_no += 1
yield db
db.close()
def db_with_no_comparator(request, tempdir) -> KeyValueStorageLeveldb:
global db_no
if request.param == 'leveldb':
db = KeyValueStorageLeveldb(tempdir, 'kv{}'.format(db_no))
else:
db = KeyValueStorageRocksdb(tempdir, 'kv{}'.format(db_no))
db_no += 1
yield db
db.close()
def storage(request, tdir) -> KeyValueStorage:
global db_no
if request.param == 'leveldb':
db = KeyValueStorageLeveldb(tdir, 'metrics_ldb_{}'.format(db_no))
else:
db = KeyValueStorageRocksdb(tdir, 'metrics_rdb_{}'.format(db_no))
db_no += 1
yield db
db.close()
def __init__(self, dataLocation):
HasFileStorage.__init__(self, dataLocation)
self.clientDataLocation = self.dataLocation
if not os.path.exists(self.clientDataLocation):
os.makedirs(self.clientDataLocation)
# self.transactionLog = TextFileStore(self.clientDataLocation,
# "transactions")
self.transactionLog = KeyValueStorageLeveldb(self.clientDataLocation,
"transactions")
self.serializer = ledger_txn_serializer
def kv(request, tempdir) -> KeyValueStorage:
global i
if request.param == 'leveldb':
kv = KeyValueStorageLeveldb(tempdir, 'kv{}'.format(i))
elif request.param == 'rocksdb':
kv = KeyValueStorageRocksdb(tempdir, 'kv{}'.format(i))
else:
kv = KeyValueStorageInMemory()
i += 1
yield kv
kv.close()
def __init__(self, name, baseDir=None):
self.dataDir = "data/clients"
self.name = name
HasFileStorage.__init__(self, name,
baseDir=baseDir,
dataDir=self.dataDir)
self.clientDataLocation = self.dataLocation
if not os.path.exists(self.clientDataLocation):
os.makedirs(self.clientDataLocation)
# self.transactionLog = TextFileStore(self.clientDataLocation,
# "transactions")
self.transactionLog = KeyValueStorageLeveldb(
self.clientDataLocation, "transactions")
self.serializer = ledger_txn_serializer
def initKeyValueStorage(keyValueType, dataLocation,
keyValueStorageName) -> KeyValueStorage:
if keyValueType == KeyValueStorageType.Leveldb:
return KeyValueStorageLeveldb(dataLocation, keyValueStorageName)
elif keyValueType == KeyValueStorageType.Memory:
return KeyValueStorageInMemory()
else:
raise KeyValueStorageConfigNotFound
def state(request, tmpdir_factory) -> State:
if request.param == 'memory':
db = KeyValueStorageInMemory()
if request.param == 'leveldb':
db = KeyValueStorageLeveldb(tmpdir_factory.mktemp('').strpath,
'some_db')
state = PruningState(db)
yield state
state.close()
def parametrised_storage(request, tmpdir_factory) -> KeyValueStorage:
if request.param == 'memory':
db = KeyValueStorageInMemory()
elif request.param == 'leveldb':
db = KeyValueStorageLeveldb(
tmpdir_factory.mktemp('').strpath, 'some_db')
else:
raise ValueError('Unsupported storage')
yield db
db.close()
def test_leveldb_and_text_store_equality(tmpdir):
"""
This test verifies that TextFileStore and LeveldbStore behave equally
"""
isLineNoKey = True
storeContentHash = False
ensureDurability = True
dbDir = str(tmpdir)
text_store = TextFileStore(dbDir=dbDir,
dbName="text_data",
isLineNoKey=isLineNoKey,
storeContentHash=storeContentHash,
ensureDurability=ensureDurability)
leveldb_store = KeyValueStorageLeveldb(dbDir, "leveldb_data")
for i in range(1, 10):
value = "Some data {}".format(str(i))
text_store.put(None, value)
leveldb_store.put(str(i), value)
assert text_store.get(str(i))
assert leveldb_store.get(str(i)).decode() == text_store.get(str(i))
assert list(v.decode() for k, v in leveldb_store.iterator()) == \
list(v for k, v in text_store.iterator())
def initKeyValueStorage(keyValueType, dataLocation, keyValueStorageName,
open=True, read_only=False, db_config=None) -> KeyValueStorage:
from storage.kv_store_leveldb import KeyValueStorageLeveldb
from storage.kv_store_rocksdb import KeyValueStorageRocksdb
if keyValueType == KeyValueStorageType.Leveldb:
return KeyValueStorageLeveldb(dataLocation, keyValueStorageName, open,
read_only)
if keyValueType == KeyValueStorageType.Rocksdb:
return KeyValueStorageRocksdb(dataLocation, keyValueStorageName, open,
read_only, db_config)
elif keyValueType == KeyValueStorageType.Memory:
return KeyValueStorageInMemory()
else:
raise KeyValueStorageConfigNotFound
def initKeyValueStorage(keyValueType,
dataLocation,
keyValueStorageName,
open=True,
read_only=False) -> KeyValueStorage:
if keyValueType == KeyValueStorageType.Leveldb:
return KeyValueStorageLeveldb(dataLocation, keyValueStorageName, open,
read_only)
if keyValueType == KeyValueStorageType.Rocksdb:
return KeyValueStorageRocksdb(dataLocation, keyValueStorageName, open,
read_only)
elif keyValueType == KeyValueStorageType.Memory:
return KeyValueStorageInMemory()
else:
raise KeyValueStorageConfigNotFound
def initKeyValueStorage(keyValueType, dataLocation, keyValueStorageName,
open=True, read_only=False, db_config=None, txn_serializer=None) -> KeyValueStorage:
from storage.kv_store_leveldb import KeyValueStorageLeveldb
from storage.kv_store_rocksdb import KeyValueStorageRocksdb
if keyValueType == KeyValueStorageType.Leveldb:
return KeyValueStorageLeveldb(dataLocation, keyValueStorageName, open,
read_only)
if keyValueType == KeyValueStorageType.Rocksdb:
return KeyValueStorageRocksdb(dataLocation, keyValueStorageName, open,
read_only, db_config)
if keyValueType == KeyValueStorageType.Memory:
return KeyValueStorageInMemory()
if keyValueType == KeyValueStorageType.ChunkedBinaryFile:
def chunk_creator(name):
return BinarySerializerBasedFileStore(txn_serializer,
os.path.join(dataLocation, keyValueStorageName),
name,
isLineNoKey=True,
storeContentHash=False,
ensureDurability=False)
return ChunkedFileStore(dataLocation,
keyValueStorageName,
isLineNoKey=True,
chunkSize=5,
chunk_creator=chunk_creator,
storeContentHash=False,
ensureDurability=False)
if keyValueType == KeyValueStorageType.BinaryFile:
return BinaryFileStore(dataLocation, keyValueStorageName,
delimiter=b'\0x8b\0xad\0xf0\0x0d\0x8b\0xad\0xf0\0x0d',
lineSep=b'\0xde\0xad\0xbe\0xef\0xde\0xad\0xbe\0xef',
storeContentHash=False)
raise KeyValueStorageConfigNotFound
class ClientTxnLog(HasFileStorage):
"""
An immutable log of transactions made by the client.
"""
def __init__(self, name, baseDir=None):
self.dataDir = "data/clients"
self.name = name
HasFileStorage.__init__(self, name,
baseDir=baseDir,
dataDir=self.dataDir)
self.clientDataLocation = self.dataLocation
if not os.path.exists(self.clientDataLocation):
os.makedirs(self.clientDataLocation)
# self.transactionLog = TextFileStore(self.clientDataLocation,
# "transactions")
self.transactionLog = KeyValueStorageLeveldb(
self.clientDataLocation, "transactions")
self.serializer = ledger_txn_serializer
def close(self):
self.transactionLog.close()
@property
def txnFieldOrdering(self):
fields = getTxnOrderedFields()
return updateFieldsWithSeqNo(fields)
def append(self, identifier: str, reqId, txn):
key = '{}{}'.format(identifier, reqId)
self.transactionLog.put(
key=key, value=self.serializer.serialize(
txn, fields=self.txnFieldOrdering, toBytes=False))
def hasTxn(self, identifier, reqId) -> bool:
key = '{}{}'.format(identifier, reqId)
return key in self.transactionLog
def reset(self):
self.transactionLog.reset()
def kv(request, tempdir) -> KeyValueStorage:
global i
if request.param == 'leveldb':
kv = KeyValueStorageLeveldb(tempdir, 'kv{}'.format(i))
else:
kv = KeyValueStorageRocksdb(tempdir, 'kv{}'.format(i))
assert kv.read_only is False
kv.put('k1', 'v1')
kv.put('k2', 'v2')
kv.put('k3', 'v3')
kv.close()
if request.param == 'leveldb':
kv = KeyValueStorageLeveldb(tempdir, 'kv{}'.format(i), read_only=True)
else:
kv = KeyValueStorageRocksdb(tempdir, 'kv{}'.format(i), read_only=True)
i += 1
yield kv
kv.close()
def kv(tempdir) -> KeyValueStorageLeveldb:
global i
kv = KeyValueStorageLeveldb(tempdir, 'kv{}'.format(i))
i += 1
yield kv
kv.close()
def state2(tempdir) -> State:
global i
state = PruningState(KeyValueStorageLeveldb(tempdir, 'kv2{}'.format(i)))
yield state
state.close()
def kv(request, tempdir) -> KeyValueStorage:
global i
if request.param == 'leveldb':
kv = KeyValueStorageLeveldb(tempdir, 'kv{}'.format(i))
else:
kv = KeyValueStorageRocksdb(tempdir, 'kv{}'.format(i))
assert kv.read_only is False
kv.put('k1', 'v1')
kv.put('k2', 'v2')
kv.put('k3', 'v3')
kv.close()
if request.param == 'leveldb':
kv = KeyValueStorageLeveldb(tempdir, 'kv{}'.format(i), read_only=True)
else:
kv = KeyValueStorageRocksdb(tempdir, 'kv{}'.format(i), read_only=True)
i += 1
yield kv
kv.close()
def db(request, tempdir) -> KeyValueStorage:
if request.param == 'leveldb':
return KeyValueStorageLeveldb(tempdir, 'kv{}'.format(i))
if request.param == 'rocksdb':
return KeyValueStorageRocksdb(tempdir, 'kv{}'.format(i))
return KeyValueStorageInMemory()
class LevelDbHashStore(HashStore):
def __init__(self, dataDir, fileNamePrefix=""):
self.dataDir = dataDir
self.nodesDb = None
self.leavesDb = None
self._leafCount = 0
self.nodes_db_name = fileNamePrefix + '_merkleNodes'
self.leaves_db_name = fileNamePrefix + '_merkleLeaves'
self.open()
@property
def is_persistent(self) -> bool:
return True
def writeLeaf(self, leafHash):
self.leavesDb.put(str(self.leafCount + 1), leafHash)
self.leafCount += 1
def writeNode(self, node):
start, height, nodeHash = node
seqNo = self.getNodePosition(start, height)
self.nodesDb.put(str(seqNo), nodeHash)
def readLeaf(self, seqNo):
return self._readOne(seqNo, self.leavesDb)
def readNode(self, seqNo):
return self._readOne(seqNo, self.nodesDb)
def _readOne(self, pos, db):
self._validatePos(pos)
try:
# Converting any bytearray to bytes
return bytes(db.get(str(pos)))
except KeyError:
logger.error("{} does not have position {}".format(db, pos))
def readLeafs(self, start, end):
return self._readMultiple(start, end, self.leavesDb)
def readNodes(self, start, end):
return self._readMultiple(start, end, self.nodesDb)
def _readMultiple(self, start, end, db):
"""
Returns a list of hashes with serial numbers between start
and end, both inclusive.
"""
self._validatePos(start, end)
# Converting any bytearray to bytes
return [bytes(db.get(str(pos))) for pos in range(start, end + 1)]
@property
def leafCount(self) -> int:
return self._leafCount
@property
def nodeCount(self) -> int:
return self.nodesDb.size
@leafCount.setter
def leafCount(self, count: int) -> None:
self._leafCount = count
@property
def closed(self):
return (self.nodesDb is None and self.leavesDb is None) \
or \
(self.nodesDb.closed and self.leavesDb.closed)
def open(self):
self.nodesDb = KeyValueStorageLeveldb(self.dataDir, self.nodes_db_name)
self.leavesDb = KeyValueStorageLeveldb(
self.dataDir, self.leaves_db_name)
self._leafCount = self.leavesDb.size
def close(self):
self.nodesDb.close()
self.leavesDb.close()
def reset(self) -> bool:
self.nodesDb.reset()
self.leavesDb.reset()
self.leafCount = 0
return True
def open(self):
self.nodesDb = KeyValueStorageLeveldb(self.dataDir, self.nodes_db_name)
self.leavesDb = KeyValueStorageLeveldb(
self.dataDir, self.leaves_db_name)
self._leafCount = self.leavesDb.size
def db(request, tempdir) -> State:
if request == 'leveldb':
return KeyValueStorageLeveldb(tempdir, 'kv{}'.format(i))
return KeyValueStorageInMemory()