def __init__(self, dataLocation):
HasFileStorage.__init__(self, dataLocation)
self.clientDataLocation = self.dataLocation
if not os.path.exists(self.clientDataLocation):
os.makedirs(self.clientDataLocation)
self.transactionLog = KeyValueStorageRocksdb(
self.clientDataLocation, "transactions")
self.serializer = ledger_txn_serializer
def database_manager(tdir_for_func):
db = DatabaseManager()
db.register_new_database(
LEDGER_ID, Ledger(CompactMerkleTree(), dataDir=tdir_for_func),
PruningState(KeyValueStorageRocksdb(tdir_for_func, 'kv1')))
db.register_new_store(
TS_LABEL,
StateTsDbStorage('test',
{1: KeyValueStorageRocksdb(tdir_for_func, 'kv2')}))
return db
def db_with_no_comparator(request, tempdir) -> KeyValueStorageLeveldb:
global db_no
if request.param == 'leveldb':
db = KeyValueStorageLeveldb(tempdir, 'kv{}'.format(db_no))
else:
db = KeyValueStorageRocksdb(tempdir, 'kv{}'.format(db_no))
db_no += 1
yield db
db.close()
def storage(request, tdir) -> KeyValueStorage:
global db_no
if request.param == 'leveldb':
db = KeyValueStorageLeveldb(tdir, 'metrics_ldb_{}'.format(db_no))
else:
db = KeyValueStorageRocksdb(tdir, 'metrics_rdb_{}'.format(db_no))
db_no += 1
yield db
db.close()
def initKeyValueStorage(keyValueType, dataLocation,
keyValueStorageName) -> KeyValueStorage:
if keyValueType == KeyValueStorageType.Leveldb:
return KeyValueStorageLeveldb(dataLocation, keyValueStorageName)
if keyValueType == KeyValueStorageType.Rocksdb:
return KeyValueStorageRocksdb(dataLocation, keyValueStorageName)
elif keyValueType == KeyValueStorageType.Memory:
return KeyValueStorageInMemory()
else:
raise KeyValueStorageConfigNotFound
def state(request, tmpdir_factory) -> State:
if request.param == 'leveldb':
db = KeyValueStorageLeveldb(tmpdir_factory.mktemp('').strpath,
'some_db')
elif request.param == 'rocksdb':
db = KeyValueStorageRocksdb(tmpdir_factory.mktemp('').strpath,
'some_db')
else:
db = KeyValueStorageInMemory()
state = PruningState(db)
yield state
state.close()
class ClientTxnLog(HasFileStorage):
"""
An immutable log of transactions made by the client.
"""
def __init__(self, dataLocation):
HasFileStorage.__init__(self, dataLocation)
self.clientDataLocation = self.dataLocation
if not os.path.exists(self.clientDataLocation):
os.makedirs(self.clientDataLocation)
self.transactionLog = KeyValueStorageRocksdb(
self.clientDataLocation, "transactions")
self.serializer = ledger_txn_serializer
def close(self):
self.transactionLog.close()
@property
def txnFieldOrdering(self):
fields = getTxnOrderedFields()
return updateFieldsWithSeqNo(fields)
def append(self, identifier: str, reqId, txn):
key = '{}{}'.format(identifier, reqId)
self.transactionLog.put(
key=key, value=self.serializer.serialize(
txn, fields=self.txnFieldOrdering, toBytes=False))
def hasTxn(self, identifier, reqId) -> bool:
key = '{}{}'.format(identifier, reqId)
return key in self.transactionLog
def reset(self):
self.transactionLog.reset()
def kv(request, tempdir) -> KeyValueStorage:
global i
if request.param == 'leveldb':
kv = KeyValueStorageLeveldb(tempdir, 'kv{}'.format(i))
else:
kv = KeyValueStorageRocksdb(tempdir, 'kv{}'.format(i))
assert kv.read_only is False
kv.put('k1', 'v1')
kv.put('k2', 'v2')
kv.put('k3', 'v3')
kv.close()
if request.param == 'leveldb':
kv = KeyValueStorageLeveldb(tempdir, 'kv{}'.format(i), read_only=True)
else:
kv = KeyValueStorageRocksdb(tempdir, 'kv{}'.format(i), read_only=True)
i += 1
yield kv
kv.close()
def initKeyValueStorage(keyValueType, dataLocation, keyValueStorageName,
open=True, read_only=False, db_config=None) -> KeyValueStorage:
from storage.kv_store_leveldb import KeyValueStorageLeveldb
from storage.kv_store_rocksdb import KeyValueStorageRocksdb
if keyValueType == KeyValueStorageType.Leveldb:
return KeyValueStorageLeveldb(dataLocation, keyValueStorageName, open,
read_only)
if keyValueType == KeyValueStorageType.Rocksdb:
return KeyValueStorageRocksdb(dataLocation, keyValueStorageName, open,
read_only, db_config)
elif keyValueType == KeyValueStorageType.Memory:
return KeyValueStorageInMemory()
else:
raise KeyValueStorageConfigNotFound
def initKeyValueStorage(keyValueType, dataLocation, keyValueStorageName,
open=True, read_only=False, db_config=None, txn_serializer=None) -> KeyValueStorage:
from storage.kv_store_leveldb import KeyValueStorageLeveldb
from storage.kv_store_rocksdb import KeyValueStorageRocksdb
if keyValueType == KeyValueStorageType.Leveldb:
return KeyValueStorageLeveldb(dataLocation, keyValueStorageName, open,
read_only)
if keyValueType == KeyValueStorageType.Rocksdb:
return KeyValueStorageRocksdb(dataLocation, keyValueStorageName, open,
read_only, db_config)
if keyValueType == KeyValueStorageType.Memory:
return KeyValueStorageInMemory()
if keyValueType == KeyValueStorageType.ChunkedBinaryFile:
def chunk_creator(name):
return BinarySerializerBasedFileStore(txn_serializer,
os.path.join(dataLocation, keyValueStorageName),
name,
isLineNoKey=True,
storeContentHash=False,
ensureDurability=False)
return ChunkedFileStore(dataLocation,
keyValueStorageName,
isLineNoKey=True,
chunkSize=5,
chunk_creator=chunk_creator,
storeContentHash=False,
ensureDurability=False)
if keyValueType == KeyValueStorageType.BinaryFile:
return BinaryFileStore(dataLocation, keyValueStorageName,
delimiter=b'\0x8b\0xad\0xf0\0x0d\0x8b\0xad\0xf0\0x0d',
lineSep=b'\0xde\0xad\0xbe\0xef\0xde\0xad\0xbe\0xef',
storeContentHash=False)
raise KeyValueStorageConfigNotFound
def db(request, tempdir) -> KeyValueStorage:
if request.param == 'leveldb':
return KeyValueStorageLeveldb(tempdir, 'kv{}'.format(i))
if request.param == 'rocksdb':
return KeyValueStorageRocksdb(tempdir, 'kv{}'.format(i))
return KeyValueStorageInMemory()
