def testIncorrectWrites(tempdir):
fhs = FileHashStore(tempdir, leafSize=50, nodeSize=50)
with pytest.raises(ValueError):
fhs.writeLeaf(b"less than 50")
with pytest.raises(ValueError):
fhs.writeNode((8, 1, b"also less than 50"))
with pytest.raises(ValueError):
fhs.writeLeaf(b"more than 50" + b'1' * 50)
with pytest.raises(ValueError):
fhs.writeNode((4, 1, b"also more than 50" + b'1' * 50))
def testIncorrectWrites(tempdir):
fhs = FileHashStore(tempdir, leafSize=50, nodeSize=50)
with pytest.raises(ValueError):
fhs.writeLeaf(b"less than 50")
with pytest.raises(ValueError):
fhs.writeNode((8, 1, b"also less than 50"))
with pytest.raises(ValueError):
fhs.writeLeaf(b"more than 50" + b'1'*50)
with pytest.raises(ValueError):
fhs.writeNode((4, 1, b"also more than 50" + b'1'*50))
def testRecoverLedgerNewFieldsToTxnsAdded(tempdir):
fhs = FileHashStore(tempdir)
tree = CompactMerkleTree(hashStore=fhs)
ledger = Ledger(tree=tree, dataDir=tempdir, serializer=ledgerSerializer)
for d in range(10):
ledger.add({
"identifier": "i{}".format(d),
"reqId": d,
"op": "operation"
})
updatedTree = ledger.tree
ledger.stop()
newOrderedFields = OrderedDict([("identifier", (str, str)),
("reqId", (str, int)), ("op", (str, str)),
("newField", (str, str))])
newLedgerSerializer = CompactSerializer(newOrderedFields)
tree = CompactMerkleTree(hashStore=fhs)
restartedLedger = Ledger(tree=tree,
dataDir=tempdir,
serializer=newLedgerSerializer)
assert restartedLedger.size == ledger.size
assert restartedLedger.root_hash == ledger.root_hash
assert restartedLedger.tree.hashes == updatedTree.hashes
assert restartedLedger.tree.root_hash == updatedTree.root_hash
def hashStore(request, tempdir):
if request.param == 'File':
# with TemporaryDirectory() as tempdir:
fhs = FileHashStore(tempdir)
yield fhs
elif request.param == 'Memory':
yield MemoryHashStore()
def updateGenesisPoolTxnFile(genesisTxnDir, genesisTxnFile, txn):
# The lock is an advisory lock, it might not work on linux filesystems
# not mounted with option `-o mand`, another approach can be to use a .lock
# file to indicate presence or absence of .lock
try:
# Exclusively lock file in a non blocking manner. Locking is neccessary
# since there might be multiple clients running on a machine so genesis
# files should be updated safely.
# TODO: There is no automated test in the codebase that confirms it.
# It has only been manaully tested in the python terminal. Add a test
# for it using multiple processes writing concurrently
with portalocker.Lock(os.path.join(genesisTxnDir, genesisTxnFile),
truncate=None,
flags=portalocker.LOCK_EX | portalocker.LOCK_NB):
seqNo = txn[F.seqNo.name]
ledger = Ledger(CompactMerkleTree(hashStore=FileHashStore(
dataDir=genesisTxnDir)),
dataDir=genesisTxnDir,
fileName=genesisTxnFile)
ledgerSize = len(ledger)
if seqNo - ledgerSize == 1:
ledger.add({k: v for k, v in txn.items() if k != F.seqNo.name})
logger.debug('Adding transaction with sequence number {} in'
' genesis pool transaction file'.format(seqNo))
else:
logger.debug('Already {} genesis pool transactions present so '
'transaction with sequence number {} '
'not applicable'.format(ledgerSize, seqNo))
except (portalocker.LockException, portalocker.LockException) as ex:
return
def ledger(tempdir):
ledger = Ledger(
CompactMerkleTree(hashStore=FileHashStore(dataDir=tempdir)),
dataDir=tempdir,
serializer=ledgerSerializer)
ledger.reset()
return ledger
def hashStore(request, tdir):
if request.param == 'File':
fhs = FileHashStore(tdir)
assert fhs.is_persistent
yield fhs
elif request.param == 'Memory':
mhs = MemoryHashStore()
assert not mhs.is_persistent
yield mhs
def testConsistencyVerificationOnStartupCase1(tempdir):
"""
One more node was added to nodes file
"""
fhs = FileHashStore(tempdir)
tree = CompactMerkleTree(hashStore=fhs)
ledger = Ledger(tree=tree, dataDir=tempdir)
tranzNum = 10
for d in range(tranzNum):
ledger.add(str(d).encode())
ledger.stop()
# Writing one more node without adding of it to leaf and transaction logs
badNode = (None, None, ('X' * 32))
fhs.writeNode(badNode)
with pytest.raises(ConsistencyVerificationFailed):
tree = CompactMerkleTree(hashStore=fhs)
ledger = NoTransactionRecoveryLedger(tree=tree, dataDir=tempdir)
ledger.recoverTreeFromHashStore()
ledger.stop()
def testSimpleReadWrite(nodesLeaves, tempdir):
nodes, leaves = nodesLeaves
fhs = FileHashStore(tempdir)
for leaf in leaves:
fhs.writeLeaf(leaf)
for i, leaf in enumerate(leaves):
assert leaf == fhs.readLeaf(i + 1)
for node in nodes:
fhs.writeNode(node)
for i, node in enumerate(nodes):
assert node[2] == fhs.readNode(i + 1)
lvs = fhs.readLeafs(1, len(leaves))
for i, l in enumerate(lvs):
assert leaves[i] == l
nds = fhs.readNodes(1, len(nodes))
for i, n in enumerate(nds):
assert nodes[i][2] == n
def testConsistencyVerificationOnStartupCase1(tempdir):
"""
One more node was added to nodes file
"""
fhs = FileHashStore(tempdir)
tree = CompactMerkleTree(hashStore=fhs)
ledger = Ledger(tree=tree, dataDir=tempdir)
tranzNum = 10
for d in range(tranzNum):
ledger.add(str(d).encode())
ledger.stop()
# Writing one more node without adding of it to leaf and transaction logs
badNode = (None, None, ('X' * 32))
fhs.writeNode(badNode)
with pytest.raises(ConsistencyVerificationFailed):
tree = CompactMerkleTree(hashStore=fhs)
ledger = NoTransactionRecoveryLedger(tree=tree, dataDir=tempdir)
ledger.recoverTreeFromHashStore()
ledger.stop()
def writtenFhs(tempdir, nodes, leaves):
fhs = FileHashStore(tempdir)
for leaf in leaves:
fhs.writeLeaf(leaf)
for node in nodes:
fhs.writeNode(node)
return fhs
def testRecoverLedgerFromHashStore(tempdir):
fhs = FileHashStore(tempdir)
tree = CompactMerkleTree(hashStore=fhs)
ledger = Ledger(tree=tree, dataDir=tempdir)
for d in range(10):
ledger.add(str(d).encode())
updatedTree = ledger.tree
ledger.stop()
tree = CompactMerkleTree(hashStore=fhs)
restartedLedger = Ledger(tree=tree, dataDir=tempdir)
assert restartedLedger.size == ledger.size
assert restartedLedger.root_hash == ledger.root_hash
assert restartedLedger.tree.hashes == updatedTree.hashes
assert restartedLedger.tree.root_hash == updatedTree.root_hash
def updateGenesisPoolTxnFile(genesisTxnDir,
genesisTxnFile,
txn,
waitTimeIfAlreadyLocked=5):
# The lock is an advisory lock, it might not work on linux filesystems
# not mounted with option `-o mand`, another approach can be to use a .lock
# file to indicate presence or absence of .lock
genesisFilePath = open(os.path.join(genesisTxnDir, genesisTxnFile), 'a+')
try:
# Exclusively lock file in a non blocking manner. Locking is neccessary
# since there might be multiple clients running on a machine so genesis
# files should be updated safely.
# TODO: There is no automated test in the codebase that confirms it.
# It has only been manaully tested in the python terminal. Add a test
# for it using multiple processes writing concurrently
portalocker.Lock(genesisFilePath,
truncate=None,
flags=portalocker.LOCK_EX | portalocker.LOCK_NB)
seqNo = txn[F.seqNo.name]
ledger = Ledger(
CompactMerkleTree(hashStore=FileHashStore(dataDir=genesisTxnDir)),
dataDir=genesisTxnDir,
fileName=genesisTxnFile)
ledgerSize = len(ledger)
if seqNo - ledgerSize == 1:
ledger.add({k: v for k, v in txn.items() if k != F.seqNo.name})
logger.debug('Adding transaction with sequence number {} in'
' genesis pool transaction file'.format(seqNo))
else:
logger.debug('Already {} genesis pool transactions present so '
'transaction with sequence number {} '
'not applicable'.format(ledgerSize, seqNo))
portalocker.unlock(genesisFilePath)
except portalocker.AlreadyLocked as ex:
logger.info(
"file is already locked: {}, will retry in few seconds".format(
genesisFilePath))
if waitTimeIfAlreadyLocked <= 15:
time.sleep(waitTimeIfAlreadyLocked)
updateGenesisPoolTxnFile(genesisTxnDir, genesisTxnFile, txn,
waitTimeIfAlreadyLocked + 5)
else:
logger.error(
"already locked error even after few attempts {}: {}".format(
genesisFilePath, str(ex)))
except portalocker.LockException as ex:
logger.error("error occurred during locking file {}: {}".format(
genesisFilePath, str(ex)))
def ledger(self):
if self._ledger is None:
if not self.hasLedger:
defaultTxnFile = os.path.join(self.basedirpath,
self.ledgerFile)
if not os.path.isfile(defaultTxnFile):
raise FileNotFoundError("Pool transactions file not "
"found: {}".format(defaultTxnFile))
else:
shutil.copy(defaultTxnFile, self.ledgerLocation)
dataDir = self.ledgerLocation
self._ledger = Ledger(
CompactMerkleTree(hashStore=FileHashStore(dataDir=dataDir)),
dataDir=dataDir,
fileName=self.ledgerFile,
ensureDurability=self.config.EnsureLedgerDurability)
return self._ledger
def ledger(self):
if self._ledger is None:
defaultTxnFile = os.path.join(self.basedirpath,
self.ledgerFile)
if not os.path.exists(defaultTxnFile):
logger.debug("Not using default initialization file for "
"pool ledger, since it does not exist: {}"
.format(defaultTxnFile))
defaultTxnFile = None
dataDir = self.ledgerLocation
self.hashStore = FileHashStore(dataDir=dataDir)
self._ledger = Ledger(CompactMerkleTree(hashStore=self.hashStore),
dataDir=dataDir,
fileName=self.ledgerFile,
ensureDurability=self.config.EnsureLedgerDurability,
defaultFile=defaultTxnFile)
return self._ledger
def testConsistencyVerificationOnStartupCase2(tempdir):
"""
One more transaction added to transactions file
"""
fhs = FileHashStore(tempdir)
tree = CompactMerkleTree(hashStore=fhs)
ledger = Ledger(tree=tree, dataDir=tempdir)
tranzNum = 10
for d in range(tranzNum):
ledger.add(str(d).encode())
# Adding one more entry to transaction log without adding it to merkle tree
badData = 'X' * 32
value = ledger.leafSerializer.serialize(badData, toBytes=False)
key = str(tranzNum + 1)
ledger._transactionLog.put(key=key, value=value)
ledger.stop()
with pytest.raises(ConsistencyVerificationFailed):
tree = CompactMerkleTree(hashStore=fhs)
ledger = NoTransactionRecoveryLedger(tree=tree, dataDir=tempdir)
ledger.recoverTreeFromHashStore()
ledger.stop()
def hasherAndTree(hasher):
tdir = TemporaryDirectory().name
store = FileHashStore(tdir)
m = CompactMerkleTree(hasher=hasher, hashStore=store)
return hasher, m
def hashStore(request, tdir):
if request.param == 'File':
fhs = FileHashStore(tdir)
yield fhs
elif request.param == 'Memory':
yield MemoryHashStore()
def testSimpleReadWrite(nodesLeaves, tempdir):
nodes, leaves = nodesLeaves
fhs = FileHashStore(tempdir)
for leaf in leaves:
fhs.writeLeaf(leaf)
for i, leaf in enumerate(leaves):
assert leaf == fhs.readLeaf(i + 1)
for node in nodes:
fhs.writeNode(node)
for i, node in enumerate(nodes):
assert node[2] == fhs.readNode(i + 1)
lvs = fhs.readLeafs(1, len(leaves))
for i, l in enumerate(lvs):
assert leaves[i] == l
nds = fhs.readNodes(1, len(nodes))
for i, n in enumerate(nds):
assert nodes[i][2] == n
# Check that hash store can be closed and re-opened and the contents remain same
leaf_count = fhs.leafCount
node_count = fhs.nodeCount
fhs.close()
reopened_hash_store = FileHashStore(tempdir)
assert reopened_hash_store.leafCount == leaf_count
assert reopened_hash_store.nodeCount == node_count
def getConfigLedger(self):
return Ledger(CompactMerkleTree(hashStore=FileHashStore(
fileNamePrefix='config', dataDir=self.dataLocation)),
dataDir=self.dataLocation,
fileName=self.config.configTransactionsFile,
ensureDurability=self.config.EnsureLedgerDurability)