def testSimpleReadWrite(nodesLeaves, tempdir): nodes, leaves = nodesLeaves fhs = FileHashStore(tempdir) for leaf in leaves: fhs.writeLeaf(leaf) for i, leaf in enumerate(leaves): assert leaf == fhs.readLeaf(i + 1) for node in nodes: fhs.writeNode(node) for i, node in enumerate(nodes): assert node[2] == fhs.readNode(i + 1) lvs = fhs.readLeafs(1, len(leaves)) for i, l in enumerate(lvs): assert leaves[i] == l nds = fhs.readNodes(1, len(nodes)) for i, n in enumerate(nds): assert nodes[i][2] == n # Check that hash store can be closed and re-opened and the contents remain same leaf_count = fhs.leafCount node_count = fhs.nodeCount fhs.close() reopened_hash_store = FileHashStore(tempdir) assert reopened_hash_store.leafCount == leaf_count assert reopened_hash_store.nodeCount == node_count
def hashStore(request, tempdir): if request.param == 'File': # with TemporaryDirectory() as tempdir: fhs = FileHashStore(tempdir) yield fhs elif request.param == 'Memory': yield MemoryHashStore()
def testRecoverLedgerNewFieldsToTxnsAdded(tempdir): fhs = FileHashStore(tempdir) tree = CompactMerkleTree(hashStore=fhs) ledger = Ledger(tree=tree, dataDir=tempdir, serializer=ledgerSerializer) for d in range(10): ledger.add({ "identifier": "i{}".format(d), "reqId": d, "op": "operation" }) updatedTree = ledger.tree ledger.stop() newOrderedFields = OrderedDict([("identifier", (str, str)), ("reqId", (str, int)), ("op", (str, str)), ("newField", (str, str))]) newLedgerSerializer = CompactSerializer(newOrderedFields) tree = CompactMerkleTree(hashStore=fhs) restartedLedger = Ledger(tree=tree, dataDir=tempdir, serializer=newLedgerSerializer) assert restartedLedger.size == ledger.size assert restartedLedger.root_hash == ledger.root_hash assert restartedLedger.tree.hashes == updatedTree.hashes assert restartedLedger.tree.root_hash == updatedTree.root_hash
def writtenFhs(tempdir, nodes, leaves): fhs = FileHashStore(tempdir) for leaf in leaves: fhs.writeLeaf(leaf) for node in nodes: fhs.writeNode(node) return fhs
def ledger(tempdir): ledger = Ledger( CompactMerkleTree(hashStore=FileHashStore(dataDir=tempdir)), dataDir=tempdir, serializer=ledgerSerializer) ledger.reset() return ledger
def updateGenesisPoolTxnFile(genesisTxnDir, genesisTxnFile, txn): # The lock is an advisory lock, it might not work on linux filesystems # not mounted with option `-o mand`, another approach can be to use a .lock # file to indicate presence or absence of .lock try: # Exclusively lock file in a non blocking manner. Locking is neccessary # since there might be multiple clients running on a machine so genesis # files should be updated safely. # TODO: There is no automated test in the codebase that confirms it. # It has only been manaully tested in the python terminal. Add a test # for it using multiple processes writing concurrently with portalocker.Lock(os.path.join(genesisTxnDir, genesisTxnFile), truncate=None, flags=portalocker.LOCK_EX | portalocker.LOCK_NB): seqNo = txn[F.seqNo.name] ledger = Ledger(CompactMerkleTree(hashStore=FileHashStore( dataDir=genesisTxnDir)), dataDir=genesisTxnDir, fileName=genesisTxnFile) ledgerSize = len(ledger) if seqNo - ledgerSize == 1: ledger.add({k: v for k, v in txn.items() if k != F.seqNo.name}) logger.debug('Adding transaction with sequence number {} in' ' genesis pool transaction file'.format(seqNo)) else: logger.debug('Already {} genesis pool transactions present so ' 'transaction with sequence number {} ' 'not applicable'.format(ledgerSize, seqNo)) except (portalocker.LockException, portalocker.LockException) as ex: return
def hashStore(request, tdir): if request.param == 'File': fhs = FileHashStore(tdir) assert fhs.is_persistent yield fhs elif request.param == 'Memory': mhs = MemoryHashStore() assert not mhs.is_persistent yield mhs
def testIncorrectWrites(tempdir): fhs = FileHashStore(tempdir, leafSize=50, nodeSize=50) with pytest.raises(ValueError): fhs.writeLeaf(b"less than 50") with pytest.raises(ValueError): fhs.writeNode((8, 1, b"also less than 50")) with pytest.raises(ValueError): fhs.writeLeaf(b"more than 50" + b'1' * 50) with pytest.raises(ValueError): fhs.writeNode((4, 1, b"also more than 50" + b'1' * 50))
def testRecoverLedgerFromHashStore(tempdir): fhs = FileHashStore(tempdir) tree = CompactMerkleTree(hashStore=fhs) ledger = Ledger(tree=tree, dataDir=tempdir) for d in range(10): ledger.add(str(d).encode()) updatedTree = ledger.tree ledger.stop() tree = CompactMerkleTree(hashStore=fhs) restartedLedger = Ledger(tree=tree, dataDir=tempdir) assert restartedLedger.size == ledger.size assert restartedLedger.root_hash == ledger.root_hash assert restartedLedger.tree.hashes == updatedTree.hashes assert restartedLedger.tree.root_hash == updatedTree.root_hash
def updateGenesisPoolTxnFile(genesisTxnDir, genesisTxnFile, txn, waitTimeIfAlreadyLocked=5): # The lock is an advisory lock, it might not work on linux filesystems # not mounted with option `-o mand`, another approach can be to use a .lock # file to indicate presence or absence of .lock genesisFilePath = open(os.path.join(genesisTxnDir, genesisTxnFile), 'a+') try: # Exclusively lock file in a non blocking manner. Locking is neccessary # since there might be multiple clients running on a machine so genesis # files should be updated safely. # TODO: There is no automated test in the codebase that confirms it. # It has only been manaully tested in the python terminal. Add a test # for it using multiple processes writing concurrently portalocker.Lock(genesisFilePath, truncate=None, flags=portalocker.LOCK_EX | portalocker.LOCK_NB) seqNo = txn[F.seqNo.name] ledger = Ledger( CompactMerkleTree(hashStore=FileHashStore(dataDir=genesisTxnDir)), dataDir=genesisTxnDir, fileName=genesisTxnFile) ledgerSize = len(ledger) if seqNo - ledgerSize == 1: ledger.add({k: v for k, v in txn.items() if k != F.seqNo.name}) logger.debug('Adding transaction with sequence number {} in' ' genesis pool transaction file'.format(seqNo)) else: logger.debug('Already {} genesis pool transactions present so ' 'transaction with sequence number {} ' 'not applicable'.format(ledgerSize, seqNo)) portalocker.unlock(genesisFilePath) except portalocker.AlreadyLocked as ex: logger.info( "file is already locked: {}, will retry in few seconds".format( genesisFilePath)) if waitTimeIfAlreadyLocked <= 15: time.sleep(waitTimeIfAlreadyLocked) updateGenesisPoolTxnFile(genesisTxnDir, genesisTxnFile, txn, waitTimeIfAlreadyLocked + 5) else: logger.error( "already locked error even after few attempts {}: {}".format( genesisFilePath, str(ex))) except portalocker.LockException as ex: logger.error("error occurred during locking file {}: {}".format( genesisFilePath, str(ex)))
def ledger(self): if self._ledger is None: defaultTxnFile = os.path.join(self.basedirpath, self.ledgerFile) if not os.path.exists(defaultTxnFile): logger.debug("Not using default initialization file for " "pool ledger, since it does not exist: {}" .format(defaultTxnFile)) defaultTxnFile = None dataDir = self.ledgerLocation self.hashStore = FileHashStore(dataDir=dataDir) self._ledger = Ledger(CompactMerkleTree(hashStore=self.hashStore), dataDir=dataDir, fileName=self.ledgerFile, ensureDurability=self.config.EnsureLedgerDurability, defaultFile=defaultTxnFile) return self._ledger
def ledger(self): if self._ledger is None: if not self.hasLedger: defaultTxnFile = os.path.join(self.basedirpath, self.ledgerFile) if not os.path.isfile(defaultTxnFile): raise FileNotFoundError("Pool transactions file not " "found: {}".format(defaultTxnFile)) else: shutil.copy(defaultTxnFile, self.ledgerLocation) dataDir = self.ledgerLocation self._ledger = Ledger( CompactMerkleTree(hashStore=FileHashStore(dataDir=dataDir)), dataDir=dataDir, fileName=self.ledgerFile, ensureDurability=self.config.EnsureLedgerDurability) return self._ledger
def testConsistencyVerificationOnStartupCase1(tempdir): """ One more node was added to nodes file """ fhs = FileHashStore(tempdir) tree = CompactMerkleTree(hashStore=fhs) ledger = Ledger(tree=tree, dataDir=tempdir) tranzNum = 10 for d in range(tranzNum): ledger.add(str(d).encode()) ledger.stop() # Writing one more node without adding of it to leaf and transaction logs badNode = (None, None, ('X' * 32)) fhs.writeNode(badNode) with pytest.raises(ConsistencyVerificationFailed): tree = CompactMerkleTree(hashStore=fhs) ledger = NoTransactionRecoveryLedger(tree=tree, dataDir=tempdir) ledger.recoverTreeFromHashStore() ledger.stop()
def testSimpleReadWrite(nodesLeaves, tempdir): nodes, leaves = nodesLeaves fhs = FileHashStore(tempdir) for leaf in leaves: fhs.writeLeaf(leaf) for i, leaf in enumerate(leaves): assert leaf == fhs.readLeaf(i + 1) for node in nodes: fhs.writeNode(node) for i, node in enumerate(nodes): assert node[2] == fhs.readNode(i + 1) lvs = fhs.readLeafs(1, len(leaves)) for i, l in enumerate(lvs): assert leaves[i] == l nds = fhs.readNodes(1, len(nodes)) for i, n in enumerate(nds): assert nodes[i][2] == n
def testConsistencyVerificationOnStartupCase2(tempdir): """ One more transaction added to transactions file """ fhs = FileHashStore(tempdir) tree = CompactMerkleTree(hashStore=fhs) ledger = Ledger(tree=tree, dataDir=tempdir) tranzNum = 10 for d in range(tranzNum): ledger.add(str(d).encode()) # Adding one more entry to transaction log without adding it to merkle tree badData = 'X' * 32 value = ledger.leafSerializer.serialize(badData, toBytes=False) key = str(tranzNum + 1) ledger._transactionLog.put(key=key, value=value) ledger.stop() with pytest.raises(ConsistencyVerificationFailed): tree = CompactMerkleTree(hashStore=fhs) ledger = NoTransactionRecoveryLedger(tree=tree, dataDir=tempdir) ledger.recoverTreeFromHashStore() ledger.stop()
def getConfigLedger(self): return Ledger(CompactMerkleTree(hashStore=FileHashStore( fileNamePrefix='config', dataDir=self.dataLocation)), dataDir=self.dataLocation, fileName=self.config.configTransactionsFile, ensureDurability=self.config.EnsureLedgerDurability)
def hasherAndTree(hasher): tdir = TemporaryDirectory().name store = FileHashStore(tdir) m = CompactMerkleTree(hasher=hasher, hashStore=store) return hasher, m
def hashStore(request, tdir): if request.param == 'File': fhs = FileHashStore(tdir) yield fhs elif request.param == 'Memory': yield MemoryHashStore()