def test_parse_non_base58_txn_type_field_raises_descriptive_error( tdirWithLedger, tdir): with pytest.raises(ValueError) as excinfo: ledger = Ledger(CompactMerkleTree(), dataDir=tdir) _, _, nodeKeys = TxnStackManager.parseLedgerForHaAndKeys(ledger) assert ("verkey" in str(excinfo.value)) ledger.stop()
def testRecoverLedgerNewFieldsToTxnsAdded(tempdir): fhs = FileHashStore(tempdir) tree = CompactMerkleTree(hashStore=fhs) ledger = Ledger(tree=tree, dataDir=tempdir, serializer=ledgerSerializer) for d in range(10): ledger.add({ "identifier": "i{}".format(d), "reqId": d, "op": "operation" }) updatedTree = ledger.tree ledger.stop() newOrderedFields = OrderedDict([("identifier", (str, str)), ("reqId", (str, int)), ("op", (str, str)), ("newField", (str, str))]) newLedgerSerializer = CompactSerializer(newOrderedFields) tree = CompactMerkleTree(hashStore=fhs) restartedLedger = Ledger(tree=tree, dataDir=tempdir, serializer=newLedgerSerializer) assert restartedLedger.size == ledger.size assert restartedLedger.root_hash == ledger.root_hash assert restartedLedger.tree.hashes == updatedTree.hashes assert restartedLedger.tree.root_hash == updatedTree.root_hash
def testRecoverMerkleTreeFromLedger(tempdir): ledger2 = Ledger(CompactMerkleTree(), dataDir=tempdir, serializer=ledgerSerializer) assert ledger2.tree.root_hash is not None ledger2.reset() ledger2.stop()
def createGenesisTxnFile(genesisTxns, targetDir, fileName, fieldOrdering, reset=True): ledger = Ledger(CompactMerkleTree(), dataDir=targetDir, serializer=CompactSerializer(fields=fieldOrdering), fileName=fileName) if reset: ledger.reset() reqIds = {} for txn in genesisTxns: identifier = txn.get(f.IDENTIFIER.nm, "") if identifier not in reqIds: reqIds[identifier] = 0 reqIds[identifier] += 1 txn.update({ f.REQ_ID.nm: reqIds[identifier], f.IDENTIFIER.nm: identifier }) ledger.add(txn) ledger.stop()
def addTxnToFile(dir, file, txns, fields=getTxnOrderedFields()): ledger = Ledger(CompactMerkleTree(), dataDir=dir, serializer=CompactSerializer(fields=fields), fileName=file) for txn in txns: ledger.add(txn) ledger.stop()
def tdirWithPoolTxns(poolTxnData, tdir, tconf): ledger = Ledger(CompactMerkleTree(), dataDir=tdir, fileName=tconf.poolTransactionsFile) for item in poolTxnData["txns"]: if item.get(TXN_TYPE) == NODE: ledger.add(item) ledger.stop() return tdir
def _open_new_ledger(data_directory, new_ledger_file, hash_store_name): # open new Ledger with leveldb hash store (to re-init it) logger.info("Open new ledger folder: {}".format( os.path.join(data_directory, new_ledger_file))) new_ledger = Ledger(CompactMerkleTree( hashStore=LevelDbHashStore( dataDir=data_directory, fileNamePrefix=hash_store_name)), dataDir=data_directory, fileName=new_ledger_file) new_ledger.stop()
def __open_new_ledger(data_directory, new_ledger_file, hash_store_name): # open new Ledger with leveldb hash store (to re-init it) logger.info("Open new ledger folder: {}".format( os.path.join(data_directory, new_ledger_file))) new_ledger = Ledger(CompactMerkleTree( hashStore=LevelDbHashStore( dataDir=data_directory, fileNamePrefix=hash_store_name)), dataDir=data_directory, fileName=new_ledger_file) new_ledger.stop()
def test_parse_verkey_non_base58_txn_type_field_raises_SystemExit_has_descriptive_error( invalid_verkey_tdir, tdir_for_func): """ Test that invalid base58 TARGET_NYM in pool_transaction raises the proper exception (INDY-150) """ with pytest.raises(SystemExit) as excinfo: ledger = Ledger(CompactMerkleTree(), dataDir=tdir_for_func) _, _, nodeKeys = TxnStackManager.parseLedgerForHaAndKeys(ledger) assert excinfo.value.code == 'Invalid verkey. Rebuild pool transactions.' ledger.stop()
def test_parse_identifier_non_base58_txn_type_field_raises_SystemExit_has_descriptive_error( invalid_identifier_tdir, tdir_for_func): """ Test that invalid base58 IDENTIFIER in pool_transaction raises the proper exception (INDY-150) """ with pytest.raises(SystemExit) as excinfo: ledger = Ledger(CompactMerkleTree(), dataDir=tdir_for_func) _, _, nodeKeys = TxnStackManager.parseLedgerForHaAndKeys(ledger) assert excinfo.value.code == errMsg2 ledger.stop()
def tdirWithDomainTxns(poolTxnData, tdir, tconf, domainTxnOrderedFields): ledger = Ledger(CompactMerkleTree(), dataDir=tdir, serializer=CompactSerializer(fields=domainTxnOrderedFields), fileName=tconf.domainTransactionsFile) for item in poolTxnData["txns"]: if item.get(TXN_TYPE) == NYM: ledger.add(item) ledger.stop() return tdir
def invalid_identifier_tdir(tdir_for_func): ledger = Ledger(CompactMerkleTree(), dataDir=tdir_for_func) txn = Steward.node_txn(nym=base58.b58encode(b'whatever').decode("utf-8"), steward_nym="invalid====", node_name='test' + str(2), ip='127.0.0.1', node_port=8080, client_port=8081, client_ip='127.0.0.1') ledger.add(txn) ledger.stop()
def tdirWithPoolTxns(poolTxnData, tdir, tconf): import getpass logging.debug("current user when creating new pool txn file: {}".format( getpass.getuser())) ledger = Ledger(CompactMerkleTree(), dataDir=tdir, fileName=tconf.poolTransactionsFile) for item in poolTxnData["txns"]: if item.get(TXN_TYPE) == NODE: ledger.add(item) ledger.stop() return tdir
def invalid_verkey_tdir(tdir_for_func): ledger = Ledger(CompactMerkleTree(), dataDir=tdir_for_func) for d in range(3): txn = Steward.node_txn(steward_nym="Th7MpTaRZVRYnPiabds81Y", node_name='test' + str(d), nym=base58.b58encode(b'whatever') if d != 1 else "invalid====", ip='127.0.0.1', node_port=8080, client_port=8081, client_ip='127.0.0.1') ledger.add(txn) ledger.stop()
def invalid_identifier_tdir(tdir_for_func): ledger = Ledger(CompactMerkleTree(), dataDir=tdir_for_func) txn = {TXN_TYPE: '0', TARGET_NYM: base58.b58encode(b'whatever').decode("utf-8"), IDENTIFIER: "invalid====", DATA: { NAME: str(2), ALIAS: 'test' + str(2), SERVICES: [VALIDATOR], } } ledger.add(txn) ledger.stop()
def __migrate_ledger(data_directory, old_ledger_file, new_ledger_file, serializer: MappingSerializer = None): """ Test for the directory, open old and new ledger, migrate data, rename directories """ # we should have ChunkedFileStorage implementation of the Ledger if not os.path.isdir(os.path.join(data_directory, old_ledger_file)): msg = 'Could not find directory {} for migration.'.format( old_ledger_file) logger.error(msg) raise Exception(msg) # open the old ledger using the specified serializer old_ledger_file_backup = old_ledger_file + "_new" old_txn_log_store = ChunkedFileStore(data_directory, old_ledger_file_backup, isLineNoKey=True, storeContentHash=False) old_ledger = Ledger(CompactMerkleTree(), dataDir=data_directory, txn_serializer=serializer, hash_serializer=serializer, fileName=old_ledger_file_backup, transactionLogStore=old_txn_log_store) # open the new ledger with new serialization new_ledger = Ledger(CompactMerkleTree(), dataDir=data_directory, fileName=new_ledger_file) logger.info("new size for {}: {}".format( old_ledger_file_backup, str(new_ledger.size))) # add all txns into the old ledger for _, txn in new_ledger.getAllTxn(): old_ledger.add(txn) logger.info("old size for {}: {}".format( new_ledger_file, str(old_ledger.size))) old_ledger.stop() new_ledger.stop() # now that everything succeeded, remove the new files and move the old # files into place shutil.rmtree( os.path.join(data_directory, new_ledger_file)) os.rename( os.path.join(data_directory, old_ledger_file_backup), os.path.join(data_directory, old_ledger_file))
def _migrate_ledger(data_directory, old_ledger_file, new_ledger_file, serializer: MappingSerializer = None): """ Test for the directory, open old and new ledger, migrate data, rename directories """ # we should have ChunkedFileStorage implementation of the Ledger if not os.path.isdir(os.path.join(data_directory, old_ledger_file)): msg = 'Could not find directory {} for migration.'.format( old_ledger_file) logger.error(msg) raise Exception(msg) # open the old ledger using the specified serializer old_ledger_file_backup = old_ledger_file + "_new" old_txn_log_store = ChunkedFileStore(data_directory, old_ledger_file_backup, isLineNoKey=True, storeContentHash=False) old_ledger = Ledger(CompactMerkleTree(), dataDir=data_directory, txn_serializer=serializer, hash_serializer=serializer, fileName=old_ledger_file_backup, transactionLogStore=old_txn_log_store) # open the new ledger with new serialization new_ledger = Ledger(CompactMerkleTree(), dataDir=data_directory, fileName=new_ledger_file) logger.info("new size for {}: {}".format( old_ledger_file_backup, str(new_ledger.size))) # add all txns into the old ledger for _, txn in new_ledger.getAllTxn(): old_ledger.add(txn) logger.info("old size for {}: {}".format( new_ledger_file, str(old_ledger.size))) old_ledger.stop() new_ledger.stop() # now that everything succeeded, remove the new files and move the old # files into place shutil.rmtree( os.path.join(data_directory, new_ledger_file)) os.rename( os.path.join(data_directory, old_ledger_file_backup), os.path.join(data_directory, old_ledger_file))
def testRecoverLedgerFromHashStore(tempdir): fhs = FileHashStore(tempdir) tree = CompactMerkleTree(hashStore=fhs) ledger = Ledger(tree=tree, dataDir=tempdir) for d in range(10): ledger.add(str(d).encode()) updatedTree = ledger.tree ledger.stop() tree = CompactMerkleTree(hashStore=fhs) restartedLedger = Ledger(tree=tree, dataDir=tempdir) assert restartedLedger.size == ledger.size assert restartedLedger.root_hash == ledger.root_hash assert restartedLedger.tree.hashes == updatedTree.hashes assert restartedLedger.tree.root_hash == updatedTree.root_hash
def testRecoverLedgerFromHashStore(odbhs, tdir): cleanup(odbhs) tree = CompactMerkleTree(hashStore=odbhs) ledger = Ledger(tree=tree, dataDir=tdir) for d in range(10): ledger.add(str(d).encode()) updatedTree = ledger.tree ledger.stop() tree = CompactMerkleTree(hashStore=odbhs) restartedLedger = Ledger(tree=tree, dataDir=tdir) assert restartedLedger.size == ledger.size assert restartedLedger.root_hash == ledger.root_hash assert restartedLedger.tree.hashes == updatedTree.hashes assert restartedLedger.tree.root_hash == updatedTree.root_hash
def invalid_verkey_tdir(tdir_for_func): ledger = Ledger(CompactMerkleTree(), dataDir=tdir_for_func) for d in range(3): txn = {TXN_TYPE: '0', TARGET_NYM: base58.b58encode(b'whatever').decode("utf-8"), IDENTIFIER: "Th7MpTaRZVRYnPiabds81Y", DATA: { NAME: str(d), ALIAS: 'test' + str(d), SERVICES: [VALIDATOR], } } if d == 1: txn[TARGET_NYM] = "invalid====" ledger.add(txn) ledger.stop()
def testRecoverLedgerFromHashStore(hashStore, tconf, tdir): cleanup(hashStore) tree = CompactMerkleTree(hashStore=hashStore) ledger = Ledger(tree=tree, dataDir=tdir) for d in range(10): ledger.add(str(d).encode()) updatedTree = ledger.tree ledger.stop() tree = CompactMerkleTree(hashStore=hashStore) restartedLedger = Ledger(tree=tree, dataDir=tdir) assert restartedLedger.size == ledger.size assert restartedLedger.root_hash == ledger.root_hash assert restartedLedger.tree.hashes == updatedTree.hashes assert restartedLedger.tree.root_hash == updatedTree.root_hash restartedLedger.stop()
def _open_old_ledger(data_directory, old_ledger_file, hash_store_name, serializer): # open old Ledger with leveldb hash store (to re-init it) old_txn_log_store = ChunkedFileStore(data_directory, old_ledger_file, isLineNoKey=True, storeContentHash=False) old_ledger = Ledger(CompactMerkleTree(hashStore=LevelDbHashStore( dataDir=data_directory, fileNamePrefix=hash_store_name)), dataDir=data_directory, txn_serializer=serializer, hash_serializer=serializer, fileName=old_ledger_file, transactionLogStore=old_txn_log_store) old_ledger.stop()
def _open_old_ledger(data_directory, old_ledger_file, hash_store_name, serializer): # open old Ledger with leveldb hash store (to re-init it) old_txn_log_store = ChunkedFileStore(data_directory, old_ledger_file, isLineNoKey=True, storeContentHash=False) old_ledger = Ledger(CompactMerkleTree( hashStore=LevelDbHashStore( dataDir=data_directory, fileNamePrefix=hash_store_name)), dataDir=data_directory, txn_serializer=serializer, hash_serializer=serializer, fileName=old_ledger_file, transactionLogStore=old_txn_log_store) old_ledger.stop()
def testConsistencyVerificationOnStartupCase1(tempdir): """ One more node was added to nodes file """ fhs = FileHashStore(tempdir) tree = CompactMerkleTree(hashStore=fhs) ledger = Ledger(tree=tree, dataDir=tempdir) tranzNum = 10 for d in range(tranzNum): ledger.add(str(d).encode()) ledger.stop() # Writing one more node without adding of it to leaf and transaction logs badNode = (None, None, ('X' * 32)) fhs.writeNode(badNode) with pytest.raises(ConsistencyVerificationFailed): tree = CompactMerkleTree(hashStore=fhs) ledger = NoTransactionRecoveryLedger(tree=tree, dataDir=tempdir) ledger.recoverTreeFromHashStore() ledger.stop()
def testRecoverLedgerNewFieldsToTxnsAdded(tempdir): fhs = FileHashStore(tempdir) tree = CompactMerkleTree(hashStore=fhs) ledger = Ledger(tree=tree, dataDir=tempdir, serializer=ledgerSerializer) for d in range(10): ledger.add({"identifier": "i{}".format(d), "reqId": d, "op": "operation"}) updatedTree = ledger.tree ledger.stop() newOrderedFields = OrderedDict([ ("identifier", (str, str)), ("reqId", (str, int)), ("op", (str, str)), ("newField", (str, str)) ]) newLedgerSerializer = CompactSerializer(newOrderedFields) tree = CompactMerkleTree(hashStore=fhs) restartedLedger = Ledger(tree=tree, dataDir=tempdir, serializer=newLedgerSerializer) assert restartedLedger.size == ledger.size assert restartedLedger.root_hash == ledger.root_hash assert restartedLedger.tree.hashes == updatedTree.hashes assert restartedLedger.tree.root_hash == updatedTree.root_hash
def testConsistencyVerificationOnStartupCase2(tempdir): """ One more transaction added to transactions file """ fhs = FileHashStore(tempdir) tree = CompactMerkleTree(hashStore=fhs) ledger = Ledger(tree=tree, dataDir=tempdir) tranzNum = 10 for d in range(tranzNum): ledger.add(str(d).encode()) # Adding one more entry to transaction log without adding it to merkle tree badData = 'X' * 32 value = ledger.leafSerializer.serialize(badData, toBytes=False) key = str(tranzNum + 1) ledger._transactionLog.put(key=key, value=value) ledger.stop() with pytest.raises(ConsistencyVerificationFailed): tree = CompactMerkleTree(hashStore=fhs) ledger = NoTransactionRecoveryLedger(tree=tree, dataDir=tempdir) ledger.recoverTreeFromHashStore() ledger.stop()
def bootstrapTestNodesCore(baseDir, poolTransactionsFile, domainTransactionsFile, domainTxnFieldOrder, ips, nodeCount, clientCount, nodeNum, startingPort): if not ips: ips = ['127.0.0.1'] * nodeCount else: ips = ips.split(",") if len(ips) != nodeCount: if len(ips) > nodeCount: ips = ips[:nodeCount] else: ips = ips + ['127.0.0.1'] * (nodeCount - len(ips)) poolLedger = Ledger(CompactMerkleTree(), dataDir=baseDir, fileName=poolTransactionsFile) poolLedger.reset() domainLedger = Ledger(CompactMerkleTree(), serializer=CompactSerializer(fields= domainTxnFieldOrder), dataDir=baseDir, fileName=domainTransactionsFile) domainLedger.reset() steward1Nym = None for num in range(1, nodeCount + 1): stewardName = "Steward" + str(num) sigseed = TestNetworkSetup.getSigningSeed(stewardName) verkey = Signer(sigseed).verhex stewardNym = TestNetworkSetup.getNymFromVerkey(verkey) txn = { TARGET_NYM: stewardNym, TXN_TYPE: NYM, ROLE: STEWARD, ALIAS: stewardName, TXN_ID: sha256(stewardName.encode()).hexdigest() } if num == 1: steward1Nym = stewardNym else: # The first steward adds every steward txn[f.IDENTIFIER.nm] = steward1Nym domainLedger.add(txn) nodeName = "Node" + str(num) nodePort, clientPort = startingPort + (num * 2 - 1), startingPort \ + (num * 2) ip = ips[num - 1] sigseed = TestNetworkSetup.getSigningSeed(nodeName) if nodeNum == num: _, verkey = initLocalKeep(nodeName, baseDir, sigseed, True) verkey = verkey.encode() print("This node with name {} will use ports {} and {} for " "nodestack and clientstack respectively" .format(nodeName, nodePort, clientPort)) else: verkey = Signer(sigseed).verhex txn = { TARGET_NYM: TestNetworkSetup.getNymFromVerkey(verkey), TXN_TYPE: NEW_NODE, f.IDENTIFIER.nm: stewardNym, DATA: { CLIENT_IP: ip, ALIAS: nodeName, CLIENT_PORT: clientPort, NODE_IP: ip, NODE_PORT: nodePort }, TXN_ID: sha256(nodeName.encode()).hexdigest() } poolLedger.add(txn) for num in range(1, clientCount + 1): clientName = "Client" + str(num) sigseed = TestNetworkSetup.getSigningSeed(clientName) verkey = Signer(sigseed).verhex txn = { f.IDENTIFIER.nm: steward1Nym, TARGET_NYM: TestNetworkSetup.getNymFromVerkey(verkey), TXN_TYPE: NYM, ALIAS: clientName, TXN_ID: sha256(clientName.encode()).hexdigest() } domainLedger.add(txn) poolLedger.stop() domainLedger.stop()
def bootstrapTestNodesCore(config, envName, appendToLedgers, domainTxnFieldOrder, ips, nodeCount, clientCount, nodeNum, startingPort): baseDir = config.baseDir if not os.path.exists(baseDir): os.makedirs(baseDir, exist_ok=True) if not ips: ips = ['127.0.0.1'] * nodeCount else: ips = ips.split(",") if len(ips) != nodeCount: if len(ips) > nodeCount: ips = ips[:nodeCount] else: ips += ['127.0.0.1'] * (nodeCount - len(ips)) if hasattr(config, "ENVS") and envName: poolTxnFile = config.ENVS[envName].poolLedger domainTxnFile = config.ENVS[envName].domainLedger else: poolTxnFile = config.poolTransactionsFile domainTxnFile = config.domainTransactionsFile poolLedger = Ledger(CompactMerkleTree(), dataDir=baseDir, fileName=poolTxnFile) domainLedger = Ledger( CompactMerkleTree(), serializer=CompactSerializer(fields=domainTxnFieldOrder), dataDir=baseDir, fileName=domainTxnFile) if not appendToLedgers: poolLedger.reset() domainLedger.reset() steward1Nym = None for num in range(1, nodeCount + 1): stewardName = "Steward" + str(num) sigseed = TestNetworkSetup.getSigningSeed(stewardName) verkey = Signer(sigseed).verhex stewardNym = TestNetworkSetup.getNymFromVerkey(verkey) txn = { TARGET_NYM: stewardNym, TXN_TYPE: NYM, ROLE: STEWARD, ALIAS: stewardName, TXN_ID: sha256(stewardName.encode()).hexdigest() } if num == 1: steward1Nym = stewardNym else: # The first steward adds every steward txn[f.IDENTIFIER.nm] = steward1Nym domainLedger.add(txn) nodeName = "Node" + str(num) nodePort, clientPort = startingPort + (num * 2 - 1), startingPort \ + (num * 2) ip = ips[num - 1] sigseed = TestNetworkSetup.getSigningSeed(nodeName) if nodeNum == num: _, verkey = initLocalKeep(nodeName, baseDir, sigseed, True) verkey = verkey.encode() print("This node with name {} will use ports {} and {} for " "nodestack and clientstack respectively".format( nodeName, nodePort, clientPort)) else: verkey = Signer(sigseed).verhex txn = { TARGET_NYM: TestNetworkSetup.getNymFromVerkey(verkey), TXN_TYPE: NODE, f.IDENTIFIER.nm: stewardNym, DATA: { CLIENT_IP: ip, ALIAS: nodeName, CLIENT_PORT: clientPort, NODE_IP: ip, NODE_PORT: nodePort, SERVICES: [VALIDATOR] }, TXN_ID: sha256(nodeName.encode()).hexdigest() } poolLedger.add(txn) for num in range(1, clientCount + 1): clientName = "Client" + str(num) sigseed = TestNetworkSetup.getSigningSeed(clientName) verkey = Signer(sigseed).verhex txn = { f.IDENTIFIER.nm: steward1Nym, TARGET_NYM: TestNetworkSetup.getNymFromVerkey(verkey), TXN_TYPE: NYM, ALIAS: clientName, TXN_ID: sha256(clientName.encode()).hexdigest() } domainLedger.add(txn) poolLedger.stop() domainLedger.stop()