def testRecoverLedgerNewFieldsToTxnsAdded(tempdir):
fhs = FileHashStore(tempdir)
tree = CompactMerkleTree(hashStore=fhs)
ledger = Ledger(tree=tree, dataDir=tempdir, serializer=ledgerSerializer)
for d in range(10):
ledger.add({
"identifier": "i{}".format(d),
"reqId": d,
"op": "operation"
})
updatedTree = ledger.tree
ledger.stop()
newOrderedFields = OrderedDict([("identifier", (str, str)),
("reqId", (str, int)), ("op", (str, str)),
("newField", (str, str))])
newLedgerSerializer = CompactSerializer(newOrderedFields)
tree = CompactMerkleTree(hashStore=fhs)
restartedLedger = Ledger(tree=tree,
dataDir=tempdir,
serializer=newLedgerSerializer)
assert restartedLedger.size == ledger.size
assert restartedLedger.root_hash == ledger.root_hash
assert restartedLedger.tree.hashes == updatedTree.hashes
assert restartedLedger.tree.root_hash == updatedTree.root_hash
def __migrate_ledger(data_directory,
old_ledger_file, new_ledger_file,
serializer: MappingSerializer = None):
"""
Test for the directory, open old and new ledger, migrate data, rename directories
"""
# we should have ChunkedFileStorage implementation of the Ledger
if not os.path.isdir(os.path.join(data_directory, old_ledger_file)):
msg = 'Could not find directory {} for migration.'.format(
old_ledger_file)
logger.error(msg)
raise Exception(msg)
# open the old ledger using the specified serializer
old_ledger_file_backup = old_ledger_file + "_new"
old_txn_log_store = ChunkedFileStore(data_directory,
old_ledger_file_backup,
isLineNoKey=True,
storeContentHash=False)
old_ledger = Ledger(CompactMerkleTree(),
dataDir=data_directory,
txn_serializer=serializer,
hash_serializer=serializer,
fileName=old_ledger_file_backup,
transactionLogStore=old_txn_log_store)
# open the new ledger with new serialization
new_ledger = Ledger(CompactMerkleTree(),
dataDir=data_directory,
fileName=new_ledger_file)
logger.info("new size for {}: {}".format(
old_ledger_file_backup, str(new_ledger.size)))
# add all txns into the old ledger
for _, txn in new_ledger.getAllTxn():
old_ledger.add(txn)
logger.info("old size for {}: {}".format(
new_ledger_file, str(old_ledger.size)))
old_ledger.stop()
new_ledger.stop()
# now that everything succeeded, remove the new files and move the old
# files into place
shutil.rmtree(
os.path.join(data_directory, new_ledger_file))
os.rename(
os.path.join(data_directory, old_ledger_file_backup),
os.path.join(data_directory, old_ledger_file))
def testRecoverLedgerFromHashStore(tempdir):
fhs = FileHashStore(tempdir)
tree = CompactMerkleTree(hashStore=fhs)
ledger = Ledger(tree=tree, dataDir=tempdir)
for d in range(10):
ledger.add(str(d).encode())
updatedTree = ledger.tree
ledger.stop()
tree = CompactMerkleTree(hashStore=fhs)
restartedLedger = Ledger(tree=tree, dataDir=tempdir)
assert restartedLedger.size == ledger.size
assert restartedLedger.root_hash == ledger.root_hash
assert restartedLedger.tree.hashes == updatedTree.hashes
assert restartedLedger.tree.root_hash == updatedTree.root_hash
def tdirWithPoolTxns(poolTxnData, tdir, tconf):
ledger = Ledger(CompactMerkleTree(),
dataDir=tdir,
fileName=tconf.poolTransactionsFile)
for item in poolTxnData["txns"]:
ledger.add(item)
return tdir
def ledger(tempdir):
ledger = Ledger(
CompactMerkleTree(hashStore=FileHashStore(dataDir=tempdir)),
dataDir=tempdir,
serializer=ledgerSerializer)
ledger.reset()
return ledger
def testRecoverMerkleTreeFromLedger(tempdir):
ledger2 = Ledger(CompactMerkleTree(),
dataDir=tempdir,
serializer=ledgerSerializer)
assert ledger2.tree.root_hash is not None
ledger2.reset()
ledger2.stop()
def createGenesisTxnFile(genesisTxns,
targetDir,
fileName,
fieldOrdering,
reset=True):
ledger = Ledger(CompactMerkleTree(),
dataDir=targetDir,
serializer=CompactSerializer(fields=fieldOrdering),
fileName=fileName)
if reset:
ledger.reset()
reqIds = {}
for txn in genesisTxns:
identifier = txn.get(f.IDENTIFIER.nm, "")
if identifier not in reqIds:
reqIds[identifier] = 0
reqIds[identifier] += 1
txn.update({
f.REQ_ID.nm: reqIds[identifier],
f.IDENTIFIER.nm: identifier
})
ledger.add(txn)
ledger.stop()
def updateGenesisPoolTxnFile(genesisTxnDir, genesisTxnFile, txn):
# The lock is an advisory lock, it might not work on linux filesystems
# not mounted with option `-o mand`, another approach can be to use a .lock
# file to indicate presence or absence of .lock
try:
# Exclusively lock file in a non blocking manner. Locking is neccessary
# since there might be multiple clients running on a machine so genesis
# files should be updated safely.
# TODO: There is no automated test in the codebase that confirms it.
# It has only been manaully tested in the python terminal. Add a test
# for it using multiple processes writing concurrently
with portalocker.Lock(os.path.join(genesisTxnDir, genesisTxnFile),
truncate=None,
flags=portalocker.LOCK_EX | portalocker.LOCK_NB):
seqNo = txn[F.seqNo.name]
ledger = Ledger(CompactMerkleTree(hashStore=FileHashStore(
dataDir=genesisTxnDir)),
dataDir=genesisTxnDir,
fileName=genesisTxnFile)
ledgerSize = len(ledger)
if seqNo - ledgerSize == 1:
ledger.add({k: v for k, v in txn.items() if k != F.seqNo.name})
logger.debug('Adding transaction with sequence number {} in'
' genesis pool transaction file'.format(seqNo))
else:
logger.debug('Already {} genesis pool transactions present so '
'transaction with sequence number {} '
'not applicable'.format(ledgerSize, seqNo))
except (portalocker.LockException, portalocker.LockException) as ex:
return
def testRecoverLedgerFromHashStore(hashStore, tconf, tdir):
cleanup(hashStore)
tree = CompactMerkleTree(hashStore=hashStore)
ledger = Ledger(tree=tree, dataDir=tdir)
for d in range(10):
ledger.add(str(d).encode())
updatedTree = ledger.tree
ledger.stop()
tree = CompactMerkleTree(hashStore=hashStore)
restartedLedger = Ledger(tree=tree, dataDir=tdir)
assert restartedLedger.size == ledger.size
assert restartedLedger.root_hash == ledger.root_hash
assert restartedLedger.tree.hashes == updatedTree.hashes
assert restartedLedger.tree.root_hash == updatedTree.root_hash
restartedLedger.stop()
def test_parse_non_base58_txn_type_field_raises_descriptive_error(
tdirWithLedger, tdir):
with pytest.raises(ValueError) as excinfo:
ledger = Ledger(CompactMerkleTree(), dataDir=tdir)
_, _, nodeKeys = TxnStackManager.parseLedgerForHaAndKeys(ledger)
assert ("verkey" in str(excinfo.value))
ledger.stop()
def tdirWithPoolTxns(poolTxnData, tdir, tconf):
ledger = Ledger(CompactMerkleTree(),
dataDir=tdir,
fileName=tconf.poolTransactionsFile)
for item in poolTxnData["txns"]:
if item.get(TXN_TYPE) in (NEW_NODE, CHANGE_HA, CHANGE_KEYS):
ledger.add(item)
return tdir
def updatedDomainTxnFile(tdir, tdirWithDomainTxns, genesisTxns,
domainTxnOrderedFields, tconf):
ledger = Ledger(CompactMerkleTree(),
dataDir=tdir,
serializer=CompactSerializer(fields=domainTxnOrderedFields),
fileName=tconf.domainTransactionsFile)
for txn in genesisTxns:
ledger.add(txn)
def create_default_ledger(tempdir, init_genesis_txn_file=None):
genesis_txn_initiator = GenesisTxnInitiatorFromFile(
tempdir, init_genesis_txn_file) if init_genesis_txn_file else None
ledger = Ledger(
CompactMerkleTree(hashStore=FileHashStore(dataDir=tempdir)),
dataDir=tempdir,
genesis_txn_initiator=genesis_txn_initiator)
return ledger
def init_pool_ledger(cls, appendToLedgers, baseDir, config, envName):
poolTxnFile = cls.pool_ledger_file_name(config, envName)
pool_ledger = Ledger(CompactMerkleTree(),
dataDir=baseDir,
fileName=poolTxnFile)
if not appendToLedgers:
pool_ledger.reset()
return pool_ledger
def addTxnToFile(dir, file, txns, fields=getTxnOrderedFields()):
ledger = Ledger(CompactMerkleTree(),
dataDir=dir,
serializer=CompactSerializer(fields=fields),
fileName=file)
for txn in txns:
ledger.add(txn)
ledger.stop()
def tdirWithPoolTxns(poolTxnData, tdir, tconf):
ledger = Ledger(CompactMerkleTree(),
dataDir=tdir,
fileName=tconf.poolTransactionsFile)
for item in poolTxnData["txns"]:
if item.get(TXN_TYPE) == NODE:
ledger.add(item)
ledger.stop()
return tdir
def tdirWithDomainTxns(poolTxnData, tdir, tconf, domainTxnOrderedFields):
ledger = Ledger(
CompactMerkleTree(),
dataDir=tdir,
serializer=CompactSerializer(fields=domainTxnOrderedFields),
fileName=tconf.domainTransactionsFile)
for item in poolTxnData["txns"]:
if item.get(TXN_TYPE) == NYM:
ledger.add(item)
return tdir
def test_parse_identifier_non_base58_txn_type_field_raises_SystemExit_has_descriptive_error(
invalid_identifier_tdir, tdir_for_func):
"""
Test that invalid base58 IDENTIFIER in pool_transaction raises the proper exception (INDY-150)
"""
with pytest.raises(SystemExit) as excinfo:
ledger = Ledger(CompactMerkleTree(), dataDir=tdir_for_func)
_, _, nodeKeys = TxnStackManager.parseLedgerForHaAndKeys(ledger)
assert excinfo.value.code == errMsg2
ledger.stop()
def _open_new_ledger(data_directory, new_ledger_file, hash_store_name):
# open new Ledger with leveldb hash store (to re-init it)
logger.info("Open new ledger folder: {}".format(
os.path.join(data_directory, new_ledger_file)))
new_ledger = Ledger(CompactMerkleTree(
hashStore=LevelDbHashStore(
dataDir=data_directory, fileNamePrefix=hash_store_name)),
dataDir=data_directory,
fileName=new_ledger_file)
new_ledger.stop()
def test_parse_verkey_non_base58_txn_type_field_raises_SystemExit_has_descriptive_error(
invalid_verkey_tdir, tdir_for_func):
"""
Test that invalid base58 TARGET_NYM in pool_transaction raises the proper exception (INDY-150)
"""
with pytest.raises(SystemExit) as excinfo:
ledger = Ledger(CompactMerkleTree(), dataDir=tdir_for_func)
_, _, nodeKeys = TxnStackManager.parseLedgerForHaAndKeys(ledger)
assert excinfo.value.code == 'Invalid verkey. Rebuild pool transactions.'
ledger.stop()
def nodeSetLedger(nodeSet, tdir):
"""
Overrides the fixture from conftest.py
"""
for n in nodeSet:
dirPath = os.path.join(tdir, n.name, "temp")
if not os.path.exists(dirPath):
os.makedirs(dirPath)
n.txnStore = Ledger(CompactMerkleTree(), dirPath)
yield nodeSet
def init_domain_ledger(cls, appendToLedgers, baseDir, config, envName,
domainTxnFieldOrder):
domainTxnFile = cls.domain_ledger_file_name(config, envName)
ser = CompactSerializer(fields=domainTxnFieldOrder)
domain_ledger = Ledger(CompactMerkleTree(),
serializer=ser,
dataDir=baseDir,
fileName=domainTxnFile)
if not appendToLedgers:
domain_ledger.reset()
return domain_ledger
def invalid_identifier_tdir(tdir_for_func):
ledger = Ledger(CompactMerkleTree(), dataDir=tdir_for_func)
txn = Steward.node_txn(nym=base58.b58encode(b'whatever').decode("utf-8"),
steward_nym="invalid====",
node_name='test' + str(2),
ip='127.0.0.1',
node_port=8080,
client_port=8081,
client_ip='127.0.0.1')
ledger.add(txn)
ledger.stop()
def ledger():
ledger = Ledger()
transactions = (
("2015-01-16", "john", "mary", "120.50"),
("2015-01-17", "john", "supermarket", "20.00"),
("2015-01-17", "mary", "insurance", "100.00"),
("2015-01-18", "john", "insurance", "50.00"),
)
ledger._process_transactions(transactions)
return ledger
def create_initiator_ledger(self) -> Ledger:
store = TextFileStore(self.__data_dir,
self.__db_name,
isLineNoKey=True,
storeContentHash=False,
ensureDurability=False)
return Ledger(CompactMerkleTree(),
dataDir=self.__data_dir,
txn_serializer=self.__serializer,
fileName=self.__db_name,
transactionLogStore=store)
def tdirWithPoolTxns(poolTxnData, tdir, tconf):
import getpass
logging.debug("current user when creating new pool txn file: {}".format(
getpass.getuser()))
ledger = Ledger(CompactMerkleTree(),
dataDir=tdir,
fileName=tconf.poolTransactionsFile)
for item in poolTxnData["txns"]:
if item.get(TXN_TYPE) == NODE:
ledger.add(item)
ledger.stop()
return tdir
def invalid_verkey_tdir(tdir_for_func):
ledger = Ledger(CompactMerkleTree(), dataDir=tdir_for_func)
for d in range(3):
txn = Steward.node_txn(steward_nym="Th7MpTaRZVRYnPiabds81Y",
node_name='test' + str(d),
nym=base58.b58encode(b'whatever') if d != 1 else "invalid====",
ip='127.0.0.1',
node_port=8080,
client_port=8081,
client_ip='127.0.0.1')
ledger.add(txn)
ledger.stop()
def invalid_identifier_tdir(tdir_for_func):
ledger = Ledger(CompactMerkleTree(), dataDir=tdir_for_func)
txn = {TXN_TYPE: '0',
TARGET_NYM: base58.b58encode(b'whatever').decode("utf-8"),
IDENTIFIER: "invalid====",
DATA: {
NAME: str(2),
ALIAS: 'test' + str(2),
SERVICES: [VALIDATOR],
}
}
ledger.add(txn)
ledger.stop()
def __create_ledger(store,
txn_serializer,
hash_serializer,
tempdir,
init_genesis_txn_file=None):
genesis_txn_initiator = GenesisTxnInitiatorFromFile(
tempdir, init_genesis_txn_file) if init_genesis_txn_file else None
ledger = Ledger(
CompactMerkleTree(hashStore=FileHashStore(dataDir=tempdir)),
dataDir=tempdir,
transactionLogStore=store,
txn_serializer=txn_serializer,
hash_serializer=hash_serializer,
genesis_txn_initiator=genesis_txn_initiator)
return ledger
def getPoolTxnStore(self) -> Ledger:
"""
Create and return the pool transaction ledger.
"""
basedirpath = self.basedirpath
if not self.node.hasFile(self.poolTransactionsFile):
defaultTxnFile = os.path.join(basedirpath,
self.poolTransactionsFile)
if not os.path.isfile(defaultTxnFile):
raise FileNotFoundError("Pool transactions file not found")
else:
shutil.copy(defaultTxnFile, self.node.getDataLocation())
ledger = Ledger(CompactMerkleTree(), dataDir=self.node.getDataLocation(),
fileName=self.poolTransactionsFile)
return ledger
