def updateGenesisPoolTxnFile(genesisTxnDir, genesisTxnFile, txn):
# The lock is an advisory lock, it might not work on linux filesystems
# not mounted with option `-o mand`, another approach can be to use a .lock
# file to indicate presence or absence of .lock
try:
# Exclusively lock file in a non blocking manner. Locking is neccessary
# since there might be multiple clients running on a machine so genesis
# files should be updated safely.
# TODO: There is no automated test in the codebase that confirms it.
# It has only been manaully tested in the python terminal. Add a test
# for it using multiple processes writing concurrently
with portalocker.Lock(os.path.join(genesisTxnDir, genesisTxnFile),
truncate=None,
flags=portalocker.LOCK_EX | portalocker.LOCK_NB):
seqNo = txn[F.seqNo.name]
ledger = Ledger(CompactMerkleTree(hashStore=FileHashStore(
dataDir=genesisTxnDir)),
dataDir=genesisTxnDir,
fileName=genesisTxnFile)
ledgerSize = len(ledger)
if seqNo - ledgerSize == 1:
ledger.add({k: v for k, v in txn.items() if k != F.seqNo.name})
logger.debug('Adding transaction with sequence number {} in'
' genesis pool transaction file'.format(seqNo))
else:
logger.debug('Already {} genesis pool transactions present so '
'transaction with sequence number {} '
'not applicable'.format(ledgerSize, seqNo))
except (portalocker.LockException, portalocker.LockException) as ex:
return
def test_parse_non_base58_txn_type_field_raises_descriptive_error(
tdirWithLedger, tdir):
with pytest.raises(ValueError) as excinfo:
ledger = Ledger(CompactMerkleTree(), dataDir=tdir)
_, _, nodeKeys = TxnStackManager.parseLedgerForHaAndKeys(ledger)
assert ("verkey" in str(excinfo.value))
ledger.stop()
def updateGenesisPoolTxnFile(genesisTxnDir, genesisTxnFile, txn):
# The lock is an advisory lock, it might not work on linux filesystems
# not mounted with option `-o mand`, another approach can be to use a .lock
# file to indicate presence or absence of .lock
try:
# Exclusively lock file in a non blocking manner. Locking is neccessary
# since there might be multiple clients running on a machine so genesis
# files should be updated safely.
# TODO: There is no automated test in the codebase that confirms it.
# It has only been manaully tested in the python terminal. Add a test
# for it using multiple processes writing concurrently
with portalocker.Lock(os.path.join(genesisTxnDir, genesisTxnFile),
flags=portalocker.LOCK_EX | portalocker.LOCK_NB):
seqNo = txn[F.seqNo.name]
ledger = Ledger(CompactMerkleTree(hashStore=FileHashStore(
dataDir=genesisTxnDir)), dataDir=genesisTxnDir,
fileName=genesisTxnFile)
ledgerSize = len(ledger)
if seqNo - ledgerSize == 1:
ledger.add({k:v for k,v in txn.items() if k != F.seqNo.name})
logger.debug('Adding transaction with sequence number {} in'
' genesis pool transaction file'.format(seqNo))
else:
logger.debug('Already {} genesis pool transactions present so '
'transaction with sequence number {} '
'not applicable'.format(ledgerSize, seqNo))
except (portalocker.exceptions.LockException,
portalocker.exceptions.LockException) as ex:
return
def tdirWithPoolTxns(poolTxnData, tdir, tconf):
ledger = Ledger(CompactMerkleTree(),
dataDir=tdir,
fileName=tconf.poolTransactionsFile)
for item in poolTxnData["txns"]:
ledger.add(item)
return tdir
def ledger(tempdir):
ledger = Ledger(
CompactMerkleTree(hashStore=FileHashStore(dataDir=tempdir)),
dataDir=tempdir,
serializer=ledgerSerializer)
ledger.reset()
return ledger
def tdirWithPoolTxns(poolTxnData, tdir, tconf):
ledger = Ledger(CompactMerkleTree(),
dataDir=tdir,
fileName=tconf.poolTransactionsFile)
for item in poolTxnData["txns"]:
ledger.add(item)
return tdir
def check_audit_txn(txn,
view_no,
pp_seq_no,
seq_no,
txn_time,
txn_roots,
state_roots,
pool_size,
domain_size,
config_size,
last_domain_seqno,
last_pool_seqno,
last_config_seqno,
primaries,
digest='',
other_sizes={}):
expectedLedgerRoots = {}
txn_roots = {k: Ledger.hashToStr(v) for k, v in txn_roots.items()}
state_roots = {k: Ledger.hashToStr(v) for k, v in state_roots.items()}
# we expect deltas here, that is a difference from the current audit ledger txn to
# the audit txn where the corresponding ledger was updated
if last_domain_seqno:
expectedLedgerRoots[1] = seq_no - last_domain_seqno
if last_pool_seqno:
expectedLedgerRoots[0] = seq_no - last_pool_seqno
if last_config_seqno:
expectedLedgerRoots[2] = seq_no - last_config_seqno
expectedLedgerRoots.update(txn_roots)
ledger_size = {0: pool_size, 1: domain_size, 2: config_size}
ledger_size.update(other_sizes)
expected = {
"reqSignature": {},
"txn": {
"data": {
"ledgerRoot": expectedLedgerRoots,
"ver": "1",
"viewNo": view_no,
"ppSeqNo": pp_seq_no,
"ledgerSize": ledger_size,
"stateRoot": state_roots,
"primaries": primaries,
"digest": digest,
},
"metadata": {},
"protocolVersion": CURRENT_PROTOCOL_VERSION,
"type": "2", # AUDIT
},
"txnMetadata": {
"seqNo": seq_no,
"txnTime": txn_time
},
"ver": "1"
}
txn = JsonSerializer().serialize(txn)
expected = JsonSerializer().serialize(expected)
print(txn)
print(expected)
assert expected == txn
def addTxnToFile(dir, file, txns, fields=getTxnOrderedFields()):
ledger = Ledger(CompactMerkleTree(),
dataDir=dir,
serializer=CompactSerializer(fields=fields),
fileName=file)
for txn in txns:
ledger.add(txn)
ledger.stop()
def tdirWithPoolTxns(poolTxnData, tdir, tconf):
ledger = Ledger(CompactMerkleTree(),
dataDir=tdir,
fileName=tconf.poolTransactionsFile)
for item in poolTxnData["txns"]:
if item.get(TXN_TYPE) in (NEW_NODE, CHANGE_HA, CHANGE_KEYS):
ledger.add(item)
return tdir
def init_pool_ledger(cls, appendToLedgers, baseDir, config, envName):
poolTxnFile = cls.pool_ledger_file_name(config, envName)
pool_ledger = Ledger(CompactMerkleTree(),
dataDir=baseDir,
fileName=poolTxnFile)
if not appendToLedgers:
pool_ledger.reset()
return pool_ledger
def tdirWithPoolTxns(poolTxnData, tdir, tconf):
ledger = Ledger(CompactMerkleTree(),
dataDir=tdir,
fileName=tconf.poolTransactionsFile)
for item in poolTxnData["txns"]:
if item.get(TXN_TYPE) in (NEW_NODE, CHANGE_HA, CHANGE_KEYS):
ledger.add(item)
return tdir
def updatedDomainTxnFile(tdir, tdirWithDomainTxns, genesisTxns,
domainTxnOrderedFields, tconf):
ledger = Ledger(CompactMerkleTree(),
dataDir=tdir,
serializer=CompactSerializer(fields=domainTxnOrderedFields),
fileName=tconf.domainTransactionsFile)
for txn in genesisTxns:
ledger.add(txn)
def tdirWithDomainTxns(poolTxnData, tdir, tconf, domainTxnOrderedFields):
ledger = Ledger(CompactMerkleTree(),
dataDir=tdir,
serializer=CompactSerializer(fields=domainTxnOrderedFields),
fileName=tconf.domainTransactionsFile)
for item in poolTxnData["txns"]:
if item.get(TXN_TYPE) == NYM:
ledger.add(item)
return tdir
def test_parse_verkey_non_base58_txn_type_field_raises_SystemExit_has_descriptive_error(
invalid_verkey_tdir, tdir_for_func):
"""
Test that invalid base58 TARGET_NYM in pool_transaction raises the proper exception (INDY-150)
"""
with pytest.raises(SystemExit) as excinfo:
ledger = Ledger(CompactMerkleTree(), dataDir=tdir_for_func)
_, _, nodeKeys = TxnStackManager.parseLedgerForHaAndKeys(ledger)
assert excinfo.value.code == 'Invalid verkey. Rebuild pool transactions.'
ledger.stop()
def test_parse_identifier_non_base58_txn_type_field_raises_SystemExit_has_descriptive_error(
invalid_identifier_tdir, tdir_for_func):
"""
Test that invalid base58 IDENTIFIER in pool_transaction raises the proper exception (INDY-150)
"""
with pytest.raises(SystemExit) as excinfo:
ledger = Ledger(CompactMerkleTree(), dataDir=tdir_for_func)
_, _, nodeKeys = TxnStackManager.parseLedgerForHaAndKeys(ledger)
assert excinfo.value.code == errMsg2
ledger.stop()
def _open_new_ledger(data_directory, new_ledger_file, hash_store_name):
# open new Ledger with leveldb hash store (to re-init it)
logger.info("Open new ledger folder: {}".format(
os.path.join(data_directory, new_ledger_file)))
new_ledger = Ledger(CompactMerkleTree(
hashStore=LevelDbHashStore(
dataDir=data_directory, fileNamePrefix=hash_store_name)),
dataDir=data_directory,
fileName=new_ledger_file)
new_ledger.stop()
def __open_new_ledger(data_directory, new_ledger_file, hash_store_name):
# open new Ledger with leveldb hash store (to re-init it)
logger.info("Open new ledger folder: {}".format(
os.path.join(data_directory, new_ledger_file)))
new_ledger = Ledger(CompactMerkleTree(
hashStore=LevelDbHashStore(
dataDir=data_directory, fileNamePrefix=hash_store_name)),
dataDir=data_directory,
fileName=new_ledger_file)
new_ledger.stop()
def tdirWithDomainTxns(poolTxnData, tdir, tconf, domainTxnOrderedFields):
ledger = Ledger(
CompactMerkleTree(),
dataDir=tdir,
serializer=CompactSerializer(fields=domainTxnOrderedFields),
fileName=tconf.domainTransactionsFile)
for item in poolTxnData["txns"]:
if item.get(TXN_TYPE) == NYM:
ledger.add(item)
return tdir
def _do_apply_batch(self, batch):
reqs = [Request(**req_dict) for req_dict in batch[f.REQUESTS.nm]]
ledger_id = batch[f.LEDGER_ID.nm]
three_pc_batch = ThreePcBatch.from_batch_committed_dict(batch)
self._node.apply_reqs(reqs, three_pc_batch)
# We need hashes in apply and str in commit
three_pc_batch.txn_root = Ledger.hashToStr(three_pc_batch.txn_root)
three_pc_batch.state_root = Ledger.hashToStr(three_pc_batch.state_root)
self._node.get_executer(ledger_id)(three_pc_batch)
def init_domain_ledger(cls, appendToLedgers, baseDir, config, envName,
domainTxnFieldOrder):
domainTxnFile = cls.domain_ledger_file_name(config, envName)
ser = CompactSerializer(fields=domainTxnFieldOrder)
domain_ledger = Ledger(CompactMerkleTree(),
serializer=ser,
dataDir=baseDir,
fileName=domainTxnFile)
if not appendToLedgers:
domain_ledger.reset()
return domain_ledger
def ledger():
ledger = Ledger()
transactions = (
("2015-01-16", "john", "mary", "120.50"),
("2015-01-17", "john", "supermarket", "20.00"),
("2015-01-17", "mary", "insurance", "100.00"),
("2015-01-18", "john", "insurance", "50.00"),
)
ledger._process_transactions(transactions)
return ledger
def ledger(tempdir):
store = ChunkedFileStore(tempdir,
'transactions',
isLineNoKey=True,
chunkSize=chunk_size,
storeContentHash=False,
ensureDurability=False)
ledger = Ledger(CompactMerkleTree(hashStore=FileHashStore(dataDir=tempdir)),
dataDir=tempdir, serializer=JsonSerializer(),
transactionLogStore=store)
ledger.reset()
return ledger
def init_ledger_from_genesis_txn(self, ledger: Ledger):
# TODO: it's possible that the file to be used for initialization does not exist.
# This is not considered as an error as of now.
init_file = os.path.join(self.__data_dir, self.__db_name)
if not os.path.exists(init_file):
logger.display("File that should be used for initialization of "
"Ledger does not exist: {}".format(init_file))
return
with open(init_file, 'r') as f:
for line in store_utils.cleanLines(f):
txn = self.__serializer.deserialize(line)
ledger.add(txn)
def __fill_ledger_root_hash(self, txn, lid, ledger, last_audit_txn, three_pc_batch):
last_audit_txn_data = get_payload_data(last_audit_txn) if last_audit_txn is not None else None
if lid == three_pc_batch.ledger_id:
txn[AUDIT_TXN_LEDGER_ROOT][lid] = Ledger.hashToStr(ledger.uncommitted_root_hash)
txn[AUDIT_TXN_STATE_ROOT][lid] = Ledger.hashToStr(self.database_manager.get_state(lid).headHash)
# 1. it is the first batch and we have something
elif last_audit_txn_data is None and ledger.uncommitted_size:
txn[AUDIT_TXN_LEDGER_ROOT][lid] = Ledger.hashToStr(ledger.uncommitted_root_hash)
txn[AUDIT_TXN_STATE_ROOT][lid] = Ledger.hashToStr(self.database_manager.get_state(lid).headHash)
# 1.1. Rare case -- we have previous audit txns but don't have this ledger i.e. new plugins
elif last_audit_txn_data is not None and last_audit_txn_data[AUDIT_TXN_LEDGERS_SIZE][lid] is None and \
len(ledger.uncommittedTxns):
txn[AUDIT_TXN_LEDGER_ROOT][lid] = Ledger.hashToStr(ledger.uncommitted_root_hash)
txn[AUDIT_TXN_STATE_ROOT][lid] = Ledger.hashToStr(self.database_manager.get_state(lid).headHash)
# 2. Usual case -- this ledger was updated since the last audit txn
elif last_audit_txn_data is not None and last_audit_txn_data[AUDIT_TXN_LEDGERS_SIZE][lid] is not None and \
ledger.uncommitted_size > last_audit_txn_data[AUDIT_TXN_LEDGERS_SIZE][lid]:
txn[AUDIT_TXN_LEDGER_ROOT][lid] = Ledger.hashToStr(ledger.uncommitted_root_hash)
txn[AUDIT_TXN_STATE_ROOT][lid] = Ledger.hashToStr(self.database_manager.get_state(lid).headHash)
# 3. This ledger is never audited, so do not add the key
elif last_audit_txn_data is None or lid not in last_audit_txn_data[AUDIT_TXN_LEDGER_ROOT]:
return
# 4. ledger is not changed in last batch => delta = delta + 1
elif isinstance(last_audit_txn_data[AUDIT_TXN_LEDGER_ROOT][lid], int):
txn[AUDIT_TXN_LEDGER_ROOT][lid] = last_audit_txn_data[AUDIT_TXN_LEDGER_ROOT][lid] + 1
# 5. ledger is changed in last batch but not changed now => delta = 1
elif last_audit_txn_data:
txn[AUDIT_TXN_LEDGER_ROOT][lid] = 1
def createGenesisTxnFile(genesisTxns,
targetDir,
fileName,
fieldOrdering,
reset=True):
ledger = Ledger(CompactMerkleTree(),
dataDir=targetDir,
serializer=CompactSerializer(fields=fieldOrdering),
fileName=fileName)
if reset:
ledger.reset()
reqIds = {}
for txn in genesisTxns:
identifier = txn.get(f.IDENTIFIER.nm, "")
if identifier not in reqIds:
reqIds[identifier] = 0
reqIds[identifier] += 1
txn.update({
f.REQ_ID.nm: reqIds[identifier],
f.IDENTIFIER.nm: identifier
})
ledger.add(txn)
ledger.stop()
def testRecoverLedgerFromHashStore(odbhs, tdir):
cleanup(odbhs)
tree = CompactMerkleTree(hashStore=odbhs)
ledger = Ledger(tree=tree, dataDir=tdir)
for d in range(10):
ledger.add(str(d).encode())
updatedTree = ledger.tree
ledger.stop()
tree = CompactMerkleTree(hashStore=odbhs)
restartedLedger = Ledger(tree=tree, dataDir=tdir)
assert restartedLedger.size == ledger.size
assert restartedLedger.root_hash == ledger.root_hash
assert restartedLedger.tree.hashes == updatedTree.hashes
assert restartedLedger.tree.root_hash == updatedTree.root_hash
def updateGenesisPoolTxnFile(genesisTxnDir,
genesisTxnFile,
txn,
waitTimeIfAlreadyLocked=5):
# The lock is an advisory lock, it might not work on linux filesystems
# not mounted with option `-o mand`, another approach can be to use a .lock
# file to indicate presence or absence of .lock
genesisFilePath = open(os.path.join(genesisTxnDir, genesisTxnFile), 'a+')
try:
# Exclusively lock file in a non blocking manner. Locking is neccessary
# since there might be multiple clients running on a machine so genesis
# files should be updated safely.
# TODO: There is no automated test in the codebase that confirms it.
# It has only been manaully tested in the python terminal. Add a test
# for it using multiple processes writing concurrently
portalocker.Lock(genesisFilePath,
truncate=None,
flags=portalocker.LOCK_EX | portalocker.LOCK_NB)
seqNo = txn[F.seqNo.name]
ledger = Ledger(
CompactMerkleTree(hashStore=FileHashStore(dataDir=genesisTxnDir)),
dataDir=genesisTxnDir,
fileName=genesisTxnFile)
ledgerSize = len(ledger)
if seqNo - ledgerSize == 1:
ledger.add({k: v for k, v in txn.items() if k != F.seqNo.name})
logger.debug('Adding transaction with sequence number {} in'
' genesis pool transaction file'.format(seqNo))
else:
logger.debug('Already {} genesis pool transactions present so '
'transaction with sequence number {} '
'not applicable'.format(ledgerSize, seqNo))
portalocker.unlock(genesisFilePath)
except portalocker.AlreadyLocked as ex:
logger.info(
"file is already locked: {}, will retry in few seconds".format(
genesisFilePath))
if waitTimeIfAlreadyLocked <= 15:
time.sleep(waitTimeIfAlreadyLocked)
updateGenesisPoolTxnFile(genesisTxnDir, genesisTxnFile, txn,
waitTimeIfAlreadyLocked + 5)
else:
logger.error(
"already locked error even after few attempts {}: {}".format(
genesisFilePath, str(ex)))
except portalocker.LockException as ex:
logger.error("error occurred during locking file {}: {}".format(
genesisFilePath, str(ex)))
def emulate_ordered_processing(self, msg: Ordered):
three_pc_batch = ThreePcBatch.from_ordered(msg)
three_pc_batch.txn_root = Ledger.hashToStr(three_pc_batch.txn_root)
three_pc_batch.state_root = Ledger.hashToStr(three_pc_batch.state_root)
self._write_manager.commit_batch(three_pc_batch)
possible_added = set(three_pc_batch.node_reg) - set(self._previous_node_reg)
possible_removed = set(self._previous_node_reg) - set(three_pc_batch.node_reg)
if possible_added:
for node_name in list(possible_added):
self._internal_bus.send(NeedAddNode(node_name, self.name))
if possible_removed:
for node_name in list(possible_removed):
self._internal_bus.send(NeedRemoveNode(node_name, self.name))
if possible_added or possible_removed:
self._previous_node_reg = three_pc_batch.node_reg
def _open_old_ledger(data_directory, old_ledger_file, hash_store_name,
serializer):
# open old Ledger with leveldb hash store (to re-init it)
old_txn_log_store = ChunkedFileStore(data_directory,
old_ledger_file,
isLineNoKey=True,
storeContentHash=False)
old_ledger = Ledger(CompactMerkleTree(hashStore=LevelDbHashStore(
dataDir=data_directory, fileNamePrefix=hash_store_name)),
dataDir=data_directory,
txn_serializer=serializer,
hash_serializer=serializer,
fileName=old_ledger_file,
transactionLogStore=old_txn_log_store)
old_ledger.stop()
def _create_audit_txn_data(self, three_pc_batch, last_audit_txn):
# 1. general format and (view_no, pp_seq_no)
txn = {
TXN_VERSION: "1",
AUDIT_TXN_VIEW_NO: three_pc_batch.view_no,
AUDIT_TXN_PP_SEQ_NO: three_pc_batch.pp_seq_no,
AUDIT_TXN_LEDGERS_SIZE: {},
AUDIT_TXN_LEDGER_ROOT: {},
AUDIT_TXN_STATE_ROOT: {},
AUDIT_TXN_PRIMARIES: None
}
for lid, ledger in self.database_manager.ledgers.items():
if lid == AUDIT_LEDGER_ID:
continue
# 2. ledger size
txn[AUDIT_TXN_LEDGERS_SIZE][lid] = ledger.uncommitted_size
# 3. ledger root (either root_hash or seq_no to last changed)
# TODO: support setting for multiple ledgers
self.__fill_ledger_root_hash(txn, three_pc_batch, lid, last_audit_txn)
# 4. state root hash
txn[AUDIT_TXN_STATE_ROOT][three_pc_batch.ledger_id] = Ledger.hashToStr(three_pc_batch.state_root)
# 5. set primaries field
self.__fill_primaries(txn, three_pc_batch, last_audit_txn)
return txn
def __fill_ledger_root_hash(self, txn, three_pc_batch, lid,
last_audit_txn):
target_ledger_id = three_pc_batch.ledger_id
last_audit_txn_data = get_payload_data(
last_audit_txn) if last_audit_txn is not None else None
# 1. ledger is changed in this batch => root_hash
if lid == target_ledger_id:
txn[AUDIT_TXN_LEDGER_ROOT][str(lid)] = Ledger.hashToStr(
three_pc_batch.txn_root)
# 2. This ledger is never audited, so do not add the key
elif last_audit_txn_data is None or str(
lid) not in last_audit_txn_data[AUDIT_TXN_LEDGER_ROOT]:
return
# 3. ledger is not changed in last batch => the same audit seq no
elif isinstance(last_audit_txn_data[AUDIT_TXN_LEDGER_ROOT][str(lid)],
int):
txn[AUDIT_TXN_LEDGER_ROOT][str(
lid)] = last_audit_txn_data[AUDIT_TXN_LEDGER_ROOT][str(lid)]
# 4. ledger is changed in last batch but not changed now => seq_no of last audit txn
elif last_audit_txn_data:
txn[AUDIT_TXN_LEDGER_ROOT][str(lid)] = get_seq_no(last_audit_txn)
def testRecoverLedgerFromHashStore(hashStore, tconf, tdir):
cleanup(hashStore)
tree = CompactMerkleTree(hashStore=hashStore)
ledger = Ledger(tree=tree, dataDir=tdir)
for d in range(10):
ledger.add(str(d).encode())
updatedTree = ledger.tree
ledger.stop()
tree = CompactMerkleTree(hashStore=hashStore)
restartedLedger = Ledger(tree=tree, dataDir=tdir)
assert restartedLedger.size == ledger.size
assert restartedLedger.root_hash == ledger.root_hash
assert restartedLedger.tree.hashes == updatedTree.hashes
assert restartedLedger.tree.root_hash == updatedTree.root_hash
restartedLedger.stop()
def check_audit_txn(txn, view_no, pp_seq_no, seq_no, txn_time, ledger_id,
txn_root, state_root, pool_size, domain_size, config_size,
last_domain_seqno, last_pool_seqno, last_config_seqno):
expectedLedgerRoots = {}
# we expect deltas here, that is a difference from the current audit ledger txn to
# the audit txn where the corresponding ledger was updated
if last_domain_seqno:
expectedLedgerRoots[1] = seq_no - last_domain_seqno
if last_pool_seqno:
expectedLedgerRoots[0] = seq_no - last_pool_seqno
if last_config_seqno:
expectedLedgerRoots[2] = seq_no - last_config_seqno
expectedLedgerRoots[ledger_id] = Ledger.hashToStr(txn_root)
expected = {
"reqSignature": {},
"txn": {
"data": {
"ledgerRoot": expectedLedgerRoots,
"ver": "1",
"viewNo": view_no,
"ppSeqNo": pp_seq_no,
"ledgerSize": {
0: pool_size,
1: domain_size,
2: config_size
},
"stateRoot": {
ledger_id: Ledger.hashToStr(state_root),
}
},
"metadata": {},
"protocolVersion": CURRENT_PROTOCOL_VERSION,
"type": "2", # AUDIT
},
"txnMetadata": {
"seqNo": seq_no,
"txnTime": txn_time
},
"ver": "1"
}
txn = JsonSerializer().serialize(txn)
expected = JsonSerializer().serialize(expected)
print(txn)
print(expected)
assert expected == txn
def create_default_ledger(tempdir, init_genesis_txn_file=None):
genesis_txn_initiator = GenesisTxnInitiatorFromFile(
tempdir, init_genesis_txn_file) if init_genesis_txn_file else None
ledger = Ledger(
CompactMerkleTree(hashStore=FileHashStore(dataDir=tempdir)),
dataDir=tempdir,
genesis_txn_initiator=genesis_txn_initiator)
return ledger
def tdirWithLedger(tdir):
tree = CompactMerkleTree()
ledger = Ledger(CompactMerkleTree(), dataDir=tdir)
for d in range(3):
txn = {
TXN_TYPE: '0',
TARGET_NYM: base58.b58encode(b'whatever'),
DATA: {
NAME: str(d),
ALIAS: 'test' + str(d),
SERVICES: [VALIDATOR],
}
}
if d == 1:
txn[TARGET_NYM] = "invalid===="
ledger.add(txn)
return ledger
def _migrate_ledger(data_directory,
old_ledger_file, new_ledger_file,
serializer: MappingSerializer = None):
"""
Test for the directory, open old and new ledger, migrate data, rename directories
"""
# we should have ChunkedFileStorage implementation of the Ledger
if not os.path.isdir(os.path.join(data_directory, old_ledger_file)):
msg = 'Could not find directory {} for migration.'.format(
old_ledger_file)
logger.error(msg)
raise Exception(msg)
# open the old ledger using the specified serializer
old_ledger_file_backup = old_ledger_file + "_new"
old_txn_log_store = ChunkedFileStore(data_directory,
old_ledger_file_backup,
isLineNoKey=True,
storeContentHash=False)
old_ledger = Ledger(CompactMerkleTree(),
dataDir=data_directory,
txn_serializer=serializer,
hash_serializer=serializer,
fileName=old_ledger_file_backup,
transactionLogStore=old_txn_log_store)
# open the new ledger with new serialization
new_ledger = Ledger(CompactMerkleTree(),
dataDir=data_directory,
fileName=new_ledger_file)
logger.info("new size for {}: {}".format(
old_ledger_file_backup, str(new_ledger.size)))
# add all txns into the old ledger
for _, txn in new_ledger.getAllTxn():
old_ledger.add(txn)
logger.info("old size for {}: {}".format(
new_ledger_file, str(old_ledger.size)))
old_ledger.stop()
new_ledger.stop()
# now that everything succeeded, remove the new files and move the old
# files into place
shutil.rmtree(
os.path.join(data_directory, new_ledger_file))
os.rename(
os.path.join(data_directory, old_ledger_file_backup),
os.path.join(data_directory, old_ledger_file))
def testRecoverLedgerNewFieldsToTxnsAdded(tempdir):
fhs = FileHashStore(tempdir)
tree = CompactMerkleTree(hashStore=fhs)
ledger = Ledger(tree=tree, dataDir=tempdir, serializer=ledgerSerializer)
for d in range(10):
ledger.add({
"identifier": "i{}".format(d),
"reqId": d,
"op": "operation"
})
updatedTree = ledger.tree
ledger.stop()
newOrderedFields = OrderedDict([("identifier", (str, str)),
("reqId", (str, int)), ("op", (str, str)),
("newField", (str, str))])
newLedgerSerializer = CompactSerializer(newOrderedFields)
tree = CompactMerkleTree(hashStore=fhs)
restartedLedger = Ledger(tree=tree,
dataDir=tempdir,
serializer=newLedgerSerializer)
assert restartedLedger.size == ledger.size
assert restartedLedger.root_hash == ledger.root_hash
assert restartedLedger.tree.hashes == updatedTree.hashes
assert restartedLedger.tree.root_hash == updatedTree.root_hash
def _open_old_ledger(data_directory, old_ledger_file,
hash_store_name, serializer):
# open old Ledger with leveldb hash store (to re-init it)
old_txn_log_store = ChunkedFileStore(data_directory,
old_ledger_file,
isLineNoKey=True,
storeContentHash=False)
old_ledger = Ledger(CompactMerkleTree(
hashStore=LevelDbHashStore(
dataDir=data_directory,
fileNamePrefix=hash_store_name)),
dataDir=data_directory,
txn_serializer=serializer,
hash_serializer=serializer,
fileName=old_ledger_file,
transactionLogStore=old_txn_log_store)
old_ledger.stop()
def check_audit_txn(txn, view_no, pp_seq_no, seq_no, txn_time, ledger_id,
txn_root, state_root, pool_size, domain_size, config_size,
last_domain_seqno, last_pool_seqno, last_config_seqno):
expectedLedgerRoots = {}
if last_domain_seqno:
expectedLedgerRoots["1"] = last_domain_seqno
if last_pool_seqno:
expectedLedgerRoots["0"] = last_pool_seqno
if last_config_seqno:
expectedLedgerRoots["2"] = last_config_seqno
expectedLedgerRoots[str(ledger_id)] = Ledger.hashToStr(txn_root)
expected = {
"reqSignature": {},
"txn": {
"data": {
"ledgerRoot": expectedLedgerRoots,
"ver": "1",
"viewNo": view_no,
"ppSeqNo": pp_seq_no,
"ledgerSize": {
"0": pool_size,
"1": domain_size,
"2": config_size
},
"stateRoot": {
str(ledger_id): Ledger.hashToStr(state_root),
}
},
"metadata": {},
"protocolVersion": CURRENT_PROTOCOL_VERSION,
"type": "2", # AUDIT
},
"txnMetadata": {
"seqNo": seq_no,
"txnTime": txn_time
},
"ver": "1"
}
txn = JsonSerializer().serialize(txn)
expected = JsonSerializer().serialize(expected)
print(txn)
print(expected)
assert expected == txn
def nodeSetLedger(nodeSet, tdir):
"""
Overrides the fixture from conftest.py
"""
for n in nodeSet:
dirPath = os.path.join(tdir, n.name, "temp")
if not os.path.exists(dirPath):
os.makedirs(dirPath)
n.txnStore = Ledger(CompactMerkleTree(), dirPath)
yield nodeSet
def testConsistencyVerificationOnStartupCase1(tempdir):
"""
One more node was added to nodes file
"""
fhs = FileHashStore(tempdir)
tree = CompactMerkleTree(hashStore=fhs)
ledger = Ledger(tree=tree, dataDir=tempdir)
tranzNum = 10
for d in range(tranzNum):
ledger.add(str(d).encode())
ledger.stop()
# Writing one more node without adding of it to leaf and transaction logs
badNode = (None, None, ('X' * 32))
fhs.writeNode(badNode)
with pytest.raises(ConsistencyVerificationFailed):
tree = CompactMerkleTree(hashStore=fhs)
ledger = NoTransactionRecoveryLedger(tree=tree, dataDir=tempdir)
ledger.recoverTreeFromHashStore()
ledger.stop()
def testRecoverLedgerNewFieldsToTxnsAdded(tempdir):
fhs = FileHashStore(tempdir)
tree = CompactMerkleTree(hashStore=fhs)
ledger = Ledger(tree=tree, dataDir=tempdir, serializer=ledgerSerializer)
for d in range(10):
ledger.add({"identifier": "i{}".format(d), "reqId": d, "op": "operation"})
updatedTree = ledger.tree
ledger.stop()
newOrderedFields = OrderedDict([
("identifier", (str, str)),
("reqId", (str, int)),
("op", (str, str)),
("newField", (str, str))
])
newLedgerSerializer = CompactSerializer(newOrderedFields)
tree = CompactMerkleTree(hashStore=fhs)
restartedLedger = Ledger(tree=tree, dataDir=tempdir, serializer=newLedgerSerializer)
assert restartedLedger.size == ledger.size
assert restartedLedger.root_hash == ledger.root_hash
assert restartedLedger.tree.hashes == updatedTree.hashes
assert restartedLedger.tree.root_hash == updatedTree.root_hash
def testConsistencyVerificationOnStartupCase2(tempdir):
"""
One more transaction added to transactions file
"""
fhs = FileHashStore(tempdir)
tree = CompactMerkleTree(hashStore=fhs)
ledger = Ledger(tree=tree, dataDir=tempdir)
tranzNum = 10
for d in range(tranzNum):
ledger.add(str(d).encode())
# Adding one more entry to transaction log without adding it to merkle tree
badData = 'X' * 32
value = ledger.leafSerializer.serialize(badData, toBytes=False)
key = str(tranzNum + 1)
ledger._transactionLog.put(key=key, value=value)
ledger.stop()
with pytest.raises(ConsistencyVerificationFailed):
tree = CompactMerkleTree(hashStore=fhs)
ledger = NoTransactionRecoveryLedger(tree=tree, dataDir=tempdir)
ledger.recoverTreeFromHashStore()
ledger.stop()
def createGenesisTxnFile(genesisTxns, targetDir, fileName, fieldOrdering,
reset=True):
ledger = Ledger(CompactMerkleTree(), dataDir=targetDir,
serializer=CompactSerializer(fields=fieldOrdering),
fileName=fileName)
if reset:
ledger.reset()
reqIds = {}
for txn in genesisTxns:
identifier = txn.get(f.IDENTIFIER.nm, "")
if identifier not in reqIds:
reqIds[identifier] = 0
reqIds[identifier] += 1
txn.update({
f.REQ_ID.nm: reqIds[identifier],
f.IDENTIFIER.nm: identifier
})
ledger.add(txn)
ledger.stop()
def _processCatchupReplies(self, ledgerType, ledger: Ledger,
catchUpReplies: List):
# Removing transactions for sequence numbers are already
# present in the ledger
numProcessed = sum(1 for s, _ in catchUpReplies if s <= ledger.size)
catchUpReplies = catchUpReplies[numProcessed:]
if numProcessed:
logger.debug("{} found {} already processed transactions in the "
"catchup replies".format(self, numProcessed))
if catchUpReplies:
seqNo = catchUpReplies[0][0]
if seqNo - ledger.seqNo == 1:
result, nodeName, toBeProcessed = self.hasValidCatchupReplies(
ledgerType, ledger, seqNo, catchUpReplies)
if result:
for _, txn in catchUpReplies[:toBeProcessed]:
merkleInfo = ledger.add(txn)
txn[F.seqNo.name] = merkleInfo[F.seqNo.name]
self.ledgers[ledgerType]["postTxnAddedToLedgerClbk"](
ledgerType, txn)
self._removePrcdCatchupReply(ledgerType, nodeName, seqNo)
return numProcessed + toBeProcessed + \
self._processCatchupReplies(ledgerType, ledger,
catchUpReplies[toBeProcessed:])
else:
if self.ownedByNode:
self.owner.blacklistNode(nodeName,
reason="Sent transactions "
"that could not be "
"verified")
self._removePrcdCatchupReply(ledgerType, nodeName,
seqNo)
# Invalid transactions have to be discarded so letting
# the caller know how many txns have to removed from
# `self.receivedCatchUpReplies`
return numProcessed + toBeProcessed
return numProcessed
def ledger(tempdir):
ledger = Ledger(CompactMerkleTree(hashStore=FileHashStore(dataDir=tempdir)),
dataDir=tempdir, serializer=ledgerSerializer)
ledger.reset()
return ledger
def bootstrapTestNodesCore(baseDir,
poolTransactionsFile,
domainTransactionsFile,
domainTxnFieldOrder,
ips, nodeCount, clientCount,
nodeNum, startingPort):
if not ips:
ips = ['127.0.0.1'] * nodeCount
else:
ips = ips.split(",")
if len(ips) != nodeCount:
if len(ips) > nodeCount:
ips = ips[:nodeCount]
else:
ips = ips + ['127.0.0.1'] * (nodeCount - len(ips))
poolLedger = Ledger(CompactMerkleTree(),
dataDir=baseDir,
fileName=poolTransactionsFile)
poolLedger.reset()
domainLedger = Ledger(CompactMerkleTree(),
serializer=CompactSerializer(fields=
domainTxnFieldOrder),
dataDir=baseDir,
fileName=domainTransactionsFile)
domainLedger.reset()
steward1Nym = None
for num in range(1, nodeCount + 1):
stewardName = "Steward" + str(num)
sigseed = TestNetworkSetup.getSigningSeed(stewardName)
verkey = Signer(sigseed).verhex
stewardNym = TestNetworkSetup.getNymFromVerkey(verkey)
txn = {
TARGET_NYM: stewardNym,
TXN_TYPE: NYM,
ROLE: STEWARD,
ALIAS: stewardName,
TXN_ID: sha256(stewardName.encode()).hexdigest()
}
if num == 1:
steward1Nym = stewardNym
else:
# The first steward adds every steward
txn[f.IDENTIFIER.nm] = steward1Nym
domainLedger.add(txn)
nodeName = "Node" + str(num)
nodePort, clientPort = startingPort + (num * 2 - 1), startingPort \
+ (num * 2)
ip = ips[num - 1]
sigseed = TestNetworkSetup.getSigningSeed(nodeName)
if nodeNum == num:
_, verkey = initLocalKeep(nodeName, baseDir, sigseed, True)
verkey = verkey.encode()
print("This node with name {} will use ports {} and {} for "
"nodestack and clientstack respectively"
.format(nodeName, nodePort, clientPort))
else:
verkey = Signer(sigseed).verhex
txn = {
TARGET_NYM: TestNetworkSetup.getNymFromVerkey(verkey),
TXN_TYPE: NEW_NODE,
f.IDENTIFIER.nm: stewardNym,
DATA: {
CLIENT_IP: ip,
ALIAS: nodeName,
CLIENT_PORT: clientPort,
NODE_IP: ip,
NODE_PORT: nodePort
},
TXN_ID: sha256(nodeName.encode()).hexdigest()
}
poolLedger.add(txn)
for num in range(1, clientCount + 1):
clientName = "Client" + str(num)
sigseed = TestNetworkSetup.getSigningSeed(clientName)
verkey = Signer(sigseed).verhex
txn = {
f.IDENTIFIER.nm: steward1Nym,
TARGET_NYM: TestNetworkSetup.getNymFromVerkey(verkey),
TXN_TYPE: NYM,
ALIAS: clientName,
TXN_ID: sha256(clientName.encode()).hexdigest()
}
domainLedger.add(txn)
poolLedger.stop()
domainLedger.stop()
def testRecoverMerkleTreeFromLedger(tempdir):
ledger2 = Ledger(CompactMerkleTree(), dataDir=tempdir,
serializer=ledgerSerializer)
assert ledger2.tree.root_hash is not None
ledger2.reset()
ledger2.stop()