def _finish(self, cons_proof: Optional[ConsistencyProof] = None):
logger.info("{} finished with consistency proof {}".format(
self, cons_proof))
till = CatchupTill(
start_size=cons_proof.seqNoStart,
final_size=cons_proof.seqNoEnd,
final_hash=cons_proof.newMerkleRoot) if cons_proof else None
nodes_ledger_sizes = {
frm: proof.seqNoEnd
for frm, proof in self._cons_proofs.items() if proof is not None
}
# Stop requesting last consistency proofs and ledger statuses.
self._is_working = False
self._same_ledger_status = set()
self._cons_proofs = {}
self._cancel_reask()
self._output.put_nowait(
LedgerCatchupStart(ledger_id=self._ledger_id,
catchup_till=till,
nodes_ledger_sizes=nodes_ledger_sizes))
def _add_txns_to_ledger(node, looper, sdk_wallet_client, num_txns_in_reply, reply_count):
'''
Add txn_count transactions to node's ledger and return
ConsistencyProof for all new transactions and list of CatchupReplies
:return: ConsistencyProof, list of CatchupReplies
'''
txn_count = num_txns_in_reply * reply_count
ledger_manager = node.ledgerManager
ledger = ledger_manager.ledgerRegistry[ledger_id].ledger
catchup_rep_service = ledger_manager._node_leecher._leechers[ledger_id]._catchup_rep_service
reqs = sdk_signed_random_requests(looper, sdk_wallet_client, txn_count)
# add transactions to ledger
for req in reqs:
txn = append_txn_metadata(reqToTxn(req), txn_time=12345678)
catchup_rep_service._add_txn(txn)
# generate CatchupReps
replies = []
for i in range(ledger.seqNo - txn_count + 1, ledger.seqNo + 1, num_txns_in_reply):
start = i
end = i + num_txns_in_reply - 1
cons_proof = ledger_manager._node_seeder._make_consistency_proof(ledger, end, ledger.size)
txns = {}
for seq_no, txn in ledger.getAllTxn(start, end):
txns[str(seq_no)] = ledger_manager.owner.update_txn_with_extra_data(txn)
replies.append(CatchupRep(ledger_id,
SortedDict(txns),
cons_proof))
three_pc_key = node.three_phase_key_for_txn_seq_no(ledger_id, ledger.seqNo)
view_no, pp_seq_no = three_pc_key if three_pc_key else (0, 0)
return CatchupTill(start_size=ledger.seqNo - txn_count,
final_size=ledger.seqNo,
final_hash=Ledger.hashToStr(ledger.tree.merkle_tree_hash(0, ledger.seqNo)),
view_no=view_no,
pp_seq_no=pp_seq_no), replies
def test_missing_txn_request(ledger_no_genesis):
"""
Testing LedgerManager's `_missing_txns`
"""
ledger = ledger_no_genesis
for i in range(20):
txn = random_txn(i)
ledger.add(txn)
service = create_fake_catchup_rep_service(ledger)
assert service._num_missing_txns() == 0
# Ledger is already ahead
ct = CatchupTill(start_size=1,
final_size=10,
final_hash='Gv9AdSeib9EnBakfpgkU79dPMtjcnFWXvXeiCX4QAgAC',
view_no=0,
pp_seq_no=0)
service._catchup_till = ct
service._received_catchup_txns = [(i, {}) for i in range(1, 15)]
assert service._num_missing_txns() == 0
# Ledger is behind but catchup replies present
ct = CatchupTill(start_size=1,
final_size=30,
final_hash='EEUnqHf2GWEpvmibiXDCZbNDSpuRgqdvCpJjgp3KFbNC',
view_no=0,
pp_seq_no=0)
service._catchup_till = ct
service._received_catchup_txns = [(i, {}) for i in range(21, 31)]
assert service._num_missing_txns() == 0
service._received_catchup_txns = [(i, {}) for i in range(21, 35)]
assert service._num_missing_txns() == 0
# Ledger is behind
ct = CatchupTill(start_size=1,
final_size=30,
final_hash='EEUnqHf2GWEpvmibiXDCZbNDSpuRgqdvCpJjgp3KFbNC',
view_no=0,
pp_seq_no=0)
service._catchup_till = ct
service._received_catchup_txns = [(i, {}) for i in range(21, 26)]
assert service._num_missing_txns() == 5
service._received_catchup_txns = [(i, {}) for i in range(26, 31)]
assert service._num_missing_txns() == 5
def _calc_catchup_till(self) -> Dict[int, CatchupTill]:
audit_ledger = self._provider.ledger(AUDIT_LEDGER_ID)
last_audit_txn = audit_ledger.get_last_committed_txn()
if last_audit_txn is None:
return {}
catchup_till = {}
last_audit_txn = get_payload_data(last_audit_txn)
for ledger_id, final_size in last_audit_txn[
AUDIT_TXN_LEDGERS_SIZE].items():
ledger = self._provider.ledger(ledger_id)
if ledger is None:
logger.debug(
"{} has audit ledger with references to nonexistent "
"ledger with ID {}. Maybe it was frozen.".format(
self, ledger_id))
continue
start_size = ledger.size
final_hash = last_audit_txn[AUDIT_TXN_LEDGER_ROOT].get(ledger_id)
if final_hash is None:
if final_size != ledger.size:
logger.error(
"{} has corrupted audit ledger: "
"it indicates that ledger {} has new transactions but doesn't have new txn root"
.format(self, ledger_id))
return {}
final_hash = Ledger.hashToStr(
ledger.tree.root_hash) if final_size > 0 else None
if isinstance(final_hash, int):
audit_txn = audit_ledger.getBySeqNo(audit_ledger.size -
final_hash)
if audit_txn is None:
logger.error(
"{} has corrupted audit ledger: "
"its txn root for ledger {} references nonexistent txn with seq_no {} - {} = {}"
.format(self, ledger_id, audit_ledger.size, final_hash,
audit_ledger.size - final_hash))
return {}
audit_txn = get_payload_data(audit_txn)
final_hash = audit_txn[AUDIT_TXN_LEDGER_ROOT].get(ledger_id)
if not isinstance(final_hash, str):
logger.error(
"{} has corrupted audit ledger: "
"its txn root for ledger {} references txn with seq_no {} - {} = {} "
"which doesn't contain txn root".format(
self, ledger_id, audit_ledger.size, final_hash,
audit_ledger.size - final_hash))
return {}
catchup_till[ledger_id] = CatchupTill(start_size=start_size,
final_size=final_size,
final_hash=final_hash)
return catchup_till
def stop(self, cons_proof: Optional[ConsistencyProof] = None):
# Stop requesting last consistency proofs and ledger statuses.
self._is_working = False
self._same_ledger_status = set()
self._cons_proofs = {}
self._requested_consistency_proof = set()
self._cancel_reask()
till = CatchupTill(
start_size=cons_proof.seqNoStart,
final_size=cons_proof.seqNoEnd,
final_hash=cons_proof.newMerkleRoot,
view_no=cons_proof.viewNo,
pp_seq_no=cons_proof.ppSeqNo) if cons_proof else None
self._output.put_nowait(
LedgerCatchupStart(ledger_id=self._ledger_id, catchup_till=till))
