def test_checkpoints_removed_on_backup_replica_after_catchup_during_view_change(
chkFreqPatched, txnPoolNodeSet, view_change_in_progress,
clear_checkpoints):
backup_replicas = getAllReplicas(txnPoolNodeSet, 1)
replica = backup_replicas[-1]
others = backup_replicas[:-1]
node = replica.node
node.master_replica.last_ordered_3pc = (2, 12)
replica._checkpointer._mark_checkpoint_stable(10)
replica._checkpointer._received_checkpoints[cp_key(
2, 15)] = [r.name for r in others]
replica._checkpointer._received_checkpoints[cp_key(
2, 20)] = [r.name for r in others]
replica._checkpointer._received_checkpoints[cp_key(2,
25)] = [others[0].name]
# Simulate catch-up completion
node.ledgerManager.last_caught_up_3PC = (2, 20)
audit_ledger = node.getLedger(AUDIT_LEDGER_ID)
txn_with_last_seq_no = {
'txn': {
'data': {
AUDIT_TXN_VIEW_NO: 2,
AUDIT_TXN_PP_SEQ_NO: 20,
AUDIT_TXN_PRIMARIES: ['Gamma', 'Delta']
}
}
}
audit_ledger.get_last_committed_txn = lambda *args: txn_with_last_seq_no
node.allLedgersCaughtUp()
check_num_received_checkpoints(replica, 0)
def test_checkpoints_removed_on_master_non_primary_replica_after_catchup(
chkFreqPatched, txnPoolNodeSet, view_setup, clear_checkpoints):
replica = getNonPrimaryReplicas(txnPoolNodeSet, 0)[-1]
others = set(getAllReplicas(txnPoolNodeSet, 0)) - {replica}
node = replica.node
node.master_replica.last_ordered_3pc = (2, 12)
replica._checkpointer._mark_checkpoint_stable(10)
replica._checkpointer._received_checkpoints[cp_key(2, 15)] = [r.name for r in others]
replica._checkpointer._received_checkpoints[cp_key(2, 20)] = [r.name for r in others]
replica._checkpointer._received_checkpoints[cp_key(2, 25)] = [next(iter(others)).name]
# Simulate catch-up completion
node.ledgerManager.last_caught_up_3PC = (2, 20)
audit_ledger = node.getLedger(AUDIT_LEDGER_ID)
txn_with_last_seq_no = {'txn': {'data': {AUDIT_TXN_VIEW_NO: 2,
AUDIT_TXN_PP_SEQ_NO: 20,
AUDIT_TXN_PRIMARIES: ['Gamma', 'Delta']}}}
audit_ledger.get_last_committed_txn = lambda *args: txn_with_last_seq_no
node.allLedgersCaughtUp()
check_num_received_checkpoints(replica, 1)
check_last_received_checkpoint(replica, 25, view_no=2)
def test_process_checkpoint(checkpoint_service, checkpoint, pre_prepare, tconf,
ordered, validators, is_master):
checkpoint_stabilized_handler = Mock()
checkpoint_service._bus.subscribe(CheckpointStabilized,
checkpoint_stabilized_handler)
quorum = checkpoint_service._data.quorums.checkpoint.value
n = len(validators)
assert quorum == n - getMaxFailures(n) - 1
senders = ["sender{}".format(i) for i in range(quorum + 1)]
till_seq_no = tconf.CHK_FREQ
checkpoint_service._received_checkpoints[cp_key(checkpoint.viewNo,
1)] = {"frm"}
# For now, on checkpoint stabilization phase all checkpoints
# with ppSeqNo less then stable_checkpoint will be removed
checkpoint_service._received_checkpoints[cp_key(
checkpoint.viewNo + 1, till_seq_no + 100)] = {"frm"}
pre_prepare.ppSeqNo = till_seq_no
pre_prepare.auditTxnRootHash = cp_digest(till_seq_no)
ordered.ppSeqNo = pre_prepare.ppSeqNo
ordered.auditTxnRootHash = pre_prepare.auditTxnRootHash
checkpoint_service._data.preprepared.append(
preprepare_to_batch_id(pre_prepare))
checkpoint_service.process_ordered(ordered)
_check_checkpoint(checkpoint_service,
till_seq_no,
pre_prepare,
check_shared_data=True)
for sender in senders[:quorum - 1]:
checkpoint_service.process_checkpoint(checkpoint, sender)
assert checkpoint_service._data.stable_checkpoint < till_seq_no
# send the last checkpoint to stable it
checkpoint_service.process_checkpoint(checkpoint, senders[quorum - 1])
assert checkpoint_service._data.stable_checkpoint == till_seq_no
# check _remove_stashed_checkpoints()
assert sum(1 for cp in checkpoint_service._received_checkpoints
if cp.view_no == checkpoint.viewNo) == 0
assert sum(1 for cp in checkpoint_service._received_checkpoints
if cp.view_no == checkpoint.viewNo + 1) > 0
# check watermarks
assert checkpoint_service._data.low_watermark == checkpoint.seqNoEnd
# check that a Cleanup msg has been sent
checkpoint_stabilized_handler.assert_called_once_with(
CheckpointStabilized(inst_id=checkpoint_service._data.inst_id,
last_stable_3pc=(checkpoint.viewNo,
checkpoint.seqNoEnd)))
def test_remove_stashed_checkpoints_doesnt_crash_when_current_view_no_is_greater_than_last_stashed_checkpoint(
checkpoint_service):
till_3pc_key = (1, 1)
checkpoint_service._received_checkpoints[cp_key(1, 1)] = {"some_node", "other_node"}
checkpoint_service._data.view_no = 2
checkpoint_service._remove_received_checkpoints(till_3pc_key)
assert not checkpoint_service._received_checkpoints
def test_received_checkpoints_removed_on_backup_primary_replica_after_catchup(
chkFreqPatched, txnPoolNodeSet, view_setup, clear_checkpoints):
replica = getPrimaryReplica(txnPoolNodeSet, 1)
others = set(getAllReplicas(txnPoolNodeSet, 1)) - {replica}
node = replica.node
node.master_replica.last_ordered_3pc = (2, 12)
replica._consensus_data.stable_checkpoint = 15
replica._checkpointer._received_checkpoints[cp_key(
2, 20)] = [next(iter(others)).name]
# Simulate catch-up completion
node.ledgerManager.last_caught_up_3PC = (2, 20)
audit_ledger = node.getLedger(AUDIT_LEDGER_ID)
txn_with_last_seq_no = {
'txn': {
'data': {
AUDIT_TXN_VIEW_NO: 2,
AUDIT_TXN_PP_SEQ_NO: 20,
AUDIT_TXN_PRIMARIES: ['Gamma', 'Delta']
}
}
}
audit_ledger.get_last_committed_txn = lambda *args: txn_with_last_seq_no
node.write_manager.node_reg_handler.on_catchup_finished = lambda *args: None
node.allLedgersCaughtUp()
check_num_received_checkpoints(replica, 1)
check_last_received_checkpoint(replica, 20, view_no=2)
def test_process_backup_catchup_msg(checkpoint_service, tconf, checkpoint):
till_seq_no = tconf.CHK_FREQ
key = cp_key(checkpoint.viewNo, till_seq_no)
new_till_seq_no = till_seq_no = tconf.CHK_FREQ
new_key = cp_key(checkpoint.viewNo, new_till_seq_no)
checkpoint_service._data.last_ordered_3pc = (checkpoint_service.view_no, 0)
checkpoint_service._data.stable_checkpoint = 1
checkpoint_service._received_checkpoints[key] = {"frm"}
checkpoint_service._received_checkpoints[new_key] = {"frm"}
checkpoint_service._data.checkpoints.append(checkpoint)
checkpoint_service._data.last_ordered_3pc = (checkpoint_service.view_no, till_seq_no)
checkpoint_service.caught_up_till_3pc(checkpoint_service._data.last_ordered_3pc)
assert checkpoint_service._data.low_watermark == till_seq_no
assert checkpoint in checkpoint_service._data.checkpoints
assert checkpoint_service._data.stable_checkpoint == till_seq_no
assert key not in checkpoint_service._received_checkpoints
assert new_key not in checkpoint_service._received_checkpoints
