def g(instId): allReplicas = getAllReplicas(txnPoolNodeSet, instId) primaryReplica = getPrimaryReplica(txnPoolNodeSet, instId) def replicas_gets_correct_num_of_COMMITs(): """ num of commit messages must be = n when zero fault; n = num of nodes and greater than or equal to n-f with faults. """ passes = 0 numOfMsgsWithZFN = quorums.commit.value numOfMsgsWithFault = quorums.commit.value key = (primaryReplica.viewNo, primaryReplica.lastPrePrepareSeqNo) for r in allReplicas: if key in r.commits: rcvdCommitRqst = r.commits[key] actualMsgsReceived = len(rcvdCommitRqst.voters) passes += int( msgCountOK(nodeCount, faultyNodes, actualMsgsReceived, numOfMsgsWithZFN, numOfMsgsWithFault)) assert passes >= min( len(allReplicas) - faultyNodes, numOfMsgsWithZFN) replicas_gets_correct_num_of_COMMITs()
def test_checkpoints_removed_on_master_non_primary_replica_after_catchup( chkFreqPatched, txnPoolNodeSet, view_setup, clear_checkpoints): replica = getNonPrimaryReplicas(txnPoolNodeSet, 0)[-1] others = set(getAllReplicas(txnPoolNodeSet, 0)) - {replica} node = replica.node node.master_replica.last_ordered_3pc = (2, 12) replica._checkpointer._mark_checkpoint_stable(10) replica._checkpointer._received_checkpoints[cp_key(2, 15)] = [r.name for r in others] replica._checkpointer._received_checkpoints[cp_key(2, 20)] = [r.name for r in others] replica._checkpointer._received_checkpoints[cp_key(2, 25)] = [next(iter(others)).name] # Simulate catch-up completion node.ledgerManager.last_caught_up_3PC = (2, 20) audit_ledger = node.getLedger(AUDIT_LEDGER_ID) txn_with_last_seq_no = {'txn': {'data': {AUDIT_TXN_VIEW_NO: 2, AUDIT_TXN_PP_SEQ_NO: 20, AUDIT_TXN_PRIMARIES: ['Gamma', 'Delta']}}} audit_ledger.get_last_committed_txn = lambda *args: txn_with_last_seq_no node.allLedgersCaughtUp() check_num_received_checkpoints(replica, 1) check_last_received_checkpoint(replica, 25, view_no=2)
def test_checkpoints_removed_on_backup_replica_after_catchup_during_view_change( chkFreqPatched, txnPoolNodeSet, view_change_in_progress, clear_checkpoints): backup_replicas = getAllReplicas(txnPoolNodeSet, 1) replica = backup_replicas[-1] others = backup_replicas[:-1] node = replica.node node.master_replica.last_ordered_3pc = (2, 12) replica._checkpointer._mark_checkpoint_stable(10) replica._checkpointer._received_checkpoints[cp_key( 2, 15)] = [r.name for r in others] replica._checkpointer._received_checkpoints[cp_key( 2, 20)] = [r.name for r in others] replica._checkpointer._received_checkpoints[cp_key(2, 25)] = [others[0].name] # Simulate catch-up completion node.ledgerManager.last_caught_up_3PC = (2, 20) audit_ledger = node.getLedger(AUDIT_LEDGER_ID) txn_with_last_seq_no = { 'txn': { 'data': { AUDIT_TXN_VIEW_NO: 2, AUDIT_TXN_PP_SEQ_NO: 20, AUDIT_TXN_PRIMARIES: ['Gamma', 'Delta'] } } } audit_ledger.get_last_committed_txn = lambda *args: txn_with_last_seq_no node.allLedgersCaughtUp() check_num_received_checkpoints(replica, 0)
def test_received_checkpoints_removed_on_backup_primary_replica_after_catchup( chkFreqPatched, txnPoolNodeSet, view_setup, clear_checkpoints): replica = getPrimaryReplica(txnPoolNodeSet, 1) others = set(getAllReplicas(txnPoolNodeSet, 1)) - {replica} node = replica.node node.master_replica.last_ordered_3pc = (2, 12) replica._consensus_data.stable_checkpoint = 15 replica._checkpointer._received_checkpoints[cp_key( 2, 20)] = [next(iter(others)).name] # Simulate catch-up completion node.ledgerManager.last_caught_up_3PC = (2, 20) audit_ledger = node.getLedger(AUDIT_LEDGER_ID) txn_with_last_seq_no = { 'txn': { 'data': { AUDIT_TXN_VIEW_NO: 2, AUDIT_TXN_PP_SEQ_NO: 20, AUDIT_TXN_PRIMARIES: ['Gamma', 'Delta'] } } } audit_ledger.get_last_committed_txn = lambda *args: txn_with_last_seq_no node.write_manager.node_reg_handler.on_catchup_finished = lambda *args: None node.allLedgersCaughtUp() check_num_received_checkpoints(replica, 1) check_last_received_checkpoint(replica, 20, view_no=2)
def test_checkpoints_removed_on_backup_replica_after_catchup_during_view_change( chkFreqPatched, txnPoolNodeSet, view_change_in_progress, clear_checkpoints): backup_replicas = getAllReplicas(txnPoolNodeSet, 1) replica = backup_replicas[-1] others = backup_replicas[:-1] node = replica.node node.master_replica.last_ordered_3pc = (2, 12) replica.checkpoints[(6, 10)] = CheckpointState( seqNo=10, digests=[], digest='digest-6-10', receivedDigests={r.name: 'digest-6-10' for r in others}, isStable=True) replica.checkpoints[(11, 15)] = CheckpointState( seqNo=13, digests=['digest-11', 'digest-12', 'digest-13'], digest=None, receivedDigests={}, isStable=False) replica.stashedRecvdCheckpoints[2] = {} replica.stashedRecvdCheckpoints[2][(11, 15)] = {} for r in others: replica.stashedRecvdCheckpoints[2][(11, 15)][r.name] = \ Checkpoint(instId=1, viewNo=2, seqNoStart=11, seqNoEnd=15, digest='digest-11-15') replica.stashedRecvdCheckpoints[2][(16, 20)] = {} for r in others: replica.stashedRecvdCheckpoints[2][(16, 20)][r.name] = \ Checkpoint(instId=1, viewNo=2, seqNoStart=16, seqNoEnd=20, digest='digest-16-20') replica.stashedRecvdCheckpoints[2][(21, 25)] = {} replica.stashedRecvdCheckpoints[2][(21, 25)][next(iter(others)).name] = \ Checkpoint(instId=1, viewNo=2, seqNoStart=21, seqNoEnd=25, digest='digest-21-25') # Simulate catch-up completion node.ledgerManager.last_caught_up_3PC = (2, 20) node.allLedgersCaughtUp() assert len(replica.checkpoints) == 0 assert len(replica.stashedRecvdCheckpoints) == 0
def test_checkpoints_removed_on_master_non_primary_replica_after_catchup( chkFreqPatched, txnPoolNodeSet, view_setup, clear_checkpoints): replica = getNonPrimaryReplicas(txnPoolNodeSet, 0)[-1] others = set(getAllReplicas(txnPoolNodeSet, 0)) - {replica} node = replica.node node.master_replica.last_ordered_3pc = (2, 12) replica.checkpoints[(6, 10)] = CheckpointState(seqNo=10, digests=[], digest='digest-6-10', receivedDigests={r.name: 'digest-6-10' for r in others}, isStable=True) replica.checkpoints[(11, 15)] = CheckpointState(seqNo=12, digests=['digest-11', 'digest-12'], digest=None, receivedDigests={}, isStable=False) replica.stashedRecvdCheckpoints[2] = {} replica.stashedRecvdCheckpoints[2][(11, 15)] = {} for r in others: replica.stashedRecvdCheckpoints[2][(11, 15)][r.name] = \ Checkpoint(instId=0, viewNo=2, seqNoStart=11, seqNoEnd=15, digest='digest-11-15') replica.stashedRecvdCheckpoints[2][(16, 20)] = {} for r in others: replica.stashedRecvdCheckpoints[2][(16, 20)][r.name] = \ Checkpoint(instId=0, viewNo=2, seqNoStart=16, seqNoEnd=20, digest='digest-16-20') replica.stashedRecvdCheckpoints[2][(21, 25)] = {} replica.stashedRecvdCheckpoints[2][(21, 25)][next(iter(others)).name] = \ Checkpoint(instId=0, viewNo=2, seqNoStart=21, seqNoEnd=25, digest='digest-21-25') # Simulate catch-up completion node.ledgerManager.last_caught_up_3PC = (2, 20) node.allLedgersCaughtUp() assert len(replica.checkpoints) == 0 assert len(replica.stashedRecvdCheckpoints) == 1 assert 2 in replica.stashedRecvdCheckpoints assert len(replica.stashedRecvdCheckpoints[2]) == 1 assert (21, 25) in replica.stashedRecvdCheckpoints[2] assert len(replica.stashedRecvdCheckpoints[2][(21, 25)]) == 1
def g(instId): allReplicas = getAllReplicas(txnPoolNodeSet, instId) primaryReplica = getPrimaryReplica(txnPoolNodeSet, instId) def replicas_gets_correct_num_of_COMMITs(): """ num of commit messages must be = n when zero fault; n = num of nodes and greater than or equal to n-f with faults. """ passes = 0 numOfMsgsWithZFN = quorums.commit.value numOfMsgsWithFault = quorums.commit.value key = (primaryReplica.viewNo, primaryReplica.lastPrePrepareSeqNo) for r in allReplicas: if key in r.commits: rcvdCommitRqst = r.commits[key] actualMsgsReceived = len(rcvdCommitRqst.voters) passes += int(msgCountOK(nodeCount, faultyNodes, actualMsgsReceived, numOfMsgsWithZFN, numOfMsgsWithFault)) assert passes >= min(len(allReplicas) - faultyNodes, numOfMsgsWithZFN) replicas_gets_correct_num_of_COMMITs()
def test_checkpoints_removed_on_backup_primary_replica_after_catchup( chkFreqPatched, txnPoolNodeSet, view_setup, clear_checkpoints): replica = getPrimaryReplica(txnPoolNodeSet, 1) others = set(getAllReplicas(txnPoolNodeSet, 1)) - {replica} node = replica.node node.master_replica.last_ordered_3pc = (2, 12) replica._checkpointer._checkpoint_state[(11, 15)] = CheckpointState( seqNo=15, digests=[], digest=cp_digest(11, 15), receivedDigests={r.name: cp_digest(11, 15) for r in others}, isStable=True) replica._checkpointer._checkpoint_state[(16, 20)] = CheckpointState( seqNo=19, digests=['digest-16', 'digest-17', 'digest-18', 'digest-19'], digest=None, receivedDigests={}, isStable=False) replica._checkpointer._stashed_recvd_checkpoints[2] = {} replica._checkpointer._stashed_recvd_checkpoints[2][(16, 20)] = {} replica._checkpointer._stashed_recvd_checkpoints[2][(16, 20)][next(iter(others)).name] = \ Checkpoint(instId=1, viewNo=2, seqNoStart=16, seqNoEnd=20, digest=cp_digest(16, 20)) # Simulate catch-up completion node.ledgerManager.last_caught_up_3PC = (2, 20) audit_ledger = node.getLedger(AUDIT_LEDGER_ID) txn_with_last_seq_no = { 'txn': { 'data': { AUDIT_TXN_VIEW_NO: 2, AUDIT_TXN_PP_SEQ_NO: 20, AUDIT_TXN_PRIMARIES: ['Gamma', 'Delta'] } } } audit_ledger.get_last_committed_txn = lambda *args: txn_with_last_seq_no node.allLedgersCaughtUp() assert len(replica._checkpointer._checkpoint_state) == 0 assert (11, 15) not in replica._checkpointer._checkpoint_state assert (16, 20) not in replica._checkpointer._checkpoint_state assert len(replica._checkpointer._stashed_recvd_checkpoints) == 1 assert 2 in replica._checkpointer._stashed_recvd_checkpoints assert len(replica._checkpointer._stashed_recvd_checkpoints[2]) == 1 assert (16, 20) in replica._checkpointer._stashed_recvd_checkpoints[2] assert len(replica._checkpointer._stashed_recvd_checkpoints[2][(16, 20)]) == 1
def g(instId): allReplicas = getAllReplicas(nodeSet, instId) primaryReplica = getPrimaryReplica(nodeSet, instId) def replicasSeesCorrectNumOfCOMMITs(): """ num of commit messages must be = n when zero fault; n = num of nodes and greater than or equal to 2f + 1 with faults. """ passes = 0 numOfMsgsWithZFN = nodeCount numOfMsgsWithFault = (2 * f) + 1 key = (primaryReplica.viewNo, primaryReplica.lastPrePrepareSeqNo) for r in allReplicas: if key in r.commits: rcvdCommitRqst = r.commits[key] actualMsgsReceived = len(rcvdCommitRqst.voters) passes += int(msgCountOK(nodeCount, faultyNodes, actualMsgsReceived, numOfMsgsWithZFN, numOfMsgsWithFault)) assert passes >= len(allReplicas) - faultyNodes def replicasReceivesCorrectNumberOfCOMMITs(): """ num of commit messages seen by replica must be equal to n - 1; when zero fault and greater than or equal to 2f+1 with faults. """ passes = 0 numOfMsgsWithZFN = nodeCount - 1 numOfMsgsWithFault = 2 * f for r in allReplicas: args = getAllArgs(r, r.processCommit) actualMsgsReceived = len(args) passes += int(msgCountOK(nodeCount, faultyNodes, actualMsgsReceived, numOfMsgsWithZFN, numOfMsgsWithFault)) for arg in args: assert arg['commit'].viewNo == primaryReplica.viewNo and \ arg['commit'].ppSeqNo == primaryReplica.lastPrePrepareSeqNo and \ arg['commit'].digest == prepared1.digest assert r.name != arg['sender'] assert passes >= len(allReplicas) - faultyNodes replicasReceivesCorrectNumberOfCOMMITs() replicasSeesCorrectNumOfCOMMITs()
def test_checkpoints_removed_on_backup_replica_after_catchup_during_view_change( chkFreqPatched, txnPoolNodeSet, view_change_in_progress, clear_checkpoints): backup_replicas = getAllReplicas(txnPoolNodeSet, 1) replica = backup_replicas[-1] others = backup_replicas[:-1] node = replica.node node.master_replica.last_ordered_3pc = (2, 12) replica.checkpoints[(6, 10)] = CheckpointState(seqNo=10, digests=[], digest='digest-6-10', receivedDigests={r.name: 'digest-6-10' for r in others}, isStable=True) replica.checkpoints[(11, 15)] = CheckpointState(seqNo=13, digests=['digest-11', 'digest-12', 'digest-13'], digest=None, receivedDigests={}, isStable=False) replica.stashedRecvdCheckpoints[2] = {} replica.stashedRecvdCheckpoints[2][(11, 15)] = {} for r in others: replica.stashedRecvdCheckpoints[2][(11, 15)][r.name] = \ Checkpoint(instId=1, viewNo=2, seqNoStart=11, seqNoEnd=15, digest='digest-11-15') replica.stashedRecvdCheckpoints[2][(16, 20)] = {} for r in others: replica.stashedRecvdCheckpoints[2][(16, 20)][r.name] = \ Checkpoint(instId=1, viewNo=2, seqNoStart=16, seqNoEnd=20, digest='digest-16-20') replica.stashedRecvdCheckpoints[2][(21, 25)] = {} replica.stashedRecvdCheckpoints[2][(21, 25)][next(iter(others)).name] = \ Checkpoint(instId=1, viewNo=2, seqNoStart=21, seqNoEnd=25, digest='digest-21-25') # Simulate catch-up completion node.ledgerManager.last_caught_up_3PC = (2, 20) node.allLedgersCaughtUp() assert len(replica.checkpoints) == 0 assert len(replica.stashedRecvdCheckpoints) == 0
def g(instId): allReplicas = getAllReplicas(nodeSet, instId) primaryReplica = getPrimaryReplica(nodeSet, instId) def replicasSeesCorrectNumOfCOMMITs(): """ num of commit messages must be = n when zero fault; n = num of nodes and greater than or equal to 2f + 1 with faults. """ passes = 0 numOfMsgsWithZFN = nodeCount numOfMsgsWithFault = (2 * f) + 1 key = (primaryReplica.viewNo, primaryReplica.lastPrePrepareSeqNo) for r in allReplicas: if key in r.commits: rcvdCommitRqst = r.commits[key] actualMsgsReceived = len(rcvdCommitRqst.voters) passes += int( msgCountOK(nodeCount, faultyNodes, actualMsgsReceived, numOfMsgsWithZFN, numOfMsgsWithFault)) assert passes >= len(allReplicas) - faultyNodes def replicasReceivesCorrectNumberOfCOMMITs(): """ num of commit messages seen by replica must be equal to n - 1; when zero fault and greater than or equal to 2f+1 with faults. """ passes = 0 numOfMsgsWithZFN = nodeCount - 1 numOfMsgsWithFault = 2 * f for r in allReplicas: args = getAllArgs(r, r.processCommit) actualMsgsReceived = len(args) passes += int( msgCountOK(nodeCount, faultyNodes, actualMsgsReceived, numOfMsgsWithZFN, numOfMsgsWithFault)) for arg in args: assert arg['commit'].viewNo == primaryReplica.viewNo and \ arg['commit'].ppSeqNo == primaryReplica.lastPrePrepareSeqNo and \ arg['commit'].digest == prepared1.digest assert r.name != arg['sender'] assert passes >= len(allReplicas) - faultyNodes replicasReceivesCorrectNumberOfCOMMITs() replicasSeesCorrectNumOfCOMMITs()
def test_checkpoints_not_removed_on_backup_primary_replica_after_catchup( chkFreqPatched, txnPoolNodeSet, view_setup, clear_checkpoints): replica = getPrimaryReplica(txnPoolNodeSet, 1) others = set(getAllReplicas(txnPoolNodeSet, 1)) - {replica} node = replica.node node.master_replica.last_ordered_3pc = (2, 12) replica.checkpoints[(11, 15)] = CheckpointState( seqNo=15, digests=[], digest='digest-11-15', receivedDigests={r.name: 'digest-11-15' for r in others}, isStable=True) replica.checkpoints[(16, 20)] = CheckpointState( seqNo=19, digests=['digest-16', 'digest-17', 'digest-18', 'digest-19'], digest=None, receivedDigests={}, isStable=False) replica.stashedRecvdCheckpoints[2] = {} replica.stashedRecvdCheckpoints[2][(16, 20)] = {} replica.stashedRecvdCheckpoints[2][(16, 20)][next(iter(others)).name] = \ Checkpoint(instId=1, viewNo=2, seqNoStart=16, seqNoEnd=20, digest='digest-16-20') # Simulate catch-up completion node.ledgerManager.last_caught_up_3PC = (2, 20) node.allLedgersCaughtUp() assert len(replica.checkpoints) == 2 assert (11, 15) in replica.checkpoints assert (16, 20) in replica.checkpoints assert len(replica.stashedRecvdCheckpoints) == 1 assert 2 in replica.stashedRecvdCheckpoints assert len(replica.stashedRecvdCheckpoints[2]) == 1 assert (16, 20) in replica.stashedRecvdCheckpoints[2] assert len(replica.stashedRecvdCheckpoints[2][(16, 20)]) == 1
def g(instId): allReplicas = getAllReplicas(txnPoolNodeSet, instId) primary = getPrimaryReplica(txnPoolNodeSet, instId) nonPrimaryReplicas = getNonPrimaryReplicas(txnPoolNodeSet, instId) def primaryDontSendAnyPREPAREs(): """ 1. no of PREPARE sent by primary should be 0 """ for r in allReplicas: for param in getAllArgs(r, Replica.processPrepare): sender = param['sender'] assert sender != primary.name def allReplicasSeeCorrectNumberOfPREPAREs(): """ 1. no of PREPARE received by replicas must be n - 1; n = num of nodes without fault, and greater than or equal to n-f-1 with faults. """ passes = 0 numOfMsgsWithZFN = nodeCount - 1 numOfMsgsWithFaults = quorums.prepare.value for replica in allReplicas: key = primary.viewNo, primary.lastPrePrepareSeqNo if key in replica.prepares: actualMsgs = len(replica.prepares[key].voters) passes += int( msgCountOK(nodeCount, faultyNodes, actualMsgs, numOfMsgsWithZFN, numOfMsgsWithFaults)) assert passes >= len(allReplicas) - faultyNodes def primaryReceivesCorrectNumberOfPREPAREs(): """ num of PREPARE seen by primary replica is n - 1; n = num of nodes without fault, and greater than or equal to n-f-1 with faults. """ actualMsgs = len([ param for param in getAllArgs(primary, primary.processPrepare) if (param['prepare'].instId, param['prepare'].viewNo, param['prepare'].ppSeqNo) == (primary.instId, primary.viewNo, primary.lastPrePrepareSeqNo) and param['sender'] != primary.name ]) numOfMsgsWithZFN = nodeCount - 1 numOfMsgsWithFaults = quorums.prepare.value assert msgCountOK(nodeCount, faultyNodes, actualMsgs, numOfMsgsWithZFN, numOfMsgsWithFaults) # TODO what if the primary is faulty? def nonPrimaryReplicasReceiveCorrectNumberOfPREPAREs(): """ num of PREPARE seen by Non primary replica is n - 2 without faults and n-f-2 with faults. """ passes = 0 numOfMsgsWithZFN = nodeCount - 2 numOfMsgsWithFaults = quorums.prepare.value - 1 for npr in nonPrimaryReplicas: actualMsgs = len([ param for param in getAllArgs(npr, npr.processPrepare) if (param['prepare'].instId, param['prepare'].viewNo, param['prepare'].ppSeqNo) == ( primary.instId, primary.viewNo, primary.lastPrePrepareSeqNo) ]) passes += int( msgCountOK(nodeCount, faultyNodes, actualMsgs, numOfMsgsWithZFN, numOfMsgsWithFaults)) assert passes >= len(nonPrimaryReplicas) - faultyNodes # TODO how do we know if one of the faulty nodes is a primary or # not? primaryDontSendAnyPREPAREs() allReplicasSeeCorrectNumberOfPREPAREs() primaryReceivesCorrectNumberOfPREPAREs() nonPrimaryReplicasReceiveCorrectNumberOfPREPAREs()
def test_checkpoints_removed_on_master_non_primary_replica_after_catchup( chkFreqPatched, txnPoolNodeSet, view_setup, clear_checkpoints): replica = getNonPrimaryReplicas(txnPoolNodeSet, 0)[-1] others = set(getAllReplicas(txnPoolNodeSet, 0)) - {replica} node = replica.node node.master_replica.last_ordered_3pc = (2, 12) replica.checkpoints[(6, 10)] = CheckpointState( seqNo=10, digests=[], digest='digest-6-10', receivedDigests={r.name: 'digest-6-10' for r in others}, isStable=True) replica.checkpoints[(11, 15)] = CheckpointState( seqNo=12, digests=['digest-11', 'digest-12'], digest=None, receivedDigests={}, isStable=False) replica.stashedRecvdCheckpoints[2] = {} replica.stashedRecvdCheckpoints[2][(11, 15)] = {} for r in others: replica.stashedRecvdCheckpoints[2][(11, 15)][r.name] = \ Checkpoint(instId=0, viewNo=2, seqNoStart=11, seqNoEnd=15, digest='digest-11-15') replica.stashedRecvdCheckpoints[2][(16, 20)] = {} for r in others: replica.stashedRecvdCheckpoints[2][(16, 20)][r.name] = \ Checkpoint(instId=0, viewNo=2, seqNoStart=16, seqNoEnd=20, digest='digest-16-20') replica.stashedRecvdCheckpoints[2][(21, 25)] = {} replica.stashedRecvdCheckpoints[2][(21, 25)][next(iter(others)).name] = \ Checkpoint(instId=0, viewNo=2, seqNoStart=21, seqNoEnd=25, digest='digest-21-25') # Simulate catch-up completion node.ledgerManager.last_caught_up_3PC = (2, 20) audit_ledger = node.getLedger(AUDIT_LEDGER_ID) txn_with_last_seq_no = { 'txn': { 'data': { AUDIT_TXN_VIEW_NO: 2, AUDIT_TXN_PP_SEQ_NO: 20, AUDIT_TXN_PRIMARIES: ['Gamma', 'Delta'] } } } audit_ledger.get_last_committed_txn = lambda *args: txn_with_last_seq_no node.allLedgersCaughtUp() assert len(replica.checkpoints) == 0 assert len(replica.stashedRecvdCheckpoints) == 1 assert 2 in replica.stashedRecvdCheckpoints assert len(replica.stashedRecvdCheckpoints[2]) == 1 assert (21, 25) in replica.stashedRecvdCheckpoints[2] assert len(replica.stashedRecvdCheckpoints[2][(21, 25)]) == 1
def g(instId): allReplicas = getAllReplicas(txnPoolNodeSet, instId) primary = getPrimaryReplica(txnPoolNodeSet, instId) nonPrimaryReplicas = getNonPrimaryReplicas(txnPoolNodeSet, instId) def primaryDontSendAnyPREPAREs(): """ 1. no of PREPARE sent by primary should be 0 """ for r in allReplicas: for param in getAllArgs(r, Replica.processPrepare): sender = param['sender'] assert sender != primary.name def allReplicasSeeCorrectNumberOfPREPAREs(): """ 1. no of PREPARE received by replicas must be n - 1; n = num of nodes without fault, and greater than or equal to n-f-1 with faults. """ passes = 0 numOfMsgsWithZFN = nodeCount - 1 numOfMsgsWithFaults = quorums.prepare.value for replica in allReplicas: key = primary.viewNo, primary.lastPrePrepareSeqNo if key in replica.prepares: actualMsgs = len(replica.prepares[key].voters) passes += int(msgCountOK(nodeCount, faultyNodes, actualMsgs, numOfMsgsWithZFN, numOfMsgsWithFaults)) assert passes >= len(allReplicas) - faultyNodes def primaryReceivesCorrectNumberOfPREPAREs(): """ num of PREPARE seen by primary replica is n - 1; n = num of nodes without fault, and greater than or equal to n-f-1 with faults. """ actualMsgs = len([param for param in getAllArgs(primary, primary.processPrepare) if (param['prepare'].instId, param['prepare'].viewNo, param['prepare'].ppSeqNo) == ( primary.instId, primary.viewNo, primary.lastPrePrepareSeqNo) and param['sender'] != primary.name]) numOfMsgsWithZFN = nodeCount - 1 numOfMsgsWithFaults = quorums.prepare.value assert msgCountOK(nodeCount, faultyNodes, actualMsgs, numOfMsgsWithZFN, numOfMsgsWithFaults) # TODO what if the primary is faulty? def nonPrimaryReplicasReceiveCorrectNumberOfPREPAREs(): """ num of PREPARE seen by Non primary replica is n - 2 without faults and n-f-2 with faults. """ passes = 0 numOfMsgsWithZFN = nodeCount - 2 numOfMsgsWithFaults = quorums.prepare.value - 1 for npr in nonPrimaryReplicas: actualMsgs = len( [ param for param in getAllArgs( npr, npr.processPrepare) if ( param['prepare'].instId, param['prepare'].viewNo, param['prepare'].ppSeqNo) == ( primary.instId, primary.viewNo, primary.lastPrePrepareSeqNo)]) passes += int(msgCountOK(nodeCount, faultyNodes, actualMsgs, numOfMsgsWithZFN, numOfMsgsWithFaults)) assert passes >= len(nonPrimaryReplicas) - faultyNodes # TODO how do we know if one of the faulty nodes is a primary or # not? primaryDontSendAnyPREPAREs() allReplicasSeeCorrectNumberOfPREPAREs() primaryReceivesCorrectNumberOfPREPAREs() nonPrimaryReplicasReceiveCorrectNumberOfPREPAREs()
def test_checkpoints_removed_on_master_replica_after_catchup_during_view_change( chkFreqPatched, txnPoolNodeSet, view_change_in_progress, clear_checkpoints): master_replicas = getAllReplicas(txnPoolNodeSet, 0) replica = master_replicas[-1] others = master_replicas[:-1] node = replica.node node.master_replica.last_ordered_3pc = (2, 12) replica._checkpointer._checkpoint_state[(6, 10)] = CheckpointState( seqNo=10, digests=[], digest=cp_digest(6, 10), receivedDigests={r.name: cp_digest(6, 10) for r in others}, isStable=True) replica._checkpointer._checkpoint_state[(11, 15)] = CheckpointState( seqNo=12, digests=['digest-11', 'digest-12'], digest=None, receivedDigests={}, isStable=False) replica._checkpointer._stashed_recvd_checkpoints[2] = {} replica._checkpointer._stashed_recvd_checkpoints[2][(11, 15)] = {} for r in others: replica._checkpointer._stashed_recvd_checkpoints[2][(11, 15)][r.name] = \ Checkpoint(instId=0, viewNo=2, seqNoStart=11, seqNoEnd=15, digest=cp_digest(11, 15)) replica._checkpointer._stashed_recvd_checkpoints[2][(16, 20)] = {} for r in others: replica._checkpointer._stashed_recvd_checkpoints[2][(16, 20)][r.name] = \ Checkpoint(instId=0, viewNo=2, seqNoStart=16, seqNoEnd=20, digest=cp_digest(16, 20)) replica._checkpointer._stashed_recvd_checkpoints[2][(21, 25)] = {} replica._checkpointer._stashed_recvd_checkpoints[2][(21, 25)][others[0].name] = \ Checkpoint(instId=0, viewNo=2, seqNoStart=21, seqNoEnd=25, digest=cp_digest(21, 25)) # Simulate catch-up completion node.ledgerManager.last_caught_up_3PC = (2, 20) audit_ledger = node.getLedger(AUDIT_LEDGER_ID) txn_with_last_seq_no = { 'txn': { 'data': { AUDIT_TXN_VIEW_NO: 2, AUDIT_TXN_PP_SEQ_NO: 20, AUDIT_TXN_PRIMARIES: ['Gamma', 'Delta'] } } } audit_ledger.get_last_committed_txn = lambda *args: txn_with_last_seq_no node.allLedgersCaughtUp() assert len(replica._checkpointer._checkpoint_state) == 0 assert len(replica._checkpointer._stashed_recvd_checkpoints) == 1 assert 2 in replica._checkpointer._stashed_recvd_checkpoints assert len(replica._checkpointer._stashed_recvd_checkpoints[2]) == 1 assert (21, 25) in replica._checkpointer._stashed_recvd_checkpoints[2] assert len(replica._checkpointer._stashed_recvd_checkpoints[2][(21, 25)]) == 1