def g(instId):
allReplicas = getAllReplicas(txnPoolNodeSet, instId)
primaryReplica = getPrimaryReplica(txnPoolNodeSet, instId)
def replicas_gets_correct_num_of_COMMITs():
"""
num of commit messages must be = n when zero fault;
n = num of nodes and greater than or equal to
n-f with faults.
"""
passes = 0
numOfMsgsWithZFN = quorums.commit.value
numOfMsgsWithFault = quorums.commit.value
key = (primaryReplica.viewNo, primaryReplica.lastPrePrepareSeqNo)
for r in allReplicas:
if key in r.commits:
rcvdCommitRqst = r.commits[key]
actualMsgsReceived = len(rcvdCommitRqst.voters)
passes += int(
msgCountOK(nodeCount, faultyNodes, actualMsgsReceived,
numOfMsgsWithZFN, numOfMsgsWithFault))
assert passes >= min(
len(allReplicas) - faultyNodes, numOfMsgsWithZFN)
replicas_gets_correct_num_of_COMMITs()
def test_checkpoints_removed_on_master_non_primary_replica_after_catchup(
chkFreqPatched, txnPoolNodeSet, view_setup, clear_checkpoints):
replica = getNonPrimaryReplicas(txnPoolNodeSet, 0)[-1]
others = set(getAllReplicas(txnPoolNodeSet, 0)) - {replica}
node = replica.node
node.master_replica.last_ordered_3pc = (2, 12)
replica._checkpointer._mark_checkpoint_stable(10)
replica._checkpointer._received_checkpoints[cp_key(2, 15)] = [r.name for r in others]
replica._checkpointer._received_checkpoints[cp_key(2, 20)] = [r.name for r in others]
replica._checkpointer._received_checkpoints[cp_key(2, 25)] = [next(iter(others)).name]
# Simulate catch-up completion
node.ledgerManager.last_caught_up_3PC = (2, 20)
audit_ledger = node.getLedger(AUDIT_LEDGER_ID)
txn_with_last_seq_no = {'txn': {'data': {AUDIT_TXN_VIEW_NO: 2,
AUDIT_TXN_PP_SEQ_NO: 20,
AUDIT_TXN_PRIMARIES: ['Gamma', 'Delta']}}}
audit_ledger.get_last_committed_txn = lambda *args: txn_with_last_seq_no
node.allLedgersCaughtUp()
check_num_received_checkpoints(replica, 1)
check_last_received_checkpoint(replica, 25, view_no=2)
def test_checkpoints_removed_on_backup_replica_after_catchup_during_view_change(
chkFreqPatched, txnPoolNodeSet, view_change_in_progress,
clear_checkpoints):
backup_replicas = getAllReplicas(txnPoolNodeSet, 1)
replica = backup_replicas[-1]
others = backup_replicas[:-1]
node = replica.node
node.master_replica.last_ordered_3pc = (2, 12)
replica._checkpointer._mark_checkpoint_stable(10)
replica._checkpointer._received_checkpoints[cp_key(
2, 15)] = [r.name for r in others]
replica._checkpointer._received_checkpoints[cp_key(
2, 20)] = [r.name for r in others]
replica._checkpointer._received_checkpoints[cp_key(2,
25)] = [others[0].name]
# Simulate catch-up completion
node.ledgerManager.last_caught_up_3PC = (2, 20)
audit_ledger = node.getLedger(AUDIT_LEDGER_ID)
txn_with_last_seq_no = {
'txn': {
'data': {
AUDIT_TXN_VIEW_NO: 2,
AUDIT_TXN_PP_SEQ_NO: 20,
AUDIT_TXN_PRIMARIES: ['Gamma', 'Delta']
}
}
}
audit_ledger.get_last_committed_txn = lambda *args: txn_with_last_seq_no
node.allLedgersCaughtUp()
check_num_received_checkpoints(replica, 0)
def test_received_checkpoints_removed_on_backup_primary_replica_after_catchup(
chkFreqPatched, txnPoolNodeSet, view_setup, clear_checkpoints):
replica = getPrimaryReplica(txnPoolNodeSet, 1)
others = set(getAllReplicas(txnPoolNodeSet, 1)) - {replica}
node = replica.node
node.master_replica.last_ordered_3pc = (2, 12)
replica._consensus_data.stable_checkpoint = 15
replica._checkpointer._received_checkpoints[cp_key(
2, 20)] = [next(iter(others)).name]
# Simulate catch-up completion
node.ledgerManager.last_caught_up_3PC = (2, 20)
audit_ledger = node.getLedger(AUDIT_LEDGER_ID)
txn_with_last_seq_no = {
'txn': {
'data': {
AUDIT_TXN_VIEW_NO: 2,
AUDIT_TXN_PP_SEQ_NO: 20,
AUDIT_TXN_PRIMARIES: ['Gamma', 'Delta']
}
}
}
audit_ledger.get_last_committed_txn = lambda *args: txn_with_last_seq_no
node.write_manager.node_reg_handler.on_catchup_finished = lambda *args: None
node.allLedgersCaughtUp()
check_num_received_checkpoints(replica, 1)
check_last_received_checkpoint(replica, 20, view_no=2)
def test_checkpoints_removed_on_backup_replica_after_catchup_during_view_change(
chkFreqPatched, txnPoolNodeSet, view_change_in_progress,
clear_checkpoints):
backup_replicas = getAllReplicas(txnPoolNodeSet, 1)
replica = backup_replicas[-1]
others = backup_replicas[:-1]
node = replica.node
node.master_replica.last_ordered_3pc = (2, 12)
replica.checkpoints[(6, 10)] = CheckpointState(
seqNo=10,
digests=[],
digest='digest-6-10',
receivedDigests={r.name: 'digest-6-10'
for r in others},
isStable=True)
replica.checkpoints[(11, 15)] = CheckpointState(
seqNo=13,
digests=['digest-11', 'digest-12', 'digest-13'],
digest=None,
receivedDigests={},
isStable=False)
replica.stashedRecvdCheckpoints[2] = {}
replica.stashedRecvdCheckpoints[2][(11, 15)] = {}
for r in others:
replica.stashedRecvdCheckpoints[2][(11, 15)][r.name] = \
Checkpoint(instId=1,
viewNo=2,
seqNoStart=11,
seqNoEnd=15,
digest='digest-11-15')
replica.stashedRecvdCheckpoints[2][(16, 20)] = {}
for r in others:
replica.stashedRecvdCheckpoints[2][(16, 20)][r.name] = \
Checkpoint(instId=1,
viewNo=2,
seqNoStart=16,
seqNoEnd=20,
digest='digest-16-20')
replica.stashedRecvdCheckpoints[2][(21, 25)] = {}
replica.stashedRecvdCheckpoints[2][(21, 25)][next(iter(others)).name] = \
Checkpoint(instId=1,
viewNo=2,
seqNoStart=21,
seqNoEnd=25,
digest='digest-21-25')
# Simulate catch-up completion
node.ledgerManager.last_caught_up_3PC = (2, 20)
node.allLedgersCaughtUp()
assert len(replica.checkpoints) == 0
assert len(replica.stashedRecvdCheckpoints) == 0
def test_checkpoints_removed_on_master_non_primary_replica_after_catchup(
chkFreqPatched, txnPoolNodeSet, view_setup, clear_checkpoints):
replica = getNonPrimaryReplicas(txnPoolNodeSet, 0)[-1]
others = set(getAllReplicas(txnPoolNodeSet, 0)) - {replica}
node = replica.node
node.master_replica.last_ordered_3pc = (2, 12)
replica.checkpoints[(6, 10)] = CheckpointState(seqNo=10,
digests=[],
digest='digest-6-10',
receivedDigests={r.name: 'digest-6-10' for r in others},
isStable=True)
replica.checkpoints[(11, 15)] = CheckpointState(seqNo=12,
digests=['digest-11', 'digest-12'],
digest=None,
receivedDigests={},
isStable=False)
replica.stashedRecvdCheckpoints[2] = {}
replica.stashedRecvdCheckpoints[2][(11, 15)] = {}
for r in others:
replica.stashedRecvdCheckpoints[2][(11, 15)][r.name] = \
Checkpoint(instId=0,
viewNo=2,
seqNoStart=11,
seqNoEnd=15,
digest='digest-11-15')
replica.stashedRecvdCheckpoints[2][(16, 20)] = {}
for r in others:
replica.stashedRecvdCheckpoints[2][(16, 20)][r.name] = \
Checkpoint(instId=0,
viewNo=2,
seqNoStart=16,
seqNoEnd=20,
digest='digest-16-20')
replica.stashedRecvdCheckpoints[2][(21, 25)] = {}
replica.stashedRecvdCheckpoints[2][(21, 25)][next(iter(others)).name] = \
Checkpoint(instId=0,
viewNo=2,
seqNoStart=21,
seqNoEnd=25,
digest='digest-21-25')
# Simulate catch-up completion
node.ledgerManager.last_caught_up_3PC = (2, 20)
node.allLedgersCaughtUp()
assert len(replica.checkpoints) == 0
assert len(replica.stashedRecvdCheckpoints) == 1
assert 2 in replica.stashedRecvdCheckpoints
assert len(replica.stashedRecvdCheckpoints[2]) == 1
assert (21, 25) in replica.stashedRecvdCheckpoints[2]
assert len(replica.stashedRecvdCheckpoints[2][(21, 25)]) == 1
def g(instId):
allReplicas = getAllReplicas(txnPoolNodeSet, instId)
primaryReplica = getPrimaryReplica(txnPoolNodeSet, instId)
def replicas_gets_correct_num_of_COMMITs():
"""
num of commit messages must be = n when zero fault;
n = num of nodes and greater than or equal to
n-f with faults.
"""
passes = 0
numOfMsgsWithZFN = quorums.commit.value
numOfMsgsWithFault = quorums.commit.value
key = (primaryReplica.viewNo, primaryReplica.lastPrePrepareSeqNo)
for r in allReplicas:
if key in r.commits:
rcvdCommitRqst = r.commits[key]
actualMsgsReceived = len(rcvdCommitRqst.voters)
passes += int(msgCountOK(nodeCount,
faultyNodes,
actualMsgsReceived,
numOfMsgsWithZFN,
numOfMsgsWithFault))
assert passes >= min(len(allReplicas) - faultyNodes,
numOfMsgsWithZFN)
replicas_gets_correct_num_of_COMMITs()
def test_checkpoints_removed_on_backup_primary_replica_after_catchup(
chkFreqPatched, txnPoolNodeSet, view_setup, clear_checkpoints):
replica = getPrimaryReplica(txnPoolNodeSet, 1)
others = set(getAllReplicas(txnPoolNodeSet, 1)) - {replica}
node = replica.node
node.master_replica.last_ordered_3pc = (2, 12)
replica._checkpointer._checkpoint_state[(11, 15)] = CheckpointState(
seqNo=15,
digests=[],
digest=cp_digest(11, 15),
receivedDigests={r.name: cp_digest(11, 15)
for r in others},
isStable=True)
replica._checkpointer._checkpoint_state[(16, 20)] = CheckpointState(
seqNo=19,
digests=['digest-16', 'digest-17', 'digest-18', 'digest-19'],
digest=None,
receivedDigests={},
isStable=False)
replica._checkpointer._stashed_recvd_checkpoints[2] = {}
replica._checkpointer._stashed_recvd_checkpoints[2][(16, 20)] = {}
replica._checkpointer._stashed_recvd_checkpoints[2][(16, 20)][next(iter(others)).name] = \
Checkpoint(instId=1,
viewNo=2,
seqNoStart=16,
seqNoEnd=20,
digest=cp_digest(16, 20))
# Simulate catch-up completion
node.ledgerManager.last_caught_up_3PC = (2, 20)
audit_ledger = node.getLedger(AUDIT_LEDGER_ID)
txn_with_last_seq_no = {
'txn': {
'data': {
AUDIT_TXN_VIEW_NO: 2,
AUDIT_TXN_PP_SEQ_NO: 20,
AUDIT_TXN_PRIMARIES: ['Gamma', 'Delta']
}
}
}
audit_ledger.get_last_committed_txn = lambda *args: txn_with_last_seq_no
node.allLedgersCaughtUp()
assert len(replica._checkpointer._checkpoint_state) == 0
assert (11, 15) not in replica._checkpointer._checkpoint_state
assert (16, 20) not in replica._checkpointer._checkpoint_state
assert len(replica._checkpointer._stashed_recvd_checkpoints) == 1
assert 2 in replica._checkpointer._stashed_recvd_checkpoints
assert len(replica._checkpointer._stashed_recvd_checkpoints[2]) == 1
assert (16, 20) in replica._checkpointer._stashed_recvd_checkpoints[2]
assert len(replica._checkpointer._stashed_recvd_checkpoints[2][(16,
20)]) == 1
def g(instId):
allReplicas = getAllReplicas(nodeSet, instId)
primaryReplica = getPrimaryReplica(nodeSet, instId)
def replicasSeesCorrectNumOfCOMMITs():
"""
num of commit messages must be = n when zero fault;
n = num of nodes and greater than or equal to
2f + 1 with faults.
"""
passes = 0
numOfMsgsWithZFN = nodeCount
numOfMsgsWithFault = (2 * f) + 1
key = (primaryReplica.viewNo, primaryReplica.lastPrePrepareSeqNo)
for r in allReplicas:
if key in r.commits:
rcvdCommitRqst = r.commits[key]
actualMsgsReceived = len(rcvdCommitRqst.voters)
passes += int(msgCountOK(nodeCount,
faultyNodes,
actualMsgsReceived,
numOfMsgsWithZFN,
numOfMsgsWithFault))
assert passes >= len(allReplicas) - faultyNodes
def replicasReceivesCorrectNumberOfCOMMITs():
"""
num of commit messages seen by replica must be equal to n - 1;
when zero fault and greater than or equal to
2f+1 with faults.
"""
passes = 0
numOfMsgsWithZFN = nodeCount - 1
numOfMsgsWithFault = 2 * f
for r in allReplicas:
args = getAllArgs(r, r.processCommit)
actualMsgsReceived = len(args)
passes += int(msgCountOK(nodeCount,
faultyNodes,
actualMsgsReceived,
numOfMsgsWithZFN,
numOfMsgsWithFault))
for arg in args:
assert arg['commit'].viewNo == primaryReplica.viewNo and \
arg['commit'].ppSeqNo == primaryReplica.lastPrePrepareSeqNo and \
arg['commit'].digest == prepared1.digest
assert r.name != arg['sender']
assert passes >= len(allReplicas) - faultyNodes
replicasReceivesCorrectNumberOfCOMMITs()
replicasSeesCorrectNumOfCOMMITs()
def test_checkpoints_removed_on_backup_replica_after_catchup_during_view_change(
chkFreqPatched, txnPoolNodeSet, view_change_in_progress, clear_checkpoints):
backup_replicas = getAllReplicas(txnPoolNodeSet, 1)
replica = backup_replicas[-1]
others = backup_replicas[:-1]
node = replica.node
node.master_replica.last_ordered_3pc = (2, 12)
replica.checkpoints[(6, 10)] = CheckpointState(seqNo=10,
digests=[],
digest='digest-6-10',
receivedDigests={r.name: 'digest-6-10' for r in others},
isStable=True)
replica.checkpoints[(11, 15)] = CheckpointState(seqNo=13,
digests=['digest-11', 'digest-12', 'digest-13'],
digest=None,
receivedDigests={},
isStable=False)
replica.stashedRecvdCheckpoints[2] = {}
replica.stashedRecvdCheckpoints[2][(11, 15)] = {}
for r in others:
replica.stashedRecvdCheckpoints[2][(11, 15)][r.name] = \
Checkpoint(instId=1,
viewNo=2,
seqNoStart=11,
seqNoEnd=15,
digest='digest-11-15')
replica.stashedRecvdCheckpoints[2][(16, 20)] = {}
for r in others:
replica.stashedRecvdCheckpoints[2][(16, 20)][r.name] = \
Checkpoint(instId=1,
viewNo=2,
seqNoStart=16,
seqNoEnd=20,
digest='digest-16-20')
replica.stashedRecvdCheckpoints[2][(21, 25)] = {}
replica.stashedRecvdCheckpoints[2][(21, 25)][next(iter(others)).name] = \
Checkpoint(instId=1,
viewNo=2,
seqNoStart=21,
seqNoEnd=25,
digest='digest-21-25')
# Simulate catch-up completion
node.ledgerManager.last_caught_up_3PC = (2, 20)
node.allLedgersCaughtUp()
assert len(replica.checkpoints) == 0
assert len(replica.stashedRecvdCheckpoints) == 0
def g(instId):
allReplicas = getAllReplicas(nodeSet, instId)
primaryReplica = getPrimaryReplica(nodeSet, instId)
def replicasSeesCorrectNumOfCOMMITs():
"""
num of commit messages must be = n when zero fault;
n = num of nodes and greater than or equal to
2f + 1 with faults.
"""
passes = 0
numOfMsgsWithZFN = nodeCount
numOfMsgsWithFault = (2 * f) + 1
key = (primaryReplica.viewNo, primaryReplica.lastPrePrepareSeqNo)
for r in allReplicas:
if key in r.commits:
rcvdCommitRqst = r.commits[key]
actualMsgsReceived = len(rcvdCommitRqst.voters)
passes += int(
msgCountOK(nodeCount, faultyNodes, actualMsgsReceived,
numOfMsgsWithZFN, numOfMsgsWithFault))
assert passes >= len(allReplicas) - faultyNodes
def replicasReceivesCorrectNumberOfCOMMITs():
"""
num of commit messages seen by replica must be equal to n - 1;
when zero fault and greater than or equal to
2f+1 with faults.
"""
passes = 0
numOfMsgsWithZFN = nodeCount - 1
numOfMsgsWithFault = 2 * f
for r in allReplicas:
args = getAllArgs(r, r.processCommit)
actualMsgsReceived = len(args)
passes += int(
msgCountOK(nodeCount, faultyNodes, actualMsgsReceived,
numOfMsgsWithZFN, numOfMsgsWithFault))
for arg in args:
assert arg['commit'].viewNo == primaryReplica.viewNo and \
arg['commit'].ppSeqNo == primaryReplica.lastPrePrepareSeqNo and \
arg['commit'].digest == prepared1.digest
assert r.name != arg['sender']
assert passes >= len(allReplicas) - faultyNodes
replicasReceivesCorrectNumberOfCOMMITs()
replicasSeesCorrectNumOfCOMMITs()
def test_checkpoints_not_removed_on_backup_primary_replica_after_catchup(
chkFreqPatched, txnPoolNodeSet, view_setup, clear_checkpoints):
replica = getPrimaryReplica(txnPoolNodeSet, 1)
others = set(getAllReplicas(txnPoolNodeSet, 1)) - {replica}
node = replica.node
node.master_replica.last_ordered_3pc = (2, 12)
replica.checkpoints[(11, 15)] = CheckpointState(
seqNo=15,
digests=[],
digest='digest-11-15',
receivedDigests={r.name: 'digest-11-15'
for r in others},
isStable=True)
replica.checkpoints[(16, 20)] = CheckpointState(
seqNo=19,
digests=['digest-16', 'digest-17', 'digest-18', 'digest-19'],
digest=None,
receivedDigests={},
isStable=False)
replica.stashedRecvdCheckpoints[2] = {}
replica.stashedRecvdCheckpoints[2][(16, 20)] = {}
replica.stashedRecvdCheckpoints[2][(16, 20)][next(iter(others)).name] = \
Checkpoint(instId=1,
viewNo=2,
seqNoStart=16,
seqNoEnd=20,
digest='digest-16-20')
# Simulate catch-up completion
node.ledgerManager.last_caught_up_3PC = (2, 20)
node.allLedgersCaughtUp()
assert len(replica.checkpoints) == 2
assert (11, 15) in replica.checkpoints
assert (16, 20) in replica.checkpoints
assert len(replica.stashedRecvdCheckpoints) == 1
assert 2 in replica.stashedRecvdCheckpoints
assert len(replica.stashedRecvdCheckpoints[2]) == 1
assert (16, 20) in replica.stashedRecvdCheckpoints[2]
assert len(replica.stashedRecvdCheckpoints[2][(16, 20)]) == 1
def g(instId):
allReplicas = getAllReplicas(txnPoolNodeSet, instId)
primary = getPrimaryReplica(txnPoolNodeSet, instId)
nonPrimaryReplicas = getNonPrimaryReplicas(txnPoolNodeSet, instId)
def primaryDontSendAnyPREPAREs():
"""
1. no of PREPARE sent by primary should be 0
"""
for r in allReplicas:
for param in getAllArgs(r, Replica.processPrepare):
sender = param['sender']
assert sender != primary.name
def allReplicasSeeCorrectNumberOfPREPAREs():
"""
1. no of PREPARE received by replicas must be n - 1;
n = num of nodes without fault, and greater than or equal to
n-f-1 with faults.
"""
passes = 0
numOfMsgsWithZFN = nodeCount - 1
numOfMsgsWithFaults = quorums.prepare.value
for replica in allReplicas:
key = primary.viewNo, primary.lastPrePrepareSeqNo
if key in replica.prepares:
actualMsgs = len(replica.prepares[key].voters)
passes += int(
msgCountOK(nodeCount, faultyNodes, actualMsgs,
numOfMsgsWithZFN, numOfMsgsWithFaults))
assert passes >= len(allReplicas) - faultyNodes
def primaryReceivesCorrectNumberOfPREPAREs():
"""
num of PREPARE seen by primary replica is n - 1;
n = num of nodes without fault, and greater than or equal to
n-f-1 with faults.
"""
actualMsgs = len([
param for param in getAllArgs(primary, primary.processPrepare)
if (param['prepare'].instId, param['prepare'].viewNo,
param['prepare'].ppSeqNo) == (primary.instId,
primary.viewNo,
primary.lastPrePrepareSeqNo)
and param['sender'] != primary.name
])
numOfMsgsWithZFN = nodeCount - 1
numOfMsgsWithFaults = quorums.prepare.value
assert msgCountOK(nodeCount, faultyNodes, actualMsgs,
numOfMsgsWithZFN, numOfMsgsWithFaults)
# TODO what if the primary is faulty?
def nonPrimaryReplicasReceiveCorrectNumberOfPREPAREs():
"""
num of PREPARE seen by Non primary replica is n - 2 without
faults and n-f-2 with faults.
"""
passes = 0
numOfMsgsWithZFN = nodeCount - 2
numOfMsgsWithFaults = quorums.prepare.value - 1
for npr in nonPrimaryReplicas:
actualMsgs = len([
param for param in getAllArgs(npr, npr.processPrepare)
if (param['prepare'].instId, param['prepare'].viewNo,
param['prepare'].ppSeqNo) == (
primary.instId, primary.viewNo,
primary.lastPrePrepareSeqNo)
])
passes += int(
msgCountOK(nodeCount, faultyNodes, actualMsgs,
numOfMsgsWithZFN, numOfMsgsWithFaults))
assert passes >= len(nonPrimaryReplicas) - faultyNodes
# TODO how do we know if one of the faulty nodes is a primary or
# not?
primaryDontSendAnyPREPAREs()
allReplicasSeeCorrectNumberOfPREPAREs()
primaryReceivesCorrectNumberOfPREPAREs()
nonPrimaryReplicasReceiveCorrectNumberOfPREPAREs()
def test_checkpoints_removed_on_master_non_primary_replica_after_catchup(
chkFreqPatched, txnPoolNodeSet, view_setup, clear_checkpoints):
replica = getNonPrimaryReplicas(txnPoolNodeSet, 0)[-1]
others = set(getAllReplicas(txnPoolNodeSet, 0)) - {replica}
node = replica.node
node.master_replica.last_ordered_3pc = (2, 12)
replica.checkpoints[(6, 10)] = CheckpointState(
seqNo=10,
digests=[],
digest='digest-6-10',
receivedDigests={r.name: 'digest-6-10'
for r in others},
isStable=True)
replica.checkpoints[(11, 15)] = CheckpointState(
seqNo=12,
digests=['digest-11', 'digest-12'],
digest=None,
receivedDigests={},
isStable=False)
replica.stashedRecvdCheckpoints[2] = {}
replica.stashedRecvdCheckpoints[2][(11, 15)] = {}
for r in others:
replica.stashedRecvdCheckpoints[2][(11, 15)][r.name] = \
Checkpoint(instId=0,
viewNo=2,
seqNoStart=11,
seqNoEnd=15,
digest='digest-11-15')
replica.stashedRecvdCheckpoints[2][(16, 20)] = {}
for r in others:
replica.stashedRecvdCheckpoints[2][(16, 20)][r.name] = \
Checkpoint(instId=0,
viewNo=2,
seqNoStart=16,
seqNoEnd=20,
digest='digest-16-20')
replica.stashedRecvdCheckpoints[2][(21, 25)] = {}
replica.stashedRecvdCheckpoints[2][(21, 25)][next(iter(others)).name] = \
Checkpoint(instId=0,
viewNo=2,
seqNoStart=21,
seqNoEnd=25,
digest='digest-21-25')
# Simulate catch-up completion
node.ledgerManager.last_caught_up_3PC = (2, 20)
audit_ledger = node.getLedger(AUDIT_LEDGER_ID)
txn_with_last_seq_no = {
'txn': {
'data': {
AUDIT_TXN_VIEW_NO: 2,
AUDIT_TXN_PP_SEQ_NO: 20,
AUDIT_TXN_PRIMARIES: ['Gamma', 'Delta']
}
}
}
audit_ledger.get_last_committed_txn = lambda *args: txn_with_last_seq_no
node.allLedgersCaughtUp()
assert len(replica.checkpoints) == 0
assert len(replica.stashedRecvdCheckpoints) == 1
assert 2 in replica.stashedRecvdCheckpoints
assert len(replica.stashedRecvdCheckpoints[2]) == 1
assert (21, 25) in replica.stashedRecvdCheckpoints[2]
assert len(replica.stashedRecvdCheckpoints[2][(21, 25)]) == 1
def g(instId):
allReplicas = getAllReplicas(txnPoolNodeSet, instId)
primary = getPrimaryReplica(txnPoolNodeSet, instId)
nonPrimaryReplicas = getNonPrimaryReplicas(txnPoolNodeSet, instId)
def primaryDontSendAnyPREPAREs():
"""
1. no of PREPARE sent by primary should be 0
"""
for r in allReplicas:
for param in getAllArgs(r, Replica.processPrepare):
sender = param['sender']
assert sender != primary.name
def allReplicasSeeCorrectNumberOfPREPAREs():
"""
1. no of PREPARE received by replicas must be n - 1;
n = num of nodes without fault, and greater than or equal to
n-f-1 with faults.
"""
passes = 0
numOfMsgsWithZFN = nodeCount - 1
numOfMsgsWithFaults = quorums.prepare.value
for replica in allReplicas:
key = primary.viewNo, primary.lastPrePrepareSeqNo
if key in replica.prepares:
actualMsgs = len(replica.prepares[key].voters)
passes += int(msgCountOK(nodeCount,
faultyNodes,
actualMsgs,
numOfMsgsWithZFN,
numOfMsgsWithFaults))
assert passes >= len(allReplicas) - faultyNodes
def primaryReceivesCorrectNumberOfPREPAREs():
"""
num of PREPARE seen by primary replica is n - 1;
n = num of nodes without fault, and greater than or equal to
n-f-1 with faults.
"""
actualMsgs = len([param for param in
getAllArgs(primary,
primary.processPrepare)
if (param['prepare'].instId,
param['prepare'].viewNo,
param['prepare'].ppSeqNo) == (
primary.instId, primary.viewNo,
primary.lastPrePrepareSeqNo) and
param['sender'] != primary.name])
numOfMsgsWithZFN = nodeCount - 1
numOfMsgsWithFaults = quorums.prepare.value
assert msgCountOK(nodeCount,
faultyNodes,
actualMsgs,
numOfMsgsWithZFN,
numOfMsgsWithFaults)
# TODO what if the primary is faulty?
def nonPrimaryReplicasReceiveCorrectNumberOfPREPAREs():
"""
num of PREPARE seen by Non primary replica is n - 2 without
faults and n-f-2 with faults.
"""
passes = 0
numOfMsgsWithZFN = nodeCount - 2
numOfMsgsWithFaults = quorums.prepare.value - 1
for npr in nonPrimaryReplicas:
actualMsgs = len(
[
param for param in getAllArgs(
npr,
npr.processPrepare) if (
param['prepare'].instId,
param['prepare'].viewNo,
param['prepare'].ppSeqNo) == (
primary.instId,
primary.viewNo,
primary.lastPrePrepareSeqNo)])
passes += int(msgCountOK(nodeCount,
faultyNodes,
actualMsgs,
numOfMsgsWithZFN,
numOfMsgsWithFaults))
assert passes >= len(nonPrimaryReplicas) - faultyNodes
# TODO how do we know if one of the faulty nodes is a primary or
# not?
primaryDontSendAnyPREPAREs()
allReplicasSeeCorrectNumberOfPREPAREs()
primaryReceivesCorrectNumberOfPREPAREs()
nonPrimaryReplicasReceiveCorrectNumberOfPREPAREs()
def test_checkpoints_removed_on_master_replica_after_catchup_during_view_change(
chkFreqPatched, txnPoolNodeSet, view_change_in_progress,
clear_checkpoints):
master_replicas = getAllReplicas(txnPoolNodeSet, 0)
replica = master_replicas[-1]
others = master_replicas[:-1]
node = replica.node
node.master_replica.last_ordered_3pc = (2, 12)
replica._checkpointer._checkpoint_state[(6, 10)] = CheckpointState(
seqNo=10,
digests=[],
digest=cp_digest(6, 10),
receivedDigests={r.name: cp_digest(6, 10)
for r in others},
isStable=True)
replica._checkpointer._checkpoint_state[(11, 15)] = CheckpointState(
seqNo=12,
digests=['digest-11', 'digest-12'],
digest=None,
receivedDigests={},
isStable=False)
replica._checkpointer._stashed_recvd_checkpoints[2] = {}
replica._checkpointer._stashed_recvd_checkpoints[2][(11, 15)] = {}
for r in others:
replica._checkpointer._stashed_recvd_checkpoints[2][(11, 15)][r.name] = \
Checkpoint(instId=0,
viewNo=2,
seqNoStart=11,
seqNoEnd=15,
digest=cp_digest(11, 15))
replica._checkpointer._stashed_recvd_checkpoints[2][(16, 20)] = {}
for r in others:
replica._checkpointer._stashed_recvd_checkpoints[2][(16, 20)][r.name] = \
Checkpoint(instId=0,
viewNo=2,
seqNoStart=16,
seqNoEnd=20,
digest=cp_digest(16, 20))
replica._checkpointer._stashed_recvd_checkpoints[2][(21, 25)] = {}
replica._checkpointer._stashed_recvd_checkpoints[2][(21, 25)][others[0].name] = \
Checkpoint(instId=0,
viewNo=2,
seqNoStart=21,
seqNoEnd=25,
digest=cp_digest(21, 25))
# Simulate catch-up completion
node.ledgerManager.last_caught_up_3PC = (2, 20)
audit_ledger = node.getLedger(AUDIT_LEDGER_ID)
txn_with_last_seq_no = {
'txn': {
'data': {
AUDIT_TXN_VIEW_NO: 2,
AUDIT_TXN_PP_SEQ_NO: 20,
AUDIT_TXN_PRIMARIES: ['Gamma', 'Delta']
}
}
}
audit_ledger.get_last_committed_txn = lambda *args: txn_with_last_seq_no
node.allLedgersCaughtUp()
assert len(replica._checkpointer._checkpoint_state) == 0
assert len(replica._checkpointer._stashed_recvd_checkpoints) == 1
assert 2 in replica._checkpointer._stashed_recvd_checkpoints
assert len(replica._checkpointer._stashed_recvd_checkpoints[2]) == 1
assert (21, 25) in replica._checkpointer._stashed_recvd_checkpoints[2]
assert len(replica._checkpointer._stashed_recvd_checkpoints[2][(21,
25)]) == 1
