def test_checkpoints_removed_on_backup_replica_after_catchup_during_view_change(
chkFreqPatched, txnPoolNodeSet, view_change_in_progress,
clear_checkpoints):
backup_replicas = getAllReplicas(txnPoolNodeSet, 1)
replica = backup_replicas[-1]
others = backup_replicas[:-1]
node = replica.node
node.master_replica.last_ordered_3pc = (2, 12)
replica.checkpoints[(6, 10)] = CheckpointState(
seqNo=10,
digests=[],
digest='digest-6-10',
receivedDigests={r.name: 'digest-6-10'
for r in others},
isStable=True)
replica.checkpoints[(11, 15)] = CheckpointState(
seqNo=13,
digests=['digest-11', 'digest-12', 'digest-13'],
digest=None,
receivedDigests={},
isStable=False)
replica.stashedRecvdCheckpoints[2] = {}
replica.stashedRecvdCheckpoints[2][(11, 15)] = {}
for r in others:
replica.stashedRecvdCheckpoints[2][(11, 15)][r.name] = \
Checkpoint(instId=1,
viewNo=2,
seqNoStart=11,
seqNoEnd=15,
digest='digest-11-15')
replica.stashedRecvdCheckpoints[2][(16, 20)] = {}
for r in others:
replica.stashedRecvdCheckpoints[2][(16, 20)][r.name] = \
Checkpoint(instId=1,
viewNo=2,
seqNoStart=16,
seqNoEnd=20,
digest='digest-16-20')
replica.stashedRecvdCheckpoints[2][(21, 25)] = {}
replica.stashedRecvdCheckpoints[2][(21, 25)][next(iter(others)).name] = \
Checkpoint(instId=1,
viewNo=2,
seqNoStart=21,
seqNoEnd=25,
digest='digest-21-25')
# Simulate catch-up completion
node.ledgerManager.last_caught_up_3PC = (2, 20)
node.allLedgersCaughtUp()
assert len(replica.checkpoints) == 0
assert len(replica.stashedRecvdCheckpoints) == 0
def test_process_checkpoint(checkpoint_service, checkpoint, pre_prepare, tconf,
ordered, validators, is_master):
global caught_msg
caught_msg = None
checkpoint_service._bus.subscribe(Cleanup, catch_msg)
quorum = checkpoint_service._data.quorums.checkpoint.value
n = len(validators)
assert quorum == n - getMaxFailures(n) - 1
senders = ["sender{}".format(i) for i in range(quorum + 1)]
key = (1, tconf.CHK_FREQ)
old_key = (-1, 0)
checkpoint_service._stash_checkpoint(
Checkpoint(1, checkpoint.viewNo, 1, 1, "1"), "frm")
checkpoint_service._stash_checkpoint(
Checkpoint(1, checkpoint.viewNo + 1, 1, 1, "1"), "frm")
checkpoint_service._checkpoint_state[old_key] = CheckpointState(
1, ["digest"] * (tconf.CHK_FREQ - 1), None, {}, False)
checkpoint_service._checkpoint_state[key] = CheckpointState(
key[1] - 1, ["digest"] * (tconf.CHK_FREQ - 1), None, {}, False)
pre_prepare.ppSeqNo = key[1]
ordered.ppSeqNo = pre_prepare.ppSeqNo
checkpoint_service._data.preprepared.append(pre_prepare)
checkpoint_service.process_ordered(ordered)
_check_checkpoint(checkpoint_service,
key[0],
key[1],
pre_prepare,
check_shared_data=True)
state = updateNamedTuple(checkpoint_service._checkpoint_state[key],
digest=checkpoint.digest)
checkpoint_service._checkpoint_state[key] = state
for sender in senders[:quorum - 1]:
assert checkpoint_service.process_checkpoint(checkpoint, sender)
assert checkpoint_service._checkpoint_state[key].receivedDigests[
sender] == checkpoint.digest
assert not checkpoint_service._checkpoint_state[key].isStable
# send the last checkpoint to stable it
assert checkpoint_service.process_checkpoint(checkpoint,
senders[quorum - 1])
assert checkpoint_service._checkpoint_state[key].isStable
# check _remove_stashed_checkpoints()
assert checkpoint.viewNo not in checkpoint_service._stashed_recvd_checkpoints
assert checkpoint.viewNo + 1 in checkpoint_service._stashed_recvd_checkpoints
# check watermarks
assert checkpoint_service._data.low_watermark == checkpoint.seqNoEnd
# check that a Cleanup msg has been sent
assert isinstance(caught_msg, Cleanup)
assert caught_msg.cleanup_till_3pc == (checkpoint.viewNo,
checkpoint.seqNoEnd)
# check that old checkpoint_states has been removed
assert old_key not in checkpoint_service._checkpoint_state
def test_checkpoints_removed_on_backup_primary_replica_after_catchup(
chkFreqPatched, txnPoolNodeSet, view_setup, clear_checkpoints):
replica = getPrimaryReplica(txnPoolNodeSet, 1)
others = set(getAllReplicas(txnPoolNodeSet, 1)) - {replica}
node = replica.node
node.master_replica.last_ordered_3pc = (2, 12)
replica._checkpointer._checkpoint_state[(11, 15)] = CheckpointState(
seqNo=15,
digests=[],
digest=cp_digest(11, 15),
receivedDigests={r.name: cp_digest(11, 15)
for r in others},
isStable=True)
replica._checkpointer._checkpoint_state[(16, 20)] = CheckpointState(
seqNo=19,
digests=['digest-16', 'digest-17', 'digest-18', 'digest-19'],
digest=None,
receivedDigests={},
isStable=False)
replica._checkpointer._stashed_recvd_checkpoints[2] = {}
replica._checkpointer._stashed_recvd_checkpoints[2][(16, 20)] = {}
replica._checkpointer._stashed_recvd_checkpoints[2][(16, 20)][next(iter(others)).name] = \
Checkpoint(instId=1,
viewNo=2,
seqNoStart=16,
seqNoEnd=20,
digest=cp_digest(16, 20))
# Simulate catch-up completion
node.ledgerManager.last_caught_up_3PC = (2, 20)
audit_ledger = node.getLedger(AUDIT_LEDGER_ID)
txn_with_last_seq_no = {
'txn': {
'data': {
AUDIT_TXN_VIEW_NO: 2,
AUDIT_TXN_PP_SEQ_NO: 20,
AUDIT_TXN_PRIMARIES: ['Gamma', 'Delta']
}
}
}
audit_ledger.get_last_committed_txn = lambda *args: txn_with_last_seq_no
node.allLedgersCaughtUp()
assert len(replica._checkpointer._checkpoint_state) == 0
assert (11, 15) not in replica._checkpointer._checkpoint_state
assert (16, 20) not in replica._checkpointer._checkpoint_state
assert len(replica._checkpointer._stashed_recvd_checkpoints) == 1
assert 2 in replica._checkpointer._stashed_recvd_checkpoints
assert len(replica._checkpointer._stashed_recvd_checkpoints[2]) == 1
assert (16, 20) in replica._checkpointer._stashed_recvd_checkpoints[2]
assert len(replica._checkpointer._stashed_recvd_checkpoints[2][(16,
20)]) == 1
def test_process_backup_catchup_msg(checkpoint_service, tconf, checkpoint):
checkpoint_service._data.last_ordered_3pc = (checkpoint_service.view_no, 0)
key = (1, tconf.CHK_FREQ)
new_key = (key[1] + 1, key[1] + tconf.CHK_FREQ)
checkpoint_service._data.stable_checkpoint = 1
checkpoint_service._stash_checkpoint(
Checkpoint(1, checkpoint.viewNo, new_key[0], new_key[1],
cp_digest(1, 1)), "frm")
checkpoint_service._stash_checkpoint(
Checkpoint(1, checkpoint.viewNo, key[0], key[1], cp_digest(1, 1)),
"frm")
checkpoint_service._checkpoint_state[key] = CheckpointState(
key[1] - 1, ["digest"] * (tconf.CHK_FREQ - 1), None, {}, False)
checkpoint_service._data.checkpoints.append(checkpoint)
checkpoint_service._data.last_ordered_3pc = (checkpoint_service.view_no,
key[1])
checkpoint_service.caught_up_till_3pc(
checkpoint_service._data.last_ordered_3pc)
assert checkpoint_service._data.low_watermark == key[1]
assert not checkpoint_service._checkpoint_state
assert not checkpoint_service._data.checkpoints
assert checkpoint_service._data.stable_checkpoint == 0
assert key not in checkpoint_service._stashed_recvd_checkpoints[
checkpoint_service.view_no]
assert new_key in checkpoint_service._stashed_recvd_checkpoints[
checkpoint_service.view_no]
def test_checkpoints_not_removed_on_backup_primary_replica_after_catchup(
chkFreqPatched, txnPoolNodeSet, view_setup, clear_checkpoints):
replica = getPrimaryReplica(txnPoolNodeSet, 1)
others = set(getAllReplicas(txnPoolNodeSet, 1)) - {replica}
node = replica.node
node.master_replica.last_ordered_3pc = (2, 12)
replica.checkpoints[(11, 15)] = CheckpointState(
seqNo=15,
digests=[],
digest='digest-11-15',
receivedDigests={r.name: 'digest-11-15'
for r in others},
isStable=True)
replica.checkpoints[(16, 20)] = CheckpointState(
seqNo=19,
digests=['digest-16', 'digest-17', 'digest-18', 'digest-19'],
digest=None,
receivedDigests={},
isStable=False)
replica.stashedRecvdCheckpoints[2] = {}
replica.stashedRecvdCheckpoints[2][(16, 20)] = {}
replica.stashedRecvdCheckpoints[2][(16, 20)][next(iter(others)).name] = \
Checkpoint(instId=1,
viewNo=2,
seqNoStart=16,
seqNoEnd=20,
digest='digest-16-20')
# Simulate catch-up completion
node.ledgerManager.last_caught_up_3PC = (2, 20)
node.allLedgersCaughtUp()
assert len(replica.checkpoints) == 2
assert (11, 15) in replica.checkpoints
assert (16, 20) in replica.checkpoints
assert len(replica.stashedRecvdCheckpoints) == 1
assert 2 in replica.stashedRecvdCheckpoints
assert len(replica.stashedRecvdCheckpoints[2]) == 1
assert (16, 20) in replica.stashedRecvdCheckpoints[2]
assert len(replica.stashedRecvdCheckpoints[2][(16, 20)]) == 1
def test_process_checkpoint_with_incorrect_digest(checkpoint_service,
checkpoint, tconf,
is_master):
key = (checkpoint.seqNoStart, checkpoint.seqNoEnd)
sender = "sender"
checkpoint_service._checkpoint_state[key] = CheckpointState(
1, [], "other_digest", {}, False)
assert checkpoint_service.process_checkpoint(checkpoint, sender)
if is_master:
assert sender not in checkpoint_service._checkpoint_state[
key].receivedDigests
else:
assert sender in checkpoint_service._checkpoint_state[
key].receivedDigests
def _add_to_checkpoint(self, ppSeqNo, digest, ledger_id, view_no):
for (s, e) in self._checkpoint_state.keys():
if s <= ppSeqNo <= e:
state = self._checkpoint_state[s, e] # type: CheckpointState
state.digests.append(digest)
state = updateNamedTuple(state, seqNo=ppSeqNo)
self._checkpoint_state[s, e] = state
break
else:
s, e = ppSeqNo, math.ceil(
ppSeqNo / self._config.CHK_FREQ) * self._config.CHK_FREQ
self._logger.debug("{} adding new checkpoint state for {}".format(
self, (s, e)))
state = CheckpointState(ppSeqNo, [
digest,
], None, {}, False)
self._checkpoint_state[s, e] = state
if state.seqNo == e:
if len(state.digests) == self._config.CHK_FREQ:
self._do_checkpoint(state, s, e, ledger_id, view_no)
self._process_stashed_checkpoints((s, e), view_no)
def test_checkpoints_removed_on_master_non_primary_replica_after_catchup(
chkFreqPatched, txnPoolNodeSet, view_setup, clear_checkpoints):
replica = getNonPrimaryReplicas(txnPoolNodeSet, 0)[-1]
others = set(getAllReplicas(txnPoolNodeSet, 0)) - {replica}
node = replica.node
node.master_replica.last_ordered_3pc = (2, 12)
replica.checkpoints[(6, 10)] = CheckpointState(
seqNo=10,
digests=[],
digest='digest-6-10',
receivedDigests={r.name: 'digest-6-10'
for r in others},
isStable=True)
replica.checkpoints[(11, 15)] = CheckpointState(
seqNo=12,
digests=['digest-11', 'digest-12'],
digest=None,
receivedDigests={},
isStable=False)
replica.stashedRecvdCheckpoints[2] = {}
replica.stashedRecvdCheckpoints[2][(11, 15)] = {}
for r in others:
replica.stashedRecvdCheckpoints[2][(11, 15)][r.name] = \
Checkpoint(instId=0,
viewNo=2,
seqNoStart=11,
seqNoEnd=15,
digest='digest-11-15')
replica.stashedRecvdCheckpoints[2][(16, 20)] = {}
for r in others:
replica.stashedRecvdCheckpoints[2][(16, 20)][r.name] = \
Checkpoint(instId=0,
viewNo=2,
seqNoStart=16,
seqNoEnd=20,
digest='digest-16-20')
replica.stashedRecvdCheckpoints[2][(21, 25)] = {}
replica.stashedRecvdCheckpoints[2][(21, 25)][next(iter(others)).name] = \
Checkpoint(instId=0,
viewNo=2,
seqNoStart=21,
seqNoEnd=25,
digest='digest-21-25')
# Simulate catch-up completion
node.ledgerManager.last_caught_up_3PC = (2, 20)
audit_ledger = node.getLedger(AUDIT_LEDGER_ID)
txn_with_last_seq_no = {
'txn': {
'data': {
AUDIT_TXN_VIEW_NO: 2,
AUDIT_TXN_PP_SEQ_NO: 20,
AUDIT_TXN_PRIMARIES: ['Gamma', 'Delta']
}
}
}
audit_ledger.get_last_committed_txn = lambda *args: txn_with_last_seq_no
node.allLedgersCaughtUp()
assert len(replica.checkpoints) == 0
assert len(replica.stashedRecvdCheckpoints) == 1
assert 2 in replica.stashedRecvdCheckpoints
assert len(replica.stashedRecvdCheckpoints[2]) == 1
assert (21, 25) in replica.stashedRecvdCheckpoints[2]
assert len(replica.stashedRecvdCheckpoints[2][(21, 25)]) == 1
def test_checkpoints_removed_on_master_replica_after_catchup_during_view_change(
chkFreqPatched, txnPoolNodeSet, view_change_in_progress,
clear_checkpoints):
master_replicas = getAllReplicas(txnPoolNodeSet, 0)
replica = master_replicas[-1]
others = master_replicas[:-1]
node = replica.node
node.master_replica.last_ordered_3pc = (2, 12)
replica._checkpointer._checkpoint_state[(6, 10)] = CheckpointState(
seqNo=10,
digests=[],
digest=cp_digest(6, 10),
receivedDigests={r.name: cp_digest(6, 10)
for r in others},
isStable=True)
replica._checkpointer._checkpoint_state[(11, 15)] = CheckpointState(
seqNo=12,
digests=['digest-11', 'digest-12'],
digest=None,
receivedDigests={},
isStable=False)
replica._checkpointer._stashed_recvd_checkpoints[2] = {}
replica._checkpointer._stashed_recvd_checkpoints[2][(11, 15)] = {}
for r in others:
replica._checkpointer._stashed_recvd_checkpoints[2][(11, 15)][r.name] = \
Checkpoint(instId=0,
viewNo=2,
seqNoStart=11,
seqNoEnd=15,
digest=cp_digest(11, 15))
replica._checkpointer._stashed_recvd_checkpoints[2][(16, 20)] = {}
for r in others:
replica._checkpointer._stashed_recvd_checkpoints[2][(16, 20)][r.name] = \
Checkpoint(instId=0,
viewNo=2,
seqNoStart=16,
seqNoEnd=20,
digest=cp_digest(16, 20))
replica._checkpointer._stashed_recvd_checkpoints[2][(21, 25)] = {}
replica._checkpointer._stashed_recvd_checkpoints[2][(21, 25)][others[0].name] = \
Checkpoint(instId=0,
viewNo=2,
seqNoStart=21,
seqNoEnd=25,
digest=cp_digest(21, 25))
# Simulate catch-up completion
node.ledgerManager.last_caught_up_3PC = (2, 20)
audit_ledger = node.getLedger(AUDIT_LEDGER_ID)
txn_with_last_seq_no = {
'txn': {
'data': {
AUDIT_TXN_VIEW_NO: 2,
AUDIT_TXN_PP_SEQ_NO: 20,
AUDIT_TXN_PRIMARIES: ['Gamma', 'Delta']
}
}
}
audit_ledger.get_last_committed_txn = lambda *args: txn_with_last_seq_no
node.allLedgersCaughtUp()
assert len(replica._checkpointer._checkpoint_state) == 0
assert len(replica._checkpointer._stashed_recvd_checkpoints) == 1
assert 2 in replica._checkpointer._stashed_recvd_checkpoints
assert len(replica._checkpointer._stashed_recvd_checkpoints[2]) == 1
assert (21, 25) in replica._checkpointer._stashed_recvd_checkpoints[2]
assert len(replica._checkpointer._stashed_recvd_checkpoints[2][(21,
25)]) == 1
