def test_checkpoints_removed_on_backup_replica_after_catchup_during_view_change( chkFreqPatched, txnPoolNodeSet, view_change_in_progress, clear_checkpoints): backup_replicas = getAllReplicas(txnPoolNodeSet, 1) replica = backup_replicas[-1] others = backup_replicas[:-1] node = replica.node node.master_replica.last_ordered_3pc = (2, 12) replica.checkpoints[(6, 10)] = CheckpointState( seqNo=10, digests=[], digest='digest-6-10', receivedDigests={r.name: 'digest-6-10' for r in others}, isStable=True) replica.checkpoints[(11, 15)] = CheckpointState( seqNo=13, digests=['digest-11', 'digest-12', 'digest-13'], digest=None, receivedDigests={}, isStable=False) replica.stashedRecvdCheckpoints[2] = {} replica.stashedRecvdCheckpoints[2][(11, 15)] = {} for r in others: replica.stashedRecvdCheckpoints[2][(11, 15)][r.name] = \ Checkpoint(instId=1, viewNo=2, seqNoStart=11, seqNoEnd=15, digest='digest-11-15') replica.stashedRecvdCheckpoints[2][(16, 20)] = {} for r in others: replica.stashedRecvdCheckpoints[2][(16, 20)][r.name] = \ Checkpoint(instId=1, viewNo=2, seqNoStart=16, seqNoEnd=20, digest='digest-16-20') replica.stashedRecvdCheckpoints[2][(21, 25)] = {} replica.stashedRecvdCheckpoints[2][(21, 25)][next(iter(others)).name] = \ Checkpoint(instId=1, viewNo=2, seqNoStart=21, seqNoEnd=25, digest='digest-21-25') # Simulate catch-up completion node.ledgerManager.last_caught_up_3PC = (2, 20) node.allLedgersCaughtUp() assert len(replica.checkpoints) == 0 assert len(replica.stashedRecvdCheckpoints) == 0
def test_process_checkpoint(checkpoint_service, checkpoint, pre_prepare, tconf, ordered, validators, is_master): global caught_msg caught_msg = None checkpoint_service._bus.subscribe(Cleanup, catch_msg) quorum = checkpoint_service._data.quorums.checkpoint.value n = len(validators) assert quorum == n - getMaxFailures(n) - 1 senders = ["sender{}".format(i) for i in range(quorum + 1)] key = (1, tconf.CHK_FREQ) old_key = (-1, 0) checkpoint_service._stash_checkpoint( Checkpoint(1, checkpoint.viewNo, 1, 1, "1"), "frm") checkpoint_service._stash_checkpoint( Checkpoint(1, checkpoint.viewNo + 1, 1, 1, "1"), "frm") checkpoint_service._checkpoint_state[old_key] = CheckpointState( 1, ["digest"] * (tconf.CHK_FREQ - 1), None, {}, False) checkpoint_service._checkpoint_state[key] = CheckpointState( key[1] - 1, ["digest"] * (tconf.CHK_FREQ - 1), None, {}, False) pre_prepare.ppSeqNo = key[1] ordered.ppSeqNo = pre_prepare.ppSeqNo checkpoint_service._data.preprepared.append(pre_prepare) checkpoint_service.process_ordered(ordered) _check_checkpoint(checkpoint_service, key[0], key[1], pre_prepare, check_shared_data=True) state = updateNamedTuple(checkpoint_service._checkpoint_state[key], digest=checkpoint.digest) checkpoint_service._checkpoint_state[key] = state for sender in senders[:quorum - 1]: assert checkpoint_service.process_checkpoint(checkpoint, sender) assert checkpoint_service._checkpoint_state[key].receivedDigests[ sender] == checkpoint.digest assert not checkpoint_service._checkpoint_state[key].isStable # send the last checkpoint to stable it assert checkpoint_service.process_checkpoint(checkpoint, senders[quorum - 1]) assert checkpoint_service._checkpoint_state[key].isStable # check _remove_stashed_checkpoints() assert checkpoint.viewNo not in checkpoint_service._stashed_recvd_checkpoints assert checkpoint.viewNo + 1 in checkpoint_service._stashed_recvd_checkpoints # check watermarks assert checkpoint_service._data.low_watermark == checkpoint.seqNoEnd # check that a Cleanup msg has been sent assert isinstance(caught_msg, Cleanup) assert caught_msg.cleanup_till_3pc == (checkpoint.viewNo, checkpoint.seqNoEnd) # check that old checkpoint_states has been removed assert old_key not in checkpoint_service._checkpoint_state
def test_checkpoints_removed_on_backup_primary_replica_after_catchup( chkFreqPatched, txnPoolNodeSet, view_setup, clear_checkpoints): replica = getPrimaryReplica(txnPoolNodeSet, 1) others = set(getAllReplicas(txnPoolNodeSet, 1)) - {replica} node = replica.node node.master_replica.last_ordered_3pc = (2, 12) replica._checkpointer._checkpoint_state[(11, 15)] = CheckpointState( seqNo=15, digests=[], digest=cp_digest(11, 15), receivedDigests={r.name: cp_digest(11, 15) for r in others}, isStable=True) replica._checkpointer._checkpoint_state[(16, 20)] = CheckpointState( seqNo=19, digests=['digest-16', 'digest-17', 'digest-18', 'digest-19'], digest=None, receivedDigests={}, isStable=False) replica._checkpointer._stashed_recvd_checkpoints[2] = {} replica._checkpointer._stashed_recvd_checkpoints[2][(16, 20)] = {} replica._checkpointer._stashed_recvd_checkpoints[2][(16, 20)][next(iter(others)).name] = \ Checkpoint(instId=1, viewNo=2, seqNoStart=16, seqNoEnd=20, digest=cp_digest(16, 20)) # Simulate catch-up completion node.ledgerManager.last_caught_up_3PC = (2, 20) audit_ledger = node.getLedger(AUDIT_LEDGER_ID) txn_with_last_seq_no = { 'txn': { 'data': { AUDIT_TXN_VIEW_NO: 2, AUDIT_TXN_PP_SEQ_NO: 20, AUDIT_TXN_PRIMARIES: ['Gamma', 'Delta'] } } } audit_ledger.get_last_committed_txn = lambda *args: txn_with_last_seq_no node.allLedgersCaughtUp() assert len(replica._checkpointer._checkpoint_state) == 0 assert (11, 15) not in replica._checkpointer._checkpoint_state assert (16, 20) not in replica._checkpointer._checkpoint_state assert len(replica._checkpointer._stashed_recvd_checkpoints) == 1 assert 2 in replica._checkpointer._stashed_recvd_checkpoints assert len(replica._checkpointer._stashed_recvd_checkpoints[2]) == 1 assert (16, 20) in replica._checkpointer._stashed_recvd_checkpoints[2] assert len(replica._checkpointer._stashed_recvd_checkpoints[2][(16, 20)]) == 1
def test_process_backup_catchup_msg(checkpoint_service, tconf, checkpoint): checkpoint_service._data.last_ordered_3pc = (checkpoint_service.view_no, 0) key = (1, tconf.CHK_FREQ) new_key = (key[1] + 1, key[1] + tconf.CHK_FREQ) checkpoint_service._data.stable_checkpoint = 1 checkpoint_service._stash_checkpoint( Checkpoint(1, checkpoint.viewNo, new_key[0], new_key[1], cp_digest(1, 1)), "frm") checkpoint_service._stash_checkpoint( Checkpoint(1, checkpoint.viewNo, key[0], key[1], cp_digest(1, 1)), "frm") checkpoint_service._checkpoint_state[key] = CheckpointState( key[1] - 1, ["digest"] * (tconf.CHK_FREQ - 1), None, {}, False) checkpoint_service._data.checkpoints.append(checkpoint) checkpoint_service._data.last_ordered_3pc = (checkpoint_service.view_no, key[1]) checkpoint_service.caught_up_till_3pc( checkpoint_service._data.last_ordered_3pc) assert checkpoint_service._data.low_watermark == key[1] assert not checkpoint_service._checkpoint_state assert not checkpoint_service._data.checkpoints assert checkpoint_service._data.stable_checkpoint == 0 assert key not in checkpoint_service._stashed_recvd_checkpoints[ checkpoint_service.view_no] assert new_key in checkpoint_service._stashed_recvd_checkpoints[ checkpoint_service.view_no]
def test_checkpoints_not_removed_on_backup_primary_replica_after_catchup( chkFreqPatched, txnPoolNodeSet, view_setup, clear_checkpoints): replica = getPrimaryReplica(txnPoolNodeSet, 1) others = set(getAllReplicas(txnPoolNodeSet, 1)) - {replica} node = replica.node node.master_replica.last_ordered_3pc = (2, 12) replica.checkpoints[(11, 15)] = CheckpointState( seqNo=15, digests=[], digest='digest-11-15', receivedDigests={r.name: 'digest-11-15' for r in others}, isStable=True) replica.checkpoints[(16, 20)] = CheckpointState( seqNo=19, digests=['digest-16', 'digest-17', 'digest-18', 'digest-19'], digest=None, receivedDigests={}, isStable=False) replica.stashedRecvdCheckpoints[2] = {} replica.stashedRecvdCheckpoints[2][(16, 20)] = {} replica.stashedRecvdCheckpoints[2][(16, 20)][next(iter(others)).name] = \ Checkpoint(instId=1, viewNo=2, seqNoStart=16, seqNoEnd=20, digest='digest-16-20') # Simulate catch-up completion node.ledgerManager.last_caught_up_3PC = (2, 20) node.allLedgersCaughtUp() assert len(replica.checkpoints) == 2 assert (11, 15) in replica.checkpoints assert (16, 20) in replica.checkpoints assert len(replica.stashedRecvdCheckpoints) == 1 assert 2 in replica.stashedRecvdCheckpoints assert len(replica.stashedRecvdCheckpoints[2]) == 1 assert (16, 20) in replica.stashedRecvdCheckpoints[2] assert len(replica.stashedRecvdCheckpoints[2][(16, 20)]) == 1
def test_process_checkpoint_with_incorrect_digest(checkpoint_service, checkpoint, tconf, is_master): key = (checkpoint.seqNoStart, checkpoint.seqNoEnd) sender = "sender" checkpoint_service._checkpoint_state[key] = CheckpointState( 1, [], "other_digest", {}, False) assert checkpoint_service.process_checkpoint(checkpoint, sender) if is_master: assert sender not in checkpoint_service._checkpoint_state[ key].receivedDigests else: assert sender in checkpoint_service._checkpoint_state[ key].receivedDigests
def _add_to_checkpoint(self, ppSeqNo, digest, ledger_id, view_no): for (s, e) in self._checkpoint_state.keys(): if s <= ppSeqNo <= e: state = self._checkpoint_state[s, e] # type: CheckpointState state.digests.append(digest) state = updateNamedTuple(state, seqNo=ppSeqNo) self._checkpoint_state[s, e] = state break else: s, e = ppSeqNo, math.ceil( ppSeqNo / self._config.CHK_FREQ) * self._config.CHK_FREQ self._logger.debug("{} adding new checkpoint state for {}".format( self, (s, e))) state = CheckpointState(ppSeqNo, [ digest, ], None, {}, False) self._checkpoint_state[s, e] = state if state.seqNo == e: if len(state.digests) == self._config.CHK_FREQ: self._do_checkpoint(state, s, e, ledger_id, view_no) self._process_stashed_checkpoints((s, e), view_no)
def test_checkpoints_removed_on_master_non_primary_replica_after_catchup( chkFreqPatched, txnPoolNodeSet, view_setup, clear_checkpoints): replica = getNonPrimaryReplicas(txnPoolNodeSet, 0)[-1] others = set(getAllReplicas(txnPoolNodeSet, 0)) - {replica} node = replica.node node.master_replica.last_ordered_3pc = (2, 12) replica.checkpoints[(6, 10)] = CheckpointState( seqNo=10, digests=[], digest='digest-6-10', receivedDigests={r.name: 'digest-6-10' for r in others}, isStable=True) replica.checkpoints[(11, 15)] = CheckpointState( seqNo=12, digests=['digest-11', 'digest-12'], digest=None, receivedDigests={}, isStable=False) replica.stashedRecvdCheckpoints[2] = {} replica.stashedRecvdCheckpoints[2][(11, 15)] = {} for r in others: replica.stashedRecvdCheckpoints[2][(11, 15)][r.name] = \ Checkpoint(instId=0, viewNo=2, seqNoStart=11, seqNoEnd=15, digest='digest-11-15') replica.stashedRecvdCheckpoints[2][(16, 20)] = {} for r in others: replica.stashedRecvdCheckpoints[2][(16, 20)][r.name] = \ Checkpoint(instId=0, viewNo=2, seqNoStart=16, seqNoEnd=20, digest='digest-16-20') replica.stashedRecvdCheckpoints[2][(21, 25)] = {} replica.stashedRecvdCheckpoints[2][(21, 25)][next(iter(others)).name] = \ Checkpoint(instId=0, viewNo=2, seqNoStart=21, seqNoEnd=25, digest='digest-21-25') # Simulate catch-up completion node.ledgerManager.last_caught_up_3PC = (2, 20) audit_ledger = node.getLedger(AUDIT_LEDGER_ID) txn_with_last_seq_no = { 'txn': { 'data': { AUDIT_TXN_VIEW_NO: 2, AUDIT_TXN_PP_SEQ_NO: 20, AUDIT_TXN_PRIMARIES: ['Gamma', 'Delta'] } } } audit_ledger.get_last_committed_txn = lambda *args: txn_with_last_seq_no node.allLedgersCaughtUp() assert len(replica.checkpoints) == 0 assert len(replica.stashedRecvdCheckpoints) == 1 assert 2 in replica.stashedRecvdCheckpoints assert len(replica.stashedRecvdCheckpoints[2]) == 1 assert (21, 25) in replica.stashedRecvdCheckpoints[2] assert len(replica.stashedRecvdCheckpoints[2][(21, 25)]) == 1
def test_checkpoints_removed_on_master_replica_after_catchup_during_view_change( chkFreqPatched, txnPoolNodeSet, view_change_in_progress, clear_checkpoints): master_replicas = getAllReplicas(txnPoolNodeSet, 0) replica = master_replicas[-1] others = master_replicas[:-1] node = replica.node node.master_replica.last_ordered_3pc = (2, 12) replica._checkpointer._checkpoint_state[(6, 10)] = CheckpointState( seqNo=10, digests=[], digest=cp_digest(6, 10), receivedDigests={r.name: cp_digest(6, 10) for r in others}, isStable=True) replica._checkpointer._checkpoint_state[(11, 15)] = CheckpointState( seqNo=12, digests=['digest-11', 'digest-12'], digest=None, receivedDigests={}, isStable=False) replica._checkpointer._stashed_recvd_checkpoints[2] = {} replica._checkpointer._stashed_recvd_checkpoints[2][(11, 15)] = {} for r in others: replica._checkpointer._stashed_recvd_checkpoints[2][(11, 15)][r.name] = \ Checkpoint(instId=0, viewNo=2, seqNoStart=11, seqNoEnd=15, digest=cp_digest(11, 15)) replica._checkpointer._stashed_recvd_checkpoints[2][(16, 20)] = {} for r in others: replica._checkpointer._stashed_recvd_checkpoints[2][(16, 20)][r.name] = \ Checkpoint(instId=0, viewNo=2, seqNoStart=16, seqNoEnd=20, digest=cp_digest(16, 20)) replica._checkpointer._stashed_recvd_checkpoints[2][(21, 25)] = {} replica._checkpointer._stashed_recvd_checkpoints[2][(21, 25)][others[0].name] = \ Checkpoint(instId=0, viewNo=2, seqNoStart=21, seqNoEnd=25, digest=cp_digest(21, 25)) # Simulate catch-up completion node.ledgerManager.last_caught_up_3PC = (2, 20) audit_ledger = node.getLedger(AUDIT_LEDGER_ID) txn_with_last_seq_no = { 'txn': { 'data': { AUDIT_TXN_VIEW_NO: 2, AUDIT_TXN_PP_SEQ_NO: 20, AUDIT_TXN_PRIMARIES: ['Gamma', 'Delta'] } } } audit_ledger.get_last_committed_txn = lambda *args: txn_with_last_seq_no node.allLedgersCaughtUp() assert len(replica._checkpointer._checkpoint_state) == 0 assert len(replica._checkpointer._stashed_recvd_checkpoints) == 1 assert 2 in replica._checkpointer._stashed_recvd_checkpoints assert len(replica._checkpointer._stashed_recvd_checkpoints[2]) == 1 assert (21, 25) in replica._checkpointer._stashed_recvd_checkpoints[2] assert len(replica._checkpointer._stashed_recvd_checkpoints[2][(21, 25)]) == 1