def test_calc_batches_respects_checkpoint(builder): cp = Checkpoint(instId=0, viewNo=0, seqNoStart=0, seqNoEnd=10, digest=cp_digest(0, 0)) vc = ViewChange(viewNo=0, stableCheckpoint=0, prepared=[(0, 1, "digest1"), (0, 2, "digest2")], preprepared=[(0, 1, "digest1"), (0, 2, "digest2")], checkpoints=[cp]) vcs = [vc, vc, vc, vc] assert builder.calc_batches(cp, vcs) == [] cp = Checkpoint(instId=0, viewNo=0, seqNoStart=0, seqNoEnd=10, digest=cp_digest(0, 0)) vc = ViewChange(viewNo=0, stableCheckpoint=0, prepared=[(0, 10, "digest10"), (0, 11, "digest11"), (1, 12, "digest12")], preprepared=[(0, 10, "digest10"), (0, 11, "digest11"), (1, 12, "digest12")], checkpoints=[cp]) vcs = [vc, vc, vc, vc] assert builder.calc_batches( cp, vcs) == [BatchID(0, 11, "digest11"), BatchID(1, 12, "digest12")]
def test_checkpoints_removed_on_backup_replica_after_catchup_during_view_change( chkFreqPatched, txnPoolNodeSet, view_change_in_progress, clear_checkpoints): backup_replicas = getAllReplicas(txnPoolNodeSet, 1) replica = backup_replicas[-1] others = backup_replicas[:-1] node = replica.node node.master_replica.last_ordered_3pc = (2, 12) replica.checkpoints[(6, 10)] = CheckpointState( seqNo=10, digests=[], digest='digest-6-10', receivedDigests={r.name: 'digest-6-10' for r in others}, isStable=True) replica.checkpoints[(11, 15)] = CheckpointState( seqNo=13, digests=['digest-11', 'digest-12', 'digest-13'], digest=None, receivedDigests={}, isStable=False) replica.stashedRecvdCheckpoints[2] = {} replica.stashedRecvdCheckpoints[2][(11, 15)] = {} for r in others: replica.stashedRecvdCheckpoints[2][(11, 15)][r.name] = \ Checkpoint(instId=1, viewNo=2, seqNoStart=11, seqNoEnd=15, digest='digest-11-15') replica.stashedRecvdCheckpoints[2][(16, 20)] = {} for r in others: replica.stashedRecvdCheckpoints[2][(16, 20)][r.name] = \ Checkpoint(instId=1, viewNo=2, seqNoStart=16, seqNoEnd=20, digest='digest-16-20') replica.stashedRecvdCheckpoints[2][(21, 25)] = {} replica.stashedRecvdCheckpoints[2][(21, 25)][next(iter(others)).name] = \ Checkpoint(instId=1, viewNo=2, seqNoStart=21, seqNoEnd=25, digest='digest-21-25') # Simulate catch-up completion node.ledgerManager.last_caught_up_3PC = (2, 20) node.allLedgersCaughtUp() assert len(replica.checkpoints) == 0 assert len(replica.stashedRecvdCheckpoints) == 0
def test_view_change_data_multiple(view_change_service, data): # view 0 -> 1 data.view_no = 0 cp1 = Checkpoint(instId=0, viewNo=0, seqNoStart=0, seqNoEnd=10, digest='empty') data.checkpoints.add(cp1) data.stable_checkpoint = 0 data.prepared = [BatchID(0, 1, "digest1"), BatchID(0, 2, "digest2")] data.preprepared = [ BatchID(0, 1, "digest1"), BatchID(0, 2, "digest2"), BatchID(0, 3, "digest3") ] view_change_service.start_view_change() assert data.prepared == [] assert data.preprepared == [] assert data.view_no == 1 msg = get_view_change(view_change_service) assert msg.viewNo == 1 assert msg.prepared == [(0, 1, "digest1"), (0, 2, "digest2")] assert msg.preprepared == [(0, 1, "digest1"), (0, 2, "digest2"), (0, 3, "digest3")] assert msg.stableCheckpoint == 0 assert msg.checkpoints == [cp1] # view 1 -> 2 data.view_no = 1 cp2 = Checkpoint(instId=0, viewNo=1, seqNoStart=10, seqNoEnd=20, digest='empty') data.checkpoints.add(cp2) data.stable_checkpoint = 0 data.prepared = [BatchID(1, 11, "digest11"), BatchID(1, 12, "digest12")] data.preprepared = [ BatchID(1, 11, "digest11"), BatchID(1, 12, "digest12"), BatchID(1, 13, "digest13") ] view_change_service.start_view_change() assert data.prepared == [] assert data.preprepared == [] assert data.view_no == 2 msg = get_view_change(view_change_service) assert msg.viewNo == 2 assert msg.prepared == [(0, 1, "digest1"), (0, 2, "digest2"), (1, 11, "digest11"), (1, 12, "digest12")] assert msg.preprepared == [(0, 1, "digest1"), (0, 2, "digest2"), (0, 3, "digest3"), (1, 11, "digest11"), (1, 12, "digest12"), (1, 13, "digest13")] assert msg.stableCheckpoint == 0 assert msg.checkpoints == [cp1, cp2]
def test_process_backup_catchup_msg(checkpoint_service, tconf, checkpoint): checkpoint_service._data.last_ordered_3pc = (checkpoint_service.view_no, 0) key = (1, tconf.CHK_FREQ) new_key = (key[1] + 1, key[1] + tconf.CHK_FREQ) checkpoint_service._data.stable_checkpoint = 1 checkpoint_service._stash_checkpoint( Checkpoint(1, checkpoint.viewNo, new_key[0], new_key[1], cp_digest(1, 1)), "frm") checkpoint_service._stash_checkpoint( Checkpoint(1, checkpoint.viewNo, key[0], key[1], cp_digest(1, 1)), "frm") checkpoint_service._checkpoint_state[key] = CheckpointState( key[1] - 1, ["digest"] * (tconf.CHK_FREQ - 1), None, {}, False) checkpoint_service._data.checkpoints.append(checkpoint) checkpoint_service._data.last_ordered_3pc = (checkpoint_service.view_no, key[1]) checkpoint_service.caught_up_till_3pc( checkpoint_service._data.last_ordered_3pc) assert checkpoint_service._data.low_watermark == key[1] assert not checkpoint_service._checkpoint_state assert not checkpoint_service._data.checkpoints assert checkpoint_service._data.stable_checkpoint == 0 assert key not in checkpoint_service._stashed_recvd_checkpoints[ checkpoint_service.view_no] assert new_key in checkpoint_service._stashed_recvd_checkpoints[ checkpoint_service.view_no]
def test_calc_batches_takes_prepared_only(builder): cp = Checkpoint(instId=0, viewNo=0, seqNoStart=0, seqNoEnd=0, digest=cp_digest(0, 0)) vc = ViewChange(viewNo=0, stableCheckpoint=0, prepared=[], preprepared=[(0, 1, "digest1"), (0, 2, "digest2"), (0, 3, "digest3"), (0, 4, "digest4")], checkpoints=[cp]) vcs = [vc, vc, vc, vc] assert builder.calc_batches(cp, vcs) == [] cp = Checkpoint(instId=0, viewNo=0, seqNoStart=0, seqNoEnd=0, digest=cp_digest(0, 0)) vc = ViewChange(viewNo=0, stableCheckpoint=0, prepared=[(0, 1, "digest1"), (0, 2, "digest2")], preprepared=[(0, 1, "digest1"), (0, 2, "digest2"), (0, 3, "digest3"), (0, 4, "digest4")], checkpoints=[cp]) vcs = [vc, vc, vc, vc] assert builder.calc_batches( cp, vcs) == [BatchID(0, 1, "digest1"), BatchID(0, 2, "digest2")]
def test_calc_checkpoints_quorum(builder): cp1 = Checkpoint(instId=0, viewNo=0, seqNoStart=0, seqNoEnd=0, digest='d1') cp2 = Checkpoint(instId=0, viewNo=0, seqNoStart=0, seqNoEnd=10, digest='d2') vc1 = ViewChange(viewNo=0, stableCheckpoint=0, prepared=[(1, 1, "digest1")], preprepared=[(1, 1, "digest1")], checkpoints=[cp1]) vc2 = ViewChange(viewNo=0, stableCheckpoint=0, prepared=[(1, 1, "digest1")], preprepared=[(1, 1, "digest1")], checkpoints=[cp2]) vc2_stable = ViewChange(viewNo=0, stableCheckpoint=10, prepared=[(1, 1, "digest1")], preprepared=[(1, 1, "digest1")], checkpoints=[cp2]) vcs = [vc1, vc1, vc1, vc1] assert builder.calc_checkpoint(vcs) == cp1 vcs = [vc2, vc2, vc2, vc2] assert builder.calc_checkpoint(vcs) == cp2 vcs = [vc2_stable, vc2_stable, vc2_stable, vc2_stable] assert builder.calc_checkpoint(vcs) == cp2 vcs = [vc2, vc1, vc1, vc1] assert builder.calc_checkpoint(vcs) == cp1 vcs = [vc2, vc2, vc1, vc1] assert builder.calc_checkpoint(vcs) == cp2 vcs = [vc2, vc2, vc2, vc1] assert builder.calc_checkpoint(vcs) == cp2 vcs = [vc2_stable, vc1, vc1, vc1] assert builder.calc_checkpoint(vcs) == cp1 vcs = [vc2_stable, vc2_stable, vc1, vc1] assert builder.calc_checkpoint(vcs) == cp2 vcs = [vc2_stable, vc2_stable, vc2_stable, vc1] assert builder.calc_checkpoint(vcs) == cp2 vcs = [vc2_stable, vc2, vc2, vc2] assert builder.calc_checkpoint(vcs) == cp2 vcs = [vc2_stable, vc2_stable, vc2, vc2] assert builder.calc_checkpoint(vcs) == cp2 vcs = [vc2_stable, vc2_stable, vc2_stable, vc2] assert builder.calc_checkpoint(vcs) == cp2
def test_process_checkpoint(checkpoint_service, checkpoint, pre_prepare, tconf, ordered, validators, is_master): global caught_msg caught_msg = None checkpoint_service._bus.subscribe(Cleanup, catch_msg) quorum = checkpoint_service._data.quorums.checkpoint.value n = len(validators) assert quorum == n - getMaxFailures(n) - 1 senders = ["sender{}".format(i) for i in range(quorum + 1)] key = (1, tconf.CHK_FREQ) old_key = (-1, 0) checkpoint_service._stash_checkpoint( Checkpoint(1, checkpoint.viewNo, 1, 1, "1"), "frm") checkpoint_service._stash_checkpoint( Checkpoint(1, checkpoint.viewNo + 1, 1, 1, "1"), "frm") checkpoint_service._checkpoint_state[old_key] = CheckpointState( 1, ["digest"] * (tconf.CHK_FREQ - 1), None, {}, False) checkpoint_service._checkpoint_state[key] = CheckpointState( key[1] - 1, ["digest"] * (tconf.CHK_FREQ - 1), None, {}, False) pre_prepare.ppSeqNo = key[1] ordered.ppSeqNo = pre_prepare.ppSeqNo checkpoint_service._data.preprepared.append(pre_prepare) checkpoint_service.process_ordered(ordered) _check_checkpoint(checkpoint_service, key[0], key[1], pre_prepare, check_shared_data=True) state = updateNamedTuple(checkpoint_service._checkpoint_state[key], digest=checkpoint.digest) checkpoint_service._checkpoint_state[key] = state for sender in senders[:quorum - 1]: assert checkpoint_service.process_checkpoint(checkpoint, sender) assert checkpoint_service._checkpoint_state[key].receivedDigests[ sender] == checkpoint.digest assert not checkpoint_service._checkpoint_state[key].isStable # send the last checkpoint to stable it assert checkpoint_service.process_checkpoint(checkpoint, senders[quorum - 1]) assert checkpoint_service._checkpoint_state[key].isStable # check _remove_stashed_checkpoints() assert checkpoint.viewNo not in checkpoint_service._stashed_recvd_checkpoints assert checkpoint.viewNo + 1 in checkpoint_service._stashed_recvd_checkpoints # check watermarks assert checkpoint_service._data.low_watermark == checkpoint.seqNoEnd # check that a Cleanup msg has been sent assert isinstance(caught_msg, Cleanup) assert caught_msg.cleanup_till_3pc == (checkpoint.viewNo, checkpoint.seqNoEnd) # check that old checkpoint_states has been removed assert old_key not in checkpoint_service._checkpoint_state
def test_view_change_data_multiple_respects_checkpoint(view_change_service, data): # view 0 -> 1 data.view_no = 0 cp1 = Checkpoint(instId=0, viewNo=0, seqNoStart=0, seqNoEnd=10, digest=cp_digest(10)) data.checkpoints.add(cp1) data.stable_checkpoint = 0 data.prepared = [BatchID(0, 1, "digest1"), BatchID(0, 2, "digest2")] data.preprepared = [ BatchID(0, 1, "digest1"), BatchID(0, 2, "digest2"), BatchID(0, 3, "digest3") ] view_change_service._bus.send(NeedViewChange()) assert data.view_no == 1 msg = get_view_change(view_change_service) assert msg.viewNo == 1 assert msg.prepared == [(0, 1, "digest1"), (0, 2, "digest2")] assert msg.preprepared == [(0, 1, "digest1"), (0, 2, "digest2"), (0, 3, "digest3")] assert msg.stableCheckpoint == 0 assert msg.checkpoints == [data.initial_checkpoint, cp1] # view 1 -> 2 data.view_no = 1 cp2 = Checkpoint(instId=0, viewNo=1, seqNoStart=0, seqNoEnd=20, digest=cp_digest(20)) data.checkpoints.add(cp2) data.stable_checkpoint = 10 # Here we simulate checkpoint stabilization logic of CheckpointService, which # clears all checkpoints below stabilized one data.checkpoints.remove(data.initial_checkpoint) data.prepared = [BatchID(1, 11, "digest11"), BatchID(1, 12, "digest12")] data.preprepared = [ BatchID(1, 11, "digest11"), BatchID(1, 12, "digest12"), BatchID(1, 13, "digest13") ] view_change_service._bus.send(NeedViewChange()) assert data.view_no == 2 msg = get_view_change(view_change_service) assert msg.viewNo == 2 assert msg.prepared == [(1, 11, "digest11"), (1, 12, "digest12")] assert msg.preprepared == [(1, 11, "digest11"), (1, 12, "digest12"), (1, 13, "digest13")] assert msg.stableCheckpoint == 10 assert msg.checkpoints == [cp1, cp2]
def test_calc_checkpoints_selects_max(builder): cp1 = Checkpoint(instId=0, viewNo=0, seqNoStart=0, seqNoEnd=0, digest=cp_digest(0)) cp2 = Checkpoint(instId=0, viewNo=0, seqNoStart=0, seqNoEnd=10, digest=cp_digest(10)) cp3 = Checkpoint(instId=0, viewNo=0, seqNoStart=0, seqNoEnd=20, digest=cp_digest(20)) vc1 = ViewChange(viewNo=2, stableCheckpoint=0, prepared=[BatchID(1, 1, 1, "digest1")], preprepared=[BatchID(1, 1, 1, "digest1")], checkpoints=[cp1]) vc2_not_stable = ViewChange(viewNo=2, stableCheckpoint=0, prepared=[BatchID(1, 1, 1, "digest1")], preprepared=[BatchID(1, 1, 1, "digest1")], checkpoints=[cp2]) vc2_stable = ViewChange(viewNo=2, stableCheckpoint=10, prepared=[BatchID(1, 1, 1, "digest1")], preprepared=[BatchID(1, 1, 1, "digest1")], checkpoints=[cp2]) vc3_not_stable = ViewChange(viewNo=2, stableCheckpoint=10, prepared=[BatchID(1, 1, 1, "digest1")], preprepared=[BatchID(1, 1, 1, "digest1")], checkpoints=[cp3]) vc3_stable = ViewChange(viewNo=2, stableCheckpoint=10, prepared=[BatchID(1, 1, 1, "digest1")], preprepared=[BatchID(1, 1, 1, "digest1")], checkpoints=[cp3]) for vc3 in (vc3_not_stable, vc3_stable): for vc2 in (vc2_not_stable, vc2_stable): vcs = [vc1, vc2, vc3, vc3] assert builder.calc_checkpoint(vcs) == cp3 vcs = [vc1, vc3, vc3, vc3] assert builder.calc_checkpoint(vcs) == cp3 vcs = [vc2, vc3, vc3, vc3] assert builder.calc_checkpoint(vcs) == cp3 vcs = [vc2, vc2, vc3, vc3] assert builder.calc_checkpoint(vcs) == cp3 vcs = [vc1, vc1, vc3, vc3] assert builder.calc_checkpoint(vcs) == cp3 vcs = [vc1, vc1, vc1, vc3] assert builder.calc_checkpoint(vcs) == cp1 vcs = [vc1, vc1, vc2, vc2] assert builder.calc_checkpoint(vcs) == cp2 vcs = [vc2, vc2, vc2, vc3] assert builder.calc_checkpoint(vcs) == cp2
def test_calc_checkpoints_digest(builder): d1 = cp_digest(0) d2 = cp_digest(10) cp1_d1 = Checkpoint(instId=0, viewNo=0, seqNoStart=0, seqNoEnd=0, digest=d1) cp2_d2 = Checkpoint(instId=0, viewNo=0, seqNoStart=0, seqNoEnd=10, digest=d2) cp2_d1 = Checkpoint(instId=0, viewNo=0, seqNoStart=0, seqNoEnd=10, digest=d1) vc1_d1 = ViewChange(viewNo=0, stableCheckpoint=0, prepared=[(1, 1, "digest1")], preprepared=[(1, 1, "digest1")], checkpoints=[cp1_d1]) vc2_d2 = ViewChange(viewNo=0, stableCheckpoint=0, prepared=[(1, 1, "digest1")], preprepared=[(1, 1, "digest1")], checkpoints=[cp2_d2]) vc2_d1 = ViewChange(viewNo=0, stableCheckpoint=0, prepared=[(1, 1, "digest1")], preprepared=[(1, 1, "digest1")], checkpoints=[cp2_d1]) vcs = [vc1_d1, vc1_d1, vc2_d1, vc2_d2] assert builder.calc_checkpoint(vcs) == cp1_d1 vcs = [vc1_d1, vc2_d1, vc2_d2, vc2_d2] assert builder.calc_checkpoint(vcs) == cp2_d2 vcs = [vc1_d1, vc2_d2, vc2_d1, vc2_d1] assert builder.calc_checkpoint(vcs) == cp2_d1 # Here we have 2 nodes malicious (f=1), but calc_checkpoint returns a value depending on the order # Is it OK, or calc_checkpoint should return None (indicating that there is no valid quorum)? vcs = [vc2_d1, vc2_d1, vc2_d2, vc2_d2] assert builder.calc_checkpoint(vcs) == cp2_d1 vcs = [vc2_d2, vc2_d2, vc2_d1, vc2_d1] assert builder.calc_checkpoint(vcs) == cp2_d2
def some_checkpoint(random: SimRandom, view_no: int, pp_seq_no: int) -> Checkpoint: return Checkpoint(instId=0, viewNo=view_no, seqNoStart=pp_seq_no, seqNoEnd=pp_seq_no, digest=random.string(40))
def __init__(self, name: str, validators: List[str], primary_name: str, timer: TimerService, bus: InternalBus, network: ExternalBus, write_manager: WriteRequestManager, bls_bft_replica: BlsBftReplica=None): self._data = ConsensusSharedData(name, validators, 0) self._data.primary_name = primary_name config = getConfig() stasher = StashingRouter(config.REPLICA_STASH_LIMIT, buses=[bus, network]) self._orderer = OrderingService(data=self._data, timer=timer, bus=bus, network=network, write_manager=write_manager, bls_bft_replica=bls_bft_replica, freshness_checker=FreshnessChecker( freshness_timeout=config.STATE_FRESHNESS_UPDATE_INTERVAL), stasher=stasher) self._checkpointer = CheckpointService(self._data, bus, network, stasher, write_manager.database_manager) self._view_changer = ViewChangeService(self._data, timer, bus, network, stasher) # TODO: This is just for testing purposes only self._data.checkpoints.append( Checkpoint(instId=0, viewNo=0, seqNoStart=0, seqNoEnd=0, digest='4F7BsTMVPKFshM1MwLf6y23cid6fL3xMpazVoF9krzUw'))
def _do_checkpoint(self, state, s, e, ledger_id, view_no, audit_txn_root_hash): # TODO CheckpointState/Checkpoint is not a namedtuple anymore # 1. check if updateNamedTuple works for the new message type # 2. choose another name # TODO: This is hack of hacks, should be removed when refactoring is complete if not self.is_master and audit_txn_root_hash is None: audit_txn_root_hash = "7RJ5bkAKRy2CCvarRij2jiHC16SVPjHcrpVdNsboiQGv" state = updateNamedTuple(state, digest=audit_txn_root_hash, digests=[]) self._checkpoint_state[s, e] = state self._logger.info( "{} sending Checkpoint {} view {} checkpointState digest {}. Ledger {} " "txn root hash {}. Committed state root hash {} Uncommitted state root hash {}" .format( self, (s, e), view_no, state.digest, ledger_id, self._db_manager.get_txn_root_hash(ledger_id), self._db_manager.get_state_root_hash(ledger_id, committed=True), self._db_manager.get_state_root_hash(ledger_id, committed=False))) checkpoint = Checkpoint(self._data.inst_id, view_no, s, e, state.digest) self._network.send(checkpoint) self._data.checkpoints.append(checkpoint)
def test_view_change_data(view_change_service, data): data.view_no = 1 data.checkpoints.clear() cp = Checkpoint(instId=0, viewNo=1, seqNoStart=0, seqNoEnd=10, digest=cp_digest(10)) data.checkpoints.add(cp) data.stable_checkpoint = 10 data.prepared = [BatchID(0, 0, 1, "digest1"), BatchID(0, 0, 2, "digest2")] data.preprepared = [ BatchID(0, 0, 1, "digest1"), BatchID(0, 0, 2, "digest2"), BatchID(0, 0, 3, "digest3") ] view_change_service._bus.send(NeedViewChange()) assert data.view_no == 2 msg = get_view_change(view_change_service) assert msg.viewNo == 2 assert msg.prepared == [(0, 0, 1, "digest1"), (0, 0, 2, "digest2")] assert msg.preprepared == [(0, 0, 1, "digest1"), (0, 0, 2, "digest2"), (0, 0, 3, "digest3")] assert msg.stableCheckpoint == 10 assert msg.checkpoints == [cp]
def _do_checkpoint(self, pp_seq_no, view_no, audit_txn_root_hash): self._logger.info("{} sending Checkpoint {} view {} audit txn root hash {}". format(self, pp_seq_no, view_no, audit_txn_root_hash)) checkpoint = Checkpoint(self._data.inst_id, view_no, 0, pp_seq_no, audit_txn_root_hash) self._network.send(checkpoint) self._data.checkpoints.add(checkpoint)
def test_discard_checkpoint_msg_for_stable_checkpoint(chkFreqPatched, tconf, looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, reqs_for_checkpoint): sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, reqs_for_checkpoint) next_checkpoint = tconf.CHK_FREQ for inst_id in txnPoolNodeSet[0].replicas.keys(): looper.run( eventually(check_for_instance, txnPoolNodeSet, inst_id, check_stable_checkpoint, next_checkpoint, retryWait=1)) node1 = txnPoolNodeSet[0] rep1 = node1.replicas[0] # TODO: Use old checkpoint message when we retain stabilized checkpoint # oldChkpointMsg = rep1._consensus_data.checkpoints[0] oldChkpointMsg = Checkpoint( instId=0, viewNo=0, seqNoStart=0, seqNoEnd=next_checkpoint, digest=node1.db_manager.get_txn_root_hash(AUDIT_LEDGER_ID)) rep1.send(oldChkpointMsg) recvReplicas = [n.replicas[0].stasher for n in txnPoolNodeSet[1:]] looper.run( eventually(checkDiscardMsg, recvReplicas, oldChkpointMsg, ALREADY_STABLE, retryWait=1))
def test_calc_batches_must_be_in_pre_prepare(builder): cp = Checkpoint(instId=0, viewNo=0, seqNoStart=0, seqNoEnd=0, digest=cp_digest(0, 0)) vc = ViewChange(viewNo=0, stableCheckpoint=0, prepared=[(0, 1, "digest1"), (0, 2, "digest2")], preprepared=[(0, 1, "digest1")], checkpoints=[cp]) vcs = [vc, vc, vc, vc] # all nodes are malicious here since all added (0, 2) into prepared without adding to pre-prepared # so, None here means we can not calculate NewView reliably assert builder.calc_batches(cp, vcs) is None vc1 = ViewChange(viewNo=0, stableCheckpoint=0, prepared=[(0, 1, "digest1"), (0, 2, "digest2")], preprepared=[(0, 1, "digest1")], checkpoints=[cp]) vc2 = ViewChange(viewNo=0, stableCheckpoint=0, prepared=[(0, 1, "digest1")], preprepared=[(0, 1, "digest1")], checkpoints=[cp]) vcs = [vc1, vc2, vc2, vc2] assert builder.calc_batches(cp, vcs) == [BatchID(0, 1, "digest1")]
def test_calc_batches_takes_next_view_one_prepared_if_weak_quorum_of_preprepared( builder): cp = Checkpoint(instId=0, viewNo=0, seqNoStart=0, seqNoEnd=0, digest=cp_digest(0, 0)) vc1 = ViewChange(viewNo=0, stableCheckpoint=0, prepared=[(0, 1, "digest1"), (1, 2, "digest2")], preprepared=[(0, 1, "digest1"), (1, 2, "digest2")], checkpoints=[cp]) vc2 = ViewChange(viewNo=0, stableCheckpoint=0, prepared=[(0, 1, "digest1")], preprepared=[(0, 1, "digest1"), (1, 2, "digest2")], checkpoints=[cp]) vc3 = ViewChange(viewNo=0, stableCheckpoint=0, prepared=[(0, 1, "digest1")], preprepared=[(0, 1, "digest1")], checkpoints=[cp]) vc4 = ViewChange(viewNo=0, stableCheckpoint=0, prepared=[(0, 1, "digest1")], preprepared=[(0, 1, "digest1")], checkpoints=[cp]) vcs = [vc1, vc2, vc3, vc4] assert builder.calc_batches( cp, vcs) == [BatchID(0, 1, "digest1"), (1, 2, "digest2")]
def __init__(self, name: str, validators: List[str], primary_name: str, timer: TimerService, bus: InternalBus, network: ExternalBus, write_manager: WriteRequestManager = None, bls_bft_replica: BlsBftReplica = None): self._data = ConsensusSharedData(name, validators, 0) self._data.primary_name = primary_name self._orderer = OrderingService(data=self._data, timer=timer, bus=bus, network=network, write_manager=write_manager, bls_bft_replica=bls_bft_replica) self._checkpointer = CheckpointService(self._data, bus, network) self._view_changer = ViewChangeService(self._data, timer, bus, network) # TODO: This is just for testing purposes only self._data.checkpoints.append( Checkpoint(instId=0, viewNo=0, seqNoStart=0, seqNoEnd=0, digest='empty'))
def test_calc_batches_takes_quorum_of_prepared(builder): cp = Checkpoint(instId=0, viewNo=0, seqNoStart=0, seqNoEnd=0, digest=cp_digest(0)) vc1 = ViewChange(viewNo=1, stableCheckpoint=0, prepared=[BatchID(0, 0, 1, "digest2")], preprepared=[BatchID(0, 0, 1, "digest2")], checkpoints=[cp]) vc2 = ViewChange(viewNo=1, stableCheckpoint=0, prepared=[BatchID(0, 0, 1, "digest1")], preprepared=[BatchID(0, 0, 1, "digest1")], checkpoints=[cp]) vc3 = ViewChange(viewNo=1, stableCheckpoint=0, prepared=[], preprepared=[BatchID(0, 0, 1, "digest1")], checkpoints=[cp]) vcs = [vc1, vc2, vc2, vc2] assert builder.calc_batches(cp, vcs) == [BatchID(0, 0, 1, "digest1")] vcs = [vc3, vc2, vc2, vc2] assert builder.calc_batches(cp, vcs) == [BatchID(0, 0, 1, "digest1")] vcs = [vc3, vc3, vc3, vc3] assert builder.calc_batches(cp, vcs) == [] vcs = [vc1, vc1, vc2, vc2] assert builder.calc_batches(cp, vcs) is None # since we have enough pre-prepares vcs = [vc2, vc3, vc3, vc3] assert builder.calc_batches(cp, vcs) == [BatchID(0, 0, 1, "digest1")] vcs = [vc2, vc2, vc3, vc3] assert builder.calc_batches(cp, vcs) == [BatchID(0, 0, 1, "digest1")]
def test_calc_batches_empty(builder): cp = Checkpoint(instId=0, viewNo=0, seqNoStart=0, seqNoEnd=0, digest=cp_digest(0, 0)) vcs = [ ViewChange(viewNo=0, stableCheckpoint=0, prepared=[], preprepared=[], checkpoints=[cp]), ViewChange(viewNo=0, stableCheckpoint=0, prepared=[], preprepared=[], checkpoints=[cp]), ViewChange(viewNo=0, stableCheckpoint=0, prepared=[], preprepared=[], checkpoints=[cp]), ViewChange(viewNo=0, stableCheckpoint=0, prepared=[], preprepared=[], checkpoints=[cp]), ] assert [] == builder.calc_batches(cp, vcs)
def test_calc_batches_takes_prepared_if_preprepared_in_next_view(builder): cp = Checkpoint(instId=0, viewNo=0, seqNoStart=0, seqNoEnd=0, digest='empty') vc1 = ViewChange(viewNo=0, stableCheckpoint=0, prepared=[(1, 1, "digest2")], preprepared=[(0, 1, "digest1"), (2, 1, "digest2")], checkpoints=[cp]) vc2 = ViewChange(viewNo=0, stableCheckpoint=0, prepared=[(0, 1, "digest1")], preprepared=[(0, 1, "digest1"), (2, 1, "digest2")], checkpoints=[cp]) vc3 = ViewChange(viewNo=0, stableCheckpoint=0, prepared=[(0, 1, "digest1")], preprepared=[(0, 1, "digest1")], checkpoints=[cp]) vc4 = ViewChange(viewNo=0, stableCheckpoint=0, prepared=[(0, 1, "digest1")], preprepared=[(0, 1, "digest1")], checkpoints=[cp]) vcs = [vc1, vc2, vc3, vc4] assert builder.calc_batches(cp, vcs) == [BatchID(1, 1, "digest2")]
def test_calc_batches_takes_prepared_with_same_batchid_only(builder): cp = Checkpoint(instId=0, viewNo=0, seqNoStart=0, seqNoEnd=0, digest=cp_digest(0, 0)) vc1 = ViewChange(viewNo=0, stableCheckpoint=0, prepared=[(1, 1, "digest1")], preprepared=[(1, 1, "digest1")], checkpoints=[cp]) vc2 = ViewChange(viewNo=0, stableCheckpoint=0, prepared=[(1, 1, "digest1")], preprepared=[(1, 1, "digest1")], checkpoints=[cp]) vc3 = ViewChange(viewNo=0, stableCheckpoint=0, prepared=[(1, 1, "digest2")], preprepared=[(1, 1, "digest2")], checkpoints=[cp]) vc4 = ViewChange(viewNo=0, stableCheckpoint=0, prepared=[], preprepared=[(1, 1, "digest1")], checkpoints=[cp]) vcs = [vc1, vc2, vc3, vc4] assert builder.calc_batches(cp, vcs) == [BatchID(1, 1, "digest1")]
def some_checkpoint(random: SimRandom, view_no: int, pp_seq_no: int) -> Checkpoint: return Checkpoint(instId=0, viewNo=view_no, seqNoStart=pp_seq_no, seqNoEnd=pp_seq_no, digest=base58.b58encode(random.string(32)).decode())
def test_view_change_data(view_change_service, data): data.view_no = 1 cp = Checkpoint(instId=0, viewNo=1, seqNoStart=0, seqNoEnd=10, digest='empty') data.checkpoints.add(cp) data.stable_checkpoint = 10 data.prepared = [BatchID(0, 1, "digest1"), BatchID(0, 2, "digest2")] data.preprepared = [ BatchID(0, 1, "digest1"), BatchID(0, 2, "digest2"), BatchID(0, 3, "digest3") ] view_change_service.start_view_change() assert data.prepared == [] assert data.preprepared == [] assert data.view_no == 2 msg = get_view_change(view_change_service) assert msg.viewNo == 2 assert msg.prepared == [(0, 1, "digest1"), (0, 2, "digest2")] assert msg.preprepared == [(0, 1, "digest1"), (0, 2, "digest2"), (0, 3, "digest3")] assert msg.stableCheckpoint == 10 assert msg.checkpoints == [cp]
def checkpoint(view_no, inst_id, seq_no_start, seq_no_end): return Checkpoint(instId=inst_id, viewNo=view_no, seqNoStart=seq_no_start, seqNoEnd=seq_no_end, digest='digest-{}-{}'.format(str(seq_no_start), str(seq_no_end)))
def _mark_checkpoint_stable(self, pp_seq_no): self._data.stable_checkpoint = pp_seq_no stable_checkpoints = self._data.checkpoints.irange_key( min_key=pp_seq_no, max_key=pp_seq_no) if len(list(stable_checkpoints)) == 0: # TODO: Is it okay to get view_no like this? view_no = self._data.last_ordered_3pc[0] checkpoint = Checkpoint(instId=self._data.inst_id, viewNo=view_no, seqNoStart=0, seqNoEnd=pp_seq_no, digest=self._audit_txn_root_hash( view_no, pp_seq_no)) self._data.checkpoints.add(checkpoint) for cp in self._data.checkpoints.copy(): if cp.seqNoEnd < pp_seq_no: self._logger.trace("{} removing previous checkpoint {}".format( self, cp)) self._data.checkpoints.remove(cp) self.set_watermarks(low_watermark=pp_seq_no) self._remove_received_checkpoints(till_3pc_key=(self.view_no, pp_seq_no)) self._bus.send(CheckpointStabilized( (self.view_no, pp_seq_no))) # call OrderingService.gc() self._logger.info("{} marked stable checkpoint {}".format( self, pp_seq_no))
def checkpoint(ordered, tconf): start = ordered.ppSeqNo % tconf.CHK_FREQ return Checkpoint(instId=ordered.instId, viewNo=ordered.viewNo, seqNoStart=0, seqNoEnd=start + tconf.CHK_FREQ - 1, digest=cp_digest(start + tconf.CHK_FREQ - 1))
def test_checkpoints_removed_on_backup_primary_replica_after_catchup( chkFreqPatched, txnPoolNodeSet, view_setup, clear_checkpoints): replica = getPrimaryReplica(txnPoolNodeSet, 1) others = set(getAllReplicas(txnPoolNodeSet, 1)) - {replica} node = replica.node node.master_replica.last_ordered_3pc = (2, 12) replica._checkpointer._checkpoint_state[(11, 15)] = CheckpointState( seqNo=15, digests=[], digest=cp_digest(11, 15), receivedDigests={r.name: cp_digest(11, 15) for r in others}, isStable=True) replica._checkpointer._checkpoint_state[(16, 20)] = CheckpointState( seqNo=19, digests=['digest-16', 'digest-17', 'digest-18', 'digest-19'], digest=None, receivedDigests={}, isStable=False) replica._checkpointer._stashed_recvd_checkpoints[2] = {} replica._checkpointer._stashed_recvd_checkpoints[2][(16, 20)] = {} replica._checkpointer._stashed_recvd_checkpoints[2][(16, 20)][next(iter(others)).name] = \ Checkpoint(instId=1, viewNo=2, seqNoStart=16, seqNoEnd=20, digest=cp_digest(16, 20)) # Simulate catch-up completion node.ledgerManager.last_caught_up_3PC = (2, 20) audit_ledger = node.getLedger(AUDIT_LEDGER_ID) txn_with_last_seq_no = { 'txn': { 'data': { AUDIT_TXN_VIEW_NO: 2, AUDIT_TXN_PP_SEQ_NO: 20, AUDIT_TXN_PRIMARIES: ['Gamma', 'Delta'] } } } audit_ledger.get_last_committed_txn = lambda *args: txn_with_last_seq_no node.allLedgersCaughtUp() assert len(replica._checkpointer._checkpoint_state) == 0 assert (11, 15) not in replica._checkpointer._checkpoint_state assert (16, 20) not in replica._checkpointer._checkpoint_state assert len(replica._checkpointer._stashed_recvd_checkpoints) == 1 assert 2 in replica._checkpointer._stashed_recvd_checkpoints assert len(replica._checkpointer._stashed_recvd_checkpoints[2]) == 1 assert (16, 20) in replica._checkpointer._stashed_recvd_checkpoints[2] assert len(replica._checkpointer._stashed_recvd_checkpoints[2][(16, 20)]) == 1
def create_checkpoints(view_no): return [ Checkpoint(instId=0, viewNo=view_no, seqNoStart=0, seqNoEnd=200, digest=cp_digest(0, 200)) ]