def test_calc_batches_takes_prepared_only(builder):
cp = Checkpoint(instId=0,
viewNo=0,
seqNoStart=0,
seqNoEnd=0,
digest=cp_digest(0, 0))
vc = ViewChange(viewNo=0,
stableCheckpoint=0,
prepared=[],
preprepared=[(0, 1, "digest1"), (0, 2, "digest2"),
(0, 3, "digest3"), (0, 4, "digest4")],
checkpoints=[cp])
vcs = [vc, vc, vc, vc]
assert builder.calc_batches(cp, vcs) == []
cp = Checkpoint(instId=0,
viewNo=0,
seqNoStart=0,
seqNoEnd=0,
digest=cp_digest(0, 0))
vc = ViewChange(viewNo=0,
stableCheckpoint=0,
prepared=[(0, 1, "digest1"), (0, 2, "digest2")],
preprepared=[(0, 1, "digest1"), (0, 2, "digest2"),
(0, 3, "digest3"), (0, 4, "digest4")],
checkpoints=[cp])
vcs = [vc, vc, vc, vc]
assert builder.calc_batches(
cp, vcs) == [BatchID(0, 1, "digest1"),
BatchID(0, 2, "digest2")]
def test_process_backup_catchup_msg(checkpoint_service, tconf, checkpoint):
checkpoint_service._data.last_ordered_3pc = (checkpoint_service.view_no, 0)
key = (1, tconf.CHK_FREQ)
new_key = (key[1] + 1, key[1] + tconf.CHK_FREQ)
checkpoint_service._data.stable_checkpoint = 1
checkpoint_service._stash_checkpoint(
Checkpoint(1, checkpoint.viewNo, new_key[0], new_key[1],
cp_digest(1, 1)), "frm")
checkpoint_service._stash_checkpoint(
Checkpoint(1, checkpoint.viewNo, key[0], key[1], cp_digest(1, 1)),
"frm")
checkpoint_service._checkpoint_state[key] = CheckpointState(
key[1] - 1, ["digest"] * (tconf.CHK_FREQ - 1), None, {}, False)
checkpoint_service._data.checkpoints.append(checkpoint)
checkpoint_service._data.last_ordered_3pc = (checkpoint_service.view_no,
key[1])
checkpoint_service.caught_up_till_3pc(
checkpoint_service._data.last_ordered_3pc)
assert checkpoint_service._data.low_watermark == key[1]
assert not checkpoint_service._checkpoint_state
assert not checkpoint_service._data.checkpoints
assert checkpoint_service._data.stable_checkpoint == 0
assert key not in checkpoint_service._stashed_recvd_checkpoints[
checkpoint_service.view_no]
assert new_key in checkpoint_service._stashed_recvd_checkpoints[
checkpoint_service.view_no]
def test_calc_batches_respects_checkpoint(builder):
cp = Checkpoint(instId=0,
viewNo=0,
seqNoStart=0,
seqNoEnd=10,
digest=cp_digest(0, 0))
vc = ViewChange(viewNo=0,
stableCheckpoint=0,
prepared=[(0, 1, "digest1"), (0, 2, "digest2")],
preprepared=[(0, 1, "digest1"), (0, 2, "digest2")],
checkpoints=[cp])
vcs = [vc, vc, vc, vc]
assert builder.calc_batches(cp, vcs) == []
cp = Checkpoint(instId=0,
viewNo=0,
seqNoStart=0,
seqNoEnd=10,
digest=cp_digest(0, 0))
vc = ViewChange(viewNo=0,
stableCheckpoint=0,
prepared=[(0, 10, "digest10"), (0, 11, "digest11"),
(1, 12, "digest12")],
preprepared=[(0, 10, "digest10"), (0, 11, "digest11"),
(1, 12, "digest12")],
checkpoints=[cp])
vcs = [vc, vc, vc, vc]
assert builder.calc_batches(
cp, vcs) == [BatchID(0, 11, "digest11"),
BatchID(1, 12, "digest12")]
def test_view_change_data_multiple(view_change_service, data):
# view 0 -> 1
data.view_no = 0
cp1 = Checkpoint(instId=0,
viewNo=0,
seqNoStart=0,
seqNoEnd=10,
digest=cp_digest(10))
data.checkpoints.add(cp1)
data.stable_checkpoint = 0
data.prepared = [BatchID(0, 0, 1, "digest1"), BatchID(0, 0, 2, "digest2")]
data.preprepared = [
BatchID(0, 0, 1, "digest1"),
BatchID(0, 0, 2, "digest2"),
BatchID(0, 0, 3, "digest3")
]
view_change_service._bus.send(NeedViewChange())
assert data.view_no == 1
msg = get_view_change(view_change_service)
assert msg.viewNo == 1
assert msg.prepared == [(0, 0, 1, "digest1"), (0, 0, 2, "digest2")]
assert msg.preprepared == [(0, 0, 1, "digest1"), (0, 0, 2, "digest2"),
(0, 0, 3, "digest3")]
assert msg.stableCheckpoint == 0
assert msg.checkpoints == [data.initial_checkpoint, cp1]
# view 1 -> 2
data.view_no = 1
cp2 = Checkpoint(instId=0,
viewNo=1,
seqNoStart=0,
seqNoEnd=20,
digest=cp_digest(20))
data.checkpoints.add(cp2)
data.stable_checkpoint = 0
data.prepared = [
BatchID(1, 1, 11, "digest11"),
BatchID(1, 1, 12, "digest12")
]
data.preprepared = [
BatchID(1, 1, 11, "digest11"),
BatchID(1, 1, 12, "digest12"),
BatchID(1, 1, 13, "digest13")
]
view_change_service._bus.send(NeedViewChange())
assert data.view_no == 2
msg = get_view_change(view_change_service)
assert msg.viewNo == 2
assert msg.prepared == [(0, 0, 1, "digest1"), (0, 0, 2, "digest2"),
(1, 1, 11, "digest11"), (1, 1, 12, "digest12")]
assert msg.preprepared == [(0, 0, 1, "digest1"), (0, 0, 2, "digest2"),
(0, 0, 3, "digest3"), (1, 1, 11, "digest11"),
(1, 1, 12, "digest12"), (1, 1, 13, "digest13")]
assert msg.stableCheckpoint == 0
assert msg.checkpoints == [data.initial_checkpoint, cp1, cp2]
def test_checkpoints_removed_on_backup_primary_replica_after_catchup(
chkFreqPatched, txnPoolNodeSet, view_setup, clear_checkpoints):
replica = getPrimaryReplica(txnPoolNodeSet, 1)
others = set(getAllReplicas(txnPoolNodeSet, 1)) - {replica}
node = replica.node
node.master_replica.last_ordered_3pc = (2, 12)
replica._checkpointer._checkpoint_state[(11, 15)] = CheckpointState(
seqNo=15,
digests=[],
digest=cp_digest(11, 15),
receivedDigests={r.name: cp_digest(11, 15)
for r in others},
isStable=True)
replica._checkpointer._checkpoint_state[(16, 20)] = CheckpointState(
seqNo=19,
digests=['digest-16', 'digest-17', 'digest-18', 'digest-19'],
digest=None,
receivedDigests={},
isStable=False)
replica._checkpointer._stashed_recvd_checkpoints[2] = {}
replica._checkpointer._stashed_recvd_checkpoints[2][(16, 20)] = {}
replica._checkpointer._stashed_recvd_checkpoints[2][(16, 20)][next(iter(others)).name] = \
Checkpoint(instId=1,
viewNo=2,
seqNoStart=16,
seqNoEnd=20,
digest=cp_digest(16, 20))
# Simulate catch-up completion
node.ledgerManager.last_caught_up_3PC = (2, 20)
audit_ledger = node.getLedger(AUDIT_LEDGER_ID)
txn_with_last_seq_no = {
'txn': {
'data': {
AUDIT_TXN_VIEW_NO: 2,
AUDIT_TXN_PP_SEQ_NO: 20,
AUDIT_TXN_PRIMARIES: ['Gamma', 'Delta']
}
}
}
audit_ledger.get_last_committed_txn = lambda *args: txn_with_last_seq_no
node.allLedgersCaughtUp()
assert len(replica._checkpointer._checkpoint_state) == 0
assert (11, 15) not in replica._checkpointer._checkpoint_state
assert (16, 20) not in replica._checkpointer._checkpoint_state
assert len(replica._checkpointer._stashed_recvd_checkpoints) == 1
assert 2 in replica._checkpointer._stashed_recvd_checkpoints
assert len(replica._checkpointer._stashed_recvd_checkpoints[2]) == 1
assert (16, 20) in replica._checkpointer._stashed_recvd_checkpoints[2]
assert len(replica._checkpointer._stashed_recvd_checkpoints[2][(16,
20)]) == 1
def test_view_change_data_multiple_respects_checkpoint(view_change_service,
data):
# view 0 -> 1
data.view_no = 0
cp1 = Checkpoint(instId=0,
viewNo=0,
seqNoStart=0,
seqNoEnd=10,
digest=cp_digest(10))
data.checkpoints.add(cp1)
data.stable_checkpoint = 0
data.prepared = [BatchID(0, 1, "digest1"), BatchID(0, 2, "digest2")]
data.preprepared = [
BatchID(0, 1, "digest1"),
BatchID(0, 2, "digest2"),
BatchID(0, 3, "digest3")
]
view_change_service._bus.send(NeedViewChange())
assert data.view_no == 1
msg = get_view_change(view_change_service)
assert msg.viewNo == 1
assert msg.prepared == [(0, 1, "digest1"), (0, 2, "digest2")]
assert msg.preprepared == [(0, 1, "digest1"), (0, 2, "digest2"),
(0, 3, "digest3")]
assert msg.stableCheckpoint == 0
assert msg.checkpoints == [data.initial_checkpoint, cp1]
# view 1 -> 2
data.view_no = 1
cp2 = Checkpoint(instId=0,
viewNo=1,
seqNoStart=0,
seqNoEnd=20,
digest=cp_digest(20))
data.checkpoints.add(cp2)
data.stable_checkpoint = 10
# Here we simulate checkpoint stabilization logic of CheckpointService, which
# clears all checkpoints below stabilized one
data.checkpoints.remove(data.initial_checkpoint)
data.prepared = [BatchID(1, 11, "digest11"), BatchID(1, 12, "digest12")]
data.preprepared = [
BatchID(1, 11, "digest11"),
BatchID(1, 12, "digest12"),
BatchID(1, 13, "digest13")
]
view_change_service._bus.send(NeedViewChange())
assert data.view_no == 2
msg = get_view_change(view_change_service)
assert msg.viewNo == 2
assert msg.prepared == [(1, 11, "digest11"), (1, 12, "digest12")]
assert msg.preprepared == [(1, 11, "digest11"), (1, 12, "digest12"),
(1, 13, "digest13")]
assert msg.stableCheckpoint == 10
assert msg.checkpoints == [cp1, cp2]
def test_calc_checkpoints_selects_max(builder):
cp1 = Checkpoint(instId=0, viewNo=0, seqNoStart=0, seqNoEnd=0, digest=cp_digest(0))
cp2 = Checkpoint(instId=0, viewNo=0, seqNoStart=0, seqNoEnd=10, digest=cp_digest(10))
cp3 = Checkpoint(instId=0, viewNo=0, seqNoStart=0, seqNoEnd=20, digest=cp_digest(20))
vc1 = ViewChange(viewNo=2, stableCheckpoint=0,
prepared=[BatchID(1, 1, 1, "digest1")],
preprepared=[BatchID(1, 1, 1, "digest1")],
checkpoints=[cp1])
vc2_not_stable = ViewChange(viewNo=2, stableCheckpoint=0,
prepared=[BatchID(1, 1, 1, "digest1")],
preprepared=[BatchID(1, 1, 1, "digest1")],
checkpoints=[cp2])
vc2_stable = ViewChange(viewNo=2, stableCheckpoint=10,
prepared=[BatchID(1, 1, 1, "digest1")],
preprepared=[BatchID(1, 1, 1, "digest1")],
checkpoints=[cp2])
vc3_not_stable = ViewChange(viewNo=2, stableCheckpoint=10,
prepared=[BatchID(1, 1, 1, "digest1")],
preprepared=[BatchID(1, 1, 1, "digest1")],
checkpoints=[cp3])
vc3_stable = ViewChange(viewNo=2, stableCheckpoint=10,
prepared=[BatchID(1, 1, 1, "digest1")],
preprepared=[BatchID(1, 1, 1, "digest1")],
checkpoints=[cp3])
for vc3 in (vc3_not_stable, vc3_stable):
for vc2 in (vc2_not_stable, vc2_stable):
vcs = [vc1, vc2, vc3, vc3]
assert builder.calc_checkpoint(vcs) == cp3
vcs = [vc1, vc3, vc3, vc3]
assert builder.calc_checkpoint(vcs) == cp3
vcs = [vc2, vc3, vc3, vc3]
assert builder.calc_checkpoint(vcs) == cp3
vcs = [vc2, vc2, vc3, vc3]
assert builder.calc_checkpoint(vcs) == cp3
vcs = [vc1, vc1, vc3, vc3]
assert builder.calc_checkpoint(vcs) == cp3
vcs = [vc1, vc1, vc1, vc3]
assert builder.calc_checkpoint(vcs) == cp1
vcs = [vc1, vc1, vc2, vc2]
assert builder.calc_checkpoint(vcs) == cp2
vcs = [vc2, vc2, vc2, vc3]
assert builder.calc_checkpoint(vcs) == cp2
def test_calc_checkpoints_quorum(builder):
cp1 = Checkpoint(instId=0, viewNo=0, seqNoStart=0, seqNoEnd=0, digest=cp_digest(0))
cp2 = Checkpoint(instId=0, viewNo=0, seqNoStart=0, seqNoEnd=10, digest=cp_digest(10))
vc1 = ViewChange(viewNo=2, stableCheckpoint=0,
prepared=[BatchID(1, 1, 1, "digest1")],
preprepared=[BatchID(1, 1, 1, "digest1")],
checkpoints=[cp1])
vc2 = ViewChange(viewNo=2, stableCheckpoint=0,
prepared=[BatchID(1, 1, 1, "digest1")],
preprepared=[BatchID(1, 1, 1, "digest1")],
checkpoints=[cp2])
vc2_stable = ViewChange(viewNo=2, stableCheckpoint=10,
prepared=[BatchID(1, 1, 1, "digest1")],
preprepared=[BatchID(1, 1, 1, "digest1")],
checkpoints=[cp2])
vcs = [vc1, vc1, vc1, vc1]
assert builder.calc_checkpoint(vcs) == cp1
vcs = [vc2, vc2, vc2, vc2]
assert builder.calc_checkpoint(vcs) == cp2
vcs = [vc2_stable, vc2_stable, vc2_stable, vc2_stable]
assert builder.calc_checkpoint(vcs) == cp2
vcs = [vc2, vc1, vc1, vc1]
assert builder.calc_checkpoint(vcs) == cp1
vcs = [vc2, vc2, vc1, vc1]
assert builder.calc_checkpoint(vcs) == cp2
vcs = [vc2, vc2, vc2, vc1]
assert builder.calc_checkpoint(vcs) == cp2
vcs = [vc2_stable, vc1, vc1, vc1]
assert builder.calc_checkpoint(vcs) == cp1
vcs = [vc2_stable, vc2_stable, vc1, vc1]
assert builder.calc_checkpoint(vcs) == cp2
vcs = [vc2_stable, vc2_stable, vc2_stable, vc1]
assert builder.calc_checkpoint(vcs) == cp2
vcs = [vc2_stable, vc2, vc2, vc2]
assert builder.calc_checkpoint(vcs) == cp2
vcs = [vc2_stable, vc2_stable, vc2, vc2]
assert builder.calc_checkpoint(vcs) == cp2
vcs = [vc2_stable, vc2_stable, vc2_stable, vc2]
assert builder.calc_checkpoint(vcs) == cp2
def test_calc_checkpoints_digest(builder):
d1 = cp_digest(0)
d2 = cp_digest(10)
cp1_d1 = Checkpoint(instId=0,
viewNo=0,
seqNoStart=0,
seqNoEnd=0,
digest=d1)
cp2_d2 = Checkpoint(instId=0,
viewNo=0,
seqNoStart=0,
seqNoEnd=10,
digest=d2)
cp2_d1 = Checkpoint(instId=0,
viewNo=0,
seqNoStart=0,
seqNoEnd=10,
digest=d1)
vc1_d1 = ViewChange(viewNo=0,
stableCheckpoint=0,
prepared=[(1, 1, "digest1")],
preprepared=[(1, 1, "digest1")],
checkpoints=[cp1_d1])
vc2_d2 = ViewChange(viewNo=0,
stableCheckpoint=0,
prepared=[(1, 1, "digest1")],
preprepared=[(1, 1, "digest1")],
checkpoints=[cp2_d2])
vc2_d1 = ViewChange(viewNo=0,
stableCheckpoint=0,
prepared=[(1, 1, "digest1")],
preprepared=[(1, 1, "digest1")],
checkpoints=[cp2_d1])
vcs = [vc1_d1, vc1_d1, vc2_d1, vc2_d2]
assert builder.calc_checkpoint(vcs) == cp1_d1
vcs = [vc1_d1, vc2_d1, vc2_d2, vc2_d2]
assert builder.calc_checkpoint(vcs) == cp2_d2
vcs = [vc1_d1, vc2_d2, vc2_d1, vc2_d1]
assert builder.calc_checkpoint(vcs) == cp2_d1
# Here we have 2 nodes malicious (f=1), but calc_checkpoint returns a value depending on the order
# Is it OK, or calc_checkpoint should return None (indicating that there is no valid quorum)?
vcs = [vc2_d1, vc2_d1, vc2_d2, vc2_d2]
assert builder.calc_checkpoint(vcs) == cp2_d1
vcs = [vc2_d2, vc2_d2, vc2_d1, vc2_d1]
assert builder.calc_checkpoint(vcs) == cp2_d2
def test_view_change_data(view_change_service, data):
data.view_no = 1
data.checkpoints.clear()
cp = Checkpoint(instId=0,
viewNo=1,
seqNoStart=0,
seqNoEnd=10,
digest=cp_digest(10))
data.checkpoints.add(cp)
data.stable_checkpoint = 10
data.prepared = [BatchID(0, 0, 1, "digest1"), BatchID(0, 0, 2, "digest2")]
data.preprepared = [
BatchID(0, 0, 1, "digest1"),
BatchID(0, 0, 2, "digest2"),
BatchID(0, 0, 3, "digest3")
]
view_change_service._bus.send(NeedViewChange())
assert data.view_no == 2
msg = get_view_change(view_change_service)
assert msg.viewNo == 2
assert msg.prepared == [(0, 0, 1, "digest1"), (0, 0, 2, "digest2")]
assert msg.preprepared == [(0, 0, 1, "digest1"), (0, 0, 2, "digest2"),
(0, 0, 3, "digest3")]
assert msg.stableCheckpoint == 10
assert msg.checkpoints == [cp]
def test_calc_batches_empty(builder):
cp = Checkpoint(instId=0,
viewNo=0,
seqNoStart=0,
seqNoEnd=0,
digest=cp_digest(0, 0))
vcs = [
ViewChange(viewNo=0,
stableCheckpoint=0,
prepared=[],
preprepared=[],
checkpoints=[cp]),
ViewChange(viewNo=0,
stableCheckpoint=0,
prepared=[],
preprepared=[],
checkpoints=[cp]),
ViewChange(viewNo=0,
stableCheckpoint=0,
prepared=[],
preprepared=[],
checkpoints=[cp]),
ViewChange(viewNo=0,
stableCheckpoint=0,
prepared=[],
preprepared=[],
checkpoints=[cp]),
]
assert [] == builder.calc_batches(cp, vcs)
def test_calc_batches_takes_next_view_one_prepared_if_weak_quorum_of_preprepared(
builder):
cp = Checkpoint(instId=0,
viewNo=0,
seqNoStart=0,
seqNoEnd=0,
digest=cp_digest(0, 0))
vc1 = ViewChange(viewNo=0,
stableCheckpoint=0,
prepared=[(0, 1, "digest1"), (1, 2, "digest2")],
preprepared=[(0, 1, "digest1"), (1, 2, "digest2")],
checkpoints=[cp])
vc2 = ViewChange(viewNo=0,
stableCheckpoint=0,
prepared=[(0, 1, "digest1")],
preprepared=[(0, 1, "digest1"), (1, 2, "digest2")],
checkpoints=[cp])
vc3 = ViewChange(viewNo=0,
stableCheckpoint=0,
prepared=[(0, 1, "digest1")],
preprepared=[(0, 1, "digest1")],
checkpoints=[cp])
vc4 = ViewChange(viewNo=0,
stableCheckpoint=0,
prepared=[(0, 1, "digest1")],
preprepared=[(0, 1, "digest1")],
checkpoints=[cp])
vcs = [vc1, vc2, vc3, vc4]
assert builder.calc_batches(
cp, vcs) == [BatchID(0, 1, "digest1"), (1, 2, "digest2")]
def test_calc_batches_takes_prepared_with_same_batchid_only(builder):
cp = Checkpoint(instId=0,
viewNo=0,
seqNoStart=0,
seqNoEnd=0,
digest=cp_digest(0, 0))
vc1 = ViewChange(viewNo=0,
stableCheckpoint=0,
prepared=[(1, 1, "digest1")],
preprepared=[(1, 1, "digest1")],
checkpoints=[cp])
vc2 = ViewChange(viewNo=0,
stableCheckpoint=0,
prepared=[(1, 1, "digest1")],
preprepared=[(1, 1, "digest1")],
checkpoints=[cp])
vc3 = ViewChange(viewNo=0,
stableCheckpoint=0,
prepared=[(1, 1, "digest2")],
preprepared=[(1, 1, "digest2")],
checkpoints=[cp])
vc4 = ViewChange(viewNo=0,
stableCheckpoint=0,
prepared=[],
preprepared=[(1, 1, "digest1")],
checkpoints=[cp])
vcs = [vc1, vc2, vc3, vc4]
assert builder.calc_batches(cp, vcs) == [BatchID(1, 1, "digest1")]
def test_calc_batches_takes_quorum_of_prepared(builder):
cp = Checkpoint(instId=0, viewNo=0, seqNoStart=0, seqNoEnd=0, digest=cp_digest(0))
vc1 = ViewChange(viewNo=1, stableCheckpoint=0,
prepared=[BatchID(0, 0, 1, "digest2")],
preprepared=[BatchID(0, 0, 1, "digest2")],
checkpoints=[cp])
vc2 = ViewChange(viewNo=1, stableCheckpoint=0,
prepared=[BatchID(0, 0, 1, "digest1")],
preprepared=[BatchID(0, 0, 1, "digest1")],
checkpoints=[cp])
vc3 = ViewChange(viewNo=1, stableCheckpoint=0,
prepared=[],
preprepared=[BatchID(0, 0, 1, "digest1")],
checkpoints=[cp])
vcs = [vc1, vc2, vc2, vc2]
assert builder.calc_batches(cp, vcs) == [BatchID(0, 0, 1, "digest1")]
vcs = [vc3, vc2, vc2, vc2]
assert builder.calc_batches(cp, vcs) == [BatchID(0, 0, 1, "digest1")]
vcs = [vc3, vc3, vc3, vc3]
assert builder.calc_batches(cp, vcs) == []
vcs = [vc1, vc1, vc2, vc2]
assert builder.calc_batches(cp, vcs) is None
# since we have enough pre-prepares
vcs = [vc2, vc3, vc3, vc3]
assert builder.calc_batches(cp, vcs) == [BatchID(0, 0, 1, "digest1")]
vcs = [vc2, vc2, vc3, vc3]
assert builder.calc_batches(cp, vcs) == [BatchID(0, 0, 1, "digest1")]
def checkpoint(ordered, tconf):
start = ordered.ppSeqNo % tconf.CHK_FREQ
return Checkpoint(instId=ordered.instId,
viewNo=ordered.viewNo,
seqNoStart=0,
seqNoEnd=start + tconf.CHK_FREQ - 1,
digest=cp_digest(start + tconf.CHK_FREQ - 1))
def test_calc_batches_must_be_in_pre_prepare(builder):
cp = Checkpoint(instId=0,
viewNo=0,
seqNoStart=0,
seqNoEnd=0,
digest=cp_digest(0, 0))
vc = ViewChange(viewNo=0,
stableCheckpoint=0,
prepared=[(0, 1, "digest1"), (0, 2, "digest2")],
preprepared=[(0, 1, "digest1")],
checkpoints=[cp])
vcs = [vc, vc, vc, vc]
# all nodes are malicious here since all added (0, 2) into prepared without adding to pre-prepared
# so, None here means we can not calculate NewView reliably
assert builder.calc_batches(cp, vcs) is None
vc1 = ViewChange(viewNo=0,
stableCheckpoint=0,
prepared=[(0, 1, "digest1"), (0, 2, "digest2")],
preprepared=[(0, 1, "digest1")],
checkpoints=[cp])
vc2 = ViewChange(viewNo=0,
stableCheckpoint=0,
prepared=[(0, 1, "digest1")],
preprepared=[(0, 1, "digest1")],
checkpoints=[cp])
vcs = [vc1, vc2, vc2, vc2]
assert builder.calc_batches(cp, vcs) == [BatchID(0, 1, "digest1")]
def test_calc_checkpoints_equal_stable(builder):
cp = Checkpoint(instId=0, viewNo=0, seqNoStart=0, seqNoEnd=10, digest=cp_digest(0))
vc = ViewChange(viewNo=2, stableCheckpoint=10,
prepared=[BatchID(1, 1, 1, "digest1")],
preprepared=[BatchID(1, 1, 1, "digest1")],
checkpoints=[cp])
vcs = [vc, vc, vc, vc]
assert builder.calc_checkpoint(vcs) == cp
def initial_checkpoints(initial_view_no):
return [
Checkpoint(instId=0,
viewNo=initial_view_no,
seqNoStart=0,
seqNoEnd=0,
digest=cp_digest(0))
]
def create_checkpoints(view_no):
return [
Checkpoint(instId=0,
viewNo=view_no,
seqNoStart=0,
seqNoEnd=200,
digest=cp_digest(0, 200))
]
def test_calc_batches_same_data(builder):
cp = Checkpoint(instId=0, viewNo=0, seqNoStart=0, seqNoEnd=0, digest=cp_digest(0))
vc = ViewChange(viewNo=1, stableCheckpoint=0,
prepared=[BatchID(0, 0, 1, "digest1"), BatchID(0, 0, 2, "digest2")],
preprepared=[BatchID(0, 0, 1, "digest1"), BatchID(0, 0, 2, "digest2")],
checkpoints=[cp])
vcs = [vc, vc, vc, vc]
assert builder.calc_batches(cp, vcs) == [BatchID(0, 0, 1, "digest1"), BatchID(0, 0, 2, "digest2")]
def test_calc_batches_combinations(builder, random):
MAX_PP_SEQ_NO = 4
MAX_VIEW_NO = 2
MAX_DIGEST_ID = 4
cp = Checkpoint(instId=0, viewNo=1, seqNoStart=0, seqNoEnd=0, digest=cp_digest(0))
for i in range(200):
for vc_count in range(N - F, N + 1):
view_changes = []
# 1. INIT
for i in range(vc_count):
# PRE-PREPARED
num_preprepares = random.integer(0, MAX_PP_SEQ_NO)
pre_prepares = []
for i in range(1, num_preprepares + 1):
view_no = random.integer(0, MAX_VIEW_NO)
batch_id = (view_no, random.integer(0, view_no),
i, "digest{}".format(random.integer(1, MAX_DIGEST_ID)))
pre_prepares.append(batch_id)
# PREPARED
prepares_mode = random.sample(['all-preprepared', 'half-preprepared', 'random-preprepared', 'random'],
1)
if prepares_mode == ['all-preprepared']:
prepares = pre_prepares
elif prepares_mode == ['half-preprepared']:
prepares = pre_prepares[:(num_preprepares // 2)]
elif prepares_mode == ['random-preprepared']:
prepares = random.sample(pre_prepares, len(pre_prepares))
elif prepares_mode == ['random']:
num_prepares = random.integer(0, MAX_PP_SEQ_NO)
prepares = []
for i in range(1, num_prepares + 1):
view_no = random.integer(0, MAX_VIEW_NO)
batch_id = (view_no, random.integer(0, view_no),
i, "digest{}".format(random.integer(1, MAX_DIGEST_ID)))
prepares.append(batch_id)
else:
assert False, str(prepares_mode)
# CHECKPOINTS
view_changes.append(ViewChange(
viewNo=MAX_VIEW_NO,
stableCheckpoint=0,
prepared=prepares,
preprepared=pre_prepares,
checkpoints=[cp]
))
# 2. EXECUTE
batches = builder.calc_batches(cp, view_changes)
# 3. VALIDATE
committed = calc_committed(view_changes, MAX_PP_SEQ_NO, N, F)
if committed and batches is not None:
assert set(committed) <= set(batches)
def test_calc_batches_diff_pp_viewno_in_prepare_and_preprepare(builder):
cp = Checkpoint(instId=0, viewNo=0, seqNoStart=0, seqNoEnd=0, digest=cp_digest(0))
vc = ViewChange(viewNo=2, stableCheckpoint=0,
prepared=[BatchID(1, 0, 1, "digest1"), BatchID(1, 0, 2, "digest2")],
preprepared=[BatchID(1, 1, 1, "digest1"), BatchID(1, 1, 2, "digest2")],
checkpoints=[cp])
vcs = [vc, vc, vc, vc]
assert builder.calc_batches(cp, vcs) is None
def create_view_change(initial_view_no, stable_cp=10, batches=None):
if batches is None:
batches = create_batches(initial_view_no)
digest = cp_digest(stable_cp)
cp = Checkpoint(instId=0, viewNo=initial_view_no, seqNoStart=0, seqNoEnd=stable_cp, digest=digest)
return ViewChange(viewNo=initial_view_no + 1,
stableCheckpoint=stable_cp,
prepared=batches,
preprepared=batches,
checkpoints=[cp])
def test_process_checkpoint(checkpoint_service, checkpoint, pre_prepare, tconf,
ordered, validators, is_master):
checkpoint_stabilized_handler = Mock()
checkpoint_service._bus.subscribe(CheckpointStabilized,
checkpoint_stabilized_handler)
quorum = checkpoint_service._data.quorums.checkpoint.value
n = len(validators)
assert quorum == n - getMaxFailures(n) - 1
senders = ["sender{}".format(i) for i in range(quorum + 1)]
till_seq_no = tconf.CHK_FREQ
checkpoint_service._received_checkpoints[cp_key(checkpoint.viewNo,
1)] = {"frm"}
# For now, on checkpoint stabilization phase all checkpoints
# with ppSeqNo less then stable_checkpoint will be removed
checkpoint_service._received_checkpoints[cp_key(
checkpoint.viewNo + 1, till_seq_no + 100)] = {"frm"}
pre_prepare.ppSeqNo = till_seq_no
pre_prepare.auditTxnRootHash = cp_digest(till_seq_no)
ordered.ppSeqNo = pre_prepare.ppSeqNo
ordered.auditTxnRootHash = pre_prepare.auditTxnRootHash
checkpoint_service._data.preprepared.append(
preprepare_to_batch_id(pre_prepare))
checkpoint_service.process_ordered(ordered)
_check_checkpoint(checkpoint_service,
till_seq_no,
pre_prepare,
check_shared_data=True)
for sender in senders[:quorum - 1]:
checkpoint_service.process_checkpoint(checkpoint, sender)
assert checkpoint_service._data.stable_checkpoint < till_seq_no
# send the last checkpoint to stable it
checkpoint_service.process_checkpoint(checkpoint, senders[quorum - 1])
assert checkpoint_service._data.stable_checkpoint == till_seq_no
# check _remove_stashed_checkpoints()
assert sum(1 for cp in checkpoint_service._received_checkpoints
if cp.view_no == checkpoint.viewNo) == 0
assert sum(1 for cp in checkpoint_service._received_checkpoints
if cp.view_no == checkpoint.viewNo + 1) > 0
# check watermarks
assert checkpoint_service._data.low_watermark == checkpoint.seqNoEnd
# check that a Cleanup msg has been sent
checkpoint_stabilized_handler.assert_called_once_with(
CheckpointStabilized(inst_id=checkpoint_service._data.inst_id,
last_stable_3pc=(checkpoint.viewNo,
checkpoint.seqNoEnd)))
def test_start_catchup_on_quorum_of_stashed_checkpoints(
checkpoint_service, checkpoint, pre_prepare, tconf, ordered,
validators, is_master):
master_catchup_handler = Mock()
backup_catchup_handler = Mock()
checkpoint_service._bus.subscribe(NeedMasterCatchup,
master_catchup_handler)
checkpoint_service._bus.subscribe(NeedBackupCatchup,
backup_catchup_handler)
def check_catchup_not_started():
master_catchup_handler.assert_not_called()
backup_catchup_handler.assert_not_called()
def check_catchup_started(till_seq_no: int):
if is_master:
master_catchup_handler.assert_called_once_with(NeedMasterCatchup())
backup_catchup_handler.assert_not_called()
else:
master_catchup_handler.assert_not_called()
backup_catchup_handler.assert_called_once_with(
NeedBackupCatchup(
inst_id=checkpoint_service._data.inst_id,
caught_up_till_3pc=(checkpoint_service.view_no,
till_seq_no)))
quorum = checkpoint_service._data.quorums.checkpoint.value
n = len(validators)
assert quorum == n - getMaxFailures(n) - 1
senders = ["sender{}".format(i) for i in range(quorum + 1)]
till_seq_no = 2 * tconf.CHK_FREQ
new_checkpoint = Checkpoint(instId=ordered.instId,
viewNo=ordered.viewNo,
seqNoStart=0,
seqNoEnd=till_seq_no,
digest=cp_digest(till_seq_no))
key = checkpoint_service._checkpoint_key(checkpoint)
for sender in senders[:quorum]:
checkpoint_service.process_checkpoint(checkpoint, sender)
assert sender in checkpoint_service._received_checkpoints[key]
check_catchup_not_started()
new_key = checkpoint_service._checkpoint_key(new_checkpoint)
for sender in senders[:quorum - 1]:
checkpoint_service.process_checkpoint(new_checkpoint, sender)
assert sender in checkpoint_service._received_checkpoints[new_key]
check_catchup_not_started()
checkpoint_service.process_checkpoint(new_checkpoint, senders[quorum - 1])
check_catchup_started(till_seq_no)
def test_send_instance_change_on_new_view_with_incorrect_checkpoint(
internal_bus, external_bus, validators, primary,
view_change_service_builder, initial_view_no, some_item, is_master):
# TODO: Need to decide on how we handle this case
if not is_master:
return
next_view_no = initial_view_no + 1
primary_name = primary(next_view_no)
non_primary_name = some_item(validators, exclude=[primary_name])
service = view_change_service_builder(non_primary_name)
vc = create_view_change(initial_view_no)
service._data.preprepared = vc.preprepared
service._data.prepared = vc.prepared
service._data.stable_checkpoint = vc.stableCheckpoint
service._data.checkpoints = vc.checkpoints
# start view change
internal_bus.send(NeedViewChange())
external_bus.sent_messages.clear()
# receive quorum of ViewChanges and ViewChangeAcks
non_primaries = [item for item in validators if item != primary_name]
non_primaries = random.sample(non_primaries,
service._data.quorums.view_change.value)
for vc_frm in non_primaries:
external_bus.process_incoming(
vc, generateName(vc_frm, service._data.inst_id))
for ack, ack_frm in create_view_change_acks(vc, vc_frm, non_primaries):
external_bus.process_incoming(
ack, generateName(ack_frm, service._data.inst_id))
cp = Checkpoint(instId=0,
viewNo=initial_view_no,
seqNoStart=0,
seqNoEnd=1000,
digest=cp_digest(1000))
new_view = create_new_view_from_vc(vc, non_primaries, checkpoint=cp)
# send NewView by Primary
init_network_msg_count = len(external_bus.sent_messages)
external_bus.process_incoming(
new_view, generateName(primary_name, service._data.inst_id))
# we don't go to new view, just send Instance Change
assert service._data.view_no == initial_view_no + 1
assert init_network_msg_count + 1 == len(external_bus.sent_messages)
msg, dst = external_bus.sent_messages[-1]
assert dst is None # broadcast
assert isinstance(msg, InstanceChange)
assert msg.viewNo == initial_view_no + 2
assert msg.reason == Suspicions.NEW_VIEW_INVALID_CHECKPOINTS.code
def _service(name):
data = consensus_data(name)
digest = cp_digest(DEFAULT_STABLE_CHKP)
cp = Checkpoint(instId=0,
viewNo=initial_view_no,
seqNoStart=0,
seqNoEnd=DEFAULT_STABLE_CHKP,
digest=digest)
data.checkpoints.append(cp)
service = ViewChangeService(data, timer, internal_bus, external_bus,
stasher)
return service
def _view_change(view_no: int):
vc = ViewChange(viewNo=view_no,
stableCheckpoint=4,
prepared=[],
preprepared=[],
checkpoints=[
Checkpoint(instId=0,
viewNo=view_no,
seqNoStart=0,
seqNoEnd=4,
digest=cp_digest(4))
])
return vc
def test_caught_up_till_3pc_stabilizes_checkpoint(
checkpoint_service, tconf, chkFreqPatched, is_master, initial_view_no,
caughtup_view_no, caughtup_pp_seq_no, expected_stable_checkpoint):
checkpoint_service._get_view_no_from_audit = lambda x: initial_view_no
checkpoint_service._get_digest_from_audit = lambda x, y: cp_digest(9999)
data = checkpoint_service._data
last_ordered = (caughtup_view_no, caughtup_pp_seq_no)
# emulate finish of catchup
data.last_ordered_3pc = last_ordered
data.view_no = last_ordered[0]
# call caught_up_till_3pc
checkpoint_service.caught_up_till_3pc(last_ordered)
# stable checkpoint must be updated
assert data.stable_checkpoint == expected_stable_checkpoint
assert data.last_checkpoint.seqNoEnd == data.stable_checkpoint
expected_view_no = initial_view_no if caughtup_pp_seq_no >= 100 else 0
assert data.last_checkpoint.viewNo == expected_view_no
expected_digest = cp_digest(9999) if caughtup_pp_seq_no >= 100 else None
assert data.last_checkpoint.digest == expected_digest
assert len(data.checkpoints) == 1
def _service(name):
data = consensus_data(name)
digest = cp_digest(DEFAULT_STABLE_CHKP)
cp = Checkpoint(instId=0,
viewNo=initial_view_no,
seqNoStart=0,
seqNoEnd=DEFAULT_STABLE_CHKP,
digest=digest)
data.checkpoints.append(cp)
primaries_selector = RoundRobinConstantNodesPrimariesSelector(
validators)
service = ViewChangeService(data, timer, internal_bus, external_bus,
stasher, primaries_selector)
return service
