def test_cleanup_after_checkpoint_stabilize(orderer):
pre_prepares = [create_pre_prepare_no_bls(generate_state_root(), view_no=0, pp_seq_no=1),
create_pre_prepare_no_bls(generate_state_root(), view_no=1, pp_seq_no=2),
create_pre_prepare_no_bls(generate_state_root(), view_no=1, pp_seq_no=3)]
dicts_to_cleaning = [orderer.pre_prepare_tss,
orderer.sent_preprepares,
orderer.prePrepares,
orderer.prepares,
orderer.commits,
orderer.batches,
orderer.pre_prepares_stashed_for_incorrect_time]
lists_to_cleaning = [orderer._data.prepared,
orderer._data.preprepared]
for pp in pre_prepares:
for dict_to_cleaning in dicts_to_cleaning:
dict_to_cleaning[(pp.viewNo, pp.ppSeqNo)] = pp
for list_to_cleaning in lists_to_cleaning:
list_to_cleaning.append(preprepare_to_batch_id(pp))
orderer._bus.send(CheckpointStabilized(last_stable_3pc=(1, 2)))
for pp in pre_prepares[:2]:
for dict_to_cleaning in dicts_to_cleaning:
assert (pp.viewNo, pp.ppSeqNo) not in dict_to_cleaning
for list_to_cleaning in lists_to_cleaning:
assert preprepare_to_batch_id(pp) not in list_to_cleaning
for dict_to_cleaning in dicts_to_cleaning:
assert (pre_prepares[2].viewNo, pre_prepares[2].ppSeqNo) in dict_to_cleaning
for list_to_cleaning in lists_to_cleaning:
assert preprepare_to_batch_id(pre_prepares[2]) in list_to_cleaning
def test_process_oredered(checkpoint_service, ordered, pre_prepare, tconf):
with pytest.raises(
LogicError,
match="CheckpointService | Can't process Ordered msg because "
"ppSeqNo {} not in preprepared".format(ordered.ppSeqNo)):
checkpoint_service.process_ordered(ordered)
checkpoint_service._data.preprepared.append(
preprepare_to_batch_id(pre_prepare))
checkpoint_service.process_ordered(ordered)
_check_checkpoint(checkpoint_service, 1, tconf.CHK_FREQ, pre_prepare)
pre_prepare.ppSeqNo = tconf.CHK_FREQ
ordered.ppSeqNo = pre_prepare.ppSeqNo
checkpoint_service._data.preprepared.append(
preprepare_to_batch_id(pre_prepare))
state = updateNamedTuple(
checkpoint_service._checkpoint_state[1, tconf.CHK_FREQ],
digests=["digest"] * (tconf.CHK_FREQ - 1))
checkpoint_service._checkpoint_state[1, tconf.CHK_FREQ] = state
checkpoint_service.process_ordered(ordered)
_check_checkpoint(checkpoint_service,
1,
tconf.CHK_FREQ,
pre_prepare,
check_shared_data=True)
pre_prepare.ppSeqNo += 1
ordered.ppSeqNo = pre_prepare.ppSeqNo
checkpoint_service._data.preprepared.append(
preprepare_to_batch_id(pre_prepare))
checkpoint_service.process_ordered(ordered)
_check_checkpoint(checkpoint_service, tconf.CHK_FREQ + 1,
tconf.CHK_FREQ * 2, pre_prepare)
def test_process_ordered(checkpoint_service, ordered, pre_prepare, tconf):
with pytest.raises(
LogicError,
match="CheckpointService | Can't process Ordered msg because "
"ppSeqNo {} not in preprepared".format(ordered.ppSeqNo)):
checkpoint_service.process_ordered(ordered)
checkpoint_service._data.preprepared.append(
preprepare_to_batch_id(pre_prepare))
checkpoint_service.process_ordered(ordered)
_check_checkpoint(checkpoint_service, tconf.CHK_FREQ, pre_prepare)
pre_prepare.ppSeqNo = tconf.CHK_FREQ
ordered.ppSeqNo = pre_prepare.ppSeqNo
checkpoint_service._data.preprepared.append(
preprepare_to_batch_id(pre_prepare))
checkpoint_service.process_ordered(ordered)
_check_checkpoint(checkpoint_service,
tconf.CHK_FREQ,
pre_prepare,
check_shared_data=True)
pre_prepare.ppSeqNo += 1
ordered.ppSeqNo = pre_prepare.ppSeqNo
checkpoint_service._data.preprepared.append(
preprepare_to_batch_id(pre_prepare))
checkpoint_service.process_ordered(ordered)
_check_checkpoint(checkpoint_service, tconf.CHK_FREQ * 2, pre_prepare)
def clear_batch(self, pp: PrePrepare):
"""
When 3pc batch processed, it removed from _prepared list
"""
if preprepare_to_batch_id(pp) in self.consensus_data.preprepared:
self.consensus_data.preprepared.remove(preprepare_to_batch_id(pp))
if preprepare_to_batch_id(pp) in self.consensus_data.prepared:
self.consensus_data.prepared.remove(preprepare_to_batch_id(pp))
def preprepare_batch(self, pp: PrePrepare):
"""
After pp had validated, it placed into _preprepared list
"""
if preprepare_to_batch_id(pp) in self.consensus_data.preprepared:
raise LogicError('New pp cannot be stored in preprepared')
if self.consensus_data.checkpoints and pp.ppSeqNo < self.consensus_data.last_checkpoint.seqNoEnd:
raise LogicError('ppSeqNo cannot be lower than last checkpoint')
self.consensus_data.preprepared.append(preprepare_to_batch_id(pp))
def test_pp_storages_freeing(pre_prepare,
consensus_data_helper: ConsensusDataHelper):
consensus_data_helper.consensus_data.prepared.append(
preprepare_to_batch_id(pre_prepare))
consensus_data_helper.consensus_data.preprepared.append(
preprepare_to_batch_id(pre_prepare))
assert consensus_data_helper.consensus_data.preprepared
assert consensus_data_helper.consensus_data.prepared
consensus_data_helper.clear_all_batches()
assert not consensus_data_helper.consensus_data.preprepared
assert not consensus_data_helper.consensus_data.prepared
def test_node_reg_in_ordered_from_audit_no_node_reg(test_node):
replica = test_node.master_replica
# run multiple times to have a situation when we write node reg to non-empty audit ledger
for i in range(1, 4):
uncommitted_node_reg = ['Alpha', 'Beta', 'Gamma', 'Node{}'.format(i)]
test_node.write_manager.node_reg_handler.uncommitted_node_reg = uncommitted_node_reg
pre_prepare = create_pre_prepare_no_bls(
state_root=generate_state_root(), pp_seq_no=i)
key = (pre_prepare.viewNo, pre_prepare.ppSeqNo)
replica._ordering_service.prePrepares[key] = pre_prepare
replica._consensus_data.preprepared.append(
preprepare_to_batch_id(pre_prepare))
test_node.primaries = ["Alpha", "Beta"]
three_pc_batch = ThreePcBatch.from_pre_prepare(
pre_prepare=pre_prepare,
state_root=pre_prepare.stateRootHash,
txn_root=pre_prepare.txnRootHash,
primaries=["Alpha", "Beta", "Gamma"],
valid_digests=pre_prepare.reqIdr)
three_pc_batch.node_reg = None
test_node.write_manager.audit_b_handler.post_batch_applied(
three_pc_batch)
replica._ordering_service._order_3pc_key(key)
ordered = replica.outBox.pop()
# we expect it equal to the current uncommitted node reg
assert ordered.nodeReg == uncommitted_node_reg
def test_node_reg_in_ordered_from_audit_for_tree_txns(test_node):
node_regs = {}
replica = test_node.master_replica
test_node.primaries = ["Alpha", "Beta"]
node_reg = ["Alpha", "Beta", "Gamma", "Delta", "Eta"]
for i in range(3):
pp = create_pre_prepare_no_bls(state_root=generate_state_root(),
pp_seq_no=i)
key = (pp.viewNo, pp.ppSeqNo)
replica._ordering_service.prePrepares[key] = pp
replica._consensus_data.preprepared.append(preprepare_to_batch_id(pp))
three_pc_batch = ThreePcBatch.from_pre_prepare(
pre_prepare=pp,
state_root=pp.stateRootHash,
txn_root=pp.txnRootHash,
primaries=["Node{}".format(num) for num in range(i + 1)],
valid_digests=pp.reqIdr)
three_pc_batch.node_reg = node_reg + ["Node{}".format(i + 10)]
test_node.write_manager.audit_b_handler.post_batch_applied(
three_pc_batch)
node_regs[key] = three_pc_batch.node_reg
for key in reversed(list(node_regs.keys())):
replica._ordering_service._order_3pc_key(key)
for ordered in replica.outBox:
if not isinstance(ordered, Ordered):
continue
assert ordered.nodeReg == node_regs[(ordered.viewNo, ordered.ppSeqNo)]
def test_send_reply_on_old_view_pre_prepares_request(
external_bus, orderer, initial_view_no, stored_old_view_pre_prepares,
requested_old_view_pre_prepares):
# Setup
orderer._data.view_no = initial_view_no + 2
orderer._update_old_view_preprepares(stored_old_view_pre_prepares)
# Receive OldViewPrePrepareRequest req
batches = [
preprepare_to_batch_id(pp) for pp in requested_old_view_pre_prepares
]
req = OldViewPrePrepareRequest(0, batches)
frm = "node1"
orderer._network.process_incoming(req,
generateName(frm, orderer._data.inst_id))
# Check that OldViewPrePrepareReply is sent for all requested PrePrepares
if not orderer.is_master:
assert len(external_bus.sent_messages) == 0
return
# equal to set's union operation
expected_pps = [
i for i in stored_old_view_pre_prepares
if i in requested_old_view_pre_prepares
]
expected_pps = sorted(expected_pps, key=lambda pp: pp.ppSeqNo)
check_reply_old_view_preprepares_sent(external_bus, frm, expected_pps)
def test_re_order_pre_prepares(looper, txnPoolNodeSet, sdk_wallet_client,
sdk_pool_handle):
# 0. use new 3PC validator
for n in txnPoolNodeSet:
ordering_service = n.master_replica._ordering_service
ordering_service._validator = OrderingServiceMsgValidator(
ordering_service._data)
# 1. drop Prepares and Commits on 4thNode
# Order a couple of requests on Nodes 1-3
lagging_node = txnPoolNodeSet[-1]
other_nodes = txnPoolNodeSet[:-1]
with delay_rules_without_processing(lagging_node.nodeIbStasher, cDelay(),
pDelay()):
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 3)
assert all(n.master_last_ordered_3PC == (0, 3) for n in other_nodes)
# 2. simulate view change start so that
# all PrePrepares/Prepares/Commits are cleared
# and uncommitted txns are reverted
for n in txnPoolNodeSet:
n.replicas.send_to_internal_bus(ViewChangeStarted(view_no=1))
master_ordering_service = n.master_replica._ordering_service
assert not master_ordering_service.prePrepares
assert not master_ordering_service.prepares
assert not master_ordering_service.commits
assert master_ordering_service.old_view_preprepares
ledger = n.db_manager.ledgers[DOMAIN_LEDGER_ID]
state = n.db_manager.states[DOMAIN_LEDGER_ID]
assert len(ledger.uncommittedTxns) == 0
assert ledger.uncommitted_root_hash == ledger.tree.root_hash
assert state.committedHead == state.head
# 3. Simulate View Change finish to re-order the same PrePrepare
assert lagging_node.master_last_ordered_3PC == (0, 0)
new_master = txnPoolNodeSet[1]
batches = sorted([
preprepare_to_batch_id(pp) for _, pp in new_master.master_replica.
_ordering_service.old_view_preprepares.items()
])
new_view_msg = NewViewCheckpointsApplied(view_no=0,
view_changes=[],
checkpoint=None,
batches=batches)
for n in txnPoolNodeSet:
n.master_replica._consensus_data.prev_view_prepare_cert = batches[
-1].pp_seq_no
n.master_replica._ordering_service._bus.send(new_view_msg)
# 4. Make sure that the nodes 1-3 (that already ordered the requests) sent Prepares and Commits so that
# the request was eventually ordered on Node4 as well
waitNodeDataEquality(looper, lagging_node, *other_nodes)
assert lagging_node.master_last_ordered_3PC == (0, 4)
sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_client,
sdk_pool_handle)
def test_process_checkpoint(checkpoint_service, checkpoint, pre_prepare, tconf,
ordered, validators, is_master):
checkpoint_stabilized_handler = Mock()
checkpoint_service._bus.subscribe(CheckpointStabilized,
checkpoint_stabilized_handler)
quorum = checkpoint_service._data.quorums.checkpoint.value
n = len(validators)
assert quorum == n - getMaxFailures(n) - 1
senders = ["sender{}".format(i) for i in range(quorum + 1)]
till_seq_no = tconf.CHK_FREQ
checkpoint_service._received_checkpoints[cp_key(checkpoint.viewNo,
1)] = {"frm"}
# For now, on checkpoint stabilization phase all checkpoints
# with ppSeqNo less then stable_checkpoint will be removed
checkpoint_service._received_checkpoints[cp_key(
checkpoint.viewNo + 1, till_seq_no + 100)] = {"frm"}
pre_prepare.ppSeqNo = till_seq_no
pre_prepare.auditTxnRootHash = cp_digest(till_seq_no)
ordered.ppSeqNo = pre_prepare.ppSeqNo
ordered.auditTxnRootHash = pre_prepare.auditTxnRootHash
checkpoint_service._data.preprepared.append(
preprepare_to_batch_id(pre_prepare))
checkpoint_service.process_ordered(ordered)
_check_checkpoint(checkpoint_service,
till_seq_no,
pre_prepare,
check_shared_data=True)
for sender in senders[:quorum - 1]:
checkpoint_service.process_checkpoint(checkpoint, sender)
assert checkpoint_service._data.stable_checkpoint < till_seq_no
# send the last checkpoint to stable it
checkpoint_service.process_checkpoint(checkpoint, senders[quorum - 1])
assert checkpoint_service._data.stable_checkpoint == till_seq_no
# check _remove_stashed_checkpoints()
assert sum(1 for cp in checkpoint_service._received_checkpoints
if cp.view_no == checkpoint.viewNo) == 0
assert sum(1 for cp in checkpoint_service._received_checkpoints
if cp.view_no == checkpoint.viewNo + 1) > 0
# check watermarks
assert checkpoint_service._data.low_watermark == checkpoint.seqNoEnd
# check that a Cleanup msg has been sent
checkpoint_stabilized_handler.assert_called_once_with(
CheckpointStabilized(inst_id=checkpoint_service._data.inst_id,
last_stable_3pc=(checkpoint.viewNo,
checkpoint.seqNoEnd)))
def test_pp_storages_freeing_till(pre_prepare,
consensus_data_helper: ConsensusDataHelper):
pre_prepare.ppSeqNo = 3
consensus_data_helper.consensus_data.prepared.append(
preprepare_to_batch_id(pre_prepare))
consensus_data_helper.consensus_data.preprepared.append(
preprepare_to_batch_id(pre_prepare))
assert consensus_data_helper.consensus_data.preprepared
assert consensus_data_helper.consensus_data.prepared
consensus_data_helper.clear_batch_till_seq_no(4)
assert not consensus_data_helper.consensus_data.preprepared
assert not consensus_data_helper.consensus_data.prepared
pre_prepare.ppSeqNo = 4
consensus_data_helper.consensus_data.prepared.append(
preprepare_to_batch_id(pre_prepare))
consensus_data_helper.consensus_data.preprepared.append(
preprepare_to_batch_id(pre_prepare))
consensus_data_helper.clear_batch_till_seq_no(4)
assert consensus_data_helper.consensus_data.preprepared
assert consensus_data_helper.consensus_data.prepared
def test_process_message_rep_already_ordered_preprepare(message_req_service: MessageReqService,
external_bus, data, pp):
key = (pp.viewNo, pp.ppSeqNo)
data.preprepared.append(preprepare_to_batch_id(pp))
message_req_service.handlers[PREPREPARE].requested_messages[key] = None
message_rep = MessageRep(**{
f.MSG_TYPE.nm: PREPREPARE,
f.PARAMS.nm: {f.INST_ID.nm: data.inst_id,
f.VIEW_NO.nm: key[0],
f.PP_SEQ_NO.nm: key[1]},
f.MSG.nm: dict(pp.items())
})
frm = "frm"
network_handler = Mock()
external_bus.subscribe(PrePrepare, network_handler)
message_req_service.process_message_rep(message_rep, frm)
network_handler.assert_not_called()
def test_node_reg_in_ordered_from_audit(test_node):
pre_prepare = create_pre_prepare_no_bls(state_root=generate_state_root(), pp_seq_no=1)
replica = test_node.master_replica
key = (pre_prepare.viewNo, pre_prepare.ppSeqNo)
replica._ordering_service.prePrepares[key] = pre_prepare
replica._consensus_data.preprepared.append(preprepare_to_batch_id(pre_prepare))
test_node.primaries = ["Alpha", "Beta"]
three_pc_batch = ThreePcBatch.from_pre_prepare(pre_prepare=pre_prepare,
state_root=pre_prepare.stateRootHash,
txn_root=pre_prepare.txnRootHash,
primaries=["Alpha", "Beta", "Gamma"],
valid_digests=pre_prepare.reqIdr)
three_pc_batch.node_reg = ["Alpha", "Beta", "Gamma", "Delta", "Eta"]
test_node.write_manager.audit_b_handler.post_batch_applied(three_pc_batch)
replica._ordering_service._order_3pc_key(key)
ordered = replica.outBox.pop()
assert ordered.nodeReg == three_pc_batch.node_reg
def create_batches_from_preprepares(preprepares):
return [preprepare_to_batch_id(pp) for pp in preprepares]
def test_process_checkpoint(checkpoint_service, checkpoint, pre_prepare, tconf,
ordered, validators, is_master):
checkpoint_stabilized_handler = Mock()
checkpoint_service._bus.subscribe(CheckpointStabilized,
checkpoint_stabilized_handler)
quorum = checkpoint_service._data.quorums.checkpoint.value
n = len(validators)
assert quorum == n - getMaxFailures(n) - 1
senders = ["sender{}".format(i) for i in range(quorum + 1)]
key = (1, tconf.CHK_FREQ)
old_key = (-1, 0)
checkpoint_service._stash_checkpoint(
Checkpoint(1, checkpoint.viewNo, 1, 1, cp_digest(1, 1)), "frm")
checkpoint_service._stash_checkpoint(
Checkpoint(1, checkpoint.viewNo + 1, 1, 1, cp_digest(1, 1)), "frm")
checkpoint_service._checkpoint_state[old_key] = CheckpointState(
1, ["digest"] * (tconf.CHK_FREQ - 1), None, {}, False)
checkpoint_service._checkpoint_state[key] = CheckpointState(
key[1] - 1, ["digest"] * (tconf.CHK_FREQ - 1), None, {}, False)
pre_prepare.ppSeqNo = key[1]
ordered.ppSeqNo = pre_prepare.ppSeqNo
checkpoint_service._data.preprepared.append(
preprepare_to_batch_id(pre_prepare))
checkpoint_service.process_ordered(ordered)
_check_checkpoint(checkpoint_service,
key[0],
key[1],
pre_prepare,
check_shared_data=True)
state = updateNamedTuple(checkpoint_service._checkpoint_state[key],
digest=checkpoint.digest)
checkpoint_service._checkpoint_state[key] = state
for sender in senders[:quorum - 1]:
assert checkpoint_service.process_checkpoint(checkpoint, sender)
assert checkpoint_service._checkpoint_state[key].receivedDigests[
sender] == checkpoint.digest
assert not checkpoint_service._checkpoint_state[key].isStable
# send the last checkpoint to stable it
assert checkpoint_service.process_checkpoint(checkpoint,
senders[quorum - 1])
assert checkpoint_service._checkpoint_state[key].isStable
# check _remove_stashed_checkpoints()
assert checkpoint.viewNo not in checkpoint_service._stashed_recvd_checkpoints
assert checkpoint.viewNo + 1 in checkpoint_service._stashed_recvd_checkpoints
# check watermarks
assert checkpoint_service._data.low_watermark == checkpoint.seqNoEnd
# check that a Cleanup msg has been sent
checkpoint_stabilized_handler.assert_called_once_with(
CheckpointStabilized(inst_id=checkpoint_service._data.inst_id,
last_stable_3pc=(checkpoint.viewNo,
checkpoint.seqNoEnd)))
# check that old checkpoint_states has been removed
assert old_key not in checkpoint_service._checkpoint_state
def prepare_batch(self, pp: PrePrepare):
"""
After prepared certificate for pp had collected,
it removed from _preprepared and placed into _prepared list
"""
self.consensus_data.prepared.append(preprepare_to_batch_id(pp))
def test_process_preprepare_on_new_view_checkpoint_applied(
internal_bus, external_bus, orderer, is_primary, all_ordered,
initial_view_no, pre_prepares, stored_old_view_pre_prepares):
# !!!SETUP!!!
orderer._data.view_no = initial_view_no + 1
batches = create_batches_from_preprepares(pre_prepares)
orderer._data.prev_view_prepare_cert = batches[-1].pp_seq_no
new_view = create_new_view(initial_view_no=initial_view_no,
stable_cp=200,
batches=batches)
# emulate that we received all PrePrepares before View Change
orderer._update_old_view_preprepares(stored_old_view_pre_prepares)
# emulate that we've already ordered the PrePrepares
if all_ordered and stored_old_view_pre_prepares:
orderer.last_ordered_3pc = (initial_view_no,
stored_old_view_pre_prepares[-1].ppSeqNo)
# !!!EXECUTE!!!
# send NewViewCheckpointsApplied
internal_bus.send(
NewViewCheckpointsApplied(view_no=initial_view_no + 1,
view_changes=new_view.viewChanges,
checkpoint=new_view.checkpoint,
batches=new_view.batches))
# !!!CHECK!!!
if not orderer.is_master:
# no re-ordering is expected on non-master
assert orderer._data.preprepared == []
assert orderer._data.prepared == []
return
# check that PPs were added
stored_batch_ids = [
preprepare_to_batch_id(pp) for pp in stored_old_view_pre_prepares
]
assert orderer._data.preprepared == [
BatchID(view_no=initial_view_no + 1,
pp_view_no=initial_view_no,
pp_seq_no=batch_id.pp_seq_no,
pp_digest=batch_id.pp_digest) for batch_id in new_view.batches
if batch_id in stored_batch_ids
]
# check that sentPrePrepares is updated in case of Primary and prePrepares in case of non-primary
updated_prepares_collection = orderer.prePrepares if not is_primary else orderer.sent_preprepares
non_updated_prepares_collection = orderer.sent_preprepares if not is_primary else orderer.prePrepares
for pp in stored_old_view_pre_prepares:
new_pp = updateNamedTuple(pp,
viewNo=initial_view_no + 1,
originalViewNo=pp.viewNo)
assert (initial_view_no + 1,
new_pp.ppSeqNo) in updated_prepares_collection
assert updated_prepares_collection[(initial_view_no + 1,
new_pp.ppSeqNo)] == new_pp
assert not non_updated_prepares_collection
# check that Prepare is sent in case of non primary
if not is_primary:
check_prepares_sent(external_bus, stored_old_view_pre_prepares,
initial_view_no + 1)
else:
# only MessageReqs are sent
assert len(external_bus.sent_messages
) == len(pre_prepares) - len(stored_old_view_pre_prepares)
# we don't have a quorum of Prepares yet
assert orderer._data.prepared == []
# check that missing PrePrepares have been requested
expected_requested_batches = [
batch_id for batch_id in new_view.batches
if batch_id not in stored_batch_ids
]
check_request_old_view_preprepares_sent(external_bus,
expected_requested_batches)
