def test_update_shared_data_on_mew_view_checkpoint_applied(internal_bus, orderer):
orderer._data.preprepared = []
orderer._data.prepared = []
old_data = copy_shared_data(orderer._data)
initial_view_no = 3
new_view = create_new_view(initial_view_no=initial_view_no, stable_cp=200)
internal_bus.send(NewViewCheckpointsApplied(view_no=initial_view_no + 1,
view_changes=new_view.viewChanges,
checkpoint=new_view.checkpoint,
batches=new_view.batches))
new_data = copy_shared_data(orderer._data)
check_service_changed_only_owned_fields_in_shared_data(OrderingService, old_data, new_data)
# preprepared are created for new view
if orderer.is_master:
assert orderer._data.preprepared
assert orderer._data.preprepared == [BatchID(view_no=initial_view_no + 1, pp_seq_no=batch_id.pp_seq_no,
pp_digest=batch_id.pp_digest)
for batch_id in new_view.batches]
assert orderer._data.prepared == []
else:
assert orderer._data.preprepared == []
assert orderer._data.prepared == []
def process_new_view_accepted(self, msg: NewViewAccepted):
if self.is_master:
cp = msg.checkpoint
# do not update stable checkpoints if the node doesn't have this checkpoint
# the node is lagging behind in this case and will start catchup after receiving
# one more quorum of checkpoints from other nodes
if cp not in self._data.checkpoints:
return
self._mark_checkpoint_stable(cp.seqNoEnd)
self.set_watermarks(low_watermark=cp.seqNoEnd)
else:
# TODO: This is kind of hackery, but proper way would require introducing more
# messages specifically for backup replicas. Hope we can live with it for now.
self._data.waiting_for_new_view = False
self._data.pp_seq_no = 0
self.set_watermarks(low_watermark=0)
self._reset_checkpoints()
self._data.stable_checkpoint = 0
self._remove_received_checkpoints()
self._bus.send(
NewViewCheckpointsApplied(view_no=msg.view_no,
view_changes=msg.view_changes,
checkpoint=msg.checkpoint,
batches=msg.batches))
def test_re_order_pre_prepares_no_pre_prepares(looper, txnPoolNodeSet,
sdk_wallet_client, sdk_pool_handle):
# 1. drop PrePrepars, Prepares and Commits on 4thNode
# Order a couple of requests on Nodes 1-3
lagging_node = txnPoolNodeSet[-1]
other_nodes = txnPoolNodeSet[:-1]
with delay_rules_without_processing(lagging_node.nodeIbStasher, delay_3pc()):
sdk_send_random_and_check(looper, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client, 3)
assert all(n.master_last_ordered_3PC == (0, 3) for n in other_nodes)
with delay_rules_without_processing(lagging_node.nodeIbStasher,
msg_rep_delay(types_to_delay=[PREPREPARE, PREPARE, COMMIT])):
# 2. simulate view change start so that
# all PrePrepares/Prepares/Commits are cleared
# and uncommitted txns are reverted
for n in txnPoolNodeSet:
n.replicas.send_to_internal_bus(ViewChangeStarted(view_no=1))
master_ordering_service = n.master_replica._ordering_service
assert not master_ordering_service.prePrepares
assert not master_ordering_service.prepares
assert not master_ordering_service.commits
ledger = n.db_manager.ledgers[DOMAIN_LEDGER_ID]
state = n.db_manager.states[DOMAIN_LEDGER_ID]
assert len(ledger.uncommittedTxns) == 0
assert ledger.uncommitted_root_hash == ledger.tree.root_hash
assert state.committedHead == state.head
# check that all nodes but the lagging one have old_view_pps stored
for n in other_nodes:
assert n.master_replica._ordering_service.old_view_preprepares
assert not lagging_node.master_replica._ordering_service.old_view_preprepares
# 3. Simulate View Change finish to re-order the same PrePrepare
assert lagging_node.master_last_ordered_3PC == (0, 0)
new_master = txnPoolNodeSet[1]
batches = sorted([preprepare_to_batch_id(pp) for _, pp in
new_master.master_replica._ordering_service.old_view_preprepares.items()])
new_view_msg = NewView(viewNo=0,
viewChanges=[],
checkpoint=None,
batches=batches)
new_view_chk_applied_msg = NewViewCheckpointsApplied(view_no=0,
view_changes=[],
checkpoint=None,
batches=batches)
for n in txnPoolNodeSet:
n.master_replica._consensus_data.new_view = new_view_msg
n.master_replica._consensus_data.prev_view_prepare_cert = batches[-1].pp_seq_no
n.master_replica._ordering_service._bus.send(new_view_chk_applied_msg)
# 4. Make sure that the nodes 1-3 (that already ordered the requests) sent Prepares and Commits so that
# the request was eventually ordered on Node4 as well
waitNodeDataEquality(looper, lagging_node, *other_nodes, customTimeout=60)
assert lagging_node.master_last_ordered_3PC == (0, 4)
sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle)
def test_re_order_pre_prepares(looper, txnPoolNodeSet, sdk_wallet_client,
sdk_pool_handle):
# 0. use new 3PC validator
for n in txnPoolNodeSet:
ordering_service = n.master_replica._ordering_service
ordering_service._validator = OrderingServiceMsgValidator(
ordering_service._data)
# 1. drop Prepares and Commits on 4thNode
# Order a couple of requests on Nodes 1-3
lagging_node = txnPoolNodeSet[-1]
other_nodes = txnPoolNodeSet[:-1]
with delay_rules_without_processing(lagging_node.nodeIbStasher, cDelay(),
pDelay()):
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 3)
assert all(n.master_last_ordered_3PC == (0, 3) for n in other_nodes)
# 2. simulate view change start so that
# all PrePrepares/Prepares/Commits are cleared
# and uncommitted txns are reverted
for n in txnPoolNodeSet:
n.replicas.send_to_internal_bus(ViewChangeStarted(view_no=1))
master_ordering_service = n.master_replica._ordering_service
assert not master_ordering_service.prePrepares
assert not master_ordering_service.prepares
assert not master_ordering_service.commits
assert master_ordering_service.old_view_preprepares
ledger = n.db_manager.ledgers[DOMAIN_LEDGER_ID]
state = n.db_manager.states[DOMAIN_LEDGER_ID]
assert len(ledger.uncommittedTxns) == 0
assert ledger.uncommitted_root_hash == ledger.tree.root_hash
assert state.committedHead == state.head
# 3. Simulate View Change finish to re-order the same PrePrepare
assert lagging_node.master_last_ordered_3PC == (0, 0)
new_master = txnPoolNodeSet[1]
batches = [
preprepare_to_batch_id(pp) for _, pp in new_master.master_replica.
_ordering_service.old_view_preprepares.items()
]
new_view_msg = NewViewCheckpointsApplied(view_no=0,
view_changes=[],
checkpoint=None,
batches=batches)
for n in txnPoolNodeSet:
n.master_replica._ordering_service._bus.send(new_view_msg)
# 4. Make sure that the nodes 1-3 (that already ordered the requests) sent Prepares and Commits so that
# the request was eventually ordered on Node4 as well
waitNodeDataEquality(looper, lagging_node, *other_nodes)
assert lagging_node.master_last_ordered_3PC == (0, 3)
sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_client,
sdk_pool_handle)
def process_new_view_accepted(self, msg: NewViewAccepted):
# 1. update shared data
cp = msg.checkpoint
if cp not in self._data.checkpoints:
self._data.checkpoints.append(cp)
self._mark_checkpoint_stable(cp.seqNoEnd)
self.set_watermarks(low_watermark=cp.seqNoEnd)
# 2. send NewViewCheckpointsApplied
self._bus.send(
NewViewCheckpointsApplied(view_no=msg.view_no,
view_changes=msg.view_changes,
checkpoint=msg.checkpoint,
batches=msg.batches))
return PROCESS, None
def test_do_nothing_on_new_view_checkpoint_applied(internal_bus, view_change_service):
view_change_service._data.waiting_for_new_view = False
view_change_service._data.view_no = 1
view_change_service._data.primary_name = "Alpha"
view_change_service._data.primaries = ["Alpha", "Beta"]
old_data = copy_shared_data(view_change_service._data)
new_view = create_new_view(initial_view_no=3, stable_cp=200)
internal_bus.send(NewViewCheckpointsApplied(view_no=4,
view_changes=new_view.viewChanges,
checkpoint=new_view.checkpoint,
batches=new_view.batches))
new_data = copy_shared_data(view_change_service._data)
assert old_data == new_data
def test_do_nothing_on_new_view_checkpoint_applied(internal_bus, checkpoint_service):
checkpoint_service._data.checkpoints.clear()
checkpoint_service._data.checkpoints.update(create_checkpoints(view_no=0))
checkpoint_service._data.stable_checkpoint = 100
checkpoint_service._data.low_watermark = 100
checkpoint_service._data.high_watermark = checkpoint_service._data.low_watermark + 300
old_data = copy_shared_data(checkpoint_service._data)
initial_view_no = 3
new_view = create_new_view(initial_view_no=initial_view_no, stable_cp=200)
internal_bus.send(NewViewCheckpointsApplied(view_no=initial_view_no + 1,
view_changes=new_view.viewChanges,
checkpoint=new_view.checkpoint,
batches=new_view.batches))
new_data = copy_shared_data(checkpoint_service._data)
assert old_data == new_data
def test_view_change_finished_sends_new_view_checkpoint_applied(
internal_bus, checkpoint_service):
handler = Mock()
internal_bus.subscribe(NewViewCheckpointsApplied, handler)
initial_view_no = 3
new_view = create_new_view(initial_view_no=initial_view_no, stable_cp=200)
internal_bus.send(
NewViewAccepted(view_no=initial_view_no + 1,
view_changes=new_view.viewChanges,
checkpoint=new_view.checkpoint,
batches=new_view.batches))
expected_apply_new_view = NewViewCheckpointsApplied(
view_no=initial_view_no + 1,
view_changes=new_view.viewChanges,
checkpoint=new_view.checkpoint,
batches=new_view.batches)
handler.assert_called_once_with(expected_apply_new_view)
def test_unstash_waiting_new_view_on_new_view_checkpoint_applied(
external_bus, internal_bus, replica_service):
replica_service._data.view_no = 2
replica_service._data.node_mode = Mode.participating
replica_service._data.waiting_for_new_view = True
external_bus.process_incoming(create_commit_no_bls_sig(req_key=(2, 10)),
replica_service._data.primary_name)
assert replica_service.stasher.stash_size(STASH_WAITING_NEW_VIEW) == 1
new_view = create_new_view(initial_view_no=1, stable_cp=200)
replica_service._data.waiting_for_new_view = False
internal_bus.send(
NewViewCheckpointsApplied(view_no=2,
view_changes=new_view.viewChanges,
checkpoint=new_view.checkpoint,
batches=new_view.batches))
assert replica_service.stasher.stash_size(STASH_WAITING_NEW_VIEW) == 0
def test_update_shared_data_on_new_view_checkpoint_applied(
internal_bus, orderer):
initial_view_no = 3
orderer._data.preprepared = []
orderer._data.prepared = []
orderer._data.view_no = initial_view_no + 1
old_data = copy_shared_data(orderer._data)
new_view = create_new_view(initial_view_no=initial_view_no, stable_cp=200)
internal_bus.send(
NewViewCheckpointsApplied(view_no=initial_view_no + 1,
view_changes=new_view.viewChanges,
checkpoint=new_view.checkpoint,
batches=new_view.batches))
new_data = copy_shared_data(orderer._data)
check_service_changed_only_owned_fields_in_shared_data(
OrderingService, old_data, new_data)
# Since we didn't order the PrePrepare from Batches, it should not be added into shared data
# (we will request the PrePrepares instead, see next tests)
assert orderer._data.preprepared == []
assert orderer._data.prepared == []
def test_view_change_finished_sends_new_view_checkpoint_applied(internal_bus, checkpoint_service, is_master):
# TODO: Need to decide on how we handle this case
if not is_master:
return
handler = Mock()
internal_bus.subscribe(NewViewCheckpointsApplied, handler)
initial_view_no = 3
checkpoint_service._data.checkpoints.clear()
checkpoint_service._data.checkpoints.update(
[Checkpoint(instId=0, viewNo=3, seqNoStart=0, seqNoEnd=200, digest=cp_digest(200))])
new_view = create_new_view(initial_view_no=initial_view_no, stable_cp=200)
internal_bus.send(NewViewAccepted(view_no=initial_view_no + 1,
view_changes=new_view.viewChanges,
checkpoint=new_view.checkpoint,
batches=new_view.batches))
expected_apply_new_view = NewViewCheckpointsApplied(view_no=initial_view_no + 1,
view_changes=new_view.viewChanges,
checkpoint=new_view.checkpoint,
batches=new_view.batches)
handler.assert_called_once_with(expected_apply_new_view)
def new_view(view_no):
return NewViewCheckpointsApplied(view_no, [], [], [])
def test_process_preprepare_on_new_view_checkpoint_applied(
internal_bus, external_bus, orderer, is_primary, all_ordered,
initial_view_no, pre_prepares, stored_old_view_pre_prepares):
# !!!SETUP!!!
orderer._data.view_no = initial_view_no + 1
batches = create_batches_from_preprepares(pre_prepares)
orderer._data.prev_view_prepare_cert = batches[-1].pp_seq_no
new_view = create_new_view(initial_view_no=initial_view_no,
stable_cp=200,
batches=batches)
# emulate that we received all PrePrepares before View Change
orderer._update_old_view_preprepares(stored_old_view_pre_prepares)
# emulate that we've already ordered the PrePrepares
if all_ordered and stored_old_view_pre_prepares:
orderer.last_ordered_3pc = (initial_view_no,
stored_old_view_pre_prepares[-1].ppSeqNo)
# !!!EXECUTE!!!
# send NewViewCheckpointsApplied
internal_bus.send(
NewViewCheckpointsApplied(view_no=initial_view_no + 1,
view_changes=new_view.viewChanges,
checkpoint=new_view.checkpoint,
batches=new_view.batches))
# !!!CHECK!!!
if not orderer.is_master:
# no re-ordering is expected on non-master
assert orderer._data.preprepared == []
assert orderer._data.prepared == []
return
# check that PPs were added
stored_batch_ids = [
preprepare_to_batch_id(pp) for pp in stored_old_view_pre_prepares
]
assert orderer._data.preprepared == [
BatchID(view_no=initial_view_no + 1,
pp_view_no=initial_view_no,
pp_seq_no=batch_id.pp_seq_no,
pp_digest=batch_id.pp_digest) for batch_id in new_view.batches
if batch_id in stored_batch_ids
]
# check that sentPrePrepares is updated in case of Primary and prePrepares in case of non-primary
updated_prepares_collection = orderer.prePrepares if not is_primary else orderer.sent_preprepares
non_updated_prepares_collection = orderer.sent_preprepares if not is_primary else orderer.prePrepares
for pp in stored_old_view_pre_prepares:
new_pp = updateNamedTuple(pp,
viewNo=initial_view_no + 1,
originalViewNo=pp.viewNo)
assert (initial_view_no + 1,
new_pp.ppSeqNo) in updated_prepares_collection
assert updated_prepares_collection[(initial_view_no + 1,
new_pp.ppSeqNo)] == new_pp
assert not non_updated_prepares_collection
# check that Prepare is sent in case of non primary
if not is_primary:
check_prepares_sent(external_bus, stored_old_view_pre_prepares,
initial_view_no + 1)
else:
# only MessageReqs are sent
assert len(external_bus.sent_messages
) == len(pre_prepares) - len(stored_old_view_pre_prepares)
# we don't have a quorum of Prepares yet
assert orderer._data.prepared == []
# check that missing PrePrepares have been requested
expected_requested_batches = [
batch_id for batch_id in new_view.batches
if batch_id not in stored_batch_ids
]
check_request_old_view_preprepares_sent(external_bus,
expected_requested_batches)