def test_unstash_waiting_for_first_batch_ordered_after_catchup(
looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle, tconf):
lagged_node = txnPoolNodeSet[-1]
other_nodes = list(set(txnPoolNodeSet) - {lagged_node})
other_stashers = [n.nodeIbStasher for n in other_nodes]
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
last_ordered_lagged_before = lagged_node.master_last_ordered_3PC
# do not process any message reqs for PrePrepares
with delay_rules_without_processing(
lagged_node.nodeIbStasher,
msg_rep_delay(types_to_delay=[PREPARE, PREPREPARE])):
with delay_rules(lagged_node.nodeIbStasher, cDelay()):
ensure_view_change(looper, txnPoolNodeSet)
looper.run(eventually(check_not_in_view_change, txnPoolNodeSet))
ensureElectionsDone(looper,
other_nodes,
instances_list=range(
getRequiredInstances(len(txnPoolNodeSet))))
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
# delay Commits on all nodes so that there are some PrePrepares still stashed after catchup
with delay_rules(other_stashers, cDelay()):
pre_prep_before = len(recvdPrePrepareForInstId(lagged_node, 0))
sdk_send_random_requests(looper, sdk_pool_handle,
sdk_wallet_client, 2)
# wait till lagged node recives the new PrePrepares
# they will be stashed as WAITING_FIRST_BATCH_IN_VIEW
looper.run(
eventually(lambda: assertExp(
len(recvdPrePrepareForInstId(lagged_node, 0)) ==
pre_prep_before + 2)))
# catchup the lagged node
# the latest 2 PrePrepares are still stashed
lagged_node.start_catchup()
looper.run(
eventually(
lambda: assertExp(lagged_node.master_last_ordered_3PC >
last_ordered_lagged_before)))
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 2)
ensureElectionsDone(looper, txnPoolNodeSet, customTimeout=30)
ensure_all_nodes_have_same_data(looper, txnPoolNodeSet, custom_timeout=30)
def check_drop():
# Node should have not received PrePrepare, Prepares and Commits for master instance
assert len(recvdPrePrepareForInstId(lagged_node, 0)) == 0
assert len(recvdPrepareForInstId(lagged_node, 0)) == 0
assert len(recvdCommitForInstId(lagged_node, 0)) == 0
# Request object should be dropped by timeout
assert len(lagged_node.requests) == 0
def check_propagates_requested():
# Node should have received delayed PrePrepare
assert len(recvdPrePrepareForInstId(lagged_node, 0)) >= 1
# Check that PROPAGATEs are requested by both replicas as PrePrepare has been
# received for request that was dropped.
tmp = getAllArgs(lagged_node, TestNode.request_propagates)
assert len(tmp) == 2
def check_drop():
# Node should have not received PrePrepare, Prepares and Commits for master instance
assert len(recvdPrePrepareForInstId(lagged_node, 0)) == 0
assert len(recvdPrepareForInstId(lagged_node, 0)) == 0
assert len(recvdCommitForInstId(lagged_node, 0)) == 0
# Request object should be dropped by timeout
assert len(lagged_node.requests) == 0
def check_propagates_requested():
# Node should have received delayed PrePrepare
assert len(recvdPrePrepareForInstId(lagged_node, 0)) >= 1
# Check that PROPAGATEs are requested by at least one replica as PrePrepare has been
# received for request that was dropped.
# We can check that the number of requested propagates is more or equal to 1,
# since the first replica who sees non-finalized requests sends MessageReq for Propagates,
# and it can receive Propagates before a PrePrepare for the next replica is received,
# so that the for the second replica all requests will be already finalized.
assert len(getAllArgs(lagged_node, TestNode.request_propagates)) == 2
def check_preprepares_delayed():
# Node should have received a request from the client
assert len(recvdRequest(lagged_node)) == 1
# Node should not have received a PROPAGATE
assert len(recvdPropagate(lagged_node)) == 3
# Node should have sent a PROPAGATE
assert len(sentPropagate(lagged_node)) == 1
# Node should have not received PrePrepares for master instance
assert len(recvdPrePrepareForInstId(lagged_node, 0)) == 0
# Node should have not received Prepares for master instance
assert len(recvdPrepareForInstId(lagged_node, 0)) == 0
# Node should have not received Commits for master instance
assert len(recvdCommitForInstId(lagged_node, 0)) == 0
# Node should have 1 request in requests queue
assert len(lagged_node.requests) == 1
def check_propagates_and_3pc_delayed():
# Node should have received a request from the client
assert len(recvdRequest(lagged_node)) == 1
# Node should not have received a PROPAGATE
assert len(recvdPropagate(lagged_node)) == 0
# Node should have sent a PROPAGATE
assert len(sentPropagate(lagged_node)) == 1
# Node should have not received PrePrepares for master instance
assert len(recvdPrePrepareForInstId(lagged_node, 0)) == 0
# Node should have not received Prepares for master instance
assert len(recvdPrepareForInstId(lagged_node, 0)) == 0
# Node should have not received Commits for master instance
assert len(recvdCommitForInstId(lagged_node, 0)) == 0
# Node should have 1 request in requests queue
assert len(lagged_node.requests) == 1
def check_propagates_requested():
# Node should have received delayed PrePrepare
assert len(recvdPrePrepareForInstId(lagged_node, 0)) == 1
# Check that PROPAGATEs are requested by both replicas as PrePrepare has been
# received for request that was dropped.
assert len(getAllArgs(lagged_node, TestNode.request_propagates)) == 2
