def test_unstash_waiting_for_first_batch_ordered_after_catchup( looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle, tconf): lagged_node = txnPoolNodeSet[-1] other_nodes = list(set(txnPoolNodeSet) - {lagged_node}) other_stashers = [n.nodeIbStasher for n in other_nodes] sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) last_ordered_lagged_before = lagged_node.master_last_ordered_3PC # do not process any message reqs for PrePrepares with delay_rules_without_processing( lagged_node.nodeIbStasher, msg_rep_delay(types_to_delay=[PREPARE, PREPREPARE])): with delay_rules(lagged_node.nodeIbStasher, cDelay()): ensure_view_change(looper, txnPoolNodeSet) looper.run(eventually(check_not_in_view_change, txnPoolNodeSet)) ensureElectionsDone(looper, other_nodes, instances_list=range( getRequiredInstances(len(txnPoolNodeSet)))) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) # delay Commits on all nodes so that there are some PrePrepares still stashed after catchup with delay_rules(other_stashers, cDelay()): pre_prep_before = len(recvdPrePrepareForInstId(lagged_node, 0)) sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 2) # wait till lagged node recives the new PrePrepares # they will be stashed as WAITING_FIRST_BATCH_IN_VIEW looper.run( eventually(lambda: assertExp( len(recvdPrePrepareForInstId(lagged_node, 0)) == pre_prep_before + 2))) # catchup the lagged node # the latest 2 PrePrepares are still stashed lagged_node.start_catchup() looper.run( eventually( lambda: assertExp(lagged_node.master_last_ordered_3PC > last_ordered_lagged_before))) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 2) ensureElectionsDone(looper, txnPoolNodeSet, customTimeout=30) ensure_all_nodes_have_same_data(looper, txnPoolNodeSet, custom_timeout=30)
def check_drop(): # Node should have not received PrePrepare, Prepares and Commits for master instance assert len(recvdPrePrepareForInstId(lagged_node, 0)) == 0 assert len(recvdPrepareForInstId(lagged_node, 0)) == 0 assert len(recvdCommitForInstId(lagged_node, 0)) == 0 # Request object should be dropped by timeout assert len(lagged_node.requests) == 0
def check_propagates_requested(): # Node should have received delayed PrePrepare assert len(recvdPrePrepareForInstId(lagged_node, 0)) >= 1 # Check that PROPAGATEs are requested by both replicas as PrePrepare has been # received for request that was dropped. tmp = getAllArgs(lagged_node, TestNode.request_propagates) assert len(tmp) == 2
def check_propagates_requested(): # Node should have received delayed PrePrepare assert len(recvdPrePrepareForInstId(lagged_node, 0)) >= 1 # Check that PROPAGATEs are requested by at least one replica as PrePrepare has been # received for request that was dropped. # We can check that the number of requested propagates is more or equal to 1, # since the first replica who sees non-finalized requests sends MessageReq for Propagates, # and it can receive Propagates before a PrePrepare for the next replica is received, # so that the for the second replica all requests will be already finalized. assert len(getAllArgs(lagged_node, TestNode.request_propagates)) == 2
def check_preprepares_delayed(): # Node should have received a request from the client assert len(recvdRequest(lagged_node)) == 1 # Node should not have received a PROPAGATE assert len(recvdPropagate(lagged_node)) == 3 # Node should have sent a PROPAGATE assert len(sentPropagate(lagged_node)) == 1 # Node should have not received PrePrepares for master instance assert len(recvdPrePrepareForInstId(lagged_node, 0)) == 0 # Node should have not received Prepares for master instance assert len(recvdPrepareForInstId(lagged_node, 0)) == 0 # Node should have not received Commits for master instance assert len(recvdCommitForInstId(lagged_node, 0)) == 0 # Node should have 1 request in requests queue assert len(lagged_node.requests) == 1
def check_propagates_and_3pc_delayed(): # Node should have received a request from the client assert len(recvdRequest(lagged_node)) == 1 # Node should not have received a PROPAGATE assert len(recvdPropagate(lagged_node)) == 0 # Node should have sent a PROPAGATE assert len(sentPropagate(lagged_node)) == 1 # Node should have not received PrePrepares for master instance assert len(recvdPrePrepareForInstId(lagged_node, 0)) == 0 # Node should have not received Prepares for master instance assert len(recvdPrepareForInstId(lagged_node, 0)) == 0 # Node should have not received Commits for master instance assert len(recvdCommitForInstId(lagged_node, 0)) == 0 # Node should have 1 request in requests queue assert len(lagged_node.requests) == 1
def check_propagates_requested(): # Node should have received delayed PrePrepare assert len(recvdPrePrepareForInstId(lagged_node, 0)) == 1 # Check that PROPAGATEs are requested by both replicas as PrePrepare has been # received for request that was dropped. assert len(getAllArgs(lagged_node, TestNode.request_propagates)) == 2