def test_pool_reaches_quorum_after_f_plus_2_nodes_turned_off_and_later_on( looper, allPluginsPath, tdir, tconf, txnPoolNodeSet, wallet1, client1, client1Connected): nodes = txnPoolNodeSet initial_view_no = nodes[0].viewNo request = sendRandomRequest(wallet1, client1) waitForSufficientRepliesForRequests(looper, client1, requests=[request]) stop_node(nodes[0], looper, nodes) waitForViewChange(looper, nodes[1:], expectedViewNo=initial_view_no + 1) ensureElectionsDone(looper, nodes[1:], numInstances=getRequiredInstances(nodeCount)) request = sendRandomRequest(wallet1, client1) waitForSufficientRepliesForRequests(looper, client1, requests=[request]) stop_node(nodes[1], looper, nodes) looper.runFor(tconf.ToleratePrimaryDisconnection + 2) checkViewNoForNodes(nodes[2:], initial_view_no + 1) request = sendRandomRequest(wallet1, client1) verify_request_not_replied_and_not_ordered(request, looper, client1, nodes) stop_node(nodes[2], looper, nodes) looper.runFor(tconf.ToleratePrimaryDisconnection + 2) checkViewNoForNodes(nodes[3:], initial_view_no + 1) request = sendRandomRequest(wallet1, client1) verify_request_not_replied_and_not_ordered(request, looper, client1, nodes) nodes[2] = start_stopped_node(nodes[2], looper, tconf, tdir, allPluginsPath) looper.runFor(waits.expectedPoolElectionTimeout(len(nodes))) request = sendRandomRequest(wallet1, client1) verify_request_not_replied_and_not_ordered(request, looper, client1, nodes) nodes[1] = start_stopped_node(nodes[1], looper, tconf, tdir, allPluginsPath) ensureElectionsDone(looper, nodes[1:], numInstances=getRequiredInstances(nodeCount)) waitForViewChange(looper, nodes[1:], expectedViewNo=initial_view_no + 1) request = sendRandomRequest(wallet1, client1) waitForSufficientRepliesForRequests(looper, client1, requests=[request]) nodes[0] = start_stopped_node(nodes[0], looper, tconf, tdir, allPluginsPath) ensureElectionsDone(looper, nodes, numInstances=getRequiredInstances(nodeCount)) waitForViewChange(looper, nodes, expectedViewNo=initial_view_no + 1) request = sendRandomRequest(wallet1, client1) waitForSufficientRepliesForRequests(looper, client1, requests=[request])
def do_view_change_with_propagate_primary_on_one_delayed_node( slow_node, nodes, looper, sdk_pool_handle, sdk_wallet_client): slow_stasher = slow_node.nodeIbStasher fast_nodes = [n for n in nodes if n != slow_node] stashers = [n.nodeIbStasher for n in nodes] # Get last prepared certificate in pool lpc = last_prepared_certificate(nodes) # Get pool current view no view_no = lpc[0] with delay_rules(slow_stasher, icDelay()): with delay_rules(slow_stasher, vcd_delay()): with delay_rules(stashers, cDelay()): # Send request request = sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client) # Wait until this request is prepared on N-f nodes looper.run(eventually(check_last_prepared_certificate_on_quorum, nodes, (lpc[0], lpc[1] + 1))) # Trigger view change for n in nodes: n.view_changer.on_master_degradation() # Wait until view change is completed on all nodes except slow one waitForViewChange(looper, fast_nodes, expectedViewNo=view_no + 1, customTimeout=waits.expectedPoolViewChangeStartedTimeout(len(nodes))) wait_for_elections_done_on_given_nodes(looper, fast_nodes, getRequiredInstances(len(nodes)), timeout=waits.expectedPoolElectionTimeout(len(nodes))) # Now all the nodes receive Commits # The slow node will accept Commits and order the 3PC-batch in the old view looper.runFor(waits.expectedOrderingTime(getNoInstances(len(nodes)))) # Now slow node receives ViewChangeDones waitForViewChange(looper, [slow_node], expectedViewNo=view_no + 1, customTimeout=waits.expectedPoolViewChangeStartedTimeout(len(nodes))) wait_for_elections_done_on_given_nodes(looper, [slow_node], getRequiredInstances(len(nodes)), timeout=waits.expectedPoolElectionTimeout(len(nodes))) # Now slow node receives InstanceChanges but discards them because already # started propagate primary to the same view. # Finish request gracefully sdk_get_reply(looper, request)
def test_quorum_after_f_plus_2_nodes_including_primary_turned_off_and_later_on( looper, allPluginsPath, tdir, tconf, txnPoolNodeSet, wallet1, client1): nodes = txnPoolNodeSet request1 = sendRandomRequest(wallet1, client1) waitForSufficientRepliesForRequests(looper, client1, requests=[request1]) stop_node(nodes[0], looper, nodes) waitForViewChange(looper, nodes[1:], expectedViewNo=1) ensureElectionsDone(looper, nodes[1:], numInstances=getRequiredInstances(nodeCount)) request2 = sendRandomRequest(wallet1, client1) waitForSufficientRepliesForRequests(looper, client1, requests=[request2]) stop_node(nodes[1], looper, nodes) looper.runFor(tconf.ToleratePrimaryDisconnection + waits.expectedPoolElectionTimeout(len(nodes))) checkViewNoForNodes(nodes[2:], expectedViewNo=1) request3 = sendRandomRequest(wallet1, client1) verify_request_not_replied_and_not_ordered(request3, looper, client1, nodes) stop_node(nodes[2], looper, nodes) looper.runFor(tconf.ToleratePrimaryDisconnection + waits.expectedPoolElectionTimeout(len(nodes))) checkViewNoForNodes(nodes[3:], expectedViewNo=1) request4 = sendRandomRequest(wallet1, client1) verify_request_not_replied_and_not_ordered(request4, looper, client1, nodes) nodes[2] = start_stopped_node(nodes[2], looper, tconf, tdir, allPluginsPath) looper.runFor(waits.expectedPoolElectionTimeout(len(nodes))) checkViewNoForNodes(nodes[3:], expectedViewNo=1) request5 = sendRandomRequest(wallet1, client1) verify_request_not_replied_and_not_ordered(request5, looper, client1, nodes) nodes[1] = start_stopped_node(nodes[1], looper, tconf, tdir, allPluginsPath) ensureElectionsDone(looper, nodes[1:], numInstances=getRequiredInstances(nodeCount)) checkViewNoForNodes(nodes[1:], expectedViewNo=1) request6 = sendRandomRequest(wallet1, client1) waitForSufficientRepliesForRequests(looper, client1, requests=[request6]) nodes[0] = start_stopped_node(nodes[0], looper, tconf, tdir, allPluginsPath) ensureElectionsDone(looper, nodes, numInstances=getRequiredInstances(nodeCount)) checkViewNoForNodes(nodes, expectedViewNo=1) request7 = sendRandomRequest(wallet1, client1) waitForSufficientRepliesForRequests(looper, client1, requests=[request7])
def fake_view_changer(request, tconf): node_count = 4 node_stack = FakeSomething( name="fake stack", connecteds={"Alpha", "Beta", "Gamma", "Delta"}, conns={"Alpha", "Beta", "Gamma", "Delta"} ) monitor = FakeSomething( isMasterDegraded=lambda: False, areBackupsDegraded=lambda: [], prettymetrics='' ) node = FakeSomething( name="SomeNode", viewNo=request.param, quorums=Quorums(getValueFromModule(request, 'nodeCount', default=node_count)), nodestack=node_stack, utc_epoch=lambda *args: get_utc_epoch(), config=tconf, monitor=monitor, discard=lambda a, b, c: print(b), primaries_disconnection_times=[None] * getRequiredInstances(node_count), master_primary_name='Alpha', master_replica=FakeSomething(instId=0) ) view_changer = ViewChanger(node) return view_changer
def fake_view_changer(request, tconf): node_count = 4 node_stack = FakeSomething(name="fake stack", connecteds={"Alpha", "Beta", "Gamma", "Delta"}, conns={"Alpha", "Beta", "Gamma", "Delta"}) monitor = FakeSomething(isMasterDegraded=lambda: False, areBackupsDegraded=lambda: [], prettymetrics='') node = FakeSomething(name="SomeNode", timer=QueueTimer(), viewNo=request.param, quorums=Quorums( getValueFromModule(request, 'nodeCount', default=node_count)), nodestack=node_stack, utc_epoch=lambda *args: get_utc_epoch(), config=tconf, monitor=monitor, discard=lambda a, b, c, d: print(b), primaries_disconnection_times=[None] * getRequiredInstances(node_count), master_primary_name='Alpha', master_replica=FakeSomething( instId=0, viewNo=request.param, _consensus_data=FakeSomething( view_no=request.param, waiting_for_new_view=False)), nodeStatusDB=None) view_changer = create_view_changer(node) # TODO: This is a hack for tests compatibility, do something better view_changer.node = node return view_changer
def fake_view_changer(request, tconf): node_count = 4 node_stack = FakeSomething(name="fake stack", connecteds={"Alpha", "Beta", "Gamma", "Delta"}, conns={"Alpha", "Beta", "Gamma", "Delta"}) monitor = FakeSomething(isMasterDegraded=lambda: False, areBackupsDegraded=lambda: [], prettymetrics='') node = FakeSomething(name="SomeNode", viewNo=request.param, quorums=Quorums( getValueFromModule(request, 'nodeCount', default=node_count)), nodestack=node_stack, utc_epoch=lambda *args: get_utc_epoch(), config=tconf, monitor=monitor, discard=lambda a, b, c: print(b), primaries_disconnection_times=[None] * getRequiredInstances(node_count), master_primary_name='Alpha', master_replica=FakeSomething(instId=0)) view_changer = ViewChanger(node) return view_changer
def test_view_change_without_primary(nodeSet, looper, patched_view_change_timeout): first, others = stop_nodes_and_remove_first(looper, nodeSet) start_and_connect_nodes(looper, others) timeout = waits.expectedPoolElectionTimeout(len(nodeSet)) + patched_view_change_timeout checkProtocolInstanceSetup(looper=looper, nodes=others, retryWait=1, customTimeout=timeout, numInstances=getRequiredInstances(len(nodeSet)))
def test_view_change_without_primary(txnPoolNodeSet, looper, patched_view_change_timeout): first, others = stop_nodes_and_remove_first(looper, txnPoolNodeSet) start_and_connect_nodes(looper, others) timeout = waits.expectedPoolElectionTimeout(len(txnPoolNodeSet)) + patched_view_change_timeout #looper.runFor(40) checkProtocolInstanceSetup(looper=looper, nodes=txnPoolNodeSet, retryWait=1, customTimeout=timeout, instances=range(getRequiredInstances(len(txnPoolNodeSet))))
def test_unstash_waiting_for_first_batch_ordered_after_catchup( looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle, tconf): lagged_node = txnPoolNodeSet[-1] other_nodes = list(set(txnPoolNodeSet) - {lagged_node}) other_stashers = [n.nodeIbStasher for n in other_nodes] sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) last_ordered_lagged_before = lagged_node.master_last_ordered_3PC # do not process any message reqs for PrePrepares with delay_rules_without_processing( lagged_node.nodeIbStasher, msg_rep_delay(types_to_delay=[PREPARE, PREPREPARE])): with delay_rules(lagged_node.nodeIbStasher, cDelay()): ensure_view_change(looper, txnPoolNodeSet) looper.run(eventually(check_not_in_view_change, txnPoolNodeSet)) ensureElectionsDone(looper, other_nodes, instances_list=range( getRequiredInstances(len(txnPoolNodeSet)))) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) # delay Commits on all nodes so that there are some PrePrepares still stashed after catchup with delay_rules(other_stashers, cDelay()): pre_prep_before = len(recvdPrePrepareForInstId(lagged_node, 0)) sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 2) # wait till lagged node recives the new PrePrepares # they will be stashed as WAITING_FIRST_BATCH_IN_VIEW looper.run( eventually(lambda: assertExp( len(recvdPrePrepareForInstId(lagged_node, 0)) == pre_prep_before + 2))) # catchup the lagged node # the latest 2 PrePrepares are still stashed lagged_node.start_catchup() looper.run( eventually( lambda: assertExp(lagged_node.master_last_ordered_3PC > last_ordered_lagged_before))) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 2) ensureElectionsDone(looper, txnPoolNodeSet, customTimeout=30) ensure_all_nodes_have_same_data(looper, txnPoolNodeSet, custom_timeout=30)
def test_view_change_without_primary(txnPoolNodeSet, looper, tconf): first, others = stop_nodes_and_remove_first(looper, txnPoolNodeSet) start_and_connect_nodes(looper, others) timeout = waits.expectedPoolElectionTimeout( len(txnPoolNodeSet)) + tconf.NEW_VIEW_TIMEOUT # looper.runFor(40) checkProtocolInstanceSetup(looper=looper, nodes=txnPoolNodeSet, retryWait=1, customTimeout=timeout, instances=range( getRequiredInstances(len(txnPoolNodeSet))))
def test_unstash_waiting_for_first_batch_ordered(looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle, tconf): lagged_node = txnPoolNodeSet[-1] other_nodes = list(set(txnPoolNodeSet) - {lagged_node}) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) with delay_rules(lagged_node.nodeIbStasher, cDelay()): ensure_view_change(looper, txnPoolNodeSet) ensureElectionsDone(looper, other_nodes, instances_list=range( getRequiredInstances(len(txnPoolNodeSet)))) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 2) ensureElectionsDone(looper, txnPoolNodeSet) ensure_all_nodes_have_same_data(looper, txnPoolNodeSet)
def test_quorum_after_f_plus_2_nodes_including_primary_turned_off_and_later_on( looper, allPluginsPath, tdir, tconf, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client): timeout = sdk_eval_timeout(1, len(txnPoolNodeSet)) nodes = txnPoolNodeSet sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) stop_node(nodes[0], looper, nodes) waitForViewChange(looper, nodes[1:], expectedViewNo=1) ensureElectionsDone(looper, nodes[1:], numInstances=getRequiredInstances(nodeCount)) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) stop_node(nodes[1], looper, nodes) looper.runFor(tconf.ToleratePrimaryDisconnection + waits.expectedPoolElectionTimeout(len(nodes))) checkViewNoForNodes(nodes[2:], expectedViewNo=1) sdk_reqs3 = sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 1) with pytest.raises(TimeoutError): req_res = sdk_get_replies(looper, sdk_reqs3, timeout=timeout) sdk_check_reply(req_res[0]) stop_node(nodes[2], looper, nodes) looper.runFor(tconf.ToleratePrimaryDisconnection + waits.expectedPoolElectionTimeout(len(nodes))) checkViewNoForNodes(nodes[3:], expectedViewNo=1) sdk_reqs4 = sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 1) with pytest.raises(TimeoutError): req_res = sdk_get_replies(looper, sdk_reqs4, timeout=timeout) sdk_check_reply(req_res[0]) nodes[2] = start_stopped_node(nodes[2], looper, tconf, tdir, allPluginsPath) looper.runFor(waits.expectedPoolElectionTimeout(len(nodes))) checkViewNoForNodes(nodes[3:], expectedViewNo=1) sdk_reqs5 = sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 1) with pytest.raises(TimeoutError): req_res = sdk_get_replies(looper, sdk_reqs5, timeout=timeout) sdk_check_reply(req_res[0]) nodes[1] = start_stopped_node(nodes[1], looper, tconf, tdir, allPluginsPath) ensureElectionsDone(looper, nodes[1:], numInstances=getRequiredInstances(nodeCount)) checkViewNoForNodes(nodes[1:], expectedViewNo=1) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) nodes[0] = start_stopped_node(nodes[0], looper, tconf, tdir, allPluginsPath) ensureElectionsDone(looper, nodes, numInstances=getRequiredInstances(nodeCount)) checkViewNoForNodes(nodes, expectedViewNo=1) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1)
def test_disconnected_node_with_lagged_view_pulls_up_its_view_on_reconnection( looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle, tconf): """ Verifies that a disconnected node with a lagged view accepts the current view from the other nodes on re-connection. Steps: 1. Provoke view change to 1. 2. Ensure that all the nodes complete view change to 1. 3. Disconnect one node from the rest of the nodes in the pool. 4. Provoke view change to 2. 5. Ensure that all the nodes except for the disconnected one complete view change to 2 and the disconnected node remains in the view 1. 6. Provoke view change to 3. 5. Ensure that all the nodes except for the disconnected one complete view change to 3 and the disconnected node remains in the view 1. 8. Connect the disconnected node to the rest of the nodes in the pool. 9. Ensure that the re-connected node completes view change to 3. 10. Ensure that all the nodes participate in consensus. """ checkViewNoForNodes(txnPoolNodeSet, 0) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) ensure_view_change(looper, txnPoolNodeSet) ensureElectionsDone(looper, txnPoolNodeSet) ensure_all_nodes_have_same_data(looper, txnPoolNodeSet) checkViewNoForNodes(txnPoolNodeSet, 1) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) lagged_node = getNonPrimaryReplicas(txnPoolNodeSet)[-1].node disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, lagged_node, stopNode=False) other_nodes = list(set(txnPoolNodeSet) - {lagged_node}) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) ensure_view_change(looper, other_nodes) ensureElectionsDone(looper, other_nodes, instances_list=range( getRequiredInstances(len(txnPoolNodeSet)))) ensure_all_nodes_have_same_data(looper, other_nodes) checkViewNoForNodes(other_nodes, 2) checkViewNoForNodes([lagged_node], 1) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) ensure_view_change(looper, other_nodes) ensureElectionsDone(looper, other_nodes, instances_list=range( getRequiredInstances(len(txnPoolNodeSet)))) ensure_all_nodes_have_same_data(looper, other_nodes) checkViewNoForNodes(other_nodes, 3) checkViewNoForNodes([lagged_node], 1) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) reconnect_node_and_ensure_connected(looper, txnPoolNodeSet, lagged_node) # The node can start view change, receive NEW_VIEW and start waiting for first ordered in the new view # But since the node is lagged for more than checkpoint, it can not do re-ordering, and has to wait until the # catchjup by checkpoints is started waitForViewChange(looper, [lagged_node], 3, customTimeout=waits.expectedPoolElectionTimeout( len(txnPoolNodeSet))) sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, num_reqs=2 * tconf.CHK_FREQ) ensure_all_nodes_have_same_data(looper, txnPoolNodeSet) checkViewNoForNodes(txnPoolNodeSet, 3) ensureElectionsDone(looper, txnPoolNodeSet) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) ensure_all_nodes_have_same_data(looper, txnPoolNodeSet)
def test_quorum_after_f_plus_2_nodes_including_primary_turned_off_and_later_on( looper, allPluginsPath, tdir, tconf, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client): timeout = sdk_eval_timeout(1, len(txnPoolNodeSet)) nodes = txnPoolNodeSet sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) stop_node(nodes[0], looper, nodes) waitForViewChange(looper, nodes[1:], expectedViewNo=1) ensureElectionsDone(looper, nodes[1:], instances_list=range(getRequiredInstances(nodeCount))) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) stop_node(nodes[1], looper, nodes) looper.runFor(tconf.ToleratePrimaryDisconnection + waits.expectedPoolElectionTimeout(len(nodes))) checkViewNoForNodes(nodes[2:], expectedViewNo=1) sdk_reqs3 = sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 1) with pytest.raises(PoolLedgerTimeoutException): req_res = sdk_get_replies(looper, sdk_reqs3, timeout=timeout) sdk_check_reply(req_res[0]) stop_node(nodes[2], looper, nodes) looper.runFor(tconf.ToleratePrimaryDisconnection + waits.expectedPoolElectionTimeout(len(nodes))) checkViewNoForNodes(nodes[3:], expectedViewNo=1) sdk_reqs4 = sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 1) with pytest.raises(PoolLedgerTimeoutException): req_res = sdk_get_replies(looper, sdk_reqs4, timeout=timeout) sdk_check_reply(req_res[0]) nodes[2] = start_stopped_node(nodes[2], looper, tconf, tdir, allPluginsPath) looper.runFor(waits.expectedPoolElectionTimeout(len(nodes))) checkViewNoForNodes(nodes[3:], expectedViewNo=1) sdk_reqs5 = sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 1) with pytest.raises(PoolLedgerTimeoutException): req_res = sdk_get_replies(looper, sdk_reqs5, timeout=timeout) sdk_check_reply(req_res[0]) nodes[1] = start_stopped_node(nodes[1], looper, tconf, tdir, allPluginsPath) ensureElectionsDone(looper, nodes[1:], instances_list=range(getRequiredInstances(nodeCount)), customTimeout=60) checkViewNoForNodes(nodes[1:], expectedViewNo=1) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) nodes[0] = start_stopped_node(nodes[0], looper, tconf, tdir, allPluginsPath) ensureElectionsDone(looper, nodes, instances_list=range(getRequiredInstances(nodeCount)), customTimeout=60) checkViewNoForNodes(nodes, expectedViewNo=1) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1)
def test_quorum_after_f_plus_2_nodes_but_not_primary_turned_off_and_later_on( looper, allPluginsPath, tdir, tconf, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client): nodes = txnPoolNodeSet sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) stop_node(nodes[4], looper, nodes) looper.runFor(tconf.ToleratePrimaryDisconnection + waits.expectedPoolElectionTimeout(len(nodes))) checkViewNoForNodes(nodes[:4], expectedViewNo=0) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) stop_node(nodes[3], looper, nodes) looper.runFor(tconf.ToleratePrimaryDisconnection + waits.expectedPoolElectionTimeout(len(nodes))) checkViewNoForNodes(nodes[:3], expectedViewNo=0) sdk_reqs3 = sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 1) with pytest.raises(PoolLedgerTimeoutException): req_res = sdk_get_replies(looper, sdk_reqs3) sdk_check_reply(req_res[0]) stop_node(nodes[2], looper, nodes) looper.runFor(tconf.ToleratePrimaryDisconnection + waits.expectedPoolElectionTimeout(len(nodes))) checkViewNoForNodes(nodes[:2], expectedViewNo=0) sdk_reqs4 = sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 1) with pytest.raises(PoolLedgerTimeoutException): req_res = sdk_get_replies(looper, sdk_reqs4) sdk_check_reply(req_res[0]) nodes[4] = start_stopped_node(nodes[4], looper, tconf, tdir, allPluginsPath) looper.runFor(waits.expectedPoolElectionTimeout(len(nodes))) checkViewNoForNodes(nodes[:2] + nodes[4:], expectedViewNo=0) sdk_reqs5 = sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 1) with pytest.raises(PoolLedgerTimeoutException): req_res = sdk_get_replies(looper, sdk_reqs5) sdk_check_reply(req_res[0]) nodes[3] = start_stopped_node(nodes[3], looper, tconf, tdir, allPluginsPath) ensureElectionsDone(looper, nodes[:2] + nodes[3:], numInstances=getRequiredInstances(nodeCount)) checkViewNoForNodes(nodes[:2] + nodes[3:], expectedViewNo=0) sdk_reqs6 = sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 1) sdk_get_replies(looper, [sdk_reqs3[0], sdk_reqs4[0], sdk_reqs5[0], sdk_reqs6[0]]) nodes[2] = start_stopped_node(nodes[2], looper, tconf, tdir, allPluginsPath) ensureElectionsDone(looper, nodes, numInstances=getRequiredInstances(nodeCount)) checkViewNoForNodes(nodes, expectedViewNo=0) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1)
def test_disconnected_node_with_lagged_view_pulls_up_its_view_on_reconnection( looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle): """ Verifies that a disconnected node with a lagged view accepts the current view from the other nodes on re-connection. Steps: 1. Provoke view change to 1. 2. Ensure that all the nodes complete view change to 1. 3. Disconnect one node from the rest of the nodes in the pool. 4. Provoke view change to 2. 5. Ensure that that all the nodes except for the disconnected one complete view change to 2 and the disconnected node remains in the view 1. 6. Provoke view change to 3. 5. Ensure that that all the nodes except for the disconnected one complete view change to 3 and the disconnected node remains in the view 1. 8. Connect the disconnected node to the rest of the nodes in the pool. 9. Ensure that the re-connected node completes view change to 3. 10. Ensure that all the nodes participate in consensus. """ checkViewNoForNodes(txnPoolNodeSet, 0) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) ensure_view_change(looper, txnPoolNodeSet) ensureElectionsDone(looper, txnPoolNodeSet) ensure_all_nodes_have_same_data(looper, txnPoolNodeSet) checkViewNoForNodes(txnPoolNodeSet, 1) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) lagged_node = getNonPrimaryReplicas(txnPoolNodeSet)[-1].node disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, lagged_node, stopNode=False) other_nodes = list(set(txnPoolNodeSet) - {lagged_node}) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) ensure_view_change(looper, other_nodes) ensureElectionsDone(looper, other_nodes, instances_list=range(getRequiredInstances(len(txnPoolNodeSet)))) ensure_all_nodes_have_same_data(looper, other_nodes) checkViewNoForNodes(other_nodes, 2) checkViewNoForNodes([lagged_node], 1) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) ensure_view_change(looper, other_nodes) ensureElectionsDone(looper, other_nodes, instances_list=range(getRequiredInstances(len(txnPoolNodeSet)))) ensure_all_nodes_have_same_data(looper, other_nodes) checkViewNoForNodes(other_nodes, 3) checkViewNoForNodes([lagged_node], 1) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) reconnect_node_and_ensure_connected(looper, txnPoolNodeSet, lagged_node) waitForViewChange(looper, [lagged_node], 3, customTimeout=waits.expectedPoolElectionTimeout( len(txnPoolNodeSet))) ensureElectionsDone(looper, txnPoolNodeSet) ensure_all_nodes_have_same_data(looper, txnPoolNodeSet) checkViewNoForNodes(txnPoolNodeSet, 3) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) ensure_all_nodes_have_same_data(looper, txnPoolNodeSet)
def test_disconnected_node_with_lagged_view_pulls_up_its_view_on_reconnection( looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle): """ Verifies that a disconnected node with a lagged view accepts the current view from the other nodes on re-connection. Steps: 1. Provoke view change to 1. 2. Ensure that all the nodes complete view change to 1. 3. Disconnect one node from the rest of the nodes in the pool. 4. Provoke view change to 2. 5. Ensure that that all the nodes except for the disconnected one complete view change to 2 and the disconnected node remains in the view 1. 6. Provoke view change to 3. 5. Ensure that that all the nodes except for the disconnected one complete view change to 3 and the disconnected node remains in the view 1. 8. Connect the disconnected node to the rest of the nodes in the pool. 9. Ensure that the re-connected node completes view change to 3. 10. Ensure that all the nodes participate in consensus. """ checkViewNoForNodes(txnPoolNodeSet, 0) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) ensure_view_change(looper, txnPoolNodeSet) ensureElectionsDone(looper, txnPoolNodeSet) ensure_all_nodes_have_same_data(looper, txnPoolNodeSet) checkViewNoForNodes(txnPoolNodeSet, 1) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) lagged_node = getNonPrimaryReplicas(txnPoolNodeSet)[-1].node disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, lagged_node, stopNode=False) other_nodes = list(set(txnPoolNodeSet) - {lagged_node}) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) ensure_view_change(looper, other_nodes) ensureElectionsDone(looper, other_nodes, numInstances=getRequiredInstances(len(txnPoolNodeSet))) ensure_all_nodes_have_same_data(looper, other_nodes) checkViewNoForNodes(other_nodes, 2) checkViewNoForNodes([lagged_node], 1) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) ensure_view_change(looper, other_nodes) ensureElectionsDone(looper, other_nodes, numInstances=getRequiredInstances(len(txnPoolNodeSet))) ensure_all_nodes_have_same_data(looper, other_nodes) checkViewNoForNodes(other_nodes, 3) checkViewNoForNodes([lagged_node], 1) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) reconnect_node_and_ensure_connected(looper, txnPoolNodeSet, lagged_node) waitForViewChange(looper, [lagged_node], 3, customTimeout=waits.expectedPoolElectionTimeout( len(txnPoolNodeSet))) ensureElectionsDone(looper, txnPoolNodeSet) ensure_all_nodes_have_same_data(looper, txnPoolNodeSet) checkViewNoForNodes(txnPoolNodeSet, 3) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) ensure_all_nodes_have_same_data(looper, txnPoolNodeSet)
def test_quorum_after_f_plus_2_nodes_but_not_primary_turned_off_and_later_on( looper, allPluginsPath, tdir, tconf, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client): nodes = txnPoolNodeSet sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) stop_node(nodes[4], looper, nodes) looper.runFor(tconf.ToleratePrimaryDisconnection + waits.expectedPoolElectionTimeout(len(nodes))) checkViewNoForNodes(nodes[:4], expectedViewNo=0) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) stop_node(nodes[3], looper, nodes) looper.runFor(tconf.ToleratePrimaryDisconnection + waits.expectedPoolElectionTimeout(len(nodes))) checkViewNoForNodes(nodes[:3], expectedViewNo=0) sdk_reqs3 = sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 1) with pytest.raises(PoolLedgerTimeoutException): req_res = sdk_get_replies(looper, sdk_reqs3) sdk_check_reply(req_res[0]) stop_node(nodes[2], looper, nodes) looper.runFor(tconf.ToleratePrimaryDisconnection + waits.expectedPoolElectionTimeout(len(nodes))) checkViewNoForNodes(nodes[:2], expectedViewNo=0) sdk_reqs4 = sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 1) with pytest.raises(PoolLedgerTimeoutException): req_res = sdk_get_replies(looper, sdk_reqs4) sdk_check_reply(req_res[0]) nodes[4] = start_stopped_node(nodes[4], looper, tconf, tdir, allPluginsPath) looper.runFor(waits.expectedPoolElectionTimeout(len(nodes))) checkViewNoForNodes(nodes[:2] + nodes[4:], expectedViewNo=0) sdk_reqs5 = sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 1) with pytest.raises(PoolLedgerTimeoutException): req_res = sdk_get_replies(looper, sdk_reqs5) sdk_check_reply(req_res[0]) nodes[3] = start_stopped_node(nodes[3], looper, tconf, tdir, allPluginsPath) ensureElectionsDone(looper, nodes[:2] + nodes[3:], instances_list=range(getRequiredInstances(nodeCount))) checkViewNoForNodes(nodes[:2] + nodes[3:], expectedViewNo=0) sdk_reqs6 = sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 1) sdk_get_replies(looper, sdk_reqs6) nodes[2] = start_stopped_node(nodes[2], looper, tconf, tdir, allPluginsPath) ensureElectionsDone(looper, nodes, instances_list=range(getRequiredInstances(nodeCount))) checkViewNoForNodes(nodes, expectedViewNo=0) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1)