def test_demote_promote_restart_after_promotion_7_nodes(txnPoolNodeSet, looper, sdk_pool_handle, sdk_wallet_steward, tdir, tconf, allPluginsPath): demoted_node = txnPoolNodeSet[-1] rest_nodes = [n for n in txnPoolNodeSet if n != demoted_node] starting_view_no = checkViewNoForNodes(txnPoolNodeSet) demote_node(looper, sdk_wallet_steward, sdk_pool_handle, demoted_node) waitForViewChange(looper, rest_nodes, expectedViewNo=starting_view_no + 1) ensureElectionsDone(looper, rest_nodes) ensure_all_nodes_have_same_data(looper, rest_nodes) sdk_send_random_and_check(looper, rest_nodes, sdk_pool_handle, sdk_wallet_steward, 5) starting_view_no = checkViewNoForNodes(rest_nodes) promote_node(looper, sdk_wallet_steward, sdk_pool_handle, demoted_node) waitForViewChange(looper, rest_nodes, expectedViewNo=starting_view_no + 1) ensureElectionsDone(looper, rest_nodes, instances_list=[0, 1, 2]) ensure_all_nodes_have_same_data(looper, rest_nodes) restart_node(looper, txnPoolNodeSet, demoted_node, tconf, tdir, allPluginsPath) ensureElectionsDone(looper, txnPoolNodeSet) sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_steward, sdk_pool_handle)
def testStewardSuspendsNode(looper, txnPoolNodeSet, tdir, tconf, sdk_pool_handle, sdk_wallet_steward, sdk_node_theta_added, poolTxnStewardData, allPluginsPath): new_steward_wallet, new_node = sdk_node_theta_added demote_node(looper, new_steward_wallet, sdk_pool_handle, new_node) # Check suspended node does not exist in any nodeReg or remotes of # nodes or clients txnPoolNodeSet = txnPoolNodeSet[:-1] for node in txnPoolNodeSet: looper.run(eventually(checkNodeNotInNodeReg, node, new_node.name)) # Check that a node does not connect to the suspended # node sdk_ensure_pool_functional(looper, txnPoolNodeSet, new_steward_wallet, sdk_pool_handle) with pytest.raises(RemoteNotFound): looper.loop.run_until_complete(sendMessageAndCheckDelivery(txnPoolNodeSet[0], new_node)) new_node.stop() looper.removeProdable(new_node) # Check that a node whose suspension is revoked can reconnect to other # nodes and clients can also connect to that node promote_node(looper, new_steward_wallet, sdk_pool_handle, new_node) nodeTheta = start_stopped_node(new_node, looper, tconf, tdir, allPluginsPath, delay_instance_change_msgs=False) txnPoolNodeSet.append(nodeTheta) looper.run(checkNodesConnected(txnPoolNodeSet)) sdk_pool_refresh(looper, sdk_pool_handle) sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_steward, sdk_pool_handle)
def testSuspendNode(looper, sdk_pool_handle, sdk_wallet_trustee, nodeSet, tdir, tconf, allPluginsPath): """ Suspend a node and then cancel suspension. Suspend while suspended to test that there is no error """ start_view_no = nodeSet[0].viewNo new_steward_wallet, new_node = sdk_node_theta_added( looper, nodeSet, tdir, tconf, sdk_pool_handle, sdk_wallet_trustee, allPluginsPath, node_config_helper_class=NodeConfigHelper, testNodeClass=TestNode, name="Node-" + randomString(5)) waitForViewChange(looper=looper, txnPoolNodeSet=nodeSet, expectedViewNo=start_view_no + 1) ensureElectionsDone(looper=looper, nodes=nodeSet) demote_node(looper, sdk_wallet_trustee, sdk_pool_handle, new_node) _wait_view_change_finish(looper, nodeSet[:-1], start_view_no + 1) demote_node(looper, sdk_wallet_trustee, sdk_pool_handle, new_node) promote_node(looper, sdk_wallet_trustee, sdk_pool_handle, new_node) _wait_view_change_finish(looper, nodeSet[:-1], start_view_no + 2) promote_node(looper, sdk_wallet_trustee, sdk_pool_handle, new_node)
def testSuspendNode(looper, sdk_pool_handle, sdk_wallet_trustee, newNodeAdded): """ Suspend a node and then cancel suspension. Suspend while suspended to test that there is no error """ new_steward_wallet, new_node = newNodeAdded demote_node(looper, sdk_wallet_trustee, sdk_pool_handle, new_node) demote_node(looper, sdk_wallet_trustee, sdk_pool_handle, new_node) promote_node(looper, sdk_wallet_trustee, sdk_pool_handle, new_node) promote_node(looper, sdk_wallet_trustee, sdk_pool_handle, new_node)
def test_promotion_before_view_change(looper, txnPoolNodeSet, tdir, tconf, allPluginsPath, sdk_wallet_stewards, sdk_pool_handle): sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_stewards[0], 1) assert txnPoolNodeSet[0].master_replica.isPrimary assert txnPoolNodeSet[1].replicas[1].isPrimary assert txnPoolNodeSet[2].replicas[2].isPrimary starting_view_number = checkViewNoForNodes(txnPoolNodeSet) node_2 = txnPoolNodeSet[1] node_3 = txnPoolNodeSet[2] node_5 = txnPoolNodeSet[4] # Demote node 2 steward_2 = sdk_wallet_stewards[1] demote_node(looper, steward_2, sdk_pool_handle, node_2) disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, node_2) looper.removeProdable(node_2) txnPoolNodeSet.remove(node_2) # Checking that view change happened # we are expecting 2 view changes here since Beta is selected as a master Primary on view=1 # (since node reg at the beginning of view 0 is used to select it), but it's not available (demoted), # so we do view change to view=2 by timeout waitForViewChange(looper, txnPoolNodeSet, expectedViewNo=starting_view_number + 2) ensureElectionsDone(looper, txnPoolNodeSet, instances_list=[0, 1]) assert node_3.master_replica.isPrimary # Promoting node 3, increasing replica count node_2 = start_stopped_node(node_2, looper, tconf, tdir, allPluginsPath) promote_node(looper, steward_2, sdk_pool_handle, node_2) txnPoolNodeSet.append(node_2) looper.run(checkNodesConnected(txnPoolNodeSet)) waitForViewChange(looper, txnPoolNodeSet, expectedViewNo=starting_view_number + 3) ensureElectionsDone(looper, txnPoolNodeSet, instances_list=[0, 1, 2]) # node 5 is a primary since promoted node is added at the end of the list assert node_5.master_replica.isPrimary sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_stewards[0], 2) ensure_all_nodes_have_same_data(looper, txnPoolNodeSet)
def test_node_txn_promote_by_endorser(txnPoolNodeSet, sdk_pool_handle, sdk_wallet_trustee, looper, sdk_wallet_handle): validators_before = get_pool_validator_count(txnPoolNodeSet) new_end_did, new_end_verkey = looper.loop.run_until_complete( did.create_and_store_my_did(sdk_wallet_trustee[0], "{}")) # Step 1. Demote node using default auth rule demote_node(looper, sdk_wallet_trustee, sdk_pool_handle, txnPoolNodeSet[-1]) # Check, that node was demoted assert validators_before - get_pool_validator_count( txnPoolNodeSet[:-1]) == 1 # Step 2. Add new Endorser sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_trustee, 'newEndorser', ENDORSER_STRING, verkey=new_end_verkey, dest=new_end_did) new_constraint = AuthConstraint(ENDORSER, 1) # Step 3. Change auth rule, to allowing endorser promote node back sdk_send_and_check_auth_rule_request(looper, sdk_pool_handle, sdk_wallet_trustee, auth_action=EDIT_PREFIX, auth_type=promote_action.txn_type, field=promote_action.field, new_value=promote_action.new_value, old_value=promote_action.old_value, constraint=new_constraint.as_dict) # Step 4. Promote node back, using new Endorser promote_node(looper, (sdk_wallet_handle, new_end_did), sdk_pool_handle, txnPoolNodeSet[-1]) # Check, that all other nodes return previous demoted node back assert validators_before == get_pool_validator_count(txnPoolNodeSet[:-1])
def test_promotion_before_view_change(looper, txnPoolNodeSet, tdir, tconf, allPluginsPath, sdk_wallet_stewards, sdk_pool_handle): sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_stewards[0], 1) assert txnPoolNodeSet[0].master_replica.isPrimary assert txnPoolNodeSet[1].replicas[1].isPrimary assert txnPoolNodeSet[2].replicas[2].isPrimary starting_view_number = checkViewNoForNodes(txnPoolNodeSet) node_2 = txnPoolNodeSet[1] node_3 = txnPoolNodeSet[2] node_4 = txnPoolNodeSet[3] # Demote node 2 steward_2 = sdk_wallet_stewards[1] demote_node(looper, steward_2, sdk_pool_handle, node_2) disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, node_2) looper.removeProdable(node_2) txnPoolNodeSet.remove(node_2) # Checking that view change happened waitForViewChange(looper, txnPoolNodeSet, expectedViewNo=starting_view_number + 1) ensureElectionsDone(looper, txnPoolNodeSet, instances_list=[0, 1]) assert node_3.master_replica.isPrimary # Promoting node 3, increasing replica count node_2 = start_stopped_node(node_2, looper, tconf, tdir, allPluginsPath) promote_node(looper, steward_2, sdk_pool_handle, node_2) txnPoolNodeSet.append(node_2) looper.run(checkNodesConnected(txnPoolNodeSet)) waitForViewChange(looper, txnPoolNodeSet, expectedViewNo=starting_view_number + 3) ensureElectionsDone(looper, txnPoolNodeSet, instances_list=[0, 1, 2]) assert node_4.master_replica.isPrimary sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_stewards[0], 2) ensure_all_nodes_have_same_data(looper, txnPoolNodeSet)
def testSuspendNode(looper, sdk_pool_handle, sdk_wallet_trustee, newNodeAdded, nodeSet): """ Suspend a node and then cancel suspension. Suspend while suspended to test that there is no error """ start_view_no = nodeSet[0].viewNo new_steward_wallet, new_node = newNodeAdded demote_node(looper, sdk_wallet_trustee, sdk_pool_handle, new_node) _wait_view_change_finish(looper, nodeSet[:-1], start_view_no + 1) demote_node(looper, sdk_wallet_trustee, sdk_pool_handle, new_node) promote_node(looper, sdk_wallet_trustee, sdk_pool_handle, new_node) _wait_view_change_finish(looper, nodeSet[:-1], start_view_no + 3) promote_node(looper, sdk_wallet_trustee, sdk_pool_handle, new_node)
def test_demote_promote_restart_after_promotion_from_10_to_4_nodes(txnPoolNodeSet, looper, sdk_pool_handle, sdk_wallet_steward, tdir, tconf, allPluginsPath): """ We expect that 2 changes for f value should be happened """ def demote_another_one(rest_pool): demoted_node = rest_pool[-1] rest_pool = [n for n in rest_pool if n != demoted_node] starting_view_no = checkViewNoForNodes(rest_pool) demote_node(looper, sdk_wallet_steward, sdk_pool_handle, demoted_node) waitForViewChange(looper, rest_pool, expectedViewNo=starting_view_no + 1) ensureElectionsDone(looper, rest_pool, customTimeout=60) ensure_all_nodes_have_same_data(looper, rest_pool) return rest_pool rest_nodes = txnPoolNodeSet etalon_node = txnPoolNodeSet[-1] while len(rest_nodes) > 4: rest_nodes = demote_another_one(rest_nodes) sdk_send_random_and_check(looper, rest_nodes, sdk_pool_handle, sdk_wallet_steward, 5) starting_view_no = checkViewNoForNodes(rest_nodes) promote_node(looper, sdk_wallet_steward, sdk_pool_handle, etalon_node) waitForViewChange(looper, rest_nodes, expectedViewNo=starting_view_no + 1) ensure_all_nodes_have_same_data(looper, rest_nodes) rest_nodes.append(etalon_node) restart_node(looper, rest_nodes, etalon_node, tconf, tdir, allPluginsPath) ensureElectionsDone(looper, rest_nodes) sdk_ensure_pool_functional(looper, rest_nodes, sdk_wallet_steward, sdk_pool_handle)
def test_promotion_leads_to_correct_primary_selection(looper, txnPoolNodeSet, tdir, tconf, allPluginsPath, sdk_wallet_stewards, sdk_pool_handle): # We are saving pool state at moment of last view_change to send it # to newly connected nodes so they could restore primaries basing on this node set. # When current primaries getting edited because of promotion/demotion we don't take this into account. # That lead us to primary inconsistency on different nodes sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_stewards[0], 1) assert txnPoolNodeSet[0].master_replica.isPrimary assert txnPoolNodeSet[1].replicas._replicas[1].isPrimary assert txnPoolNodeSet[2].replicas._replicas[2].isPrimary starting_view_number = checkViewNoForNodes(txnPoolNodeSet) node_1 = txnPoolNodeSet[0] node_3 = txnPoolNodeSet[2] # Demote node 3 steward_3 = sdk_wallet_stewards[2] demote_node(looper, steward_3, sdk_pool_handle, node_3) disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, node_3) looper.removeProdable(node_3) txnPoolNodeSet.remove(node_3) # Checking that view change happened waitForViewChange(looper, txnPoolNodeSet, starting_view_number + 1) assert all(node.replicas.primary_name_by_inst_id == node_1.replicas.primary_name_by_inst_id for node in txnPoolNodeSet) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_stewards[0], 2) for node in txnPoolNodeSet: assert node.f == 1 assert node.replicas.num_replicas == 2 # restart Node1 disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, node_1) looper.removeProdable(node_1) txnPoolNodeSet.remove(node_1) node_1 = start_stopped_node(node_1, looper, tconf, tdir, allPluginsPath) txnPoolNodeSet.append(node_1) # Wait so node_1 could start and catch up waitForViewChange(looper, txnPoolNodeSet, starting_view_number + 1) assert all(node.replicas.primary_name_by_inst_id == node_1.replicas.primary_name_by_inst_id for node in txnPoolNodeSet) ensure_all_nodes_have_same_data(looper, txnPoolNodeSet) # Promoting node 3, increasing replica count node_3 = start_stopped_node(node_3, looper, tconf, tdir, allPluginsPath) promote_node(looper, steward_3, sdk_pool_handle, node_3) txnPoolNodeSet.append(node_3) looper.run(checkNodesConnected(txnPoolNodeSet)) # Wait for view change after promotion waitForViewChange(looper, txnPoolNodeSet, starting_view_number + 2) ensureElectionsDone(looper, txnPoolNodeSet, instances_list=[0, 1, 2]) # Node 3 able to do ordering sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_stewards[0], 2) ensure_all_nodes_have_same_data(looper, txnPoolNodeSet)
def test_promotion_leads_to_primary_inconsistency(looper, txnPoolNodeSet, tdir, tconf, allPluginsPath, sdk_wallet_stewards, sdk_pool_handle): # We are saving pool state at moment of last view_change to send it # to newly connected nodes so they could restore primaries basing on this node set. # When current primaries getting edited because of promotion/demotion we don't take this into account. # That lead us to primary inconsistency on different nodes sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_stewards[0], 1) assert txnPoolNodeSet[0].master_replica.isPrimary assert txnPoolNodeSet[1].replicas._replicas[1].isPrimary assert txnPoolNodeSet[2].replicas._replicas[2].isPrimary starting_view_number = checkViewNoForNodes(txnPoolNodeSet) # Demote node 3 node_3 = txnPoolNodeSet[2] steward_3 = sdk_wallet_stewards[2] demote_node(looper, steward_3, sdk_pool_handle, node_3) disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, node_3) looper.removeProdable(node_3) txnPoolNodeSet.remove(node_3) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_stewards[0], 2) for node in txnPoolNodeSet: assert node.f == 1 assert node.replicas.num_replicas == 2 # Force a view change by stopping master. In this moment we are saving pool state (without 3rd node) node_1 = txnPoolNodeSet[0] disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, node_1) looper.removeProdable(node_1) txnPoolNodeSet.remove(node_1) # Checking that view change happened ensureElectionsDone(looper, txnPoolNodeSet, instances_list=[0, 1]) view_number = checkViewNoForNodes(txnPoolNodeSet) assert view_number == starting_view_number + 1 node_1 = start_stopped_node(node_1, looper, tconf, tdir, allPluginsPath) txnPoolNodeSet.append(node_1) # Wait so node_1 could start and finish view_change looper.runFor(1) # Promoting node 3, increasing replica count node_3 = start_stopped_node(node_3, looper, tconf, tdir, allPluginsPath) promote_node(looper, steward_3, sdk_pool_handle, node_3) txnPoolNodeSet.append(node_3) looper.run(checkNodesConnected(txnPoolNodeSet)) ensureElectionsDone(looper, txnPoolNodeSet, instances_list=[0, 1, 2]) # Node 3 able to do ordering sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_stewards[0], 2) view_number = checkViewNoForNodes(txnPoolNodeSet) assert view_number == starting_view_number + 2 ensure_all_nodes_have_same_data(looper, txnPoolNodeSet) # But it has different primary, cause it uses nodeReg without itself to calculate primaries assert all(node.replicas.primary_name_by_inst_id == node_1.replicas.primary_name_by_inst_id for node in txnPoolNodeSet if node is not node_3) # Fails assert all(node.replicas.primary_name_by_inst_id == node_1.replicas.primary_name_by_inst_id for node in txnPoolNodeSet)