def txnPoolNodeSet(txnPoolNodeSet, looper, sdk_pool_handle, sdk_wallet_steward, tconf, tdirWithPoolTxns, allPluginsPath): logger.debug("Do several view changes to round the list of primaries") assert txnPoolNodeSet[0].viewNo == len(txnPoolNodeSet) - 1 logger.debug( "Do view change to reach viewNo {}".format(txnPoolNodeSet[0].viewNo + 1)) ensure_view_change_complete(looper, txnPoolNodeSet) logger.debug("Send requests to ensure that pool is working properly, " "viewNo: {}".format(txnPoolNodeSet[0].viewNo)) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 3) logger.debug("Pool is ready, current viewNo: {}".format( txnPoolNodeSet[0].viewNo)) # TODO find out and fix why additional view change could happen # because of degarded master. It's critical for current test to have # view change completed for the time when new node is joining. # Thus, disable master degradation check as it won't impact the case # and guarantees necessary state. for node in txnPoolNodeSet: node.monitor.isMasterDegraded = lambda: False return txnPoolNodeSet
def test_restart_half_to_lower_view(looper, txnPoolNodeSet, tconf, tdir, allPluginsPath, sdk_pool_handle, sdk_wallet_client): # Add transaction to ledger sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) # Move to higher view ensure_view_change_complete(looper, txnPoolNodeSet) # Restart half of nodes tm = tconf.ToleratePrimaryDisconnection + waits.expectedPoolElectionTimeout( len(txnPoolNodeSet)) nodes_before_restart = txnPoolNodeSet.copy() restart_nodes(looper, txnPoolNodeSet, txnPoolNodeSet[2:], tconf, tdir, allPluginsPath, after_restart_timeout=tm, start_one_by_one=False) # Check that nodes didn't think they may have inconsistent 3PC state for node in nodes_before_restart: assert node.spylog.count(node.on_inconsistent_3pc_state) == 0 # Check that nodes don't think they may have inconsistent 3PC state for node in txnPoolNodeSet: assert node.spylog.count(node.on_inconsistent_3pc_state) == 0 # Check that all nodes are still functional sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle)
def test_last_ordered_3pc_reset_if_more_than_new_view(txnPoolNodeSet, looper, sdk_pool_handle, sdk_wallet_client): """ Check that if last_ordered_3pc's viewNo on a Replica is greater than the new viewNo after view change, then last_ordered_3pc is reset to (0,0). It can be that last_ordered_3pc was set for the previous view, since it's set during catch-up Example: a Node has last_ordered = (1, 300), and then the whole pool except this node restarted. The new viewNo is 0, but last_ordered is (1, 300), so all new requests will be discarded by this Node if we don't reset last_ordered_3pc """ old_view_no = checkViewNoForNodes(txnPoolNodeSet) for node in txnPoolNodeSet: node.master_replica.last_ordered_3pc = (old_view_no + 2, 100) ensure_view_change_complete(looper, txnPoolNodeSet, customTimeout=60) view_no = checkViewNoForNodes(txnPoolNodeSet) for node in txnPoolNodeSet: assert (view_no, 0) == node.master_replica.last_ordered_3pc # Make sure the pool is working sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 5) ensure_all_nodes_have_same_data(looper, txnPoolNodeSet)
def test_ledger_status_after_catchup(looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle): # we expect last ordered 3PC is not None for Domain ledger only, as there is a txn added to Domain ledger sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) ensure_view_change_complete(looper, txnPoolNodeSet) ensure_all_nodes_have_same_data(looper, txnPoolNodeSet) check_ledger_statuses(txnPoolNodeSet)
def test_revert_auth_rule_changing(looper, txnPoolNodeSet, sdk_wallet_trustee, sdk_wallet_steward, sdk_pool_handle): node_stashers = [n.nodeIbStasher for n in txnPoolNodeSet] wh, _ = sdk_wallet_trustee new_steward_did, new_steward_verkey = create_verkey_did(looper, wh) new_steward_did2, new_steward_verkey2 = create_verkey_did(looper, wh) """We try to change rule for adding new steward. For this case we """ changed_constraint = AuthConstraint(role=STEWARD, sig_count=1) action = AuthActionAdd(txn_type=NYM, field=ROLE, value=STEWARD) with delay_rules_without_processing( node_stashers, pDelay(), cDelay(), msg_rep_delay(types_to_delay=[PREPARE, COMMIT])): sdk_send_and_check_auth_rule_request( looper, sdk_pool_handle, sdk_wallet_trustee, auth_action=ADD_PREFIX, auth_type=action.txn_type, field=action.field, new_value=action.value, old_value=None, constraint=changed_constraint.as_dict, no_wait=True) looper.runFor(waits.expectedPrePrepareTime(len(txnPoolNodeSet))) """ Try to add new steward by already existed trustee. Validation should raise exception because we change uncommitted state by adding new rule, that "Only steward can add new steward" """ with pytest.raises(RequestRejectedException, match="Not enough STEWARD signatures"): sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_trustee, 'newSteward1', STEWARD_STRING, dest=new_steward_did, verkey=new_steward_verkey) looper.runFor(waits.expectedPrePrepareTime(len(txnPoolNodeSet))) """ Catchup should revert config_state and discard rule changing """ for n in txnPoolNodeSet: n.requests.clear() ensure_view_change_complete(looper, txnPoolNodeSet) """ Try to create new steward by steward We can not do this, because AUTH_RULE txn was reverted """ with pytest.raises(RequestRejectedException, match="Not enough TRUSTEE signatures"): sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_steward, 'newSteward2', STEWARD_STRING, dest=new_steward_did2, verkey=new_steward_verkey2)
def test_revert_for_all_after_view_change(looper, helpers, nodeSetWithIntegratedTokenPlugin, sdk_pool_handle, fees_set, mint_tokens, addresses, fees): node_set = nodeSetWithIntegratedTokenPlugin current_amount = get_amount_from_token_txn(mint_tokens) seq_no = get_seq_no(mint_tokens) reverted_node = nodeSetWithIntegratedTokenPlugin[-1] current_amount, seq_no, _ = send_and_check_nym_with_fees( helpers, fees_set, seq_no, looper, addresses, current_amount) current_amount, seq_no, _ = send_and_check_transfer( helpers, addresses, fees, looper, current_amount, seq_no) ensure_all_nodes_have_same_data(looper, node_set) with delay_rules([n.nodeIbStasher for n in node_set], cDelay(), pDelay()): len_batches_before = len(reverted_node.master_replica.batches) current_amount, seq_no, resp1 = send_and_check_transfer( helpers, addresses, fees, looper, current_amount, seq_no, check_reply=False) current_amount, seq_no, resp2 = send_and_check_nym_with_fees( helpers, fees_set, seq_no, looper, addresses, current_amount, check_reply=False) looper.runFor( waits.expectedPrePrepareTime( len(nodeSetWithIntegratedTokenPlugin))) len_batches_after = len(reverted_node.master_replica.batches) """ Checks, that we have a 2 new batches """ assert len_batches_after - len_batches_before == 2 for n in node_set: n.view_changer.on_master_degradation() ensure_view_change_complete(looper, nodeSetWithIntegratedTokenPlugin) looper.run( eventually( lambda: assertExp(reverted_node.mode == Mode.participating))) ensure_all_nodes_have_same_data(looper, node_set) sdk_get_and_check_replies(looper, resp1) sdk_get_and_check_replies(looper, resp2) send_and_check_nym_with_fees(helpers, fees_set, seq_no, looper, addresses, current_amount) ensure_all_nodes_have_same_data(looper, node_set)
def test_ledger_status_after_catchup(looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle): # we expect last ordered 3PC is not None for Domain ledger only, as there is a txn added to Domain ledger sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) ensure_view_change_complete(looper, txnPoolNodeSet) ensure_all_nodes_have_same_data(looper, txnPoolNodeSet) check_ledger_statuses(txnPoolNodeSet, pool_last_ordered_3pc=(None, None), domain_last_ordered_3pc=txnPoolNodeSet[0].master_last_ordered_3PC, config_last_ordered_3pc=(None, None))
def test_view_change_can_finish_despite_perpetual_catchup(txnPoolNodeSet, looper, tconf): # Make nodes think that there is perpetual catchup old_methods = [n.num_txns_caught_up_in_last_catchup for n in txnPoolNodeSet] for n in txnPoolNodeSet: n.num_txns_caught_up_in_last_catchup = types.MethodType(lambda _n: 10, n) ensure_view_change_complete(looper, txnPoolNodeSet) # Restore node behaviour for n, old_method in zip(txnPoolNodeSet, old_methods): n.num_txns_caught_up_in_last_catchup = old_method
def test_restart_majority_to_lower_view(looper, txnPoolNodeSet, tconf, tdir, allPluginsPath, sdk_pool_handle, sdk_wallet_client): # Add transaction to ledger sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) # Move to higher view ensure_view_change_complete(looper, txnPoolNodeSet) majority = txnPoolNodeSet[:3] minority = txnPoolNodeSet[3:] # Restart majority group tm = tconf.ToleratePrimaryDisconnection + waits.expectedPoolElectionTimeout( len(txnPoolNodeSet)) majority_before_restart = majority.copy() restart_nodes(looper, txnPoolNodeSet, majority, tconf, tdir, allPluginsPath, after_restart_timeout=tm, start_one_by_one=False, wait_for_elections=False) ensureElectionsDone(looper, majority, numInstances=2) # Check that nodes in minority group are aware that they might have inconsistent 3PC state for node in minority: assert node.spylog.count(node.on_inconsistent_3pc_state) == 1 # Check that nodes in majority group didn't think they might have inconsistent 3PC state for node in majority_before_restart: assert node.spylog.count(node.on_inconsistent_3pc_state) == 0 # Check that nodes in majority group don't think they might have inconsistent 3PC state for node in majority: assert node.spylog.count(node.on_inconsistent_3pc_state) == 0 # Restart minority group restart_nodes(looper, txnPoolNodeSet, minority, tconf, tdir, allPluginsPath, after_restart_timeout=tm, start_one_by_one=False) # Check that all nodes are still functional sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle)
def txnPoolNodeSet(txnPoolNodeSet, looper, client1, wallet1, client1Connected, tconf, tdirWithPoolTxns, allPluginsPath): logger.debug("Do several view changes to round the list of primaries") assert txnPoolNodeSet[0].viewNo == len(txnPoolNodeSet) - 1 logger.debug( "Do view change to reach viewNo {}".format(txnPoolNodeSet[0].viewNo + 1)) ensure_view_change_complete(looper, txnPoolNodeSet) logger.debug("Send requests to ensure that pool is working properly, " "viewNo: {}".format(txnPoolNodeSet[0].viewNo)) sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, numReqs=3) return txnPoolNodeSet
def test_view_change_can_finish_despite_perpetual_catchup( txnPoolNodeSet, looper, tconf): # Make nodes think that there is perpetual catchup old_methods = [ n.num_txns_caught_up_in_last_catchup for n in txnPoolNodeSet ] for n in txnPoolNodeSet: n.num_txns_caught_up_in_last_catchup = types.MethodType( lambda _n: 10, n) ensure_view_change_complete(looper, txnPoolNodeSet) # Restore node behaviour for n, old_method in zip(txnPoolNodeSet, old_methods): n.num_txns_caught_up_in_last_catchup = old_method
def test_last_ordered_3pc_not_reset_if_more_than_new_view( txnPoolNodeSet, looper, sdk_pool_handle, sdk_wallet_client): """ Check that if last_ordered_3pc's viewNo on a Replica is greater than the new viewNo after view change, then last_ordered_3pc is not reset. It can be that last_ordered_3pc was set for the next view, since it's set during catch-up """ old_view_no = checkViewNoForNodes(txnPoolNodeSet) for node in txnPoolNodeSet: node.master_replica.last_ordered_3pc = (old_view_no + 2, 100) ensure_view_change_complete(looper, txnPoolNodeSet, customTimeout=60) for node in txnPoolNodeSet: assert (old_view_no + 2, 100) == node.master_replica.last_ordered_3pc
def test_last_ordered_3pc_not_reset_if_less_than_new_view( txnPoolNodeSet, looper, sdk_pool_handle, sdk_wallet_client): """ Check that if last_ordered_3pc's viewNo on a Replica is less than the new viewNo after view change, then last_ordered_3pc is not reset. """ old_view_no = checkViewNoForNodes(txnPoolNodeSet) for node in txnPoolNodeSet: node.master_replica.last_ordered_3pc = (old_view_no, 100) ensure_view_change_complete(looper, txnPoolNodeSet, customTimeout=60) for node in txnPoolNodeSet: assert (old_view_no + 1, 1) == node.master_replica.last_ordered_3pc # Make sure the pool is working sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 5) ensure_all_nodes_have_same_data(looper, txnPoolNodeSet)
def test_last_ordered_3pc_not_reset_if_equal_to_new_view( txnPoolNodeSet, looper, sdk_pool_handle, sdk_wallet_client): """ Check that if last_ordered_3pc's viewNo on a Replica is equal to the new viewNo after view change, then last_ordered_3pc is not reset. It can be that last_ordered_3pc was set for the next view, since it's set during catch-up """ global batches_count old_view_no = checkViewNoForNodes(txnPoolNodeSet) for node in txnPoolNodeSet: node.master_replica.last_ordered_3pc = (old_view_no + 1, batches_count) node.master_replica._ordering_service.lastPrePrepareSeqNo = batches_count ensure_view_change_complete(looper, txnPoolNodeSet, customTimeout=60) batches_count += 1 for node in txnPoolNodeSet: assert (old_view_no + 1, batches_count) == node.master_replica.last_ordered_3pc
def test_last_ordered_3pc_not_reset_if_less_than_new_view(txnPoolNodeSet, looper, sdk_pool_handle, sdk_wallet_client): """ Check that if last_ordered_3pc's viewNo on a Replica is equal to the new viewNo after view change, then last_ordered_3pc is reset to (0,0). It can be that last_ordered_3pc was set for the previous view, since it's set during catch-up Example: a Node has last_ordered = (1, 300), and then the whole pool except this node restarted. The new viewNo is 0, but last_ordered is (1, 300), so all new requests will be discarded by this Node if we don't reset last_ordered_3pc """ old_view_no = checkViewNoForNodes(txnPoolNodeSet) for node in txnPoolNodeSet: node.master_replica.last_ordered_3pc = (old_view_no, 100) ensure_view_change_complete(looper, txnPoolNodeSet, customTimeout=60) for node in txnPoolNodeSet: assert (old_view_no, 100) == node.master_replica.last_ordered_3pc # Make sure the pool is working sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 5) ensure_all_nodes_have_same_data(looper, txnPoolNodeSet)
def test_restart_majority_to_lower_view(looper, txnPoolNodeSet, tconf, tdir, allPluginsPath, sdk_pool_handle, sdk_wallet_client): # Add transaction to ledger sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) # Move to higher view ensure_view_change_complete(looper, txnPoolNodeSet) majority = txnPoolNodeSet[:3] minority = txnPoolNodeSet[3:] # Restart majority group tm = tconf.ToleratePrimaryDisconnection + waits.expectedPoolElectionTimeout(len(txnPoolNodeSet)) majority_before_restart = majority.copy() restart_nodes(looper, txnPoolNodeSet, majority, tconf, tdir, allPluginsPath, after_restart_timeout=tm, start_one_by_one=False, wait_for_elections=False) ensureElectionsDone(looper, majority, instances_list=range(2)) # Check that nodes in minority group are aware that they might have inconsistent 3PC state for node in minority: assert node.spylog.count(node.on_inconsistent_3pc_state) == 1 # Check that nodes in majority group didn't think they might have inconsistent 3PC state for node in majority_before_restart: assert node.spylog.count(node.on_inconsistent_3pc_state) == 0 # Check that nodes in majority group don't think they might have inconsistent 3PC state for node in majority: assert node.spylog.count(node.on_inconsistent_3pc_state) == 0 # Restart minority group restart_nodes(looper, txnPoolNodeSet, minority, tconf, tdir, allPluginsPath, after_restart_timeout=tm, start_one_by_one=False) # Check that all nodes are still functional sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle)
def test_restart_half_to_lower_view(looper, txnPoolNodeSet, tconf, tdir, allPluginsPath, sdk_pool_handle, sdk_wallet_client): # Add transaction to ledger sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) # Move to higher view ensure_view_change_complete(looper, txnPoolNodeSet) # Restart half of nodes tm = tconf.ToleratePrimaryDisconnection + waits.expectedPoolElectionTimeout(len(txnPoolNodeSet)) nodes_before_restart = txnPoolNodeSet.copy() restart_nodes(looper, txnPoolNodeSet, txnPoolNodeSet[2:], tconf, tdir, allPluginsPath, after_restart_timeout=tm, start_one_by_one=False) # Check that nodes didn't think they may have inconsistent 3PC state for node in nodes_before_restart: assert node.spylog.count(node.on_inconsistent_3pc_state) == 0 # Check that nodes don't think they may have inconsistent 3PC state for node in txnPoolNodeSet: assert node.spylog.count(node.on_inconsistent_3pc_state) == 0 # Check that all nodes are still functional sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle, num_reqs=2, num_batches=1)
def test_primary_selection_after_primary_demotion_and_view_changes( looper, txnPoolNodeSet, stewardAndWalletForMasterNode, txnPoolMasterNodes): """ Demote primary and do multiple view changes forcing primaries rotation. Demoted primary should be skipped without additional view changes. """ viewNo0 = checkViewNoForNodes(txnPoolNodeSet) logger.info( "1. turn off the node which has primary replica for master instanse, " " this should trigger view change") master_node = txnPoolMasterNodes[0] client, wallet = stewardAndWalletForMasterNode node_data = {ALIAS: master_node.name, SERVICES: []} updateNodeData(looper, client, wallet, master_node, node_data) restNodes = [node for node in txnPoolNodeSet \ if node.name != master_node.name] ensureElectionsDone(looper, restNodes) viewNo1 = checkViewNoForNodes(restNodes) assert viewNo1 == viewNo0 + 1 assert master_node.viewNo == viewNo0 assert len(restNodes[0].replicas) == 1 # only one instance left assert restNodes[0].replicas[0].primaryName != master_node.name # ensure pool is working properly sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, numReqs=3) logger.info("2. force view change 2 and check final viewNo") ensure_view_change_complete(looper, restNodes) viewNo2 = checkViewNoForNodes(restNodes) assert restNodes[0].replicas[0].primaryName != master_node.name assert viewNo2 == viewNo1 + 1 sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, numReqs=3) logger.info("3. force view change 3 and check final viewNo") ensure_view_change_complete(looper, restNodes) viewNo3 = checkViewNoForNodes(restNodes) assert restNodes[0].replicas[0].primaryName != master_node.name assert viewNo3 == viewNo2 + 1 sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, numReqs=3) logger.info("4. force view change 4 and check final viewNo") ensure_view_change_complete(looper, restNodes) viewNo4 = checkViewNoForNodes(restNodes) assert restNodes[0].replicas[0].primaryName != master_node.name assert viewNo4 == viewNo3 + 1 sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, numReqs=3)
def txnPoolNodeSet(txnPoolNodeSet, looper, sdk_pool_handle, sdk_wallet_steward, tconf, tdirWithPoolTxns, allPluginsPath): logger.debug("Do several view changes to round the list of primaries") assert txnPoolNodeSet[0].viewNo == len(txnPoolNodeSet) - 1 logger.debug("Do view change to reach viewNo {}".format(txnPoolNodeSet[0].viewNo + 1)) ensure_view_change_complete(looper, txnPoolNodeSet) logger.debug("Send requests to ensure that pool is working properly, " "viewNo: {}".format(txnPoolNodeSet[0].viewNo)) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 3) logger.debug("Pool is ready, current viewNo: {}".format(txnPoolNodeSet[0].viewNo)) # TODO find out and fix why additional view change could happen # because of degarded master. It's critical for current test to have # view change completed for the time when new node is joining. # Thus, disable master degradation check as it won't impact the case # and guarantees necessary state. for node in txnPoolNodeSet: node.monitor.isMasterDegraded = lambda: False return txnPoolNodeSet
def test_view_change_gc_in_between_3pc_all_nodes_delays( looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client): """ Test that garbage collector compares the whole 3PC key (viewNo, ppSeqNo) and does not remove messages from node's queues that have higher viewNo than last ordered one even if their ppSeqNo are less or equal """ numNodes = len(txnPoolNodeSet) viewNo = checkViewNoForNodes(txnPoolNodeSet) # 1 send two messages one by one separately to make # node pool working with two batches # -> last_ordered_3pc = (+0, 2) [+0 means from the initial state] # (last_ordered_3pc here and futher is tracked # for master instances only cause non-master ones have # specific logic of its management which we don't care in # the test, see Replica::_setup_for_non_master) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) last_ordered_3pc = (viewNo, 2) check_nodes_last_ordered_3pc(txnPoolNodeSet, last_ordered_3pc) check_nodes_requests_size(txnPoolNodeSet, 2) # 2 do view change # -> GC should remove it from nodes' queues # -> viewNo = +1 ensure_view_change_complete(looper, txnPoolNodeSet) viewNo = checkViewNoForNodes(txnPoolNodeSet, viewNo + 1) check_nodes_last_ordered_3pc(txnPoolNodeSet, last_ordered_3pc) check_nodes_requests_size(txnPoolNodeSet, 0) # 3 slow processing 3PC messages for all nodes (all replica instances) # randomly and send one more message # -> not ordered (last_ordered_3pc still equal (+0, 2)) but primaries # should at least send PRE-PREPAREs # TODO could it be not enough for wainting that at least primary # has sent PRE-PREPARE propagationTimeout = waits.expectedClientRequestPropagationTime(numNodes) delay_3pc_messages(txnPoolNodeSet, 0, delay=propagationTimeout * 2) delay_3pc_messages(txnPoolNodeSet, 1, delay=propagationTimeout * 2) requests = sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client) def checkPrePrepareSentAtLeastByPrimary(): for node in txnPoolNodeSet: for replica in node.replicas.values(): if replica.isPrimary: assert len(replica.sentPrePrepares) looper.run(eventually(checkPrePrepareSentAtLeastByPrimary, retryWait=0.1, timeout=propagationTimeout)) # 4 do view change # -> GC shouldn't remove anything because # last_ordered_3pc (+0, 1) < last message's 3pc key (+1, 1) # -> viewNo = 2 ensure_view_change_complete(looper, txnPoolNodeSet) viewNoNew = checkViewNoForNodes(txnPoolNodeSet) # another view change could happen because of slow nodes assert viewNoNew - viewNo in (1, 2) viewNo = viewNoNew check_nodes_last_ordered_3pc(txnPoolNodeSet, last_ordered_3pc) check_nodes_requests_size(txnPoolNodeSet, 1) # 5 reset delays and wait for replies # -> new primaries should send new 3pc for last message # with 3pc key (+2, 1) # -> they should be ordered # -> last_ordered_3pc = (+2, 1) reset_delays_and_process_delayeds(txnPoolNodeSet) sdk_get_replies(looper, [requests]) checkViewNoForNodes(txnPoolNodeSet, viewNo) last_ordered_3pc = (viewNo, 1) check_nodes_last_ordered_3pc(txnPoolNodeSet, last_ordered_3pc) check_nodes_requests_size(txnPoolNodeSet, 1) # 6 do view change # -> GC should remove them ensure_view_change_complete(looper, txnPoolNodeSet) viewNo = checkViewNoForNodes(txnPoolNodeSet, viewNo + 1) check_nodes_last_ordered_3pc(txnPoolNodeSet, last_ordered_3pc) check_nodes_requests_size(txnPoolNodeSet, 0)
def test_view_change_gc_in_between_3pc_all_nodes_delays( looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client): """ Test that garbage collector compares the whole 3PC key (viewNo, ppSeqNo) and does not remove messages from node's queues that have higher viewNo than last ordered one even if their ppSeqNo are less or equal """ numNodes = len(txnPoolNodeSet) viewNo = checkViewNoForNodes(txnPoolNodeSet) # 1 send two messages one by one separately to make # node pool working with two batches # -> last_ordered_3pc = (+0, 2) [+0 means from the initial state] # (last_ordered_3pc here and futher is tracked # for master instances only cause non-master ones have # specific logic of its management which we don't care in # the test, see Replica::_setup_for_non_master) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) batches_count = get_pp_seq_no(txnPoolNodeSet) last_ordered_3pc = (viewNo, batches_count) check_nodes_last_ordered_3pc(txnPoolNodeSet, last_ordered_3pc) check_nodes_requests_size(txnPoolNodeSet, 2) # 2 do view change # -> GC should remove it from nodes' queues # -> viewNo = +1 ensure_view_change_complete(looper, txnPoolNodeSet) batches_count += 1 viewNo = checkViewNoForNodes(txnPoolNodeSet, viewNo + 1) looper.run( eventually(check_nodes_last_ordered_3pc, txnPoolNodeSet, (viewNo, batches_count))) check_nodes_requests_size(txnPoolNodeSet, 0) # 3 slow processing 3PC messages for all nodes (all replica instances) # randomly and send one more message # -> not ordered (last_ordered_3pc still equal (+0, 2)) but primaries # should at least send PRE-PREPAREs # TODO could it be not enough for wainting that at least primary # has sent PRE-PREPARE propagationTimeout = waits.expectedClientRequestPropagationTime(numNodes) delay_3pc_messages(txnPoolNodeSet, 0, delay=propagationTimeout * 2) delay_3pc_messages(txnPoolNodeSet, 1, delay=propagationTimeout * 2) requests = sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client) def checkPrePrepareSentAtLeastByPrimary(): for node in txnPoolNodeSet: for replica in node.replicas.values(): if replica.isPrimary: assert len(replica._ordering_service.sent_preprepares) looper.run( eventually(checkPrePrepareSentAtLeastByPrimary, retryWait=0.1, timeout=propagationTimeout)) # 4 do view change # -> GC shouldn't remove anything because # last_ordered_3pc (+0, 1) < last message's 3pc key (+1, 1) # -> viewNo = 2 ensure_view_change_complete(looper, txnPoolNodeSet) batches_count += 1 viewNoNew = checkViewNoForNodes(txnPoolNodeSet) # another view change could happen because of slow nodes assert viewNoNew - viewNo in (1, 2) viewNo = viewNoNew check_nodes_last_ordered_3pc(txnPoolNodeSet, (last_ordered_3pc[0] + 1, batches_count - 1)) check_nodes_requests_size(txnPoolNodeSet, 1) # 5 reset delays and wait for replies # -> new primaries should send new 3pc for last message # with 3pc key (+2, 1) # -> they should be ordered # -> last_ordered_3pc = (+2, 1) reset_delays_and_process_delayeds(txnPoolNodeSet) sdk_get_replies(looper, [requests]) batches_count += 1 checkViewNoForNodes(txnPoolNodeSet, viewNo) last_ordered_3pc = (viewNo, batches_count) check_nodes_last_ordered_3pc(txnPoolNodeSet, last_ordered_3pc) check_nodes_requests_size(txnPoolNodeSet, 1) # 6 do view change # -> GC should remove them ensure_view_change_complete(looper, txnPoolNodeSet) batches_count += 1 viewNo = checkViewNoForNodes(txnPoolNodeSet, viewNo + 1) check_nodes_last_ordered_3pc(txnPoolNodeSet, (last_ordered_3pc[0] + 1, batches_count)) check_nodes_requests_size(txnPoolNodeSet, 0)
def view_setup(looper, txnPoolNodeSet): for i in range(2): ensure_view_change_complete(looper, txnPoolNodeSet) for node in txnPoolNodeSet: assert node.viewNo == 2
def test_primary_selection_after_demoted_node_promotion( looper, txnPoolNodeSet, sdk_node_theta_added, sdk_pool_handle, tconf, tdir, allPluginsPath): """ Demote non-primary node Promote it again Restart one node to get the following difference with others: - not restarted - node registry and related pool parameters are kept in memory in some state which is expected as the same as in the pool ledger - restarted one - loaded node registry and pool parameters from the pool ledger at startup Do several view changes and check that all nodes will choose previously demoted / promoted node as a primary for some instanse """ new_steward_wallet, new_node = sdk_node_theta_added # viewNo0 = checkViewNoForNodes(txnPoolNodeSet) check_all_nodes_the_same_pool_list(txnPoolNodeSet) logger.info("1. Demote node Theta") node_dest = hexToFriendly(new_node.nodestack.verhex) sdk_send_update_node(looper, new_steward_wallet, sdk_pool_handle, node_dest, new_node.name, None, None, None, None, []) remainingNodes = list(set(txnPoolNodeSet) - {new_node}) check_all_nodes_the_same_pool_list(remainingNodes) # ensure pool is working properly sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, new_steward_wallet, 3) # TODO view change might happen unexpectedly by unknown reason # checkViewNoForNodes(remainingNodes, expectedViewNo=viewNo0) logger.info("2. Promote node Theta back") sdk_send_update_node(looper, new_steward_wallet, sdk_pool_handle, node_dest, new_node.name, None, None, None, None, [VALIDATOR]) check_all_nodes_the_same_pool_list(txnPoolNodeSet) # ensure pool is working properly sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, new_steward_wallet, 3) # checkViewNoForNodes(txnPoolNodeSet, expectedViewNo=viewNo0) logger.info("3. Restart one node") stopped_node = txnPoolNodeSet[0] disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, stopped_node, stopNode=True) looper.removeProdable(stopped_node) remainingNodes = list(set(txnPoolNodeSet) - {stopped_node}) ensureElectionsDone(looper, remainingNodes) # ensure pool is working properly sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, new_steward_wallet, 3) # checkViewNoForNodes(remainingNodes, expectedViewNo=viewNo0) # start node restartedNode = start_stopped_node(stopped_node, looper, tconf, tdir, allPluginsPath) txnPoolNodeSet = remainingNodes + [restartedNode] ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet) # ensure pool is working properly sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, new_steward_wallet, 3) # checkViewNoForNodes(txnPoolNodeSet, expectedViewNo=viewNo0) logger.info("4. Do view changes to check that nodeTheta will be chosen " "as a primary for some instance by all nodes after some rounds") while txnPoolNodeSet[0].viewNo < 4: ensure_view_change_complete(looper, txnPoolNodeSet) # ensure pool is working properly sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, new_steward_wallet, 3)
def test_primary_selection_after_primary_demotion_and_view_changes(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, txnPoolMasterNodes): """ Demote primary and do multiple view changes forcing primaries rotation. Demoted primary should be skipped without additional view changes. """ viewNo0 = checkViewNoForNodes(txnPoolNodeSet) logger.info("1. turn off the node which has primary replica for master instanse, " " this should trigger view change") master_node = txnPoolMasterNodes[0] node_dest = hexToFriendly(master_node.nodestack.verhex) sdk_send_update_node(looper, sdk_wallet_steward, sdk_pool_handle, node_dest, master_node.name, None, None, None, None, services=[]) restNodes = [node for node in txnPoolNodeSet \ if node.name != master_node.name] ensureElectionsDone(looper, restNodes) viewNo1 = checkViewNoForNodes(restNodes) assert viewNo1 == viewNo0 + 1 assert master_node.viewNo == viewNo0 assert len(restNodes[0].replicas) == 1 # only one instance left assert restNodes[0].replicas[0].primaryName != master_node.name # ensure pool is working properly sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 3) logger.info("2. force view change 2 and check final viewNo") ensure_view_change_complete(looper, restNodes) viewNo2 = checkViewNoForNodes(restNodes) assert restNodes[0].replicas[0].primaryName != master_node.name assert viewNo2 == viewNo1 + 1 sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 3) logger.info("3. force view change 3 and check final viewNo") ensure_view_change_complete(looper, restNodes) viewNo3 = checkViewNoForNodes(restNodes) assert restNodes[0].replicas[0].primaryName != master_node.name assert viewNo3 == viewNo2 + 1 sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 3) logger.info("4. force view change 4 and check final viewNo") ensure_view_change_complete(looper, restNodes) viewNo4 = checkViewNoForNodes(restNodes) assert restNodes[0].replicas[0].primaryName != master_node.name assert viewNo4 == viewNo3 + 1 sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 3)
def view_setup(looper, txnPoolNodeSet): for i in range(2): ensure_view_change_complete(looper, txnPoolNodeSet) for node in txnPoolNodeSet: assert node.viewNo == 2
def test_primary_selection_after_demoted_node_promotion( looper, txnPoolNodeSet, sdk_node_theta_added, sdk_pool_handle, tconf, tdir, allPluginsPath): """ Demote non-primary node Promote it again Restart one node to get the following difference with others: - not restarted - node registry and related pool parameters are kept in memory in some state which is expected as the same as in the pool ledger - restarted one - loaded node registry and pool parameters from the pool ledger at startup Do several view changes and check that all nodes will choose previously demoted / promoted node as a primary for some instanse """ new_steward_wallet, new_node = sdk_node_theta_added # viewNo0 = checkViewNoForNodes(txnPoolNodeSet) check_all_nodes_the_same_pool_list(txnPoolNodeSet) logger.info("1. Demote node Theta") node_dest = hexToFriendly(new_node.nodestack.verhex) sdk_send_update_node(looper, new_steward_wallet, sdk_pool_handle, node_dest, new_node.name, None, None, None, None, []) remainingNodes = list(set(txnPoolNodeSet) - {new_node}) check_all_nodes_the_same_pool_list(remainingNodes) # ensure pool is working properly sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, new_steward_wallet, 3) # TODO view change might happen unexpectedly by unknown reason # checkViewNoForNodes(remainingNodes, expectedViewNo=viewNo0) logger.info("2. Promote node Theta back") sdk_send_update_node(looper, new_steward_wallet, sdk_pool_handle, node_dest, new_node.name, None, None, None, None, [VALIDATOR]) check_all_nodes_the_same_pool_list(txnPoolNodeSet) # ensure pool is working properly sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, new_steward_wallet, 3) # checkViewNoForNodes(txnPoolNodeSet, expectedViewNo=viewNo0) logger.info("3. Restart one node") stopped_node = txnPoolNodeSet[0] disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, stopped_node, stopNode=True) looper.removeProdable(stopped_node) remainingNodes = list(set(txnPoolNodeSet) - {stopped_node}) ensureElectionsDone(looper, remainingNodes) # ensure pool is working properly sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, new_steward_wallet, 3) # checkViewNoForNodes(remainingNodes, expectedViewNo=viewNo0) # start node restartedNode = start_stopped_node(stopped_node, looper, tconf, tdir, allPluginsPath) txnPoolNodeSet = remainingNodes + [restartedNode] ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet) # ensure pool is working properly sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, new_steward_wallet, 3) # checkViewNoForNodes(txnPoolNodeSet, expectedViewNo=viewNo0) logger.info( "4. Do view changes to check that nodeTheta will be chosen " "as a primary for some instance by all nodes after some rounds") while txnPoolNodeSet[0].viewNo < 4: ensure_view_change_complete(looper, txnPoolNodeSet) # ensure pool is working properly sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, new_steward_wallet, 3)
def test_revert_auth_rule_changing(looper, txnPoolNodeSet, sdk_wallet_trustee, sdk_wallet_steward, sdk_pool_handle): node_stashers = [n.nodeIbStasher for n in txnPoolNodeSet] wh, _ = sdk_wallet_trustee new_steward_did, new_steward_verkey = create_verkey_did(looper, wh) new_steward_did2, new_steward_verkey2 = create_verkey_did(looper, wh) """We try to change rule for adding new steward. For this case we """ changed_constraint = AuthConstraint(role=STEWARD, sig_count=1) action = AuthActionAdd(txn_type=NYM, field=ROLE, value=STEWARD) with delay_rules_without_processing( node_stashers, pDelay(), cDelay(), msg_rep_delay(types_to_delay=[PREPARE, COMMIT])): sdk_send_and_check_auth_rule_request( looper, sdk_pool_handle, sdk_wallet_trustee, auth_action=ADD_PREFIX, auth_type=action.txn_type, field=action.field, new_value=action.value, old_value=None, constraint=changed_constraint.as_dict, no_wait=True) looper.runFor(waits.expectedPrePrepareTime(len(txnPoolNodeSet))) """ Try to add new steward by already existed trustee. Validation should raise exception because we change uncommitted state by adding new rule, that "Only steward can add new steward" """ with pytest.raises(RequestRejectedException, match="Not enough STEWARD signatures"): sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_trustee, 'newSteward1', STEWARD_STRING, dest=new_steward_did, verkey=new_steward_verkey) looper.runFor(waits.expectedPrePrepareTime(len(txnPoolNodeSet))) """ Catchup should revert config_state and discard rule changing """ for n in txnPoolNodeSet: n.start_catchup() # clear all request queues to not re-send the AuthRule txns again for n in txnPoolNodeSet: n.requests.clear() for r in n.replicas.values(): r._ordering_service.requestQueues[DOMAIN_LEDGER_ID].clear() r._ordering_service.requestQueues[CONFIG_LEDGER_ID].clear() looper.run( eventually( lambda nodes: assertExp( all([n.mode == Mode.participating for n in nodes])), txnPoolNodeSet)) # do view change to not send PrePrepares with the same ppSeqNo and viewNo ensure_view_change_complete(looper, txnPoolNodeSet) """ Try to create new steward by steward We can not do this, because AUTH_RULE txn was reverted """ with pytest.raises(RequestRejectedException, match="Not enough TRUSTEE signatures"): sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_steward, 'newSteward2', STEWARD_STRING, dest=new_steward_did2, verkey=new_steward_verkey2)
def test_primary_selection_after_demoted_node_promotion( looper, txnPoolNodeSet, nodeThetaAdded, tconf, tdirWithPoolTxns, allPluginsPath): """ Demote non-primary node Promote it again Restart one node to get the following difference with others: - not restarted - node registry and related pool parameters are kept in memory in some state which is expected as the same as in the pool ledger - restarted one - loaded node registry and pool parameters from the pool ledger at startup Do several view changes and check that all nodes will choose previously demoted / promoted node as a primary for some instanse """ nodeThetaSteward, nodeThetaStewardWallet, nodeTheta = nodeThetaAdded # viewNo0 = checkViewNoForNodes(txnPoolNodeSet) check_all_nodes_the_same_pool_list(txnPoolNodeSet) logger.info("1. Demote node Theta") node_data = { ALIAS: nodeTheta.name, SERVICES: [] } updateNodeData(looper, nodeThetaSteward, nodeThetaStewardWallet, nodeTheta, node_data) remainingNodes = list(set(txnPoolNodeSet) - {nodeTheta}) check_all_nodes_the_same_pool_list(remainingNodes) # ensure pool is working properly sendReqsToNodesAndVerifySuffReplies(looper, nodeThetaStewardWallet, nodeThetaSteward, numReqs=3) # TODO view change might happen unexpectedly by unknown reason # checkViewNoForNodes(remainingNodes, expectedViewNo=viewNo0) logger.info("2. Promote node Theta back") node_data = { ALIAS: nodeTheta.name, SERVICES: [VALIDATOR] } updateNodeData(looper, nodeThetaSteward, nodeThetaStewardWallet, nodeTheta, node_data) check_all_nodes_the_same_pool_list(txnPoolNodeSet) # ensure pool is working properly sendReqsToNodesAndVerifySuffReplies(looper, nodeThetaStewardWallet, nodeThetaSteward, numReqs=3) # checkViewNoForNodes(txnPoolNodeSet, expectedViewNo=viewNo0) logger.info("3. Restart one node") stopped_node = txnPoolNodeSet[0] disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, stopped_node, stopNode=True) looper.removeProdable(stopped_node) remainingNodes = list(set(txnPoolNodeSet) - {stopped_node}) # ensure pool is working properly sendReqsToNodesAndVerifySuffReplies(looper, nodeThetaStewardWallet, nodeThetaSteward, numReqs=3) # checkViewNoForNodes(remainingNodes, expectedViewNo=viewNo0) # start node restartedNode = start_stopped_node(stopped_node, looper, tconf, tdirWithPoolTxns, allPluginsPath) txnPoolNodeSet = remainingNodes + [restartedNode] ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet) # ensure pool is working properly sendReqsToNodesAndVerifySuffReplies(looper, nodeThetaStewardWallet, nodeThetaSteward, numReqs=3) # checkViewNoForNodes(txnPoolNodeSet, expectedViewNo=viewNo0) logger.info("4. Do view changes to check that nodeTheta will be chosen " "as a primary for some instance by all nodes after some rounds") while txnPoolNodeSet[0].viewNo < 4: ensure_view_change_complete(looper, txnPoolNodeSet) # ensure pool is working properly sendReqsToNodesAndVerifySuffReplies(looper, nodeThetaStewardWallet, nodeThetaSteward, numReqs=3)
def test_primary_selection_after_primary_demotion_and_view_changes( looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, txnPoolMasterNodes): """ Demote primary and do multiple view changes forcing primaries rotation. Demoted primary should be skipped without additional view changes. """ viewNo0 = checkViewNoForNodes(txnPoolNodeSet) logger.info( "1. turn off the node which has primary replica for master instanse, " " this should trigger view change") master_node = txnPoolMasterNodes[0] node_dest = hexToFriendly(master_node.nodestack.verhex) sdk_send_update_node(looper, sdk_wallet_steward, sdk_pool_handle, node_dest, master_node.name, None, None, None, None, services=[]) restNodes = [node for node in txnPoolNodeSet \ if node.name != master_node.name] ensureElectionsDone(looper, restNodes) viewNo1 = checkViewNoForNodes(restNodes) assert viewNo1 == viewNo0 + 1 assert master_node.viewNo == viewNo0 assert len(restNodes[0].replicas) == 1 # only one instance left assert restNodes[0].replicas[0].primaryName != master_node.name # ensure pool is working properly sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 3) logger.info("2. force view change 2 and check final viewNo") ensure_view_change_complete(looper, restNodes) viewNo2 = checkViewNoForNodes(restNodes) assert restNodes[0].replicas[0].primaryName != master_node.name assert viewNo2 == viewNo1 + 1 sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 3) logger.info("3. force view change 3 and check final viewNo") ensure_view_change_complete(looper, restNodes) viewNo3 = checkViewNoForNodes(restNodes) assert restNodes[0].replicas[0].primaryName != master_node.name assert viewNo3 == viewNo2 + 1 sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 3) logger.info("4. force view change 4 and check final viewNo") ensure_view_change_complete(looper, restNodes) viewNo4 = checkViewNoForNodes(restNodes) assert restNodes[0].replicas[0].primaryName != master_node.name assert viewNo4 == viewNo3 + 1 sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 3)