def testNonStewardCannotAddNode(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, tdir, tconf, allPluginsPath): new_node_name = "Epsilon" with pytest.raises(RequestRejectedException) as e: sdk_add_new_node(looper, sdk_pool_handle, sdk_wallet_client, new_node_name, tdir, tconf, allPluginsPath) assert 'is not a steward so cannot add a ' in e._excinfo[1].args[0] sdk_pool_refresh(looper, sdk_pool_handle)
def testStewardCannotAddMoreThanOneNode(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, tdir, tconf, allPluginsPath): new_node_name = "Epsilon" with pytest.raises(RequestRejectedException) as e: sdk_add_new_node(looper, sdk_pool_handle, sdk_wallet_steward, new_node_name, tdir, tconf, allPluginsPath) assert 'already has a node' in e._excinfo[1].args[0] sdk_pool_refresh(looper, sdk_pool_handle)
def testNonStewardCannotAddNode(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, tdir, tconf, allPluginsPath): new_node_name = "Epsilon" with pytest.raises(RequestRejectedException) as e: sdk_add_new_node(looper, sdk_pool_handle, sdk_wallet_client, new_node_name, tdir, tconf, allPluginsPath) assert 'is not a steward so cannot add a ' in e._excinfo[1].args[0] sdk_pool_refresh(looper, sdk_pool_handle)
def test_add_node_with_not_unique_alias(looper, tdir, tconf, sdk_pool_handle, sdk_wallet_steward, allPluginsPath): new_node_name = "Alpha" new_steward_wallet, steward_did = sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_steward, alias="TEST_STEWARD1", role='STEWARD') with pytest.raises(RequestRejectedException) as e: sdk_add_new_node(looper, sdk_pool_handle, (new_steward_wallet, steward_did), new_node_name, tdir, tconf, allPluginsPath) assert 'existing data has conflicts with request data' in \ e._excinfo[1].args[0] sdk_pool_refresh(looper, sdk_pool_handle)
def testStewardCannotAddMoreThanOneNode(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, tdir, tconf, allPluginsPath): new_node_name = "Epsilon" with pytest.raises(RequestRejectedException) as e: sdk_add_new_node(looper, sdk_pool_handle, sdk_wallet_steward, new_node_name, tdir, tconf, allPluginsPath) assert 'already has a node' in e._excinfo[1].args[0] sdk_pool_refresh(looper, sdk_pool_handle)
def add_new_node(helpers, looper, node_set, sdk_wallet, current_amount, seq_no, fees_set, sdk_pool_handle, tdir, tconf, allPluginsPath, address, do_post_node_creation, node_class): new_did, verkey = helpers.wallet.create_did(sdk_wallet=sdk_wallet) req = helpers.request.nym(sdk_wallet=sdk_wallet, alias="new_steward", role=STEWARD_STRING, dest=new_did, verkey=verkey) utxos = [{ADDRESS: address, AMOUNT: current_amount, f.SEQ_NO.nm: seq_no}] req = add_fees_request_with_address(helpers, fees_set, req, address, utxos=utxos) current_amount, seq_no, _ = send_and_check_nym_with_fees(helpers, fees_set, seq_no, looper, addresses, current_amount, nym_with_fees=req) new_steward_wallet_handle = sdk_wallet[0], new_did new_node = sdk_add_new_node(looper, sdk_pool_handle, new_steward_wallet_handle, 'Epsilon', tdir, tconf, allPluginsPath=allPluginsPath, do_post_node_creation=do_post_node_creation, nodeClass=node_class) node_set.append(new_node) looper.run(checkNodesConnected(node_set)) waitNodeDataEquality(looper, new_node, *node_set[:-1])
def test_add_node_with_not_unique_alias(looper, tdir, tconf, sdk_pool_handle, sdk_wallet_steward, allPluginsPath): new_node_name = "Alpha" new_steward_wallet, steward_did = sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_steward, alias="TEST_STEWARD1", role='STEWARD') with pytest.raises(RequestRejectedException) as e: sdk_add_new_node(looper, sdk_pool_handle, (new_steward_wallet, steward_did), new_node_name, tdir, tconf, allPluginsPath) assert 'existing data has conflicts with request data' in \ e._excinfo[1].args[0] sdk_pool_refresh(looper, sdk_pool_handle)
def testDemoteNodeWhichWasNeverActive(looper, nodeSet, sdk_pool_handle, sdk_wallet_trustee, tdir, tconf, allPluginsPath): """ Add a node without services field and check that the ledger does not contain the `services` field and check that it can be demoted and the ledger has `services` as empty list """ alias = randomString(5) new_node_name = "Node-" + alias sdk_wallet_steward = sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_trustee, alias="Steward-" + alias, role='STEWARD') new_node = sdk_add_new_node(looper, sdk_pool_handle, sdk_wallet_steward, new_node_name, tdir, tconf, allPluginsPath, services=None) looper.runFor(tconf.PROPAGATE_REQUEST_DELAY * 1.5) for node in nodeSet: txn = [t for _, t in node.poolLedger.getAllTxn()][-1] txn_data = get_payload_data(txn) assert txn_data[TARGET_NYM] == hexToFriendly(new_node.nodestack.verhex) assert SERVICES not in txn_data[DATA] demote_node(looper, sdk_wallet_steward, sdk_pool_handle, new_node) for node in nodeSet: txn = [t for _, t in node.poolLedger.getAllTxn()][-1] txn_data = get_payload_data(txn) assert txn_data[TARGET_NYM] == hexToFriendly(new_node.nodestack.verhex) assert SERVICES in txn_data[DATA] and txn_data[DATA][SERVICES] == []
def test_audit_ledger_view_change(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, sdk_wallet_steward, initial_domain_size, initial_pool_size, initial_config_size, tdir, tconf, allPluginsPath, view_no, pp_seq_no, initial_seq_no, monkeypatch): ''' 1. Send a NODE transaction and add a 7th Node for adding a new instance, but delay Ordered messages. 2. Send a NYM txn. 3. Reset delays in executing force_process_ordered 4. Check that an audit txn for the NYM txn uses primary list from uncommitted audit with a new list of primaries. ''' other_nodes = txnPoolNodeSet[:-1] slow_node = txnPoolNodeSet[-1] # Add a new steward for creating a new node new_steward_wallet_handle = sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_steward, alias="newSteward", role=STEWARD_STRING) audit_size_initial = [node.auditLedger.size for node in txnPoolNodeSet] ordereds = [] monkeypatch.setattr(slow_node, 'try_processing_ordered', lambda msg: ordereds.append(msg)) with delay_rules([n.nodeIbStasher for n in txnPoolNodeSet], icDelay()): # Send NODE txn fo 7th node new_node = sdk_add_new_node(looper, sdk_pool_handle, new_steward_wallet_handle, "Theta", tdir, tconf, allPluginsPath) txnPoolNodeSet.append(new_node) looper.run(checkNodesConnected(other_nodes + [new_node])) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) check_audit_ledger_updated(audit_size_initial, [slow_node], audit_txns_added=0) looper.run(eventually(check_audit_ledger_uncommitted_updated, audit_size_initial, [slow_node], 2)) def patch_force_process_ordered(): for msg in list(ordereds): slow_node.replicas[msg.instId].outBox.append(msg) ordereds.remove(msg) monkeypatch.undo() slow_node.force_process_ordered() assert ordereds monkeypatch.setattr(slow_node, 'force_process_ordered', patch_force_process_ordered) looper.run(eventually(lambda: assertExp(all(n.viewNo == 1 for n in txnPoolNodeSet)))) ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet) looper.run(eventually(lambda: assertExp(not ordereds))) for node in txnPoolNodeSet: last_txn = node.auditLedger.get_last_txn() last_txn['txn']['data']['primaries'] = node._get_last_audited_primaries() check_audit_txn(txn=last_txn, view_no=view_no + 1, pp_seq_no=1, seq_no=initial_seq_no + 4, txn_time=node.master_replica._ordering_service.last_accepted_pre_prepare_time, txn_roots={DOMAIN_LEDGER_ID: node.getLedger(DOMAIN_LEDGER_ID).tree.root_hash}, state_roots={DOMAIN_LEDGER_ID: node.getState(DOMAIN_LEDGER_ID).committedHeadHash}, pool_size=initial_pool_size + 1, domain_size=initial_domain_size + 2, config_size=initial_config_size, last_pool_seqno=2, last_domain_seqno=1, last_config_seqno=None, primaries=node.write_manager.future_primary_handler.get_last_primaries() or node.primaries)
def check_view_change_adding_new_node(looper, tdir, tconf, allPluginsPath, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, sdk_wallet_steward, slow_nodes=[], delay_commit=False, delay_pre_prepare=False, trigger_view_change_manually=False): # Pre-requisites: viewNo=3, Primary is Node4 for viewNo in range(1, 4): trigger_view_change(txnPoolNodeSet) waitForViewChange(looper, txnPoolNodeSet, viewNo) ensureElectionsDone(looper, txnPoolNodeSet, customTimeout=30) # Delay 3PC messages on slow nodes fast_nodes = [node for node in txnPoolNodeSet if node not in slow_nodes] all_stashers = [n.nodeIbStasher for n in txnPoolNodeSet] slow_stashers = [slow_node.nodeIbStasher for slow_node in slow_nodes] delayers = [] if delay_pre_prepare: delayers.append(ppDelay()) delayers.append(msg_rep_delay(types_to_delay=[PREPREPARE])) if delay_commit: delayers.append(cDelay()) # add a new Steward before delaying. Otherwise the slow node may reject NODE client reqs # as it can not authenticate it due to lack of Steward txn applied new_steward_wallet_handle = sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_steward, alias='New_Steward', role=STEWARD_STRING) # delay NewView message to make sure that all old nodes started view change, # but finish the view change when no Commits are delayed (otherwise slow node will not be able to select backup primaries) with delay_rules(all_stashers, nv_delay()): with delay_rules_without_processing(slow_stashers, *delayers): # Add Node5 new_node = sdk_add_new_node( looper, sdk_pool_handle, new_steward_wallet_handle, 'Epsilon', tdir, tconf, allPluginsPath, autoStart=True, nodeClass=TestNode, do_post_node_creation=None, services=[VALIDATOR], wait_till_added=True) looper.run(checkNodesConnected(fast_nodes + [new_node])) old_set = list(txnPoolNodeSet) txnPoolNodeSet.append(new_node) if trigger_view_change_manually: trigger_view_change(txnPoolNodeSet) # make sure view change is started and finished eventually waitForViewChange(looper, old_set, 4) ensureElectionsDone(looper, old_set) sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle)
def test_view_change_add_one_node_uncommitted_by_next_primary( looper, tdir, tconf, allPluginsPath, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, sdk_wallet_steward): # 1. Pre-requisites: viewNo=2, Primary is Node3 for viewNo in range(1, 3): trigger_view_change(txnPoolNodeSet) waitForViewChange(looper, txnPoolNodeSet, viewNo) ensureElectionsDone(looper, txnPoolNodeSet, customTimeout=30) # 2. Add Steward for new Node new_steward_wallet_handle = sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_steward, alias="testClientSteward" + randomString(3), role=STEWARD_STRING) # 3. Send txn to add Node5 # It will not be proposed and ordered by the current Primary, but will be proposed by the next one in the new view # Make sure that the request is propagated by the next Primary old_state_root_hash = txnPoolNodeSet[0].stateRootHash( ledgerId=POOL_LEDGER_ID, isCommitted=False) primary_node = getPrimaryReplica(txnPoolNodeSet).node next_primary = txnPoolNodeSet[-1] with delay_rules_without_processing(primary_node.nodeIbStasher, ppgDelay()): sdk_add_new_node(looper, sdk_pool_handle, new_steward_wallet_handle, new_node_name="Psi", tdir=tdir, tconf=tconf, allPluginsPath=allPluginsPath, autoStart=True, nodeClass=TestNode, do_post_node_creation=None, services=[VALIDATOR], wait_till_added=False) looper.run(eventually(check_node_txn_propagated, [next_primary])) check_node_txn_not_applied(txnPoolNodeSet, old_state_root_hash) # 4. Trigger view change to view # Make sure that only the next Primary (Node4) finishes View Change to view=3 slow_nodes = txnPoolNodeSet[:3] fast_nodes = [next_primary] slow_stashers = [slow_node.nodeIbStasher for slow_node in slow_nodes] with delay_rules_without_processing( slow_stashers, nv_delay(), msg_rep_delay(types_to_delay=[NEW_VIEW])): trigger_view_change(txnPoolNodeSet) waitForViewChange(looper, txnPoolNodeSet, 3) # view change is finished on Node4 only looper.run(eventually(check_view_change_done, fast_nodes, 3)) for n in slow_nodes: assert n.master_replica._consensus_data.waiting_for_new_view # wait till fast nodes apply the Node txn in the new View (Node4 creates a new batch with it) looper.run( eventually(check_node_txn_applied, fast_nodes, old_state_root_hash)) check_node_txn_not_applied(slow_nodes, old_state_root_hash) # 5. Trigger view change to view=4, and make sure it's finished properly trigger_view_change(txnPoolNodeSet) waitForViewChange(looper, txnPoolNodeSet, 4) ensureElectionsDone(looper, txnPoolNodeSet, customTimeout=35) sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle)