def testNonStewardCannotAddNode(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, tdir, tconf,
allPluginsPath):
new_node_name = "Epsilon"
with pytest.raises(RequestRejectedException) as e:
sdk_add_new_node(looper, sdk_pool_handle, sdk_wallet_client,
new_node_name, tdir, tconf, allPluginsPath)
assert 'is not a steward so cannot add a ' in e._excinfo[1].args[0]
sdk_pool_refresh(looper, sdk_pool_handle)
def testStewardCannotAddMoreThanOneNode(looper, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_steward,
tdir, tconf, allPluginsPath):
new_node_name = "Epsilon"
with pytest.raises(RequestRejectedException) as e:
sdk_add_new_node(looper, sdk_pool_handle, sdk_wallet_steward,
new_node_name, tdir, tconf, allPluginsPath)
assert 'already has a node' in e._excinfo[1].args[0]
sdk_pool_refresh(looper, sdk_pool_handle)
def testNonStewardCannotAddNode(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, tdir, tconf,
allPluginsPath):
new_node_name = "Epsilon"
with pytest.raises(RequestRejectedException) as e:
sdk_add_new_node(looper,
sdk_pool_handle,
sdk_wallet_client,
new_node_name,
tdir,
tconf,
allPluginsPath)
assert 'is not a steward so cannot add a ' in e._excinfo[1].args[0]
sdk_pool_refresh(looper, sdk_pool_handle)
def test_add_node_with_not_unique_alias(looper, tdir, tconf, sdk_pool_handle,
sdk_wallet_steward, allPluginsPath):
new_node_name = "Alpha"
new_steward_wallet, steward_did = sdk_add_new_nym(looper,
sdk_pool_handle,
sdk_wallet_steward,
alias="TEST_STEWARD1",
role='STEWARD')
with pytest.raises(RequestRejectedException) as e:
sdk_add_new_node(looper, sdk_pool_handle,
(new_steward_wallet, steward_did), new_node_name,
tdir, tconf, allPluginsPath)
assert 'existing data has conflicts with request data' in \
e._excinfo[1].args[0]
sdk_pool_refresh(looper, sdk_pool_handle)
def testStewardCannotAddMoreThanOneNode(looper, txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_steward, tdir, tconf,
allPluginsPath):
new_node_name = "Epsilon"
with pytest.raises(RequestRejectedException) as e:
sdk_add_new_node(looper,
sdk_pool_handle,
sdk_wallet_steward,
new_node_name,
tdir,
tconf,
allPluginsPath)
assert 'already has a node' in e._excinfo[1].args[0]
sdk_pool_refresh(looper, sdk_pool_handle)
def add_new_node(helpers, looper, node_set, sdk_wallet, current_amount, seq_no,
fees_set, sdk_pool_handle, tdir, tconf, allPluginsPath,
address, do_post_node_creation, node_class):
new_did, verkey = helpers.wallet.create_did(sdk_wallet=sdk_wallet)
req = helpers.request.nym(sdk_wallet=sdk_wallet,
alias="new_steward",
role=STEWARD_STRING,
dest=new_did,
verkey=verkey)
utxos = [{ADDRESS: address, AMOUNT: current_amount, f.SEQ_NO.nm: seq_no}]
req = add_fees_request_with_address(helpers,
fees_set,
req,
address,
utxos=utxos)
current_amount, seq_no, _ = send_and_check_nym_with_fees(helpers,
fees_set,
seq_no,
looper,
addresses,
current_amount,
nym_with_fees=req)
new_steward_wallet_handle = sdk_wallet[0], new_did
new_node = sdk_add_new_node(looper,
sdk_pool_handle,
new_steward_wallet_handle,
'Epsilon',
tdir,
tconf,
allPluginsPath=allPluginsPath,
do_post_node_creation=do_post_node_creation,
nodeClass=node_class)
node_set.append(new_node)
looper.run(checkNodesConnected(node_set))
waitNodeDataEquality(looper, new_node, *node_set[:-1])
def test_add_node_with_not_unique_alias(looper,
tdir,
tconf,
sdk_pool_handle,
sdk_wallet_steward,
allPluginsPath):
new_node_name = "Alpha"
new_steward_wallet, steward_did = sdk_add_new_nym(looper,
sdk_pool_handle,
sdk_wallet_steward,
alias="TEST_STEWARD1",
role='STEWARD')
with pytest.raises(RequestRejectedException) as e:
sdk_add_new_node(looper,
sdk_pool_handle,
(new_steward_wallet, steward_did),
new_node_name,
tdir,
tconf,
allPluginsPath)
assert 'existing data has conflicts with request data' in \
e._excinfo[1].args[0]
sdk_pool_refresh(looper, sdk_pool_handle)
def testDemoteNodeWhichWasNeverActive(looper, nodeSet, sdk_pool_handle,
sdk_wallet_trustee, tdir, tconf,
allPluginsPath):
"""
Add a node without services field and check that the ledger does not
contain the `services` field and check that it can be demoted and
the ledger has `services` as empty list
"""
alias = randomString(5)
new_node_name = "Node-" + alias
sdk_wallet_steward = sdk_add_new_nym(looper,
sdk_pool_handle,
sdk_wallet_trustee,
alias="Steward-" + alias,
role='STEWARD')
new_node = sdk_add_new_node(looper,
sdk_pool_handle,
sdk_wallet_steward,
new_node_name,
tdir,
tconf,
allPluginsPath,
services=None)
looper.runFor(tconf.PROPAGATE_REQUEST_DELAY * 1.5)
for node in nodeSet:
txn = [t for _, t in node.poolLedger.getAllTxn()][-1]
txn_data = get_payload_data(txn)
assert txn_data[TARGET_NYM] == hexToFriendly(new_node.nodestack.verhex)
assert SERVICES not in txn_data[DATA]
demote_node(looper, sdk_wallet_steward, sdk_pool_handle, new_node)
for node in nodeSet:
txn = [t for _, t in node.poolLedger.getAllTxn()][-1]
txn_data = get_payload_data(txn)
assert txn_data[TARGET_NYM] == hexToFriendly(new_node.nodestack.verhex)
assert SERVICES in txn_data[DATA] and txn_data[DATA][SERVICES] == []
def test_audit_ledger_view_change(looper, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client, sdk_wallet_steward,
initial_domain_size, initial_pool_size, initial_config_size,
tdir,
tconf,
allPluginsPath,
view_no, pp_seq_no,
initial_seq_no,
monkeypatch):
'''
1. Send a NODE transaction and add a 7th Node for adding a new instance,
but delay Ordered messages.
2. Send a NYM txn.
3. Reset delays in executing force_process_ordered
4. Check that an audit txn for the NYM txn uses primary list from uncommitted
audit with a new list of primaries.
'''
other_nodes = txnPoolNodeSet[:-1]
slow_node = txnPoolNodeSet[-1]
# Add a new steward for creating a new node
new_steward_wallet_handle = sdk_add_new_nym(looper,
sdk_pool_handle,
sdk_wallet_steward,
alias="newSteward",
role=STEWARD_STRING)
audit_size_initial = [node.auditLedger.size for node in txnPoolNodeSet]
ordereds = []
monkeypatch.setattr(slow_node, 'try_processing_ordered', lambda msg: ordereds.append(msg))
with delay_rules([n.nodeIbStasher for n in txnPoolNodeSet], icDelay()):
# Send NODE txn fo 7th node
new_node = sdk_add_new_node(looper,
sdk_pool_handle,
new_steward_wallet_handle,
"Theta",
tdir,
tconf,
allPluginsPath)
txnPoolNodeSet.append(new_node)
looper.run(checkNodesConnected(other_nodes + [new_node]))
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
check_audit_ledger_updated(audit_size_initial, [slow_node],
audit_txns_added=0)
looper.run(eventually(check_audit_ledger_uncommitted_updated,
audit_size_initial, [slow_node], 2))
def patch_force_process_ordered():
for msg in list(ordereds):
slow_node.replicas[msg.instId].outBox.append(msg)
ordereds.remove(msg)
monkeypatch.undo()
slow_node.force_process_ordered()
assert ordereds
monkeypatch.setattr(slow_node, 'force_process_ordered', patch_force_process_ordered)
looper.run(eventually(lambda: assertExp(all(n.viewNo == 1 for n in txnPoolNodeSet))))
ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet)
looper.run(eventually(lambda: assertExp(not ordereds)))
for node in txnPoolNodeSet:
last_txn = node.auditLedger.get_last_txn()
last_txn['txn']['data']['primaries'] = node._get_last_audited_primaries()
check_audit_txn(txn=last_txn,
view_no=view_no + 1, pp_seq_no=1,
seq_no=initial_seq_no + 4,
txn_time=node.master_replica._ordering_service.last_accepted_pre_prepare_time,
txn_roots={DOMAIN_LEDGER_ID: node.getLedger(DOMAIN_LEDGER_ID).tree.root_hash},
state_roots={DOMAIN_LEDGER_ID: node.getState(DOMAIN_LEDGER_ID).committedHeadHash},
pool_size=initial_pool_size + 1, domain_size=initial_domain_size + 2,
config_size=initial_config_size,
last_pool_seqno=2,
last_domain_seqno=1,
last_config_seqno=None,
primaries=node.write_manager.future_primary_handler.get_last_primaries() or node.primaries)
def check_view_change_adding_new_node(looper, tdir, tconf, allPluginsPath,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
sdk_wallet_steward,
slow_nodes=[],
delay_commit=False,
delay_pre_prepare=False,
trigger_view_change_manually=False):
# Pre-requisites: viewNo=3, Primary is Node4
for viewNo in range(1, 4):
trigger_view_change(txnPoolNodeSet)
waitForViewChange(looper, txnPoolNodeSet, viewNo)
ensureElectionsDone(looper, txnPoolNodeSet, customTimeout=30)
# Delay 3PC messages on slow nodes
fast_nodes = [node for node in txnPoolNodeSet if node not in slow_nodes]
all_stashers = [n.nodeIbStasher for n in txnPoolNodeSet]
slow_stashers = [slow_node.nodeIbStasher for slow_node in slow_nodes]
delayers = []
if delay_pre_prepare:
delayers.append(ppDelay())
delayers.append(msg_rep_delay(types_to_delay=[PREPREPARE]))
if delay_commit:
delayers.append(cDelay())
# add a new Steward before delaying. Otherwise the slow node may reject NODE client reqs
# as it can not authenticate it due to lack of Steward txn applied
new_steward_wallet_handle = sdk_add_new_nym(looper,
sdk_pool_handle,
sdk_wallet_steward,
alias='New_Steward',
role=STEWARD_STRING)
# delay NewView message to make sure that all old nodes started view change,
# but finish the view change when no Commits are delayed (otherwise slow node will not be able to select backup primaries)
with delay_rules(all_stashers, nv_delay()):
with delay_rules_without_processing(slow_stashers, *delayers):
# Add Node5
new_node = sdk_add_new_node(
looper,
sdk_pool_handle,
new_steward_wallet_handle,
'Epsilon',
tdir,
tconf,
allPluginsPath,
autoStart=True,
nodeClass=TestNode,
do_post_node_creation=None,
services=[VALIDATOR],
wait_till_added=True)
looper.run(checkNodesConnected(fast_nodes + [new_node]))
old_set = list(txnPoolNodeSet)
txnPoolNodeSet.append(new_node)
if trigger_view_change_manually:
trigger_view_change(txnPoolNodeSet)
# make sure view change is started and finished eventually
waitForViewChange(looper, old_set, 4)
ensureElectionsDone(looper, old_set)
sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle)
def test_view_change_add_one_node_uncommitted_by_next_primary(
looper, tdir, tconf, allPluginsPath, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, sdk_wallet_steward):
# 1. Pre-requisites: viewNo=2, Primary is Node3
for viewNo in range(1, 3):
trigger_view_change(txnPoolNodeSet)
waitForViewChange(looper, txnPoolNodeSet, viewNo)
ensureElectionsDone(looper, txnPoolNodeSet, customTimeout=30)
# 2. Add Steward for new Node
new_steward_wallet_handle = sdk_add_new_nym(looper,
sdk_pool_handle,
sdk_wallet_steward,
alias="testClientSteward" +
randomString(3),
role=STEWARD_STRING)
# 3. Send txn to add Node5
# It will not be proposed and ordered by the current Primary, but will be proposed by the next one in the new view
# Make sure that the request is propagated by the next Primary
old_state_root_hash = txnPoolNodeSet[0].stateRootHash(
ledgerId=POOL_LEDGER_ID, isCommitted=False)
primary_node = getPrimaryReplica(txnPoolNodeSet).node
next_primary = txnPoolNodeSet[-1]
with delay_rules_without_processing(primary_node.nodeIbStasher,
ppgDelay()):
sdk_add_new_node(looper,
sdk_pool_handle,
new_steward_wallet_handle,
new_node_name="Psi",
tdir=tdir,
tconf=tconf,
allPluginsPath=allPluginsPath,
autoStart=True,
nodeClass=TestNode,
do_post_node_creation=None,
services=[VALIDATOR],
wait_till_added=False)
looper.run(eventually(check_node_txn_propagated, [next_primary]))
check_node_txn_not_applied(txnPoolNodeSet, old_state_root_hash)
# 4. Trigger view change to view
# Make sure that only the next Primary (Node4) finishes View Change to view=3
slow_nodes = txnPoolNodeSet[:3]
fast_nodes = [next_primary]
slow_stashers = [slow_node.nodeIbStasher for slow_node in slow_nodes]
with delay_rules_without_processing(
slow_stashers, nv_delay(),
msg_rep_delay(types_to_delay=[NEW_VIEW])):
trigger_view_change(txnPoolNodeSet)
waitForViewChange(looper, txnPoolNodeSet, 3)
# view change is finished on Node4 only
looper.run(eventually(check_view_change_done, fast_nodes, 3))
for n in slow_nodes:
assert n.master_replica._consensus_data.waiting_for_new_view
# wait till fast nodes apply the Node txn in the new View (Node4 creates a new batch with it)
looper.run(
eventually(check_node_txn_applied, fast_nodes,
old_state_root_hash))
check_node_txn_not_applied(slow_nodes, old_state_root_hash)
# 5. Trigger view change to view=4, and make sure it's finished properly
trigger_view_change(txnPoolNodeSet)
waitForViewChange(looper, txnPoolNodeSet, 4)
ensureElectionsDone(looper, txnPoolNodeSet, customTimeout=35)
sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_client,
sdk_pool_handle)
