def test_demote_promote_restart_after_promotion_7_nodes(txnPoolNodeSet,
looper,
sdk_pool_handle,
sdk_wallet_steward,
tdir,
tconf,
allPluginsPath):
demoted_node = txnPoolNodeSet[-1]
rest_nodes = [n for n in txnPoolNodeSet if n != demoted_node]
starting_view_no = checkViewNoForNodes(txnPoolNodeSet)
demote_node(looper, sdk_wallet_steward, sdk_pool_handle, demoted_node)
waitForViewChange(looper, rest_nodes, expectedViewNo=starting_view_no + 1)
ensureElectionsDone(looper, rest_nodes)
ensure_all_nodes_have_same_data(looper, rest_nodes)
sdk_send_random_and_check(looper, rest_nodes, sdk_pool_handle, sdk_wallet_steward, 5)
starting_view_no = checkViewNoForNodes(rest_nodes)
promote_node(looper, sdk_wallet_steward, sdk_pool_handle, demoted_node)
waitForViewChange(looper, rest_nodes, expectedViewNo=starting_view_no + 1)
ensureElectionsDone(looper, rest_nodes, instances_list=[0, 1, 2])
ensure_all_nodes_have_same_data(looper, rest_nodes)
restart_node(looper, txnPoolNodeSet, demoted_node, tconf, tdir, allPluginsPath)
ensureElectionsDone(looper, txnPoolNodeSet)
sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_steward, sdk_pool_handle)
def testStewardSuspendsNode(looper, txnPoolNodeSet,
tdir, tconf,
sdk_pool_handle,
sdk_wallet_steward,
sdk_node_theta_added,
poolTxnStewardData,
allPluginsPath):
new_steward_wallet, new_node = sdk_node_theta_added
demote_node(looper, new_steward_wallet, sdk_pool_handle, new_node)
# Check suspended node does not exist in any nodeReg or remotes of
# nodes or clients
txnPoolNodeSet = txnPoolNodeSet[:-1]
for node in txnPoolNodeSet:
looper.run(eventually(checkNodeNotInNodeReg, node, new_node.name))
# Check that a node does not connect to the suspended
# node
sdk_ensure_pool_functional(looper, txnPoolNodeSet, new_steward_wallet, sdk_pool_handle)
with pytest.raises(RemoteNotFound):
looper.loop.run_until_complete(sendMessageAndCheckDelivery(txnPoolNodeSet[0], new_node))
new_node.stop()
looper.removeProdable(new_node)
# Check that a node whose suspension is revoked can reconnect to other
# nodes and clients can also connect to that node
promote_node(looper, new_steward_wallet, sdk_pool_handle, new_node)
nodeTheta = start_stopped_node(new_node, looper, tconf,
tdir, allPluginsPath,
delay_instance_change_msgs=False)
txnPoolNodeSet.append(nodeTheta)
looper.run(checkNodesConnected(txnPoolNodeSet))
sdk_pool_refresh(looper, sdk_pool_handle)
sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_steward, sdk_pool_handle)
def testSuspendNode(looper, sdk_pool_handle, sdk_wallet_trustee, nodeSet, tdir,
tconf, allPluginsPath):
"""
Suspend a node and then cancel suspension. Suspend while suspended
to test that there is no error
"""
start_view_no = nodeSet[0].viewNo
new_steward_wallet, new_node = sdk_node_theta_added(
looper,
nodeSet,
tdir,
tconf,
sdk_pool_handle,
sdk_wallet_trustee,
allPluginsPath,
node_config_helper_class=NodeConfigHelper,
testNodeClass=TestNode,
name="Node-" + randomString(5))
waitForViewChange(looper=looper,
txnPoolNodeSet=nodeSet,
expectedViewNo=start_view_no + 1)
ensureElectionsDone(looper=looper, nodes=nodeSet)
demote_node(looper, sdk_wallet_trustee, sdk_pool_handle, new_node)
_wait_view_change_finish(looper, nodeSet[:-1], start_view_no + 1)
demote_node(looper, sdk_wallet_trustee, sdk_pool_handle, new_node)
promote_node(looper, sdk_wallet_trustee, sdk_pool_handle, new_node)
_wait_view_change_finish(looper, nodeSet[:-1], start_view_no + 2)
promote_node(looper, sdk_wallet_trustee, sdk_pool_handle, new_node)
def testSuspendNode(looper, sdk_pool_handle, sdk_wallet_trustee, newNodeAdded):
"""
Suspend a node and then cancel suspension. Suspend while suspended
to test that there is no error
"""
new_steward_wallet, new_node = newNodeAdded
demote_node(looper, sdk_wallet_trustee, sdk_pool_handle, new_node)
demote_node(looper, sdk_wallet_trustee, sdk_pool_handle, new_node)
promote_node(looper, sdk_wallet_trustee, sdk_pool_handle, new_node)
promote_node(looper, sdk_wallet_trustee, sdk_pool_handle, new_node)
def test_promotion_before_view_change(looper, txnPoolNodeSet, tdir, tconf,
allPluginsPath, sdk_wallet_stewards,
sdk_pool_handle):
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_stewards[0], 1)
assert txnPoolNodeSet[0].master_replica.isPrimary
assert txnPoolNodeSet[1].replicas[1].isPrimary
assert txnPoolNodeSet[2].replicas[2].isPrimary
starting_view_number = checkViewNoForNodes(txnPoolNodeSet)
node_2 = txnPoolNodeSet[1]
node_3 = txnPoolNodeSet[2]
node_5 = txnPoolNodeSet[4]
# Demote node 2
steward_2 = sdk_wallet_stewards[1]
demote_node(looper, steward_2, sdk_pool_handle, node_2)
disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, node_2)
looper.removeProdable(node_2)
txnPoolNodeSet.remove(node_2)
# Checking that view change happened
# we are expecting 2 view changes here since Beta is selected as a master Primary on view=1
# (since node reg at the beginning of view 0 is used to select it), but it's not available (demoted),
# so we do view change to view=2 by timeout
waitForViewChange(looper,
txnPoolNodeSet,
expectedViewNo=starting_view_number + 2)
ensureElectionsDone(looper, txnPoolNodeSet, instances_list=[0, 1])
assert node_3.master_replica.isPrimary
# Promoting node 3, increasing replica count
node_2 = start_stopped_node(node_2, looper, tconf, tdir, allPluginsPath)
promote_node(looper, steward_2, sdk_pool_handle, node_2)
txnPoolNodeSet.append(node_2)
looper.run(checkNodesConnected(txnPoolNodeSet))
waitForViewChange(looper,
txnPoolNodeSet,
expectedViewNo=starting_view_number + 3)
ensureElectionsDone(looper, txnPoolNodeSet, instances_list=[0, 1, 2])
# node 5 is a primary since promoted node is added at the end of the list
assert node_5.master_replica.isPrimary
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_stewards[0], 2)
ensure_all_nodes_have_same_data(looper, txnPoolNodeSet)
def test_node_txn_promote_by_endorser(txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_trustee, looper,
sdk_wallet_handle):
validators_before = get_pool_validator_count(txnPoolNodeSet)
new_end_did, new_end_verkey = looper.loop.run_until_complete(
did.create_and_store_my_did(sdk_wallet_trustee[0], "{}"))
# Step 1. Demote node using default auth rule
demote_node(looper, sdk_wallet_trustee, sdk_pool_handle,
txnPoolNodeSet[-1])
# Check, that node was demoted
assert validators_before - get_pool_validator_count(
txnPoolNodeSet[:-1]) == 1
# Step 2. Add new Endorser
sdk_add_new_nym(looper,
sdk_pool_handle,
sdk_wallet_trustee,
'newEndorser',
ENDORSER_STRING,
verkey=new_end_verkey,
dest=new_end_did)
new_constraint = AuthConstraint(ENDORSER, 1)
# Step 3. Change auth rule, to allowing endorser promote node back
sdk_send_and_check_auth_rule_request(looper,
sdk_pool_handle,
sdk_wallet_trustee,
auth_action=EDIT_PREFIX,
auth_type=promote_action.txn_type,
field=promote_action.field,
new_value=promote_action.new_value,
old_value=promote_action.old_value,
constraint=new_constraint.as_dict)
# Step 4. Promote node back, using new Endorser
promote_node(looper, (sdk_wallet_handle, new_end_did), sdk_pool_handle,
txnPoolNodeSet[-1])
# Check, that all other nodes return previous demoted node back
assert validators_before == get_pool_validator_count(txnPoolNodeSet[:-1])
def test_promotion_before_view_change(looper,
txnPoolNodeSet,
tdir,
tconf,
allPluginsPath,
sdk_wallet_stewards,
sdk_pool_handle):
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_stewards[0], 1)
assert txnPoolNodeSet[0].master_replica.isPrimary
assert txnPoolNodeSet[1].replicas[1].isPrimary
assert txnPoolNodeSet[2].replicas[2].isPrimary
starting_view_number = checkViewNoForNodes(txnPoolNodeSet)
node_2 = txnPoolNodeSet[1]
node_3 = txnPoolNodeSet[2]
node_4 = txnPoolNodeSet[3]
# Demote node 2
steward_2 = sdk_wallet_stewards[1]
demote_node(looper, steward_2, sdk_pool_handle, node_2)
disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, node_2)
looper.removeProdable(node_2)
txnPoolNodeSet.remove(node_2)
# Checking that view change happened
waitForViewChange(looper,
txnPoolNodeSet,
expectedViewNo=starting_view_number + 1)
ensureElectionsDone(looper, txnPoolNodeSet, instances_list=[0, 1])
assert node_3.master_replica.isPrimary
# Promoting node 3, increasing replica count
node_2 = start_stopped_node(node_2, looper, tconf, tdir, allPluginsPath)
promote_node(looper, steward_2, sdk_pool_handle, node_2)
txnPoolNodeSet.append(node_2)
looper.run(checkNodesConnected(txnPoolNodeSet))
waitForViewChange(looper,
txnPoolNodeSet,
expectedViewNo=starting_view_number + 3)
ensureElectionsDone(looper, txnPoolNodeSet, instances_list=[0, 1, 2])
assert node_4.master_replica.isPrimary
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_stewards[0], 2)
ensure_all_nodes_have_same_data(looper, txnPoolNodeSet)
def testSuspendNode(looper, sdk_pool_handle, sdk_wallet_trustee, newNodeAdded,
nodeSet):
"""
Suspend a node and then cancel suspension. Suspend while suspended
to test that there is no error
"""
start_view_no = nodeSet[0].viewNo
new_steward_wallet, new_node = newNodeAdded
demote_node(looper, sdk_wallet_trustee, sdk_pool_handle, new_node)
_wait_view_change_finish(looper, nodeSet[:-1], start_view_no + 1)
demote_node(looper, sdk_wallet_trustee, sdk_pool_handle, new_node)
promote_node(looper, sdk_wallet_trustee, sdk_pool_handle, new_node)
_wait_view_change_finish(looper, nodeSet[:-1], start_view_no + 3)
promote_node(looper, sdk_wallet_trustee, sdk_pool_handle, new_node)
def test_demote_promote_restart_after_promotion_from_10_to_4_nodes(txnPoolNodeSet,
looper,
sdk_pool_handle,
sdk_wallet_steward,
tdir,
tconf,
allPluginsPath):
"""
We expect that 2 changes for f value should be happened
"""
def demote_another_one(rest_pool):
demoted_node = rest_pool[-1]
rest_pool = [n for n in rest_pool if n != demoted_node]
starting_view_no = checkViewNoForNodes(rest_pool)
demote_node(looper, sdk_wallet_steward, sdk_pool_handle, demoted_node)
waitForViewChange(looper, rest_pool, expectedViewNo=starting_view_no + 1)
ensureElectionsDone(looper, rest_pool, customTimeout=60)
ensure_all_nodes_have_same_data(looper, rest_pool)
return rest_pool
rest_nodes = txnPoolNodeSet
etalon_node = txnPoolNodeSet[-1]
while len(rest_nodes) > 4:
rest_nodes = demote_another_one(rest_nodes)
sdk_send_random_and_check(looper, rest_nodes, sdk_pool_handle, sdk_wallet_steward, 5)
starting_view_no = checkViewNoForNodes(rest_nodes)
promote_node(looper, sdk_wallet_steward, sdk_pool_handle, etalon_node)
waitForViewChange(looper, rest_nodes, expectedViewNo=starting_view_no + 1)
ensure_all_nodes_have_same_data(looper, rest_nodes)
rest_nodes.append(etalon_node)
restart_node(looper, rest_nodes, etalon_node, tconf, tdir, allPluginsPath)
ensureElectionsDone(looper, rest_nodes)
sdk_ensure_pool_functional(looper, rest_nodes, sdk_wallet_steward, sdk_pool_handle)
def test_promotion_leads_to_correct_primary_selection(looper, txnPoolNodeSet,
tdir, tconf,
allPluginsPath,
sdk_wallet_stewards,
sdk_pool_handle):
# We are saving pool state at moment of last view_change to send it
# to newly connected nodes so they could restore primaries basing on this node set.
# When current primaries getting edited because of promotion/demotion we don't take this into account.
# That lead us to primary inconsistency on different nodes
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_stewards[0], 1)
assert txnPoolNodeSet[0].master_replica.isPrimary
assert txnPoolNodeSet[1].replicas._replicas[1].isPrimary
assert txnPoolNodeSet[2].replicas._replicas[2].isPrimary
starting_view_number = checkViewNoForNodes(txnPoolNodeSet)
node_1 = txnPoolNodeSet[0]
node_3 = txnPoolNodeSet[2]
# Demote node 3
steward_3 = sdk_wallet_stewards[2]
demote_node(looper, steward_3, sdk_pool_handle, node_3)
disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, node_3)
looper.removeProdable(node_3)
txnPoolNodeSet.remove(node_3)
# Checking that view change happened
waitForViewChange(looper, txnPoolNodeSet, starting_view_number + 1)
assert all(node.replicas.primary_name_by_inst_id ==
node_1.replicas.primary_name_by_inst_id
for node in txnPoolNodeSet)
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_stewards[0], 2)
for node in txnPoolNodeSet:
assert node.f == 1
assert node.replicas.num_replicas == 2
# restart Node1
disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, node_1)
looper.removeProdable(node_1)
txnPoolNodeSet.remove(node_1)
node_1 = start_stopped_node(node_1, looper, tconf, tdir, allPluginsPath)
txnPoolNodeSet.append(node_1)
# Wait so node_1 could start and catch up
waitForViewChange(looper, txnPoolNodeSet, starting_view_number + 1)
assert all(node.replicas.primary_name_by_inst_id ==
node_1.replicas.primary_name_by_inst_id
for node in txnPoolNodeSet)
ensure_all_nodes_have_same_data(looper, txnPoolNodeSet)
# Promoting node 3, increasing replica count
node_3 = start_stopped_node(node_3, looper, tconf, tdir, allPluginsPath)
promote_node(looper, steward_3, sdk_pool_handle, node_3)
txnPoolNodeSet.append(node_3)
looper.run(checkNodesConnected(txnPoolNodeSet))
# Wait for view change after promotion
waitForViewChange(looper, txnPoolNodeSet, starting_view_number + 2)
ensureElectionsDone(looper, txnPoolNodeSet, instances_list=[0, 1, 2])
# Node 3 able to do ordering
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_stewards[0], 2)
ensure_all_nodes_have_same_data(looper, txnPoolNodeSet)
def test_promotion_leads_to_primary_inconsistency(looper,
txnPoolNodeSet,
tdir,
tconf,
allPluginsPath,
sdk_wallet_stewards,
sdk_pool_handle):
# We are saving pool state at moment of last view_change to send it
# to newly connected nodes so they could restore primaries basing on this node set.
# When current primaries getting edited because of promotion/demotion we don't take this into account.
# That lead us to primary inconsistency on different nodes
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_stewards[0], 1)
assert txnPoolNodeSet[0].master_replica.isPrimary
assert txnPoolNodeSet[1].replicas._replicas[1].isPrimary
assert txnPoolNodeSet[2].replicas._replicas[2].isPrimary
starting_view_number = checkViewNoForNodes(txnPoolNodeSet)
# Demote node 3
node_3 = txnPoolNodeSet[2]
steward_3 = sdk_wallet_stewards[2]
demote_node(looper, steward_3, sdk_pool_handle, node_3)
disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, node_3)
looper.removeProdable(node_3)
txnPoolNodeSet.remove(node_3)
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_stewards[0], 2)
for node in txnPoolNodeSet:
assert node.f == 1
assert node.replicas.num_replicas == 2
# Force a view change by stopping master. In this moment we are saving pool state (without 3rd node)
node_1 = txnPoolNodeSet[0]
disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, node_1)
looper.removeProdable(node_1)
txnPoolNodeSet.remove(node_1)
# Checking that view change happened
ensureElectionsDone(looper, txnPoolNodeSet, instances_list=[0, 1])
view_number = checkViewNoForNodes(txnPoolNodeSet)
assert view_number == starting_view_number + 1
node_1 = start_stopped_node(node_1, looper, tconf, tdir, allPluginsPath)
txnPoolNodeSet.append(node_1)
# Wait so node_1 could start and finish view_change
looper.runFor(1)
# Promoting node 3, increasing replica count
node_3 = start_stopped_node(node_3, looper, tconf, tdir, allPluginsPath)
promote_node(looper, steward_3, sdk_pool_handle, node_3)
txnPoolNodeSet.append(node_3)
looper.run(checkNodesConnected(txnPoolNodeSet))
ensureElectionsDone(looper, txnPoolNodeSet, instances_list=[0, 1, 2])
# Node 3 able to do ordering
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_stewards[0], 2)
view_number = checkViewNoForNodes(txnPoolNodeSet)
assert view_number == starting_view_number + 2
ensure_all_nodes_have_same_data(looper, txnPoolNodeSet)
# But it has different primary, cause it uses nodeReg without itself to calculate primaries
assert all(node.replicas.primary_name_by_inst_id ==
node_1.replicas.primary_name_by_inst_id
for node in txnPoolNodeSet if node is not node_3)
# Fails
assert all(node.replicas.primary_name_by_inst_id ==
node_1.replicas.primary_name_by_inst_id
for node in txnPoolNodeSet)