def test_restart_half_to_lower_view(looper, txnPoolNodeSet, tconf, tdir, allPluginsPath, sdk_pool_handle, sdk_wallet_client): # Add transaction to ledger sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) # Move to higher view ensure_view_change_complete(looper, txnPoolNodeSet) # Restart half of nodes tm = tconf.ToleratePrimaryDisconnection + waits.expectedPoolElectionTimeout( len(txnPoolNodeSet)) nodes_before_restart = txnPoolNodeSet.copy() restart_nodes(looper, txnPoolNodeSet, txnPoolNodeSet[2:], tconf, tdir, allPluginsPath, after_restart_timeout=tm, start_one_by_one=False) # Check that nodes didn't think they may have inconsistent 3PC state for node in nodes_before_restart: assert node.spylog.count(node.on_inconsistent_3pc_state) == 0 # Check that nodes don't think they may have inconsistent 3PC state for node in txnPoolNodeSet: assert node.spylog.count(node.on_inconsistent_3pc_state) == 0 # Check that all nodes are still functional sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle)
def test_restart_majority_to_same_view(looper, txnPoolNodeSet, tconf, tdir, allPluginsPath, sdk_pool_handle, sdk_wallet_client): # Add transaction to ledger sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) majority = txnPoolNodeSet[:3] minority = txnPoolNodeSet[3:] # Restart majority group tm = tconf.ToleratePrimaryDisconnection + waits.expectedPoolElectionTimeout(len(txnPoolNodeSet)) majority_before_restart = majority.copy() restart_nodes(looper, txnPoolNodeSet, majority, tconf, tdir, allPluginsPath, after_restart_timeout=tm, start_one_by_one=False, wait_for_elections=False) ensureElectionsDone(looper, majority, instances_list=range(2)) # Check that nodes in minority group are aware that they might have inconsistent 3PC state for node in minority: assert node.spylog.count(node.on_inconsistent_3pc_state) == 1 # Check that nodes in majority group didn't think they might have inconsistent 3PC state for node in majority_before_restart: assert node.spylog.count(node.on_inconsistent_3pc_state) == 0 # Check that nodes in majority group don't think they might have inconsistent 3PC state for node in majority: assert node.spylog.count(node.on_inconsistent_3pc_state) == 0 # Restart minority group restart_nodes(looper, txnPoolNodeSet, minority, tconf, tdir, allPluginsPath, after_restart_timeout=tm, start_one_by_one=False) # Check that all nodes are still functional sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle)
def test_restart_majority_to_same_view(looper, txnPoolNodeSet, tconf, tdir, allPluginsPath, sdk_pool_handle, sdk_wallet_client): # Add transaction to ledger sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) majority = txnPoolNodeSet[:3] minority = txnPoolNodeSet[3:] # Restart majority group tm = tconf.ToleratePrimaryDisconnection + waits.expectedPoolElectionTimeout(len(txnPoolNodeSet)) majority_before_restart = majority.copy() restart_nodes(looper, txnPoolNodeSet, majority, tconf, tdir, allPluginsPath, after_restart_timeout=tm, start_one_by_one=False, wait_for_elections=False) ensureElectionsDone(looper, majority, instances_list=range(2)) # Check that nodes in minority group are aware that they might have inconsistent 3PC state for node in minority: assert node.spylog.count(node.on_inconsistent_3pc_state) == 1 # Check that nodes in majority group didn't think they might have inconsistent 3PC state for node in majority_before_restart: assert node.spylog.count(node.on_inconsistent_3pc_state) == 0 # Check that nodes in majority group don't think they might have inconsistent 3PC state for node in majority: assert node.spylog.count(node.on_inconsistent_3pc_state) == 0 # Restart minority group restart_nodes(looper, txnPoolNodeSet, minority, tconf, tdir, allPluginsPath, after_restart_timeout=tm, start_one_by_one=False) # Check that all nodes are still functional sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle)
def test_restart_groups_6_of_7_np_no_tm(looper, txnPoolNodeSet, tconf, tdir, sdk_pool_handle, sdk_wallet_client, allPluginsPath): tm = tconf.ToleratePrimaryDisconnection + waits.expectedPoolElectionTimeout(len(txnPoolNodeSet)) restart_group = get_group(txnPoolNodeSet, 6, include_primary=False) restart_nodes(looper, txnPoolNodeSet, restart_group, tconf, tdir, allPluginsPath, after_restart_timeout=tm, start_one_by_one=False) sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle)
def test_restart_to_same_view_with_killed_primary(looper, txnPoolNodeSet, tconf, tdir, allPluginsPath, sdk_pool_handle, sdk_wallet_client): restart_timeout = tconf.ToleratePrimaryDisconnection + \ waits.expectedPoolElectionTimeout(len(txnPoolNodeSet)) primary = txnPoolNodeSet[0] alive_nodes = txnPoolNodeSet[1:] minority = alive_nodes[-1:] majority = alive_nodes[:-1] # Move to higher view by killing primary primary.cleanupOnStopping = True primary.stop() looper.removeProdable(primary) ensure_node_disconnected(looper, primary, txnPoolNodeSet) waitForViewChange(looper, alive_nodes, 1, customTimeout=VIEW_CHANGE_TIMEOUT) ensureElectionsDone(looper, alive_nodes, numInstances=3) # Add transaction to ledger sdk_send_random_and_check(looper, alive_nodes, sdk_pool_handle, sdk_wallet_client, 1) # Restart majority group majority_before_restart = majority.copy() restart_nodes(looper, alive_nodes, majority, tconf, tdir, allPluginsPath, after_restart_timeout=restart_timeout, start_one_by_one=False, wait_for_elections=False) waitForViewChange(looper, majority, 1, customTimeout=2.1 * VIEW_CHANGE_TIMEOUT) ensureElectionsDone(looper, majority, numInstances=3) # Check that nodes in minority group are aware that they might have inconsistent 3PC state for node in minority: assert node.spylog.count(node.on_inconsistent_3pc_state) == 1 # Check that nodes in majority group didn't think they might have inconsistent 3PC state for node in majority_before_restart: assert node.spylog.count(node.on_inconsistent_3pc_state) == 0 # Check that nodes in majority group don't think they might have inconsistent 3PC state for node in majority: assert node.spylog.count(node.on_inconsistent_3pc_state) == 0 # Restart minority group restart_nodes(looper, alive_nodes, minority, tconf, tdir, allPluginsPath, after_restart_timeout=restart_timeout, start_one_by_one=False, wait_for_elections=False) ensureElectionsDone(looper, alive_nodes, numInstances=3) # Check that all nodes are still functional sdk_ensure_pool_functional(looper, alive_nodes, sdk_wallet_client, sdk_pool_handle)
def test_restart_to_same_view_with_killed_primary(looper, txnPoolNodeSet, tconf, tdir, allPluginsPath, sdk_pool_handle, sdk_wallet_client): restart_timeout = tconf.ToleratePrimaryDisconnection + \ waits.expectedPoolElectionTimeout(len(txnPoolNodeSet)) primary = txnPoolNodeSet[0] alive_nodes = txnPoolNodeSet[1:] minority = alive_nodes[-1:] majority = alive_nodes[:-1] # Move to higher view by killing primary primary.cleanupOnStopping = True primary.stop() looper.removeProdable(primary) ensure_node_disconnected(looper, primary, txnPoolNodeSet) waitForViewChange(looper, alive_nodes, 1, customTimeout=VIEW_CHANGE_TIMEOUT) ensureElectionsDone(looper, alive_nodes, instances_list=range(3)) # Add transaction to ledger sdk_send_random_and_check(looper, alive_nodes, sdk_pool_handle, sdk_wallet_client, 1) # Restart majority group majority_before_restart = majority.copy() restart_nodes(looper, alive_nodes, majority, tconf, tdir, allPluginsPath, after_restart_timeout=restart_timeout, start_one_by_one=False, wait_for_elections=False) waitForViewChange(looper, majority, 1, customTimeout=2.1 * VIEW_CHANGE_TIMEOUT) ensureElectionsDone(looper, majority, instances_list=range(3)) # Check that nodes in minority group are aware that they might have inconsistent 3PC state for node in minority: assert node.spylog.count(node.on_inconsistent_3pc_state) == 1 # Check that nodes in majority group didn't think they might have inconsistent 3PC state for node in majority_before_restart: assert node.spylog.count(node.on_inconsistent_3pc_state) == 0 # Check that nodes in majority group don't think they might have inconsistent 3PC state for node in majority: assert node.spylog.count(node.on_inconsistent_3pc_state) == 0 # Restart minority group restart_nodes(looper, alive_nodes, minority, tconf, tdir, allPluginsPath, after_restart_timeout=restart_timeout, start_one_by_one=False, wait_for_elections=False) ensureElectionsDone(looper, alive_nodes, instances_list=range(3)) # Check that all nodes are still functional sdk_ensure_pool_functional(looper, alive_nodes, sdk_wallet_client, sdk_pool_handle)
def test_view_change_with_instance_change_lost_due_to_restarts( looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, tconf, tdir, allPluginsPath): """ 1. some_nodes (Beta and Gamma) send InstanceChange for all nodes. 2. Restart other_nodes (Gamma and Delta) 3. last_node (Delta) send InstanceChange for all nodes. 4. Ensure elections done and pool is functional """ current_view_no = txnPoolNodeSet[0].viewNo some_nodes = txnPoolNodeSet[1:3] other_nodes = txnPoolNodeSet[2:4] for n in some_nodes: send_test_instance_change(n) def check_ic_delivery(): for node in txnPoolNodeSet: vct_service = node.master_replica._view_change_trigger_service assert all( vct_service._instance_changes.has_inst_chng_from( current_view_no + 1, sender.name) for sender in some_nodes) looper.run(eventually(check_ic_delivery)) restart_nodes(looper, txnPoolNodeSet, other_nodes, tconf, tdir, allPluginsPath, start_one_by_one=False) last_node = txnPoolNodeSet[-1] send_test_instance_change(last_node) waitForViewChange(looper, txnPoolNodeSet, current_view_no + 1, customTimeout=3 * FRESHNESS_TIMEOUT) ensureElectionsDone(looper, txnPoolNodeSet) sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle)
def test_first_audit_catchup_during_ordering(monkeypatch, looper, tconf, tdir, allPluginsPath, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client): # 1. patch primaries in audit ledger for n in txnPoolNodeSet: patch_primaries_in_audit(n, monkeypatch) # 2. order a txn sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) # 3. restart Nodes 3 and 4 restart_nodes(looper, txnPoolNodeSet, txnPoolNodeSet[2:], tconf, tdir, allPluginsPath, start_one_by_one=False) for n in txnPoolNodeSet[2:]: patch_primaries_in_audit(n, monkeypatch) # 5. make sure that all node have equal Priamries and can order ensureElectionsDone(looper, txnPoolNodeSet, customTimeout=30) ensure_all_nodes_have_same_data(looper, txnPoolNodeSet, custom_timeout=20) sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle)
def test_view_change_with_instance_change_lost_due_to_restarts( looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, tconf, tdir, allPluginsPath): """ 1. some_nodes (Beta and Gamma) send InstanceChange for all nodes. 2. Restart other_nodes (Gamma and Delta) 3. last_node (Delta) send InstanceChange for all nodes. 4. Ensure elections done and pool is functional """ current_view_no = txnPoolNodeSet[0].viewNo some_nodes = txnPoolNodeSet[1:3] other_nodes = txnPoolNodeSet[2:4] for n in some_nodes: n.view_changer.on_master_degradation() def check_ic_delivery(): for node in txnPoolNodeSet: assert node.view_changer.instanceChanges._votes_count( current_view_no + 1) == 2 looper.run(eventually(check_ic_delivery)) restart_nodes(looper, txnPoolNodeSet, other_nodes, tconf, tdir, allPluginsPath, start_one_by_one=False) last_node = txnPoolNodeSet[-1] last_node.view_changer.on_master_degradation() waitForViewChange(looper, txnPoolNodeSet, current_view_no + 1, customTimeout=3 * FRESHNESS_TIMEOUT) ensureElectionsDone(looper, txnPoolNodeSet) sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle)
def test_restart_half_to_lower_view(looper, txnPoolNodeSet, tconf, tdir, allPluginsPath, sdk_pool_handle, sdk_wallet_client): # Add transaction to ledger sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) # Move to higher view ensure_view_change_complete(looper, txnPoolNodeSet) # Restart half of nodes tm = tconf.ToleratePrimaryDisconnection + waits.expectedPoolElectionTimeout(len(txnPoolNodeSet)) nodes_before_restart = txnPoolNodeSet.copy() restart_nodes(looper, txnPoolNodeSet, txnPoolNodeSet[2:], tconf, tdir, allPluginsPath, after_restart_timeout=tm, start_one_by_one=False) # Check that nodes didn't think they may have inconsistent 3PC state for node in nodes_before_restart: assert node.spylog.count(node.on_inconsistent_3pc_state) == 0 # Check that nodes don't think they may have inconsistent 3PC state for node in txnPoolNodeSet: assert node.spylog.count(node.on_inconsistent_3pc_state) == 0 # Check that all nodes are still functional sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle, num_reqs=2, num_batches=1)