def test_freshness_batch_updates_last_ordered_non_primary( looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, tconf, tdir, allPluginsPath): looper.run( eventually(check_freshness_updated_for_all, txnPoolNodeSet, timeout=FRESHNESS_TIMEOUT * 2)) bls_multi_sigs_after_first_update = get_all_multi_sig_values_for_all_nodes( txnPoolNodeSet) looper.run( eventually(check_updated_bls_multi_sig_for_all_ledgers, txnPoolNodeSet, bls_multi_sigs_after_first_update, FRESHNESS_TIMEOUT, timeout=FRESHNESS_TIMEOUT * 2)) restart_node(looper, txnPoolNodeSet, txnPoolNodeSet[1], tconf, tdir, allPluginsPath) assert txnPoolNodeSet[0].db_manager.get_txn_root_hash(DOMAIN_LEDGER_ID) == \ txnPoolNodeSet[1].db_manager.get_txn_root_hash(DOMAIN_LEDGER_ID) # node caught up till actual last_ordered_3pc assert txnPoolNodeSet[0].master_replica.last_ordered_3pc == \ txnPoolNodeSet[1].master_replica.last_ordered_3pc # correct ordering sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 1) # domain ledger equeal assert txnPoolNodeSet[0].db_manager.get_txn_root_hash(DOMAIN_LEDGER_ID) == \ txnPoolNodeSet[1].db_manager.get_txn_root_hash(DOMAIN_LEDGER_ID)
def test_demote_promote_restart_after_promotion_7_nodes(txnPoolNodeSet, looper, sdk_pool_handle, sdk_wallet_steward, tdir, tconf, allPluginsPath): demoted_node = txnPoolNodeSet[-1] rest_nodes = [n for n in txnPoolNodeSet if n != demoted_node] starting_view_no = checkViewNoForNodes(txnPoolNodeSet) demote_node(looper, sdk_wallet_steward, sdk_pool_handle, demoted_node) waitForViewChange(looper, rest_nodes, expectedViewNo=starting_view_no + 1) ensureElectionsDone(looper, rest_nodes) ensure_all_nodes_have_same_data(looper, rest_nodes) sdk_send_random_and_check(looper, rest_nodes, sdk_pool_handle, sdk_wallet_steward, 5) starting_view_no = checkViewNoForNodes(rest_nodes) promote_node(looper, sdk_wallet_steward, sdk_pool_handle, demoted_node) waitForViewChange(looper, rest_nodes, expectedViewNo=starting_view_no + 1) ensureElectionsDone(looper, rest_nodes, instances_list=[0, 1, 2]) ensure_all_nodes_have_same_data(looper, rest_nodes) restart_node(looper, txnPoolNodeSet, demoted_node, tconf, tdir, allPluginsPath) ensureElectionsDone(looper, txnPoolNodeSet) sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_steward, sdk_pool_handle)
def test_view_change_not_happen_if_ic_is_discarded(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, tconf, tdir, allPluginsPath): """ 1. panic_node (Delta) send InstanceChange for all nodes. 2. Restart nodes_to_restart (Beta, Gamma). 3. Wait OUTDATED_INSTANCE_CHANGES_CHECK_INTERVAL sec. 4. nodes_to_restart send InstanceChanges for all nodes. 5. View change doesn't happen since old InstanceChange from panic_node was discarded due to timeout. 5. Ensure elections done """ nodes_to_restart = txnPoolNodeSet[1:3] panic_node = txnPoolNodeSet[-1] view_no = txnPoolNodeSet[0].viewNo send_test_instance_change(panic_node) for n in nodes_to_restart: restart_node(looper, txnPoolNodeSet, n, tconf, tdir, allPluginsPath) nodes_to_restart = txnPoolNodeSet[1:3] # waiting to discard InstanceChange def check_old_ic_discarded(): vct_services = [ n.master_replica._view_change_trigger_service for n in txnPoolNodeSet ] assert all(not vct_service._instance_changes.has_inst_chng_from( view_no + 1, panic_node.name) for vct_service in vct_services) looper.run( eventually(check_old_ic_discarded, timeout=tconf.OUTDATED_INSTANCE_CHANGES_CHECK_INTERVAL + 1)) for n in nodes_to_restart: send_test_instance_change(n) def check_ic(): for node in txnPoolNodeSet: vct_service = node.master_replica._view_change_trigger_service assert all( vct_service._instance_changes.has_inst_chng_from( view_no + 1, n.name) for n in nodes_to_restart) looper.run(eventually(check_ic)) ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet) ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet) for node in txnPoolNodeSet: assert node.viewNo == view_no
def test_freshness_batch_updates_last_ordered(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, tconf, tdir, allPluginsPath): assert txnPoolNodeSet[0].master_replica.isPrimary looper.run( eventually(check_freshness_updated_for_all, txnPoolNodeSet, timeout=FRESHNESS_TIMEOUT * 2)) bls_multi_sigs_after_first_update = get_all_multi_sig_values_for_all_nodes( txnPoolNodeSet) looper.run( eventually(check_updated_bls_multi_sig_for_all_ledgers, txnPoolNodeSet, bls_multi_sigs_after_first_update, FRESHNESS_TIMEOUT, timeout=FRESHNESS_TIMEOUT * 2)) restart_node(looper, txnPoolNodeSet, txnPoolNodeSet[0], tconf, tdir, allPluginsPath) # no view change happened assert txnPoolNodeSet[0].master_replica.isPrimary assert txnPoolNodeSet[0].master_replica.txnRootHash(DOMAIN_LEDGER_ID) == \ txnPoolNodeSet[1].master_replica.txnRootHash(DOMAIN_LEDGER_ID) # node caught up till actual last_ordered_3pc assert txnPoolNodeSet[0].master_replica.last_ordered_3pc == \ txnPoolNodeSet[1].master_replica.last_ordered_3pc old_discard = len(getSpecificDiscardedMsg(txnPoolNodeSet[1], PrePrepare)) # correct ordering sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 1) # domain ledger equeal assert txnPoolNodeSet[0].master_replica.txnRootHash(DOMAIN_LEDGER_ID) == \ txnPoolNodeSet[1].master_replica.txnRootHash(DOMAIN_LEDGER_ID) # no discard happened assert len(getSpecificDiscardedMsg(txnPoolNodeSet[1], PrePrepare)) == old_discard
def test_vc_started_in_different_time(looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle, tconf, tdir, allPluginsPath): alpha, beta, gamma, delta = txnPoolNodeSet # Delta and Gamma send InstanceChange for all nodes. for node in [gamma, delta]: send_test_instance_change(node) looper.run(eventually(nodes_received_ic, txnPoolNodeSet, node, 1)) # Restart Alpha, Beta, Gamma for node in [alpha, beta, gamma]: restart_node(looper, txnPoolNodeSet, node, tconf, tdir, allPluginsPath) alpha, beta, gamma, delta = txnPoolNodeSet # Send InstanceChange from Beta for all nodes send_test_instance_change(beta) # Ensure that pool is still functional sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) # Restart Alpha, Beta for i, node in enumerate([alpha, beta]): restart_node(looper, txnPoolNodeSet, node, tconf, tdir, allPluginsPath, wait_node_data_equality=False) alpha, beta, gamma, delta = txnPoolNodeSet # Alpha, Gamma send InstanceChange for all nodes. for node in [alpha, gamma]: send_test_instance_change(node) # Ensure that pool is still functional sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) ensure_all_nodes_have_same_data(looper, txnPoolNodeSet)
def test_demote_promote_restart_after_promotion_from_10_to_4_nodes(txnPoolNodeSet, looper, sdk_pool_handle, sdk_wallet_steward, tdir, tconf, allPluginsPath): """ We expect that 2 changes for f value should be happened """ def demote_another_one(rest_pool): demoted_node = rest_pool[-1] rest_pool = [n for n in rest_pool if n != demoted_node] starting_view_no = checkViewNoForNodes(rest_pool) demote_node(looper, sdk_wallet_steward, sdk_pool_handle, demoted_node) waitForViewChange(looper, rest_pool, expectedViewNo=starting_view_no + 1) ensureElectionsDone(looper, rest_pool, customTimeout=60) ensure_all_nodes_have_same_data(looper, rest_pool) return rest_pool rest_nodes = txnPoolNodeSet etalon_node = txnPoolNodeSet[-1] while len(rest_nodes) > 4: rest_nodes = demote_another_one(rest_nodes) sdk_send_random_and_check(looper, rest_nodes, sdk_pool_handle, sdk_wallet_steward, 5) starting_view_no = checkViewNoForNodes(rest_nodes) promote_node(looper, sdk_wallet_steward, sdk_pool_handle, etalon_node) waitForViewChange(looper, rest_nodes, expectedViewNo=starting_view_no + 1) ensure_all_nodes_have_same_data(looper, rest_nodes) rest_nodes.append(etalon_node) restart_node(looper, rest_nodes, etalon_node, tconf, tdir, allPluginsPath) ensureElectionsDone(looper, rest_nodes) sdk_ensure_pool_functional(looper, rest_nodes, sdk_wallet_steward, sdk_pool_handle)
def test_vc_finished_when_less_than_quorum_started(looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle, tconf, tdir, allPluginsPath): alpha, beta, gamma, delta = txnPoolNodeSet # Delta and Gamma send InstanceChange for all nodes. for node in [gamma, delta]: node.view_changer.on_master_degradation() looper.run(eventually(nodes_received_ic, txnPoolNodeSet, node, 1)) # Restart Alpha, Beta, Gamma for node in [alpha, beta, gamma]: restart_node(looper, txnPoolNodeSet, node, tconf, tdir, allPluginsPath) alpha, beta, gamma, delta = txnPoolNodeSet # Send InstanceChange from Beta for all nodes beta.view_changer.on_master_degradation() # Ensure that pool is still functional sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) # Alpha and Gamma send InstanceChange for all nodes. for node in [gamma, alpha]: node.view_changer.on_master_degradation() ensureElectionsDone(looper, txnPoolNodeSet) waitForViewChange(looper, txnPoolNodeSet, expectedViewNo=1, customTimeout=tconf.VIEW_CHANGE_TIMEOUT) # Ensure that pool is still functional sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) ensure_all_nodes_have_same_data(looper, txnPoolNodeSet)