def test_freshness_batch_updates_last_ordered_non_primary(
looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, tconf,
tdir, allPluginsPath):
looper.run(
eventually(check_freshness_updated_for_all,
txnPoolNodeSet,
timeout=FRESHNESS_TIMEOUT * 2))
bls_multi_sigs_after_first_update = get_all_multi_sig_values_for_all_nodes(
txnPoolNodeSet)
looper.run(
eventually(check_updated_bls_multi_sig_for_all_ledgers,
txnPoolNodeSet,
bls_multi_sigs_after_first_update,
FRESHNESS_TIMEOUT,
timeout=FRESHNESS_TIMEOUT * 2))
restart_node(looper, txnPoolNodeSet, txnPoolNodeSet[1], tconf, tdir,
allPluginsPath)
assert txnPoolNodeSet[0].db_manager.get_txn_root_hash(DOMAIN_LEDGER_ID) == \
txnPoolNodeSet[1].db_manager.get_txn_root_hash(DOMAIN_LEDGER_ID)
# node caught up till actual last_ordered_3pc
assert txnPoolNodeSet[0].master_replica.last_ordered_3pc == \
txnPoolNodeSet[1].master_replica.last_ordered_3pc
# correct ordering
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, 1)
# domain ledger equeal
assert txnPoolNodeSet[0].db_manager.get_txn_root_hash(DOMAIN_LEDGER_ID) == \
txnPoolNodeSet[1].db_manager.get_txn_root_hash(DOMAIN_LEDGER_ID)
def test_demote_promote_restart_after_promotion_7_nodes(txnPoolNodeSet,
looper,
sdk_pool_handle,
sdk_wallet_steward,
tdir,
tconf,
allPluginsPath):
demoted_node = txnPoolNodeSet[-1]
rest_nodes = [n for n in txnPoolNodeSet if n != demoted_node]
starting_view_no = checkViewNoForNodes(txnPoolNodeSet)
demote_node(looper, sdk_wallet_steward, sdk_pool_handle, demoted_node)
waitForViewChange(looper, rest_nodes, expectedViewNo=starting_view_no + 1)
ensureElectionsDone(looper, rest_nodes)
ensure_all_nodes_have_same_data(looper, rest_nodes)
sdk_send_random_and_check(looper, rest_nodes, sdk_pool_handle, sdk_wallet_steward, 5)
starting_view_no = checkViewNoForNodes(rest_nodes)
promote_node(looper, sdk_wallet_steward, sdk_pool_handle, demoted_node)
waitForViewChange(looper, rest_nodes, expectedViewNo=starting_view_no + 1)
ensureElectionsDone(looper, rest_nodes, instances_list=[0, 1, 2])
ensure_all_nodes_have_same_data(looper, rest_nodes)
restart_node(looper, txnPoolNodeSet, demoted_node, tconf, tdir, allPluginsPath)
ensureElectionsDone(looper, txnPoolNodeSet)
sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_steward, sdk_pool_handle)
def test_view_change_not_happen_if_ic_is_discarded(looper, txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client, tconf,
tdir, allPluginsPath):
"""
1. panic_node (Delta) send InstanceChange for all nodes.
2. Restart nodes_to_restart (Beta, Gamma).
3. Wait OUTDATED_INSTANCE_CHANGES_CHECK_INTERVAL sec.
4. nodes_to_restart send InstanceChanges for all nodes.
5. View change doesn't happen since old InstanceChange from panic_node was discarded due to timeout.
5. Ensure elections done
"""
nodes_to_restart = txnPoolNodeSet[1:3]
panic_node = txnPoolNodeSet[-1]
view_no = txnPoolNodeSet[0].viewNo
send_test_instance_change(panic_node)
for n in nodes_to_restart:
restart_node(looper, txnPoolNodeSet, n, tconf, tdir, allPluginsPath)
nodes_to_restart = txnPoolNodeSet[1:3]
# waiting to discard InstanceChange
def check_old_ic_discarded():
vct_services = [
n.master_replica._view_change_trigger_service
for n in txnPoolNodeSet
]
assert all(not vct_service._instance_changes.has_inst_chng_from(
view_no + 1, panic_node.name) for vct_service in vct_services)
looper.run(
eventually(check_old_ic_discarded,
timeout=tconf.OUTDATED_INSTANCE_CHANGES_CHECK_INTERVAL + 1))
for n in nodes_to_restart:
send_test_instance_change(n)
def check_ic():
for node in txnPoolNodeSet:
vct_service = node.master_replica._view_change_trigger_service
assert all(
vct_service._instance_changes.has_inst_chng_from(
view_no + 1, n.name) for n in nodes_to_restart)
looper.run(eventually(check_ic))
ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet)
ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet)
for node in txnPoolNodeSet:
assert node.viewNo == view_no
def test_freshness_batch_updates_last_ordered(looper, txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_steward, tconf, tdir,
allPluginsPath):
assert txnPoolNodeSet[0].master_replica.isPrimary
looper.run(
eventually(check_freshness_updated_for_all,
txnPoolNodeSet,
timeout=FRESHNESS_TIMEOUT * 2))
bls_multi_sigs_after_first_update = get_all_multi_sig_values_for_all_nodes(
txnPoolNodeSet)
looper.run(
eventually(check_updated_bls_multi_sig_for_all_ledgers,
txnPoolNodeSet,
bls_multi_sigs_after_first_update,
FRESHNESS_TIMEOUT,
timeout=FRESHNESS_TIMEOUT * 2))
restart_node(looper, txnPoolNodeSet, txnPoolNodeSet[0], tconf, tdir,
allPluginsPath)
# no view change happened
assert txnPoolNodeSet[0].master_replica.isPrimary
assert txnPoolNodeSet[0].master_replica.txnRootHash(DOMAIN_LEDGER_ID) == \
txnPoolNodeSet[1].master_replica.txnRootHash(DOMAIN_LEDGER_ID)
# node caught up till actual last_ordered_3pc
assert txnPoolNodeSet[0].master_replica.last_ordered_3pc == \
txnPoolNodeSet[1].master_replica.last_ordered_3pc
old_discard = len(getSpecificDiscardedMsg(txnPoolNodeSet[1], PrePrepare))
# correct ordering
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, 1)
# domain ledger equeal
assert txnPoolNodeSet[0].master_replica.txnRootHash(DOMAIN_LEDGER_ID) == \
txnPoolNodeSet[1].master_replica.txnRootHash(DOMAIN_LEDGER_ID)
# no discard happened
assert len(getSpecificDiscardedMsg(txnPoolNodeSet[1],
PrePrepare)) == old_discard
def test_vc_started_in_different_time(looper, txnPoolNodeSet,
sdk_wallet_client, sdk_pool_handle,
tconf, tdir, allPluginsPath):
alpha, beta, gamma, delta = txnPoolNodeSet
# Delta and Gamma send InstanceChange for all nodes.
for node in [gamma, delta]:
send_test_instance_change(node)
looper.run(eventually(nodes_received_ic, txnPoolNodeSet, node, 1))
# Restart Alpha, Beta, Gamma
for node in [alpha, beta, gamma]:
restart_node(looper, txnPoolNodeSet, node, tconf, tdir, allPluginsPath)
alpha, beta, gamma, delta = txnPoolNodeSet
# Send InstanceChange from Beta for all nodes
send_test_instance_change(beta)
# Ensure that pool is still functional
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
# Restart Alpha, Beta
for i, node in enumerate([alpha, beta]):
restart_node(looper,
txnPoolNodeSet,
node,
tconf,
tdir,
allPluginsPath,
wait_node_data_equality=False)
alpha, beta, gamma, delta = txnPoolNodeSet
# Alpha, Gamma send InstanceChange for all nodes.
for node in [alpha, gamma]:
send_test_instance_change(node)
# Ensure that pool is still functional
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
ensure_all_nodes_have_same_data(looper, txnPoolNodeSet)
def test_demote_promote_restart_after_promotion_from_10_to_4_nodes(txnPoolNodeSet,
looper,
sdk_pool_handle,
sdk_wallet_steward,
tdir,
tconf,
allPluginsPath):
"""
We expect that 2 changes for f value should be happened
"""
def demote_another_one(rest_pool):
demoted_node = rest_pool[-1]
rest_pool = [n for n in rest_pool if n != demoted_node]
starting_view_no = checkViewNoForNodes(rest_pool)
demote_node(looper, sdk_wallet_steward, sdk_pool_handle, demoted_node)
waitForViewChange(looper, rest_pool, expectedViewNo=starting_view_no + 1)
ensureElectionsDone(looper, rest_pool, customTimeout=60)
ensure_all_nodes_have_same_data(looper, rest_pool)
return rest_pool
rest_nodes = txnPoolNodeSet
etalon_node = txnPoolNodeSet[-1]
while len(rest_nodes) > 4:
rest_nodes = demote_another_one(rest_nodes)
sdk_send_random_and_check(looper, rest_nodes, sdk_pool_handle, sdk_wallet_steward, 5)
starting_view_no = checkViewNoForNodes(rest_nodes)
promote_node(looper, sdk_wallet_steward, sdk_pool_handle, etalon_node)
waitForViewChange(looper, rest_nodes, expectedViewNo=starting_view_no + 1)
ensure_all_nodes_have_same_data(looper, rest_nodes)
rest_nodes.append(etalon_node)
restart_node(looper, rest_nodes, etalon_node, tconf, tdir, allPluginsPath)
ensureElectionsDone(looper, rest_nodes)
sdk_ensure_pool_functional(looper, rest_nodes, sdk_wallet_steward, sdk_pool_handle)
def test_vc_finished_when_less_than_quorum_started(looper, txnPoolNodeSet,
sdk_wallet_client,
sdk_pool_handle, tconf,
tdir, allPluginsPath):
alpha, beta, gamma, delta = txnPoolNodeSet
# Delta and Gamma send InstanceChange for all nodes.
for node in [gamma, delta]:
node.view_changer.on_master_degradation()
looper.run(eventually(nodes_received_ic, txnPoolNodeSet, node, 1))
# Restart Alpha, Beta, Gamma
for node in [alpha, beta, gamma]:
restart_node(looper, txnPoolNodeSet, node, tconf, tdir, allPluginsPath)
alpha, beta, gamma, delta = txnPoolNodeSet
# Send InstanceChange from Beta for all nodes
beta.view_changer.on_master_degradation()
# Ensure that pool is still functional
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
# Alpha and Gamma send InstanceChange for all nodes.
for node in [gamma, alpha]:
node.view_changer.on_master_degradation()
ensureElectionsDone(looper, txnPoolNodeSet)
waitForViewChange(looper,
txnPoolNodeSet,
expectedViewNo=1,
customTimeout=tconf.VIEW_CHANGE_TIMEOUT)
# Ensure that pool is still functional
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
ensure_all_nodes_have_same_data(looper, txnPoolNodeSet)
