def view_change_in_between_3pc_random_delays(looper,
nodes,
slow_nodes,
sdk_pool_handle,
sdk_wallet_client,
tconf,
min_delay=0,
max_delay=0):
sdk_send_random_and_check(looper, nodes, sdk_pool_handle,
sdk_wallet_client, 4)
# max delay should not be more than catchup timeout.
max_delay = max_delay or tconf.NEW_VIEW_TIMEOUT - 1
delay_3pc_messages(slow_nodes, 0, min_delay=min_delay, max_delay=max_delay)
sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 10)
ensure_view_change_complete(looper,
nodes,
customTimeout=2 * tconf.NEW_VIEW_TIMEOUT +
max_delay,
exclude_from_check=['check_last_ordered_3pc'])
reset_delays_and_process_delayeds(slow_nodes)
sdk_send_random_and_check(looper, nodes, sdk_pool_handle,
sdk_wallet_client, 10)
def view_change_in_between_3pc_random_delays(looper,
nodes,
slow_nodes,
wallet,
client,
tconf,
min_delay=0,
max_delay=0):
send_reqs_to_nodes_and_verify_all_replies(looper, wallet, client, 4)
# max delay should not be more than catchup timeout.
max_delay = max_delay or tconf.MIN_TIMEOUT_CATCHUPS_DONE_DURING_VIEW_CHANGE - 1
delay_3pc_messages(slow_nodes, 0, min_delay=min_delay, max_delay=max_delay)
sendRandomRequests(wallet, client, 10)
ensure_view_change_complete(looper,
nodes,
customTimeout=2 * tconf.VIEW_CHANGE_TIMEOUT +
max_delay,
exclude_from_check=['check_last_ordered_3pc'])
reset_delays_and_process_delayeds(slow_nodes)
send_reqs_to_nodes_and_verify_all_replies(looper, wallet, client, 10)
def view_change_in_between_3pc(looper,
nodes,
slow_nodes,
sdk_pool_handle,
sdk_wallet_client,
slow_delay=1,
wait=None):
sdk_send_random_and_check(looper, nodes, sdk_pool_handle,
sdk_wallet_client, 4)
delay_3pc_messages(slow_nodes, 0, delay=slow_delay)
sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 10)
if wait:
looper.runFor(wait)
ensure_view_change(looper, nodes)
looper.run(eventually(check_not_in_view_change, nodes))
reset_delays_and_process_delayeds(slow_nodes)
ensureElectionsDone(looper=looper, nodes=nodes)
ensure_all_nodes_have_same_data(looper, nodes)
sdk_send_random_and_check(looper,
nodes,
sdk_pool_handle,
sdk_wallet_client,
5,
total_timeout=30)
sdk_send_random_and_check(looper,
nodes,
sdk_pool_handle,
sdk_wallet_client,
5,
total_timeout=30)
def view_change_in_between_3pc(looper,
nodes,
slow_nodes,
sdk_pool_handle,
sdk_wallet_client,
slow_delay=1,
wait=None):
sdk_send_random_and_check(looper, nodes, sdk_pool_handle,
sdk_wallet_client, 4)
delay_3pc_messages(slow_nodes, 0, delay=slow_delay)
sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 10)
if wait:
looper.runFor(wait)
ensure_view_change_complete(looper, nodes, customTimeout=60)
reset_delays_and_process_delayeds(slow_nodes)
sdk_send_random_and_check(looper,
nodes,
sdk_pool_handle,
sdk_wallet_client,
5,
total_timeout=30)
sdk_send_random_and_check(looper,
nodes,
sdk_pool_handle,
sdk_wallet_client,
5,
total_timeout=30)
def view_change_in_between_3pc(looper,
nodes,
slow_nodes,
wallet,
client,
slow_delay=1,
wait=None):
send_reqs_to_nodes_and_verify_all_replies(looper, wallet, client, 4)
delay_3pc_messages(slow_nodes, 0, delay=slow_delay)
sendRandomRequests(wallet, client, 10)
if wait:
looper.runFor(wait)
ensure_view_change_complete(looper, nodes, customTimeout=60)
reset_delays_and_process_delayeds(slow_nodes)
sendReqsToNodesAndVerifySuffReplies(looper,
wallet,
client,
5,
total_timeout=30)
send_reqs_to_nodes_and_verify_all_replies(looper,
wallet,
client,
5,
total_timeout=30)
def test_view_change_timeout(nodeSet, looper, up, wallet1, client1):
"""
Check view change restarted if it is not completed in time
"""
m_primary_node = get_master_primary_node(list(nodeSet.nodes.values()))
initial_view_no = waitForViewChange(looper, nodeSet)
# Setting view change timeout to low value to make test pass quicker
for node in nodeSet:
node._view_change_timeout = 5
# Delaying view change messages to make first view change fail
# due to timeout
for node in nodeSet:
node.nodeIbStasher.delay(vcd_delay(delay=50))
# Delaying preprepae messages from nodes and
# sending request to force view change
#for i in range(3):
# delayNonPrimaries(nodeSet, instId=i, delay=10)
#sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 4)
times = {}
for node in nodeSet:
times[node.name] = {
'called': get_count(node, node._check_view_change_completed),
'returned_true': len(getAllReturnVals(
node, node._check_view_change_completed, compare_val_to=True))
}
for node in nodeSet:
node.startViewChange(initial_view_no + 1)
# First view change should fail, because of delayed
# instance change messages. This then leads to new view change that we need.
with pytest.raises(AssertionError):
ensureElectionsDone(looper=looper, nodes=nodeSet, customTimeout=10)
# Resetting delays to let second view change go well
reset_delays_and_process_delayeds(nodeSet)
# This view change should be completed with no problems
ensureElectionsDone(looper=looper, nodes=nodeSet)
ensure_all_nodes_have_same_data(looper, nodes=nodeSet)
new_m_primary_node = get_master_primary_node(list(nodeSet.nodes.values()))
assert m_primary_node.name != new_m_primary_node.name
# The timeout method has been called at least once
for node in nodeSet:
assert get_count(node, node._check_view_change_completed) > times[node.name]['called']
assert len(getAllReturnVals(node,
node._check_view_change_completed,
compare_val_to=True)) > times[node.name]['returned_true']
# Multiple view changes have been initiated
for node in nodeSet:
assert (node.viewNo - initial_view_no) > 1
ensure_pool_functional(looper, nodeSet, wallet1, client1)
def test_multiple_view_change_retries_by_timeouts(txnPoolNodeSet, looper,
setup, sdk_pool_handle,
sdk_wallet_client):
"""
Verifies that a view change is restarted each time
when the previous one is timed out
"""
_, initial_view_no, timeout_callback_stats = setup
delay_view_change_done_msg(txnPoolNodeSet)
start_view_change(txnPoolNodeSet, initial_view_no + 1)
def check_timeout_callback_called(times):
for node in txnPoolNodeSet:
assert get_count(node,
node._check_view_change_completed) - \
timeout_callback_stats[node.name]['called'] == times
assert len(getAllReturnVals(node,
node._check_view_change_completed,
compare_val_to=True)) - \
timeout_callback_stats[node.name]['returned_true'] == times
# Check that the timeout method was called 3 times
# during the triple view_change_timeout period plus a margin
looper.run(
eventually(check_timeout_callback_called,
3,
retryWait=1,
timeout=3 * view_change_timeout + 2))
# Check that the last view change has failed
with pytest.raises(AssertionError):
ensureElectionsDone(looper=looper,
nodes=txnPoolNodeSet,
customTimeout=1)
# Reset delays to let the next view change go well
reset_delays_and_process_delayeds(txnPoolNodeSet)
# This view change must be completed with no problems
ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet)
ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet)
# 4 view changes must have been initiated (initial one + 3 retries)
for node in txnPoolNodeSet:
assert node.viewNo - initial_view_no == 4
sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_client,
sdk_pool_handle)
def test_view_change_retry_by_timeout(
txnPoolNodeSet, looper, setup, sdk_pool_handle, sdk_wallet_client):
"""
Verifies that a view change is restarted if it is not completed in time
"""
m_primary_node, initial_view_no, timeout_callback_stats = setup
delay_view_change_done_msg(txnPoolNodeSet)
start_view_change(txnPoolNodeSet, initial_view_no + 1)
# First view change should fail, because of delayed ViewChangeDone
# messages. This then leads to new view change that we need.
with pytest.raises(AssertionError):
ensureElectionsDone(looper=looper,
nodes=txnPoolNodeSet,
customTimeout=view_change_timeout + 2)
# Resetting delays to let second view change go well
reset_delays_and_process_delayeds(txnPoolNodeSet)
# This view change should be completed with no problems
ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet)
ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet)
new_m_primary_node = get_master_primary_node(list(txnPoolNodeSet))
assert m_primary_node.name != new_m_primary_node.name
# The timeout method was called one time
for node in txnPoolNodeSet:
assert get_count(node,
node._check_view_change_completed) - \
timeout_callback_stats[node.name]['called'] == 1
assert len(getAllReturnVals(node,
node._check_view_change_completed,
compare_val_to=True)) - \
timeout_callback_stats[node.name]['returned_true'] == 1
# 2 view changes have been initiated
for node in txnPoolNodeSet:
assert node.viewNo - initial_view_no == 2
sdk_ensure_pool_functional(looper, txnPoolNodeSet,
sdk_wallet_client,
sdk_pool_handle)
def view_change_in_between_3pc_random_delays(looper,
nodes,
slow_nodes,
wallet,
client,
min_delay=0,
max_delay=5):
send_reqs_to_nodes_and_verify_all_replies(looper, wallet, client, 4)
delay_3pc_messages(slow_nodes, 0, min_delay=min_delay, max_delay=max_delay)
sendRandomRequests(wallet, client, 10)
ensure_view_change(looper, nodes)
ensureElectionsDone(looper=looper, nodes=nodes)
ensure_all_nodes_have_same_data(looper, nodes=nodes)
reset_delays_and_process_delayeds(slow_nodes)
send_reqs_to_nodes_and_verify_all_replies(looper, wallet, client, 10)
def test_view_change_timeout(nodeSet, looper, up, setup, wallet1, client1):
"""
Check view change restarted if it is not completed in time
"""
m_primary_node, initial_view_no, times = setup
delay_view_change_msg(nodeSet)
start_view_change(nodeSet, initial_view_no + 1)
# First view change should fail, because of delayed
# instance change messages. This then leads to new view change that we
# need.
with pytest.raises(AssertionError):
ensureElectionsDone(looper=looper, nodes=nodeSet, customTimeout=10)
# Resetting delays to let second view change go well
reset_delays_and_process_delayeds(nodeSet)
# This view change should be completed with no problems
ensureElectionsDone(looper=looper, nodes=nodeSet)
ensure_all_nodes_have_same_data(looper, nodes=nodeSet)
new_m_primary_node = get_master_primary_node(list(nodeSet.nodes.values()))
assert m_primary_node.name != new_m_primary_node.name
# The timeout method has been called at least once
for node in nodeSet:
assert get_count(
node,
node._check_view_change_completed) > times[node.name]['called']
assert len(
getAllReturnVals(
node, node._check_view_change_completed,
compare_val_to=True)) > times[node.name]['returned_true']
# Multiple view changes have been initiated
for node in nodeSet:
assert (node.viewNo - initial_view_no) > 1
ensure_pool_functional(looper, nodeSet, wallet1, client1)
def test_view_change_on_start(tconf, txnPoolNodeSet, looper, sdk_pool_handle,
sdk_wallet_client):
"""
Do view change on a without any requests
"""
old_view_no = txnPoolNodeSet[0].viewNo
master_primary = get_master_primary_node(txnPoolNodeSet)
other_nodes = [n for n in txnPoolNodeSet if n != master_primary]
delay_3pc = 10
delay_3pc_messages(txnPoolNodeSet, 0, delay_3pc)
sent_batches = 2
sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client,
sent_batches * tconf.Max3PCBatchSize)
def chk1():
t_root, s_root = check_uncommitteds_equal(other_nodes)
assert master_primary.domainLedger.uncommittedRootHash != t_root
assert master_primary.states[DOMAIN_LEDGER_ID].headHash != s_root
looper.run(eventually(chk1, retryWait=1))
timeout = tconf.PerfCheckFreq + \
waits.expectedPoolElectionTimeout(len(txnPoolNodeSet))
waitForViewChange(looper,
txnPoolNodeSet,
old_view_no + 1,
customTimeout=timeout)
ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet)
check_uncommitteds_equal(txnPoolNodeSet)
reset_delays_and_process_delayeds(txnPoolNodeSet)
sdk_send_random_and_check(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
2 * Max3PCBatchSize,
add_delay_to_timeout=delay_3pc)
ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet)
def test_view_change_gc_in_between_3pc_all_nodes_delays(
looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client):
"""
Test that garbage collector compares the whole 3PC key (viewNo, ppSeqNo)
and does not remove messages from node's queues that have higher
viewNo than last ordered one even if their ppSeqNo are less or equal
"""
numNodes = len(txnPoolNodeSet)
viewNo = checkViewNoForNodes(txnPoolNodeSet)
# 1 send two messages one by one separately to make
# node pool working with two batches
# -> last_ordered_3pc = (+0, 2) [+0 means from the initial state]
# (last_ordered_3pc here and futher is tracked
# for master instances only cause non-master ones have
# specific logic of its management which we don't care in
# the test, see Replica::_setup_for_non_master)
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
last_ordered_3pc = (viewNo, 2)
check_nodes_last_ordered_3pc(txnPoolNodeSet, last_ordered_3pc)
check_nodes_requests_size(txnPoolNodeSet, 2)
# 2 do view change
# -> GC should remove it from nodes' queues
# -> viewNo = +1
ensure_view_change_complete(looper, txnPoolNodeSet)
viewNo = checkViewNoForNodes(txnPoolNodeSet, viewNo + 1)
check_nodes_last_ordered_3pc(txnPoolNodeSet, last_ordered_3pc)
check_nodes_requests_size(txnPoolNodeSet, 0)
# 3 slow processing 3PC messages for all nodes (all replica instances)
# randomly and send one more message
# -> not ordered (last_ordered_3pc still equal (+0, 2)) but primaries
# should at least send PRE-PREPAREs
# TODO could it be not enough for wainting that at least primary
# has sent PRE-PREPARE
propagationTimeout = waits.expectedClientRequestPropagationTime(numNodes)
delay_3pc_messages(txnPoolNodeSet,
0,
delay=propagationTimeout * 2)
delay_3pc_messages(txnPoolNodeSet,
1,
delay=propagationTimeout * 2)
requests = sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client)
def checkPrePrepareSentAtLeastByPrimary():
for node in txnPoolNodeSet:
for replica in node.replicas.values():
if replica.isPrimary:
assert len(replica.sentPrePrepares)
looper.run(eventually(checkPrePrepareSentAtLeastByPrimary,
retryWait=0.1,
timeout=propagationTimeout))
# 4 do view change
# -> GC shouldn't remove anything because
# last_ordered_3pc (+0, 1) < last message's 3pc key (+1, 1)
# -> viewNo = 2
ensure_view_change_complete(looper, txnPoolNodeSet)
viewNoNew = checkViewNoForNodes(txnPoolNodeSet)
# another view change could happen because of slow nodes
assert viewNoNew - viewNo in (1, 2)
viewNo = viewNoNew
check_nodes_last_ordered_3pc(txnPoolNodeSet, last_ordered_3pc)
check_nodes_requests_size(txnPoolNodeSet, 1)
# 5 reset delays and wait for replies
# -> new primaries should send new 3pc for last message
# with 3pc key (+2, 1)
# -> they should be ordered
# -> last_ordered_3pc = (+2, 1)
reset_delays_and_process_delayeds(txnPoolNodeSet)
sdk_get_replies(looper, [requests])
checkViewNoForNodes(txnPoolNodeSet, viewNo)
last_ordered_3pc = (viewNo, 1)
check_nodes_last_ordered_3pc(txnPoolNodeSet, last_ordered_3pc)
check_nodes_requests_size(txnPoolNodeSet, 1)
# 6 do view change
# -> GC should remove them
ensure_view_change_complete(looper, txnPoolNodeSet)
viewNo = checkViewNoForNodes(txnPoolNodeSet, viewNo + 1)
check_nodes_last_ordered_3pc(txnPoolNodeSet, last_ordered_3pc)
check_nodes_requests_size(txnPoolNodeSet, 0)
def test_view_change_gc_in_between_3pc_all_nodes_delays(
looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client):
"""
Test that garbage collector compares the whole 3PC key (viewNo, ppSeqNo)
and does not remove messages from node's queues that have higher
viewNo than last ordered one even if their ppSeqNo are less or equal
"""
numNodes = len(txnPoolNodeSet)
viewNo = checkViewNoForNodes(txnPoolNodeSet)
# 1 send two messages one by one separately to make
# node pool working with two batches
# -> last_ordered_3pc = (+0, 2) [+0 means from the initial state]
# (last_ordered_3pc here and futher is tracked
# for master instances only cause non-master ones have
# specific logic of its management which we don't care in
# the test, see Replica::_setup_for_non_master)
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
batches_count = get_pp_seq_no(txnPoolNodeSet)
last_ordered_3pc = (viewNo, batches_count)
check_nodes_last_ordered_3pc(txnPoolNodeSet, last_ordered_3pc)
check_nodes_requests_size(txnPoolNodeSet, 2)
# 2 do view change
# -> GC should remove it from nodes' queues
# -> viewNo = +1
ensure_view_change_complete(looper, txnPoolNodeSet)
batches_count += 1
viewNo = checkViewNoForNodes(txnPoolNodeSet, viewNo + 1)
looper.run(
eventually(check_nodes_last_ordered_3pc, txnPoolNodeSet,
(viewNo, batches_count)))
check_nodes_requests_size(txnPoolNodeSet, 0)
# 3 slow processing 3PC messages for all nodes (all replica instances)
# randomly and send one more message
# -> not ordered (last_ordered_3pc still equal (+0, 2)) but primaries
# should at least send PRE-PREPAREs
# TODO could it be not enough for wainting that at least primary
# has sent PRE-PREPARE
propagationTimeout = waits.expectedClientRequestPropagationTime(numNodes)
delay_3pc_messages(txnPoolNodeSet, 0, delay=propagationTimeout * 2)
delay_3pc_messages(txnPoolNodeSet, 1, delay=propagationTimeout * 2)
requests = sdk_send_random_request(looper, sdk_pool_handle,
sdk_wallet_client)
def checkPrePrepareSentAtLeastByPrimary():
for node in txnPoolNodeSet:
for replica in node.replicas.values():
if replica.isPrimary:
assert len(replica._ordering_service.sent_preprepares)
looper.run(
eventually(checkPrePrepareSentAtLeastByPrimary,
retryWait=0.1,
timeout=propagationTimeout))
# 4 do view change
# -> GC shouldn't remove anything because
# last_ordered_3pc (+0, 1) < last message's 3pc key (+1, 1)
# -> viewNo = 2
ensure_view_change_complete(looper, txnPoolNodeSet)
batches_count += 1
viewNoNew = checkViewNoForNodes(txnPoolNodeSet)
# another view change could happen because of slow nodes
assert viewNoNew - viewNo in (1, 2)
viewNo = viewNoNew
check_nodes_last_ordered_3pc(txnPoolNodeSet,
(last_ordered_3pc[0] + 1, batches_count - 1))
check_nodes_requests_size(txnPoolNodeSet, 1)
# 5 reset delays and wait for replies
# -> new primaries should send new 3pc for last message
# with 3pc key (+2, 1)
# -> they should be ordered
# -> last_ordered_3pc = (+2, 1)
reset_delays_and_process_delayeds(txnPoolNodeSet)
sdk_get_replies(looper, [requests])
batches_count += 1
checkViewNoForNodes(txnPoolNodeSet, viewNo)
last_ordered_3pc = (viewNo, batches_count)
check_nodes_last_ordered_3pc(txnPoolNodeSet, last_ordered_3pc)
check_nodes_requests_size(txnPoolNodeSet, 1)
# 6 do view change
# -> GC should remove them
ensure_view_change_complete(looper, txnPoolNodeSet)
batches_count += 1
viewNo = checkViewNoForNodes(txnPoolNodeSet, viewNo + 1)
check_nodes_last_ordered_3pc(txnPoolNodeSet,
(last_ordered_3pc[0] + 1, batches_count))
check_nodes_requests_size(txnPoolNodeSet, 0)
def cancel_slow_catch_up(nodes):
reset_delays_and_process_delayeds(nodes)
reset_delays_and_process_delayeds_for_client(nodes)
