def test_restarted_node_complete_vc_by_current_state(looper, txnPoolNodeSet,
tconf, tdir,
allPluginsPath):
node_to_restart = txnPoolNodeSet[-1]
disconnect_node_and_ensure_disconnected(looper,
txnPoolNodeSet,
node_to_restart,
stopNode=True)
looper.removeProdable(node_to_restart)
old_completed_view_no = get_last_completed_view_no(txnPoolNodeSet[:-1])
ensure_view_change(looper, txnPoolNodeSet[:-1])
ensureElectionsDone(looper,
txnPoolNodeSet[:-1],
customTimeout=tconf.VIEW_CHANGE_TIMEOUT)
current_completed_view_no = get_last_completed_view_no(txnPoolNodeSet[:-1])
assert current_completed_view_no > old_completed_view_no
# Delay VIEW_CHANGE_DONE messages for all nodes
for node in txnPoolNodeSet[:-1]:
node.nodeIbStasher.delay(vcd_delay(1000))
ensure_view_change(looper, txnPoolNodeSet[:-1])
# Start stopped node until other nodes do view_change
node_to_restart = start_stopped_node(node_to_restart, looper, tconf, tdir,
allPluginsPath)
node_to_restart.nodeIbStasher.delay(vcd_delay(1000))
# check, that restarted node use last completed view no from pool, instead of proposed
looper.run(
eventually(complete_propagate_primary,
node_to_restart,
current_completed_view_no,
timeout=tconf.VIEW_CHANGE_TIMEOUT))
def test_restarted_node_complete_vc_by_current_state(looper,
txnPoolNodeSet,
tconf,
tdir,
allPluginsPath):
node_to_restart = txnPoolNodeSet[-1]
disconnect_node_and_ensure_disconnected(looper,
txnPoolNodeSet,
node_to_restart,
stopNode=True)
looper.removeProdable(node_to_restart)
old_completed_view_no = get_last_completed_view_no(txnPoolNodeSet[:-1])
ensure_view_change(looper,
txnPoolNodeSet[:-1])
ensureElectionsDone(looper, txnPoolNodeSet[:-1], customTimeout=tconf.VIEW_CHANGE_TIMEOUT)
current_completed_view_no = get_last_completed_view_no(txnPoolNodeSet[:-1])
assert current_completed_view_no > old_completed_view_no
# Delay VIEW_CHANGE_DONE messages for all nodes
for node in txnPoolNodeSet[:-1]:
node.nodeIbStasher.delay(vcd_delay(1000))
ensure_view_change(looper, txnPoolNodeSet[:-1])
# Start stopped node until other nodes do view_change
node_to_restart = start_stopped_node(node_to_restart,
looper,
tconf,
tdir,
allPluginsPath)
node_to_restart.nodeIbStasher.delay(vcd_delay(1000))
# check, that restarted node use last completed view no from pool, instead of proposed
looper.run(eventually(complete_propagate_primary,
node_to_restart,
current_completed_view_no,
timeout=tconf.VIEW_CHANGE_TIMEOUT))
def test_view_change_retry_by_timeout(txnPoolNodeSet, looper, tconf, setup,
sdk_pool_handle, sdk_wallet_client):
"""
Verifies that a view change is restarted if it is not completed in time
"""
m_primary_node, initial_view_no, timeout_callback_stats = setup
stashers = [n.nodeIbStasher for n in txnPoolNodeSet]
with delay_rules(stashers, vcd_delay()):
start_view_change(txnPoolNodeSet, initial_view_no + 1)
# First view change should fail, because of delayed ViewChangeDone
# messages. This then leads to new view change that we need.
with pytest.raises(AssertionError):
ensureElectionsDone(looper=looper,
nodes=txnPoolNodeSet,
customTimeout=1.5 * VIEW_CHANGE_TIMEOUT)
# Now as ViewChangeDone messages are unblocked view changes should finish successfully
ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet)
ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet)
new_m_primary_node = get_master_primary_node(list(txnPoolNodeSet))
assert m_primary_node.name != new_m_primary_node.name
# The timeout method was called one time
check_watchdog_called_expected_times(txnPoolNodeSet,
timeout_callback_stats, 1)
# 2 view changes have been initiated
for node in txnPoolNodeSet:
assert node.viewNo - initial_view_no == 2
sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_client,
sdk_pool_handle)
def test_multiple_view_change_retries_by_timeouts(
txnPoolNodeSet, looper, tconf, setup,
sdk_pool_handle, sdk_wallet_client):
"""
Verifies that a view change is restarted each time
when the previous one is timed out
"""
_, initial_view_no, timeout_callback_stats = setup
stashers = [n.nodeIbStasher for n in txnPoolNodeSet]
with delay_rules(stashers, vcd_delay()):
start_view_change(txnPoolNodeSet, initial_view_no + 1)
# Wait until timeout callback is called 3 times
looper.run(eventually(check_watchdog_called_expected_times,
txnPoolNodeSet, timeout_callback_stats, 3,
retryWait=1,
timeout=3 * VIEW_CHANGE_TIMEOUT + 2))
# View changes should fail
with pytest.raises(AssertionError):
ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet, customTimeout=1)
# This view change must be completed with no problems
ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet)
ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet)
# 4 view changes must have been initiated (initial one + 3 retries)
for node in txnPoolNodeSet:
assert node.viewNo - initial_view_no == 4
sdk_ensure_pool_functional(looper, txnPoolNodeSet,
sdk_wallet_client,
sdk_pool_handle)
def test_multiple_view_change_retries_by_timeouts(
txnPoolNodeSet, looper, tconf, setup,
sdk_pool_handle, sdk_wallet_client):
"""
Verifies that a view change is restarted each time
when the previous one is timed out
"""
_, initial_view_no, timeout_callback_stats = setup
stashers = [n.nodeIbStasher for n in txnPoolNodeSet]
with delay_rules(stashers, vcd_delay()):
start_view_change(txnPoolNodeSet, initial_view_no + 1)
# Wait until timeout callback is called 3 times
looper.run(eventually(check_watchdog_called_expected_times,
txnPoolNodeSet, timeout_callback_stats, 3,
retryWait=1,
timeout=3 * VIEW_CHANGE_TIMEOUT + 2))
# View changes should fail
with pytest.raises(AssertionError):
ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet, customTimeout=1)
# This view change must be completed with no problems
ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet)
ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet)
# 4 view changes must have been initiated (initial one + 3 retries)
for node in txnPoolNodeSet:
assert node.viewNo - initial_view_no == 4
sdk_ensure_pool_functional(looper, txnPoolNodeSet,
sdk_wallet_client,
sdk_pool_handle)
def test_view_change_retry_by_timeout(
txnPoolNodeSet, looper, tconf, setup, sdk_pool_handle, sdk_wallet_client):
"""
Verifies that a view change is restarted if it is not completed in time
"""
m_primary_node, initial_view_no, timeout_callback_stats = setup
stashers = [n.nodeIbStasher for n in txnPoolNodeSet]
with delay_rules(stashers, vcd_delay()):
start_view_change(txnPoolNodeSet, initial_view_no + 1)
# First view change should fail, because of delayed ViewChangeDone
# messages. This then leads to new view change that we need.
with pytest.raises(AssertionError):
ensureElectionsDone(looper=looper,
nodes=txnPoolNodeSet,
customTimeout=1.5 * VIEW_CHANGE_TIMEOUT)
# Now as ViewChangeDone messages are unblocked view changes should finish successfully
ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet)
ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet)
new_m_primary_node = get_master_primary_node(list(txnPoolNodeSet))
assert m_primary_node.name != new_m_primary_node.name
# The timeout method was called one time
check_watchdog_called_expected_times(txnPoolNodeSet, timeout_callback_stats, 1)
# 2 view changes have been initiated
for node in txnPoolNodeSet:
assert node.viewNo - initial_view_no == 2
sdk_ensure_pool_functional(looper, txnPoolNodeSet,
sdk_wallet_client,
sdk_pool_handle)
def testQueueingReqFromFutureView(delayed_perf_chk, looper, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client):
"""
Test if every node queues 3 Phase requests(PRE-PREPARE, PREPARE and COMMIT)
that come from a view which is greater than the current view.
- Delay reception and processing of view change messages by a non primary for master instance
=> it starts receiving 3 phase commit messages for next view
"""
lagging_node = get_last_master_non_primary_node(txnPoolNodeSet)
old_view_no = lagging_node.viewNo
# Delay processing of InstanceChange and ViewChangeDone so node stashes
# 3PC messages
delay_ic = 60
lagging_node.nodeIbStasher.delay(icDelay(delay_ic))
lagging_node.nodeIbStasher.delay(vcd_delay(delay_ic))
logger.debug('{} will delay its view change'.format(lagging_node))
def chk_fut_view(view_no, is_empty):
length = len(lagging_node.msgsForFutureViews.get(view_no, ()))
if is_empty:
assert length == 0
else:
assert length > 0
return length
# No messages queued for future view
chk_fut_view(old_view_no + 1, is_empty=True)
logger.debug(
'{} does not have any messages for future views'.format(lagging_node))
# Every node except Node A should do a view change
ensure_view_change(looper,
[n for n in txnPoolNodeSet if n != lagging_node],
[lagging_node])
# send more requests that will be queued for the lagged node
reqs = sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client,
5)
l = looper.run(
eventually(chk_fut_view, old_view_no + 1, False, retryWait=1))
logger.debug('{} has {} messages for future views'.format(lagging_node, l))
sdk_get_replies(looper, reqs)
# reset delays for the lagging_node node so that it finally makes view
# change
lagging_node.reset_delays_and_process_delayeds()
# Eventually no messages queued for future view
looper.run(
eventually(chk_fut_view,
old_view_no + 1,
True,
retryWait=1,
timeout=delay_ic + 10))
logger.debug(
'{} exhausted pending messages for future views'.format(lagging_node))
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 2)
def test_view_change_timeout(nodeSet, looper, up, wallet1, client1):
"""
Check view change restarted if it is not completed in time
"""
m_primary_node = get_master_primary_node(list(nodeSet.nodes.values()))
initial_view_no = waitForViewChange(looper, nodeSet)
# Setting view change timeout to low value to make test pass quicker
for node in nodeSet:
node._view_change_timeout = 5
# Delaying view change messages to make first view change fail
# due to timeout
for node in nodeSet:
node.nodeIbStasher.delay(vcd_delay(delay=50))
# Delaying preprepae messages from nodes and
# sending request to force view change
#for i in range(3):
# delayNonPrimaries(nodeSet, instId=i, delay=10)
#sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 4)
times = {}
for node in nodeSet:
times[node.name] = {
'called': get_count(node, node._check_view_change_completed),
'returned_true': len(getAllReturnVals(
node, node._check_view_change_completed, compare_val_to=True))
}
for node in nodeSet:
node.startViewChange(initial_view_no + 1)
# First view change should fail, because of delayed
# instance change messages. This then leads to new view change that we need.
with pytest.raises(AssertionError):
ensureElectionsDone(looper=looper, nodes=nodeSet, customTimeout=10)
# Resetting delays to let second view change go well
reset_delays_and_process_delayeds(nodeSet)
# This view change should be completed with no problems
ensureElectionsDone(looper=looper, nodes=nodeSet)
ensure_all_nodes_have_same_data(looper, nodes=nodeSet)
new_m_primary_node = get_master_primary_node(list(nodeSet.nodes.values()))
assert m_primary_node.name != new_m_primary_node.name
# The timeout method has been called at least once
for node in nodeSet:
assert get_count(node, node._check_view_change_completed) > times[node.name]['called']
assert len(getAllReturnVals(node,
node._check_view_change_completed,
compare_val_to=True)) > times[node.name]['returned_true']
# Multiple view changes have been initiated
for node in nodeSet:
assert (node.viewNo - initial_view_no) > 1
ensure_pool_functional(looper, nodeSet, wallet1, client1)
def test_delayed_instance_changes_after_vcd_for_next_view(looper, txnPoolNodeSet):
'''
A node is doing view change to view=1, while the other nodes already finished view change to view=2.
The node receives a quorum of VCD messages for view=2 before a quorum of InstanceChange messages for view=2.
Nevertheless, the node should not start a view change to view=2 without a quorum of InstanceChanges,
that is it should not go to propagate primary mode since it's already in view chanage state.
The node should eventually finish view change to view=2 once receives all VCD and IS msgs for view=2
'''
nodes = txnPoolNodeSet
slow_node = nodes[-1]
fast_nodes = [n for n in nodes if n != slow_node]
slow_stasher = slow_node.nodeIbStasher
# 1. DO FIRST VIEW CHANGE
# delay VCD for the first ViewChange
with delay_rules(slow_stasher, vcd_delay()):
# Trigger view change
for n in nodes:
n.view_changer.on_master_degradation()
waitForViewChange(looper, nodes, expectedViewNo=1)
# make sure view change is finished on all nodes except the slow one
ensureElectionsDone(looper, fast_nodes, instances_list=range(3))
# drop all VCD to view=1
slow_stasher.drop_delayeds()
# 2. DO SECOND VIEW CHANGE
# delay Instance Changes and
# so that the slow node receives VCD for view=2 before
# a quorum of InstanceChanges for that view while still doing view change to view=1
with delay_rules(slow_stasher, icDelay()):
# Trigger view change
for n in nodes:
n.view_changer.on_master_degradation()
waitForViewChange(looper, fast_nodes, expectedViewNo=2)
# make sure view change is finished on all nodes except the slow one
ensureElectionsDone(looper, fast_nodes, instances_list=range(3))
# slow node is still on view=1
assert slow_node.viewNo == 1
assert slow_node.view_change_in_progress
# make sure that the slow node receives VCD msgs for view=2
# and didn't receive IS msgs for view=2
check_vcd_msgs(slow_node, expected_view_no=2, expected_count=len(fast_nodes), )
check_no_ic_msgs(slow_node, expected_view_no=2)
# 3. RESET DELAYS AND CHECK
waitForViewChange(looper, nodes, expectedViewNo=2)
ensureElectionsDone(looper, nodes)
assert not slow_node.view_change_in_progress
ensure_all_nodes_have_same_data(looper, nodes=nodes)
def test_view_change_done_delayed(txnPoolNodeSet, looper, sdk_pool_handle,
sdk_wallet_client):
"""
A node is slow so is behind other nodes, after view change, it catches up
but it also gets view change message as delayed, a node should start
participating only when caught up and ViewChangeCone quorum received.
"""
nprs = [r.node for r in getNonPrimaryReplicas(txnPoolNodeSet, 0)]
slow_node = nprs[-1]
other_nodes = [n for n in txnPoolNodeSet if n != slow_node]
delay_3pc = 10
delay_vcd = 25
delay_3pc_messages([slow_node], 0, delay_3pc)
slow_node.nodeIbStasher.delay(vcd_delay(delay_vcd))
def chk(node):
assert node.view_changer.has_acceptable_view_change_quorum
assert node.view_changer._primary_verified
assert node.isParticipating
assert None not in {r.isPrimary for r in node.replicas.values()}
sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client,
5 * 4, 4)
ensure_view_change(looper, nodes=txnPoolNodeSet)
# After view change, the slow node successfully completes catchup
waitNodeDataEquality(looper, slow_node, *other_nodes)
# Other nodes complete view change, select primary and participate
for node in other_nodes:
looper.run(eventually(chk, node, retryWait=1))
# Since `ViewChangeCone` is delayed, slow_node is not able to select primary
# and participate
assert not slow_node.view_changer.has_acceptable_view_change_quorum
assert not slow_node.view_changer._primary_verified
assert not slow_node.isParticipating
assert {r.isPrimary for r in slow_node.replicas.values()} == {None}
# Send requests to make sure pool is functional
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 5)
# Repair network
slow_node.reset_delays_and_process_delayeds()
# `slow_node` selects primary and participate
looper.run(eventually(chk, slow_node, retryWait=1))
# Processes requests received during lack of primary
waitNodeDataEquality(looper, slow_node, *other_nodes)
# Send more requests and compare data of all nodes
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 5)
ensure_all_nodes_have_same_data(looper, txnPoolNodeSet)
def test_delayed_instance_changes_after_vcd_for_next_view(looper, txnPoolNodeSet):
'''
A node is doing view change to view=1, while the other nodes already finished view change to view=2.
The node receives a quorum of VCD messages for view=2 before a quorum of InstanceChange messages for view=2.
Nevertheless, the node should not start a view change to view=2 without a quorum of InstanceChanges,
that is it should not go to propagate primary mode since it's already in view chanage state.
The node should eventually finish view change to view=2 once receives all VCD and IS msgs for view=2
'''
nodes = txnPoolNodeSet
slow_node = nodes[-1]
fast_nodes = [n for n in nodes if n != slow_node]
slow_stasher = slow_node.nodeIbStasher
# 1. DO FIRST VIEW CHANGE
# delay VCD for the first ViewChange
with delay_rules(slow_stasher, vcd_delay()):
# Trigger view change
for n in nodes:
n.view_changer.on_master_degradation()
waitForViewChange(looper, nodes, expectedViewNo=1)
# make sure view change is finished on all nodes except the slow one
ensureElectionsDone(looper, fast_nodes, instances_list=range(3))
# drop all VCD to view=1
slow_stasher.drop_delayeds()
# 2. DO SECOND VIEW CHANGE
# delay Instance Changes and
# so that the slow node receives VCD for view=2 before
# a quorum of InstanceChanges for that view while still doing view change to view=1
with delay_rules(slow_stasher, icDelay()):
# Trigger view change
for n in nodes:
n.view_changer.on_master_degradation()
waitForViewChange(looper, fast_nodes, expectedViewNo=2)
# make sure view change is finished on all nodes except the slow one
ensureElectionsDone(looper, fast_nodes, instances_list=range(3))
# slow node is still on view=1
assert slow_node.viewNo == 1
assert slow_node.view_change_in_progress
# make sure that the slow node receives VCD msgs for view=2
# and didn't receive IS msgs for view=2
check_vcd_msgs(slow_node, expected_view_no=2, expected_count=len(fast_nodes), )
check_no_ic_msgs(slow_node, expected_view_no=2)
# 3. RESET DELAYS AND CHECK
waitForViewChange(looper, nodes, expectedViewNo=2)
ensureElectionsDone(looper, nodes)
assert not slow_node.view_change_in_progress
ensure_all_nodes_have_same_data(looper, nodes=nodes)
def do_view_change_with_propagate_primary_on_one_delayed_node(
slow_node, nodes, looper, sdk_pool_handle, sdk_wallet_client):
slow_stasher = slow_node.nodeIbStasher
fast_nodes = [n for n in nodes if n != slow_node]
stashers = [n.nodeIbStasher for n in nodes]
# Get last prepared certificate in pool
lpc = last_prepared_certificate(nodes)
# Get pool current view no
view_no = lpc[0]
with delay_rules(slow_stasher, icDelay()):
with delay_rules(slow_stasher, vcd_delay()):
with delay_rules(stashers, cDelay()):
# Send request
request = sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client)
# Wait until this request is prepared on N-f nodes
looper.run(eventually(check_last_prepared_certificate_on_quorum, nodes, (lpc[0], lpc[1] + 1)))
# Trigger view change
for n in nodes:
n.view_changer.on_master_degradation()
# Wait until view change is completed on all nodes except slow one
waitForViewChange(looper,
fast_nodes,
expectedViewNo=view_no + 1,
customTimeout=waits.expectedPoolViewChangeStartedTimeout(len(nodes)))
wait_for_elections_done_on_given_nodes(looper,
fast_nodes,
getRequiredInstances(len(nodes)),
timeout=waits.expectedPoolElectionTimeout(len(nodes)))
# Now all the nodes receive Commits
# The slow node will accept Commits and order the 3PC-batch in the old view
looper.runFor(waits.expectedOrderingTime(getNoInstances(len(nodes))))
# Now slow node receives ViewChangeDones
waitForViewChange(looper,
[slow_node],
expectedViewNo=view_no + 1,
customTimeout=waits.expectedPoolViewChangeStartedTimeout(len(nodes)))
wait_for_elections_done_on_given_nodes(looper,
[slow_node],
getRequiredInstances(len(nodes)),
timeout=waits.expectedPoolElectionTimeout(len(nodes)))
# Now slow node receives InstanceChanges but discards them because already
# started propagate primary to the same view.
# Finish request gracefully
sdk_get_reply(looper, request)
def do_view_change_with_propagate_primary_on_one_delayed_node(
slow_node, nodes, looper, sdk_pool_handle, sdk_wallet_client):
slow_stasher = slow_node.nodeIbStasher
fast_nodes = [n for n in nodes if n != slow_node]
stashers = [n.nodeIbStasher for n in nodes]
# Get last prepared certificate in pool
lpc = last_prepared_certificate(nodes)
# Get pool current view no
view_no = lpc[0]
with delay_rules(slow_stasher, icDelay()):
with delay_rules(slow_stasher, vcd_delay()):
with delay_rules(stashers, cDelay()):
# Send request
request = sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client)
# Wait until this request is prepared on N-f nodes
looper.run(eventually(check_last_prepared_certificate_on_quorum, nodes, (lpc[0], lpc[1] + 1)))
# Trigger view change
for n in nodes:
n.view_changer.on_master_degradation()
# Wait until view change is completed on all nodes except slow one
waitForViewChange(looper,
fast_nodes,
expectedViewNo=view_no + 1,
customTimeout=waits.expectedPoolViewChangeStartedTimeout(len(nodes)))
wait_for_elections_done_on_given_nodes(looper,
fast_nodes,
getRequiredInstances(len(nodes)),
timeout=waits.expectedPoolElectionTimeout(len(nodes)))
# Now all the nodes receive Commits
# The slow node will accept Commits and order the 3PC-batch in the old view
looper.runFor(waits.expectedOrderingTime(getNoInstances(len(nodes))))
# Now slow node receives ViewChangeDones
waitForViewChange(looper,
[slow_node],
expectedViewNo=view_no + 1,
customTimeout=waits.expectedPoolViewChangeStartedTimeout(len(nodes)))
wait_for_elections_done_on_given_nodes(looper,
[slow_node],
getRequiredInstances(len(nodes)),
timeout=waits.expectedPoolElectionTimeout(len(nodes)))
# Now slow node receives InstanceChanges but discards them because already
# started propagate primary to the same view.
# Finish request gracefully
sdk_get_reply(looper, request)
def test_prepare_in_queue_before_vc(looper, txnPoolNodeSet, sdk_wallet_steward,
sdk_pool_handle):
"""
Test steps:
1. Sent N random requests.
2. Patching processNodeInBox method for node Delta.
This method will process only not Prepare messages and store in nodeInBox queue Prepare messages
3. Sent one request and check, that all Prepares are stored in nodeInBox queue and there is quorum of it
4. Compare last_ordered_3pc_key and last_prepared_certificate. Last_prepared_certificate must be greater then last ordered
5. ppSeqNo in last_prepared_certificate must be at least as ppSeqNo for queued Prepares msgs in nodeInBox queue
"""
def chk_quorumed_prepares_count(prepares, count):
pp_qourum = slow_node.quorums.prepare.value
assert len([
pp for key, pp in prepares.items()
if prepares.hasQuorum(pp.msg, pp_qourum)
]) == count
def patched_start_view_change(self, *args, **kwargs):
self.node.processNodeInBox = functools.partial(
TestNode.processNodeInBox, self.node)
ViewChanger.start_view_change(self, *args, **kwargs)
while stashed_msgs:
self.node.nodestack.rxMsgs.append(stashed_msgs.popleft())
"""Send REQ_COUNT txns"""
slow_node = txnPoolNodeSet[-1]
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, REQ_COUNT)
"""Check that there is REQ_COUNT prepares with quorum in queue"""
chk_quorumed_prepares_count(
slow_node.master_replica._ordering_service.prepares, REQ_COUNT)
"""Patch processNodeInBox method for saving Prepares in nodeInBox queue"""
not_processing_prepare(slow_node)
"""Send 1 txn"""
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, REQ_COUNT_AFTER_SLOW)
chk_quorumed_prepares_count(
slow_node.master_replica._ordering_service.prepares, REQ_COUNT)
"""Get last ordered 3pc key (should be (0, REQ_COUNT))"""
ordered_lpc = slow_node.master_replica.last_ordered_3pc
"""Delay view_change_done messages"""
slow_node.nodeIbStasher.delay(vcd_delay(100))
"""Patch on_view_change_start method for reverting processNodeInBox method"""
slow_node.view_changer.start_view_change = functools.partial(
patched_start_view_change, slow_node.view_changer)
"""Initiate view change"""
ensure_view_change(looper, txnPoolNodeSet)
"""Last prepared certificate should take into account Prepares in nodeInBox queue too"""
expected_lpc = slow_node.master_replica.last_prepared_before_view_change
assert expected_lpc == (0, 11)
"""Last ordered key should be less than last_prepared_before_view_change"""
assert compare_3PC_keys(ordered_lpc, expected_lpc) > 0
def test_view_change_done_delayed(txnPoolNodeSet, looper, sdk_pool_handle, sdk_wallet_client):
"""
A node is slow so is behind other nodes, after view change, it catches up
but it also gets view change message as delayed, a node should start
participating only when caught up and ViewChangeCone quorum received.
"""
nprs = [r.node for r in getNonPrimaryReplicas(txnPoolNodeSet, 0)]
slow_node = nprs[-1]
other_nodes = [n for n in txnPoolNodeSet if n != slow_node]
delay_3pc = 10
delay_vcd = 25
delay_3pc_messages([slow_node], 0, delay_3pc)
slow_node.nodeIbStasher.delay(vcd_delay(delay_vcd))
def chk(node):
assert node.view_changer.has_acceptable_view_change_quorum
assert node.view_changer._primary_verified
assert node.isParticipating
assert None not in {r.isPrimary for r in node.replicas.values()}
sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 5 * 4, 4)
ensure_view_change(looper, nodes=txnPoolNodeSet)
# After view change, the slow node successfully completes catchup
waitNodeDataEquality(looper, slow_node, *other_nodes)
# Other nodes complete view change, select primary and participate
for node in other_nodes:
looper.run(eventually(chk, node, retryWait=1))
# Since `ViewChangeCone` is delayed, slow_node is not able to select primary
# and participate
assert not slow_node.view_changer.has_acceptable_view_change_quorum
assert not slow_node.view_changer._primary_verified
assert not slow_node.isParticipating
assert {r.isPrimary for r in slow_node.replicas.values()} == {None}
# Send requests to make sure pool is functional
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 5)
# Repair network
slow_node.reset_delays_and_process_delayeds()
# `slow_node` selects primary and participate
looper.run(eventually(chk, slow_node, retryWait=1))
# Processes requests received during lack of primary
waitNodeDataEquality(looper, slow_node, *other_nodes)
# Send more requests and compare data of all nodes
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 5)
ensure_all_nodes_have_same_data(looper, txnPoolNodeSet)
def test_view_change_by_order_stashed_on_3_nodes_and_catchup_on_1_node(txnPoolNodeSet, looper,
sdk_pool_handle, sdk_wallet_steward):
'''
- COMMITS are delayed on all nodes
- All nodes starts a view change with a prepared certificate (for delayed message)
- COMMITS come during view change for 3 nodes
- So these 3 nodes finish view change by processing Commits and Ordered msgs during view change (in between rounds of catchup).
- The lagging (4th) node receives missing txns as part of catch-up (during view change) and also finishes view change.
'''
slow_node = txnPoolNodeSet[-1]
fast_nodes = txnPoolNodeSet[:-1]
slow_stasher = slow_node.nodeIbStasher
fast_stashers = [n.nodeIbStasher for n in fast_nodes]
all_stashers = [n.nodeIbStasher for n in txnPoolNodeSet]
initial_last_ordered = txnPoolNodeSet[0].master_replica.last_ordered_3pc
txns_count = 4
eventual_last_ordered = initial_last_ordered[0], initial_last_ordered[1] + txns_count
batches_count = initial_last_ordered[1]
with delay_rules(all_stashers, vcd_delay()):
# the lagging node is slow in receiving Commits and Catchup mghs
with delay_rules(slow_stasher, cDelay()):
with delay_rules(slow_stasher, lsDelay(), msg_rep_delay(types_to_delay=[LEDGER_STATUS])):
# fast nodes will receive and order Commits for last_prepared_cert during view change
with delay_rules(fast_stashers, cDelay()):
with delay_rules(fast_stashers, lsDelay(), msg_rep_delay(types_to_delay=[LEDGER_STATUS])):
sdk_send_random_requests(looper, sdk_pool_handle,
sdk_wallet_steward, txns_count)
batches_count += txns_count
looper.run(eventually(check_prepare_certificate, txnPoolNodeSet, batches_count))
check_last_ordered_3pc_on_master(txnPoolNodeSet, initial_last_ordered)
# trigger view change on all nodes
ensure_view_change(looper, txnPoolNodeSet)
looper.run(eventually(check_last_prepared_certificate_after_view_change_start,
txnPoolNodeSet, eventual_last_ordered))
# check that all txns are ordered till last prepared on fast nodes
looper.run(eventually(check_last_ordered_3pc_on_master, fast_nodes, eventual_last_ordered, timeout=30))
# check that all txns are ordered till last prepared on slow node as a result of catchup
looper.run(eventually(check_last_ordered_3pc_on_master, [slow_node], eventual_last_ordered, timeout=30))
# wait for view change done on all nodes
ensureElectionsDone(looper, txnPoolNodeSet)
ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet)
# make sure that the pool is functional
sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_steward, sdk_pool_handle)
def test_view_change_by_order_stashed_on_all(txnPoolNodeSet, looper,
sdk_pool_handle,
sdk_wallet_steward):
'''
- COMMITS are delayed on all nodes
- All nodes starts a view change with a prepared certificate (for delayed message)
- COMMITS come during view change for all nodes,
- So all nodes finish view change by processing Commits and Ordered msgs during view change (in between rounds of catchup).
'''
all_stashers = [n.nodeIbStasher for n in txnPoolNodeSet]
initial_last_ordered = txnPoolNodeSet[0].master_replica.last_ordered_3pc
txns_count = 4
eventual_last_ordered = initial_last_ordered[
0], initial_last_ordered[1] + txns_count
with delay_rules(all_stashers, vcd_delay()):
with delay_rules(all_stashers, cDelay()):
with delay_rules(all_stashers, lsDelay(),
msg_rep_delay(types_to_delay=[LEDGER_STATUS])):
sdk_send_random_requests(looper, sdk_pool_handle,
sdk_wallet_steward, txns_count)
looper.run(
eventually(check_prepare_certificate, txnPoolNodeSet,
txns_count))
check_last_ordered_3pc_on_master(txnPoolNodeSet,
initial_last_ordered)
# trigger view change on all nodes
ensure_view_change(looper, txnPoolNodeSet)
looper.run(
eventually(
check_last_prepared_certificate_after_view_change_start,
txnPoolNodeSet, eventual_last_ordered))
# check that all txns are ordered till last prepared
looper.run(
eventually(check_last_ordered_3pc_on_master,
txnPoolNodeSet,
eventual_last_ordered,
timeout=30))
# wait for view change done on all nodes
ensureElectionsDone(looper, txnPoolNodeSet)
ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet)
# make sure that the pool is functional
sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_steward,
sdk_pool_handle)
def test_view_change_timeout_reset_on_next_view(txnPoolNodeSet, looper, tconf):
# Check that all nodes are in view 0
assert all(n.viewNo == 0 for n in txnPoolNodeSet)
stashers = [n.nodeIbStasher for n in txnPoolNodeSet]
with delay_rules(stashers, vcd_delay()):
# Start first view change
for n in txnPoolNodeSet:
n.view_changer.on_master_degradation()
waitForViewChange(looper, txnPoolNodeSet, expectedViewNo=1)
looper.runFor(0.6 * VIEW_CHANGE_TIMEOUT)
# Start second view change
for n in txnPoolNodeSet:
n.view_changer.on_master_degradation()
waitForViewChange(looper, txnPoolNodeSet, expectedViewNo=2)
looper.runFor(0.6 * VIEW_CHANGE_TIMEOUT)
# Ensure only 2 view changes happened
ensureElectionsDone(looper, txnPoolNodeSet)
for n in txnPoolNodeSet:
assert n.viewNo == 2
def test_view_change_timeout_reset_on_next_view(txnPoolNodeSet, looper, tconf):
# Check that all nodes are in view 0
assert all(n.viewNo == 0 for n in txnPoolNodeSet)
stashers = [n.nodeIbStasher for n in txnPoolNodeSet]
with delay_rules(stashers, vcd_delay()):
# Start first view change
for n in txnPoolNodeSet:
n.view_changer.on_master_degradation()
waitForViewChange(looper, txnPoolNodeSet, expectedViewNo=1)
looper.runFor(0.6 * NEW_VIEW_TIMEOUT)
# Start second view change
for n in txnPoolNodeSet:
n.view_changer.on_master_degradation()
waitForViewChange(looper, txnPoolNodeSet, expectedViewNo=2)
looper.runFor(0.6 * NEW_VIEW_TIMEOUT)
# Ensure only 2 view changes happened
ensureElectionsDone(looper, txnPoolNodeSet)
for n in txnPoolNodeSet:
assert n.viewNo == 2
def test_add_vc_start_msg_during_start_view_change(txnPoolNodeSet,
looper):
delayed_node = txnPoolNodeSet[-1]
current_view_no = delayed_node.viewNo
proposed_view_no = current_view_no + 1
delayed_node.nodeIbStasher.delay(vcd_delay(1000))
delayed_node.nodeIbStasher.delay(icDelay(1000))
assert delayed_node.view_changer
ensure_view_change(looper, txnPoolNodeSet[:-1])
"""
If view number was incremented, that means that instanceChange quorum was archived
"""
assert txnPoolNodeSet[0].viewNo == proposed_view_no
looper.removeProdable(delayed_node)
delayed_node.nodeInBox.append((InstanceChange(1, 25), 'Alpha'))
delayed_node.nodeInBox.append((InstanceChange(1, 25), 'Beta'))
delayed_node.nodeInBox.append((InstanceChange(1, 25), 'Gamma'))
delayed_node.processNodeInBox = functools.partial(Node.processNodeInBox, delayed_node)
looper.run(delayed_node.processNodeInBox())
looper.run(delayed_node.serviceViewChanger(None))
assert len(delayed_node.nodeInBox) == 1
m = delayed_node.nodeInBox.popleft()
assert isinstance(m[0], ViewChangeStartMessage)
def test_process_three_phase_msg_and_stashed_future_view(
txnPoolNodeSet, looper, tconf, sdk_pool_handle, sdk_wallet_steward):
"""
1. Delay ViewChangeDone messages for the slow_node.
2. Start view change on all nodes.
3. Order a new request.
4. Check that slow_node could not order this request and stashed all 3pc messages
and other nodes ordered.
6. Reset delays.
7. Check that the last request is ordered on the slow_node and stashed messages were removed.
"""
slow_node = txnPoolNodeSet[-1]
fast_nodes = txnPoolNodeSet[:-1]
view_no = slow_node.viewNo
old_stashed = {
inst_id: r.stasher.num_stashed_future_view
for inst_id, r in slow_node.replicas.items()
}
last_ordered = {
inst_id: r.last_ordered_3pc
for inst_id, r in slow_node.replicas.items()
}
with delay_rules([
slow_node.nodeIbStasher,
], msg_rep_delay(types_to_delay=[PREPREPARE, PREPARE, COMMIT])):
with delay_rules([
slow_node.nodeIbStasher,
], vcd_delay()):
for n in txnPoolNodeSet:
n.view_changer.on_master_degradation()
waitForViewChange(looper,
fast_nodes,
expectedViewNo=view_no + 1,
customTimeout=2 * tconf.VIEW_CHANGE_TIMEOUT)
ensureElectionsDone(looper=looper,
nodes=fast_nodes,
instances_list=range(
fast_nodes[0].requiredNumberOfInstances))
looper.run(
eventually(lambda: assertExp(slow_node.mode == Mode.synced)))
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, 1)
assert slow_node.view_change_in_progress
# 1 - pre-prepare msg
# (len(txnPoolNodeSet) - 2) - prepare msgs
# (len(txnPoolNodeSet) - 1) - commit msgs
stashed_master_messages = 2 * (1 + (len(txnPoolNodeSet) - 2) +
(len(txnPoolNodeSet) - 1))
stashed_backup_messages = 2 * (1 + (len(txnPoolNodeSet) - 2) +
(len(txnPoolNodeSet) - 1))
assert slow_node.master_replica.stasher.num_stashed_future_view == old_stashed[
0] + stashed_master_messages
assert all(
r.stasher.num_stashed_future_view == old_stashed[inst_id] +
stashed_backup_messages
for inst_id, r in slow_node.replicas.items() if inst_id != 0)
assert all(r.last_ordered_3pc == last_ordered[inst_id]
for inst_id, r in slow_node.replicas.items())
def chk():
for inst_id, r in slow_node.replicas.items():
if inst_id == 0:
assert r.last_ordered_3pc == (view_no + 1, 2)
else:
assert r.last_ordered_3pc == (view_no + 1, 2)
assert r.stasher.num_stashed_future_view == old_stashed[
inst_id]
looper.run(eventually(chk))
waitNodeDataEquality(looper, slow_node, *fast_nodes)
def test_checkpoints_after_view_change(tconf, looper, chkFreqPatched,
reqs_for_checkpoint, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client):
'''
Tests that there is no infinite catchups if there is
a quorum of stashed checkpoints received during the view change
'''
# Prepare nodes
lagging_node = txnPoolNodeSet[-1]
rest_nodes = txnPoolNodeSet[:-1]
initial_all_ledgers_caught_up = lagging_node.spylog.count(
Node.allLedgersCaughtUp)
initial_start_catchup = lagging_node.spylog.count(Node.start_catchup)
with delay_rules(lagging_node.nodeIbStasher, lsDelay()):
with delay_rules(lagging_node.nodeIbStasher, vcd_delay()):
ensure_view_change(looper, txnPoolNodeSet)
looper.run(
eventually(lambda: assertExp(lagging_node.
view_change_in_progress is True),
timeout=waits.expectedPoolCatchupTime(
len(txnPoolNodeSet))))
ensureElectionsDone(looper=looper,
nodes=rest_nodes,
instances_list=range(2))
assert all(n.view_change_in_progress is False for n in rest_nodes)
assert lagging_node.view_change_in_progress is True
# make sure that more requests are being ordered while catch-up is in progress on the lagging node
# stash enough stable checkpoints for starting a catch-up
num_checkpoints = Replica.STASHED_CHECKPOINTS_BEFORE_CATCHUP + 1
num_reqs = reqs_for_checkpoint * num_checkpoints + 1
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, num_reqs)
looper.run(
eventually(check_last_ordered_3pc_on_master, rest_nodes,
(1, num_reqs + 1)))
looper.run(
eventually(check_last_ordered_3pc_on_backup, rest_nodes,
(1, num_reqs + 1)))
# all good nodes stabilized checkpoint
looper.run(eventually(chkChkpoints, rest_nodes, 2, 0))
assert get_stashed_checkpoints(
lagging_node) == num_checkpoints * len(rest_nodes)
# lagging node is doing the view change and stashing all checkpoints
assert lagging_node.view_change_in_progress is True
looper.run(
eventually(lambda: assertExp(
get_stashed_checkpoints(lagging_node) == 2 * len(rest_nodes
)),
timeout=waits.expectedPoolCatchupTime(
len(txnPoolNodeSet))))
# check that view change is finished
ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet)
assert lagging_node.view_change_in_progress is False
# check that last_ordered is set
looper.run(
eventually(check_last_ordered_3pc_on_master, [lagging_node],
(1, num_reqs + 1)))
looper.run(
eventually(check_last_ordered_3pc_on_backup, [lagging_node],
(1, num_reqs + 1)))
# check that checkpoint is stabilized for master
looper.run(eventually(chk_chkpoints_for_instance, [lagging_node], 0, 2, 0))
# check that the catch-up is finished
assert lagging_node.mode == Mode.participating
assert lagging_node.spylog.count(
Node.allLedgersCaughtUp) == initial_all_ledgers_caught_up + 1
assert lagging_node.spylog.count(
Node.start_catchup) == initial_start_catchup + 1
waitNodeDataEquality(looper, *txnPoolNodeSet, customTimeout=5)
def test_unstash_three_phase_msg_after_catchup_in_view_change(
txnPoolNodeSet, looper, tconf, sdk_pool_handle, sdk_wallet_steward):
"""
1. Delay Commit on Node4
2. Order 1 req
3. Delay Commit on all nodes
4. Order 1 req
5. Delay CatchupRep on Node4
6. Delay Ledger Status and ViewChangeDones on Nodes1-3
7. Start View change on all nodes
8. Wait until Node4 got 3 stashed CatchupReps
9. Reset delaying of Commits on all Nodes
10. Reset Ledger Status on Nodes1-3
11. Check that 3 nodes finished VC while Node4 is syncing and not finished
12. Reset CatchupRep on Node4
13. Check that Node4 finished VC, and there was just 1 round of cacth-up (edited)
"""
slow_node = txnPoolNodeSet[-1]
fast_nodes = txnPoolNodeSet[:-1]
view_no = txnPoolNodeSet[0].viewNo
old_stashed = slow_node.master_replica.stasher.num_stashed_future_view
last_ordered = txnPoolNodeSet[0].master_replica.last_ordered_3pc
with delay_rules(
[n.nodeIbStasher for n in txnPoolNodeSet],
msg_rep_delay(types_to_delay=[PREPREPARE, PREPARE, COMMIT])):
# Delay Commit messages for slow_node.
slow_node.nodeIbStasher.delay(cDelay(sys.maxsize))
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, 1)
# Delay Commit messages for fast_nodes.
for n in fast_nodes:
n.nodeIbStasher.delay(cDelay(sys.maxsize))
request2 = sdk_send_random_request(looper, sdk_pool_handle,
sdk_wallet_steward)
def check_commits(commit_key):
for n in fast_nodes:
for r in n.replicas.values():
assert commit_key in r.commits
assert len(r.commits[commit_key].voters) == 1
looper.run(eventually(check_commits, (view_no, last_ordered[1] + 2)))
# Delay CatchupRep messages for the slow_node.
with delay_rules([slow_node.nodeIbStasher], cr_delay()):
with delay_rules([n.nodeIbStasher for n in fast_nodes],
vcd_delay()):
with delay_rules(
[n.nodeIbStasher for n in fast_nodes],
msg_rep_delay(types_to_delay=[LEDGER_STATUS])):
for n in txnPoolNodeSet:
n.view_changer.on_master_degradation()
looper.run(
eventually(
lambda: assertExp(slow_node.mode == Mode.syncing)))
# Reset delay Commit messages for all nodes.
for n in txnPoolNodeSet:
n.nodeIbStasher.reset_delays_and_process_delayeds(
COMMIT)
assert slow_node.view_change_in_progress
assert slow_node.mode == Mode.syncing
looper.run(
eventually(_check_nodes_stashed, fast_nodes,
old_stashed,
len(txnPoolNodeSet) - 1))
looper.run(
eventually(_check_nodes_stashed, [slow_node],
old_stashed, (len(txnPoolNodeSet) - 1) * 2))
waitForViewChange(looper,
fast_nodes,
expectedViewNo=view_no + 1,
customTimeout=2 * tconf.VIEW_CHANGE_TIMEOUT)
ensureElectionsDone(looper=looper,
nodes=fast_nodes,
instances_list=range(
fast_nodes[0].requiredNumberOfInstances),
customTimeout=2 * tconf.VIEW_CHANGE_TIMEOUT)
sdk_get_and_check_replies(looper, [request2])
waitForViewChange(looper, [slow_node],
expectedViewNo=view_no + 1,
customTimeout=2 * tconf.VIEW_CHANGE_TIMEOUT)
ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet)
_check_nodes_stashed(fast_nodes, old_stashed, 0)
assert all(n.master_replica.last_ordered_3pc == (last_ordered[0],
last_ordered[1] + 2)
for n in txnPoolNodeSet)
assert slow_node.catchup_rounds_without_txns == 1
def delay_view_change_msg(nodes):
for node in nodes:
node.nodeIbStasher.delay(vcd_delay(delay=50))
def test_view_change_after_max_catchup_rounds(txnPoolNodeSet, looper,
sdk_pool_handle,
sdk_wallet_client,
limitTestRunningTime):
"""
The node should do only a fixed rounds of catchup. For this delay Prepares
and Commits for 2 non-primary nodes by a large amount which is equivalent
to loss of Prepares and Commits. Make sure 2 nodes have a different last
prepared certificate from other two. Then do a view change, make sure view
change completes and the pool does not process the request that were
prepared by only a subset of the nodes
"""
batches_count = 3
sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client,
2 * batches_count, batches_count)
ensure_all_nodes_have_same_data(looper, txnPoolNodeSet)
ledger_summary = txnPoolNodeSet[0].ledger_summary
slow_nodes = [
r.node for r in getNonPrimaryReplicas(txnPoolNodeSet, 0)[-2:]
]
fast_nodes = [n for n in txnPoolNodeSet if n not in slow_nodes]
# Make node slow to process Prepares and Commits
for node in slow_nodes:
node.nodeIbStasher.delay(pDelay(120, 0))
node.nodeIbStasher.delay(cDelay(120, 0))
sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 5)
looper.runFor(3)
def last_prepared(nodes):
lst = [
n.master_replica._ordering_service.
l_last_prepared_certificate_in_view() for n in nodes
]
# All nodes have same last prepared
assert check_if_all_equal_in_list(lst)
return lst[0]
with delay_rules([n.nodeIbStasher for n in txnPoolNodeSet], vcd_delay()):
ensure_view_change(looper, nodes=txnPoolNodeSet)
last_prepared_slow = last_prepared(slow_nodes)
last_prepared_fast = last_prepared(fast_nodes)
# Check `slow_nodes` and `fast_nodes` set different last_prepared
assert last_prepared_fast != last_prepared_slow
for node in txnPoolNodeSet:
node.reset_delays_and_process_delayeds(vcd_delay)
# View change complete
ensureElectionsDone(looper, txnPoolNodeSet)
ensure_all_nodes_have_same_data(looper, txnPoolNodeSet)
# The requests which were prepared by only a subset of the nodes were
# not ordered
assert txnPoolNodeSet[0].ledger_summary == ledger_summary
for node in slow_nodes:
node.nodeIbStasher.reset_delays_and_process_delayeds()
# Make sure pool is functional
sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client,
10, 2)
ensure_all_nodes_have_same_data(looper, txnPoolNodeSet)
last_prepared(txnPoolNodeSet)
def test_catchup_to_next_view_during_view_change_0_to_1_then_1_to_2(
txnPoolNodeSet, looper, sdk_pool_handle, sdk_wallet_steward):
'''
1) Lagging node is not a primary for new views
2) All nodes except the lagging one go to view=1
3) All nodes except the lagging one order txns on view=1
4) All nodes except the lagging one go to view=2
5) All nodes except the lagging one order txns on view=2
6) Lagging node gets InstanceChanges for view=1 => it changes to view=1, and catches up till txns from view=2
7) Lagging node gets InstanceChanges for view=2 => it changes to view=2
8) Make sure that the lagging node is up to date, and canc participate in consensus
'''
lagging_node = txnPoolNodeSet[0]
other_nodes = txnPoolNodeSet[1:]
initial_view_no = checkViewNoForNodes(txnPoolNodeSet)
initial_last_ordered = lagging_node.master_last_ordered_3PC
with delay_rules(lagging_node.nodeIbStasher, icDelay(viewNo=2),
vcd_delay(viewNo=2), vc_delay(view_no=2)):
with delay_rules(lagging_node.nodeIbStasher, delay_for_view(viewNo=0),
delay_for_view(viewNo=1)):
# view change to viewNo=1
for n in txnPoolNodeSet:
n.view_changer.on_master_degradation()
waitForViewChange(looper,
other_nodes,
expectedViewNo=initial_view_no + 1)
checkProtocolInstanceSetup(looper=looper,
nodes=other_nodes,
instances=range(3))
ensure_all_nodes_have_same_data(looper, nodes=other_nodes)
# order some txns
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, 5)
# view change to viewNo=2
for n in txnPoolNodeSet:
n.view_changer.on_master_degradation()
waitForViewChange(looper,
other_nodes,
expectedViewNo=initial_view_no + 2)
checkProtocolInstanceSetup(looper=looper,
nodes=other_nodes,
instances=range(3))
ensure_all_nodes_have_same_data(looper, nodes=other_nodes)
# order some txns
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, 5)
assert initial_view_no == lagging_node.viewNo
assert initial_last_ordered == lagging_node.master_last_ordered_3PC
# make sure that the first View Change happened on the lagging node
waitForViewChange(looper, [lagging_node],
expectedViewNo=initial_view_no + 1,
customTimeout=20)
assert initial_view_no + 1 == lagging_node.viewNo
# make sure that the second View Change happened on the lagging node
waitForViewChange(looper, [lagging_node],
expectedViewNo=initial_view_no + 2,
customTimeout=20)
ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet)
ensure_all_nodes_have_same_data(looper, nodes=other_nodes)
# make sure that the pool is functional
sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_steward,
sdk_pool_handle)
