def do_view_change_with_unaligned_prepare_certificates(
slow_nodes, nodes, looper, sdk_pool_handle, sdk_wallet_client):
"""
Perform view change with some nodes reaching lower last prepared certificate than others.
With current implementation of view change this can result with view change taking a lot of time.
"""
fast_nodes = [n for n in nodes if n not in slow_nodes]
all_stashers = [n.nodeIbStasher for n in nodes]
slow_stashers = [n.nodeIbStasher for n in slow_nodes]
# Delay some PREPAREs and all COMMITs
with delay_rules(slow_stashers, pDelay()):
with delay_rules(all_stashers, cDelay()):
# Send request
request = sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client)
# Wait until this request is prepared on fast nodes
looper.run(eventually(check_last_prepared_certificate, fast_nodes, (0, 1)))
# Make sure its not prepared on slow nodes
looper.run(eventually(check_last_prepared_certificate, slow_nodes, None))
# Trigger view change
for n in nodes:
n.view_changer.on_master_degradation()
# Now commits are processed
# Wait until view change is complete
looper.run(eventually(check_view_change_done, nodes, 1, timeout=60))
# Finish request gracefully
sdk_get_reply(looper, request)
def test_check_cdp_pp_storages(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward):
def check_all_empty(replica, reverse=False):
check_preprepared_empty(replica, reverse)
check_prepared_empty(replica, reverse)
def check_preprepared_empty(replica, reverse=False):
statement_pp = bool(replica._consensus_data.preprepared)
statement_pp ^= reverse
assert not statement_pp
def check_prepared_empty(replica, reverse=False):
statement_p = bool(replica._consensus_data.prepared)
statement_p ^= reverse
assert not statement_p
def operation_for_replicas(operation, node_set=txnPoolNodeSet, reverse=False):
for node in node_set:
operation(node.master_replica, reverse)
node_stashers = [n.nodeIbStasher for n in txnPoolNodeSet]
with delay_rules(node_stashers, cDelay()):
with delay_rules(node_stashers, pDelay()):
with delay_rules(node_stashers, ppDelay()):
sdk_add_new_nym_without_waiting(looper, sdk_pool_handle, sdk_wallet_steward)
looper.run(eventually(operation_for_replicas, check_all_empty, txnPoolNodeSet[1:]))
looper.run(eventually(operation_for_replicas, check_preprepared_empty, txnPoolNodeSet[0:1], True))
looper.run(eventually(operation_for_replicas, check_preprepared_empty, txnPoolNodeSet, True))
looper.run(eventually(operation_for_replicas, check_prepared_empty, txnPoolNodeSet, True))
looper.run(eventually(operation_for_replicas, check_all_empty, txnPoolNodeSet, True))
def test_propagate_of_ordered_request_doesnt_stash_requests_in_authenticator(
looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client):
# Universal delayer
def stopAll(msg):
return 100000
def check_verified_req_list_is_empty():
for node in txnPoolNodeSet:
assert len(node.clientAuthNr._verified_reqs) == 0
# Order one request while cutting off last node
lastNode = txnPoolNodeSet[-1]
with delay_rules(lastNode.nodeIbStasher, stopAll), \
delay_rules(lastNode.clientIbStasher, stopAll):
sdk_send_random_and_check(looper, txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client, 1)
old_propagates = [n.spylog.count('processPropagate') for n in txnPoolNodeSet]
def check_more_propagates_delivered():
new_propagates = [n.spylog.count('processPropagate') for n in txnPoolNodeSet]
assert all(old < new for old, new in zip(old_propagates, new_propagates))
# Wait until more propagates are delivered to all nodes
looper.run(eventually(check_more_propagates_delivered))
# Make sure that verified req list will be empty eventually
looper.run(eventually(check_verified_req_list_is_empty))
def test_delayed_instance_changes_after_vcd_for_next_view(
looper, txnPoolNodeSet):
'''
A node is doing view change to view=1, while the other nodes already finished view change to view=2.
The node receives a quorum of VCD messages for view=2 before a quorum of InstanceChange messages for view=2.
Nevertheless, the node should not start a view change to view=2 without a quorum of InstanceChanges,
that is it should not go to propagate primary mode since it's already in view chanage state.
The node should eventually finish view change to view=2 once receives all VCD and IS msgs for view=2
'''
nodes = txnPoolNodeSet
slow_node = nodes[-1]
fast_nodes = [n for n in nodes if n != slow_node]
slow_stasher = slow_node.nodeIbStasher
# 1. DO FIRST VIEW CHANGE
# delay VCD for the first ViewChange
with delay_rules(slow_stasher, nv_delay()):
# Trigger view change
trigger_view_change(nodes)
waitForViewChange(looper, nodes, expectedViewNo=1)
# make sure view change is finished on all nodes except the slow one
ensureElectionsDone(looper,
fast_nodes,
instances_list=range(3),
customTimeout=30)
# drop all VCD to view=1
slow_stasher.drop_delayeds()
# 2. DO SECOND VIEW CHANGE
# delay Instance Changes and
# so that the slow node receives VCD for view=2 before
# a quorum of InstanceChanges for that view while still doing view change to view=1
with delay_rules(slow_stasher, icDelay()):
# Trigger view change
trigger_view_change(nodes)
waitForViewChange(looper, fast_nodes, expectedViewNo=2)
# make sure view change is finished on all nodes except the slow one
ensureElectionsDone(looper, fast_nodes, instances_list=range(3))
# slow node is still on view=1
assert slow_node.viewNo == 1
assert slow_node.view_change_in_progress
# make sure that the slow node receives VCD msgs for view=2
# and didn't receive IS msgs for view=2
# check_vcd_msgs(slow_node, expected_view_no=2, expected_count=len(fast_nodes), )
check_no_ic_msgs(slow_node, 2, fast_nodes)
# 3. RESET DELAYS AND CHECK
waitForViewChange(looper, nodes, expectedViewNo=2)
ensureElectionsDone(looper, nodes)
assert not slow_node.view_change_in_progress
ensure_all_nodes_have_same_data(looper, nodes=nodes)
def test_propagate_of_ordered_request_doesnt_stash_requests_in_authenticator(
looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client):
# Universal delayer
def stopAll(msg):
return 100000
def check_verified_req_list_is_empty():
for node in txnPoolNodeSet:
assert len(node.clientAuthNr._verified_reqs) == 0
# Order one request while cutting off last node
lastNode = txnPoolNodeSet[-1]
with delay_rules(lastNode.nodeIbStasher, stopAll), \
delay_rules(lastNode.clientIbStasher, stopAll):
sdk_send_random_and_check(looper, txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client, 1)
old_propagates = [n.spylog.count('processPropagate') for n in txnPoolNodeSet]
def check_more_propagates_delivered():
new_propagates = [n.spylog.count('processPropagate') for n in txnPoolNodeSet]
assert all(old < new for old, new in zip(old_propagates, new_propagates))
# Wait until more propagates are delivered to all nodes
looper.run(eventually(check_more_propagates_delivered))
# Make sure that verified req list will be empty eventually
looper.run(eventually(check_verified_req_list_is_empty))
def test_freeing_forwarded_not_preprepared_request(
looper, chkFreqPatched, reqs_for_checkpoint, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_steward, tconf, tdir, allPluginsPath):
behind_node = txnPoolNodeSet[-1]
behind_node.requests.clear()
sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_steward,
CHK_FREQ, CHK_FREQ)
with delay_rules(
behind_node.nodeIbStasher,
chk_delay(delay=sys.maxsize,
instId=behind_node.replicas.values()[-1])):
with delay_rules(behind_node.nodeIbStasher, ppDelay(delay=sys.maxsize),
pDelay(delay=sys.maxsize), cDelay(delay=sys.maxsize)):
count = behind_node.spylog.count(behind_node.allLedgersCaughtUp)
sdk_send_batches_of_random(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, req_num, req_num)
looper.run(
eventually(node_caughtup, behind_node, count, retryWait=1))
looper.run(
eventually(
lambda: assertExp(len(behind_node.requests) == req_num)))
# We execute caughtup requests
looper.run(
eventually(lambda: assertExp(len(behind_node.requests) == req_num)))
assert all(r.executed for r in behind_node.requests.values()
if behind_node.seqNoDB.get(r.request.key)[1])
def do_view_change_with_unaligned_prepare_certificates(
slow_nodes, nodes, looper, sdk_pool_handle, sdk_wallet_client):
"""
Perform view change with some nodes reaching lower last prepared certificate than others.
With current implementation of view change this can result with view change taking a lot of time.
"""
fast_nodes = [n for n in nodes if n not in slow_nodes]
all_stashers = [n.nodeIbStasher for n in nodes]
slow_stashers = [n.nodeIbStasher for n in slow_nodes]
# Delay some PREPAREs and all COMMITs
with delay_rules(slow_stashers, pDelay()):
with delay_rules(all_stashers, cDelay()):
# Send request
request = sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client)
# Wait until this request is prepared on fast nodes
looper.run(eventually(check_last_prepared_certificate, fast_nodes, (0, 1)))
# Make sure its not prepared on slow nodes
looper.run(eventually(check_last_prepared_certificate, slow_nodes, None))
# Trigger view change
for n in nodes:
n.view_changer.on_master_degradation()
# Now commits are processed
# Wait until view change is complete
looper.run(eventually(check_view_change_done, nodes, 1, timeout=60))
# Finish request gracefully
sdk_get_reply(looper, request)
def test_view_change_with_different_prepare_certificate(looper, txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client):
"""
Check that a node without pre-prepare but with quorum of prepares wouldn't
use this transaction as a last in prepare certificate
"""
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
slow_node = txnPoolNodeSet[-1]
# delay preprepares and message response with preprepares.
with delay_rules(slow_node.nodeIbStasher, ppDelay(delay=sys.maxsize)):
with delay_rules(slow_node.nodeIbStasher,
msg_rep_delay(delay=sys.maxsize,
types_to_delay=[PREPREPARE, ])):
last_ordered = slow_node.master_replica.last_ordered_3pc
sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client)
looper.run(eventually(check_prepare_certificate,
txnPoolNodeSet[0:-1],
last_ordered[1] + 1))
for n in txnPoolNodeSet:
n.view_changer.on_master_degradation()
assert slow_node.master_replica.last_prepared_certificate_in_view() == \
(0, last_ordered[1])
ensureElectionsDone(looper, txnPoolNodeSet)
def test_no_propagate_request_on_different_prepares_on_backup_before_vc(looper, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client):
'''
1. Send random request
2. Make 3 node on backup instance slow in getting prepares
3. Send random request
4. do view change
5. reset delays
=> we expect that all nodes and all instances have the same last ordered
'''
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
slow_instance = 1
slow_nodes = txnPoolNodeSet[1:3]
fast_nodes = [n for n in txnPoolNodeSet if n not in slow_nodes]
nodes_stashers = [n.nodeIbStasher for n in slow_nodes]
old_last_ordered = txnPoolNodeSet[0].master_replica.last_ordered_3pc
with delay_rules(nodes_stashers, pDelay(instId=slow_instance)):
with delay_rules(nodes_stashers, ppDelay(instId=slow_instance)):
# send one request
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
old_view_no = txnPoolNodeSet[0].viewNo
looper.run(
eventually(is_prepared,
fast_nodes,
2,
slow_instance))
# trigger view change on all nodes
ensure_view_change(looper, txnPoolNodeSet)
# wait for view change done on all nodes
ensureElectionsDone(looper, txnPoolNodeSet)
primary = getPrimaryReplica(txnPoolNodeSet, slow_instance).node
non_primaries = [n for n in txnPoolNodeSet if n is not primary]
check_last_ordered(non_primaries,
slow_instance,
(old_view_no, old_last_ordered[1] + 1))
# Backup primary replica must not advance last_ordered_3pc
# up to the master's value
check_last_ordered([primary],
slow_instance,
(old_view_no, old_last_ordered[1]))
check_last_ordered(txnPoolNodeSet,
txnPoolNodeSet[0].master_replica.instId,
(old_last_ordered[0], old_last_ordered[1] + 1))
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
looper.run(
eventually(check_last_ordered,
txnPoolNodeSet,
slow_instance,
(txnPoolNodeSet[0].viewNo, 1)))
assert all(0 == node.spylog.count(node.request_propagates)
for node in txnPoolNodeSet)
def test_view_change_with_different_prepare_certificate(
looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client):
"""
Check that a node without pre-prepare but with quorum of prepares wouldn't
use this transaction as a last in prepare certificate
"""
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
slow_node = txnPoolNodeSet[-1]
# delay preprepares and message response with preprepares.
with delay_rules(slow_node.nodeIbStasher, ppDelay(delay=sys.maxsize)):
with delay_rules(
slow_node.nodeIbStasher,
msg_rep_delay(delay=sys.maxsize, types_to_delay=[
PREPREPARE,
])):
last_ordered = slow_node.master_replica.last_ordered_3pc
sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client)
looper.run(
eventually(check_prepare_certificate, txnPoolNodeSet[0:-1],
last_ordered[1] + 1))
for n in txnPoolNodeSet:
n.view_changer.on_master_degradation()
assert slow_node.master_replica.last_prepared_certificate_in_view() == \
(0, last_ordered[1])
ensureElectionsDone(looper, txnPoolNodeSet)
def test_dequeue_and_validate_commits(looper, txnPoolNodeSet,
sdk_wallet_client, sdk_pool_handle):
slow_node = [r.node for r in getNonPrimaryReplicas(txnPoolNodeSet, 0)][-1]
other_nodes = [n for n in txnPoolNodeSet if n != slow_node]
delay = 50
with delay_rules(slow_node.nodeIbStasher, pDelay(delay),
msg_rep_delay(delay, [PREPARE, PREPREPARE])):
with delay_rules(slow_node.nodeIbStasher, ppDelay(delay)):
sdk_send_batches_of_random_and_check(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
num_reqs=1,
num_batches=1)
assert not slow_node.master_replica._ordering_service.prePrepares
assert not slow_node.master_replica._ordering_service.prepares
assert not slow_node.master_replica._ordering_service.commits
assert len(slow_node.master_replica._ordering_service.
commitsWaitingForPrepare) > 0
waitNodeDataEquality(looper, slow_node, *other_nodes)
assert check_if_all_equal_in_list([
n.master_replica._ordering_service.ordered for n in txnPoolNodeSet
])
assert slow_node.master_replica._ordering_service.prePrepares
assert slow_node.master_replica._ordering_service.prepares
assert slow_node.master_replica._ordering_service.commits
assert not slow_node.master_replica._ordering_service.commitsWaitingForPrepare
assert all(slow_node.master_replica.last_ordered_3pc ==
n.master_replica.last_ordered_3pc for n in other_nodes)
def test_check_cdp_pp_storages(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client):
def check_all_empty(replica):
assert not bool(replica._consensus_data.preprepared)
assert not bool(replica._consensus_data.prepared)
def check_preprepared_not_empty(replica):
assert bool(replica._consensus_data.preprepared)
def check_prepared_not_empty(replica):
assert bool(replica._consensus_data.prepared)
def operation_for_replicas(operation, node_set=txnPoolNodeSet):
for node in node_set:
operation(node.master_replica)
node_stashers = [n.nodeIbStasher for n in txnPoolNodeSet]
with delay_rules(node_stashers, pDelay()):
with delay_rules(node_stashers, ppDelay()):
sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client)
looper.run(
eventually(operation_for_replicas, check_all_empty,
txnPoolNodeSet[1:]))
looper.run(
eventually(operation_for_replicas, check_preprepared_not_empty,
txnPoolNodeSet[0:1]))
looper.run(
eventually(operation_for_replicas, check_preprepared_not_empty,
txnPoolNodeSet))
looper.run(
eventually(operation_for_replicas, check_prepared_not_empty,
txnPoolNodeSet))
def test_catchup_with_one_slow_node(tdir, tconf, looper, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client,
allPluginsPath, logsearch):
'''
1. Stop the node Delta
2. Order 9 txns. In sending CatchupReq in a first round every
node [Alpha, Beta, Gamma] will receive request for 3 txns.
3. Delay CatchupReq messages on Alpha
4. Start Delta
5. Check that all nodes have equality data.
6. Check that Delta re-ask CatchupRep only once.
In the second CatchupRep (first re-ask) Delta shouldn't request
CatchupRep from Alpha because it didn't answer early.
If the behavior is wrong and Delta re-ask txns form all nodes,
every node will receive request for 1 txns, Alpha will not answer
and Delta will need a new re-ask round.
'''
# Prepare nodes
lagging_node = txnPoolNodeSet[-1]
rest_nodes = txnPoolNodeSet[:-1]
# Stop one node
waitNodeDataEquality(looper, lagging_node, *rest_nodes)
disconnect_node_and_ensure_disconnected(looper,
txnPoolNodeSet,
lagging_node,
stopNode=True)
looper.removeProdable(lagging_node)
# Send more requests to active nodes
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client,
len(rest_nodes) * 3)
waitNodeDataEquality(looper, *rest_nodes)
# Restart stopped node and wait for successful catch up
lagging_node = start_stopped_node(
lagging_node,
looper,
tconf,
tdir,
allPluginsPath,
start=False,
)
log_re_ask, _ = logsearch(
msgs=['requesting .* missing transactions after timeout'])
old_re_ask_count = len(log_re_ask)
# Delay CatchupRep messages on Alpha
with delay_rules(rest_nodes[0].nodeIbStasher, cqDelay()):
with delay_rules(lagging_node.nodeIbStasher, cs_delay()):
looper.add(lagging_node)
txnPoolNodeSet[-1] = lagging_node
looper.run(checkNodesConnected(txnPoolNodeSet))
waitNodeDataEquality(looper, *txnPoolNodeSet, customTimeout=120)
assert len(
log_re_ask
) - old_re_ask_count == 2 # for audit and domain ledgers
def test_delayed_instance_changes_after_vcd_for_next_view(looper, txnPoolNodeSet):
'''
A node is doing view change to view=1, while the other nodes already finished view change to view=2.
The node receives a quorum of VCD messages for view=2 before a quorum of InstanceChange messages for view=2.
Nevertheless, the node should not start a view change to view=2 without a quorum of InstanceChanges,
that is it should not go to propagate primary mode since it's already in view chanage state.
The node should eventually finish view change to view=2 once receives all VCD and IS msgs for view=2
'''
nodes = txnPoolNodeSet
slow_node = nodes[-1]
fast_nodes = [n for n in nodes if n != slow_node]
slow_stasher = slow_node.nodeIbStasher
# 1. DO FIRST VIEW CHANGE
# delay VCD for the first ViewChange
with delay_rules(slow_stasher, vcd_delay()):
# Trigger view change
for n in nodes:
n.view_changer.on_master_degradation()
waitForViewChange(looper, nodes, expectedViewNo=1)
# make sure view change is finished on all nodes except the slow one
ensureElectionsDone(looper, fast_nodes, instances_list=range(3))
# drop all VCD to view=1
slow_stasher.drop_delayeds()
# 2. DO SECOND VIEW CHANGE
# delay Instance Changes and
# so that the slow node receives VCD for view=2 before
# a quorum of InstanceChanges for that view while still doing view change to view=1
with delay_rules(slow_stasher, icDelay()):
# Trigger view change
for n in nodes:
n.view_changer.on_master_degradation()
waitForViewChange(looper, fast_nodes, expectedViewNo=2)
# make sure view change is finished on all nodes except the slow one
ensureElectionsDone(looper, fast_nodes, instances_list=range(3))
# slow node is still on view=1
assert slow_node.viewNo == 1
assert slow_node.view_change_in_progress
# make sure that the slow node receives VCD msgs for view=2
# and didn't receive IS msgs for view=2
check_vcd_msgs(slow_node, expected_view_no=2, expected_count=len(fast_nodes), )
check_no_ic_msgs(slow_node, expected_view_no=2)
# 3. RESET DELAYS AND CHECK
waitForViewChange(looper, nodes, expectedViewNo=2)
ensureElectionsDone(looper, nodes)
assert not slow_node.view_change_in_progress
ensure_all_nodes_have_same_data(looper, nodes=nodes)
def test_view_change_during_unstash(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, tconf):
slow_node = txnPoolNodeSet[-1]
other_nodes = txnPoolNodeSet[:-1]
slow_stasher = slow_node.nodeIbStasher
other_stashers = [n.nodeIbStasher for n in other_nodes]
all_stashers = [n.nodeIbStasher for n in txnPoolNodeSet]
# Preload nodes with some transactions
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
for node in txnPoolNodeSet:
assert node.master_replica.last_ordered_3pc == (0, 1)
# Prevent ordering of some requests
start_delaying(all_stashers, delay_3pc(after=7, msgs=(Prepare, Commit)))
# Stop ordering on slow node and send requests
slow_node_after_5 = start_delaying(slow_stasher,
delay_3pc(after=5, msgs=Commit))
slow_node_until_5 = start_delaying(slow_stasher, delay_3pc(after=0))
reqs_view_0 = sdk_send_random_requests(looper, sdk_pool_handle,
sdk_wallet_client, 8)
# Make pool order first 2 batches and pause
pool_after_3 = start_delaying(other_stashers, delay_3pc(after=3))
looper.run(eventually(check_nodes_ordered_till, other_nodes, 0, 3))
# Start catchup, continue ordering everywhere (except two last batches on slow node)
with delay_rules(slow_stasher, cr_delay()):
slow_node._do_start_catchup(just_started=False)
looper.run(eventually(check_catchup_is_started, slow_node))
stop_delaying_and_process(pool_after_3)
looper.run(eventually(check_nodes_ordered_till, other_nodes, 0, 7))
# Finish catchup and continue processing on slow node
looper.run(eventually(check_catchup_is_finished, slow_node))
stop_delaying_and_process(slow_node_until_5)
looper.run(eventually(check_nodes_ordered_till, [slow_node], 0, 5))
# Start view change and allow slow node to get remaining commits
with delay_rules(all_stashers, icDelay()):
for node in txnPoolNodeSet:
node.view_changer.on_master_degradation()
looper.runFor(0.1)
stop_delaying_and_process(slow_node_after_5)
# Ensure that expected number of requests was ordered
replies = sdk_get_replies(looper, reqs_view_0)
for rep in replies[:6]:
sdk_check_reply(rep)
# Ensure that everything is ok
ensureElectionsDone(looper, txnPoolNodeSet)
ensure_all_nodes_have_same_data(looper, txnPoolNodeSet)
sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_client,
sdk_pool_handle)
def test_catchup_to_next_view_during_view_change_by_primary(
txnPoolNodeSet, looper, sdk_pool_handle, sdk_wallet_steward):
'''
1) Lagging node is a primary for view=1
2) All nodes except the lagging one start a view change (to view=1)
3) The nodes can not finish it on time since the Primary for view=1 is lagging
4) All nodes except the lagging one go to view=2 then
5) All nodes except the lagging one order txns on view=2
6) Lagging node gets InstanceChanges for view=1 => it changes to view=2, and catches up till txns from view=2
7) Lagging node gets InstanceChanges for view=2 => it changes to view=2
8) Make sure that the lagging node is up to date, and can participate in consensus
'''
lagging_node = txnPoolNodeSet[1]
other_nodes = list(set(txnPoolNodeSet) - {lagging_node})
initial_view_no = checkViewNoForNodes(txnPoolNodeSet)
initial_last_ordered = lagging_node.master_last_ordered_3PC
with delay_rules(lagging_node.nodeIbStasher, delay_for_view(viewNo=2)):
with delay_rules(lagging_node.nodeIbStasher, delay_for_view(viewNo=0),
delay_for_view(viewNo=1)):
# view change to viewNo=2 since a primary for viewNo=1 is a lagging node
for n in txnPoolNodeSet:
n.view_changer.on_master_degradation()
waitForViewChange(looper,
other_nodes,
expectedViewNo=initial_view_no + 2,
customTimeout=30)
checkProtocolInstanceSetup(looper=looper,
nodes=other_nodes,
instances=range(3))
ensure_all_nodes_have_same_data(looper, nodes=other_nodes)
# order some txns
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, 5)
assert initial_view_no == lagging_node.viewNo
assert initial_last_ordered == lagging_node.master_last_ordered_3PC
assert len(lagging_node.master_replica._ordering_service.
requestQueues[DOMAIN_LEDGER_ID]) > 0
# make sure that the first View Change happened on lagging node
waitForViewChange(looper, [lagging_node],
expectedViewNo=initial_view_no + 1,
customTimeout=20)
assert initial_view_no + 1 == lagging_node.viewNo
# make sure that the second View Change happened on lagging node
waitForViewChange(looper, [lagging_node],
expectedViewNo=initial_view_no + 2,
customTimeout=20)
ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet)
ensure_all_nodes_have_same_data(looper, nodes=other_nodes)
# make sure that the pool is functional
sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_steward,
sdk_pool_handle)
def test_no_propagate_request_on_different_prepares_on_backup_before_vc(looper, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client):
''' Send random request and do view change then fast_nodes (2,3 - with
primary backup replica) will have prepare or send preprepare on backup
replicas and slow_nodes are have not and transaction will ordered on all
master replicas. Check last ordered after view change and after another
one request.'''
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
slow_instance = 1
slow_nodes = txnPoolNodeSet[1:3]
fast_nodes = [n for n in txnPoolNodeSet if n not in slow_nodes]
nodes_stashers = [n.nodeIbStasher for n in slow_nodes]
old_last_ordered = txnPoolNodeSet[0].master_replica.last_ordered_3pc
with delay_rules(nodes_stashers, pDelay(instId=slow_instance)):
with delay_rules(nodes_stashers, ppDelay(instId=slow_instance)):
# send one request
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
old_view_no = txnPoolNodeSet[0].viewNo
looper.run(
eventually(is_prepared,
fast_nodes,
2,
slow_instance))
# trigger view change on all nodes
ensure_view_change(looper, txnPoolNodeSet)
# wait for view change done on all nodes
ensureElectionsDone(looper, txnPoolNodeSet)
primary = getPrimaryReplica(txnPoolNodeSet, slow_instance).node
non_primaries = [n for n in txnPoolNodeSet if n is not primary]
check_last_ordered(non_primaries,
slow_instance,
(old_view_no, old_last_ordered[1] + 1))
# Backup primary replica must not advance last_ordered_3pc
# up to the master's value
check_last_ordered([primary],
slow_instance,
(old_view_no, old_last_ordered[1]))
check_last_ordered(txnPoolNodeSet,
txnPoolNodeSet[0].master_replica.instId,
(old_last_ordered[0], old_last_ordered[1] + 1))
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
looper.run(
eventually(check_last_ordered,
txnPoolNodeSet,
slow_instance,
(txnPoolNodeSet[0].viewNo, 1)))
assert all(0 == node.spylog.count(node.request_propagates)
for node in txnPoolNodeSet)
def do_view_change_with_propagate_primary_on_one_delayed_node(
slow_node, nodes, looper, sdk_pool_handle, sdk_wallet_client):
slow_stasher = slow_node.nodeIbStasher
fast_nodes = [n for n in nodes if n != slow_node]
stashers = [n.nodeIbStasher for n in nodes]
# Get last prepared certificate in pool
lpc = last_prepared_certificate(nodes)
# Get pool current view no
view_no = lpc[0]
with delay_rules(slow_stasher, icDelay()):
with delay_rules(slow_stasher, vcd_delay()):
with delay_rules(stashers, cDelay()):
# Send request
request = sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client)
# Wait until this request is prepared on N-f nodes
looper.run(eventually(check_last_prepared_certificate_on_quorum, nodes, (lpc[0], lpc[1] + 1)))
# Trigger view change
for n in nodes:
n.view_changer.on_master_degradation()
# Wait until view change is completed on all nodes except slow one
waitForViewChange(looper,
fast_nodes,
expectedViewNo=view_no + 1,
customTimeout=waits.expectedPoolViewChangeStartedTimeout(len(nodes)))
wait_for_elections_done_on_given_nodes(looper,
fast_nodes,
getRequiredInstances(len(nodes)),
timeout=waits.expectedPoolElectionTimeout(len(nodes)))
# Now all the nodes receive Commits
# The slow node will accept Commits and order the 3PC-batch in the old view
looper.runFor(waits.expectedOrderingTime(getNoInstances(len(nodes))))
# Now slow node receives ViewChangeDones
waitForViewChange(looper,
[slow_node],
expectedViewNo=view_no + 1,
customTimeout=waits.expectedPoolViewChangeStartedTimeout(len(nodes)))
wait_for_elections_done_on_given_nodes(looper,
[slow_node],
getRequiredInstances(len(nodes)),
timeout=waits.expectedPoolElectionTimeout(len(nodes)))
# Now slow node receives InstanceChanges but discards them because already
# started propagate primary to the same view.
# Finish request gracefully
sdk_get_reply(looper, request)
def do_view_change_with_propagate_primary_on_one_delayed_node(
slow_node, nodes, looper, sdk_pool_handle, sdk_wallet_client):
slow_stasher = slow_node.nodeIbStasher
fast_nodes = [n for n in nodes if n != slow_node]
stashers = [n.nodeIbStasher for n in nodes]
# Get last prepared certificate in pool
lpc = last_prepared_certificate(nodes)
# Get pool current view no
view_no = lpc[0]
with delay_rules(slow_stasher, icDelay()):
with delay_rules(slow_stasher, vcd_delay()):
with delay_rules(stashers, cDelay()):
# Send request
request = sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client)
# Wait until this request is prepared on N-f nodes
looper.run(eventually(check_last_prepared_certificate_on_quorum, nodes, (lpc[0], lpc[1] + 1)))
# Trigger view change
for n in nodes:
n.view_changer.on_master_degradation()
# Wait until view change is completed on all nodes except slow one
waitForViewChange(looper,
fast_nodes,
expectedViewNo=view_no + 1,
customTimeout=waits.expectedPoolViewChangeStartedTimeout(len(nodes)))
wait_for_elections_done_on_given_nodes(looper,
fast_nodes,
getRequiredInstances(len(nodes)),
timeout=waits.expectedPoolElectionTimeout(len(nodes)))
# Now all the nodes receive Commits
# The slow node will accept Commits and order the 3PC-batch in the old view
looper.runFor(waits.expectedOrderingTime(getNoInstances(len(nodes))))
# Now slow node receives ViewChangeDones
waitForViewChange(looper,
[slow_node],
expectedViewNo=view_no + 1,
customTimeout=waits.expectedPoolViewChangeStartedTimeout(len(nodes)))
wait_for_elections_done_on_given_nodes(looper,
[slow_node],
getRequiredInstances(len(nodes)),
timeout=waits.expectedPoolElectionTimeout(len(nodes)))
# Now slow node receives InstanceChanges but discards them because already
# started propagate primary to the same view.
# Finish request gracefully
sdk_get_reply(looper, request)
def test_stabilize_checkpoint_while_unstashing_when_missing_pre_prepare(
looper, chkFreqPatched, reqs_for_checkpoint, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client):
# Prepare nodes
lagging_node = txnPoolNodeSet[-1]
lagging_master_replcia = lagging_node.master_replica
rest_nodes = txnPoolNodeSet[:-1]
# 1. send enough requests so that just 1 is left for checkpoint stabilization
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, reqs_for_checkpoint - 1)
# 2. delay PrePrepare on 1 node so that prepares and commits will be stashed
with delay_rules(lagging_node.nodeIbStasher, ppDelay()):
with delay_rules(lagging_node.nodeIbStasher, msg_rep_delay()):
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
# all good nodes stabilized checkpoint
looper.run(
eventually(check_for_nodes, rest_nodes,
check_stable_checkpoint, 5))
# bad node received checkpoints from all nodes but didn't stabilize it
looper.run(
eventually(check_for_nodes, [lagging_node],
check_stable_checkpoint, 0))
looper.run(
eventually(check_for_nodes,
[lagging_node], check_received_checkpoint_votes, 5,
len(rest_nodes)))
# bad node has all commits and prepares for the last request stashed
looper.run(
eventually(lambda: assertExp(
(0, CHK_FREQ) in lagging_master_replcia._ordering_service.
preparesWaitingForPrePrepare and len(
lagging_master_replcia._ordering_service.
preparesWaitingForPrePrepare[
(0, CHK_FREQ)]) == len(rest_nodes) - 1)))
looper.run(
eventually(lambda: assertExp(
(0, CHK_FREQ) in lagging_master_replcia._ordering_service.
commitsWaitingForPrepare and len(
lagging_master_replcia._ordering_service.
commitsWaitingForPrepare[
(0, CHK_FREQ)]) == len(rest_nodes))))
# 3. the delayed PrePrepare is processed, and stashed prepares and commits are unstashed
# checkpoint will be stabilized during unstashing, and the request will be ordered
looper.run(
eventually(check_for_nodes, [lagging_node], check_stable_checkpoint,
5))
waitNodeDataEquality(looper, *txnPoolNodeSet, customTimeout=5)
def test_delay_rules_raise_type_error_when_given_stashers_of_wrong_type():
with pytest.raises(TypeError):
with delay_rules(1, delay_twos):
pass
with pytest.raises(TypeError):
with delay_rules([1, 2], delay_twos):
pass
with pytest.raises(TypeError):
with delay_rules(range(3), delay_twos):
pass
def test_delay_rules_raise_type_error_when_given_stashers_of_wrong_type():
with pytest.raises(TypeError):
with delay_rules(1, delay_twos):
pass
with pytest.raises(TypeError):
with delay_rules([1, 2], delay_twos):
pass
with pytest.raises(TypeError):
with delay_rules(range(3), delay_twos):
pass
def test_view_change_by_order_stashed_on_3_nodes_and_catchup_on_1_node(txnPoolNodeSet, looper,
sdk_pool_handle, sdk_wallet_steward):
'''
- COMMITS are delayed on all nodes
- All nodes starts a view change with a prepared certificate (for delayed message)
- COMMITS come during view change for 3 nodes
- So these 3 nodes finish view change by processing Commits and Ordered msgs during view change (in between rounds of catchup).
- The lagging (4th) node receives missing txns as part of catch-up (during view change) and also finishes view change.
'''
slow_node = txnPoolNodeSet[-1]
fast_nodes = txnPoolNodeSet[:-1]
slow_stasher = slow_node.nodeIbStasher
fast_stashers = [n.nodeIbStasher for n in fast_nodes]
all_stashers = [n.nodeIbStasher for n in txnPoolNodeSet]
initial_last_ordered = txnPoolNodeSet[0].master_replica.last_ordered_3pc
txns_count = 4
eventual_last_ordered = initial_last_ordered[0], initial_last_ordered[1] + txns_count
batches_count = initial_last_ordered[1]
with delay_rules(all_stashers, vcd_delay()):
# the lagging node is slow in receiving Commits and Catchup mghs
with delay_rules(slow_stasher, cDelay()):
with delay_rules(slow_stasher, lsDelay(), msg_rep_delay(types_to_delay=[LEDGER_STATUS])):
# fast nodes will receive and order Commits for last_prepared_cert during view change
with delay_rules(fast_stashers, cDelay()):
with delay_rules(fast_stashers, lsDelay(), msg_rep_delay(types_to_delay=[LEDGER_STATUS])):
sdk_send_random_requests(looper, sdk_pool_handle,
sdk_wallet_steward, txns_count)
batches_count += txns_count
looper.run(eventually(check_prepare_certificate, txnPoolNodeSet, batches_count))
check_last_ordered_3pc_on_master(txnPoolNodeSet, initial_last_ordered)
# trigger view change on all nodes
ensure_view_change(looper, txnPoolNodeSet)
looper.run(eventually(check_last_prepared_certificate_after_view_change_start,
txnPoolNodeSet, eventual_last_ordered))
# check that all txns are ordered till last prepared on fast nodes
looper.run(eventually(check_last_ordered_3pc_on_master, fast_nodes, eventual_last_ordered, timeout=30))
# check that all txns are ordered till last prepared on slow node as a result of catchup
looper.run(eventually(check_last_ordered_3pc_on_master, [slow_node], eventual_last_ordered, timeout=30))
# wait for view change done on all nodes
ensureElectionsDone(looper, txnPoolNodeSet)
ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet)
# make sure that the pool is functional
sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_steward, sdk_pool_handle)
def test_process_three_phase_msg_and_stashed_future_view(
txnPoolNodeSet, looper, tconf, sdk_pool_handle, sdk_wallet_steward):
"""
1. Delay ViewChangeDone messages for the slow_node.
2. Start view change on all nodes.
3. Order a new request.
4. Check that slow_node could not order this request and stashed all 3pc messages
and other nodes ordered.
6. Reset delays.
7. Check that the last request is ordered on the slow_node and stashed messages were removed.
"""
slow_node = txnPoolNodeSet[-1]
fast_nodes = txnPoolNodeSet[:-1]
view_no = slow_node.viewNo
old_stashed = {
inst_id: r.stasher.stash_size(STASH_VIEW_3PC)
for inst_id, r in slow_node.replicas.items()
}
with delay_rules([
slow_node.nodeIbStasher,
], msg_rep_delay(types_to_delay=[PREPREPARE, PREPARE, COMMIT])):
with delay_rules([
slow_node.nodeIbStasher,
], nv_delay()):
for n in txnPoolNodeSet:
n.view_changer.on_master_degradation()
waitForViewChange(looper,
fast_nodes,
expectedViewNo=view_no + 1,
customTimeout=2 * tconf.NEW_VIEW_TIMEOUT)
ensureElectionsDone(looper=looper,
nodes=fast_nodes,
instances_list=range(
fast_nodes[0].requiredNumberOfInstances))
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, 1)
assert slow_node.view_change_in_progress
# 1 - pre-prepare msg
# (len(txnPoolNodeSet) - 2) - prepare msgs
# (len(txnPoolNodeSet) - 1) - commit msgs
stashed_master_messages = 2 * (1 + (len(txnPoolNodeSet) - 2) +
(len(txnPoolNodeSet) - 1))
assert slow_node.master_replica.stasher.stash_size(
STASH_VIEW_3PC) == old_stashed[0] + stashed_master_messages
def chk():
for inst_id, r in slow_node.replicas.items():
assert r.last_ordered_3pc[1] == 2
assert r.stasher.stash_size(STASH_VIEW_3PC) == 0
looper.run(eventually(chk))
waitNodeDataEquality(looper, slow_node, *fast_nodes)
def test_slow_catchup_while_ordering(tdir, tconf, looper, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client):
lagging_node = txnPoolNodeSet[-1]
other_lagging_node = txnPoolNodeSet[-2]
other_nodes = txnPoolNodeSet[:-1]
other_stashers = [node.nodeIbStasher for node in other_nodes]
def lagging_node_state() -> NodeLeecherService.State:
return lagging_node.ledgerManager._node_leecher._state
def check_lagging_node_is_not_syncing_audit():
assert lagging_node_state() != NodeLeecherService.State.SyncingAudit
# Prevent lagging node from ordering
with delay_rules(lagging_node.nodeIbStasher, ppDelay(), pDelay(),
cDelay()):
# Order request on all nodes except lagging one
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
# Prevent lagging node from catching up domain ledger (and finishing catchup)
with delay_rules(other_stashers, delay_domain_ledger_catchup()):
# Start catchup on lagging node
lagging_node.ledgerManager.start_catchup()
assert lagging_node_state(
) == NodeLeecherService.State.SyncingAudit
# Ensure that audit ledger is caught up by lagging node
looper.run(eventually(check_lagging_node_is_not_syncing_audit))
assert lagging_node_state() != NodeLeecherService.State.Idle
# Order one more request on all nodes except lagging one
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
# Now lagging node can catch up domain ledger which contains more transactions
# than it was when audit ledger was caught up
# Now delayed 3PC messages reach lagging node, so any transactions missed during
# catch up can be ordered, ensure that all nodes will have same data after that
ensure_all_nodes_have_same_data(looper, txnPoolNodeSet)
# Ensure that even if we disable some other node pool is still functional
# (it won't be the case if old lagging node is nonfunctional)
with delay_rules(other_lagging_node.nodeIbStasher, ppDelay(), pDelay(),
cDelay()):
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
# Ensure that all nodes will eventually have same data
ensure_all_nodes_have_same_data(looper, txnPoolNodeSet)
def test_view_change_by_order_stashed_on_all(txnPoolNodeSet, looper,
sdk_pool_handle,
sdk_wallet_steward):
'''
- COMMITS are delayed on all nodes
- All nodes starts a view change with a prepared certificate (for delayed message)
- COMMITS come during view change for all nodes,
- So all nodes finish view change by processing Commits and Ordered msgs during view change (in between rounds of catchup).
'''
all_stashers = [n.nodeIbStasher for n in txnPoolNodeSet]
initial_last_ordered = txnPoolNodeSet[0].master_replica.last_ordered_3pc
txns_count = 4
eventual_last_ordered = initial_last_ordered[
0], initial_last_ordered[1] + txns_count
with delay_rules(all_stashers, vcd_delay()):
with delay_rules(all_stashers, cDelay()):
with delay_rules(all_stashers, lsDelay(),
msg_rep_delay(types_to_delay=[LEDGER_STATUS])):
sdk_send_random_requests(looper, sdk_pool_handle,
sdk_wallet_steward, txns_count)
looper.run(
eventually(check_prepare_certificate, txnPoolNodeSet,
txns_count))
check_last_ordered_3pc_on_master(txnPoolNodeSet,
initial_last_ordered)
# trigger view change on all nodes
ensure_view_change(looper, txnPoolNodeSet)
looper.run(
eventually(
check_last_prepared_certificate_after_view_change_start,
txnPoolNodeSet, eventual_last_ordered))
# check that all txns are ordered till last prepared
looper.run(
eventually(check_last_ordered_3pc_on_master,
txnPoolNodeSet,
eventual_last_ordered,
timeout=30))
# wait for view change done on all nodes
ensureElectionsDone(looper, txnPoolNodeSet)
ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet)
# make sure that the pool is functional
sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_steward,
sdk_pool_handle)
def test_catchup_from_unequal_nodes_without_waiting(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client):
normal_node = txnPoolNodeSet[0]
lagging_node_1 = txnPoolNodeSet[1]
lagging_node_2 = txnPoolNodeSet[2]
stopped_node = txnPoolNodeSet[3]
# Make sure everyone have one batch
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1)
# Wait until all nodes have same data and store last 3PC number of node that's going to be "stopped"
ensure_all_nodes_have_same_data(looper, txnPoolNodeSet, custom_timeout=30)
last_3pc = stopped_node.master_last_ordered_3PC
with delay_rules_without_processing(stopped_node.nodeIbStasher, delay_3pc()):
# Create one more batch on all nodes except "stopped" node
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1)
with delay_rules(lagging_node_1.nodeIbStasher, delay_3pc(msgs=Commit)):
# Create one more batch on all nodes except "stopped" and first lagging node
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1)
with delay_rules(lagging_node_2.nodeIbStasher, delay_3pc(msgs=Commit)):
# Create one more batch on all nodes except "stopped" and both lagging nodes
# This time we can't wait for replies because there will be only one
reqs = sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 1)
# Wait until normal node orders txn
looper.run(eventually(lambda: assert_eq(normal_node.master_last_ordered_3PC[1],
last_3pc[1] + 3)))
# Now all nodes have different number of txns, so if we try to start a catch up
# it is guaranteed that we'll need to ask for equal consistency proofs, and
# disabled timeout ensures that node can do so without relying on timeout
stopped_node.start_catchup()
# Wait until catchup ends
looper.run(eventually(lambda: assert_eq(stopped_node.ledgerManager._node_leecher._state,
NodeLeecherService.State.Idle)))
# Ensure stopped node caught up at least one batch
assert stopped_node.master_last_ordered_3PC[1] > last_3pc[1]
# And there was no view change
assert stopped_node.master_last_ordered_3PC[0] == last_3pc[0]
# Make sure replies from last request are eventually received
sdk_get_and_check_replies(looper, reqs)
def test_unstash_waiting_for_first_batch_ordered_after_catchup(
looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle, tconf):
lagged_node = txnPoolNodeSet[-1]
other_nodes = list(set(txnPoolNodeSet) - {lagged_node})
other_stashers = [n.nodeIbStasher for n in other_nodes]
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
last_ordered_lagged_before = lagged_node.master_last_ordered_3PC
# do not process any message reqs for PrePrepares
with delay_rules_without_processing(
lagged_node.nodeIbStasher,
msg_rep_delay(types_to_delay=[PREPARE, PREPREPARE])):
with delay_rules(lagged_node.nodeIbStasher, cDelay()):
ensure_view_change(looper, txnPoolNodeSet)
looper.run(eventually(check_not_in_view_change, txnPoolNodeSet))
ensureElectionsDone(looper,
other_nodes,
instances_list=range(
getRequiredInstances(len(txnPoolNodeSet))))
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
# delay Commits on all nodes so that there are some PrePrepares still stashed after catchup
with delay_rules(other_stashers, cDelay()):
pre_prep_before = len(recvdPrePrepareForInstId(lagged_node, 0))
sdk_send_random_requests(looper, sdk_pool_handle,
sdk_wallet_client, 2)
# wait till lagged node recives the new PrePrepares
# they will be stashed as WAITING_FIRST_BATCH_IN_VIEW
looper.run(
eventually(lambda: assertExp(
len(recvdPrePrepareForInstId(lagged_node, 0)) ==
pre_prep_before + 2)))
# catchup the lagged node
# the latest 2 PrePrepares are still stashed
lagged_node.start_catchup()
looper.run(
eventually(
lambda: assertExp(lagged_node.master_last_ordered_3PC >
last_ordered_lagged_before)))
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 2)
ensureElectionsDone(looper, txnPoolNodeSet, customTimeout=30)
ensure_all_nodes_have_same_data(looper, txnPoolNodeSet, custom_timeout=30)
def test_state_proof_for_get_fee(helpers, nodeSetWithIntegratedTokenPlugin,
looper, sdk_pool_handle, sdk_wallet_trustee):
# make sure that config ledger is BLS signed by sending a txn to config ledger
send_and_check_auth_rule(looper, sdk_pool_handle, sdk_wallet_trustee)
with delay_rules([n.clientIbStasher for n in nodeSetWithIntegratedTokenPlugin[1:]], req_delay()):
resp = helpers.general.do_get_fee()
assert resp.get(STATE_PROOF, False)
assert {} == resp[FEE]
fees = {NYM_FEES_ALIAS: 5}
helpers.general.do_set_fees(fees)
with delay_rules([n.clientIbStasher for n in nodeSetWithIntegratedTokenPlugin[1:]], req_delay()):
resp = helpers.general.do_get_fee()
assert resp.get(STATE_PROOF, False)
assert fees == resp[FEE]
def test_state_proof_for_get_fees(helpers, nodeSetWithIntegratedTokenPlugin):
fees = {NYM_FEES_ALIAS: 5}
with delay_rules(
[n.clientIbStasher for n in nodeSetWithIntegratedTokenPlugin[1:]],
req_delay()):
with pytest.raises(PoolLedgerTimeoutException):
helpers.general.do_get_fees()
helpers.general.do_set_fees(fees)
with delay_rules(
[n.nodeIbStasher for n in nodeSetWithIntegratedTokenPlugin[1:]],
req_delay()):
resp = helpers.general.do_get_fees()
assert resp.get(STATE_PROOF, False)
assert fees == resp[FEES]
def test_freeing_forwarded_preprepared_request(
looper, chkFreqPatched, reqs_for_checkpoint, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_steward):
# Case, when both backup and primary had problems
behind_node = txnPoolNodeSet[-1]
sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, CHK_FREQ, CHK_FREQ)
with delay_rules(behind_node.nodeIbStasher,
pDelay(delay=sys.maxsize),
cDelay(delay=sys.maxsize), ):
count = behind_node.spylog.count(behind_node.allLedgersCaughtUp)
sdk_send_batches_of_random(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, req_num, req_num)
looper.run(eventually(node_caughtup, behind_node, count, retryWait=1))
assert len(behind_node.requests) == req_num
assert all(r.executed for r in behind_node.requests.values() if behind_node.seqNoDB.get(r.request.key)[1])
sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, CHK_FREQ, CHK_FREQ)
# Master and backup replicas do not stash new requests and succesfully order them
assert len(behind_node.requests) == req_num
def test_last_committed_after_catchup(looper, helpers,
nodeSetWithIntegratedTokenPlugin,
sdk_pool_handle,
fees_set, address_main, mint_tokens):
node_set = nodeSetWithIntegratedTokenPlugin
reverted_node = node_set[-1]
amount = get_amount_from_token_txn(mint_tokens)
init_seq_no = 1
request_1, request_2 = nyms_with_fees(2,
helpers,
fees_set,
address_main,
amount,
init_seq_no=init_seq_no)
reverted_last_committed = get_last_committed_from_tracker(reverted_node)
not_reverted_last_committed = get_last_committed_from_tracker(node_set[-1])
assert reverted_last_committed == not_reverted_last_committed
with delay_rules(reverted_node.nodeIbStasher, cDelay()):
"""
Send NYM with FEES and wait for reply.
"""
r = sdk_sign_and_submit_req_obj(looper, sdk_pool_handle, helpers.request._steward_wallet, request_1)
sdk_get_and_check_replies(looper, [r])
"""
Start catchup. Uncommitted batch for reverted_node should be rejected and it will get
NYM with FEES during catchup procedure.
"""
reverted_node.start_catchup()
looper.run(eventually(lambda: assertExp(reverted_node.mode == Mode.participating)))
assert get_last_committed_from_tracker(reverted_node) ==\
get_last_committed_from_tracker(node_set[0])
def test_no_propagated_future_view_change_until_synced(txnPoolNodeSet, looper, mode):
# the last node is a lagging one, which will receive ViewChangeDone messages for future view
viewNo = checkViewNoForNodes(txnPoolNodeSet)
lagged_node_index = (viewNo + 3) % len(txnPoolNodeSet)
lagged_node = txnPoolNodeSet[lagged_node_index]
other_nodes = list(set(txnPoolNodeSet) - {lagged_node})
# emulate catchup by setting non-synced status
lagged_node.mode = mode
old_view_no = checkViewNoForNodes([lagged_node])
check_future_vcd_count(lagged_node, 0)
# delay INSTANCE CHANGE on lagged nodes, so all nodes except the lagging one finish View Change
with delay_rules(lagged_node.nodeIbStasher, icDelay()):
# make sure that View Change happened on all nodes but the lagging one
ensure_view_change(looper, other_nodes)
checkProtocolInstanceSetup(looper=looper, nodes=other_nodes, instances=range(2))
ensure_all_nodes_have_same_data(looper, nodes=other_nodes)
check_no_view_change(looper, lagged_node)
assert old_view_no == checkViewNoForNodes([lagged_node])
# emulate finishing of catchup by setting Participating status
lagged_node.mode = Mode.participating
# make sure that View Change happened on lagging node
waitForViewChange(looper, [lagged_node], expectedViewNo=old_view_no + 1,
customTimeout=10)
ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet)
def test_view_change_retry_by_timeout(txnPoolNodeSet, looper, tconf, setup,
sdk_pool_handle, sdk_wallet_client):
"""
Verifies that a view change is restarted if it is not completed in time
"""
m_primary_node, initial_view_no, timeout_callback_stats = setup
stashers = [n.nodeIbStasher for n in txnPoolNodeSet]
with delay_rules(stashers, nv_delay()):
start_view_change(txnPoolNodeSet, initial_view_no + 1)
# First view change should fail, because of delayed ViewChangeDone
# messages. This then leads to new view change that we need.
with pytest.raises(AssertionError):
ensureElectionsDone(looper=looper,
nodes=txnPoolNodeSet,
customTimeout=1.5 * NEW_VIEW_TIMEOUT)
# Now as ViewChangeDone messages are unblocked view changes should finish successfully
ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet)
ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet)
new_m_primary_node = get_master_primary_node(list(txnPoolNodeSet))
assert m_primary_node.name != new_m_primary_node.name
# The timeout method was called one time
check_watchdog_called_expected_times(txnPoolNodeSet,
timeout_callback_stats, 1)
# 2 view changes have been initiated
for node in txnPoolNodeSet:
assert node.viewNo - initial_view_no == 2
sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_client,
sdk_pool_handle)
def test_deletion_non_forwarded_request(
looper, chkFreqPatched, reqs_for_checkpoint, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_steward, tconf, tdir, allPluginsPath):
behind_node = txnPoolNodeSet[-1]
[behind_node.replicas.values()[1].discard_req_key(1, key) for key in behind_node.requests]
behind_node.requests.clear()
sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, CHK_FREQ, CHK_FREQ)
behind_node.quorums.propagate = Quorum(len(txnPoolNodeSet) + 1)
with delay_rules(behind_node.nodeIbStasher,
ppDelay(delay=sys.maxsize),
pDelay(delay=sys.maxsize),
cDelay(delay=sys.maxsize)):
count = behind_node.spylog.count(behind_node.allLedgersCaughtUp)
sdk_send_batches_of_random(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, req_num, req_num)
looper.run(eventually(node_caughtup, behind_node, count, retryWait=1))
# We clear caughtup requests
assert len(behind_node.requests) == 0
assert all([len(q) == 0 for r in behind_node.replicas.values() for q in r.requestQueues.values()])
assert len(behind_node.clientAuthNr._verified_reqs) == 0
assert len(behind_node.requestSender) == 0
def do_view_change_with_delayed_commits_on_all_but_one(nodes, nodes_without_one_stashers,
except_node,
looper,
sdk_pool_handle,
sdk_wallet_client):
new_view_no = except_node.viewNo + 1
old_last_ordered = except_node.master_replica.last_ordered_3pc
# delay commits for all nodes except node X
with delay_rules(nodes_without_one_stashers, cDelay(sys.maxsize)):
# send one request
requests2 = sdk_send_random_requests(looper, sdk_pool_handle,
sdk_wallet_client, 1)
def last_ordered(node: Node, last_ordered):
assert node.master_replica.last_ordered_3pc == last_ordered
# wait until except_node ordered txn
looper.run(
eventually(last_ordered, except_node, (except_node.viewNo,
old_last_ordered[1] + 1)))
# trigger view change on all nodes
for node in nodes:
node.view_changer.on_master_degradation()
# wait for view change done on all nodes
looper.run(eventually(view_change_done, nodes, new_view_no))
sdk_get_replies(looper, requests2)
def test_use_modified_rules_from_uncommitted(looper, txnPoolNodeSet,
sdk_wallet_trustee,
sdk_wallet_steward,
sdk_pool_handle):
node_stashers = [n.nodeIbStasher for n in txnPoolNodeSet]
wh, _ = sdk_wallet_trustee
new_steward_did, new_steward_verkey = create_verkey_did(looper, wh)
changed_constraint = AuthConstraint(role=STEWARD, sig_count=1)
with delay_rules(node_stashers, cDelay()):
r_auth = sdk_send_and_check_auth_rule_request(
looper,
sdk_pool_handle,
sdk_wallet_trustee,
auth_action=ADD_PREFIX,
auth_type=NYM,
field=ROLE,
new_value=STEWARD,
old_value=None,
constraint=changed_constraint.as_dict,
no_wait=True)
looper.runFor(waits.expectedPrePrepareTime(len(txnPoolNodeSet)))
r_add_steward = sdk_add_new_nym(looper,
sdk_pool_handle,
sdk_wallet_steward,
'newSteward2',
STEWARD_STRING,
dest=new_steward_did,
verkey=new_steward_verkey,
no_wait=True)
sdk_get_and_check_replies(looper, [r_auth])
sdk_get_and_check_replies(looper, [r_add_steward])
def test_ordered_request_freed_on_replica_removal(looper, txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
chkFreqPatched, view_change):
node = txnPoolNodeSet[0]
# Stabilize checkpoint
# Send one more request to stabilize checkpoint
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
old_stable_checkpoint = node.master_replica._consensus_data.stable_checkpoint
with delay_rules(node.nodeIbStasher, cDelay(),
msg_rep_delay(types_to_delay=[COMMIT])):
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
f_d, f_r = get_forwarded_to_all(node)
assert f_d
node.replicas.remove_replica(node.replicas.num_replicas - 1)
assert node.requests[f_d].forwardedTo == node.replicas.num_replicas
looper.run(
eventually(check_for_nodes, txnPoolNodeSet, check_stable_checkpoint,
old_stable_checkpoint))
# Send one more request to stabilize checkpoint
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, CHK_FREQ - 1)
looper.run(
eventually(check_for_nodes, txnPoolNodeSet, check_stable_checkpoint,
old_stable_checkpoint + CHK_FREQ))
def test_multiple_view_change_retries_by_timeouts(
txnPoolNodeSet, looper, tconf, setup,
sdk_pool_handle, sdk_wallet_client):
"""
Verifies that a view change is restarted each time
when the previous one is timed out
"""
_, initial_view_no, timeout_callback_stats = setup
stashers = [n.nodeIbStasher for n in txnPoolNodeSet]
with delay_rules(stashers, vcd_delay()):
start_view_change(txnPoolNodeSet, initial_view_no + 1)
# Wait until timeout callback is called 3 times
looper.run(eventually(check_watchdog_called_expected_times,
txnPoolNodeSet, timeout_callback_stats, 3,
retryWait=1,
timeout=3 * VIEW_CHANGE_TIMEOUT + 2))
# View changes should fail
with pytest.raises(AssertionError):
ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet, customTimeout=1)
# This view change must be completed with no problems
ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet)
ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet)
# 4 view changes must have been initiated (initial one + 3 retries)
for node in txnPoolNodeSet:
assert node.viewNo - initial_view_no == 4
sdk_ensure_pool_functional(looper, txnPoolNodeSet,
sdk_wallet_client,
sdk_pool_handle)
def test_view_change_retry_by_timeout(
txnPoolNodeSet, looper, tconf, setup, sdk_pool_handle, sdk_wallet_client):
"""
Verifies that a view change is restarted if it is not completed in time
"""
m_primary_node, initial_view_no, timeout_callback_stats = setup
stashers = [n.nodeIbStasher for n in txnPoolNodeSet]
with delay_rules(stashers, vcd_delay()):
start_view_change(txnPoolNodeSet, initial_view_no + 1)
# First view change should fail, because of delayed ViewChangeDone
# messages. This then leads to new view change that we need.
with pytest.raises(AssertionError):
ensureElectionsDone(looper=looper,
nodes=txnPoolNodeSet,
customTimeout=1.5 * VIEW_CHANGE_TIMEOUT)
# Now as ViewChangeDone messages are unblocked view changes should finish successfully
ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet)
ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet)
new_m_primary_node = get_master_primary_node(list(txnPoolNodeSet))
assert m_primary_node.name != new_m_primary_node.name
# The timeout method was called one time
check_watchdog_called_expected_times(txnPoolNodeSet, timeout_callback_stats, 1)
# 2 view changes have been initiated
for node in txnPoolNodeSet:
assert node.viewNo - initial_view_no == 2
sdk_ensure_pool_functional(looper, txnPoolNodeSet,
sdk_wallet_client,
sdk_pool_handle)
def test_delay_rules_enable_delays_on_entry_and_disables_them_on_exit():
s = Stasher(deque())
with delay_rules(s, delay_twos):
assert delay_twos in s.delayRules
assert delay_twos not in s.delayRules
def do_view_change_with_delayed_commits_on_all_but_one(nodes, nodes_without_one_stashers,
except_node,
looper,
sdk_pool_handle,
sdk_wallet_client):
new_view_no = except_node.viewNo + 1
old_last_ordered = except_node.master_replica.last_ordered_3pc
# delay commits for all nodes except node X
with delay_rules(nodes_without_one_stashers, cDelay(sys.maxsize)):
# send one request
requests2 = sdk_send_random_requests(looper, sdk_pool_handle,
sdk_wallet_client, 1)
def last_ordered(node: Node, last_ordered):
assert node.master_replica.last_ordered_3pc == last_ordered
# wait until except_node ordered txn
looper.run(
eventually(last_ordered, except_node, (except_node.viewNo,
old_last_ordered[1] + 1)))
# trigger view change on all nodes
for node in nodes:
node.view_changer.on_master_degradation()
# wait for view change done on all nodes
looper.run(eventually(view_change_done, nodes, new_view_no))
sdk_get_replies(looper, requests2)
def test_watermarks_after_view_change(tdir, tconf,
looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client):
"""
Delay commit, checkpoint, InstanceChange and ViewChangeDone messages for lagging_node.
Start ViewChange.
Check that ViewChange finished.
Reset delays.
Check that lagging_node can order transactions and has same data with other nodes.
"""
lagging_node = txnPoolNodeSet[-1]
lagging_node.master_replica.config.LOG_SIZE = LOG_SIZE
start_view_no = lagging_node.viewNo
with delay_rules(lagging_node.nodeIbStasher, cDelay(), chk_delay(), icDelay(), nv_delay()):
trigger_view_change(txnPoolNodeSet)
waitForViewChange(looper,
txnPoolNodeSet[:-1],
expectedViewNo=start_view_no + 1,
customTimeout=waits.expectedPoolViewChangeStartedTimeout(len(txnPoolNodeSet)))
ensure_all_nodes_have_same_data(looper, txnPoolNodeSet[:-1])
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 6)
ensure_all_nodes_have_same_data(looper, txnPoolNodeSet)
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
ensure_all_nodes_have_same_data(looper, txnPoolNodeSet)
def test_no_propagate_request_on_different_last_ordered_on_master_before_vc(
looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client):
''' Send random request and do view change then fast_nodes (1, 4 - without
primary after next view change) are already ordered transaction on master
and slow_nodes are not. Check ordering on slow_nodes.'''
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
master_instance = txnPoolNodeSet[0].master_replica.instId
slow_nodes = txnPoolNodeSet[1:3]
fast_nodes = [n for n in txnPoolNodeSet if n not in slow_nodes]
nodes_stashers = [n.nodeIbStasher for n in slow_nodes]
old_last_ordered = txnPoolNodeSet[0].master_replica.last_ordered_3pc
with delay_rules(nodes_stashers, cDelay(delay=sys.maxsize)):
# send one request
requests = sdk_send_random_requests(looper, sdk_pool_handle,
sdk_wallet_client, 1)
last_ordered_for_slow = slow_nodes[0].master_replica.last_ordered_3pc
old_view_no = txnPoolNodeSet[0].viewNo
looper.run(
eventually(check_last_ordered, fast_nodes, master_instance,
(old_view_no, old_last_ordered[1] + 1)))
# trigger view change on all nodes
for node in txnPoolNodeSet:
node.view_changer.on_master_degradation()
# wait for view change done on all nodes
ensureElectionsDone(looper, txnPoolNodeSet)
sdk_get_replies(looper, requests)
looper.run(
eventually(check_last_ordered, slow_nodes, master_instance,
(old_view_no, last_ordered_for_slow[1] + 1)))
assert all(0 == node.spylog.count(node.request_propagates)
for node in txnPoolNodeSet)
def test_delay_rules_dont_touch_other_delays():
s = Stasher(deque())
s.delay(delay_threes)
with delay_rules(s, delay_twos):
assert delay_threes in s.delayRules
assert delay_threes in s.delayRules
def test_delay_rules_can_use_multiple_delayers():
s = Stasher(deque())
with delay_rules(s, delay_twos, delay_threes):
assert delay_twos in s.delayRules
assert delay_threes in s.delayRules
assert delay_twos not in s.delayRules
assert delay_threes not in s.delayRules
def test_delay_rules_can_use_multiple_stashers():
s1 = Stasher(deque())
s2 = Stasher(deque())
with delay_rules([s1, s2], delay_twos):
assert delay_twos in s1.delayRules
assert delay_twos in s2.delayRules
assert delay_twos not in s1.delayRules
assert delay_twos not in s2.delayRules
def test_delay_rules_can_use_generator_expressions():
stashers = [Stasher(deque(), name="{}".format(i)) for i in range(3)]
with delay_rules((s for s in stashers if s.name != "1"), delay_twos):
assert delay_twos in stashers[0].delayRules
assert delay_twos not in stashers[1].delayRules
assert delay_twos in stashers[2].delayRules
assert delay_twos not in stashers[0].delayRules
assert delay_twos not in stashers[1].delayRules
assert delay_twos not in stashers[2].delayRules
def test_no_propagate_request_on_different_last_ordered_on_backup_before_vc(looper, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client):
''' Send random request and do view change then fast_nodes (1, 4 - without
primary backup replicas) are already ordered transaction on master and some backup replica
and slow_nodes are not on backup replica. Wait ordering on slow_nodes.'''
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
slow_instance = 1
slow_nodes = txnPoolNodeSet[1:3]
fast_nodes = [n for n in txnPoolNodeSet if n not in slow_nodes]
nodes_stashers = [n.nodeIbStasher for n in slow_nodes]
old_last_ordered = txnPoolNodeSet[0].replicas[slow_instance].last_ordered_3pc
with delay_rules(nodes_stashers, cDelay(instId=slow_instance)):
# send one request
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
old_view_no = txnPoolNodeSet[0].viewNo
looper.run(
eventually(check_last_ordered,
fast_nodes,
slow_instance,
(old_view_no, old_last_ordered[1] + 1)))
check_last_ordered(slow_nodes, slow_instance, old_last_ordered)
# trigger view change on all nodes
ensure_view_change(looper, txnPoolNodeSet)
# wait for view change done on all nodes
ensureElectionsDone(looper, txnPoolNodeSet)
primary = getPrimaryReplica(txnPoolNodeSet, slow_instance).node
non_primaries = [n for n in txnPoolNodeSet if n is not primary]
check_last_ordered(non_primaries,
slow_instance,
(old_view_no, old_last_ordered[1] + 1))
# Backup primary replica must not advance last_ordered_3pc
# up to the master's value
check_last_ordered([primary],
slow_instance,
(old_view_no, old_last_ordered[1]))
check_last_ordered(txnPoolNodeSet,
txnPoolNodeSet[0].master_replica.instId,
(old_last_ordered[0], old_last_ordered[1] + 1))
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
assert all(0 == node.spylog.count(node.request_propagates)
for node in txnPoolNodeSet)
def test_delay_rules_return_delayed_items_to_list_on_exit():
q = deque([1, 2, 3])
s = Stasher(q)
s.delay(delay_threes)
with delay_rules(s, delay_twos):
s.process()
assert 1 in q
assert 2 not in q
assert 3 not in q
assert 1 in q
assert 2 in q
assert 3 not in q
def do_view_change_with_pending_request_and_one_fast_node(fast_node,
nodes, looper, sdk_pool_handle, sdk_wallet_client):
"""
Perform view change while processing request, with one node receiving commits much sooner than others.
With current implementation of view change this will result in corrupted state of fast node
"""
fast_stasher = fast_node.nodeIbStasher
slow_nodes = [n for n in nodes if n != fast_node]
slow_stashers = [n.nodeIbStasher for n in slow_nodes]
# Get last prepared certificate in pool
lpc = last_prepared_certificate(nodes)
# Get pool current view no
view_no = lpc[0]
# Delay all COMMITs
with delay_rules(slow_stashers, cDelay()):
with delay_rules(fast_stasher, cDelay()):
# Send request
request = sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client)
# Wait until this request is prepared on N-f nodes
looper.run(eventually(check_last_prepared_certificate_on_quorum, nodes, (lpc[0], lpc[1] + 1)))
# Trigger view change
for n in nodes:
n.view_changer.on_master_degradation()
# Now commits are processed on fast node
# Wait until view change is complete
looper.run(eventually(check_view_change_done, nodes, view_no + 1, timeout=60))
# Finish request gracefully
sdk_get_reply(looper, request)
def test_freeing_forwarded_not_preprepared_request(
looper, chkFreqPatched, reqs_for_checkpoint, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_steward, tconf, tdir, allPluginsPath):
behind_node = txnPoolNodeSet[-1]
behind_node.requests.clear()
sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, CHK_FREQ, CHK_FREQ)
with delay_rules(behind_node.nodeIbStasher,
ppDelay(delay=sys.maxsize),
pDelay(delay=sys.maxsize),
cDelay(delay=sys.maxsize)):
count = behind_node.spylog.count(behind_node.allLedgersCaughtUp)
sdk_send_batches_of_random(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, req_num, req_num)
looper.run(eventually(node_caughtup, behind_node, count, retryWait=1))
# We execute caughtup requests
assert len(behind_node.requests) == req_num
assert all(r.executed for r in behind_node.requests.values() if behind_node.seqNoDB.get(r.request.key)[1])
def do_test_replica_removing_with_backup_degraded(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
tconf):
"""
Node will change view even though it does not find the master to be degraded
when a quorum of nodes agree that master performance degraded
"""
start_replicas_count = txnPoolNodeSet[0].replicas.num_replicas
view_no = txnPoolNodeSet[0].viewNo
instance_to_remove = 1
stashers = [node.nodeIbStasher for node in txnPoolNodeSet]
with delay_rules(stashers, cDelay(delay=sys.maxsize, instId=instance_to_remove)):
sdk_send_batches_of_random_and_check(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
num_reqs=10,
num_batches=5)
# check that replicas were removed
def check_replica_removed_on_all_nodes(inst_id=instance_to_remove):
for n in txnPoolNodeSet:
check_replica_removed(n,
start_replicas_count,
inst_id)
assert not n.monitor.isMasterDegraded()
looper.run(eventually(check_replica_removed_on_all_nodes, timeout=120))
# start View Change
for node in txnPoolNodeSet:
node.view_changer.on_master_degradation()
waitForViewChange(looper, txnPoolNodeSet, expectedViewNo=view_no + 1,
customTimeout=2 * tconf.VIEW_CHANGE_TIMEOUT)
ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet)
# check that all replicas were restored
assert all(start_replicas_count == node.replicas.num_replicas
for node in txnPoolNodeSet)
def test_view_change_timeout_reset_on_next_view(txnPoolNodeSet, looper, tconf):
# Check that all nodes are in view 0
assert all(n.viewNo == 0 for n in txnPoolNodeSet)
stashers = [n.nodeIbStasher for n in txnPoolNodeSet]
with delay_rules(stashers, vcd_delay()):
# Start first view change
for n in txnPoolNodeSet:
n.view_changer.on_master_degradation()
waitForViewChange(looper, txnPoolNodeSet, expectedViewNo=1)
looper.runFor(0.6 * VIEW_CHANGE_TIMEOUT)
# Start second view change
for n in txnPoolNodeSet:
n.view_changer.on_master_degradation()
waitForViewChange(looper, txnPoolNodeSet, expectedViewNo=2)
looper.runFor(0.6 * VIEW_CHANGE_TIMEOUT)
# Ensure only 2 view changes happened
ensureElectionsDone(looper, txnPoolNodeSet)
for n in txnPoolNodeSet:
assert n.viewNo == 2
def test_no_propagate_request_on_different_last_ordered_on_master_before_vc(looper, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client):
''' Send random request and do view change then fast_nodes (1, 4 - without
primary after next view change) are already ordered transaction on master
and slow_nodes are not. Check ordering on slow_nodes.'''
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
master_instance = txnPoolNodeSet[0].master_replica.instId
slow_nodes = txnPoolNodeSet[1:3]
fast_nodes = [n for n in txnPoolNodeSet if n not in slow_nodes]
nodes_stashers = [n.nodeIbStasher for n in slow_nodes]
old_last_ordered = txnPoolNodeSet[0].master_replica.last_ordered_3pc
with delay_rules(nodes_stashers, cDelay()):
# send one request
requests = sdk_send_random_requests(looper, sdk_pool_handle,
sdk_wallet_client, 1)
last_ordered_for_slow = slow_nodes[0].master_replica.last_ordered_3pc
old_view_no = txnPoolNodeSet[0].viewNo
looper.run(
eventually(check_last_ordered,
fast_nodes,
master_instance,
(old_view_no, old_last_ordered[1] + 1)))
# trigger view change on all nodes
ensure_view_change(looper, txnPoolNodeSet)
# wait for view change done on all nodes
ensureElectionsDone(looper, txnPoolNodeSet)
replies = sdk_get_replies(looper, requests)
for reply in replies:
sdk_check_reply(reply)
check_last_ordered(slow_nodes,
master_instance,
(old_view_no, last_ordered_for_slow[1] + 1))
assert all(0 == node.spylog.count(node.request_propagates)
for node in txnPoolNodeSet)
def test_apply_stashed_partially_ordered(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client):
test_node = getNonPrimaryReplicas(txnPoolNodeSet)[0].node
test_stasher = test_node.nodeIbStasher
ledger_size = max(node.domainLedger.size for node in txnPoolNodeSet)
def check_pool_ordered_some_requests():
assert max(node.domainLedger.size for node in txnPoolNodeSet) > ledger_size
def check_test_node_has_stashed_ordered_requests():
assert len(test_node.stashedOrderedReqs) > 0
# Delay COMMITs so requests are not ordered on test node
with delay_rules(test_stasher, cDelay()):
reqs = sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, TOTAL_REQUESTS)
looper.run(eventually(check_pool_ordered_some_requests))
# Get some of txns that need to be ordered
ledger_info = test_node.ledgerManager.getLedgerInfoByType(DOMAIN_LEDGER_ID)
txns = ledger_info.ledger.uncommittedTxns
txns = txns[:len(txns) // 2]
assert len(txns) > 1
# Emulate incomplete catchup simultaneous with generation of ORDERED message
test_node.mode = Mode.syncing
test_node.master_replica.revert_unordered_batches()
looper.run(eventually(check_test_node_has_stashed_ordered_requests))
for txn in txns:
ledger_info.ledger.add(txn)
ledger_info.postTxnAddedToLedgerClbk(DOMAIN_LEDGER_ID, txn)
test_node.mode = Mode.participating
test_node.processStashedOrderedReqs()
ensure_all_nodes_have_same_data(looper, txnPoolNodeSet)
sdk_get_and_check_replies(looper, reqs)
def test_unordered_request_freed_on_replica_removal(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
chkFreqPatched,
view_change):
node = txnPoolNodeSet[0]
stashers = [n.nodeIbStasher for n in txnPoolNodeSet]
with delay_rules(stashers, cDelay(delay=sys.maxsize)):
req = sdk_send_random_requests(looper,
sdk_pool_handle,
sdk_wallet_client,
1)
looper.runFor(waits.expectedPropagateTime(len(txnPoolNodeSet)) +
waits.expectedPrePrepareTime(len(txnPoolNodeSet)) +
waits.expectedPrepareTime(len(txnPoolNodeSet)) +
waits.expectedCommittedTime(len(txnPoolNodeSet)))
assert len(node.requests) == 1
forwardedToBefore = next(iter(node.requests.values())).forwardedTo
node.replicas.remove_replica(node.replicas.num_replicas - 1)
assert len(node.requests) == 1
forwardedToAfter = next(iter(node.requests.values())).forwardedTo
assert forwardedToAfter == forwardedToBefore - 1
chkChkpoints(txnPoolNodeSet, 0)
sdk_get_replies(looper, req)
chkChkpoints(txnPoolNodeSet, 1)
# Send one more request to stabilize checkpoint
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1)
looper.run(eventually(chkChkpoints, txnPoolNodeSet, 1, 0))
assert len(node.requests) == 0
def test_clearing_forwarded_preprepared_request(
looper, chkFreqPatched, reqs_for_checkpoint, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_steward):
# Case when backup ordered correctly, but primary had problems.
# As a result, master will execute caughtup txns and will be removed
# from requests queues
behind_node = txnPoolNodeSet[-1]
sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, CHK_FREQ, CHK_FREQ)
with delay_rules(behind_node.nodeIbStasher,
pDelay(delay=sys.maxsize, instId=0),
cDelay(delay=sys.maxsize, instId=0)):
count = behind_node.spylog.count(behind_node.allLedgersCaughtUp)
sdk_send_batches_of_random(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, req_num, req_num)
looper.run(eventually(node_caughtup, behind_node, count, retryWait=1))
assert len(behind_node.requests) == 0
assert all([len(q) == 0 for r in behind_node.replicas.values() for q in r.requestQueues.values()])
assert len(behind_node.clientAuthNr._verified_reqs) == 0
assert len(behind_node.requestSender) == 0
def test_no_propagated_future_view_change_while_view_change(txnPoolNodeSet, looper):
# the last node is a lagging one, which will receive ViewChangeDone messages for future view
viewNo = checkViewNoForNodes(txnPoolNodeSet)
lagged_node = txnPoolNodeSet[-1]
other_nodes = list(set(txnPoolNodeSet) - {lagged_node})
# emulate view change in progress
lagged_node.view_changer.view_change_in_progress = True
old_view_no = checkViewNoForNodes([lagged_node])
initial_vhdc = \
lagged_node.view_changer.spylog.count(lagged_node.view_changer.process_future_view_vchd_msg.__name__)
# delay INSTANCE CHANGE on lagged nodes, so all nodes except the lagging one finish View Change
with delay_rules(lagged_node.nodeIbStasher, icDelay()):
# make sure that View Change happened on all nodes but the lagging one
ensure_view_change(looper, other_nodes)
checkProtocolInstanceSetup(looper=looper, nodes=other_nodes, instances=range(2))
ensure_all_nodes_have_same_data(looper, nodes=other_nodes)
# check that lagged node recived 3 Future VCD, but didn't start new view change
assert len(other_nodes) + initial_vhdc ==\
lagged_node.view_changer.spylog.count(lagged_node.view_changer.process_future_view_vchd_msg.__name__)
assert old_view_no == checkViewNoForNodes([lagged_node])
def test_new_primary_has_wrong_clock(tconf, looper, txnPoolNodeSet,
sdk_wallet_client, sdk_pool_handle):
"""
One of non-primary has a bad clock, it raises suspicions but orders
requests after getting PREPAREs. Then a view change happens this
non-primary with the bad clock becomes the new primary but is not able to
get any of it's PRE-PREPAREs ordered. Eventually another view change
happens and a new primary is elected the pool is functional again
:return:
"""
# The node having the bad clock, this node will be primary after view
# change
faulty_node = getNonPrimaryReplicas(txnPoolNodeSet, 0)[0].node
make_clock_faulty(faulty_node)
assert not faulty_node.master_replica.isPrimary
# faulty_node replies too
ledger_sizes = {
node.name: node.domainLedger.size for node in txnPoolNodeSet}
susp_counts = {node.name: get_timestamp_suspicion_count(
node) for node in txnPoolNodeSet}
ensure_view_change(looper, txnPoolNodeSet)
ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet)
# After view change, faulty_node is primary
assert faulty_node.master_replica.isPrimary
old_view_no = txnPoolNodeSet[0].viewNo
# Delay instance change so view change doesn't happen in the middle of this test
stashers = (n.nodeIbStasher for n in txnPoolNodeSet)
with delay_rules(stashers, icDelay()):
# Requests are sent
for _ in range(5):
sdk_send_random_requests(looper,
sdk_pool_handle,
sdk_wallet_client,
count=2)
looper.runFor(2)
def chk():
for node in txnPoolNodeSet:
assert node.viewNo == old_view_no
for node in [n for n in txnPoolNodeSet if n != faulty_node]:
# Each non faulty node raises suspicion
assert get_timestamp_suspicion_count(node) > susp_counts[node.name]
# Ledger does not change
assert node.domainLedger.size == ledger_sizes[node.name]
assert faulty_node.domainLedger.size == ledger_sizes[faulty_node.name]
looper.run(eventually(chk, retryWait=1))
# Eventually another view change happens
ensure_view_change(looper, txnPoolNodeSet)
looper.run(eventually(checkViewNoForNodes, txnPoolNodeSet, old_view_no + 1,
retryWait=1, timeout=2 * tconf.PerfCheckFreq))
ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet)
# After view change, faulty_node is no more the primary
assert not faulty_node.master_replica.isPrimary
# All nodes reply
sdk_send_random_and_check(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
count=Max3PCBatchSize * 2)
