def test_view_change_by_order_stashed_on_3_nodes_and_catchup_on_1_node(txnPoolNodeSet, looper,
sdk_pool_handle, sdk_wallet_steward):
'''
- COMMITS are delayed on all nodes
- All nodes starts a view change with a prepared certificate (for delayed message)
- COMMITS come during view change for 3 nodes
- So these 3 nodes finish view change by processing Commits and Ordered msgs during view change (in between rounds of catchup).
- The lagging (4th) node receives missing txns as part of catch-up (during view change) and also finishes view change.
'''
slow_node = txnPoolNodeSet[-1]
fast_nodes = txnPoolNodeSet[:-1]
slow_stasher = slow_node.nodeIbStasher
fast_stashers = [n.nodeIbStasher for n in fast_nodes]
all_stashers = [n.nodeIbStasher for n in txnPoolNodeSet]
initial_last_ordered = txnPoolNodeSet[0].master_replica.last_ordered_3pc
txns_count = 4
eventual_last_ordered = initial_last_ordered[0], initial_last_ordered[1] + txns_count
batches_count = initial_last_ordered[1]
with delay_rules(all_stashers, vcd_delay()):
# the lagging node is slow in receiving Commits and Catchup mghs
with delay_rules(slow_stasher, cDelay()):
with delay_rules(slow_stasher, lsDelay(), msg_rep_delay(types_to_delay=[LEDGER_STATUS])):
# fast nodes will receive and order Commits for last_prepared_cert during view change
with delay_rules(fast_stashers, cDelay()):
with delay_rules(fast_stashers, lsDelay(), msg_rep_delay(types_to_delay=[LEDGER_STATUS])):
sdk_send_random_requests(looper, sdk_pool_handle,
sdk_wallet_steward, txns_count)
batches_count += txns_count
looper.run(eventually(check_prepare_certificate, txnPoolNodeSet, batches_count))
check_last_ordered_3pc_on_master(txnPoolNodeSet, initial_last_ordered)
# trigger view change on all nodes
ensure_view_change(looper, txnPoolNodeSet)
looper.run(eventually(check_last_prepared_certificate_after_view_change_start,
txnPoolNodeSet, eventual_last_ordered))
# check that all txns are ordered till last prepared on fast nodes
looper.run(eventually(check_last_ordered_3pc_on_master, fast_nodes, eventual_last_ordered, timeout=30))
# check that all txns are ordered till last prepared on slow node as a result of catchup
looper.run(eventually(check_last_ordered_3pc_on_master, [slow_node], eventual_last_ordered, timeout=30))
# wait for view change done on all nodes
ensureElectionsDone(looper, txnPoolNodeSet)
ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet)
# make sure that the pool is functional
sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_steward, sdk_pool_handle)
def test_catchup_with_reask_cp(txnPoolNodeSet, looper, sdk_pool_handle,
sdk_wallet_steward, tconf, tdir,
allPluginsPath):
'''
Start a catchup
Delay ConsistencyProofs twice
Check that the catchup finished
'''
lagged_node = txnPoolNodeSet[-1]
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, 5)
with delay_rules_without_processing(
lagged_node.nodeIbStasher, delay_3pc(),
msg_rep_delay(types_to_delay=[COMMIT])):
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, 2)
lagged_node.nodeIbStasher.drop_delayeds()
with delay_rules_without_processing(lagged_node.nodeIbStasher, cpDelay()):
lagged_node.start_catchup()
def chk():
cp_count = 0
for msg in lagged_node.nodeIbStasher.delayeds:
if isinstance(msg.item[0], ConsistencyProof):
cp_count += 1
assert cp_count >= (len(txnPoolNodeSet) - 1) * 2
lagged_node.nodeIbStasher.drop_delayeds()
looper.run(eventually(chk))
waitNodeDataEquality(looper,
lagged_node,
*txnPoolNodeSet,
exclude_from_check=['check_last_ordered_3pc_backup'])
def test_dequeue_and_validate_commits(looper, txnPoolNodeSet,
sdk_wallet_client, sdk_pool_handle):
slow_node = [r.node for r in getNonPrimaryReplicas(txnPoolNodeSet, 0)][-1]
other_nodes = [n for n in txnPoolNodeSet if n != slow_node]
delay = 50
with delay_rules(slow_node.nodeIbStasher, pDelay(delay),
msg_rep_delay(delay, [PREPARE, PREPREPARE])):
with delay_rules(slow_node.nodeIbStasher, ppDelay(delay)):
sdk_send_batches_of_random_and_check(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
num_reqs=1,
num_batches=1)
assert not slow_node.master_replica._ordering_service.prePrepares
assert not slow_node.master_replica._ordering_service.prepares
assert not slow_node.master_replica._ordering_service.commits
assert len(slow_node.master_replica._ordering_service.
commitsWaitingForPrepare) > 0
waitNodeDataEquality(looper, slow_node, *other_nodes)
assert check_if_all_equal_in_list([
n.master_replica._ordering_service.ordered for n in txnPoolNodeSet
])
assert slow_node.master_replica._ordering_service.prePrepares
assert slow_node.master_replica._ordering_service.prepares
assert slow_node.master_replica._ordering_service.commits
assert not slow_node.master_replica._ordering_service.commitsWaitingForPrepare
assert all(slow_node.master_replica.last_ordered_3pc ==
n.master_replica.last_ordered_3pc for n in other_nodes)
def test_ordered_request_freed_on_replica_removal(looper, txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
chkFreqPatched, view_change):
node = txnPoolNodeSet[0]
# Stabilize checkpoint
# Send one more request to stabilize checkpoint
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
old_stable_checkpoint = node.master_replica._consensus_data.stable_checkpoint
with delay_rules(node.nodeIbStasher, cDelay(),
msg_rep_delay(types_to_delay=[COMMIT])):
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
f_d, f_r = get_forwarded_to_all(node)
assert f_d
node.replicas.remove_replica(node.replicas.num_replicas - 1)
assert node.requests[f_d].forwardedTo == node.replicas.num_replicas
looper.run(
eventually(check_for_nodes, txnPoolNodeSet, check_stable_checkpoint,
old_stable_checkpoint))
# Send one more request to stabilize checkpoint
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, CHK_FREQ - 1)
looper.run(
eventually(check_for_nodes, txnPoolNodeSet, check_stable_checkpoint,
old_stable_checkpoint + CHK_FREQ))
def test_view_change_with_different_prepare_certificate(
looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client):
"""
Check that a node without pre-prepare but with quorum of prepares wouldn't
use this transaction as a last in prepare certificate
"""
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
slow_node = txnPoolNodeSet[-1]
# delay preprepares and message response with preprepares.
with delay_rules(slow_node.nodeIbStasher, ppDelay(delay=sys.maxsize)):
with delay_rules(
slow_node.nodeIbStasher,
msg_rep_delay(delay=sys.maxsize, types_to_delay=[
PREPREPARE,
])):
last_ordered = slow_node.master_replica.last_ordered_3pc
sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client)
looper.run(
eventually(check_prepare_certificate, txnPoolNodeSet[0:-1],
last_ordered[1] + 1))
for n in txnPoolNodeSet:
n.view_changer.on_master_degradation()
assert slow_node.master_replica.last_prepared_certificate_in_view() == \
(0, last_ordered[1])
ensureElectionsDone(looper, txnPoolNodeSet)
def test_view_change_with_different_prepare_certificate(looper, txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client):
"""
Check that a node without pre-prepare but with quorum of prepares wouldn't
use this transaction as a last in prepare certificate
"""
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
slow_node = txnPoolNodeSet[-1]
# delay preprepares and message response with preprepares.
with delay_rules(slow_node.nodeIbStasher, ppDelay(delay=sys.maxsize)):
with delay_rules(slow_node.nodeIbStasher,
msg_rep_delay(delay=sys.maxsize,
types_to_delay=[PREPREPARE, ])):
last_ordered = slow_node.master_replica.last_ordered_3pc
sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client)
looper.run(eventually(check_prepare_certificate,
txnPoolNodeSet[0:-1],
last_ordered[1] + 1))
for n in txnPoolNodeSet:
n.view_changer.on_master_degradation()
assert slow_node.master_replica.last_prepared_certificate_in_view() == \
(0, last_ordered[1])
ensureElectionsDone(looper, txnPoolNodeSet)
def test_no_preprepare_requested(looper, txnPoolNodeSet, sdk_wallet_client,
sdk_pool_handle, teardown):
"""
Node missing Propagates hence request not finalised, hence stashes
PRE-PREPARE but does not request PRE-PREPARE on receiving PREPARE
"""
slow_node, other_nodes, _, _ = split_nodes(txnPoolNodeSet)
slow_node.nodeIbStasher.delay(ppgDelay())
slow_node.clientIbStasher.delay(req_delay())
slow_node.nodeIbStasher.delay(msg_rep_delay(1000, [
PROPAGATE,
]))
old_count_resp = count_requested_preprepare_resp(slow_node)
sdk_send_batches_of_random_and_check(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
num_reqs=4,
num_batches=2)
# The slow node is behind
checkNodeDataForInequality(slow_node, *other_nodes)
# PRE-PREPARE were not requested
assert count_requested_preprepare_resp(slow_node) == old_count_resp
slow_node.nodeIbStasher.reset_delays_and_process_delayeds()
# The slow node has processed all requests
waitNodeDataEquality(looper, slow_node, *other_nodes)
# PRE-PREPARE were not requested
assert count_requested_preprepare_resp(slow_node) == old_count_resp
def test_no_preprepare_requested(looper, txnPoolNodeSet, client1,
wallet1, client1Connected, teardown):
"""
Node missing Propagates hence request not finalised, hence stashes
PRE-PREPARE but does not request PRE-PREPARE on receiving PREPARE
"""
slow_node, other_nodes, _, _ = split_nodes(txnPoolNodeSet)
slow_node.nodeIbStasher.delay(ppgDelay(20))
slow_node.nodeIbStasher.delay(msg_rep_delay(20, [PROPAGATE, ]))
old_count_resp = count_requested_preprepare_resp(slow_node)
send_reqs_batches_and_get_suff_replies(looper, wallet1, client1, 4, 2)
# The slow node is behind
checkNodeDataForInequality(slow_node, *other_nodes)
# PRE-PREPARE were not requested
assert count_requested_preprepare_resp(slow_node) == old_count_resp
slow_node.nodeIbStasher.reset_delays_and_process_delayeds()
# The slow node has processed all requests
waitNodeDataEquality(looper, slow_node, *other_nodes)
# PRE-PREPARE were not requested
assert count_requested_preprepare_resp(slow_node) == old_count_resp
def test_no_preprepare_requested(looper, txnPoolNodeSet,
sdk_wallet_client, sdk_pool_handle,
teardown):
"""
Node missing Propagates hence request not finalised, hence stashes
PRE-PREPARE but does not request PRE-PREPARE on receiving PREPARE
"""
slow_node, other_nodes, _, _ = split_nodes(txnPoolNodeSet)
slow_node.nodeIbStasher.delay(ppgDelay(20))
slow_node.nodeIbStasher.delay(msg_rep_delay(20, [PROPAGATE, ]))
old_count_resp = count_requested_preprepare_resp(slow_node)
sdk_send_batches_of_random_and_check(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
num_reqs=4,
num_batches=2)
# The slow node is behind
checkNodeDataForInequality(slow_node, *other_nodes)
# PRE-PREPARE were not requested
assert count_requested_preprepare_resp(slow_node) == old_count_resp
slow_node.nodeIbStasher.reset_delays_and_process_delayeds()
# The slow node has processed all requests
waitNodeDataEquality(looper, slow_node, *other_nodes)
# PRE-PREPARE were not requested
assert count_requested_preprepare_resp(slow_node) == old_count_resp
def setup(txnPoolNodeSet):
A, B, C, D = txnPoolNodeSet # type: TestNode
delay(Propagate, frm=[B, C, D], to=A, howlong=howlong)
# Delay MessageRep by long simulating loss as if Propagate is missing, it
# is requested
A.nodeIbStasher.delay(msg_rep_delay(10 * howlong, [
PROPAGATE,
]))
def test_revert_auth_rule_changing(looper, txnPoolNodeSet, sdk_wallet_trustee,
sdk_wallet_steward, sdk_pool_handle):
node_stashers = [n.nodeIbStasher for n in txnPoolNodeSet]
wh, _ = sdk_wallet_trustee
new_steward_did, new_steward_verkey = create_verkey_did(looper, wh)
new_steward_did2, new_steward_verkey2 = create_verkey_did(looper, wh)
"""We try to change rule for adding new steward. For this case we """
changed_constraint = AuthConstraint(role=STEWARD, sig_count=1)
action = AuthActionAdd(txn_type=NYM, field=ROLE, value=STEWARD)
with delay_rules_without_processing(
node_stashers, pDelay(), cDelay(),
msg_rep_delay(types_to_delay=[PREPARE, COMMIT])):
sdk_send_and_check_auth_rule_request(
looper,
sdk_pool_handle,
sdk_wallet_trustee,
auth_action=ADD_PREFIX,
auth_type=action.txn_type,
field=action.field,
new_value=action.value,
old_value=None,
constraint=changed_constraint.as_dict,
no_wait=True)
looper.runFor(waits.expectedPrePrepareTime(len(txnPoolNodeSet)))
"""
Try to add new steward by already existed trustee.
Validation should raise exception because we change uncommitted state
by adding new rule, that "Only steward can add new steward"
"""
with pytest.raises(RequestRejectedException,
match="Not enough STEWARD signatures"):
sdk_add_new_nym(looper,
sdk_pool_handle,
sdk_wallet_trustee,
'newSteward1',
STEWARD_STRING,
dest=new_steward_did,
verkey=new_steward_verkey)
looper.runFor(waits.expectedPrePrepareTime(len(txnPoolNodeSet)))
"""
Catchup should revert config_state and discard rule changing
"""
for n in txnPoolNodeSet:
n.requests.clear()
ensure_view_change_complete(looper, txnPoolNodeSet)
"""
Try to create new steward by steward
We can not do this, because AUTH_RULE txn was reverted
"""
with pytest.raises(RequestRejectedException,
match="Not enough TRUSTEE signatures"):
sdk_add_new_nym(looper,
sdk_pool_handle,
sdk_wallet_steward,
'newSteward2',
STEWARD_STRING,
dest=new_steward_did2,
verkey=new_steward_verkey2)
def test_re_order_pre_prepares_no_pre_prepares(looper, txnPoolNodeSet,
sdk_wallet_client, sdk_pool_handle):
# 1. drop PrePrepars, Prepares and Commits on 4thNode
# Order a couple of requests on Nodes 1-3
lagging_node = txnPoolNodeSet[-1]
other_nodes = txnPoolNodeSet[:-1]
with delay_rules_without_processing(lagging_node.nodeIbStasher, delay_3pc()):
sdk_send_random_and_check(looper, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client, 3)
assert all(n.master_last_ordered_3PC == (0, 3) for n in other_nodes)
with delay_rules_without_processing(lagging_node.nodeIbStasher,
msg_rep_delay(types_to_delay=[PREPREPARE, PREPARE, COMMIT])):
# 2. simulate view change start so that
# all PrePrepares/Prepares/Commits are cleared
# and uncommitted txns are reverted
for n in txnPoolNodeSet:
n.replicas.send_to_internal_bus(ViewChangeStarted(view_no=1))
master_ordering_service = n.master_replica._ordering_service
assert not master_ordering_service.prePrepares
assert not master_ordering_service.prepares
assert not master_ordering_service.commits
ledger = n.db_manager.ledgers[DOMAIN_LEDGER_ID]
state = n.db_manager.states[DOMAIN_LEDGER_ID]
assert len(ledger.uncommittedTxns) == 0
assert ledger.uncommitted_root_hash == ledger.tree.root_hash
assert state.committedHead == state.head
# check that all nodes but the lagging one have old_view_pps stored
for n in other_nodes:
assert n.master_replica._ordering_service.old_view_preprepares
assert not lagging_node.master_replica._ordering_service.old_view_preprepares
# 3. Simulate View Change finish to re-order the same PrePrepare
assert lagging_node.master_last_ordered_3PC == (0, 0)
new_master = txnPoolNodeSet[1]
batches = sorted([preprepare_to_batch_id(pp) for _, pp in
new_master.master_replica._ordering_service.old_view_preprepares.items()])
new_view_msg = NewView(viewNo=0,
viewChanges=[],
checkpoint=None,
batches=batches)
new_view_chk_applied_msg = NewViewCheckpointsApplied(view_no=0,
view_changes=[],
checkpoint=None,
batches=batches)
for n in txnPoolNodeSet:
n.master_replica._consensus_data.new_view = new_view_msg
n.master_replica._consensus_data.prev_view_prepare_cert = batches[-1].pp_seq_no
n.master_replica._ordering_service._bus.send(new_view_chk_applied_msg)
# 4. Make sure that the nodes 1-3 (that already ordered the requests) sent Prepares and Commits so that
# the request was eventually ordered on Node4 as well
waitNodeDataEquality(looper, lagging_node, *other_nodes, customTimeout=60)
assert lagging_node.master_last_ordered_3PC == (0, 4)
sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle)
def test_backups_dont_order_while_reordering(txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
looper):
"""
This test needs to show that for now we stop ordering on backups
until master in reordering state after view_change
Steps:
1. Delay ordering on master replica for collecting requests to reorder after VC
2. Make sure that master didn't order
3. Delay old_view_pp_request and force VC
4. Ensure that all backup replica on all nodes cannot order
because primary waiting for reordering on master
"""
def check_pp_count(node, expected_count, inst_id=0):
assert node.replicas._replicas[inst_id].last_ordered_3pc[1] == expected_count, \
"master last ordered: {}, backup_last_ordered: {}".format(node.master_replica._ordering_service.batches,
node.replicas._replicas[
1]._ordering_service.batches)
# We expect that after VC Gamma will be primary on backup
delayed_node = txnPoolNodeSet[-2]
fast_nodes = [n for n in txnPoolNodeSet if n != delayed_node]
master_pp_seq_no_before = delayed_node.master_replica.last_ordered_3pc[1]
with delay_rules_without_processing(delayed_node.nodeIbStasher,
pDelay(instId=MASTER_REPLICA_INDEX),
cDelay(instId=MASTER_REPLICA_INDEX),
msg_req_delay(),
msg_rep_delay(),
ppDelay(instId=MASTER_REPLICA_INDEX)):
sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, REQS_FOR_REORDERING)
looper.run(eventually(check_pp_count, delayed_node, REQS_FOR_REORDERING, BACKUP_INST_ID))
assert delayed_node.master_replica.last_ordered_3pc[1] == master_pp_seq_no_before
with delay_rules([n.nodeIbStasher for n in txnPoolNodeSet], old_view_pp_request_delay()):
ensure_view_change(looper, txnPoolNodeSet)
# check that view change is finished on all nodes
looper.run(eventually(check_not_in_view_change, txnPoolNodeSet))
# check that delayed node is selected on all fast nodes but not on delayed node
def check_backup_primaries():
assert delayed_node.replicas[BACKUP_INST_ID]._consensus_data.primary_name is None
assert delayed_node.master_replica.last_ordered_3pc[1] == master_pp_seq_no_before
assert all(
n.replicas[BACKUP_INST_ID]._consensus_data.primary_name == generateName(delayed_node.name,
instId=BACKUP_INST_ID)
for n in fast_nodes
)
looper.run(eventually(check_backup_primaries))
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, REQS_FOR_REORDERING)
for node in txnPoolNodeSet:
assert node.replicas._replicas[BACKUP_INST_ID].last_ordered_3pc[1] == 0
def test_stabilize_checkpoint_while_unstashing_when_missing_pre_prepare(
looper, chkFreqPatched, reqs_for_checkpoint, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client):
# Prepare nodes
lagging_node = txnPoolNodeSet[-1]
lagging_master_replcia = lagging_node.master_replica
rest_nodes = txnPoolNodeSet[:-1]
# 1. send enough requests so that just 1 is left for checkpoint stabilization
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, reqs_for_checkpoint - 1)
# 2. delay PrePrepare on 1 node so that prepares and commits will be stashed
with delay_rules(lagging_node.nodeIbStasher, ppDelay()):
with delay_rules(lagging_node.nodeIbStasher, msg_rep_delay()):
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
# all good nodes stabilized checkpoint
looper.run(
eventually(check_for_nodes, rest_nodes,
check_stable_checkpoint, 5))
# bad node received checkpoints from all nodes but didn't stabilize it
looper.run(
eventually(check_for_nodes, [lagging_node],
check_stable_checkpoint, 0))
looper.run(
eventually(check_for_nodes,
[lagging_node], check_received_checkpoint_votes, 5,
len(rest_nodes)))
# bad node has all commits and prepares for the last request stashed
looper.run(
eventually(lambda: assertExp(
(0, CHK_FREQ) in lagging_master_replcia._ordering_service.
preparesWaitingForPrePrepare and len(
lagging_master_replcia._ordering_service.
preparesWaitingForPrePrepare[
(0, CHK_FREQ)]) == len(rest_nodes) - 1)))
looper.run(
eventually(lambda: assertExp(
(0, CHK_FREQ) in lagging_master_replcia._ordering_service.
commitsWaitingForPrepare and len(
lagging_master_replcia._ordering_service.
commitsWaitingForPrepare[
(0, CHK_FREQ)]) == len(rest_nodes))))
# 3. the delayed PrePrepare is processed, and stashed prepares and commits are unstashed
# checkpoint will be stabilized during unstashing, and the request will be ordered
looper.run(
eventually(check_for_nodes, [lagging_node], check_stable_checkpoint,
5))
waitNodeDataEquality(looper, *txnPoolNodeSet, customTimeout=5)
def test_process_three_phase_msg_and_stashed_future_view(
txnPoolNodeSet, looper, tconf, sdk_pool_handle, sdk_wallet_steward):
"""
1. Delay ViewChangeDone messages for the slow_node.
2. Start view change on all nodes.
3. Order a new request.
4. Check that slow_node could not order this request and stashed all 3pc messages
and other nodes ordered.
6. Reset delays.
7. Check that the last request is ordered on the slow_node and stashed messages were removed.
"""
slow_node = txnPoolNodeSet[-1]
fast_nodes = txnPoolNodeSet[:-1]
view_no = slow_node.viewNo
old_stashed = {
inst_id: r.stasher.stash_size(STASH_VIEW_3PC)
for inst_id, r in slow_node.replicas.items()
}
with delay_rules([
slow_node.nodeIbStasher,
], msg_rep_delay(types_to_delay=[PREPREPARE, PREPARE, COMMIT])):
with delay_rules([
slow_node.nodeIbStasher,
], nv_delay()):
for n in txnPoolNodeSet:
n.view_changer.on_master_degradation()
waitForViewChange(looper,
fast_nodes,
expectedViewNo=view_no + 1,
customTimeout=2 * tconf.NEW_VIEW_TIMEOUT)
ensureElectionsDone(looper=looper,
nodes=fast_nodes,
instances_list=range(
fast_nodes[0].requiredNumberOfInstances))
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, 1)
assert slow_node.view_change_in_progress
# 1 - pre-prepare msg
# (len(txnPoolNodeSet) - 2) - prepare msgs
# (len(txnPoolNodeSet) - 1) - commit msgs
stashed_master_messages = 2 * (1 + (len(txnPoolNodeSet) - 2) +
(len(txnPoolNodeSet) - 1))
assert slow_node.master_replica.stasher.stash_size(
STASH_VIEW_3PC) == old_stashed[0] + stashed_master_messages
def chk():
for inst_id, r in slow_node.replicas.items():
assert r.last_ordered_3pc[1] == 2
assert r.stasher.stash_size(STASH_VIEW_3PC) == 0
looper.run(eventually(chk))
waitNodeDataEquality(looper, slow_node, *fast_nodes)
def test_catchup_with_reask_ls(txnPoolNodeSet, looper, sdk_pool_handle,
sdk_wallet_steward, tconf, tdir,
allPluginsPath):
'''
Start a catchup
Delay MessageqResp with LedgerStatuses twice
Check that the catchup finished
'''
lagged_node = txnPoolNodeSet[-1]
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, 5)
lagged_node.nodeIbStasher.delay(msg_rep_delay(types_to_delay=[COMMIT]))
with delay_rules_without_processing(
lagged_node.nodeIbStasher, delay_3pc(),
msg_rep_delay(types_to_delay=[COMMIT])):
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, 2)
lagged_node.nodeIbStasher.drop_delayeds()
with delay_rules_without_processing(
lagged_node.nodeIbStasher, lsDelay(),
msg_rep_delay(types_to_delay=[LEDGER_STATUS])):
lagged_node.start_catchup()
def chk():
resp_ls_count = 0
for msg in lagged_node.nodeIbStasher.delayeds:
if isinstance(
msg.item[0],
MessageRep) and msg.item[0].msg_type == LEDGER_STATUS:
resp_ls_count += 1
assert resp_ls_count >= (len(txnPoolNodeSet) - 1) * 2
lagged_node.nodeIbStasher.drop_delayeds()
looper.run(eventually(chk))
waitNodeDataEquality(looper,
lagged_node,
*txnPoolNodeSet,
exclude_from_check=['check_last_ordered_3pc_backup'])
def setup(txnPoolNodeSet, looper, sdk_pool_handle, sdk_wallet_client):
global initial_ledger_size
A, B, C, D = txnPoolNodeSet # type: TestNode
lagged_node = A
frm = [B, C, D]
delay(Propagate, frm=frm, to=lagged_node, howlong=howlong)
# Delay MessageRep by long simulating loss as if Propagate is missing, it
# is requested
A.nodeIbStasher.delay(msg_rep_delay(10 * howlong, [PROPAGATE, ]))
initial_ledger_size = lagged_node.domainLedger.size
request_couple_json = sdk_send_random_requests(
looper, sdk_pool_handle, sdk_wallet_client, 1)
return request_couple_json
def test_view_change_by_order_stashed_on_all(txnPoolNodeSet, looper,
sdk_pool_handle,
sdk_wallet_steward):
'''
- COMMITS are delayed on all nodes
- All nodes starts a view change with a prepared certificate (for delayed message)
- COMMITS come during view change for all nodes,
- So all nodes finish view change by processing Commits and Ordered msgs during view change (in between rounds of catchup).
'''
all_stashers = [n.nodeIbStasher for n in txnPoolNodeSet]
initial_last_ordered = txnPoolNodeSet[0].master_replica.last_ordered_3pc
txns_count = 4
eventual_last_ordered = initial_last_ordered[
0], initial_last_ordered[1] + txns_count
with delay_rules(all_stashers, vcd_delay()):
with delay_rules(all_stashers, cDelay()):
with delay_rules(all_stashers, lsDelay(),
msg_rep_delay(types_to_delay=[LEDGER_STATUS])):
sdk_send_random_requests(looper, sdk_pool_handle,
sdk_wallet_steward, txns_count)
looper.run(
eventually(check_prepare_certificate, txnPoolNodeSet,
txns_count))
check_last_ordered_3pc_on_master(txnPoolNodeSet,
initial_last_ordered)
# trigger view change on all nodes
ensure_view_change(looper, txnPoolNodeSet)
looper.run(
eventually(
check_last_prepared_certificate_after_view_change_start,
txnPoolNodeSet, eventual_last_ordered))
# check that all txns are ordered till last prepared
looper.run(
eventually(check_last_ordered_3pc_on_master,
txnPoolNodeSet,
eventual_last_ordered,
timeout=30))
# wait for view change done on all nodes
ensureElectionsDone(looper, txnPoolNodeSet)
ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet)
# make sure that the pool is functional
sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_steward,
sdk_pool_handle)
def check_view_change_one_slow_node(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
vc_counts,
slow_node_is_next_primary,
delay_commit=True,
delay_pre_prepare=True):
current_view_no = checkViewNoForNodes(txnPoolNodeSet)
expected_view_no = current_view_no + vc_counts
next_primary = get_next_primary_name(txnPoolNodeSet, expected_view_no)
pretenders = [
r.node for r in getNonPrimaryReplicas(txnPoolNodeSet)
if not r.isPrimary
]
if slow_node_is_next_primary:
delayed_node = [n for n in pretenders if n.name == next_primary][0]
else:
delayed_node = [n for n in pretenders if n.name != next_primary][0]
fast_nodes = [node for node in txnPoolNodeSet if node != delayed_node]
delayers = []
if delay_pre_prepare:
delayers.append(ppDelay())
delayers.append(msg_rep_delay(types_to_delay=[PREPREPARE]))
if delay_commit:
delayers.append(cDelay())
# delay OldViewPrePrepareReply so that slow node doesn't receive PrePrepares before ReOrdering phase finishes
with delay_rules(delayed_node.nodeIbStasher, old_view_pp_reply_delay()):
with delay_rules_without_processing(delayed_node.nodeIbStasher,
*delayers):
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
trigger_view_change(txnPoolNodeSet)
if vc_counts == 2:
for node in txnPoolNodeSet:
node.master_replica.internal_bus.send(
NodeNeedViewChange(current_view_no + 2))
waitForViewChange(looper=looper,
txnPoolNodeSet=txnPoolNodeSet,
expectedViewNo=expected_view_no)
ensureElectionsDone(looper, txnPoolNodeSet, customTimeout=30)
# wait till fast nodes finish re-ordering
looper.run(eventually(check_has_commits, fast_nodes))
sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_client,
sdk_pool_handle)
ensure_all_nodes_have_same_data(looper, txnPoolNodeSet)
def test_unstash_waiting_for_first_batch_ordered_after_catchup(
looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle, tconf):
lagged_node = txnPoolNodeSet[-1]
other_nodes = list(set(txnPoolNodeSet) - {lagged_node})
other_stashers = [n.nodeIbStasher for n in other_nodes]
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
last_ordered_lagged_before = lagged_node.master_last_ordered_3PC
# do not process any message reqs for PrePrepares
with delay_rules_without_processing(
lagged_node.nodeIbStasher,
msg_rep_delay(types_to_delay=[PREPARE, PREPREPARE])):
with delay_rules(lagged_node.nodeIbStasher, cDelay()):
ensure_view_change(looper, txnPoolNodeSet)
looper.run(eventually(check_not_in_view_change, txnPoolNodeSet))
ensureElectionsDone(looper,
other_nodes,
instances_list=range(
getRequiredInstances(len(txnPoolNodeSet))))
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
# delay Commits on all nodes so that there are some PrePrepares still stashed after catchup
with delay_rules(other_stashers, cDelay()):
pre_prep_before = len(recvdPrePrepareForInstId(lagged_node, 0))
sdk_send_random_requests(looper, sdk_pool_handle,
sdk_wallet_client, 2)
# wait till lagged node recives the new PrePrepares
# they will be stashed as WAITING_FIRST_BATCH_IN_VIEW
looper.run(
eventually(lambda: assertExp(
len(recvdPrePrepareForInstId(lagged_node, 0)) ==
pre_prep_before + 2)))
# catchup the lagged node
# the latest 2 PrePrepares are still stashed
lagged_node.start_catchup()
looper.run(
eventually(
lambda: assertExp(lagged_node.master_last_ordered_3PC >
last_ordered_lagged_before)))
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 2)
ensureElectionsDone(looper, txnPoolNodeSet, customTimeout=30)
ensure_all_nodes_have_same_data(looper, txnPoolNodeSet, custom_timeout=30)
def setup(txnPoolNodeSet, looper, sdk_pool_handle, sdk_wallet_client):
global initial_ledger_size
A, B, C, D = txnPoolNodeSet # type: TestNode
lagged_node = A
frm = [B, C, D]
delay(Propagate, frm=frm, to=lagged_node, howlong=howlong)
# Delay MessageRep by long simulating loss as if Propagate is missing, it
# is requested
A.nodeIbStasher.delay(msg_rep_delay(10 * howlong, [
PROPAGATE,
]))
initial_ledger_size = lagged_node.domainLedger.size
request_couple_json = sdk_send_random_requests(looper, sdk_pool_handle,
sdk_wallet_client, 1)
return request_couple_json
def setup(txnPoolNodeSet, looper, sdk_pool_handle, sdk_wallet_client,):
def _clean(*args):
pass
A, B, C, D = txnPoolNodeSet # type: TestNode
delay(Propagate, frm=[B, C, D], to=A, howlong=howlong)
# Delay MessageRep by long simulating loss as if Propagate is missing, it
# is requested
A.nodeIbStasher.delay(msg_rep_delay(10 * howlong, [PROPAGATE, ]))
# disable _clean method which remove req.key from requests map
A.requests._clean = types.MethodType(
_clean, A.requests)
request_couple_json = sdk_send_random_requests(
looper, sdk_pool_handle, sdk_wallet_client, reqCount)
return request_couple_json
def test_unordered_request_freed_on_replica_removal(looper, txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
chkFreqPatched,
view_change):
node = txnPoolNodeSet[0]
# Stabilize checkpoint
# Send one more request to stabilize checkpoint
sdk_send_random_and_check(
looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client,
CHK_FREQ - get_pp_seq_no(txnPoolNodeSet) % CHK_FREQ)
old_stable_checkpoint = node.master_replica._consensus_data.stable_checkpoint
stashers = [n.nodeIbStasher for n in txnPoolNodeSet]
with delay_rules(stashers, cDelay(delay=sys.maxsize),
msg_rep_delay(types_to_delay=[COMMIT])):
req = sdk_send_random_requests(looper, sdk_pool_handle,
sdk_wallet_client, 1)
looper.runFor(
waits.expectedPropagateTime(len(txnPoolNodeSet)) +
waits.expectedPrePrepareTime(len(txnPoolNodeSet)) +
waits.expectedPrepareTime(len(txnPoolNodeSet)) +
waits.expectedCommittedTime(len(txnPoolNodeSet)))
f_d, f_r = get_forwarded_to_all(node)
assert f_d
node.replicas.remove_replica(node.replicas.num_replicas - 1)
assert node.requests[f_d].forwardedTo == node.replicas.num_replicas
check_for_nodes(txnPoolNodeSet, check_stable_checkpoint,
old_stable_checkpoint)
sdk_get_replies(looper, req)
check_for_nodes(txnPoolNodeSet, check_stable_checkpoint,
old_stable_checkpoint)
# Send one more request to stabilize checkpoint
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, CHK_FREQ - 1)
looper.run(
eventually(check_for_nodes, txnPoolNodeSet, check_stable_checkpoint,
old_stable_checkpoint + CHK_FREQ))
assert len(node.requests) == 0
def check_view_change_adding_new_node(looper,
tdir,
tconf,
allPluginsPath,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
sdk_wallet_steward,
slow_nodes=[],
delay_commit=False,
delay_pre_prepare=False):
# Pre-requisites: viewNo=3, Primary is Node4
for viewNo in range(1, 4):
trigger_view_change(txnPoolNodeSet)
waitForViewChange(looper, txnPoolNodeSet, viewNo)
ensureElectionsDone(looper, txnPoolNodeSet, customTimeout=30)
# Delay 3PC messages on slow nodes
fast_nodes = [node for node in txnPoolNodeSet if node not in slow_nodes]
slow_stashers = [slow_node.nodeIbStasher for slow_node in slow_nodes]
delayers = []
if delay_pre_prepare:
delayers.append(ppDelay())
delayers.append(msg_rep_delay(types_to_delay=[PREPREPARE]))
if delay_commit:
delayers.append(cDelay())
with delay_rules_without_processing(slow_stashers, *delayers):
# Add Node5
new_node = add_new_node(looper, fast_nodes, sdk_pool_handle,
sdk_wallet_steward, tdir, tconf,
allPluginsPath)
old_set = list(txnPoolNodeSet)
txnPoolNodeSet.append(new_node)
# Trigger view change
trigger_view_change(txnPoolNodeSet)
# make sure view change is finished eventually
waitForViewChange(looper, old_set, 4)
ensureElectionsDone(looper, old_set)
sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_client,
sdk_pool_handle)
def setup(
txnPoolNodeSet,
looper,
sdk_pool_handle,
sdk_wallet_client,
):
def _clean():
pass
A, B, C, D = txnPoolNodeSet # type: TestNode
delay(Propagate, frm=[B, C, D], to=A, howlong=howlong)
# Delay MessageRep by long simulating loss as if Propagate is missing, it
# is requested
A.nodeIbStasher.delay(msg_rep_delay(10 * howlong, [
PROPAGATE,
]))
# disable _clean method which remove req.key from requests map
A.requests._clean = types.MethodType(_clean, A.requests)
request_couple_json = sdk_send_random_requests(looper, sdk_pool_handle,
sdk_wallet_client, 1)
return request_couple_json
def test_view_change_after_back_to_quorum_with_disconnected_primary(
txnPoolNodeSet, looper, sdk_pool_handle, sdk_wallet_client, tdir,
tconf, allPluginsPath):
assert len(txnPoolNodeSet) == 4
pr_node = get_master_primary_node(txnPoolNodeSet)
assert pr_node.name == "Alpha"
# 1. Initiate view change be primary (Alpha) restart
nodes = ensure_view_change_by_primary_restart(
looper,
txnPoolNodeSet,
tconf,
tdir,
allPluginsPath,
customTimeout=2 * tconf.NEW_VIEW_TIMEOUT,
exclude_from_check=['check_last_ordered_3pc_backup'])
# Now primary should be Beta
pr_node = get_master_primary_node(nodes)
assert pr_node.name == "Beta"
# 2. Stop non-primary node Delta, no any view changes are expected
non_primary_to_stop = [n for n in nodes if n.name == "Delta"][0]
disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet,
non_primary_to_stop)
looper.removeProdable(non_primary_to_stop)
remaining_nodes = list(set(nodes) - {non_primary_to_stop})
# Primary is going to be stopped, remember instance change messages count
# to ensure that no view change happened as number of connected nodes is less
# than quorum.
ic_cnt = {}
for n in remaining_nodes:
ic_cnt[n.name] = n.view_changer.spylog.count(
ViewChanger.sendInstanceChange.__name__)
# 3. Disconnect primary
disconnect_node_and_ensure_disconnected(looper, remaining_nodes, pr_node)
looper.removeProdable(pr_node)
# Wait for more than ToleratePrimaryDisconnection timeout and check that no IC messages presented.
looper.runFor(tconf.ToleratePrimaryDisconnection + 5)
remaining_nodes = list(set(remaining_nodes) - {pr_node})
for n in remaining_nodes:
assert ic_cnt[n.name] == n.view_changer.spylog.count(
ViewChanger.sendInstanceChange.__name__)
view_no = checkViewNoForNodes(remaining_nodes)
# 4. Start Delta (non-primary), now primary (Beta) is disconnected but there is a quorum
# to choose a new one.
restartedNode = start_stopped_node(non_primary_to_stop,
looper,
tconf,
tdir,
allPluginsPath,
delay_instance_change_msgs=False,
start=False)
# 5. delay catchup on Delta for more that INITIAL_PROPOSE_VIEW_CHANGE_TIMEOUT,
# so that Delta proposes INSTANCE_CHANGE from view=0 to view=1
# (it doesn't yet know that the current view is 1, since it hasn't yet finished catchup)
with delay_rules(restartedNode.nodeIbStasher,
msg_rep_delay(types_to_delay=[LEDGER_STATUS])):
looper.add(restartedNode)
looper.runFor(tconf.INITIAL_PROPOSE_VIEW_CHANGE_TIMEOUT + 5)
remaining_nodes = remaining_nodes + [restartedNode]
# 6. Check that view change happened eventually because
# Delta re-send InstanceChang for view=2 after it finished catchup
waitForViewChange(looper,
remaining_nodes,
expectedViewNo=(view_no + 1),
customTimeout=3 * tconf.NEW_VIEW_TIMEOUT)
# 7. ensure pool is working properly
sdk_send_random_and_check(looper, remaining_nodes, sdk_pool_handle,
sdk_wallet_client, 3)
ensure_all_nodes_have_same_data(looper, nodes=remaining_nodes)
def test_reverted_unordered(txnPoolNodeSet, looper, sdk_pool_handle, sdk_wallet_client):
"""
Before starting catchup, revert any uncommitted changes to state and
ledger. This is to avoid any re-application of requests that were
ordered but stashed
Example scenario
prepared (1, 4)
startViewChange
start_catchup
...
....
...
committed and send Ordered (1, 2)
...
....
preLedgerCatchUp
force_process_ordered, take out (1,2) and stash (1, 2)
now process stashed Ordered(1,2), its requests will be applied again
Simulation: Delay COMMITs to a node so that it can not order requests
but has prepared them. Then trigger a view change and make sure the slow
node has not ordered same number of requests as others but has prepared
so it can order when it receives COMMITs while view change is in progress.
The slow node should revert unordered batches and but it should eventually
process the ordered requests, so delay LEDGER_STATUS too so catchup
is delayed
"""
slow_node = getNonPrimaryReplicas(txnPoolNodeSet, 0)[-1].node
fast_nodes = [n for n in txnPoolNodeSet if n != slow_node]
slow_node.nodeIbStasher.delay(cDelay(120, 0))
sent_batches = 5
sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 2 * sent_batches, sent_batches)
# Fast nodes have same last ordered and same data
last_ordered = [n.master_last_ordered_3PC for n in fast_nodes]
assert check_if_all_equal_in_list(last_ordered)
ensure_all_nodes_have_same_data(looper, fast_nodes)
# Slow nodes have different last ordered than fast nodes
assert last_ordered[0] != slow_node.master_last_ordered_3PC
# Delay LEDGER_STATUS so catchup starts late
slow_node.nodeIbStasher.delay(lsDelay(100))
slow_node.nodeIbStasher.delay(msg_rep_delay(100))
# slow_node has not reverted batches
assert sent_batches not in getAllReturnVals(
slow_node.master_replica,
slow_node.master_replica.revert_unordered_batches)
ensure_view_change(looper, txnPoolNodeSet)
def chk1():
# slow_node reverted all batches
rv = getAllReturnVals(slow_node.master_replica,
slow_node.master_replica.revert_unordered_batches)
assert sent_batches in rv
looper.run(eventually(chk1, retryWait=1))
# After the view change slow_node has prepared same requests as the fast
# nodes have ordered
assert last_ordered[0] == slow_node.master_replica.last_prepared_before_view_change
# Deliver COMMITs
slow_node.nodeIbStasher.reset_delays_and_process_delayeds(COMMIT)
def chk2():
# slow_node orders all requests as others have
assert last_ordered[0] == slow_node.master_last_ordered_3PC
looper.run(eventually(chk2, retryWait=1))
# Deliver LEDGER_STATUS so catchup can complete
slow_node.nodeIbStasher.reset_delays_and_process_delayeds(LEDGER_STATUS)
slow_node.nodeIbStasher.reset_delays_and_process_delayeds(MESSAGE_RESPONSE)
# Ensure all nodes have same data
ensure_all_nodes_have_same_data(looper, txnPoolNodeSet)
ensureElectionsDone(looper, txnPoolNodeSet)
def chk3():
# slow_node processed stashed Ordered requests successfully
rv = getAllReturnVals(slow_node,
slow_node.processStashedOrderedReqs)
assert sent_batches in rv
looper.run(eventually(chk3, retryWait=1))
# Ensure pool is functional
sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 10, 2)
ensure_all_nodes_have_same_data(looper, txnPoolNodeSet)
def test_old_view_pre_prepare_reply_processing(looper, txnPoolNodeSet, tconf,
allPluginsPath, sdk_pool_handle,
sdk_wallet_steward,
monkeypatch):
"""
Test steps:
Delay PrePrepares on `slow_node` (without processing)
Delay receiving of OldViewPrePrepareRequest on all nodes but `malicious_node`
Patch sending for OldViewPrePrepareReply on the `malicious_node` to send an invalid PrePrepare
Start a view change
Make sure it's finished on all nodes excluding `slow_node`
Make sure that the lagging node received OldViewPrePrepareReply from the malicious node
Reset delay for OldViewPrePrepareRequest on other nodes
Make sure the pool is functional and all nodes have same data
"""
start_view_no = txnPoolNodeSet[0].viewNo
slow_node = txnPoolNodeSet[-2]
malicious_node = txnPoolNodeSet[-1]
other_nodes = [
n for n in txnPoolNodeSet if n not in [slow_node, malicious_node]
]
ensureElectionsDone(looper,
txnPoolNodeSet,
customTimeout=tconf.NEW_VIEW_TIMEOUT)
timeout = waits.expectedPoolCatchupTime(nodeCount=len(txnPoolNodeSet))
ensure_all_nodes_have_same_data(looper,
txnPoolNodeSet,
custom_timeout=timeout)
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, 1)
with delay_rules_without_processing(
slow_node.nodeIbStasher, ppDelay(),
msg_rep_delay(types_to_delay=[PREPREPARE])):
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, 1)
with delay_rules([n.nodeIbStasher for n in other_nodes],
old_view_pp_request_delay()):
old_sender = malicious_node.master_replica._ordering_service._send
def patched_sender(msg, dst=None, stat=None):
if isinstance(msg, OldViewPrePrepareReply) and msg.preprepares:
pp_dict = msg.preprepares[0]._asdict()
pp_dict["digest"] = "incorrect_digest"
pp = PrePrepare(**pp_dict)
msg.preprepares[0] = pp
monkeypatch.undo()
old_sender(msg, dst, stat)
monkeypatch.setattr(malicious_node.master_replica._ordering_service,
'_send', patched_sender)
monkeypatch.setattr(slow_node.master_replica._ordering_service,
'_validate_applied_pre_prepare',
lambda a, b, c: None)
process_old_pp_num = slow_node.master_replica._ordering_service.spylog.count(
OrderingService.process_old_view_preprepare_reply)
for n in txnPoolNodeSet:
n.view_changer.on_master_degradation()
waitForViewChange(looper,
other_nodes + [malicious_node],
expectedViewNo=start_view_no + 1)
ensureElectionsDone(looper=looper,
nodes=other_nodes + [malicious_node],
instances_list=[0, 1, 2])
ensure_all_nodes_have_same_data(looper,
nodes=other_nodes + [malicious_node])
def chk():
assert process_old_pp_num + 1 == slow_node.master_replica._ordering_service.spylog.count(
OrderingService.process_old_view_preprepare_reply)
looper.run(eventually(chk))
ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet)
sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_steward,
sdk_pool_handle)
def test_reverted_unordered(txnPoolNodeSet, looper, sdk_pool_handle,
sdk_wallet_client):
"""
Before starting catchup, revert any uncommitted changes to state and
ledger. This is to avoid any re-application of requests that were
ordered but stashed
Example scenario
prepared (1, 4)
start_view_change
start_catchup
...
....
...
committed and send Ordered (1, 2)
...
....
preLedgerCatchUp
force_process_ordered, take out (1,2) and stash (1, 2)
now process stashed Ordered(1,2), its requests will be applied again
Simulation: Delay COMMITs to a node so that it can not order requests
but has prepared them. Then trigger a view change and make sure the slow
node has not ordered same number of requests as others but has prepared
so it can order when it receives COMMITs while view change is in progress.
The slow node should revert unordered batches and but it should eventually
process the ordered requests, so delay LEDGER_STATUS too so catchup
is delayed
"""
slow_node = getNonPrimaryReplicas(txnPoolNodeSet, 0)[-1].node
fast_nodes = [n for n in txnPoolNodeSet if n != slow_node]
slow_node.nodeIbStasher.delay(cDelay(120, 0))
sent_batches = 5
sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client,
2 * sent_batches, sent_batches)
# Fast nodes have same last ordered and same data
last_ordered = [n.master_last_ordered_3PC for n in fast_nodes]
assert check_if_all_equal_in_list(last_ordered)
ensure_all_nodes_have_same_data(looper, fast_nodes)
# Slow nodes have different last ordered than fast nodes
assert last_ordered[0] != slow_node.master_last_ordered_3PC
# Delay LEDGER_STATUS so catchup starts late
slow_node.nodeIbStasher.delay(lsDelay(100))
slow_node.nodeIbStasher.delay(msg_rep_delay(100))
slow_node.nodeIbStasher.delay(cr_delay(100))
# slow_node has not reverted batches
assert sent_batches not in getAllReturnVals(
slow_node.master_replica,
slow_node.master_replica.revert_unordered_batches)
ensure_view_change(looper, txnPoolNodeSet)
def chk1():
# slow_node reverted all batches
rv = getAllReturnVals(
slow_node.master_replica,
slow_node.master_replica.revert_unordered_batches)
assert sent_batches in rv
looper.run(eventually(chk1, retryWait=1))
# After the view change slow_node has prepared same requests as the fast
# nodes have ordered
assert last_ordered[
0] == slow_node.master_replica.last_prepared_before_view_change
# Deliver COMMITs
slow_node.nodeIbStasher.reset_delays_and_process_delayeds(COMMIT)
def chk2():
# slow_node stashed commits
assert slow_node.master_replica.stasher.num_stashed_catchup == \
sent_batches * (len(txnPoolNodeSet) - 1)
looper.run(eventually(chk2, retryWait=1))
# Deliver LEDGER_STATUS so catchup can complete
slow_node.nodeIbStasher.reset_delays_and_process_delayeds(LEDGER_STATUS)
slow_node.nodeIbStasher.reset_delays_and_process_delayeds(MESSAGE_RESPONSE)
slow_node.nodeIbStasher.reset_delays_and_process_delayeds(CATCHUP_REP)
# Ensure all nodes have same data
ensure_all_nodes_have_same_data(looper, txnPoolNodeSet)
ensureElectionsDone(looper, txnPoolNodeSet)
def chk3():
# slow_node processed stashed messages successfully
assert slow_node.master_replica.stasher.num_stashed_catchup == 0
looper.run(eventually(chk3, retryWait=1))
# Ensure pool is functional
sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client,
10, 2)
ensure_all_nodes_have_same_data(looper, txnPoolNodeSet)
def test_unstash_three_phase_msg_after_catchup(txnPoolNodeSet, looper, tconf,
sdk_pool_handle,
sdk_wallet_steward):
"""
1. Delay Commit on Node4
2. Order 1 req
3. Delay Commit on all nodes
4. Order 1 req
5. Delay CatchupRep on Node4
6. Delay Ledger Status and ViewChangeDones on Nodes1-3
7. Start View change on all nodes
8. Wait until Node4 got 3 stashed CatchupReps
9. Reset delaying of Commits on all Nodes
10. Reset Ledger Status on Nodes1-3
11. Check that 3 nodes finished VC while Node4 is syncing and not finished
12. Reset CatchupRep on Node4
13. Check that Node4 finished VC, and there was just 1 round of catch-up
"""
slow_node = txnPoolNodeSet[-1]
fast_nodes = txnPoolNodeSet[:-1]
view_no = txnPoolNodeSet[0].viewNo
old_stashed = slow_node.master_replica.stasher.stash_size(STASH_VIEW_3PC)
last_ordered = txnPoolNodeSet[0].master_replica.last_ordered_3pc
batches_count = last_ordered[1]
with delay_rules(
[n.nodeIbStasher for n in txnPoolNodeSet],
msg_rep_delay(types_to_delay=[PREPREPARE, PREPARE, COMMIT])):
# Delay Commit messages for slow_node.
slow_node.nodeIbStasher.delay(cDelay(sys.maxsize))
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, 1)
batches_count += 1
# Delay Commit messages for fast_nodes.
for n in fast_nodes:
n.nodeIbStasher.delay(cDelay(sys.maxsize))
request2 = sdk_send_random_request(looper, sdk_pool_handle,
sdk_wallet_steward)
batches_count += 1
def check_commits(commit_key):
for n in fast_nodes:
for r in n.replicas.values():
assert commit_key in r._ordering_service.commits
assert len(
r._ordering_service.commits[commit_key].voters) == 1
looper.run(
eventually(check_commits,
(view_no, last_ordered[1] + batches_count)))
# Delay CatchupRep messages for the slow_node.
with delay_rules([slow_node.nodeIbStasher], cr_delay()):
with delay_rules([n.nodeIbStasher for n in fast_nodes],
msg_rep_delay(types_to_delay=[LEDGER_STATUS])):
for n in txnPoolNodeSet:
n.start_catchup()
looper.run(
eventually(
lambda: assertExp(slow_node.mode == Mode.discovering)))
# Reset delay Commit messages for all nodes.
for n in txnPoolNodeSet:
n.nodeIbStasher.reset_delays_and_process_delayeds(COMMIT)
assert slow_node.mode == Mode.discovering
looper.run(
eventually(_check_nodes_stashed, fast_nodes, old_stashed,
len(txnPoolNodeSet) - 1))
looper.run(
eventually(_check_nodes_stashed, [slow_node], old_stashed,
(len(txnPoolNodeSet) - 1) * 2))
sdk_get_and_check_replies(looper, [request2])
_check_nodes_stashed(fast_nodes, old_stashed, 0)
assert get_pp_seq_no(txnPoolNodeSet) == batches_count
ensure_all_nodes_have_same_data(looper, txnPoolNodeSet)
sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_steward,
sdk_pool_handle)
def test_restart_node_with_view_changes(tdir, tconf, looper, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client,
allPluginsPath):
'''
1. Stop the node Delta
2. Patch methods for processing VCStartMsgStrategy messages
3. Delay CurrentState messages on Delta
4. Start Delta
5. Start view change with a maser degradation reason (from view 0 to 1)
6. Check that Delta start VCStartMsgStrategy after quorum of InstanceChanges
7. Reset delay for CurrentStates
8. Check that propagate primary happened.
9. Unpatch VCStartMsgStrategy methods and process catching messages.
10. Start view change with a maser degradation reason (from view 1 to 2)
11. Check that all nodes has viewNo = 2 and can order transactions.
'''
# Prepare nodes
lagging_node = txnPoolNodeSet[-1]
rest_nodes = txnPoolNodeSet[:-1]
start_view_no = lagging_node.viewNo
# Stop Delta
waitNodeDataEquality(looper, lagging_node, *rest_nodes)
disconnect_node_and_ensure_disconnected(looper,
txnPoolNodeSet,
lagging_node,
stopNode=True)
looper.removeProdable(lagging_node)
# Send more requests to active nodes
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client,
len(rest_nodes) * 3)
waitNodeDataEquality(looper, *rest_nodes)
# Restart stopped node
lagging_node = start_stopped_node(
lagging_node,
looper,
tconf,
tdir,
allPluginsPath,
start=False,
)
# Add to lagging_node node route a patched method for processing
# ViewChangeStartMessage to delay processing.
global view_change_started_messages
view_change_started_messages = []
def patch_on_view_change_started(node, msg, frm):
view_change_started_messages.append((node, msg, frm))
processor = partial(patch_on_view_change_started, lagging_node)
lagging_node.nodeMsgRouter.add((ViewChangeStartMessage, processor))
# Delay CurrentState messages on lagging_node to delay propagate primary
with delay_rules(lagging_node.nodeIbStasher, msg_rep_delay()):
# Add lagging_node to pool
looper.add(lagging_node)
txnPoolNodeSet[-1] = lagging_node
looper.run(checkNodesConnected(txnPoolNodeSet))
looper.run(
eventually(lambda: assertExp(
len(lagging_node.nodeIbStasher.delayeds) >= 3)))
# Start ViewChange (0 -> 1)
for n in rest_nodes:
n.view_changer.on_master_degradation()
# Lagging node still did not catchup, so it can't participate and process I_CH
looper.run(
eventually(
lambda: assertExp(len(view_change_started_messages) == 0)))
# Lagging node catches up till old view
looper.run(
eventually(lambda: assertExp(lagging_node.viewNo == start_view_no)))
# Unpatch ViewChangeStartMessages processing and process delayed messages
processor = partial(VCStartMsgStrategy.on_view_change_started,
lagging_node)
lagging_node.nodeMsgRouter.add((ViewChangeStartMessage, processor))
for msg in view_change_started_messages:
lagging_node.view_changer.node.nodeInBox.append(
(msg[1], lagging_node.view_changer.node.name))
waitForViewChange(looper,
txnPoolNodeSet,
expectedViewNo=start_view_no + 1,
customTimeout=waits.expectedPoolViewChangeStartedTimeout(
len(txnPoolNodeSet)))
# Start ViewChange (1 -> 2)
for n in rest_nodes:
n.view_changer.on_master_degradation()
waitForViewChange(looper,
txnPoolNodeSet,
expectedViewNo=start_view_no + 2,
customTimeout=waits.expectedPoolViewChangeStartedTimeout(
len(txnPoolNodeSet)))
ensureElectionsDone(looper=looper,
nodes=txnPoolNodeSet,
instances_list=range(
txnPoolNodeSet[0].requiredNumberOfInstances))
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
waitNodeDataEquality(looper, *txnPoolNodeSet)
def test_3pc_while_catchup_with_chkpoints_only(
tdir, tconf, looper, chkFreqPatched, reqs_for_checkpoint,
testNodeClass, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client,
allPluginsPath):
'''
Check that catch-up is not started again even if a quorum of stashed checkpoints
is received during catch-up.
Assume that only checkpoints and no 3PC messages are received.
'''
# Prepare nodes
lagging_node = txnPoolNodeSet[-1]
rest_nodes = txnPoolNodeSet[:-1]
# Check that requests executed well
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
# Stop one node
waitNodeDataEquality(looper, lagging_node, *rest_nodes)
disconnect_node_and_ensure_disconnected(looper,
txnPoolNodeSet,
lagging_node,
stopNode=True)
looper.removeProdable(lagging_node)
# Send more requests to active nodes
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
waitNodeDataEquality(looper, *rest_nodes)
# Restart stopped node and wait for successful catch up
lagging_node = start_stopped_node(
lagging_node,
looper,
tconf,
tdir,
allPluginsPath,
start=False,
)
initial_all_ledgers_caught_up = lagging_node.spylog.count(
Node.allLedgersCaughtUp)
# delay all 3PC messages on the lagged node so that it
# receives only Checkpoints and catch-up messages
lagging_node.nodeIbStasher.delay(ppDelay())
lagging_node.nodeIbStasher.delay(pDelay())
lagging_node.nodeIbStasher.delay(cDelay())
# delay CurrentState to avoid Primary Propagation (since it will lead to more catch-ups not needed in this test).
with delay_rules(lagging_node.nodeIbStasher, cs_delay()):
with delay_rules(lagging_node.nodeIbStasher, lsDelay(),
msg_rep_delay()):
looper.add(lagging_node)
txnPoolNodeSet[-1] = lagging_node
looper.run(checkNodesConnected(txnPoolNodeSet))
# wait till we got ledger statuses for messages missed while the node was offline,
# so that now we can order more messages, and they will not be caught up, but stashed
looper.run(
eventually(lambda: assertExp(
lagging_node.nodeIbStasher.num_of_stashed(LedgerStatus) >=
3),
retryWait=1,
timeout=60))
assert lagging_node.mode != Mode.participating
# make sure that more requests are being ordered while catch-up is in progress
# stash enough stable checkpoints for starting a catch-up
num_checkpoints = Replica.STASHED_CHECKPOINTS_BEFORE_CATCHUP + 1
num_reqs = reqs_for_checkpoint * num_checkpoints + 1
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, num_reqs)
looper.run(
eventually(check_last_ordered_3pc_on_all_replicas, rest_nodes,
(0, num_reqs + 2)))
# all good nodes stabilized checkpoint
looper.run(eventually(chkChkpoints, rest_nodes, 2, 0))
assert lagging_node.mode != Mode.participating
# lagging node is catching up and stashing all checkpoints
looper.run(
eventually(lambda: assertExp(
get_stashed_checkpoints(lagging_node) == num_checkpoints *
len(rest_nodes)),
timeout=waits.expectedPoolCatchupTime(
len(txnPoolNodeSet))))
# check that last_ordered is set
looper.run(
eventually(check_last_ordered_3pc_on_master, [lagging_node],
(0, num_reqs + 2)))
# check that the catch-up is finished
looper.run(
eventually(
lambda: assertExp(lagging_node.mode == Mode.participating),
retryWait=1,
timeout=waits.expectedPoolCatchupTime(len(txnPoolNodeSet))))
# check that catch-up was started only once
assert lagging_node.spylog.count(
Node.allLedgersCaughtUp) == initial_all_ledgers_caught_up + 1
looper.run(
eventually(lambda: assertExp(
lagging_node.spylog.count(Node.allLedgersCaughtUp) ==
initial_all_ledgers_caught_up + 1)))
looper.run(
eventually(lambda: assertExp(
lagging_node.spylog.count(Node.start_catchup) == 1)))
# do not check for audit ledger since we didn't catch-up audit ledger when txns were ordering durinf catch-up
waitNodeDataEquality(looper,
*txnPoolNodeSet,
exclude_from_check='check_audit',
customTimeout=5)
def test_limited_stash_3pc_while_catchup(tdir, tconf, looper, testNodeClass,
txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, allPluginsPath,
chkFreqPatched):
'''
Test that the lagging_node can process messages from catchup stash after catchup
and request lost messages from other nodes.
'''
# Prepare nodes
lagging_node = txnPoolNodeSet[-1]
rest_nodes = txnPoolNodeSet[:-1]
# Check that requests executed well
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
# Stop one node
waitNodeDataEquality(looper, lagging_node, *rest_nodes)
disconnect_node_and_ensure_disconnected(looper,
txnPoolNodeSet,
lagging_node,
stopNode=True)
looper.removeProdable(lagging_node)
# Order 2 checkpoints on rest_nodes (2 txns in 2 batches)
sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client,
2 * CHK_FREQ, 2)
waitNodeDataEquality(looper, *rest_nodes)
# Restart stopped node and wait for successful catch up
lagging_node = start_stopped_node(
lagging_node,
looper,
tconf,
tdir,
allPluginsPath,
start=False,
)
initial_all_ledgers_caught_up = lagging_node.spylog.count(
Node.allLedgersCaughtUp)
with delay_rules(lagging_node.nodeIbStasher, cs_delay(),
msg_rep_delay(types_to_delay=[PREPARE, PREPREPARE])):
with delay_rules(lagging_node.nodeIbStasher,
cr_delay(ledger_filter=DOMAIN_LEDGER_ID)):
looper.add(lagging_node)
txnPoolNodeSet[-1] = lagging_node
looper.run(checkNodesConnected(txnPoolNodeSet))
# Order 2 checkpoints in the first lagging node catchup (2 txns in 2 batches)
sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
2 * CHK_FREQ, 2)
# Check that firs txn was ordered from stash after first catchup
looper.run(
eventually(lambda: assertExp(lagging_node.master_last_ordered_3PC[
1] == txnPoolNodeSet[0].master_last_ordered_3PC[1] - 1),
retryWait=1,
timeout=waits.expectedPoolCatchupTime(
len(txnPoolNodeSet))))
# Order 2 checkpoints in the second lagging node catchup (2 txns in 2 batches)
sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client, 2 * CHK_FREQ,
2)
waitNodeDataEquality(looper, *txnPoolNodeSet, customTimeout=5)
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
looper.run(
eventually(lambda: assertExp(lagging_node.master_last_ordered_3PC == n.
master_last_ordered_3PC
for n in txnPoolNodeSet)))
# check that catch-up was started only twice
assert lagging_node.spylog.count(
Node.allLedgersCaughtUp) == initial_all_ledgers_caught_up + 2
def test_primary_send_incorrect_pp(looper, txnPoolNodeSet, tconf,
allPluginsPath, sdk_pool_handle,
sdk_wallet_steward, monkeypatch):
"""
Test steps:
Delay message requests with PrePrepares on `slow_node`
Patch sending for PrePrepare on the `malicious_primary` to send an invalid PrePrepare to slow_node
Order a new request
Start a view change
Make sure it's finished on all nodes
Make sure that the lagging node has same data with other nodes
"""
start_view_no = txnPoolNodeSet[0].viewNo
slow_node = txnPoolNodeSet[-1]
malicious_primary = txnPoolNodeSet[0]
other_nodes = [
n for n in txnPoolNodeSet if n not in [slow_node, malicious_primary]
]
timeout = waits.expectedPoolCatchupTime(nodeCount=len(txnPoolNodeSet))
ensure_all_nodes_have_same_data(looper,
txnPoolNodeSet,
custom_timeout=timeout)
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, 1)
old_sender = malicious_primary.master_replica._ordering_service._send
def patched_sender(msg, dst=None, stat=None):
if isinstance(msg, PrePrepare) and msg:
old_sender(msg, [n.name for n in other_nodes], stat)
pp_dict = msg._asdict()
pp_dict["ppTime"] += 1
pp = PrePrepare(**pp_dict)
old_sender(pp, [slow_node.name], stat)
monkeypatch.undo()
monkeypatch.setattr(malicious_primary.master_replica._ordering_service,
'_send', patched_sender)
monkeypatch.setattr(slow_node.master_replica._ordering_service,
'_validate_applied_pre_prepare', lambda a, b, c: None)
with delay_rules(slow_node.nodeIbStasher,
msg_rep_delay(types_to_delay=[PREPREPARE])):
preprepare_process_num = slow_node.master_replica._ordering_service.spylog.count(
OrderingService.process_preprepare)
resp_task = sdk_send_random_request(looper, sdk_pool_handle,
sdk_wallet_steward)
def chk():
assert preprepare_process_num + 1 == slow_node.master_replica._ordering_service.spylog.count(
OrderingService.process_preprepare)
looper.run(eventually(chk))
_, j_resp = sdk_get_and_check_replies(looper, [resp_task])[0]
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, 1)
trigger_view_change(txnPoolNodeSet)
ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet)
waitForViewChange(looper,
txnPoolNodeSet,
expectedViewNo=start_view_no + 1)
ensureElectionsDone(looper=looper,
nodes=txnPoolNodeSet,
instances_list=[0, 1])
ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet)
sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_steward,
sdk_pool_handle)
def test_stashed_messages_processed_on_backup_replica_ordering_resumption(
looper, chkFreqPatched, reqs_for_checkpoint,
one_replica_and_others_in_backup_instance,
sdk_pool_handle, sdk_wallet_client, view_change_done,
txnPoolNodeSet):
"""
Verifies resumption of ordering 3PC-batches on a backup replica
on detection of a lag in checkpoints in case it is detected after
some 3PC-messages related to the next checkpoint have already been stashed
as laying outside of the watermarks.
Please note that to verify this case the config is set up so that
LOG_SIZE == (Replica.STASHED_CHECKPOINTS_BEFORE_CATCHUP + 1) * CHK_FREQ
"""
global first_run
batches_count = get_pp_seq_no(txnPoolNodeSet)
slow_replica, other_replicas = one_replica_and_others_in_backup_instance
view_no = slow_replica.viewNo
low_watermark = slow_replica.h
# Send a request and ensure that the replica orders the batch for it
sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 1)
batches_count += 1
looper.run(
eventually(lambda *args: assertExp(slow_replica.last_ordered_3pc == (view_no, batches_count)),
slow_replica,
retryWait=1,
timeout=waits.expectedTransactionExecutionTime(nodeCount)))
# Don't receive Commits from two replicas
slow_replica.node.nodeIbStasher.delay(
cDelay(instId=1, sender_filter=other_replicas[0].node.name))
slow_replica.node.nodeIbStasher.delay(
cDelay(instId=1, sender_filter=other_replicas[1].node.name))
slow_replica.node.nodeIbStasher.delay(
msg_rep_delay(types_to_delay=[COMMIT])
)
# Send a request for which the replica will not be able to order the batch
# due to an insufficient count of Commits
sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 1)
looper.runFor(waits.expectedTransactionExecutionTime(nodeCount))
# Receive further Commits from now on
slow_replica.node.nodeIbStasher.drop_delayeds()
slow_replica.node.nodeIbStasher.resetDelays()
# Send requests but in a quantity insufficient
# for catch-up number of checkpoints
reqs_until_checkpoints = reqs_for_checkpoint - get_pp_seq_no([r.node for r in other_replicas]) % reqs_for_checkpoint
sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client,
Replica.STASHED_CHECKPOINTS_BEFORE_CATCHUP *
reqs_until_checkpoints)
looper.runFor(waits.expectedTransactionExecutionTime(nodeCount))
# Don't receive Checkpoints
slow_replica.node.nodeIbStasher.delay(chk_delay(instId=1))
# Send more requests to reach catch-up number of checkpoints
sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client,
reqs_for_checkpoint)
looper.runFor(waits.expectedTransactionExecutionTime(nodeCount))
# Ensure that there are no 3PC-messages stashed
# as laying outside of the watermarks
assert slow_replica.stasher.stash_size(STASH_WATERMARKS) == 0
# Send a request for which the batch will be outside of the watermarks
sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 1)
looper.runFor(waits.expectedTransactionExecutionTime(nodeCount))
# Ensure that the replica has not ordered any batches
# after the very first one
assert slow_replica.last_ordered_3pc == (view_no, batches_count)
# Ensure that the watermarks have not been shifted since the view start
assert slow_replica.h == low_watermark
assert slow_replica.H == (sys.maxsize if first_run else low_watermark + LOG_SIZE)
# Ensure that there are some quorumed stashed checkpoints
check_num_quorumed_received_checkpoints(slow_replica, 1)
# Ensure that now there are 3PC-messages stashed
# as laying outside of the watermarks
if not first_run:
assert slow_replica.stasher.stash_size(STASH_WATERMARKS) == incoming_3pc_msgs_count(len(txnPoolNodeSet))
# Receive belated Checkpoints
slow_replica.node.nodeIbStasher.reset_delays_and_process_delayeds()
batches_count = get_pp_seq_no([r.node for r in other_replicas])
# Ensure that the replica has ordered the batch for the last sent request
looper.run(
eventually(lambda *args: assertExp(slow_replica.last_ordered_3pc ==
(view_no, batches_count)),
slow_replica,
retryWait=1,
timeout=waits.expectedTransactionExecutionTime(nodeCount)))
# Ensure that the watermarks have been shifted so that the lower watermark
# now equals to the end of the last stable checkpoint in the instance
assert slow_replica.h == low_watermark + (Replica.STASHED_CHECKPOINTS_BEFORE_CATCHUP + 1) * CHK_FREQ
assert slow_replica.H == low_watermark + (Replica.STASHED_CHECKPOINTS_BEFORE_CATCHUP + 1) * CHK_FREQ + LOG_SIZE
# Ensure that now there are no quorumed stashed checkpoints
check_num_quorumed_received_checkpoints(slow_replica, 0)
# Ensure that now there are no 3PC-messages stashed
# as laying outside of the watermarks
assert slow_replica.stasher.stash_size(STASH_WATERMARKS) == 0
# Send a request and ensure that the replica orders the batch for it
sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 1)
batches_count += 1
looper.run(
eventually(lambda *args: assertExp(slow_replica.last_ordered_3pc ==
(view_no, batches_count)),
slow_replica,
retryWait=1,
timeout=waits.expectedTransactionExecutionTime(nodeCount)))
first_run = False
