def test_check_cdp_pp_storages(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client):
def check_all_empty(replica):
assert not bool(replica._consensus_data.preprepared)
assert not bool(replica._consensus_data.prepared)
def check_preprepared_not_empty(replica):
assert bool(replica._consensus_data.preprepared)
def check_prepared_not_empty(replica):
assert bool(replica._consensus_data.prepared)
def operation_for_replicas(operation, node_set=txnPoolNodeSet):
for node in node_set:
operation(node.master_replica)
node_stashers = [n.nodeIbStasher for n in txnPoolNodeSet]
with delay_rules(node_stashers, pDelay()):
with delay_rules(node_stashers, ppDelay()):
sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client)
looper.run(
eventually(operation_for_replicas, check_all_empty,
txnPoolNodeSet[1:]))
looper.run(
eventually(operation_for_replicas, check_preprepared_not_empty,
txnPoolNodeSet[0:1]))
looper.run(
eventually(operation_for_replicas, check_preprepared_not_empty,
txnPoolNodeSet))
looper.run(
eventually(operation_for_replicas, check_prepared_not_empty,
txnPoolNodeSet))
def test_view_change_with_different_prepare_certificate(
looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client):
"""
Check that a node without pre-prepare but with quorum of prepares wouldn't
use this transaction as a last in prepare certificate
"""
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
slow_node = txnPoolNodeSet[-1]
# delay preprepares and message response with preprepares.
with delay_rules(slow_node.nodeIbStasher, ppDelay(delay=sys.maxsize)):
with delay_rules(
slow_node.nodeIbStasher,
msg_rep_delay(delay=sys.maxsize, types_to_delay=[
PREPREPARE,
])):
last_ordered = slow_node.master_replica.last_ordered_3pc
sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client)
looper.run(
eventually(check_prepare_certificate, txnPoolNodeSet[0:-1],
last_ordered[1] + 1))
for n in txnPoolNodeSet:
n.view_changer.on_master_degradation()
assert slow_node.master_replica.last_prepared_certificate_in_view() == \
(0, last_ordered[1])
ensureElectionsDone(looper, txnPoolNodeSet)
def testOrderingCase1(looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle):
"""
Scenario -> PRE-PREPARE not received by the replica, Request not received
for ordering by the replica, but received enough commits to start ordering.
It queues up the request so when a PRE-PREPARE is received or request is
receievd for ordering, an order can be triggered
https://www.pivotaltracker.com/story/show/125239401
Reproducing by - Pick a node with no primary replica, replica ignores
forwarded request to replica and delay reception of PRE-PREPARE sufficiently
so that enough COMMITs reach to trigger ordering.
"""
delay = 10
replica = getNonPrimaryReplicas(txnPoolNodeSet, instId=0)[0]
delaysPrePrepareProcessing(replica.node, delay=delay, instId=0)
def doNotProcessReqDigest(self, _):
pass
patchedMethod = types.MethodType(doNotProcessReqDigest, replica)
replica.processRequest = patchedMethod
def chk(n):
assert replica.spylog.count(replica.doOrder.__name__) == n
sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client)
timeout = delay - 5
looper.run(eventually(chk, 0, retryWait=1, timeout=timeout))
timeout = delay + 5
looper.run(eventually(chk, 1, retryWait=1, timeout=timeout))
def test_view_change_with_different_prepare_certificate(looper, txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client):
"""
Check that a node without pre-prepare but with quorum of prepares wouldn't
use this transaction as a last in prepare certificate
"""
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
slow_node = txnPoolNodeSet[-1]
# delay preprepares and message response with preprepares.
with delay_rules(slow_node.nodeIbStasher, ppDelay(delay=sys.maxsize)):
with delay_rules(slow_node.nodeIbStasher,
msg_rep_delay(delay=sys.maxsize,
types_to_delay=[PREPREPARE, ])):
last_ordered = slow_node.master_replica.last_ordered_3pc
sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client)
looper.run(eventually(check_prepare_certificate,
txnPoolNodeSet[0:-1],
last_ordered[1] + 1))
for n in txnPoolNodeSet:
n.view_changer.on_master_degradation()
assert slow_node.master_replica.last_prepared_certificate_in_view() == \
(0, last_ordered[1])
ensureElectionsDone(looper, txnPoolNodeSet)
def testOrderingCase1(looper, txnPoolNodeSet, sdk_wallet_client,
sdk_pool_handle):
"""
Scenario -> PRE-PREPARE not received by the replica, Request not received
for ordering by the replica, but received enough commits to start ordering.
It queues up the request so when a PRE-PREPARE is received or request is
receievd for ordering, an order can be triggered
https://www.pivotaltracker.com/story/show/125239401
Reproducing by - Pick a node with no primary replica, replica ignores
forwarded request to replica and delay reception of PRE-PREPARE sufficiently
so that enough COMMITs reach to trigger ordering.
"""
delay = 10
replica = getNonPrimaryReplicas(txnPoolNodeSet, instId=0)[0]
delaysPrePrepareProcessing(replica.node, delay=delay, instId=0)
def doNotProcessReqDigest(self, _):
pass
patchedMethod = types.MethodType(doNotProcessReqDigest, replica)
replica.processRequest = patchedMethod
def chk(n):
assert replica.spylog.count(replica.doOrder.__name__) == n
sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client)
timeout = delay - 5
looper.run(eventually(chk, 0, retryWait=1, timeout=timeout))
timeout = delay + 5
looper.run(eventually(chk, 1, retryWait=1, timeout=timeout))
def test_nodes_with_bad_clock(tconf, looper, txnPoolNodeSet,
sdk_wallet_client, sdk_pool_handle):
"""
All nodes have bad clocks but they eventaully get repaired, an example of
nodes being cut off from NTP server for some time or NTP sync disabled
then without node restart NTP sync enabled
"""
sdk_send_random_and_check(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
count=Max3PCBatchSize * 3)
ledger_sizes = {node.name: node.domainLedger.size for node in
txnPoolNodeSet}
susp_counts = {node.name: get_timestamp_suspicion_count(node) for node in
txnPoolNodeSet}
for node in txnPoolNodeSet:
make_clock_faulty(
node,
clock_slow_by_sec=node.config.ACCEPTABLE_DEVIATION_PREPREPARE_SECS +
randint(
5,
15),
ppr_always_wrong=False)
for _ in range(5):
sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client)
looper.runFor(.2)
# Let some time pass
looper.runFor(3)
def chk():
for node in txnPoolNodeSet:
# Each node raises suspicion
assert get_timestamp_suspicion_count(node) > susp_counts[node.name]
# Ledger does not change
assert node.domainLedger.size == ledger_sizes[node.name]
looper.run(eventually(chk, retryWait=1))
# Fix clocks
for node in txnPoolNodeSet:
def utc_epoch(self) -> int:
return get_utc_epoch()
node.utc_epoch = types.MethodType(utc_epoch, node)
# Let some more time pass
looper.runFor(3)
# All nodes reply
sdk_send_random_and_check(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
count=Max3PCBatchSize * 2)
def testOrderingWhenPrePrepareNotReceived(looper, txnPoolNodeSet,
sdk_wallet_client, sdk_pool_handle):
"""
Send commits but delay pre-prepare and prepares such that enough
commits are received, now the request should not be ordered until
pre-prepare is received and ordering should just happen once,
"""
delay = 10
non_prim_reps = getNonPrimaryReplicas(txnPoolNodeSet, 0)
slow_rep = non_prim_reps[0]
slow_node = slow_rep.node
slow_node.nodeIbStasher.delay(ppDelay(delay, 0))
slow_node.nodeIbStasher.delay(pDelay(delay, 0))
stash_pp = []
stash_p = []
orig_pp_method = slow_rep._ordering_service.process_preprepare
orig_p_method = slow_rep._ordering_service.process_prepare
def patched_pp(self, msg, sender):
stash_pp.append((msg, sender))
def patched_p(self, msg, sender):
stash_p.append((msg, sender))
slow_rep._ordering_service.process_preprepare = \
types.MethodType(patched_pp, slow_rep)
slow_rep._ordering_service.process_prepare = \
types.MethodType(patched_p, slow_rep)
def chk1():
assert len(slow_rep._ordering_service.commitsWaitingForPrepare) > 0
sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client)
timeout = waits.expectedPrePrepareTime(len(txnPoolNodeSet)) + delay
looper.run(eventually(chk1, retryWait=1, timeout=timeout))
for m, s in stash_pp:
orig_pp_method(m, s)
for m, s in stash_p:
orig_p_method(m, s)
def chk2():
assert len(slow_rep._ordering_service.commitsWaitingForPrepare) == 0
assert slow_rep._ordering_service.spylog.count(
slow_rep._ordering_service._do_order.__name__) == 1
timeout = waits.expectedOrderingTime(len(non_prim_reps) + 1) + 2 * delay
looper.run(eventually(chk2, retryWait=1, timeout=timeout))
def testOrderingWhenPrePrepareNotReceived(looper, txnPoolNodeSet,
sdk_wallet_client, sdk_pool_handle):
"""
Send commits but delay pre-prepare and prepares such that enough
commits are received, now the request should not be ordered until
pre-prepare is received and ordering should just happen once,
"""
delay = 10
non_prim_reps = getNonPrimaryReplicas(txnPoolNodeSet, 0)
slow_rep = non_prim_reps[0]
slow_node = slow_rep.node
slow_node.nodeIbStasher.delay(ppDelay(delay, 0))
slow_node.nodeIbStasher.delay(pDelay(delay, 0))
stash_pp = []
stash_p = []
orig_pp_method = slow_rep.processPrePrepare
orig_p_method = slow_rep.processPrepare
def patched_pp(self, msg, sender):
stash_pp.append((msg, sender))
def patched_p(self, msg, sender):
stash_p.append((msg, sender))
slow_rep.processPrePrepare = \
types.MethodType(patched_pp, slow_rep)
slow_rep.processPrepare = \
types.MethodType(patched_p, slow_rep)
def chk1():
assert len(slow_rep.commitsWaitingForPrepare) > 0
sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client)
timeout = waits.expectedPrePrepareTime(len(txnPoolNodeSet)) + delay
looper.run(eventually(chk1, retryWait=1, timeout=timeout))
for m, s in stash_pp:
orig_pp_method(m, s)
for m, s in stash_p:
orig_p_method(m, s)
def chk2():
assert len(slow_rep.commitsWaitingForPrepare) == 0
assert slow_rep.spylog.count(slow_rep.doOrder.__name__) == 1
timeout = waits.expectedOrderingTime(len(non_prim_reps) + 1) + 2 * delay
looper.run(eventually(chk2, retryWait=1, timeout=timeout))
def do_view_change_with_unaligned_prepare_certificates(
slow_nodes, nodes, looper, sdk_pool_handle, sdk_wallet_client):
"""
Perform view change with some nodes reaching lower last prepared certificate than others.
With current implementation of view change this can result with view change taking a lot of time.
"""
fast_nodes = [n for n in nodes if n not in slow_nodes]
all_stashers = [n.nodeIbStasher for n in nodes]
slow_stashers = [n.nodeIbStasher for n in slow_nodes]
# Delay some PREPAREs and all COMMITs
with delay_rules(slow_stashers, pDelay()):
with delay_rules(all_stashers, cDelay()):
# Send request
request = sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client)
# Wait until this request is prepared on fast nodes
looper.run(eventually(check_last_prepared_certificate, fast_nodes, (0, 1)))
# Make sure its not prepared on slow nodes
looper.run(eventually(check_last_prepared_certificate, slow_nodes, None))
# Trigger view change
for n in nodes:
n.view_changer.on_master_degradation()
# Now commits are processed
# Wait until view change is complete
looper.run(eventually(check_view_change_done, nodes, 1, timeout=60))
# Finish request gracefully
sdk_get_reply(looper, request)
def requests(looper, sdk_wallet_client, sdk_pool_handle):
requests = []
for i in range(5):
req = sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client)
req, _ = sdk_get_reply(looper, req, timeout=sdk_eval_timeout(1, 4))
requests.append(req)
return requests
def do_view_change_with_unaligned_prepare_certificates(
slow_nodes, nodes, looper, sdk_pool_handle, sdk_wallet_client):
"""
Perform view change with some nodes reaching lower last prepared certificate than others.
With current implementation of view change this can result with view change taking a lot of time.
"""
fast_nodes = [n for n in nodes if n not in slow_nodes]
all_stashers = [n.nodeIbStasher for n in nodes]
slow_stashers = [n.nodeIbStasher for n in slow_nodes]
# Delay some PREPAREs and all COMMITs
with delay_rules(slow_stashers, pDelay()):
with delay_rules(all_stashers, cDelay()):
# Send request
request = sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client)
# Wait until this request is prepared on fast nodes
looper.run(eventually(check_last_prepared_certificate, fast_nodes, (0, 1)))
# Make sure its not prepared on slow nodes
looper.run(eventually(check_last_prepared_certificate, slow_nodes, None))
# Trigger view change
for n in nodes:
n.view_changer.on_master_degradation()
# Now commits are processed
# Wait until view change is complete
looper.run(eventually(check_view_change_done, nodes, 1, timeout=60))
# Finish request gracefully
sdk_get_reply(looper, request)
def do_view_change_with_propagate_primary_on_one_delayed_node(
slow_node, nodes, looper, sdk_pool_handle, sdk_wallet_client):
slow_stasher = slow_node.nodeIbStasher
fast_nodes = [n for n in nodes if n != slow_node]
stashers = [n.nodeIbStasher for n in nodes]
# Get last prepared certificate in pool
lpc = last_prepared_certificate(nodes)
# Get pool current view no
view_no = lpc[0]
with delay_rules(slow_stasher, icDelay()):
with delay_rules(slow_stasher, vcd_delay()):
with delay_rules(stashers, cDelay()):
# Send request
request = sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client)
# Wait until this request is prepared on N-f nodes
looper.run(eventually(check_last_prepared_certificate_on_quorum, nodes, (lpc[0], lpc[1] + 1)))
# Trigger view change
for n in nodes:
n.view_changer.on_master_degradation()
# Wait until view change is completed on all nodes except slow one
waitForViewChange(looper,
fast_nodes,
expectedViewNo=view_no + 1,
customTimeout=waits.expectedPoolViewChangeStartedTimeout(len(nodes)))
wait_for_elections_done_on_given_nodes(looper,
fast_nodes,
getRequiredInstances(len(nodes)),
timeout=waits.expectedPoolElectionTimeout(len(nodes)))
# Now all the nodes receive Commits
# The slow node will accept Commits and order the 3PC-batch in the old view
looper.runFor(waits.expectedOrderingTime(getNoInstances(len(nodes))))
# Now slow node receives ViewChangeDones
waitForViewChange(looper,
[slow_node],
expectedViewNo=view_no + 1,
customTimeout=waits.expectedPoolViewChangeStartedTimeout(len(nodes)))
wait_for_elections_done_on_given_nodes(looper,
[slow_node],
getRequiredInstances(len(nodes)),
timeout=waits.expectedPoolElectionTimeout(len(nodes)))
# Now slow node receives InstanceChanges but discards them because already
# started propagate primary to the same view.
# Finish request gracefully
sdk_get_reply(looper, request)
def do_view_change_with_propagate_primary_on_one_delayed_node(
slow_node, nodes, looper, sdk_pool_handle, sdk_wallet_client):
slow_stasher = slow_node.nodeIbStasher
fast_nodes = [n for n in nodes if n != slow_node]
stashers = [n.nodeIbStasher for n in nodes]
# Get last prepared certificate in pool
lpc = last_prepared_certificate(nodes)
# Get pool current view no
view_no = lpc[0]
with delay_rules(slow_stasher, icDelay()):
with delay_rules(slow_stasher, vcd_delay()):
with delay_rules(stashers, cDelay()):
# Send request
request = sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client)
# Wait until this request is prepared on N-f nodes
looper.run(eventually(check_last_prepared_certificate_on_quorum, nodes, (lpc[0], lpc[1] + 1)))
# Trigger view change
for n in nodes:
n.view_changer.on_master_degradation()
# Wait until view change is completed on all nodes except slow one
waitForViewChange(looper,
fast_nodes,
expectedViewNo=view_no + 1,
customTimeout=waits.expectedPoolViewChangeStartedTimeout(len(nodes)))
wait_for_elections_done_on_given_nodes(looper,
fast_nodes,
getRequiredInstances(len(nodes)),
timeout=waits.expectedPoolElectionTimeout(len(nodes)))
# Now all the nodes receive Commits
# The slow node will accept Commits and order the 3PC-batch in the old view
looper.runFor(waits.expectedOrderingTime(getNoInstances(len(nodes))))
# Now slow node receives ViewChangeDones
waitForViewChange(looper,
[slow_node],
expectedViewNo=view_no + 1,
customTimeout=waits.expectedPoolViewChangeStartedTimeout(len(nodes)))
wait_for_elections_done_on_given_nodes(looper,
[slow_node],
getRequiredInstances(len(nodes)),
timeout=waits.expectedPoolElectionTimeout(len(nodes)))
# Now slow node receives InstanceChanges but discards them because already
# started propagate primary to the same view.
# Finish request gracefully
sdk_get_reply(looper, request)
def testReplyWhenRequestAlreadyExecuted(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, sent1):
"""
When a request has already been executed the previously executed reply
will be sent again to the client. An acknowledgement will not be sent
for a repeated request.
"""
sdk_get_and_check_replies(looper, sent1)
req = sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client)
sdk_get_and_check_replies(looper, [req])
def testReplyWhenRequestAlreadyExecuted(looper, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client,
sent1):
"""
When a request has already been executed the previously executed reply
will be sent again to the client. An acknowledgement will not be sent
for a repeated request.
"""
sdk_get_and_check_replies(looper, sent1)
req = sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client)
sdk_get_and_check_replies(looper, [req])
def testMsgFromInstanceDelay(configNodeSet, looper,
sdk_pool_handle, sdk_wallet_client):
A, B, C, D = configNodeSet
sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client)
def getCommits(node: TestNode, instId: int):
replica = node.replicas[instId] # type: Replica
return list(replica.commits.values())
def checkPresence():
for node in [C, D]:
commReqs = getCommits(node, 0)
assert len(commReqs) > 0
assert Replica.generateName(A.name, 0) not in commReqs[0][0]
commReqs = getCommits(node, 1)
assert len(commReqs) > 0
assert Replica.generateName(A.name, 1) in commReqs[0][0]
numOfNodes = len(configNodeSet)
timeout = waits.expectedClientRequestPropagationTime(numOfNodes)
looper.run(eventually(checkPresence, retryWait=.5, timeout=timeout))
def testMsgFromInstanceDelay(configNodeSet, looper,
sdk_pool_handle, sdk_wallet_client):
A, B, C, D = configNodeSet
sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client)
def getCommits(node: TestNode, instId: int):
replica = node.replicas[instId] # type: Replica
return list(replica.commits.values())
def checkPresence():
for node in [C, D]:
commReqs = getCommits(node, 0)
assert len(commReqs) > 0
assert Replica.generateName(A.name, 0) not in commReqs[0][0]
commReqs = getCommits(node, 1)
assert len(commReqs) > 0
assert Replica.generateName(A.name, 1) in commReqs[0][0]
numOfNodes = len(configNodeSet)
timeout = waits.expectedClientRequestPropagationTime(numOfNodes)
looper.run(eventually(checkPresence, retryWait=.5, timeout=timeout))
def do_view_change_with_pending_request_and_one_fast_node(
fast_node, nodes, looper, sdk_pool_handle, sdk_wallet_client):
"""
Perform view change while processing request, with one node receiving commits much sooner than others.
With current implementation of view change this will result in corrupted state of fast node
"""
fast_stasher = fast_node.nodeIbStasher
slow_nodes = [n for n in nodes if n != fast_node]
slow_stashers = [n.nodeIbStasher for n in slow_nodes]
# Get last prepared certificate in pool
lpc = last_prepared_certificate(nodes)
# Get pool current view no
view_no = lpc[0]
# Delay all COMMITs
with delay_rules(slow_stashers, cDelay()):
with delay_rules(fast_stasher, cDelay()):
# Send request
request = sdk_send_random_request(looper, sdk_pool_handle,
sdk_wallet_client)
# Wait until this request is prepared on N-f nodes
looper.run(
eventually(check_last_prepared_certificate_on_quorum, nodes,
(lpc[0], lpc[1] + 1)))
# Trigger view change
for n in nodes:
n.view_changer.on_master_degradation()
# Now commits are processed on fast node
# Wait until view change is complete
looper.run(
eventually(check_view_change_done, nodes, view_no + 1, timeout=60))
# Finish request gracefully
sdk_get_reply(looper, request)
def do_view_change_with_pending_request_and_one_fast_node(fast_node,
nodes, looper, sdk_pool_handle, sdk_wallet_client):
"""
Perform view change while processing request, with one node receiving commits much sooner than others.
With current implementation of view change this will result in corrupted state of fast node
"""
fast_stasher = fast_node.nodeIbStasher
slow_nodes = [n for n in nodes if n != fast_node]
slow_stashers = [n.nodeIbStasher for n in slow_nodes]
# Get last prepared certificate in pool
lpc = last_prepared_certificate(nodes)
# Get pool current view no
view_no = lpc[0]
# Delay all COMMITs
with delay_rules(slow_stashers, cDelay()):
with delay_rules(fast_stasher, cDelay()):
# Send request
request = sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client)
# Wait until this request is prepared on N-f nodes
looper.run(eventually(check_last_prepared_certificate_on_quorum, nodes, (lpc[0], lpc[1] + 1)))
# Trigger view change
for n in nodes:
n.view_changer.on_master_degradation()
# Now commits are processed on fast node
# Wait until view change is complete
looper.run(eventually(check_view_change_done, nodes, view_no + 1, timeout=60))
# Finish request gracefully
sdk_get_reply(looper, request)
def test_unstash_three_phase_msg_after_catchup(txnPoolNodeSet, looper, tconf,
sdk_pool_handle,
sdk_wallet_steward):
"""
1. Delay Commit on Node4
2. Order 1 req
3. Delay Commit on all nodes
4. Order 1 req
5. Delay CatchupRep on Node4
6. Delay Ledger Status and ViewChangeDones on Nodes1-3
7. Start View change on all nodes
8. Wait until Node4 got 3 stashed CatchupReps
9. Reset delaying of Commits on all Nodes
10. Reset Ledger Status on Nodes1-3
11. Check that 3 nodes finished VC while Node4 is syncing and not finished
12. Reset CatchupRep on Node4
13. Check that Node4 finished VC, and there was just 1 round of catch-up
"""
slow_node = txnPoolNodeSet[-1]
fast_nodes = txnPoolNodeSet[:-1]
view_no = txnPoolNodeSet[0].viewNo
old_stashed = slow_node.master_replica.stasher.stash_size(STASH_VIEW_3PC)
last_ordered = txnPoolNodeSet[0].master_replica.last_ordered_3pc
batches_count = last_ordered[1]
with delay_rules(
[n.nodeIbStasher for n in txnPoolNodeSet],
msg_rep_delay(types_to_delay=[PREPREPARE, PREPARE, COMMIT])):
# Delay Commit messages for slow_node.
slow_node.nodeIbStasher.delay(cDelay(sys.maxsize))
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, 1)
batches_count += 1
# Delay Commit messages for fast_nodes.
for n in fast_nodes:
n.nodeIbStasher.delay(cDelay(sys.maxsize))
request2 = sdk_send_random_request(looper, sdk_pool_handle,
sdk_wallet_steward)
batches_count += 1
def check_commits(commit_key):
for n in fast_nodes:
for r in n.replicas.values():
assert commit_key in r._ordering_service.commits
assert len(
r._ordering_service.commits[commit_key].voters) == 1
looper.run(
eventually(check_commits,
(view_no, last_ordered[1] + batches_count)))
# Delay CatchupRep messages for the slow_node.
with delay_rules([slow_node.nodeIbStasher], cr_delay()):
with delay_rules([n.nodeIbStasher for n in fast_nodes],
msg_rep_delay(types_to_delay=[LEDGER_STATUS])):
for n in txnPoolNodeSet:
n.start_catchup()
looper.run(
eventually(
lambda: assertExp(slow_node.mode == Mode.discovering)))
# Reset delay Commit messages for all nodes.
for n in txnPoolNodeSet:
n.nodeIbStasher.reset_delays_and_process_delayeds(COMMIT)
assert slow_node.mode == Mode.discovering
looper.run(
eventually(_check_nodes_stashed, fast_nodes, old_stashed,
len(txnPoolNodeSet) - 1))
looper.run(
eventually(_check_nodes_stashed, [slow_node], old_stashed,
(len(txnPoolNodeSet) - 1) * 2))
sdk_get_and_check_replies(looper, [request2])
_check_nodes_stashed(fast_nodes, old_stashed, 0)
assert get_pp_seq_no(txnPoolNodeSet) == batches_count
ensure_all_nodes_have_same_data(looper, txnPoolNodeSet)
sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_steward,
sdk_pool_handle)
def test_unstash_three_phase_msg_after_catchup_in_view_change(
txnPoolNodeSet, looper, tconf, sdk_pool_handle, sdk_wallet_steward):
"""
1. Delay Commit on Node4
2. Order 1 req
3. Delay Commit on all nodes
4. Order 1 req
5. Delay CatchupRep on Node4
6. Delay Ledger Status and ViewChangeDones on Nodes1-3
7. Start View change on all nodes
8. Wait until Node4 got 3 stashed CatchupReps
9. Reset delaying of Commits on all Nodes
10. Reset Ledger Status on Nodes1-3
11. Check that 3 nodes finished VC while Node4 is syncing and not finished
12. Reset CatchupRep on Node4
13. Check that Node4 finished VC, and there was just 1 round of cacth-up (edited)
"""
slow_node = txnPoolNodeSet[-1]
fast_nodes = txnPoolNodeSet[:-1]
view_no = txnPoolNodeSet[0].viewNo
old_stashed = slow_node.master_replica.stasher.num_stashed_future_view
last_ordered = txnPoolNodeSet[0].master_replica.last_ordered_3pc
with delay_rules(
[n.nodeIbStasher for n in txnPoolNodeSet],
msg_rep_delay(types_to_delay=[PREPREPARE, PREPARE, COMMIT])):
# Delay Commit messages for slow_node.
slow_node.nodeIbStasher.delay(cDelay(sys.maxsize))
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, 1)
# Delay Commit messages for fast_nodes.
for n in fast_nodes:
n.nodeIbStasher.delay(cDelay(sys.maxsize))
request2 = sdk_send_random_request(looper, sdk_pool_handle,
sdk_wallet_steward)
def check_commits(commit_key):
for n in fast_nodes:
for r in n.replicas.values():
assert commit_key in r.commits
assert len(r.commits[commit_key].voters) == 1
looper.run(eventually(check_commits, (view_no, last_ordered[1] + 2)))
# Delay CatchupRep messages for the slow_node.
with delay_rules([slow_node.nodeIbStasher], cr_delay()):
with delay_rules([n.nodeIbStasher for n in fast_nodes],
vcd_delay()):
with delay_rules(
[n.nodeIbStasher for n in fast_nodes],
msg_rep_delay(types_to_delay=[LEDGER_STATUS])):
for n in txnPoolNodeSet:
n.view_changer.on_master_degradation()
looper.run(
eventually(
lambda: assertExp(slow_node.mode == Mode.syncing)))
# Reset delay Commit messages for all nodes.
for n in txnPoolNodeSet:
n.nodeIbStasher.reset_delays_and_process_delayeds(
COMMIT)
assert slow_node.view_change_in_progress
assert slow_node.mode == Mode.syncing
looper.run(
eventually(_check_nodes_stashed, fast_nodes,
old_stashed,
len(txnPoolNodeSet) - 1))
looper.run(
eventually(_check_nodes_stashed, [slow_node],
old_stashed, (len(txnPoolNodeSet) - 1) * 2))
waitForViewChange(looper,
fast_nodes,
expectedViewNo=view_no + 1,
customTimeout=2 * tconf.VIEW_CHANGE_TIMEOUT)
ensureElectionsDone(looper=looper,
nodes=fast_nodes,
instances_list=range(
fast_nodes[0].requiredNumberOfInstances),
customTimeout=2 * tconf.VIEW_CHANGE_TIMEOUT)
sdk_get_and_check_replies(looper, [request2])
waitForViewChange(looper, [slow_node],
expectedViewNo=view_no + 1,
customTimeout=2 * tconf.VIEW_CHANGE_TIMEOUT)
ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet)
_check_nodes_stashed(fast_nodes, old_stashed, 0)
assert all(n.master_replica.last_ordered_3pc == (last_ordered[0],
last_ordered[1] + 2)
for n in txnPoolNodeSet)
assert slow_node.catchup_rounds_without_txns == 1
def do_view_change_with_delayed_commits_and_node_restarts(
fast_nodes,
slow_nodes,
nodes_to_restart,
old_view_no,
old_last_ordered,
looper,
sdk_pool_handle,
sdk_wallet_client,
tconf,
tdir,
all_plugins_path,
wait_for_catchup=False):
"""
Delays commits without processing on `slow_nodes`, restarts `nodes_to_restart`, triggers view change, and confirms
that view changed completed successfully and that the ledgers are consistent and in sync.
:param fast_nodes: Nodes that will order the requests
:param slow_nodes: Nodes whose commits will be delay, and that will not order the requests
:param nodes_to_restart: Nodes that will be restarted
:param old_view_no: View that we started from
:param old_last_ordered: Last ordered 3pc txn before we did any requests
:param wait_for_catchup: Should we wait for restarted nodes to finish catchup
"""
nodes = fast_nodes + slow_nodes
slow_stashers = [slow_node.nodeIbStasher for slow_node in slow_nodes]
# Delay commits on `slow_nodes`
with delay_rules_without_processing(slow_stashers, cDelay()):
request = sdk_send_random_request(looper, sdk_pool_handle,
sdk_wallet_client)
# Check that all of the nodes except the slows one ordered the request
looper.run(
eventually(check_last_ordered, fast_nodes,
(old_view_no, old_last_ordered[1] + 1)))
looper.run(eventually(check_last_ordered, slow_nodes,
old_last_ordered))
# Restart nodes
for node in nodes_to_restart:
disconnect_node_and_ensure_disconnected(looper,
nodes,
node,
timeout=len(nodes_to_restart),
stopNode=True)
looper.removeProdable(node)
nodes.remove(node)
restarted_node = start_stopped_node(node, looper, tconf, tdir,
all_plugins_path)
nodes.append(restarted_node)
looper.runFor(waits.expectedNodeStartUpTimeout())
looper.run(checkNodesConnected(nodes))
if wait_for_catchup:
ensure_all_nodes_have_same_data(looper, nodes)
# Trigger view change on all nodes
for node in nodes:
node.view_changer.on_master_degradation()
assert len(nodes) == len(slow_nodes) + len(fast_nodes)
# Assert that view change was successful and that ledger data is consistent
waitForViewChange(looper,
nodes,
expectedViewNo=(old_view_no + 1),
customTimeout=waits.expectedPoolViewChangeStartedTimeout(
len(nodes)))
ensureElectionsDone(looper=looper, nodes=nodes)
ensure_all_nodes_have_same_data(looper, nodes)
sdk_get_reply(looper, request)
sdk_ensure_pool_functional(looper, nodes, sdk_wallet_client,
sdk_pool_handle)
def test_primary_send_incorrect_pp(looper, txnPoolNodeSet, tconf,
allPluginsPath, sdk_pool_handle,
sdk_wallet_steward, monkeypatch):
"""
Test steps:
Delay message requests with PrePrepares on `slow_node`
Patch sending for PrePrepare on the `malicious_primary` to send an invalid PrePrepare to slow_node
Order a new request
Start a view change
Make sure it's finished on all nodes
Make sure that the lagging node has same data with other nodes
"""
start_view_no = txnPoolNodeSet[0].viewNo
slow_node = txnPoolNodeSet[-1]
malicious_primary = txnPoolNodeSet[0]
other_nodes = [
n for n in txnPoolNodeSet if n not in [slow_node, malicious_primary]
]
timeout = waits.expectedPoolCatchupTime(nodeCount=len(txnPoolNodeSet))
ensure_all_nodes_have_same_data(looper,
txnPoolNodeSet,
custom_timeout=timeout)
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, 1)
old_sender = malicious_primary.master_replica._ordering_service._send
def patched_sender(msg, dst=None, stat=None):
if isinstance(msg, PrePrepare) and msg:
old_sender(msg, [n.name for n in other_nodes], stat)
pp_dict = msg._asdict()
pp_dict["ppTime"] += 1
pp = PrePrepare(**pp_dict)
old_sender(pp, [slow_node.name], stat)
monkeypatch.undo()
monkeypatch.setattr(malicious_primary.master_replica._ordering_service,
'_send', patched_sender)
monkeypatch.setattr(slow_node.master_replica._ordering_service,
'_validate_applied_pre_prepare', lambda a, b, c: None)
with delay_rules(slow_node.nodeIbStasher,
msg_rep_delay(types_to_delay=[PREPREPARE])):
preprepare_process_num = slow_node.master_replica._ordering_service.spylog.count(
OrderingService.process_preprepare)
resp_task = sdk_send_random_request(looper, sdk_pool_handle,
sdk_wallet_steward)
def chk():
assert preprepare_process_num + 1 == slow_node.master_replica._ordering_service.spylog.count(
OrderingService.process_preprepare)
looper.run(eventually(chk))
_, j_resp = sdk_get_and_check_replies(looper, [resp_task])[0]
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, 1)
trigger_view_change(txnPoolNodeSet)
ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet)
waitForViewChange(looper,
txnPoolNodeSet,
expectedViewNo=start_view_no + 1)
ensureElectionsDone(looper=looper,
nodes=txnPoolNodeSet,
instances_list=[0, 1])
ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet)
sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_steward,
sdk_pool_handle)
def test_sdk_new_client_send(looper, sdk_pool_handle, sdk_wallet_new_client):
resp_task = sdk_send_random_request(looper, sdk_pool_handle,
sdk_wallet_new_client)
_, j_resp = sdk_get_reply(looper, resp_task)
assert j_resp['result']
def test_sdk_new_steward_send(looper, sdk_pool_handle, sdk_wallet_new_steward):
resp_task = sdk_send_random_request(looper, sdk_pool_handle,
sdk_wallet_new_steward)
_, j_resp = sdk_get_and_check_replies(looper, [resp_task])[0]
assert j_resp['result']
def test_view_change_gc_in_between_3pc_all_nodes_delays(
looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client):
"""
Test that garbage collector compares the whole 3PC key (viewNo, ppSeqNo)
and does not remove messages from node's queues that have higher
viewNo than last ordered one even if their ppSeqNo are less or equal
"""
numNodes = len(txnPoolNodeSet)
viewNo = checkViewNoForNodes(txnPoolNodeSet)
# 1 send two messages one by one separately to make
# node pool working with two batches
# -> last_ordered_3pc = (+0, 2) [+0 means from the initial state]
# (last_ordered_3pc here and futher is tracked
# for master instances only cause non-master ones have
# specific logic of its management which we don't care in
# the test, see Replica::_setup_for_non_master)
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
batches_count = get_pp_seq_no(txnPoolNodeSet)
last_ordered_3pc = (viewNo, batches_count)
check_nodes_last_ordered_3pc(txnPoolNodeSet, last_ordered_3pc)
check_nodes_requests_size(txnPoolNodeSet, 2)
# 2 do view change
# -> GC should remove it from nodes' queues
# -> viewNo = +1
ensure_view_change_complete(looper, txnPoolNodeSet)
batches_count += 1
viewNo = checkViewNoForNodes(txnPoolNodeSet, viewNo + 1)
looper.run(
eventually(check_nodes_last_ordered_3pc, txnPoolNodeSet,
(viewNo, batches_count)))
check_nodes_requests_size(txnPoolNodeSet, 0)
# 3 slow processing 3PC messages for all nodes (all replica instances)
# randomly and send one more message
# -> not ordered (last_ordered_3pc still equal (+0, 2)) but primaries
# should at least send PRE-PREPAREs
# TODO could it be not enough for wainting that at least primary
# has sent PRE-PREPARE
propagationTimeout = waits.expectedClientRequestPropagationTime(numNodes)
delay_3pc_messages(txnPoolNodeSet, 0, delay=propagationTimeout * 2)
delay_3pc_messages(txnPoolNodeSet, 1, delay=propagationTimeout * 2)
requests = sdk_send_random_request(looper, sdk_pool_handle,
sdk_wallet_client)
def checkPrePrepareSentAtLeastByPrimary():
for node in txnPoolNodeSet:
for replica in node.replicas.values():
if replica.isPrimary:
assert len(replica._ordering_service.sent_preprepares)
looper.run(
eventually(checkPrePrepareSentAtLeastByPrimary,
retryWait=0.1,
timeout=propagationTimeout))
# 4 do view change
# -> GC shouldn't remove anything because
# last_ordered_3pc (+0, 1) < last message's 3pc key (+1, 1)
# -> viewNo = 2
ensure_view_change_complete(looper, txnPoolNodeSet)
batches_count += 1
viewNoNew = checkViewNoForNodes(txnPoolNodeSet)
# another view change could happen because of slow nodes
assert viewNoNew - viewNo in (1, 2)
viewNo = viewNoNew
check_nodes_last_ordered_3pc(txnPoolNodeSet,
(last_ordered_3pc[0] + 1, batches_count - 1))
check_nodes_requests_size(txnPoolNodeSet, 1)
# 5 reset delays and wait for replies
# -> new primaries should send new 3pc for last message
# with 3pc key (+2, 1)
# -> they should be ordered
# -> last_ordered_3pc = (+2, 1)
reset_delays_and_process_delayeds(txnPoolNodeSet)
sdk_get_replies(looper, [requests])
batches_count += 1
checkViewNoForNodes(txnPoolNodeSet, viewNo)
last_ordered_3pc = (viewNo, batches_count)
check_nodes_last_ordered_3pc(txnPoolNodeSet, last_ordered_3pc)
check_nodes_requests_size(txnPoolNodeSet, 1)
# 6 do view change
# -> GC should remove them
ensure_view_change_complete(looper, txnPoolNodeSet)
batches_count += 1
viewNo = checkViewNoForNodes(txnPoolNodeSet, viewNo + 1)
check_nodes_last_ordered_3pc(txnPoolNodeSet,
(last_ordered_3pc[0] + 1, batches_count))
check_nodes_requests_size(txnPoolNodeSet, 0)
def test_commit_signature_validation_integration(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_steward,
sdk_wallet_client,
tconf,
tdir):
'''
All nodes receive PrePrepare1(txn1 for pool_ledger)
Nodes 1, 2 ordered txn1 and nodes 3, 4 did not.
All nodes receive PrePrepare2(txn2 for domain_ledger)
Nodes 3, 4 receive commits from nodes 1, 2
Nodes 3, 4 ordered txn1
Check that all nodes ordered txn2
'''
fast_nodes = txnPoolNodeSet[:2]
slow_nodes = txnPoolNodeSet[2:]
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, 1)
# create new steward
new_steward_wallet_handle = sdk_add_new_nym(looper,
sdk_pool_handle,
sdk_wallet_steward,
alias="testClientSteward945",
role=STEWARD_STRING)
sigseed, verkey, bls_key, nodeIp, nodePort, clientIp, clientPort, key_proof = \
prepare_new_node_data(tconf, tdir, "new_node")
# create node request to add new demote node
_, steward_did = new_steward_wallet_handle
node_request = looper.loop.run_until_complete(
prepare_node_request(steward_did,
new_node_name="new_node",
clientIp=clientIp,
clientPort=clientPort,
nodeIp=nodeIp,
nodePort=nodePort,
bls_key=bls_key,
sigseed=sigseed,
services=[],
key_proof=key_proof))
first_ordered = txnPoolNodeSet[0].master_last_ordered_3PC
with ord_delay(slow_nodes):
request1 = sdk_sign_and_send_prepared_request(looper, new_steward_wallet_handle,
sdk_pool_handle, node_request)
key1 = get_key_from_req(request1[0])
def check_nodes_receive_pp(view_no, seq_no):
for node in txnPoolNodeSet:
assert node.master_replica.getPrePrepare(view_no, seq_no)
looper.run(eventually(check_nodes_receive_pp, first_ordered[0], first_ordered[1] + 1))
def check_fast_nodes_ordered_request():
for n in fast_nodes:
assert key1 not in n.requests or n.requests[key1].executed
for n in slow_nodes:
assert not n.requests[key1].executed
looper.run(eventually(check_fast_nodes_ordered_request))
request2 = sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client)
looper.run(eventually(check_nodes_receive_pp, first_ordered[0], first_ordered[1] + 2))
def check_nodes_receive_commits(view_no, seq_no):
for node in txnPoolNodeSet:
assert len(node.master_replica.commits[view_no, seq_no].voters) >= node.f + 1
looper.run(eventually(check_nodes_receive_commits, first_ordered[0], first_ordered[1] + 2))
sdk_get_and_check_replies(looper, [request1])
sdk_get_and_check_replies(looper, [request2])
def test_view_change_gc_in_between_3pc_all_nodes_delays(
looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client):
"""
Test that garbage collector compares the whole 3PC key (viewNo, ppSeqNo)
and does not remove messages from node's queues that have higher
viewNo than last ordered one even if their ppSeqNo are less or equal
"""
numNodes = len(txnPoolNodeSet)
viewNo = checkViewNoForNodes(txnPoolNodeSet)
# 1 send two messages one by one separately to make
# node pool working with two batches
# -> last_ordered_3pc = (+0, 2) [+0 means from the initial state]
# (last_ordered_3pc here and futher is tracked
# for master instances only cause non-master ones have
# specific logic of its management which we don't care in
# the test, see Replica::_setup_for_non_master)
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
last_ordered_3pc = (viewNo, 2)
check_nodes_last_ordered_3pc(txnPoolNodeSet, last_ordered_3pc)
check_nodes_requests_size(txnPoolNodeSet, 2)
# 2 do view change
# -> GC should remove it from nodes' queues
# -> viewNo = +1
ensure_view_change_complete(looper, txnPoolNodeSet)
viewNo = checkViewNoForNodes(txnPoolNodeSet, viewNo + 1)
check_nodes_last_ordered_3pc(txnPoolNodeSet, last_ordered_3pc)
check_nodes_requests_size(txnPoolNodeSet, 0)
# 3 slow processing 3PC messages for all nodes (all replica instances)
# randomly and send one more message
# -> not ordered (last_ordered_3pc still equal (+0, 2)) but primaries
# should at least send PRE-PREPAREs
# TODO could it be not enough for wainting that at least primary
# has sent PRE-PREPARE
propagationTimeout = waits.expectedClientRequestPropagationTime(numNodes)
delay_3pc_messages(txnPoolNodeSet,
0,
delay=propagationTimeout * 2)
delay_3pc_messages(txnPoolNodeSet,
1,
delay=propagationTimeout * 2)
requests = sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client)
def checkPrePrepareSentAtLeastByPrimary():
for node in txnPoolNodeSet:
for replica in node.replicas.values():
if replica.isPrimary:
assert len(replica.sentPrePrepares)
looper.run(eventually(checkPrePrepareSentAtLeastByPrimary,
retryWait=0.1,
timeout=propagationTimeout))
# 4 do view change
# -> GC shouldn't remove anything because
# last_ordered_3pc (+0, 1) < last message's 3pc key (+1, 1)
# -> viewNo = 2
ensure_view_change_complete(looper, txnPoolNodeSet)
viewNoNew = checkViewNoForNodes(txnPoolNodeSet)
# another view change could happen because of slow nodes
assert viewNoNew - viewNo in (1, 2)
viewNo = viewNoNew
check_nodes_last_ordered_3pc(txnPoolNodeSet, last_ordered_3pc)
check_nodes_requests_size(txnPoolNodeSet, 1)
# 5 reset delays and wait for replies
# -> new primaries should send new 3pc for last message
# with 3pc key (+2, 1)
# -> they should be ordered
# -> last_ordered_3pc = (+2, 1)
reset_delays_and_process_delayeds(txnPoolNodeSet)
sdk_get_replies(looper, [requests])
checkViewNoForNodes(txnPoolNodeSet, viewNo)
last_ordered_3pc = (viewNo, 1)
check_nodes_last_ordered_3pc(txnPoolNodeSet, last_ordered_3pc)
check_nodes_requests_size(txnPoolNodeSet, 1)
# 6 do view change
# -> GC should remove them
ensure_view_change_complete(looper, txnPoolNodeSet)
viewNo = checkViewNoForNodes(txnPoolNodeSet, viewNo + 1)
check_nodes_last_ordered_3pc(txnPoolNodeSet, last_ordered_3pc)
check_nodes_requests_size(txnPoolNodeSet, 0)
def test_commit_signature_validation_integration(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_steward,
sdk_wallet_client,
tconf,
tdir):
'''
All nodes receive PrePrepare1(txn1 for pool_ledger)
Nodes 1, 2 ordered txn1 and nodes 3, 4 did not.
All nodes receive PrePrepare2(txn2 for domain_ledger)
Nodes 3, 4 receive commits from nodes 1, 2
Nodes 3, 4 ordered txn1
Check that all nodes ordered txn2
'''
fast_nodes = txnPoolNodeSet[:2]
slow_nodes = txnPoolNodeSet[2:]
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, 1)
# create new steward
new_steward_wallet_handle = sdk_add_new_nym(looper,
sdk_pool_handle,
sdk_wallet_steward,
alias="testClientSteward945",
role=STEWARD_STRING)
sigseed, verkey, bls_key, nodeIp, nodePort, clientIp, clientPort, key_proof = \
prepare_new_node_data(tconf, tdir, "new_node")
# create node request to add new demote node
_, steward_did = new_steward_wallet_handle
node_request = looper.loop.run_until_complete(
prepare_node_request(steward_did,
new_node_name="new_node",
clientIp=clientIp,
clientPort=clientPort,
nodeIp=nodeIp,
nodePort=nodePort,
bls_key=bls_key,
sigseed=sigseed,
services=[],
key_proof=key_proof))
first_ordered = txnPoolNodeSet[0].master_last_ordered_3PC
with ord_delay(slow_nodes):
request1 = sdk_sign_and_send_prepared_request(looper, new_steward_wallet_handle,
sdk_pool_handle, node_request)
key1 = get_key_from_req(request1[0])
def check_nodes_receive_pp(view_no, seq_no):
for node in txnPoolNodeSet:
assert node.master_replica._ordering_service.get_preprepare(view_no, seq_no)
looper.run(eventually(check_nodes_receive_pp, first_ordered[0], first_ordered[1] + 1))
def check_fast_nodes_ordered_request():
for n in fast_nodes:
assert key1 not in n.requests or n.requests[key1].executed
for n in slow_nodes:
assert not n.requests[key1].executed
looper.run(eventually(check_fast_nodes_ordered_request))
request2 = sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client)
looper.run(eventually(check_nodes_receive_pp, first_ordered[0], first_ordered[1] + 2))
def check_nodes_receive_commits(view_no, seq_no):
for node in txnPoolNodeSet:
assert len(node.master_replica._ordering_service.commits[view_no, seq_no].voters) >= node.f + 1
looper.run(eventually(check_nodes_receive_commits, first_ordered[0], first_ordered[1] + 2))
sdk_get_and_check_replies(looper, [request1])
sdk_get_and_check_replies(looper, [request2])
