def committed1(looper, nodeSet, client1, prepared1, faultyNodes):
checkCommited(looper,
nodeSet,
prepared1,
range(getNoInstances(len(nodeSet))),
faultyNodes)
return prepared1
def testPrimarySelectionAfterViewChange( # noqa
looper,
txnPoolNodeSet,
primaryReplicas,
catchup_complete_count):
"""
Test that primary replica of a protocol instance shifts to a new node after
a view change.
"""
# TODO: This test can fail due to view change.
ensure_view_change(looper, txnPoolNodeSet)
ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet)
for n in txnPoolNodeSet:
assert n.spylog.count(
n.allLedgersCaughtUp) > catchup_complete_count[n.name]
# Primary replicas before view change
prBeforeVC = primaryReplicas
# Primary replicas after view change
instanceCount = getNoInstances(nodeCount)
prAfterVC = [getPrimaryReplica(txnPoolNodeSet, i) for i in range(instanceCount)]
# Primary replicas have moved to the next node
for br, ar in zip(prBeforeVC, prAfterVC):
assert ar.node.rank - br.node.rank == 1
check_rank_consistent_across_each_node(txnPoolNodeSet)
checkProtocolInstanceSetup(looper, txnPoolNodeSet, retryWait=1)
def testPrimarySelectionAfterViewChange( # noqa
looper, txnPoolNodeSet, primaryReplicas, catchup_complete_count,
view_change_done):
"""
Test that primary replica of a protocol instance shifts to a new node after
a view change.
"""
# TODO: This test can fail due to view change.
for n in txnPoolNodeSet:
assert n.spylog.count(
n.allLedgersCaughtUp) > catchup_complete_count[n.name]
# Primary replicas before view change
prBeforeVC = primaryReplicas
# Primary replicas after view change
instanceCount = getNoInstances(nodeCount)
prAfterVC = [
getPrimaryReplica(txnPoolNodeSet, i) for i in range(instanceCount)
]
# Primary replicas have moved to the next node
for br, ar in zip(prBeforeVC, prAfterVC):
assert ar.node.rank - br.node.rank == 1
check_rank_consistent_across_each_node(txnPoolNodeSet)
checkProtocolInstanceSetup(looper, txnPoolNodeSet, retryWait=1)
def preprepared1(looper, nodeSet, propagated1, faultyNodes):
checkPrePrepared(looper,
nodeSet,
propagated1,
range(getNoInstances(len(nodeSet))),
faultyNodes)
return propagated1
def committed1(looper, txnPoolNodeSet, prepared1, faultyNodes):
checkCommitted(looper,
txnPoolNodeSet,
prepared1,
range(getNoInstances(len(txnPoolNodeSet))),
faultyNodes)
return prepared1
def preprepared1(looper, txnPoolNodeSet, propagated1, faultyNodes):
checkPrePrepared(looper,
txnPoolNodeSet,
propagated1,
range(getNoInstances(len(txnPoolNodeSet))),
faultyNodes)
return propagated1
def test_num_of_commit_msg_with_2_of_6_faulty(evilNodes, looper,
txnPoolNodeSet, prepared1,
noRetryReq):
with pytest.raises(AssertionError):
# To raise an error pass less than the actual number of faults
checkCommitted(looper, txnPoolNodeSet, prepared1,
range(getNoInstances(len(txnPoolNodeSet))), f)
def committed1(looper, txnPoolNodeSet, prepared1, faultyNodes):
checkCommitted(looper,
txnPoolNodeSet,
prepared1,
range(getNoInstances(len(txnPoolNodeSet))),
faultyNodes)
return prepared1
def committed1(looper, nodeSet, client1, prepared1, faultyNodes):
checkCommitted(looper,
nodeSet,
prepared1,
range(getNoInstances(len(nodeSet))),
faultyNodes)
return prepared1
def check_request_is_not_returned_to_nodes(txnPoolNodeSet, request):
instances = range(getNoInstances(len(txnPoolNodeSet)))
for node, inst_id in itertools.product(txnPoolNodeSet, instances):
checkRequestNotReturnedToNode(node,
request.identifier,
request.reqId,
inst_id)
def preprepared1WithDelay(looper, txnPoolNodeSet, propagated1, faultyNodes):
timeouts = waits.expectedPrePrepareTime(len(txnPoolNodeSet)) + delayPrePrepareSec
checkPrePrepared(looper,
txnPoolNodeSet,
propagated1,
range(getNoInstances(len(txnPoolNodeSet))),
faultyNodes,
timeout=timeouts)
def testNumOfCommitMsgsWithFPlusOneFaults(afterElection, looper,
nodeSet, prepared1, noRetryReq):
with pytest.raises(AssertionError):
checkCommited(looper,
nodeSet,
prepared1,
range(getNoInstances(len(nodeSet))),
faultyNodes)
def preprepared1WithDelay(looper, txnPoolNodeSet, propagated1, faultyNodes):
timeouts = waits.expectedPrePrepareTime(len(txnPoolNodeSet)) + delayPrePrepareSec
checkPrePrepared(looper,
txnPoolNodeSet,
propagated1,
range(getNoInstances(len(txnPoolNodeSet))),
faultyNodes,
timeout=timeouts)
def testNumOfCommitMsgsWithFPlusOneFaults(afterElection, looper,
txnPoolNodeSet, prepared1,
noRetryReq):
with pytest.raises(AssertionError):
# To raise an error pass less than the actual number of faults
checkCommitted(looper, txnPoolNodeSet, prepared1,
range(getNoInstances(len(txnPoolNodeSet))),
faultyNodes - 1)
def checkOrderedCount():
instances = getNoInstances(len(nodeSet))
resp = [
requestReturnedToNode(node, wallet1.defaultId, committed1.reqId,
instId) for node in nodeSet
for instId in range(instances)
]
assert resp.count(True) >= (len(nodeSet) - faultyNodes) * instances
def testNumOfCommitMsgsWithFPlusOneFaults(afterElection, looper,
nodeSet, prepared1, noRetryReq):
with pytest.raises(AssertionError):
checkCommitted(looper,
nodeSet,
prepared1,
range(getNoInstances(len(nodeSet))),
faultyNodes)
def test_num_of_prepare_2_of_6_faulty(evilNodes, looper,
txnPoolNodeSet, preprepared1, noRetryReq):
with pytest.raises(AssertionError):
# To raise an error pass less than the actual number of faults
checkPrepared(looper,
txnPoolNodeSet,
preprepared1,
range(getNoInstances(len(txnPoolNodeSet))),
f)
def testNumOfCommitMsgsWithFPlusOneFaults(afterElection, looper,
txnPoolNodeSet, prepared1, noRetryReq):
with pytest.raises(AssertionError):
# To raise an error pass less than the actual number of faults
checkCommitted(looper,
txnPoolNodeSet,
prepared1,
range(getNoInstances(len(txnPoolNodeSet))),
faultyNodes - 1)
def do_view_change_with_propagate_primary_on_one_delayed_node(
slow_node, nodes, looper, sdk_pool_handle, sdk_wallet_client):
slow_stasher = slow_node.nodeIbStasher
fast_nodes = [n for n in nodes if n != slow_node]
stashers = [n.nodeIbStasher for n in nodes]
# Get last prepared certificate in pool
lpc = last_prepared_certificate(nodes)
# Get pool current view no
view_no = lpc[0]
with delay_rules(slow_stasher, icDelay()):
with delay_rules(slow_stasher, vcd_delay()):
with delay_rules(stashers, cDelay()):
# Send request
request = sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client)
# Wait until this request is prepared on N-f nodes
looper.run(eventually(check_last_prepared_certificate_on_quorum, nodes, (lpc[0], lpc[1] + 1)))
# Trigger view change
for n in nodes:
n.view_changer.on_master_degradation()
# Wait until view change is completed on all nodes except slow one
waitForViewChange(looper,
fast_nodes,
expectedViewNo=view_no + 1,
customTimeout=waits.expectedPoolViewChangeStartedTimeout(len(nodes)))
wait_for_elections_done_on_given_nodes(looper,
fast_nodes,
getRequiredInstances(len(nodes)),
timeout=waits.expectedPoolElectionTimeout(len(nodes)))
# Now all the nodes receive Commits
# The slow node will accept Commits and order the 3PC-batch in the old view
looper.runFor(waits.expectedOrderingTime(getNoInstances(len(nodes))))
# Now slow node receives ViewChangeDones
waitForViewChange(looper,
[slow_node],
expectedViewNo=view_no + 1,
customTimeout=waits.expectedPoolViewChangeStartedTimeout(len(nodes)))
wait_for_elections_done_on_given_nodes(looper,
[slow_node],
getRequiredInstances(len(nodes)),
timeout=waits.expectedPoolElectionTimeout(len(nodes)))
# Now slow node receives InstanceChanges but discards them because already
# started propagate primary to the same view.
# Finish request gracefully
sdk_get_reply(looper, request)
def do_view_change_with_propagate_primary_on_one_delayed_node(
slow_node, nodes, looper, sdk_pool_handle, sdk_wallet_client):
slow_stasher = slow_node.nodeIbStasher
fast_nodes = [n for n in nodes if n != slow_node]
stashers = [n.nodeIbStasher for n in nodes]
# Get last prepared certificate in pool
lpc = last_prepared_certificate(nodes)
# Get pool current view no
view_no = lpc[0]
with delay_rules(slow_stasher, icDelay()):
with delay_rules(slow_stasher, vcd_delay()):
with delay_rules(stashers, cDelay()):
# Send request
request = sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client)
# Wait until this request is prepared on N-f nodes
looper.run(eventually(check_last_prepared_certificate_on_quorum, nodes, (lpc[0], lpc[1] + 1)))
# Trigger view change
for n in nodes:
n.view_changer.on_master_degradation()
# Wait until view change is completed on all nodes except slow one
waitForViewChange(looper,
fast_nodes,
expectedViewNo=view_no + 1,
customTimeout=waits.expectedPoolViewChangeStartedTimeout(len(nodes)))
wait_for_elections_done_on_given_nodes(looper,
fast_nodes,
getRequiredInstances(len(nodes)),
timeout=waits.expectedPoolElectionTimeout(len(nodes)))
# Now all the nodes receive Commits
# The slow node will accept Commits and order the 3PC-batch in the old view
looper.runFor(waits.expectedOrderingTime(getNoInstances(len(nodes))))
# Now slow node receives ViewChangeDones
waitForViewChange(looper,
[slow_node],
expectedViewNo=view_no + 1,
customTimeout=waits.expectedPoolViewChangeStartedTimeout(len(nodes)))
wait_for_elections_done_on_given_nodes(looper,
[slow_node],
getRequiredInstances(len(nodes)),
timeout=waits.expectedPoolElectionTimeout(len(nodes)))
# Now slow node receives InstanceChanges but discards them because already
# started propagate primary to the same view.
# Finish request gracefully
sdk_get_reply(looper, request)
def test_cannot_restore_last_sent_pp_seq_no_if_replica_absent(
txnPoolNodeSet, view_no_set, setup):
node = txnPoolNodeSet[0]
assert node.viewNo == 2
absent_replica_index = getNoInstances(nodeCount)
can = node.last_sent_pp_store_helper._can_restore_last_sent_pp_seq_no(
absent_replica_index, [2, 5])
assert can is False
def test_cannot_restore_last_sent_pp_seq_no_if_replica_absent(
txnPoolNodeSet, view_no_set, setup):
node = txnPoolNodeSet[0]
assert node.viewNo == 2
absent_replica_index = getNoInstances(nodeCount)
can = node.last_sent_pp_store_helper._can_restore_last_sent_pp_seq_no(
PrePrepareKey(inst_id=absent_replica_index, view_no=2, pp_seq_no=5))
assert can is False
def sdk_check_request_is_not_returned_to_nodes(looper, nodeSet, request):
instances = range(getNoInstances(len(nodeSet)))
coros = []
for node, inst_id in itertools.product(nodeSet, instances):
c = partial(checkRequestNotReturnedToNode,
node=node,
identifier=request['identifier'],
reqId=request['reqId'],
instId=inst_id)
coros.append(c)
timeout = waits.expectedTransactionExecutionTime(len(nodeSet))
looper.run(eventuallyAll(*coros, retryWait=1, totalTimeout=timeout))
def sdk_check_request_is_not_returned_to_nodes(looper, nodeSet, request):
instances = range(getNoInstances(len(nodeSet)))
coros = []
for node, inst_id in itertools.product(nodeSet, instances):
c = partial(checkRequestNotReturnedToNode,
node=node,
identifier=request['identifier'],
reqId=request['reqId'],
instId=inst_id
)
coros.append(c)
timeout = waits.expectedTransactionExecutionTime(len(nodeSet))
looper.run(eventuallyAll(*coros, retryWait=1, totalTimeout=timeout))
def testPrimarySelectionAfterViewChange(looper, nodeSet, ready, primaryReplicas,
viewChangeDone):
"""
Test that primary replica of a protocol instance shifts to a new node after a view change.
"""
# Primary replicas before view change
prBeforeVC = primaryReplicas
# Primary replicas after view change
instanceCount = getNoInstances(nodeCount)
prAfterVC = [getPrimaryReplica(nodeSet, i) for i in range(instanceCount)]
# Primary replicas have moved to the next node
for br, ar in zip(prBeforeVC, prAfterVC):
assert ar.node.rank - br.node.rank == 1
checkProtocolInstanceSetup(looper, nodeSet, retryWait=1, timeout=5)
def replied1(looper, nodeSet, client1, committed1, wallet1, faultyNodes):
numOfNodes = len(nodeSet)
numOfInstances = getNoInstances(numOfNodes)
quorum = numOfInstances * (numOfNodes - faultyNodes)
def checkOrderedCount():
resp = [
requestReturnedToNode(node, wallet1.defaultId, committed1.reqId,
instId) for node in nodeSet
for instId in range(numOfInstances)
]
assert resp.count(True) >= quorum
orderingTimeout = waits.expectedOrderingTime(numOfInstances)
looper.run(
eventually(checkOrderedCount, retryWait=1, timeout=orderingTimeout))
waitForSufficientRepliesForRequests(looper, client1, requests=[committed1])
return committed1
def replied1(looper, nodeSet, client1, committed1):
for instId in range(getNoInstances(len(nodeSet))):
getPrimaryReplica(nodeSet, instId)
looper.run(*[eventually(checkRequestReturnedToNode,
node,
client1.defaultIdentifier,
committed1.reqId,
committed1.digest,
instId,
retryWait=1, timeout=30)
for node in nodeSet])
looper.run(eventually(
checkSufficientRepliesRecvd,
client1.inBox,
committed1.reqId,
2,
retryWait=2,
timeout=30))
return committed1
def replied1(looper, txnPoolNodeSet, sdk_wallet_client, committed1,
faultyNodes, sent1):
numOfNodes = len(txnPoolNodeSet)
numOfInstances = getNoInstances(numOfNodes)
quorum = numOfInstances * (numOfNodes - faultyNodes)
_, did = sdk_wallet_client
def checkOrderedCount():
resp = [
requestReturnedToNode(node, committed1.digest, instId)
for node in txnPoolNodeSet for instId in range(numOfInstances)
]
assert resp.count(True) >= quorum
orderingTimeout = waits.expectedOrderingTime(numOfInstances)
looper.run(
eventually(checkOrderedCount, retryWait=1, timeout=orderingTimeout))
sdk_get_and_check_replies(looper, sent1)
return committed1
def replied1(looper, txnPoolNodeSet, sdk_wallet_client,
committed1, faultyNodes, sent1):
numOfNodes = len(txnPoolNodeSet)
numOfInstances = getNoInstances(numOfNodes)
quorum = numOfInstances * (numOfNodes - faultyNodes)
_, did = sdk_wallet_client
def checkOrderedCount():
resp = [requestReturnedToNode(node,
committed1.digest,
instId)
for node in txnPoolNodeSet for instId in range(numOfInstances)]
assert resp.count(True) >= quorum
orderingTimeout = waits.expectedOrderingTime(numOfInstances)
looper.run(eventually(checkOrderedCount,
retryWait=1,
timeout=orderingTimeout))
sdk_get_and_check_replies(looper, sent1)
return committed1
def replied1(looper, nodeSet, client1, committed1, wallet1):
for instId in range(getNoInstances(len(nodeSet))):
getPrimaryReplica(nodeSet, instId)
looper.run(*[
eventually(checkRequestReturnedToNode,
node,
wallet1.defaultId,
committed1.reqId,
committed1.digest,
instId,
retryWait=1,
timeout=30) for node in nodeSet
])
looper.run(
eventually(checkSufficientRepliesRecvd,
client1.inBox,
committed1.reqId,
2,
retryWait=2,
timeout=30))
return committed1
def primaryReplicas(nodeSet, up):
instanceCount = getNoInstances(nodeCount)
return [getPrimaryReplica(nodeSet, i) for i in range(instanceCount)]
def primaryReplicas(txnPoolNodeSet):
instanceCount = getNoInstances(nodeCount)
return [getPrimaryReplica(txnPoolNodeSet, i) for i in range(instanceCount)]
def checkOrderedCount():
instances = getNoInstances(len(nodeSet))
resp = [requestReturnedToNode(node, wallet1.defaultId,
committed1.reqId, instId) for
node in nodeSet for instId in range(instances)]
assert resp.count(True) >= (len(nodeSet) - faultyNodes)*instances
def check_request_is_not_returned_to_nodes(txnPoolNodeSet, request):
instances = range(getNoInstances(len(txnPoolNodeSet)))
for node, inst_id in itertools.product(txnPoolNodeSet, instances):
checkRequestNotReturnedToNode(node,
request.key,
inst_id)
