def testNonPrimarySendsAPrePrepare(looper, nodeSet, setup, propagated1):
primaryReplica = getPrimaryReplica(nodeSet, instId)
nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet, instId)
firstNpr = nonPrimaryReplicas[0]
remainingNpr = nonPrimaryReplicas[1:]
def sendPrePrepareFromNonPrimary(replica):
firstNpr.sendPrePrepare(propagated1.reqDigest)
return PrePrepare(
replica.instId,
firstNpr.viewNo,
firstNpr.prePrepareSeqNo,
propagated1.clientId,
propagated1.reqId,
propagated1.digest)
ppr = sendPrePrepareFromNonPrimary(firstNpr)
def chk():
for r in (primaryReplica, *remainingNpr):
recvdPps = recvdPrePrepare(r)
assert len(recvdPps) == 1
assert recvdPps[0]['pp'] == ppr
nodeSuspicions = len(getNodeSuspicions(
r.node, Suspicions.PPR_FRM_NON_PRIMARY.code))
assert nodeSuspicions == 1
looper.run(eventually(chk,
retryWait=.5, timeout=5))
looper.run(eventually(partial(checkViewNoForNodes, nodeSet, 1), retryWait=1,
timeout=20))
def testQueueingReqFromFutureView(delayedPerf, looper, nodeSet, up, client1):
"""
Test if every node queues 3 Phase requests(PRE-PREPARE, PREPARE and COMMIT)
that come from a view which is greater than the current view
"""
f = getMaxFailures(nodeCount)
# Delay processing of instance change on a node
nodeA = nodeSet.Alpha
nodeA.nodeIbStasher.delay(icDelay(60))
nonPrimReps = getNonPrimaryReplicas(nodeSet, 0)
# Delay processing of PRE-PREPARE from all non primary replicas of master
# so master's throughput falls and view changes
ppDelayer = ppDelay(5, 0)
for r in nonPrimReps:
r.node.nodeIbStasher.delay(ppDelayer)
sendReqsToNodesAndVerifySuffReplies(looper, client1, 4, timeout=5 * nodeCount)
# Every node except Node A should have a view change
for node in nodeSet:
if node.name != nodeA.name:
looper.run(eventually(
partial(checkViewChangeInitiatedForNode, node, 0),
retryWait=1,
timeout=20))
# Node A's view should not have changed yet
with pytest.raises(AssertionError):
looper.run(eventually(partial(
checkViewChangeInitiatedForNode, nodeA, 0),
retryWait=1,
timeout=20))
# NodeA should not have any pending 3 phase request for a later view
for r in nodeA.replicas: # type: TestReplica
assert len(r.threePhaseMsgsForLaterView) == 0
# Reset delays on incoming messages from all nodes
for node in nodeSet:
node.nodeIbStasher.nodelay(ppDelayer)
# Send one more request
sendRandomRequest(client1)
def checkPending3PhaseReqs():
# Get all replicas that have their primary status decided
reps = [rep for rep in nodeA.replicas if rep.isPrimary is not None]
# Atleast one replica should have its primary status decided
assert len(reps) > 0
for r in reps: # type: TestReplica
logging.debug("primary status for replica {} is {}"
.format(r, r.primaryNames))
assert len(r.threePhaseMsgsForLaterView) > 0
# NodeA should now have pending 3 phase request for a later view
looper.run(eventually(checkPending3PhaseReqs, retryWait=1, timeout=30))
def setup(nodeSet, up):
primaryRep, nonPrimaryReps = getPrimaryReplica(nodeSet, 0), \
getNonPrimaryReplicas(nodeSet, 0)
# The primary replica would send PRE-PREPARE messages with incorrect digest
makeNodeFaulty(primaryRep.node, partial(send3PhaseMsgWithIncorrectDigest,
msgType=PrePrepare))
return adict(primaryRep=primaryRep, nonPrimaryReps=nonPrimaryReps)
def sendPrepareFromPrimary(instId):
primary = getPrimaryReplica(nodeSet, instId)
preprepared1.viewNo = instId
preprepared1.ppSeqNo = primary.prePrepareSeqNo
primary.sendPrepare(preprepared1)
for r in getNonPrimaryReplicas(nodeSet, instId):
l = len([param for param in getAllArgs(r, r.processPrepare)
if param['sender'] == primary.name])
assert l == 1
sendPrepareFromPrimary(0)
def setup(nodeSet, up):
primaryRep, nonPrimaryReps = getPrimaryReplica(nodeSet, 0), \
getNonPrimaryReplicas(nodeSet, 0)
# A non primary replica sends PREPARE messages with incorrect digest
faultyRep = nonPrimaryReps[0]
makeNodeFaulty(faultyRep.node, partial(send3PhaseMsgWithIncorrectDigest,
msgType=Prepare, instId=0))
return adict(primaryRep=primaryRep, nonPrimaryReps=nonPrimaryReps,
faultyRep=faultyRep)
def testReplicasRejectSamePrePrepareMsg(looper, nodeSet, client1):
"""
Replicas should not accept PRE-PREPARE for view "v" and prepare sequence
number "n" if it has already accepted a request with view number "v" and
sequence number "n"
"""
numOfNodes = 4
fValue = getMaxFailures(numOfNodes)
request1 = sendRandomRequest(client1)
result1 = looper.run(
eventually(checkSufficientRepliesRecvd, client1.inBox,
request1.reqId, fValue,
retryWait=1, timeout=5))
logging.debug("request {} gives result {}".format(request1, result1))
primaryRepl = getPrimaryReplica(nodeSet)
logging.debug("Primary Replica: {}".format(primaryRepl))
logging.debug(
"Decrementing the primary replica's pre-prepare sequence number by one...")
primaryRepl.prePrepareSeqNo -= 1
request2 = sendRandomRequest(client1)
looper.run(eventually(checkPrePrepareReqSent, primaryRepl, request2,
retryWait=1, timeout=10))
nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet)
logging.debug("Non Primary Replicas: " + str(nonPrimaryReplicas))
prePrepareReq = PrePrepare(
primaryRepl.instId,
primaryRepl.viewNo,
primaryRepl.prePrepareSeqNo,
client1.clientId,
request2.reqId,
request2.digest)
logging.debug("""Checking whether all the non primary replicas have received
the pre-prepare request with same sequence number""")
looper.run(eventually(checkPrePrepareReqRecvd,
nonPrimaryReplicas,
prePrepareReq,
retryWait=1,
timeout=10))
logging.debug("""Check that none of the non primary replicas didn't send any prepare message "
in response to the pre-prepare message""")
for npr in nonPrimaryReplicas:
with pytest.raises(AssertionError):
looper.run(eventually(checkPrepareReqSent,
npr,
client1.clientId,
request2.reqId,
retryWait=1,
timeout=10))
def viewChangeDone(nodeSet, looper, up, client1):
"""
Test that a view change is done when the performance of master goes down
"""
# Delay processing of PRE-PREPARE from all non primary replicas of master
# so master's performance falls and view changes
nonPrimReps = getNonPrimaryReplicas(nodeSet, 0)
for r in nonPrimReps:
r.node.nodeIbStasher.delay(ppDelay(10, 0))
sendReqsToNodesAndVerifySuffReplies(looper, client1, 4)
looper.run(eventually(partial(checkViewNoForNodes, nodeSet, 1),
retryWait=1, timeout=20))
def setup(nodeSet, up):
primaryRep, nonPrimaryReps = getPrimaryReplica(nodeSet, 0), \
getNonPrimaryReplicas(nodeSet, 0)
# The primary replica would send 3 duplicate PRE-PREPARE requests to
# non primary replicas
makeNodeFaulty(primaryRep.node, partial(sendDuplicate3PhaseMsg,
msgType=PrePrepare, count=3))
# The node of the primary replica above should not be blacklisted by any
# other node since we are simulating multiple PRE-PREPARE messages and
# want to check for a particular suspicion
# for node in nodeSet:
# if node != primaryRep.node:
# node.whitelistNode(primaryRep.node.name,
# Suspicions.DUPLICATE_PPR_SENT.code)
return adict(primaryRep=primaryRep, nonPrimaryReps=nonPrimaryReps)
def testPrimarySendsAPrepareAndMarkedSuspicious(looper, nodeSet, preprepared1):
def sendPrepareFromPrimary(instId):
primary = getPrimaryReplica(nodeSet, instId)
preprepared1.viewNo = instId
preprepared1.ppSeqNo = primary.prePrepareSeqNo
primary.sendPrepare(preprepared1)
for r in getNonPrimaryReplicas(nodeSet, instId):
l = len([param for param in getAllArgs(r, r.processPrepare)
if param['sender'] == primary.name])
assert l == 1
sendPrepareFromPrimary(0)
for node in nodeSet:
if node in getNonPrimaryReplicas(nodeSet, 0):
frm, reason, code = getAllArgs(node, Node.reportSuspiciousNode)
assert frm == getPrimaryReplica(nodeSet, 0).node.name
assert isinstance(reason, SuspiciousNode)
assert len(getNodeSuspicions(node, Suspicions.PR_FRM_PRIMARY.code)) \
== 10
def testElectionsAfterViewChange(delayedPerf, looper: Looper, nodeSet: TestNodeSet, up, client1):
"""
Test that a primary election does happen after a view change
"""
# Delay processing of PRE-PREPARE from all non primary replicas of master so master's throughput falls
# and view changes
nonPrimReps = getNonPrimaryReplicas(nodeSet, 0)
for r in nonPrimReps:
r.node.nodeIbStasher.delay(ppDelay(10, 0))
sendReqsToNodesAndVerifySuffReplies(looper, client1, 4)
# Ensure view change happened for both node and its primary elector
for node in nodeSet:
looper.run(eventually(partial(checkViewChangeInitiatedForNode, node, 0), retryWait=1, timeout=20))
# Ensure elections are done again and pool is setup again with appropriate protocol instances and each
# protocol instance is setup properly too
checkProtocolInstanceSetup(looper, nodeSet, retryWait=1, timeout=30)
def setup(nodeSet, up):
primaryRep, nonPrimaryReps = getPrimaryReplica(nodeSet, 0), \
getNonPrimaryReplicas(nodeSet, 0)
# A non primary replica sends duplicate PREPARE requests to all other
# replicas
faultyRep = nonPrimaryReps[0]
makeNodeFaulty(faultyRep.node, partial(sendDuplicate3PhaseMsg,
msgType=Prepare, count=3,
instId=0))
# The node of the primary replica above should not be blacklisted by any
# other node since we are simulating multiple PREPARE messages and
# want to check for a particular suspicion
whitelistNode(faultyRep.node.name,
[node for node in nodeSet if node != faultyRep.node],
Suspicions.DUPLICATE_PR_SENT.code)
return adict(primaryRep=primaryRep, nonPrimaryReps=nonPrimaryReps,
faultyRep=faultyRep)
def testViewNotChanged(looper: Looper, nodeSet: TestNodeSet, up, client1):
"""
Test that a view change is not done when the performance of master does
not go down
"""
"""
Send multiple requests to the client and delay some requests by all
backup instances to ensure master instance
is always faster than backup instances and there is no view change
"""
# Delay PRE-PREPARE for all backup protocol instances so master performs
# better
for i in range(1, F + 1):
nonPrimReps = getNonPrimaryReplicas(nodeSet, i)
# type: Iterable[TestReplica]
for r in nonPrimReps:
r.node.nodeIbStasher.delay(ppDelay(10, i))
sendReqsToNodesAndVerifySuffReplies(looper, client1, 5)
checkViewNoForNodes(nodeSet, 0)
def g(instId):
primary = getPrimaryReplica(nodeSet, instId)
nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet, instId)
def primarySeesCorrectNumberOfPREPREPAREs():
"""
no of PRE-PREPARE as seen by processPrePrepare
method for primary must be 0 with or without faults in system
"""
l1 = len([param for param in
getAllArgs(primary, primary.processPrePrepare)])
assert l1 == 0
def nonPrimarySeesCorrectNumberOfPREPREPAREs():
"""
1. no of PRE-PREPARE as seen by processPrePrepare method for
non-primaries must be 1; whn zero faulty nodes in system.
2. no of PRE-PREPARE as seen by processPrePrepare method for
non-primaries must be greater than or equal to 0;
with faults in system.
"""
expectedPrePrepareRequest = PrePrepare(
instId,
primary.viewNo,
primary.prePrepareSeqNo,
propagated1.clientId,
propagated1.reqId,
propagated1.digest)
passes = 0
for npr in nonPrimaryReplicas:
actualMsgs = len([param for param in
getAllArgs(npr, npr.processPrePrepare)
if (param['pp'],
param['sender']) == (
expectedPrePrepareRequest,
primary.name)])
numOfMsgsWithZFN = 1
numOfMsgsWithFaults = 0
passes += int(msgCountOK(nodesSize,
faultyNodes,
actualMsgs,
numOfMsgsWithZFN,
numOfMsgsWithFaults))
assert passes >= len(nonPrimaryReplicas) - faultyNodes
def primarySentsCorrectNumberOfPREPREPAREs():
"""
1. no of PRE-PREPARE sent by primary is 1 with or without
fault in system but, when primary is faulty no of sent PRE_PREPARE
will be zero and primary must be marked as malicious.
"""
actualMsgs = len([param for param in
getAllArgs(primary, primary.sendPrePrepare)
if (param['reqDigest'].clientId,
param['reqDigest'].reqId,
param['reqDigest'].digest) ==
(propagated1.clientId,
propagated1.reqId,
propagated1.digest)
])
numOfMsgsWithZFN = 1
# TODO: Considering, Primary is not faulty and will always send
# PRE-PREPARE. Write separate test for testing when Primary
# is faulty
assert msgCountOK(nodesSize,
faultyNodes,
actualMsgs,
numOfMsgsWithZFN,
numOfMsgsWithZFN)
def nonPrimaryReceivesCorrectNumberOfPREPREPAREs():
"""
1. no of PRE-PREPARE received by non-primaries must be 1
with zero faults in system, and 0 faults in system.
"""
passes = 0
for npr in nonPrimaryReplicas:
l4 = len([param for param in
getAllArgs(npr, npr.addToPrePrepares)
if (param['pp'].clientId,
param['pp'].reqId,
param['pp'].digest) == (
propagated1.clientId,
propagated1.reqId,
propagated1.digest)])
numOfMsgsWithZFN = 1
numOfMsgsWithFaults = 0
passes += msgCountOK(nodesSize,
faultyNodes,
l4,
numOfMsgsWithZFN,
numOfMsgsWithFaults)
assert passes >= len(nonPrimaryReplicas) - faultyNodes
primarySeesCorrectNumberOfPREPREPAREs()
nonPrimarySeesCorrectNumberOfPREPREPAREs()
primarySentsCorrectNumberOfPREPREPAREs()
nonPrimaryReceivesCorrectNumberOfPREPREPAREs()
def g(instId):
allReplicas = getAllReplicas(nodeSet, instId)
primary = getPrimaryReplica(nodeSet, instId)
nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet, instId)
def primaryDontSendAnyPREPAREs():
"""
1. no of PREPARE sent by primary should be 0
"""
for r in allReplicas:
for param in getAllArgs(r, Replica.processPrepare):
sender = param['sender']
assert sender != primary.name
def allReplicasSeeCorrectNumberOfPREPAREs():
"""
1. no of PREPARE received by replicas must be n - 1;
n = num of nodes without fault, and greater than or equal to
2f with faults.
"""
passes = 0
numOfMsgsWithZFN = nodeCount - 1
numOfMsgsWithFaults = 2 * f
for replica in allReplicas:
key = primary.viewNo, primary.prePrepareSeqNo
if key in replica.prepares:
actualMsgs = len(replica.prepares[key].voters)
passes += int(msgCountOK(nodeCount,
faultyNodes,
actualMsgs,
numOfMsgsWithZFN,
numOfMsgsWithFaults))
assert passes >= len(allReplicas) - faultyNodes
def primaryReceivesCorrectNumberOfPREPAREs():
"""
num of PREPARE seen by primary replica is n - 1;
n = num of nodes without fault, and greater than or equal to
2f with faults.
"""
actualMsgs = len([param for param in
getAllArgs(primary,
primary.processPrepare)
if (param['prepare'].instId,
param['prepare'].viewNo,
param['prepare'].ppSeqNo) == (primary.instId,
primary.viewNo,
primary.prePrepareSeqNo) and
param['sender'] != primary.name])
numOfMsgsWithZFN = nodeCount - 1
numOfMsgsWithFaults = 2 * f - 1
assert msgCountOK(nodeCount,
faultyNodes,
actualMsgs,
numOfMsgsWithZFN,
numOfMsgsWithFaults)
# TODO what if the primary is faulty?
def nonPrimaryReplicasReceiveCorrectNumberOfPREPAREs():
"""
num of PREPARE seen by Non primary replica is n - 2 without
faults and 2f - 1 with faults.
"""
passes = 0
numOfMsgsWithZFN = nodeCount - 2
numOfMsgsWithFaults = (2 * f) - 1
for npr in nonPrimaryReplicas:
actualMsgs = len([param for param in
getAllArgs(
npr,
npr.processPrepare)
if (param['prepare'].instId,
param['prepare'].viewNo,
param['prepare'].ppSeqNo) == (primary.instId,
primary.viewNo,
primary.prePrepareSeqNo)
])
passes += int(msgCountOK(nodeCount,
faultyNodes,
actualMsgs,
numOfMsgsWithZFN,
numOfMsgsWithFaults))
assert passes >= len(nonPrimaryReplicas) - faultyNodes
# TODO how do we know if one of the faulty nodes is a primary or
# not?
primaryDontSendAnyPREPAREs()
allReplicasSeeCorrectNumberOfPREPAREs()
primaryReceivesCorrectNumberOfPREPAREs()
nonPrimaryReplicasReceiveCorrectNumberOfPREPAREs()
