def testNonPrimarySendsAPrePrepare(looper, nodeSet, setup, propagated1):
primaryReplica = getPrimaryReplica(nodeSet, instId)
nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet, instId)
firstNpr = nonPrimaryReplicas[0]
remainingNpr = nonPrimaryReplicas[1:]
def sendPrePrepareFromNonPrimary(replica):
firstNpr.sendPrePrepare(propagated1.reqDigest)
return PrePrepare(
replica.instId,
firstNpr.viewNo,
firstNpr.prePrepareSeqNo,
propagated1.clientId,
propagated1.reqId,
propagated1.digest)
ppr = sendPrePrepareFromNonPrimary(firstNpr)
def chk():
for r in (primaryReplica, *remainingNpr):
recvdPps = recvdPrePrepare(r)
assert len(recvdPps) == 1
assert recvdPps[0]['pp'] == ppr
nodeSuspicions = len(getNodeSuspicions(
r.node, Suspicions.PPR_FRM_NON_PRIMARY.code))
assert nodeSuspicions == 1
looper.run(eventually(chk,
retryWait=.5, timeout=5))
looper.run(eventually(partial(checkViewNoForNodes, nodeSet, 1), retryWait=1,
timeout=20))
def g(instId):
allReplicas = getAllReplicas(nodeSet, instId)
primaryReplica = getPrimaryReplica(nodeSet, instId)
def replicasSeesCorrectNumOfCOMMITs():
"""
num of commit messages must be = n when zero fault;
n = num of nodes and greater than or equal to
2f + 1 with faults.
"""
passes = 0
numOfMsgsWithZFN = nodeCount
numOfMsgsWithFault = (2 * f) + 1
key = (primaryReplica.viewNo, primaryReplica.prePrepareSeqNo)
for r in allReplicas:
if key in r.commits:
rcvdCommitRqst = r.commits[key]
assert rcvdCommitRqst[0] == prepared1.digest
actualMsgsReceived = len(rcvdCommitRqst[1])
passes += int(msgCountOK(nodeCount,
faultyNodes,
actualMsgsReceived,
numOfMsgsWithZFN,
numOfMsgsWithFault))
assert passes >= len(allReplicas) - faultyNodes
def replicasReceivesCorrectNumberOfCOMMITs():
"""
num of commit messages seen by replica must be equal to n - 1;
when zero fault and greater than or equal to
2f+1 with faults.
"""
passes = 0
numOfMsgsWithZFN = nodeCount - 1
numOfMsgsWithFault = 2 * f
for r in allReplicas:
args = getAllArgs(r, r.processCommit)
actualMsgsReceived = len(args)
passes += int(msgCountOK(nodeCount,
faultyNodes,
actualMsgsReceived,
numOfMsgsWithZFN,
numOfMsgsWithFault))
for arg in args:
assert arg['commit'].viewNo == primaryReplica.viewNo and \
arg['commit'].ppSeqNo == primaryReplica.prePrepareSeqNo and \
arg['commit'].digest == prepared1.digest
assert r.name != arg['sender']
assert passes >= len(allReplicas) - faultyNodes
replicasReceivesCorrectNumberOfCOMMITs()
replicasSeesCorrectNumOfCOMMITs()
def setup(nodeSet, up):
def dontSendPrePrepareRequest(self, reqDigest: ReqDigest):
logger.debug("EVIL: {} not sending pre-prepare message for request {}".
format(self.name, reqDigest))
return
pr = getPrimaryReplica(nodeSet, instId)
evilMethod = types.MethodType(dontSendPrePrepareRequest, pr)
pr.sendPrePrepare = evilMethod
def setup(nodeSet, up):
primaryRep, nonPrimaryReps = getPrimaryReplica(nodeSet, 0), \
getNonPrimaryReplicas(nodeSet, 0)
# The primary replica would send PRE-PREPARE messages with incorrect digest
makeNodeFaulty(primaryRep.node, partial(send3PhaseMsgWithIncorrectDigest,
msgType=PrePrepare))
return adict(primaryRep=primaryRep, nonPrimaryReps=nonPrimaryReps)
def sendPrepareFromPrimary(instId):
primary = getPrimaryReplica(nodeSet, instId)
preprepared1.viewNo = instId
preprepared1.ppSeqNo = primary.prePrepareSeqNo
primary.sendPrepare(preprepared1)
for r in getNonPrimaryReplicas(nodeSet, instId):
l = len([param for param in getAllArgs(r, r.processPrepare)
if param['sender'] == primary.name])
assert l == 1
sendPrepareFromPrimary(0)
def setup(nodeSet, up):
primaryRep, nonPrimaryReps = getPrimaryReplica(nodeSet, 0), \
getNonPrimaryReplicas(nodeSet, 0)
# A non primary replica sends PREPARE messages with incorrect digest
faultyRep = nonPrimaryReps[0]
makeNodeFaulty(faultyRep.node, partial(send3PhaseMsgWithIncorrectDigest,
msgType=Prepare, instId=0))
return adict(primaryRep=primaryRep, nonPrimaryReps=nonPrimaryReps,
faultyRep=faultyRep)
def testReplicasRejectSamePrePrepareMsg(looper, nodeSet, client1):
"""
Replicas should not accept PRE-PREPARE for view "v" and prepare sequence
number "n" if it has already accepted a request with view number "v" and
sequence number "n"
"""
numOfNodes = 4
fValue = getMaxFailures(numOfNodes)
request1 = sendRandomRequest(client1)
result1 = looper.run(
eventually(checkSufficientRepliesRecvd, client1.inBox,
request1.reqId, fValue,
retryWait=1, timeout=5))
logging.debug("request {} gives result {}".format(request1, result1))
primaryRepl = getPrimaryReplica(nodeSet)
logging.debug("Primary Replica: {}".format(primaryRepl))
logging.debug(
"Decrementing the primary replica's pre-prepare sequence number by one...")
primaryRepl.prePrepareSeqNo -= 1
request2 = sendRandomRequest(client1)
looper.run(eventually(checkPrePrepareReqSent, primaryRepl, request2,
retryWait=1, timeout=10))
nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet)
logging.debug("Non Primary Replicas: " + str(nonPrimaryReplicas))
prePrepareReq = PrePrepare(
primaryRepl.instId,
primaryRepl.viewNo,
primaryRepl.prePrepareSeqNo,
client1.clientId,
request2.reqId,
request2.digest)
logging.debug("""Checking whether all the non primary replicas have received
the pre-prepare request with same sequence number""")
looper.run(eventually(checkPrePrepareReqRecvd,
nonPrimaryReplicas,
prePrepareReq,
retryWait=1,
timeout=10))
logging.debug("""Check that none of the non primary replicas didn't send any prepare message "
in response to the pre-prepare message""")
for npr in nonPrimaryReplicas:
with pytest.raises(AssertionError):
looper.run(eventually(checkPrepareReqSent,
npr,
client1.clientId,
request2.reqId,
retryWait=1,
timeout=10))
def testPrimarySelectionAfterViewChange(looper, nodeSet, ready, primaryReplicas,
viewChangeDone):
"""
Test that primary replica of a protocol instance shifts to a new node after a view change.
"""
# Primary replicas before view change
prBeforeVC = primaryReplicas
# Primary replicas after view change
instanceCount = getNoInstances(nodeCount)
prAfterVC = [getPrimaryReplica(nodeSet, i) for i in range(instanceCount)]
# Primary replicas have moved to the next node
for br, ar in zip(prBeforeVC, prAfterVC):
assert ar.node.rank - br.node.rank == 1
checkProtocolInstanceSetup(looper, nodeSet, retryWait=1, timeout=5)
def setup(nodeSet, up):
primaryRep, nonPrimaryReps = getPrimaryReplica(nodeSet, 0), \
getNonPrimaryReplicas(nodeSet, 0)
# The primary replica would send 3 duplicate PRE-PREPARE requests to
# non primary replicas
makeNodeFaulty(primaryRep.node, partial(sendDuplicate3PhaseMsg,
msgType=PrePrepare, count=3))
# The node of the primary replica above should not be blacklisted by any
# other node since we are simulating multiple PRE-PREPARE messages and
# want to check for a particular suspicion
# for node in nodeSet:
# if node != primaryRep.node:
# node.whitelistNode(primaryRep.node.name,
# Suspicions.DUPLICATE_PPR_SENT.code)
return adict(primaryRep=primaryRep, nonPrimaryReps=nonPrimaryReps)
def testPrimarySendsAPrepareAndMarkedSuspicious(looper, nodeSet, preprepared1):
def sendPrepareFromPrimary(instId):
primary = getPrimaryReplica(nodeSet, instId)
preprepared1.viewNo = instId
preprepared1.ppSeqNo = primary.prePrepareSeqNo
primary.sendPrepare(preprepared1)
for r in getNonPrimaryReplicas(nodeSet, instId):
l = len([param for param in getAllArgs(r, r.processPrepare)
if param['sender'] == primary.name])
assert l == 1
sendPrepareFromPrimary(0)
for node in nodeSet:
if node in getNonPrimaryReplicas(nodeSet, 0):
frm, reason, code = getAllArgs(node, Node.reportSuspiciousNode)
assert frm == getPrimaryReplica(nodeSet, 0).node.name
assert isinstance(reason, SuspiciousNode)
assert len(getNodeSuspicions(node, Suspicions.PR_FRM_PRIMARY.code)) \
== 10
def replied1(looper, nodeSet, client1, committed1):
for instId in range(getNoInstances(len(nodeSet))):
primaryReplica = getPrimaryReplica(nodeSet, instId)
looper.run(*[eventually(checkRequestReturnedToNode,
node,
client1.clientId,
committed1.reqId,
committed1.digest,
instId,
retryWait=1, timeout=30)
for node in nodeSet])
looper.run(eventually(
checkSufficientRepliesRecvd,
client1.inBox,
committed1.reqId,
2,
retryWait=2, timeout=30))
return committed1
def setup(nodeSet, up):
primaryRep, nonPrimaryReps = getPrimaryReplica(nodeSet, 0), \
getNonPrimaryReplicas(nodeSet, 0)
# A non primary replica sends duplicate PREPARE requests to all other
# replicas
faultyRep = nonPrimaryReps[0]
makeNodeFaulty(faultyRep.node, partial(sendDuplicate3PhaseMsg,
msgType=Prepare, count=3,
instId=0))
# The node of the primary replica above should not be blacklisted by any
# other node since we are simulating multiple PREPARE messages and
# want to check for a particular suspicion
whitelistNode(faultyRep.node.name,
[node for node in nodeSet if node != faultyRep.node],
Suspicions.DUPLICATE_PR_SENT.code)
return adict(primaryRep=primaryRep, nonPrimaryReps=nonPrimaryReps,
faultyRep=faultyRep)
def g(instId):
primary = getPrimaryReplica(nodeSet, instId)
nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet, instId)
def primarySeesCorrectNumberOfPREPREPAREs():
"""
no of PRE-PREPARE as seen by processPrePrepare
method for primary must be 0 with or without faults in system
"""
l1 = len([param for param in
getAllArgs(primary, primary.processPrePrepare)])
assert l1 == 0
def nonPrimarySeesCorrectNumberOfPREPREPAREs():
"""
1. no of PRE-PREPARE as seen by processPrePrepare method for
non-primaries must be 1; whn zero faulty nodes in system.
2. no of PRE-PREPARE as seen by processPrePrepare method for
non-primaries must be greater than or equal to 0;
with faults in system.
"""
expectedPrePrepareRequest = PrePrepare(
instId,
primary.viewNo,
primary.prePrepareSeqNo,
propagated1.clientId,
propagated1.reqId,
propagated1.digest)
passes = 0
for npr in nonPrimaryReplicas:
actualMsgs = len([param for param in
getAllArgs(npr, npr.processPrePrepare)
if (param['pp'],
param['sender']) == (
expectedPrePrepareRequest,
primary.name)])
numOfMsgsWithZFN = 1
numOfMsgsWithFaults = 0
passes += int(msgCountOK(nodesSize,
faultyNodes,
actualMsgs,
numOfMsgsWithZFN,
numOfMsgsWithFaults))
assert passes >= len(nonPrimaryReplicas) - faultyNodes
def primarySentsCorrectNumberOfPREPREPAREs():
"""
1. no of PRE-PREPARE sent by primary is 1 with or without
fault in system but, when primary is faulty no of sent PRE_PREPARE
will be zero and primary must be marked as malicious.
"""
actualMsgs = len([param for param in
getAllArgs(primary, primary.sendPrePrepare)
if (param['reqDigest'].clientId,
param['reqDigest'].reqId,
param['reqDigest'].digest) ==
(propagated1.clientId,
propagated1.reqId,
propagated1.digest)
])
numOfMsgsWithZFN = 1
# TODO: Considering, Primary is not faulty and will always send
# PRE-PREPARE. Write separate test for testing when Primary
# is faulty
assert msgCountOK(nodesSize,
faultyNodes,
actualMsgs,
numOfMsgsWithZFN,
numOfMsgsWithZFN)
def nonPrimaryReceivesCorrectNumberOfPREPREPAREs():
"""
1. no of PRE-PREPARE received by non-primaries must be 1
with zero faults in system, and 0 faults in system.
"""
passes = 0
for npr in nonPrimaryReplicas:
l4 = len([param for param in
getAllArgs(npr, npr.addToPrePrepares)
if (param['pp'].clientId,
param['pp'].reqId,
param['pp'].digest) == (
propagated1.clientId,
propagated1.reqId,
propagated1.digest)])
numOfMsgsWithZFN = 1
numOfMsgsWithFaults = 0
passes += msgCountOK(nodesSize,
faultyNodes,
l4,
numOfMsgsWithZFN,
numOfMsgsWithFaults)
assert passes >= len(nonPrimaryReplicas) - faultyNodes
primarySeesCorrectNumberOfPREPREPAREs()
nonPrimarySeesCorrectNumberOfPREPREPAREs()
primarySentsCorrectNumberOfPREPREPAREs()
nonPrimaryReceivesCorrectNumberOfPREPREPAREs()
def g(instId):
allReplicas = getAllReplicas(nodeSet, instId)
primary = getPrimaryReplica(nodeSet, instId)
nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet, instId)
def primaryDontSendAnyPREPAREs():
"""
1. no of PREPARE sent by primary should be 0
"""
for r in allReplicas:
for param in getAllArgs(r, Replica.processPrepare):
sender = param['sender']
assert sender != primary.name
def allReplicasSeeCorrectNumberOfPREPAREs():
"""
1. no of PREPARE received by replicas must be n - 1;
n = num of nodes without fault, and greater than or equal to
2f with faults.
"""
passes = 0
numOfMsgsWithZFN = nodeCount - 1
numOfMsgsWithFaults = 2 * f
for replica in allReplicas:
key = primary.viewNo, primary.prePrepareSeqNo
if key in replica.prepares:
actualMsgs = len(replica.prepares[key].voters)
passes += int(msgCountOK(nodeCount,
faultyNodes,
actualMsgs,
numOfMsgsWithZFN,
numOfMsgsWithFaults))
assert passes >= len(allReplicas) - faultyNodes
def primaryReceivesCorrectNumberOfPREPAREs():
"""
num of PREPARE seen by primary replica is n - 1;
n = num of nodes without fault, and greater than or equal to
2f with faults.
"""
actualMsgs = len([param for param in
getAllArgs(primary,
primary.processPrepare)
if (param['prepare'].instId,
param['prepare'].viewNo,
param['prepare'].ppSeqNo) == (primary.instId,
primary.viewNo,
primary.prePrepareSeqNo) and
param['sender'] != primary.name])
numOfMsgsWithZFN = nodeCount - 1
numOfMsgsWithFaults = 2 * f - 1
assert msgCountOK(nodeCount,
faultyNodes,
actualMsgs,
numOfMsgsWithZFN,
numOfMsgsWithFaults)
# TODO what if the primary is faulty?
def nonPrimaryReplicasReceiveCorrectNumberOfPREPAREs():
"""
num of PREPARE seen by Non primary replica is n - 2 without
faults and 2f - 1 with faults.
"""
passes = 0
numOfMsgsWithZFN = nodeCount - 2
numOfMsgsWithFaults = (2 * f) - 1
for npr in nonPrimaryReplicas:
actualMsgs = len([param for param in
getAllArgs(
npr,
npr.processPrepare)
if (param['prepare'].instId,
param['prepare'].viewNo,
param['prepare'].ppSeqNo) == (primary.instId,
primary.viewNo,
primary.prePrepareSeqNo)
])
passes += int(msgCountOK(nodeCount,
faultyNodes,
actualMsgs,
numOfMsgsWithZFN,
numOfMsgsWithFaults))
assert passes >= len(nonPrimaryReplicas) - faultyNodes
# TODO how do we know if one of the faulty nodes is a primary or
# not?
primaryDontSendAnyPREPAREs()
allReplicasSeeCorrectNumberOfPREPAREs()
primaryReceivesCorrectNumberOfPREPAREs()
nonPrimaryReplicasReceiveCorrectNumberOfPREPAREs()
def primaryReplicas(nodeSet):
instanceCount = getNoInstances(nodeCount)
return [getPrimaryReplica(nodeSet, i) for i in range(instanceCount)]
