def testPrimarySendsAPrepareAndMarkedSuspicious(looper, nodeSet, preprepared1):
def sendPrepareFromPrimary(instId):
primary = getPrimaryReplica(nodeSet, instId)
viewNo, ppSeqNo = next(iter(primary.sentPrePrepares.keys()))
prepare = Prepare(instId, viewNo, ppSeqNo, preprepared1.digest,
time.time())
primary.doPrepare(prepare)
def chk():
for r in getNonPrimaryReplicas(nodeSet, instId):
l = len([
param for param in getAllArgs(r, r.processPrepare)
if param['sender'] == primary.name
])
assert l == 1
looper.run(eventually(chk))
sendPrepareFromPrimary(0)
for node in nodeSet:
if node in getNonPrimaryReplicas(nodeSet, 0):
frm, reason, code = getAllArgs(node, TestNode.reportSuspiciousNode)
assert frm == getPrimaryReplica(nodeSet, 0).node.name
assert isinstance(reason, SuspiciousNode)
assert len(getNodeSuspicions(node,
Suspicions.PR_FRM_PRIMARY.code)) == 10
def testNonPrimarySendsAPrePrepare(looper, nodeSet, setup, propagated1):
primaryReplica = getPrimaryReplica(nodeSet, instId)
nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet, instId)
firstNpr = nonPrimaryReplicas[0]
remainingNpr = nonPrimaryReplicas[1:]
def sendPrePrepareFromNonPrimary(replica):
firstNpr.doPrePrepare(propagated1.reqDigest)
return PrePrepare(
replica.instId,
firstNpr.viewNo,
firstNpr.prePrepareSeqNo,
propagated1.identifier,
propagated1.reqId,
propagated1.digest,
time.time())
ppr = sendPrePrepareFromNonPrimary(firstNpr)
def chk():
for r in (primaryReplica, *remainingNpr):
recvdPps = recvdPrePrepare(r)
assert len(recvdPps) == 1
assert recvdPps[0]['pp'][:-1] == ppr[:-1]
nodeSuspicions = len(getNodeSuspicions(
r.node, Suspicions.PPR_FRM_NON_PRIMARY.code))
assert nodeSuspicions == 1
looper.run(eventually(chk,
retryWait=.5, timeout=5))
def setup(looper, startedNodes, up, wallet1, client1):
# Get the master replica of the master protocol instance
P = getPrimaryReplica(startedNodes)
# Make `Delta` small enough so throughput check passes.
for node in startedNodes:
node.monitor.Delta = .001
slowRequest = None
# make P (primary replica on master) faulty, i.e., slow to send
# PRE-PREPARE for a specific client request only
def by65SpecificPrePrepare(msg):
nonlocal slowRequest
if isinstance(msg, PrePrepare) and slowRequest is None:
slowRequest = getattr(msg, f.REQ_ID.nm)
return 65
P.outBoxTestStasher.delay(by65SpecificPrePrepare)
sendReqsToNodesAndVerifySuffReplies(looper,
wallet1,
client1,
numReqs=5,
timeoutPerReq=80)
return adict(nodes=startedNodes)
def testPrePrepareWithHighSeqNo(looper, nodeSet, propagated1):
def chk():
for r in getNonPrimaryReplicas(nodeSet, instId):
nodeSuspicions = len(
getNodeSuspicions(r.node, Suspicions.WRONG_PPSEQ_NO.code))
assert nodeSuspicions == 1
def checkPreprepare(replica, viewNo, ppSeqNo, req, numOfPrePrepares):
assert (replica.prePrepares[viewNo, ppSeqNo][0]) == \
(req.identifier, req.reqId, req.digest)
primary = getPrimaryReplica(nodeSet, instId)
nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet, instId)
req = propagated1.reqDigest
primary.doPrePrepare(req)
for np in nonPrimaryReplicas:
looper.run(
eventually(checkPreprepare,
np,
primary.viewNo,
primary.lastPrePrepareSeqNo - 1,
req,
1,
retryWait=.5,
timeout=10))
newReqDigest = ReqDigest(req.identifier, req.reqId + 1, req.digest)
incorrectPrePrepareReq = PrePrepare(instId, primary.viewNo,
primary.lastPrePrepareSeqNo + 2,
*newReqDigest, time.time())
primary.send(incorrectPrePrepareReq, TPCStat.PrePrepareSent)
looper.run(eventually(chk, retryWait=1, timeout=50))
def testPrePrepareWithHighSeqNo(looper, nodeSet, propagated1):
def chk():
for r in getNonPrimaryReplicas(nodeSet, instId):
nodeSuspicions = len(getNodeSuspicions(
r.node, Suspicions.WRONG_PPSEQ_NO.code))
assert nodeSuspicions == 1
def checkPreprepare(replica, viewNo, ppSeqNo, req, numOfPrePrepares):
assert (replica.prePrepares[viewNo, ppSeqNo]) == (req.clientId, req.reqId, req.digest)
primary = getPrimaryReplica(nodeSet, instId)
nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet, instId)
req = propagated1.reqDigest
primary.doPrePrepare(req)
for np in nonPrimaryReplicas:
looper.run(
eventually(checkPreprepare, np, primary.viewNo, primary.prePrepareSeqNo - 1,
req, 1,
retryWait=.5, timeout=10))
newReqDigest = ReqDigest(req.clientId, req.reqId + 1, req.digest)
incorrectPrePrepareReq = PrePrepare(instId,
primary.viewNo,
primary.prePrepareSeqNo + 2,
*newReqDigest)
primary.send(incorrectPrePrepareReq,TPCStat.PrePrepareSent)
looper.run(eventually(chk, retryWait=1, timeout=50))
def testReplicasRejectSamePrePrepareMsg(looper, nodeSet, client1, wallet1):
"""
Replicas should not accept PRE-PREPARE for view "v" and prepare sequence
number "n" if it has already accepted a request with view number "v" and
sequence number "n"
"""
numOfNodes = 4
fValue = getMaxFailures(numOfNodes)
request1 = sendRandomRequest(wallet1, client1)
result1 = looper.run(
eventually(checkSufficientRepliesRecvd,
client1.inBox,
request1.reqId,
fValue,
retryWait=1,
timeout=5))
logger.debug("request {} gives result {}".format(request1, result1))
primaryRepl = getPrimaryReplica(nodeSet)
logger.debug("Primary Replica: {}".format(primaryRepl))
logger.debug(
"Decrementing the primary replica's pre-prepare sequence number by "
"one...")
primaryRepl.lastPrePrepareSeqNo -= 1
request2 = sendRandomRequest(wallet1, client1)
looper.run(
eventually(checkPrePrepareReqSent,
primaryRepl,
request2,
retryWait=1,
timeout=10))
nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet)
logger.debug("Non Primary Replicas: " + str(nonPrimaryReplicas))
prePrepareReq = PrePrepare(primaryRepl.instId, primaryRepl.viewNo,
primaryRepl.lastPrePrepareSeqNo,
wallet1.defaultId, request2.reqId,
request2.digest, time.time())
logger.debug("""Checking whether all the non primary replicas have received
the pre-prepare request with same sequence number""")
looper.run(
eventually(checkPrePrepareReqRecvd,
nonPrimaryReplicas,
prePrepareReq,
retryWait=1,
timeout=10))
logger.debug("""Check that none of the non primary replicas didn't send
any prepare message "
in response to the pre-prepare message""")
for npr in nonPrimaryReplicas:
with pytest.raises(AssertionError):
looper.run(
eventually(checkPrepareReqSent,
npr,
wallet1.defaultId,
request2.reqId,
retryWait=1,
timeout=10))
def g(instId):
allReplicas = getAllReplicas(nodeSet, instId)
primaryReplica = getPrimaryReplica(nodeSet, instId)
def replicasSeesCorrectNumOfCOMMITs():
"""
num of commit messages must be = n when zero fault;
n = num of nodes and greater than or equal to
2f + 1 with faults.
"""
passes = 0
numOfMsgsWithZFN = nodeCount
numOfMsgsWithFault = (2 * f) + 1
key = (primaryReplica.viewNo, primaryReplica.prePrepareSeqNo)
for r in allReplicas:
if key in r.commits:
rcvdCommitRqst = r.commits[key]
assert rcvdCommitRqst[0] == prepared1.digest
actualMsgsReceived = len(rcvdCommitRqst[1])
passes += int(msgCountOK(nodeCount,
faultyNodes,
actualMsgsReceived,
numOfMsgsWithZFN,
numOfMsgsWithFault))
assert passes >= len(allReplicas) - faultyNodes
def replicasReceivesCorrectNumberOfCOMMITs():
"""
num of commit messages seen by replica must be equal to n - 1;
when zero fault and greater than or equal to
2f+1 with faults.
"""
passes = 0
numOfMsgsWithZFN = nodeCount - 1
numOfMsgsWithFault = 2 * f
for r in allReplicas:
args = getAllArgs(r, r.processCommit)
actualMsgsReceived = len(args)
passes += int(msgCountOK(nodeCount,
faultyNodes,
actualMsgsReceived,
numOfMsgsWithZFN,
numOfMsgsWithFault))
for arg in args:
assert arg['commit'].viewNo == primaryReplica.viewNo and \
arg['commit'].ppSeqNo == primaryReplica.prePrepareSeqNo and \
arg['commit'].digest == prepared1.digest
assert r.name != arg['sender']
assert passes >= len(allReplicas) - faultyNodes
replicasReceivesCorrectNumberOfCOMMITs()
replicasSeesCorrectNumOfCOMMITs()
def setup(nodeSet, up):
def dontSendPrePrepareRequest(self, reqDigest: ReqDigest):
logger.debug("EVIL: {} not sending pre-prepare message for request {}".
format(self.name, reqDigest))
return
pr = getPrimaryReplica(nodeSet, instId)
evilMethod = types.MethodType(dontSendPrePrepareRequest, pr)
pr.doPrePrepare = evilMethod
def testPrimaryForfeit(looper, nodeSet, up, client1, wallet1):
"""
The primary of master protocol instance of the pool forfeits the primary
status by triggering an election and not nominating itself
"""
pr = getPrimaryReplica(nodeSet, instId=0)
prNode = pr.node
# TODO: Incomplete
pass
def setup(nodeSet, up):
primaryRep, nonPrimaryReps = getPrimaryReplica(nodeSet, 0), \
getNonPrimaryReplicas(nodeSet, 0)
# The primary replica would send PRE-PREPARE messages with incorrect digest
makeNodeFaulty(primaryRep.node, partial(send3PhaseMsgWithIncorrectDigest,
msgType=PrePrepare))
return adict(primaryRep=primaryRep, nonPrimaryReps=nonPrimaryReps)
def setup(nodeSet, up):
primaryRep = getPrimaryReplica(nodeSet, 0)
nonPrimaryReps = getNonPrimaryReplicas(nodeSet, 0)
faultyRep = nonPrimaryReps[0]
makeNodeFaulty(faultyRep.node, partial(send3PhaseMsgWithIncorrectDigest,
msgType=Commit, instId=0))
return adict(primaryRep=primaryRep, nonPrimaryReps=nonPrimaryReps,
faultyRep=faultyRep)
def setup(nodeSet, up):
primaryRep, nonPrimaryReps = getPrimaryReplica(nodeSet, 0), \
getNonPrimaryReplicas(nodeSet, 0)
# The primary replica would send PRE-PREPARE messages with incorrect digest
makeNodeFaulty(
primaryRep.node,
partial(send3PhaseMsgWithIncorrectDigest, msgType=PrePrepare))
return adict(primaryRep=primaryRep, nonPrimaryReps=nonPrimaryReps)
def testReplicasRejectSamePrePrepareMsg(looper, nodeSet, client1, wallet1):
"""
Replicas should not accept PRE-PREPARE for view "v" and prepare sequence
number "n" if it has already accepted a request with view number "v" and
sequence number "n"
"""
numOfNodes = 4
fValue = getMaxFailures(numOfNodes)
request1 = sendRandomRequest(wallet1, client1)
result1 = looper.run(
eventually(checkSufficientRepliesRecvd, client1.inBox,
request1.reqId, fValue,
retryWait=1, timeout=5))
logger.debug("request {} gives result {}".format(request1, result1))
primaryRepl = getPrimaryReplica(nodeSet)
logger.debug("Primary Replica: {}".format(primaryRepl))
logger.debug(
"Decrementing the primary replica's pre-prepare sequence number by "
"one...")
primaryRepl.lastPrePrepareSeqNo -= 1
request2 = sendRandomRequest(wallet1, client1)
looper.run(eventually(checkPrePrepareReqSent, primaryRepl, request2,
retryWait=1, timeout=10))
nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet)
logger.debug("Non Primary Replicas: " + str(nonPrimaryReplicas))
prePrepareReq = PrePrepare(
primaryRepl.instId,
primaryRepl.viewNo,
primaryRepl.lastPrePrepareSeqNo,
wallet1.defaultId,
request2.reqId,
request2.digest,
time.time()
)
logger.debug("""Checking whether all the non primary replicas have received
the pre-prepare request with same sequence number""")
looper.run(eventually(checkPrePrepareReqRecvd,
nonPrimaryReplicas,
prePrepareReq,
retryWait=1,
timeout=10))
logger.debug("""Check that none of the non primary replicas didn't send
any prepare message "
in response to the pre-prepare message""")
for npr in nonPrimaryReplicas:
with pytest.raises(AssertionError):
looper.run(eventually(checkPrepareReqSent,
npr,
wallet1.defaultId,
request2.reqId,
retryWait=1,
timeout=10))
def g(instId):
allReplicas = getAllReplicas(nodeSet, instId)
primaryReplica = getPrimaryReplica(nodeSet, instId)
def replicasSeesCorrectNumOfCOMMITs():
"""
num of commit messages must be = n when zero fault;
n = num of nodes and greater than or equal to
2f + 1 with faults.
"""
passes = 0
numOfMsgsWithZFN = nodeCount
numOfMsgsWithFault = (2 * f) + 1
key = (primaryReplica.viewNo, primaryReplica.prePrepareSeqNo)
for r in allReplicas:
if key in r.commits:
rcvdCommitRqst = r.commits[key]
assert rcvdCommitRqst[0] == prepared1.digest
actualMsgsReceived = len(rcvdCommitRqst[1])
passes += int(
msgCountOK(nodeCount, faultyNodes, actualMsgsReceived,
numOfMsgsWithZFN, numOfMsgsWithFault))
assert passes >= len(allReplicas) - faultyNodes
def replicasReceivesCorrectNumberOfCOMMITs():
"""
num of commit messages seen by replica must be equal to n - 1;
when zero fault and greater than or equal to
2f+1 with faults.
"""
passes = 0
numOfMsgsWithZFN = nodeCount - 1
numOfMsgsWithFault = 2 * f
for r in allReplicas:
args = getAllArgs(r, r.processCommit)
actualMsgsReceived = len(args)
passes += int(
msgCountOK(nodeCount, faultyNodes, actualMsgsReceived,
numOfMsgsWithZFN, numOfMsgsWithFault))
for arg in args:
assert arg['commit'].viewNo == primaryReplica.viewNo and \
arg['commit'].ppSeqNo == primaryReplica.prePrepareSeqNo and \
arg['commit'].digest == prepared1.digest
assert r.name != arg['sender']
assert passes >= len(allReplicas) - faultyNodes
replicasReceivesCorrectNumberOfCOMMITs()
replicasSeesCorrectNumOfCOMMITs()
def sendPrepareFromPrimary(instId):
primary = getPrimaryReplica(nodeSet, instId)
preprepared1.viewNo = instId
preprepared1.ppSeqNo = primary.prePrepareSeqNo
primary.doPrepare(preprepared1)
for r in getNonPrimaryReplicas(nodeSet, instId):
l = len([param for param in getAllArgs(r, r.processPrepare)
if param['sender'] == primary.name])
assert l == 1
sendPrepareFromPrimary(0)
def replied1(looper, nodeSet, client1, committed1):
for instId in range(getNoInstances(len(nodeSet))):
getPrimaryReplica(nodeSet, instId)
looper.run(*[eventually(checkRequestReturnedToNode,
node,
client1.defaultIdentifier,
committed1.reqId,
committed1.digest,
instId,
retryWait=1, timeout=30)
for node in nodeSet])
looper.run(eventually(
checkSufficientRepliesRecvd,
client1.inBox,
committed1.reqId,
2,
retryWait=2,
timeout=30))
return committed1
def setup(nodeSet, up):
primaryRep, nonPrimaryReps = getPrimaryReplica(nodeSet, 0), \
getNonPrimaryReplicas(nodeSet, 0)
# A non primary replica sends PREPARE messages with incorrect digest
faultyRep = nonPrimaryReps[0]
makeNodeFaulty(faultyRep.node, partial(send3PhaseMsgWithIncorrectDigest,
msgType=Prepare, instId=0))
return adict(primaryRep=primaryRep, nonPrimaryReps=nonPrimaryReps,
faultyRep=faultyRep)
def setup(nodeSet, up):
primaryRep, nonPrimaryReps = getPrimaryReplica(nodeSet, 0), \
getNonPrimaryReplicas(nodeSet, 0)
# A non primary replica sends PREPARE messages with incorrect digest
faultyRep = nonPrimaryReps[0]
makeNodeFaulty(faultyRep.node, partial(send3PhaseMsgWithIncorrectDigest,
msgType=Prepare, instId=0))
return adict(primaryRep=primaryRep, nonPrimaryReps=nonPrimaryReps,
faultyRep=faultyRep)
def step1(looper, nodeSet, up, wallet1, client1):
startedNodes = nodeSet
"""
stand up a pool of nodes and send 5 requests to client
"""
# the master instance has a primary replica, call it P
P = getPrimaryReplica(startedNodes)
requests = sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 5)
# profile_this(sendReqsToNodesAndVerifySuffReplies, looper, client1, 5)
return adict(P=P, nodes=startedNodes, requests=requests)
def setup(nodeSet, up):
primaryRep = getPrimaryReplica(nodeSet, 0)
nonPrimaryReps = getNonPrimaryReplicas(nodeSet, 0)
faultyRep = nonPrimaryReps[0]
makeNodeFaulty(
faultyRep.node,
partial(send3PhaseMsgWithIncorrectDigest, msgType=Commit, instId=0))
return adict(primaryRep=primaryRep,
nonPrimaryReps=nonPrimaryReps,
faultyRep=faultyRep)
def sendPrepareFromPrimary(instId):
primary = getPrimaryReplica(nodeSet, instId)
preprepared1.viewNo = instId
preprepared1.ppSeqNo = primary.prePrepareSeqNo
primary.doPrepare(preprepared1)
for r in getNonPrimaryReplicas(nodeSet, instId):
l = len([
param for param in getAllArgs(r, r.processPrepare)
if param['sender'] == primary.name
])
assert l == 1
sendPrepareFromPrimary(0)
def step1(looper, startedNodes, up, client1):
"""
stand up a pool of nodes and send 5 requests to client
"""
# the master instance has a primary replica, call it P
P = getPrimaryReplica(startedNodes)
requests = sendReqsToNodesAndVerifySuffReplies(looper, client1, 5)
# profile_this(sendReqsToNodesAndVerifySuffReplies, looper, client1, 5)
return adict(P=P,
nodes=startedNodes,
requests=requests)
def setup(nodeSet, up):
primaryRep, nonPrimaryReps = getPrimaryReplica(nodeSet, 0), \
getNonPrimaryReplicas(nodeSet, 0)
# The primary replica would send 3 duplicate PRE-PREPARE requests to
# non primary replicas
makeNodeFaulty(primaryRep.node, partial(sendDuplicate3PhaseMsg,
msgType=PrePrepare, count=3))
# The node of the primary replica above should not be blacklisted by any
# other node since we are simulating multiple PRE-PREPARE messages and
# want to check for a particular suspicion
return adict(primaryRep=primaryRep, nonPrimaryReps=nonPrimaryReps)
def setup(nodeSet, up):
primaryRep, nonPrimaryReps = getPrimaryReplica(nodeSet, 0), \
getNonPrimaryReplicas(nodeSet, 0)
# The primary replica would send 3 duplicate PRE-PREPARE requests to
# non primary replicas
makeNodeFaulty(primaryRep.node, partial(sendDuplicate3PhaseMsg,
msgType=PrePrepare, count=3))
# The node of the primary replica above should not be blacklisted by any
# other node since we are simulating multiple PRE-PREPARE messages and
# want to check for a particular suspicion
return adict(primaryRep=primaryRep, nonPrimaryReps=nonPrimaryReps)
def replied1(looper, nodeSet, client1, committed1, wallet1):
for instId in range(getNoInstances(len(nodeSet))):
getPrimaryReplica(nodeSet, instId)
looper.run(*[
eventually(checkRequestReturnedToNode,
node,
wallet1.defaultId,
committed1.reqId,
committed1.digest,
instId,
retryWait=1,
timeout=30) for node in nodeSet
])
looper.run(
eventually(checkSufficientRepliesRecvd,
client1.inBox,
committed1.reqId,
2,
retryWait=2,
timeout=30))
return committed1
def sendPrepareFromPrimary(instId):
primary = getPrimaryReplica(nodeSet, instId)
viewNo, ppSeqNo = next(iter(primary.sentPrePrepares.keys()))
prepare = Prepare(instId, viewNo, ppSeqNo, preprepared1.digest,
time.time())
primary.doPrepare(prepare)
def chk():
for r in getNonPrimaryReplicas(nodeSet, instId):
l = len([
param for param in getAllArgs(r, r.processPrepare)
if param['sender'] == primary.name
])
assert l == 1
looper.run(eventually(chk))
def testPrimarySelectionAfterViewChange(looper, nodeSet, ready, primaryReplicas,
viewChangeDone):
"""
Test that primary replica of a protocol instance shifts to a new node after a view change.
"""
# Primary replicas before view change
prBeforeVC = primaryReplicas
# Primary replicas after view change
instanceCount = getNoInstances(nodeCount)
prAfterVC = [getPrimaryReplica(nodeSet, i) for i in range(instanceCount)]
# Primary replicas have moved to the next node
for br, ar in zip(prBeforeVC, prAfterVC):
assert ar.node.rank - br.node.rank == 1
checkProtocolInstanceSetup(looper, nodeSet, retryWait=1, timeout=5)
def setup(nodeSet, up):
primaryRep, nonPrimaryReps = getPrimaryReplica(nodeSet, 0), \
getNonPrimaryReplicas(nodeSet, 0)
faultyRep = nonPrimaryReps[0]
makeNodeFaulty(faultyRep.node, partial(sendDuplicate3PhaseMsg,
msgType=Commit, count=3,
instId=0))
# The node of the primary replica above should not be blacklisted by any
# other node since we are simulating multiple COMMIT messages and
# want to check for a particular suspicion
whitelistNode(faultyRep.node.name,
[node for node in nodeSet if node != faultyRep.node],
Suspicions.DUPLICATE_CM_SENT.code)
return adict(primaryRep=primaryRep, nonPrimaryReps=nonPrimaryReps,
faultyRep=faultyRep)
def setup(looper, startedNodes, up, client1):
# Get the master replica of the master protocol instance
P = getPrimaryReplica(startedNodes)
# Make `Delta` small enough so throughput check passes.
for node in startedNodes:
node.monitor.Delta = .001
# make P (primary replica on master) faulty, i.e., slow to send
# PRE-PREPARE for a specific client request only
def by65SpecificPrePrepare(msg):
if isinstance(msg, PrePrepare) and getattr(msg, f.REQ_ID.nm) == 2:
return 65
P.outBoxTestStasher.delay(by65SpecificPrePrepare)
sendReqsToNodesAndVerifySuffReplies(looper, client1, numReqs=5, timeout=80)
return adict(nodes=startedNodes)
def setup(nodeSet, up):
primaryRep, nonPrimaryReps = getPrimaryReplica(nodeSet, 0), \
getNonPrimaryReplicas(nodeSet, 0)
faultyRep = nonPrimaryReps[0]
makeNodeFaulty(
faultyRep.node,
partial(sendDuplicate3PhaseMsg, msgType=Commit, count=3, instId=0))
# The node of the primary replica above should not be blacklisted by any
# other node since we are simulating multiple COMMIT messages and
# want to check for a particular suspicion
whitelistNode(faultyRep.node.name,
[node for node in nodeSet if node != faultyRep.node],
Suspicions.DUPLICATE_CM_SENT.code)
return adict(primaryRep=primaryRep,
nonPrimaryReps=nonPrimaryReps,
faultyRep=faultyRep)
def testPrimarySendsAPrepareAndMarkedSuspicious(looper, nodeSet, preprepared1):
def sendPrepareFromPrimary(instId):
primary = getPrimaryReplica(nodeSet, instId)
preprepared1.viewNo = instId
preprepared1.ppSeqNo = primary.prePrepareSeqNo
primary.doPrepare(preprepared1)
for r in getNonPrimaryReplicas(nodeSet, instId):
l = len([param for param in getAllArgs(r, r.processPrepare)
if param['sender'] == primary.name])
assert l == 1
sendPrepareFromPrimary(0)
for node in nodeSet:
if node in getNonPrimaryReplicas(nodeSet, 0):
frm, reason, code = getAllArgs(node, Node.reportSuspiciousNode)
assert frm == getPrimaryReplica(nodeSet, 0).node.name
assert isinstance(reason, SuspiciousNode)
assert len(getNodeSuspicions(node, Suspicions.PR_FRM_PRIMARY.code)) \
== 10
def testViewChangeCase1(nodeSet, looper, up, wallet1, client1, viewNo):
"""
Node will change view even though it does not find the master to be degraded
when a quorum of nodes agree that master performance degraded
"""
# Delay processing of PRE-PREPARE from all non primary replicas of master
# so master's performance falls and view changes
delayNonPrimaries(nodeSet, 0, 10)
pr = getPrimaryReplica(nodeSet, 0)
relucatantNode = pr.node
# Count sent instance changes of all nodes
sentInstChanges = {}
instChngMethodName = Node.sendInstanceChange.__name__
for n in nodeSet:
sentInstChanges[n.name] = n.spylog.count(instChngMethodName)
# Node reluctant to change view, never says master is degraded
relucatantNode.monitor.isMasterDegraded = types.MethodType(
lambda x: False, relucatantNode.monitor)
sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 4)
# Check that view change happened for all nodes
looper.run(
eventually(partial(checkViewNoForNodes, nodeSet, viewNo + 1),
retryWait=1,
timeout=20))
# All nodes except the reluctant node should have sent a view change and
# thus must have called `sendInstanceChange`
for n in nodeSet:
if n.name != relucatantNode.name:
assert n.spylog.count(instChngMethodName) > \
sentInstChanges.get(n.name, 0)
else:
assert n.spylog.count(instChngMethodName) == \
sentInstChanges.get(n.name, 0)
def testPrimarySendsAPrepareAndMarkedSuspicious(looper, nodeSet, preprepared1):
def sendPrepareFromPrimary(instId):
primary = getPrimaryReplica(nodeSet, instId)
preprepared1.viewNo = instId
preprepared1.ppSeqNo = primary.prePrepareSeqNo
primary.doPrepare(preprepared1)
for r in getNonPrimaryReplicas(nodeSet, instId):
l = len([
param for param in getAllArgs(r, r.processPrepare)
if param['sender'] == primary.name
])
assert l == 1
sendPrepareFromPrimary(0)
for node in nodeSet:
if node in getNonPrimaryReplicas(nodeSet, 0):
frm, reason, code = getAllArgs(node, Node.reportSuspiciousNode)
assert frm == getPrimaryReplica(nodeSet, 0).node.name
assert isinstance(reason, SuspiciousNode)
assert len(getNodeSuspicions(node, Suspicions.PR_FRM_PRIMARY.code)) \
== 10
def step2(step1, looper):
"""
Sends requests to client and check the ratio of throughput of master
instance and backup instance must be greater than or equal to Delta and
verify no view change takes place.
"""
# record when Node.checkPerformance was last run
lastPerfChecks = latestPerfChecks(step1.nodes)
# wait for every node to run another checkPerformance
newPerfChecks = waitForNextPerfCheck(looper, step1.nodes, lastPerfChecks)
# verify all nodes say that P is performing OK, and that no view changes
# have been done
for n in step1.nodes:
assert n.viewNo == 0
# verify Primary is still the same
assert getPrimaryReplica(step1.nodes) == step1.P
step1.perfChecks = newPerfChecks
return step1
def step2(step1, looper):
"""
Sends requests to client and check the ratio of throughput of master
instance and backup instance must be greater than or equal to Delta and
verify no view change takes place.
"""
# record when Node.checkPerformance was last run
lastPerfChecks = latestPerfChecks(step1.nodes)
# wait for every node to run another checkPerformance
newPerfChecks = waitForNextPerfCheck(looper, step1.nodes, lastPerfChecks)
# verify all nodes say that P is performing OK, and that no view changes
# have been done
for n in step1.nodes:
assert n.viewNo == 0
# verify Primary is still the same
assert getPrimaryReplica(step1.nodes) == step1.P
step1.perfChecks = newPerfChecks
return step1
def testViewChangeCase1(nodeSet, looper, up, wallet1, client1, viewNo):
"""
Node will change view even though it does not find the master to be degraded
when a quorum of nodes agree that master performance degraded
"""
# Delay processing of PRE-PREPARE from all non primary replicas of master
# so master's performance falls and view changes
delayNonPrimaries(nodeSet, 0, 10)
pr = getPrimaryReplica(nodeSet, 0)
relucatantNode = pr.node
# Count sent instance changes of all nodes
sentInstChanges = {}
instChngMethodName = Node.sendInstanceChange.__name__
for n in nodeSet:
sentInstChanges[n.name] = n.spylog.count(instChngMethodName)
# Node reluctant to change view, never says master is degraded
relucatantNode.monitor.isMasterDegraded = types.MethodType(
lambda x: False, relucatantNode.monitor)
sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 4)
# Check that view change happened for all nodes
looper.run(eventually(partial(checkViewNoForNodes, nodeSet, viewNo + 1),
retryWait=1, timeout=20))
# All nodes except the reluctant node should have sent a view change and
# thus must have called `sendInstanceChange`
for n in nodeSet:
if n.name != relucatantNode.name:
assert n.spylog.count(instChngMethodName) > \
sentInstChanges.get(n.name, 0)
else:
assert n.spylog.count(instChngMethodName) == \
sentInstChanges.get(n.name, 0)
def testOrderingCase2(looper, nodeSet, up, client1, wallet1):
"""
Scenario -> A client sends requests, some nodes delay COMMITs to few
specific nodes such some nodes achieve commit quorum later for those
requests compared to other nodes. But all nodes `ORDER` request in the same
order of ppSeqNos
https://www.pivotaltracker.com/n/projects/1889887/stories/133655009
"""
pr, replicas = getPrimaryReplica(nodeSet, instId=0), \
getNonPrimaryReplicas(nodeSet, instId=0)
assert len(replicas) == 6
rep0 = pr
rep1 = replicas[0]
rep2 = replicas[1]
rep3 = replicas[2]
rep4 = replicas[3]
rep5 = replicas[4]
rep6 = replicas[5]
node0 = rep0.node
node1 = rep1.node
node2 = rep2.node
node3 = rep3.node
node4 = rep4.node
node5 = rep5.node
node6 = rep6.node
ppSeqsToDelay = 5
commitDelay = 3 # delay each COMMIT by this number of seconds
delayedPpSeqNos = set()
requestCount = 15
requests = sendRandomRequests(wallet1, client1, requestCount)
def specificCommits(wrappedMsg):
nonlocal node3, node4, node5
msg, sender = wrappedMsg
if isinstance(msg, PrePrepare):
if len(delayedPpSeqNos) < ppSeqsToDelay:
delayedPpSeqNos.add(msg.ppSeqNo)
logger.debug('ppSeqNo {} corresponding to request id {} would '
'be delayed'.format(msg.ppSeqNo, msg.reqId))
if isinstance(msg, Commit) and msg.instId == 0 and \
sender in (n.name for n in (node3, node4, node5)) and \
msg.ppSeqNo in delayedPpSeqNos:
return commitDelay
for node in (node1, node2):
logger.debug('{} would be delaying commits'.format(node))
node.nodeIbStasher.delay(specificCommits)
checkSufficientRepliesForRequests(looper, client1, requests)
def ensureSlowNodesHaveAllTxns():
nonlocal node1, node2
for node in node1, node2:
assert len(node.domainLedger) == requestCount
looper.run(eventually(ensureSlowNodesHaveAllTxns, retryWait=1, timeout=15))
checkAllLedgersEqual((n.domainLedger for n in (node0, node3, node4,
node5, node6)))
for node in (node1, node2):
for n in nodeSet:
if n != node:
checkLedgerEquality(node.domainLedger, n.domainLedger)
checkAllLedgersEqual((n.domainLedger for n in nodeSet))
def g(instId):
allReplicas = getAllReplicas(nodeSet, instId)
primary = getPrimaryReplica(nodeSet, instId)
nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet, instId)
def primaryDontSendAnyPREPAREs():
"""
1. no of PREPARE sent by primary should be 0
"""
for r in allReplicas:
for param in getAllArgs(r, Replica.processPrepare):
sender = param['sender']
assert sender != primary.name
def allReplicasSeeCorrectNumberOfPREPAREs():
"""
1. no of PREPARE received by replicas must be n - 1;
n = num of nodes without fault, and greater than or equal to
2f with faults.
"""
passes = 0
numOfMsgsWithZFN = nodeCount - 1
numOfMsgsWithFaults = 2 * f
for replica in allReplicas:
key = primary.viewNo, primary.lastPrePrepareSeqNo
if key in replica.prepares:
actualMsgs = len(replica.prepares[key].voters)
passes += int(
msgCountOK(nodeCount, faultyNodes, actualMsgs,
numOfMsgsWithZFN, numOfMsgsWithFaults))
assert passes >= len(allReplicas) - faultyNodes
def primaryReceivesCorrectNumberOfPREPAREs():
"""
num of PREPARE seen by primary replica is n - 1;
n = num of nodes without fault, and greater than or equal to
2f with faults.
"""
actualMsgs = len([
param for param in getAllArgs(primary, primary.processPrepare)
if (param['prepare'].instId, param['prepare'].viewNo,
param['prepare'].ppSeqNo) == (primary.instId,
primary.viewNo,
primary.lastPrePrepareSeqNo)
and param['sender'] != primary.name
])
numOfMsgsWithZFN = nodeCount - 1
numOfMsgsWithFaults = 2 * f - 1
assert msgCountOK(nodeCount, faultyNodes, actualMsgs,
numOfMsgsWithZFN, numOfMsgsWithFaults)
# TODO what if the primary is faulty?
def nonPrimaryReplicasReceiveCorrectNumberOfPREPAREs():
"""
num of PREPARE seen by Non primary replica is n - 2 without
faults and 2f - 1 with faults.
"""
passes = 0
numOfMsgsWithZFN = nodeCount - 2
numOfMsgsWithFaults = (2 * f) - 1
for npr in nonPrimaryReplicas:
actualMsgs = len([
param for param in getAllArgs(npr, npr.processPrepare)
if (param['prepare'].instId, param['prepare'].viewNo,
param['prepare'].ppSeqNo) == (
primary.instId, primary.viewNo,
primary.lastPrePrepareSeqNo)
])
passes += int(
msgCountOK(nodeCount, faultyNodes, actualMsgs,
numOfMsgsWithZFN, numOfMsgsWithFaults))
assert passes >= len(nonPrimaryReplicas) - faultyNodes
# TODO how do we know if one of the faulty nodes is a primary or
# not?
primaryDontSendAnyPREPAREs()
allReplicasSeeCorrectNumberOfPREPAREs()
primaryReceivesCorrectNumberOfPREPAREs()
nonPrimaryReplicasReceiveCorrectNumberOfPREPAREs()
def testOrderingCase1(looper, nodeSet, up, client1, wallet1):
"""
Scenario -> A client sends requests, some nodes delay COMMITs to few
specific nodes such some nodes achieve commit quorum later for those
requests compared to other nodes. But all nodes `ORDER` request in the same
order of ppSeqNos
https://www.pivotaltracker.com/n/projects/1889887/stories/133655009
"""
pr, replicas = getPrimaryReplica(nodeSet, instId=0), \
getNonPrimaryReplicas(nodeSet, instId=0)
assert len(replicas) == 6
rep0 = pr
rep1 = replicas[0]
rep2 = replicas[1]
rep3 = replicas[2]
rep4 = replicas[3]
rep5 = replicas[4]
rep6 = replicas[5]
node0 = rep0.node
node1 = rep1.node
node2 = rep2.node
node3 = rep3.node
node4 = rep4.node
node5 = rep5.node
node6 = rep6.node
requests = sendRandomRequests(wallet1, client1, 15)
ppSeqsToDelay = 5
delayedPpSeqNos = set()
def specificCommits(wrappedMsg):
nonlocal node3, node4, node5
msg, sender = wrappedMsg
if isinstance(msg, PrePrepare):
if len(delayedPpSeqNos) < ppSeqsToDelay:
delayedPpSeqNos.add(msg.ppSeqNo)
logger.debug('ppSeqNo {} corresponding to request id {} would '
'be delayed'.format(msg.ppSeqNo, msg.reqId))
if isinstance(msg, Commit) and msg.instId == 0 and \
sender in (n.name for n in (node3, node4, node5)) and \
msg.ppSeqNo in delayedPpSeqNos:
return 3
for node in (node1, node2):
logger.debug('{} would be delaying commits'.format(node))
node.nodeIbStasher.delay(specificCommits)
checkSufficientRepliesForRequests(looper, client1, requests)
def ensureSlowNodesHaveAllTxns():
nonlocal node1, node2
for node in node1, node2:
assert len(node.domainLedger) == 15
looper.run(eventually(ensureSlowNodesHaveAllTxns, retryWait=1, timeout=15))
checkAllLedgersEqual((n.domainLedger for n in (node0, node3, node4,
node5, node6)))
for node in (node1, node2):
for n in nodeSet:
if n != node:
checkLedgerEquality(node.domainLedger, n.domainLedger)
checkAllLedgersEqual((n.domainLedger for n in nodeSet))
def g(instId):
primary = getPrimaryReplica(nodeSet, instId)
nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet, instId)
def primarySeesCorrectNumberOfPREPREPAREs():
"""
no of PRE-PREPARE as seen by processPrePrepare
method for primary must be 0 with or without faults in system
"""
l1 = len([
param
for param in getAllArgs(primary, primary.processPrePrepare)
])
assert l1 == 0
def nonPrimarySeesCorrectNumberOfPREPREPAREs():
"""
1. no of PRE-PREPARE as seen by processPrePrepare method for
non-primaries must be 1; whn zero faulty nodes in system.
2. no of PRE-PREPARE as seen by processPrePrepare method for
non-primaries must be greater than or equal to 0;
with faults in system.
"""
expectedPrePrepareRequest = PrePrepare(instId, primary.viewNo,
primary.lastPrePrepareSeqNo,
propagated1.identifier,
propagated1.reqId,
propagated1.digest,
time.time())
passes = 0
for npr in nonPrimaryReplicas:
actualMsgs = len([
param for param in getAllArgs(npr, npr.processPrePrepare)
if (param['pp'][:-1],
param['sender']) == (expectedPrePrepareRequest[:-1],
primary.name)
])
numOfMsgsWithZFN = 1
numOfMsgsWithFaults = 0
passes += int(
msgCountOK(nodesSize, faultyNodes, actualMsgs,
numOfMsgsWithZFN, numOfMsgsWithFaults))
assert passes >= len(nonPrimaryReplicas) - faultyNodes
def primarySentsCorrectNumberOfPREPREPAREs():
"""
1. no of PRE-PREPARE sent by primary is 1 with or without
fault in system but, when primary is faulty no of sent PRE_PREPARE
will be zero and primary must be marked as malicious.
"""
actualMsgs = len([
param for param in getAllArgs(primary, primary.doPrePrepare)
if (param['reqDigest'].identifier, param['reqDigest'].reqId,
param['reqDigest'].digest) == (propagated1.identifier,
propagated1.reqId,
propagated1.digest)
])
numOfMsgsWithZFN = 1
# TODO: Considering, Primary is not faulty and will always send
# PRE-PREPARE. Write separate test for testing when Primary
# is faulty
assert msgCountOK(nodesSize, faultyNodes, actualMsgs,
numOfMsgsWithZFN, numOfMsgsWithZFN)
def nonPrimaryReceivesCorrectNumberOfPREPREPAREs():
"""
1. no of PRE-PREPARE received by non-primaries must be 1
with zero faults in system, and 0 faults in system.
"""
passes = 0
for npr in nonPrimaryReplicas:
l4 = len([
param for param in getAllArgs(npr, npr.addToPrePrepares)
if (param['pp'].identifier, param['pp'].reqId,
param['pp'].digest) == (propagated1.identifier,
propagated1.reqId,
propagated1.digest)
])
numOfMsgsWithZFN = 1
numOfMsgsWithFaults = 0
passes += msgCountOK(nodesSize, faultyNodes, l4,
numOfMsgsWithZFN, numOfMsgsWithFaults)
assert passes >= len(nonPrimaryReplicas) - faultyNodes
primarySeesCorrectNumberOfPREPREPAREs()
nonPrimarySeesCorrectNumberOfPREPREPAREs()
primarySentsCorrectNumberOfPREPREPAREs()
nonPrimaryReceivesCorrectNumberOfPREPREPAREs()
def g(instId):
allReplicas = getAllReplicas(nodeSet, instId)
primary = getPrimaryReplica(nodeSet, instId)
nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet, instId)
def primaryDontSendAnyPREPAREs():
"""
1. no of PREPARE sent by primary should be 0
"""
for r in allReplicas:
for param in getAllArgs(r, Replica.processPrepare):
sender = param['sender']
assert sender != primary.name
def allReplicasSeeCorrectNumberOfPREPAREs():
"""
1. no of PREPARE received by replicas must be n - 1;
n = num of nodes without fault, and greater than or equal to
2f with faults.
"""
passes = 0
numOfMsgsWithZFN = nodeCount - 1
numOfMsgsWithFaults = 2 * f
for replica in allReplicas:
key = primary.viewNo, primary.prePrepareSeqNo
if key in replica.prepares:
actualMsgs = len(replica.prepares[key].voters)
passes += int(msgCountOK(nodeCount,
faultyNodes,
actualMsgs,
numOfMsgsWithZFN,
numOfMsgsWithFaults))
assert passes >= len(allReplicas) - faultyNodes
def primaryReceivesCorrectNumberOfPREPAREs():
"""
num of PREPARE seen by primary replica is n - 1;
n = num of nodes without fault, and greater than or equal to
2f with faults.
"""
actualMsgs = len([param for param in
getAllArgs(primary,
primary.processPrepare)
if (param['prepare'].instId,
param['prepare'].viewNo,
param['prepare'].ppSeqNo) == (primary.instId,
primary.viewNo,
primary.prePrepareSeqNo) and
param['sender'] != primary.name])
numOfMsgsWithZFN = nodeCount - 1
numOfMsgsWithFaults = 2 * f - 1
assert msgCountOK(nodeCount,
faultyNodes,
actualMsgs,
numOfMsgsWithZFN,
numOfMsgsWithFaults)
# TODO what if the primary is faulty?
def nonPrimaryReplicasReceiveCorrectNumberOfPREPAREs():
"""
num of PREPARE seen by Non primary replica is n - 2 without
faults and 2f - 1 with faults.
"""
passes = 0
numOfMsgsWithZFN = nodeCount - 2
numOfMsgsWithFaults = (2 * f) - 1
for npr in nonPrimaryReplicas:
actualMsgs = len([param for param in
getAllArgs(
npr,
npr.processPrepare)
if (param['prepare'].instId,
param['prepare'].viewNo,
param['prepare'].ppSeqNo) == (primary.instId,
primary.viewNo,
primary.prePrepareSeqNo)
])
passes += int(msgCountOK(nodeCount,
faultyNodes,
actualMsgs,
numOfMsgsWithZFN,
numOfMsgsWithFaults))
assert passes >= len(nonPrimaryReplicas) - faultyNodes
# TODO how do we know if one of the faulty nodes is a primary or
# not?
primaryDontSendAnyPREPAREs()
allReplicasSeeCorrectNumberOfPREPAREs()
primaryReceivesCorrectNumberOfPREPAREs()
nonPrimaryReplicasReceiveCorrectNumberOfPREPAREs()
def primaryReplicas(nodeSet, up):
instanceCount = getNoInstances(nodeCount)
return [getPrimaryReplica(nodeSet, i) for i in range(instanceCount)]
def primaryReplicas(nodeSet, up):
instanceCount = getNoInstances(nodeCount)
return [getPrimaryReplica(nodeSet, i) for i in range(instanceCount)]
def g(instId):
primary = getPrimaryReplica(nodeSet, instId)
nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet, instId)
def primarySeesCorrectNumberOfPREPREPAREs():
"""
no of PRE-PREPARE as seen by processPrePrepare
method for primary must be 0 with or without faults in system
"""
l1 = len([param for param in
getAllArgs(primary, primary.processPrePrepare)])
assert l1 == 0
def nonPrimarySeesCorrectNumberOfPREPREPAREs():
"""
1. no of PRE-PREPARE as seen by processPrePrepare method for
non-primaries must be 1; whn zero faulty nodes in system.
2. no of PRE-PREPARE as seen by processPrePrepare method for
non-primaries must be greater than or equal to 0;
with faults in system.
"""
expectedPrePrepareRequest = PrePrepare(
instId,
primary.viewNo,
primary.prePrepareSeqNo,
propagated1.clientId,
propagated1.reqId,
propagated1.digest)
passes = 0
for npr in nonPrimaryReplicas:
actualMsgs = len([param for param in
getAllArgs(npr, npr.processPrePrepare)
if (param['pp'],
param['sender']) == (
expectedPrePrepareRequest,
primary.name)])
numOfMsgsWithZFN = 1
numOfMsgsWithFaults = 0
passes += int(msgCountOK(nodesSize,
faultyNodes,
actualMsgs,
numOfMsgsWithZFN,
numOfMsgsWithFaults))
assert passes >= len(nonPrimaryReplicas) - faultyNodes
def primarySentsCorrectNumberOfPREPREPAREs():
"""
1. no of PRE-PREPARE sent by primary is 1 with or without
fault in system but, when primary is faulty no of sent PRE_PREPARE
will be zero and primary must be marked as malicious.
"""
actualMsgs = len([param for param in
getAllArgs(primary, primary.doPrePrepare)
if (param['reqDigest'].clientId,
param['reqDigest'].reqId,
param['reqDigest'].digest) ==
(propagated1.clientId,
propagated1.reqId,
propagated1.digest)
])
numOfMsgsWithZFN = 1
# TODO: Considering, Primary is not faulty and will always send
# PRE-PREPARE. Write separate test for testing when Primary
# is faulty
assert msgCountOK(nodesSize,
faultyNodes,
actualMsgs,
numOfMsgsWithZFN,
numOfMsgsWithZFN)
def nonPrimaryReceivesCorrectNumberOfPREPREPAREs():
"""
1. no of PRE-PREPARE received by non-primaries must be 1
with zero faults in system, and 0 faults in system.
"""
passes = 0
for npr in nonPrimaryReplicas:
l4 = len([param for param in
getAllArgs(npr, npr.addToPrePrepares)
if (param['pp'].clientId,
param['pp'].reqId,
param['pp'].digest) == (
propagated1.clientId,
propagated1.reqId,
propagated1.digest)])
numOfMsgsWithZFN = 1
numOfMsgsWithFaults = 0
passes += msgCountOK(nodesSize,
faultyNodes,
l4,
numOfMsgsWithZFN,
numOfMsgsWithFaults)
assert passes >= len(nonPrimaryReplicas) - faultyNodes
primarySeesCorrectNumberOfPREPREPAREs()
nonPrimarySeesCorrectNumberOfPREPREPAREs()
primarySentsCorrectNumberOfPREPREPAREs()
nonPrimaryReceivesCorrectNumberOfPREPREPAREs()