def testPrimarySendsAPrepareAndMarkedSuspicious(looper, nodeSet, preprepared1): def sendPrepareFromPrimary(instId): primary = getPrimaryReplica(nodeSet, instId) viewNo, ppSeqNo = next(iter(primary.sentPrePrepares.keys())) prepare = Prepare(instId, viewNo, ppSeqNo, preprepared1.digest, time.time()) primary.doPrepare(prepare) def chk(): for r in getNonPrimaryReplicas(nodeSet, instId): l = len([ param for param in getAllArgs(r, r.processPrepare) if param['sender'] == primary.name ]) assert l == 1 looper.run(eventually(chk)) sendPrepareFromPrimary(0) for node in nodeSet: if node in getNonPrimaryReplicas(nodeSet, 0): frm, reason, code = getAllArgs(node, TestNode.reportSuspiciousNode) assert frm == getPrimaryReplica(nodeSet, 0).node.name assert isinstance(reason, SuspiciousNode) assert len(getNodeSuspicions(node, Suspicions.PR_FRM_PRIMARY.code)) == 10
def testNonPrimarySendsAPrePrepare(looper, nodeSet, setup, propagated1): primaryReplica = getPrimaryReplica(nodeSet, instId) nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet, instId) firstNpr = nonPrimaryReplicas[0] remainingNpr = nonPrimaryReplicas[1:] def sendPrePrepareFromNonPrimary(replica): firstNpr.doPrePrepare(propagated1.reqDigest) return PrePrepare( replica.instId, firstNpr.viewNo, firstNpr.prePrepareSeqNo, propagated1.identifier, propagated1.reqId, propagated1.digest, time.time()) ppr = sendPrePrepareFromNonPrimary(firstNpr) def chk(): for r in (primaryReplica, *remainingNpr): recvdPps = recvdPrePrepare(r) assert len(recvdPps) == 1 assert recvdPps[0]['pp'][:-1] == ppr[:-1] nodeSuspicions = len(getNodeSuspicions( r.node, Suspicions.PPR_FRM_NON_PRIMARY.code)) assert nodeSuspicions == 1 looper.run(eventually(chk, retryWait=.5, timeout=5))
def setup(looper, startedNodes, up, wallet1, client1): # Get the master replica of the master protocol instance P = getPrimaryReplica(startedNodes) # Make `Delta` small enough so throughput check passes. for node in startedNodes: node.monitor.Delta = .001 slowRequest = None # make P (primary replica on master) faulty, i.e., slow to send # PRE-PREPARE for a specific client request only def by65SpecificPrePrepare(msg): nonlocal slowRequest if isinstance(msg, PrePrepare) and slowRequest is None: slowRequest = getattr(msg, f.REQ_ID.nm) return 65 P.outBoxTestStasher.delay(by65SpecificPrePrepare) sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, numReqs=5, timeoutPerReq=80) return adict(nodes=startedNodes)
def testPrePrepareWithHighSeqNo(looper, nodeSet, propagated1): def chk(): for r in getNonPrimaryReplicas(nodeSet, instId): nodeSuspicions = len( getNodeSuspicions(r.node, Suspicions.WRONG_PPSEQ_NO.code)) assert nodeSuspicions == 1 def checkPreprepare(replica, viewNo, ppSeqNo, req, numOfPrePrepares): assert (replica.prePrepares[viewNo, ppSeqNo][0]) == \ (req.identifier, req.reqId, req.digest) primary = getPrimaryReplica(nodeSet, instId) nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet, instId) req = propagated1.reqDigest primary.doPrePrepare(req) for np in nonPrimaryReplicas: looper.run( eventually(checkPreprepare, np, primary.viewNo, primary.lastPrePrepareSeqNo - 1, req, 1, retryWait=.5, timeout=10)) newReqDigest = ReqDigest(req.identifier, req.reqId + 1, req.digest) incorrectPrePrepareReq = PrePrepare(instId, primary.viewNo, primary.lastPrePrepareSeqNo + 2, *newReqDigest, time.time()) primary.send(incorrectPrePrepareReq, TPCStat.PrePrepareSent) looper.run(eventually(chk, retryWait=1, timeout=50))
def testPrePrepareWithHighSeqNo(looper, nodeSet, propagated1): def chk(): for r in getNonPrimaryReplicas(nodeSet, instId): nodeSuspicions = len(getNodeSuspicions( r.node, Suspicions.WRONG_PPSEQ_NO.code)) assert nodeSuspicions == 1 def checkPreprepare(replica, viewNo, ppSeqNo, req, numOfPrePrepares): assert (replica.prePrepares[viewNo, ppSeqNo]) == (req.clientId, req.reqId, req.digest) primary = getPrimaryReplica(nodeSet, instId) nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet, instId) req = propagated1.reqDigest primary.doPrePrepare(req) for np in nonPrimaryReplicas: looper.run( eventually(checkPreprepare, np, primary.viewNo, primary.prePrepareSeqNo - 1, req, 1, retryWait=.5, timeout=10)) newReqDigest = ReqDigest(req.clientId, req.reqId + 1, req.digest) incorrectPrePrepareReq = PrePrepare(instId, primary.viewNo, primary.prePrepareSeqNo + 2, *newReqDigest) primary.send(incorrectPrePrepareReq,TPCStat.PrePrepareSent) looper.run(eventually(chk, retryWait=1, timeout=50))
def testReplicasRejectSamePrePrepareMsg(looper, nodeSet, client1, wallet1): """ Replicas should not accept PRE-PREPARE for view "v" and prepare sequence number "n" if it has already accepted a request with view number "v" and sequence number "n" """ numOfNodes = 4 fValue = getMaxFailures(numOfNodes) request1 = sendRandomRequest(wallet1, client1) result1 = looper.run( eventually(checkSufficientRepliesRecvd, client1.inBox, request1.reqId, fValue, retryWait=1, timeout=5)) logger.debug("request {} gives result {}".format(request1, result1)) primaryRepl = getPrimaryReplica(nodeSet) logger.debug("Primary Replica: {}".format(primaryRepl)) logger.debug( "Decrementing the primary replica's pre-prepare sequence number by " "one...") primaryRepl.lastPrePrepareSeqNo -= 1 request2 = sendRandomRequest(wallet1, client1) looper.run( eventually(checkPrePrepareReqSent, primaryRepl, request2, retryWait=1, timeout=10)) nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet) logger.debug("Non Primary Replicas: " + str(nonPrimaryReplicas)) prePrepareReq = PrePrepare(primaryRepl.instId, primaryRepl.viewNo, primaryRepl.lastPrePrepareSeqNo, wallet1.defaultId, request2.reqId, request2.digest, time.time()) logger.debug("""Checking whether all the non primary replicas have received the pre-prepare request with same sequence number""") looper.run( eventually(checkPrePrepareReqRecvd, nonPrimaryReplicas, prePrepareReq, retryWait=1, timeout=10)) logger.debug("""Check that none of the non primary replicas didn't send any prepare message " in response to the pre-prepare message""") for npr in nonPrimaryReplicas: with pytest.raises(AssertionError): looper.run( eventually(checkPrepareReqSent, npr, wallet1.defaultId, request2.reqId, retryWait=1, timeout=10))
def g(instId): allReplicas = getAllReplicas(nodeSet, instId) primaryReplica = getPrimaryReplica(nodeSet, instId) def replicasSeesCorrectNumOfCOMMITs(): """ num of commit messages must be = n when zero fault; n = num of nodes and greater than or equal to 2f + 1 with faults. """ passes = 0 numOfMsgsWithZFN = nodeCount numOfMsgsWithFault = (2 * f) + 1 key = (primaryReplica.viewNo, primaryReplica.prePrepareSeqNo) for r in allReplicas: if key in r.commits: rcvdCommitRqst = r.commits[key] assert rcvdCommitRqst[0] == prepared1.digest actualMsgsReceived = len(rcvdCommitRqst[1]) passes += int(msgCountOK(nodeCount, faultyNodes, actualMsgsReceived, numOfMsgsWithZFN, numOfMsgsWithFault)) assert passes >= len(allReplicas) - faultyNodes def replicasReceivesCorrectNumberOfCOMMITs(): """ num of commit messages seen by replica must be equal to n - 1; when zero fault and greater than or equal to 2f+1 with faults. """ passes = 0 numOfMsgsWithZFN = nodeCount - 1 numOfMsgsWithFault = 2 * f for r in allReplicas: args = getAllArgs(r, r.processCommit) actualMsgsReceived = len(args) passes += int(msgCountOK(nodeCount, faultyNodes, actualMsgsReceived, numOfMsgsWithZFN, numOfMsgsWithFault)) for arg in args: assert arg['commit'].viewNo == primaryReplica.viewNo and \ arg['commit'].ppSeqNo == primaryReplica.prePrepareSeqNo and \ arg['commit'].digest == prepared1.digest assert r.name != arg['sender'] assert passes >= len(allReplicas) - faultyNodes replicasReceivesCorrectNumberOfCOMMITs() replicasSeesCorrectNumOfCOMMITs()
def setup(nodeSet, up): def dontSendPrePrepareRequest(self, reqDigest: ReqDigest): logger.debug("EVIL: {} not sending pre-prepare message for request {}". format(self.name, reqDigest)) return pr = getPrimaryReplica(nodeSet, instId) evilMethod = types.MethodType(dontSendPrePrepareRequest, pr) pr.doPrePrepare = evilMethod
def testPrimaryForfeit(looper, nodeSet, up, client1, wallet1): """ The primary of master protocol instance of the pool forfeits the primary status by triggering an election and not nominating itself """ pr = getPrimaryReplica(nodeSet, instId=0) prNode = pr.node # TODO: Incomplete pass
def setup(nodeSet, up): primaryRep, nonPrimaryReps = getPrimaryReplica(nodeSet, 0), \ getNonPrimaryReplicas(nodeSet, 0) # The primary replica would send PRE-PREPARE messages with incorrect digest makeNodeFaulty(primaryRep.node, partial(send3PhaseMsgWithIncorrectDigest, msgType=PrePrepare)) return adict(primaryRep=primaryRep, nonPrimaryReps=nonPrimaryReps)
def setup(nodeSet, up): primaryRep = getPrimaryReplica(nodeSet, 0) nonPrimaryReps = getNonPrimaryReplicas(nodeSet, 0) faultyRep = nonPrimaryReps[0] makeNodeFaulty(faultyRep.node, partial(send3PhaseMsgWithIncorrectDigest, msgType=Commit, instId=0)) return adict(primaryRep=primaryRep, nonPrimaryReps=nonPrimaryReps, faultyRep=faultyRep)
def setup(nodeSet, up): primaryRep, nonPrimaryReps = getPrimaryReplica(nodeSet, 0), \ getNonPrimaryReplicas(nodeSet, 0) # The primary replica would send PRE-PREPARE messages with incorrect digest makeNodeFaulty( primaryRep.node, partial(send3PhaseMsgWithIncorrectDigest, msgType=PrePrepare)) return adict(primaryRep=primaryRep, nonPrimaryReps=nonPrimaryReps)
def testReplicasRejectSamePrePrepareMsg(looper, nodeSet, client1, wallet1): """ Replicas should not accept PRE-PREPARE for view "v" and prepare sequence number "n" if it has already accepted a request with view number "v" and sequence number "n" """ numOfNodes = 4 fValue = getMaxFailures(numOfNodes) request1 = sendRandomRequest(wallet1, client1) result1 = looper.run( eventually(checkSufficientRepliesRecvd, client1.inBox, request1.reqId, fValue, retryWait=1, timeout=5)) logger.debug("request {} gives result {}".format(request1, result1)) primaryRepl = getPrimaryReplica(nodeSet) logger.debug("Primary Replica: {}".format(primaryRepl)) logger.debug( "Decrementing the primary replica's pre-prepare sequence number by " "one...") primaryRepl.lastPrePrepareSeqNo -= 1 request2 = sendRandomRequest(wallet1, client1) looper.run(eventually(checkPrePrepareReqSent, primaryRepl, request2, retryWait=1, timeout=10)) nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet) logger.debug("Non Primary Replicas: " + str(nonPrimaryReplicas)) prePrepareReq = PrePrepare( primaryRepl.instId, primaryRepl.viewNo, primaryRepl.lastPrePrepareSeqNo, wallet1.defaultId, request2.reqId, request2.digest, time.time() ) logger.debug("""Checking whether all the non primary replicas have received the pre-prepare request with same sequence number""") looper.run(eventually(checkPrePrepareReqRecvd, nonPrimaryReplicas, prePrepareReq, retryWait=1, timeout=10)) logger.debug("""Check that none of the non primary replicas didn't send any prepare message " in response to the pre-prepare message""") for npr in nonPrimaryReplicas: with pytest.raises(AssertionError): looper.run(eventually(checkPrepareReqSent, npr, wallet1.defaultId, request2.reqId, retryWait=1, timeout=10))
def g(instId): allReplicas = getAllReplicas(nodeSet, instId) primaryReplica = getPrimaryReplica(nodeSet, instId) def replicasSeesCorrectNumOfCOMMITs(): """ num of commit messages must be = n when zero fault; n = num of nodes and greater than or equal to 2f + 1 with faults. """ passes = 0 numOfMsgsWithZFN = nodeCount numOfMsgsWithFault = (2 * f) + 1 key = (primaryReplica.viewNo, primaryReplica.prePrepareSeqNo) for r in allReplicas: if key in r.commits: rcvdCommitRqst = r.commits[key] assert rcvdCommitRqst[0] == prepared1.digest actualMsgsReceived = len(rcvdCommitRqst[1]) passes += int( msgCountOK(nodeCount, faultyNodes, actualMsgsReceived, numOfMsgsWithZFN, numOfMsgsWithFault)) assert passes >= len(allReplicas) - faultyNodes def replicasReceivesCorrectNumberOfCOMMITs(): """ num of commit messages seen by replica must be equal to n - 1; when zero fault and greater than or equal to 2f+1 with faults. """ passes = 0 numOfMsgsWithZFN = nodeCount - 1 numOfMsgsWithFault = 2 * f for r in allReplicas: args = getAllArgs(r, r.processCommit) actualMsgsReceived = len(args) passes += int( msgCountOK(nodeCount, faultyNodes, actualMsgsReceived, numOfMsgsWithZFN, numOfMsgsWithFault)) for arg in args: assert arg['commit'].viewNo == primaryReplica.viewNo and \ arg['commit'].ppSeqNo == primaryReplica.prePrepareSeqNo and \ arg['commit'].digest == prepared1.digest assert r.name != arg['sender'] assert passes >= len(allReplicas) - faultyNodes replicasReceivesCorrectNumberOfCOMMITs() replicasSeesCorrectNumOfCOMMITs()
def sendPrepareFromPrimary(instId): primary = getPrimaryReplica(nodeSet, instId) preprepared1.viewNo = instId preprepared1.ppSeqNo = primary.prePrepareSeqNo primary.doPrepare(preprepared1) for r in getNonPrimaryReplicas(nodeSet, instId): l = len([param for param in getAllArgs(r, r.processPrepare) if param['sender'] == primary.name]) assert l == 1 sendPrepareFromPrimary(0)
def replied1(looper, nodeSet, client1, committed1): for instId in range(getNoInstances(len(nodeSet))): getPrimaryReplica(nodeSet, instId) looper.run(*[eventually(checkRequestReturnedToNode, node, client1.defaultIdentifier, committed1.reqId, committed1.digest, instId, retryWait=1, timeout=30) for node in nodeSet]) looper.run(eventually( checkSufficientRepliesRecvd, client1.inBox, committed1.reqId, 2, retryWait=2, timeout=30)) return committed1
def setup(nodeSet, up): primaryRep, nonPrimaryReps = getPrimaryReplica(nodeSet, 0), \ getNonPrimaryReplicas(nodeSet, 0) # A non primary replica sends PREPARE messages with incorrect digest faultyRep = nonPrimaryReps[0] makeNodeFaulty(faultyRep.node, partial(send3PhaseMsgWithIncorrectDigest, msgType=Prepare, instId=0)) return adict(primaryRep=primaryRep, nonPrimaryReps=nonPrimaryReps, faultyRep=faultyRep)
def step1(looper, nodeSet, up, wallet1, client1): startedNodes = nodeSet """ stand up a pool of nodes and send 5 requests to client """ # the master instance has a primary replica, call it P P = getPrimaryReplica(startedNodes) requests = sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 5) # profile_this(sendReqsToNodesAndVerifySuffReplies, looper, client1, 5) return adict(P=P, nodes=startedNodes, requests=requests)
def setup(nodeSet, up): primaryRep = getPrimaryReplica(nodeSet, 0) nonPrimaryReps = getNonPrimaryReplicas(nodeSet, 0) faultyRep = nonPrimaryReps[0] makeNodeFaulty( faultyRep.node, partial(send3PhaseMsgWithIncorrectDigest, msgType=Commit, instId=0)) return adict(primaryRep=primaryRep, nonPrimaryReps=nonPrimaryReps, faultyRep=faultyRep)
def sendPrepareFromPrimary(instId): primary = getPrimaryReplica(nodeSet, instId) preprepared1.viewNo = instId preprepared1.ppSeqNo = primary.prePrepareSeqNo primary.doPrepare(preprepared1) for r in getNonPrimaryReplicas(nodeSet, instId): l = len([ param for param in getAllArgs(r, r.processPrepare) if param['sender'] == primary.name ]) assert l == 1 sendPrepareFromPrimary(0)
def step1(looper, startedNodes, up, client1): """ stand up a pool of nodes and send 5 requests to client """ # the master instance has a primary replica, call it P P = getPrimaryReplica(startedNodes) requests = sendReqsToNodesAndVerifySuffReplies(looper, client1, 5) # profile_this(sendReqsToNodesAndVerifySuffReplies, looper, client1, 5) return adict(P=P, nodes=startedNodes, requests=requests)
def setup(nodeSet, up): primaryRep, nonPrimaryReps = getPrimaryReplica(nodeSet, 0), \ getNonPrimaryReplicas(nodeSet, 0) # The primary replica would send 3 duplicate PRE-PREPARE requests to # non primary replicas makeNodeFaulty(primaryRep.node, partial(sendDuplicate3PhaseMsg, msgType=PrePrepare, count=3)) # The node of the primary replica above should not be blacklisted by any # other node since we are simulating multiple PRE-PREPARE messages and # want to check for a particular suspicion return adict(primaryRep=primaryRep, nonPrimaryReps=nonPrimaryReps)
def replied1(looper, nodeSet, client1, committed1, wallet1): for instId in range(getNoInstances(len(nodeSet))): getPrimaryReplica(nodeSet, instId) looper.run(*[ eventually(checkRequestReturnedToNode, node, wallet1.defaultId, committed1.reqId, committed1.digest, instId, retryWait=1, timeout=30) for node in nodeSet ]) looper.run( eventually(checkSufficientRepliesRecvd, client1.inBox, committed1.reqId, 2, retryWait=2, timeout=30)) return committed1
def sendPrepareFromPrimary(instId): primary = getPrimaryReplica(nodeSet, instId) viewNo, ppSeqNo = next(iter(primary.sentPrePrepares.keys())) prepare = Prepare(instId, viewNo, ppSeqNo, preprepared1.digest, time.time()) primary.doPrepare(prepare) def chk(): for r in getNonPrimaryReplicas(nodeSet, instId): l = len([ param for param in getAllArgs(r, r.processPrepare) if param['sender'] == primary.name ]) assert l == 1 looper.run(eventually(chk))
def testPrimarySelectionAfterViewChange(looper, nodeSet, ready, primaryReplicas, viewChangeDone): """ Test that primary replica of a protocol instance shifts to a new node after a view change. """ # Primary replicas before view change prBeforeVC = primaryReplicas # Primary replicas after view change instanceCount = getNoInstances(nodeCount) prAfterVC = [getPrimaryReplica(nodeSet, i) for i in range(instanceCount)] # Primary replicas have moved to the next node for br, ar in zip(prBeforeVC, prAfterVC): assert ar.node.rank - br.node.rank == 1 checkProtocolInstanceSetup(looper, nodeSet, retryWait=1, timeout=5)
def setup(nodeSet, up): primaryRep, nonPrimaryReps = getPrimaryReplica(nodeSet, 0), \ getNonPrimaryReplicas(nodeSet, 0) faultyRep = nonPrimaryReps[0] makeNodeFaulty(faultyRep.node, partial(sendDuplicate3PhaseMsg, msgType=Commit, count=3, instId=0)) # The node of the primary replica above should not be blacklisted by any # other node since we are simulating multiple COMMIT messages and # want to check for a particular suspicion whitelistNode(faultyRep.node.name, [node for node in nodeSet if node != faultyRep.node], Suspicions.DUPLICATE_CM_SENT.code) return adict(primaryRep=primaryRep, nonPrimaryReps=nonPrimaryReps, faultyRep=faultyRep)
def setup(looper, startedNodes, up, client1): # Get the master replica of the master protocol instance P = getPrimaryReplica(startedNodes) # Make `Delta` small enough so throughput check passes. for node in startedNodes: node.monitor.Delta = .001 # make P (primary replica on master) faulty, i.e., slow to send # PRE-PREPARE for a specific client request only def by65SpecificPrePrepare(msg): if isinstance(msg, PrePrepare) and getattr(msg, f.REQ_ID.nm) == 2: return 65 P.outBoxTestStasher.delay(by65SpecificPrePrepare) sendReqsToNodesAndVerifySuffReplies(looper, client1, numReqs=5, timeout=80) return adict(nodes=startedNodes)
def setup(nodeSet, up): primaryRep, nonPrimaryReps = getPrimaryReplica(nodeSet, 0), \ getNonPrimaryReplicas(nodeSet, 0) faultyRep = nonPrimaryReps[0] makeNodeFaulty( faultyRep.node, partial(sendDuplicate3PhaseMsg, msgType=Commit, count=3, instId=0)) # The node of the primary replica above should not be blacklisted by any # other node since we are simulating multiple COMMIT messages and # want to check for a particular suspicion whitelistNode(faultyRep.node.name, [node for node in nodeSet if node != faultyRep.node], Suspicions.DUPLICATE_CM_SENT.code) return adict(primaryRep=primaryRep, nonPrimaryReps=nonPrimaryReps, faultyRep=faultyRep)
def testPrimarySendsAPrepareAndMarkedSuspicious(looper, nodeSet, preprepared1): def sendPrepareFromPrimary(instId): primary = getPrimaryReplica(nodeSet, instId) preprepared1.viewNo = instId preprepared1.ppSeqNo = primary.prePrepareSeqNo primary.doPrepare(preprepared1) for r in getNonPrimaryReplicas(nodeSet, instId): l = len([param for param in getAllArgs(r, r.processPrepare) if param['sender'] == primary.name]) assert l == 1 sendPrepareFromPrimary(0) for node in nodeSet: if node in getNonPrimaryReplicas(nodeSet, 0): frm, reason, code = getAllArgs(node, Node.reportSuspiciousNode) assert frm == getPrimaryReplica(nodeSet, 0).node.name assert isinstance(reason, SuspiciousNode) assert len(getNodeSuspicions(node, Suspicions.PR_FRM_PRIMARY.code)) \ == 10
def testViewChangeCase1(nodeSet, looper, up, wallet1, client1, viewNo): """ Node will change view even though it does not find the master to be degraded when a quorum of nodes agree that master performance degraded """ # Delay processing of PRE-PREPARE from all non primary replicas of master # so master's performance falls and view changes delayNonPrimaries(nodeSet, 0, 10) pr = getPrimaryReplica(nodeSet, 0) relucatantNode = pr.node # Count sent instance changes of all nodes sentInstChanges = {} instChngMethodName = Node.sendInstanceChange.__name__ for n in nodeSet: sentInstChanges[n.name] = n.spylog.count(instChngMethodName) # Node reluctant to change view, never says master is degraded relucatantNode.monitor.isMasterDegraded = types.MethodType( lambda x: False, relucatantNode.monitor) sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 4) # Check that view change happened for all nodes looper.run( eventually(partial(checkViewNoForNodes, nodeSet, viewNo + 1), retryWait=1, timeout=20)) # All nodes except the reluctant node should have sent a view change and # thus must have called `sendInstanceChange` for n in nodeSet: if n.name != relucatantNode.name: assert n.spylog.count(instChngMethodName) > \ sentInstChanges.get(n.name, 0) else: assert n.spylog.count(instChngMethodName) == \ sentInstChanges.get(n.name, 0)
def testPrimarySendsAPrepareAndMarkedSuspicious(looper, nodeSet, preprepared1): def sendPrepareFromPrimary(instId): primary = getPrimaryReplica(nodeSet, instId) preprepared1.viewNo = instId preprepared1.ppSeqNo = primary.prePrepareSeqNo primary.doPrepare(preprepared1) for r in getNonPrimaryReplicas(nodeSet, instId): l = len([ param for param in getAllArgs(r, r.processPrepare) if param['sender'] == primary.name ]) assert l == 1 sendPrepareFromPrimary(0) for node in nodeSet: if node in getNonPrimaryReplicas(nodeSet, 0): frm, reason, code = getAllArgs(node, Node.reportSuspiciousNode) assert frm == getPrimaryReplica(nodeSet, 0).node.name assert isinstance(reason, SuspiciousNode) assert len(getNodeSuspicions(node, Suspicions.PR_FRM_PRIMARY.code)) \ == 10
def step2(step1, looper): """ Sends requests to client and check the ratio of throughput of master instance and backup instance must be greater than or equal to Delta and verify no view change takes place. """ # record when Node.checkPerformance was last run lastPerfChecks = latestPerfChecks(step1.nodes) # wait for every node to run another checkPerformance newPerfChecks = waitForNextPerfCheck(looper, step1.nodes, lastPerfChecks) # verify all nodes say that P is performing OK, and that no view changes # have been done for n in step1.nodes: assert n.viewNo == 0 # verify Primary is still the same assert getPrimaryReplica(step1.nodes) == step1.P step1.perfChecks = newPerfChecks return step1
def testViewChangeCase1(nodeSet, looper, up, wallet1, client1, viewNo): """ Node will change view even though it does not find the master to be degraded when a quorum of nodes agree that master performance degraded """ # Delay processing of PRE-PREPARE from all non primary replicas of master # so master's performance falls and view changes delayNonPrimaries(nodeSet, 0, 10) pr = getPrimaryReplica(nodeSet, 0) relucatantNode = pr.node # Count sent instance changes of all nodes sentInstChanges = {} instChngMethodName = Node.sendInstanceChange.__name__ for n in nodeSet: sentInstChanges[n.name] = n.spylog.count(instChngMethodName) # Node reluctant to change view, never says master is degraded relucatantNode.monitor.isMasterDegraded = types.MethodType( lambda x: False, relucatantNode.monitor) sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 4) # Check that view change happened for all nodes looper.run(eventually(partial(checkViewNoForNodes, nodeSet, viewNo + 1), retryWait=1, timeout=20)) # All nodes except the reluctant node should have sent a view change and # thus must have called `sendInstanceChange` for n in nodeSet: if n.name != relucatantNode.name: assert n.spylog.count(instChngMethodName) > \ sentInstChanges.get(n.name, 0) else: assert n.spylog.count(instChngMethodName) == \ sentInstChanges.get(n.name, 0)
def testOrderingCase2(looper, nodeSet, up, client1, wallet1): """ Scenario -> A client sends requests, some nodes delay COMMITs to few specific nodes such some nodes achieve commit quorum later for those requests compared to other nodes. But all nodes `ORDER` request in the same order of ppSeqNos https://www.pivotaltracker.com/n/projects/1889887/stories/133655009 """ pr, replicas = getPrimaryReplica(nodeSet, instId=0), \ getNonPrimaryReplicas(nodeSet, instId=0) assert len(replicas) == 6 rep0 = pr rep1 = replicas[0] rep2 = replicas[1] rep3 = replicas[2] rep4 = replicas[3] rep5 = replicas[4] rep6 = replicas[5] node0 = rep0.node node1 = rep1.node node2 = rep2.node node3 = rep3.node node4 = rep4.node node5 = rep5.node node6 = rep6.node ppSeqsToDelay = 5 commitDelay = 3 # delay each COMMIT by this number of seconds delayedPpSeqNos = set() requestCount = 15 requests = sendRandomRequests(wallet1, client1, requestCount) def specificCommits(wrappedMsg): nonlocal node3, node4, node5 msg, sender = wrappedMsg if isinstance(msg, PrePrepare): if len(delayedPpSeqNos) < ppSeqsToDelay: delayedPpSeqNos.add(msg.ppSeqNo) logger.debug('ppSeqNo {} corresponding to request id {} would ' 'be delayed'.format(msg.ppSeqNo, msg.reqId)) if isinstance(msg, Commit) and msg.instId == 0 and \ sender in (n.name for n in (node3, node4, node5)) and \ msg.ppSeqNo in delayedPpSeqNos: return commitDelay for node in (node1, node2): logger.debug('{} would be delaying commits'.format(node)) node.nodeIbStasher.delay(specificCommits) checkSufficientRepliesForRequests(looper, client1, requests) def ensureSlowNodesHaveAllTxns(): nonlocal node1, node2 for node in node1, node2: assert len(node.domainLedger) == requestCount looper.run(eventually(ensureSlowNodesHaveAllTxns, retryWait=1, timeout=15)) checkAllLedgersEqual((n.domainLedger for n in (node0, node3, node4, node5, node6))) for node in (node1, node2): for n in nodeSet: if n != node: checkLedgerEquality(node.domainLedger, n.domainLedger) checkAllLedgersEqual((n.domainLedger for n in nodeSet))
def g(instId): allReplicas = getAllReplicas(nodeSet, instId) primary = getPrimaryReplica(nodeSet, instId) nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet, instId) def primaryDontSendAnyPREPAREs(): """ 1. no of PREPARE sent by primary should be 0 """ for r in allReplicas: for param in getAllArgs(r, Replica.processPrepare): sender = param['sender'] assert sender != primary.name def allReplicasSeeCorrectNumberOfPREPAREs(): """ 1. no of PREPARE received by replicas must be n - 1; n = num of nodes without fault, and greater than or equal to 2f with faults. """ passes = 0 numOfMsgsWithZFN = nodeCount - 1 numOfMsgsWithFaults = 2 * f for replica in allReplicas: key = primary.viewNo, primary.lastPrePrepareSeqNo if key in replica.prepares: actualMsgs = len(replica.prepares[key].voters) passes += int( msgCountOK(nodeCount, faultyNodes, actualMsgs, numOfMsgsWithZFN, numOfMsgsWithFaults)) assert passes >= len(allReplicas) - faultyNodes def primaryReceivesCorrectNumberOfPREPAREs(): """ num of PREPARE seen by primary replica is n - 1; n = num of nodes without fault, and greater than or equal to 2f with faults. """ actualMsgs = len([ param for param in getAllArgs(primary, primary.processPrepare) if (param['prepare'].instId, param['prepare'].viewNo, param['prepare'].ppSeqNo) == (primary.instId, primary.viewNo, primary.lastPrePrepareSeqNo) and param['sender'] != primary.name ]) numOfMsgsWithZFN = nodeCount - 1 numOfMsgsWithFaults = 2 * f - 1 assert msgCountOK(nodeCount, faultyNodes, actualMsgs, numOfMsgsWithZFN, numOfMsgsWithFaults) # TODO what if the primary is faulty? def nonPrimaryReplicasReceiveCorrectNumberOfPREPAREs(): """ num of PREPARE seen by Non primary replica is n - 2 without faults and 2f - 1 with faults. """ passes = 0 numOfMsgsWithZFN = nodeCount - 2 numOfMsgsWithFaults = (2 * f) - 1 for npr in nonPrimaryReplicas: actualMsgs = len([ param for param in getAllArgs(npr, npr.processPrepare) if (param['prepare'].instId, param['prepare'].viewNo, param['prepare'].ppSeqNo) == ( primary.instId, primary.viewNo, primary.lastPrePrepareSeqNo) ]) passes += int( msgCountOK(nodeCount, faultyNodes, actualMsgs, numOfMsgsWithZFN, numOfMsgsWithFaults)) assert passes >= len(nonPrimaryReplicas) - faultyNodes # TODO how do we know if one of the faulty nodes is a primary or # not? primaryDontSendAnyPREPAREs() allReplicasSeeCorrectNumberOfPREPAREs() primaryReceivesCorrectNumberOfPREPAREs() nonPrimaryReplicasReceiveCorrectNumberOfPREPAREs()
def testOrderingCase1(looper, nodeSet, up, client1, wallet1): """ Scenario -> A client sends requests, some nodes delay COMMITs to few specific nodes such some nodes achieve commit quorum later for those requests compared to other nodes. But all nodes `ORDER` request in the same order of ppSeqNos https://www.pivotaltracker.com/n/projects/1889887/stories/133655009 """ pr, replicas = getPrimaryReplica(nodeSet, instId=0), \ getNonPrimaryReplicas(nodeSet, instId=0) assert len(replicas) == 6 rep0 = pr rep1 = replicas[0] rep2 = replicas[1] rep3 = replicas[2] rep4 = replicas[3] rep5 = replicas[4] rep6 = replicas[5] node0 = rep0.node node1 = rep1.node node2 = rep2.node node3 = rep3.node node4 = rep4.node node5 = rep5.node node6 = rep6.node requests = sendRandomRequests(wallet1, client1, 15) ppSeqsToDelay = 5 delayedPpSeqNos = set() def specificCommits(wrappedMsg): nonlocal node3, node4, node5 msg, sender = wrappedMsg if isinstance(msg, PrePrepare): if len(delayedPpSeqNos) < ppSeqsToDelay: delayedPpSeqNos.add(msg.ppSeqNo) logger.debug('ppSeqNo {} corresponding to request id {} would ' 'be delayed'.format(msg.ppSeqNo, msg.reqId)) if isinstance(msg, Commit) and msg.instId == 0 and \ sender in (n.name for n in (node3, node4, node5)) and \ msg.ppSeqNo in delayedPpSeqNos: return 3 for node in (node1, node2): logger.debug('{} would be delaying commits'.format(node)) node.nodeIbStasher.delay(specificCommits) checkSufficientRepliesForRequests(looper, client1, requests) def ensureSlowNodesHaveAllTxns(): nonlocal node1, node2 for node in node1, node2: assert len(node.domainLedger) == 15 looper.run(eventually(ensureSlowNodesHaveAllTxns, retryWait=1, timeout=15)) checkAllLedgersEqual((n.domainLedger for n in (node0, node3, node4, node5, node6))) for node in (node1, node2): for n in nodeSet: if n != node: checkLedgerEquality(node.domainLedger, n.domainLedger) checkAllLedgersEqual((n.domainLedger for n in nodeSet))
def g(instId): primary = getPrimaryReplica(nodeSet, instId) nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet, instId) def primarySeesCorrectNumberOfPREPREPAREs(): """ no of PRE-PREPARE as seen by processPrePrepare method for primary must be 0 with or without faults in system """ l1 = len([ param for param in getAllArgs(primary, primary.processPrePrepare) ]) assert l1 == 0 def nonPrimarySeesCorrectNumberOfPREPREPAREs(): """ 1. no of PRE-PREPARE as seen by processPrePrepare method for non-primaries must be 1; whn zero faulty nodes in system. 2. no of PRE-PREPARE as seen by processPrePrepare method for non-primaries must be greater than or equal to 0; with faults in system. """ expectedPrePrepareRequest = PrePrepare(instId, primary.viewNo, primary.lastPrePrepareSeqNo, propagated1.identifier, propagated1.reqId, propagated1.digest, time.time()) passes = 0 for npr in nonPrimaryReplicas: actualMsgs = len([ param for param in getAllArgs(npr, npr.processPrePrepare) if (param['pp'][:-1], param['sender']) == (expectedPrePrepareRequest[:-1], primary.name) ]) numOfMsgsWithZFN = 1 numOfMsgsWithFaults = 0 passes += int( msgCountOK(nodesSize, faultyNodes, actualMsgs, numOfMsgsWithZFN, numOfMsgsWithFaults)) assert passes >= len(nonPrimaryReplicas) - faultyNodes def primarySentsCorrectNumberOfPREPREPAREs(): """ 1. no of PRE-PREPARE sent by primary is 1 with or without fault in system but, when primary is faulty no of sent PRE_PREPARE will be zero and primary must be marked as malicious. """ actualMsgs = len([ param for param in getAllArgs(primary, primary.doPrePrepare) if (param['reqDigest'].identifier, param['reqDigest'].reqId, param['reqDigest'].digest) == (propagated1.identifier, propagated1.reqId, propagated1.digest) ]) numOfMsgsWithZFN = 1 # TODO: Considering, Primary is not faulty and will always send # PRE-PREPARE. Write separate test for testing when Primary # is faulty assert msgCountOK(nodesSize, faultyNodes, actualMsgs, numOfMsgsWithZFN, numOfMsgsWithZFN) def nonPrimaryReceivesCorrectNumberOfPREPREPAREs(): """ 1. no of PRE-PREPARE received by non-primaries must be 1 with zero faults in system, and 0 faults in system. """ passes = 0 for npr in nonPrimaryReplicas: l4 = len([ param for param in getAllArgs(npr, npr.addToPrePrepares) if (param['pp'].identifier, param['pp'].reqId, param['pp'].digest) == (propagated1.identifier, propagated1.reqId, propagated1.digest) ]) numOfMsgsWithZFN = 1 numOfMsgsWithFaults = 0 passes += msgCountOK(nodesSize, faultyNodes, l4, numOfMsgsWithZFN, numOfMsgsWithFaults) assert passes >= len(nonPrimaryReplicas) - faultyNodes primarySeesCorrectNumberOfPREPREPAREs() nonPrimarySeesCorrectNumberOfPREPREPAREs() primarySentsCorrectNumberOfPREPREPAREs() nonPrimaryReceivesCorrectNumberOfPREPREPAREs()
def g(instId): allReplicas = getAllReplicas(nodeSet, instId) primary = getPrimaryReplica(nodeSet, instId) nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet, instId) def primaryDontSendAnyPREPAREs(): """ 1. no of PREPARE sent by primary should be 0 """ for r in allReplicas: for param in getAllArgs(r, Replica.processPrepare): sender = param['sender'] assert sender != primary.name def allReplicasSeeCorrectNumberOfPREPAREs(): """ 1. no of PREPARE received by replicas must be n - 1; n = num of nodes without fault, and greater than or equal to 2f with faults. """ passes = 0 numOfMsgsWithZFN = nodeCount - 1 numOfMsgsWithFaults = 2 * f for replica in allReplicas: key = primary.viewNo, primary.prePrepareSeqNo if key in replica.prepares: actualMsgs = len(replica.prepares[key].voters) passes += int(msgCountOK(nodeCount, faultyNodes, actualMsgs, numOfMsgsWithZFN, numOfMsgsWithFaults)) assert passes >= len(allReplicas) - faultyNodes def primaryReceivesCorrectNumberOfPREPAREs(): """ num of PREPARE seen by primary replica is n - 1; n = num of nodes without fault, and greater than or equal to 2f with faults. """ actualMsgs = len([param for param in getAllArgs(primary, primary.processPrepare) if (param['prepare'].instId, param['prepare'].viewNo, param['prepare'].ppSeqNo) == (primary.instId, primary.viewNo, primary.prePrepareSeqNo) and param['sender'] != primary.name]) numOfMsgsWithZFN = nodeCount - 1 numOfMsgsWithFaults = 2 * f - 1 assert msgCountOK(nodeCount, faultyNodes, actualMsgs, numOfMsgsWithZFN, numOfMsgsWithFaults) # TODO what if the primary is faulty? def nonPrimaryReplicasReceiveCorrectNumberOfPREPAREs(): """ num of PREPARE seen by Non primary replica is n - 2 without faults and 2f - 1 with faults. """ passes = 0 numOfMsgsWithZFN = nodeCount - 2 numOfMsgsWithFaults = (2 * f) - 1 for npr in nonPrimaryReplicas: actualMsgs = len([param for param in getAllArgs( npr, npr.processPrepare) if (param['prepare'].instId, param['prepare'].viewNo, param['prepare'].ppSeqNo) == (primary.instId, primary.viewNo, primary.prePrepareSeqNo) ]) passes += int(msgCountOK(nodeCount, faultyNodes, actualMsgs, numOfMsgsWithZFN, numOfMsgsWithFaults)) assert passes >= len(nonPrimaryReplicas) - faultyNodes # TODO how do we know if one of the faulty nodes is a primary or # not? primaryDontSendAnyPREPAREs() allReplicasSeeCorrectNumberOfPREPAREs() primaryReceivesCorrectNumberOfPREPAREs() nonPrimaryReplicasReceiveCorrectNumberOfPREPAREs()
def primaryReplicas(nodeSet, up): instanceCount = getNoInstances(nodeCount) return [getPrimaryReplica(nodeSet, i) for i in range(instanceCount)]
def g(instId): primary = getPrimaryReplica(nodeSet, instId) nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet, instId) def primarySeesCorrectNumberOfPREPREPAREs(): """ no of PRE-PREPARE as seen by processPrePrepare method for primary must be 0 with or without faults in system """ l1 = len([param for param in getAllArgs(primary, primary.processPrePrepare)]) assert l1 == 0 def nonPrimarySeesCorrectNumberOfPREPREPAREs(): """ 1. no of PRE-PREPARE as seen by processPrePrepare method for non-primaries must be 1; whn zero faulty nodes in system. 2. no of PRE-PREPARE as seen by processPrePrepare method for non-primaries must be greater than or equal to 0; with faults in system. """ expectedPrePrepareRequest = PrePrepare( instId, primary.viewNo, primary.prePrepareSeqNo, propagated1.clientId, propagated1.reqId, propagated1.digest) passes = 0 for npr in nonPrimaryReplicas: actualMsgs = len([param for param in getAllArgs(npr, npr.processPrePrepare) if (param['pp'], param['sender']) == ( expectedPrePrepareRequest, primary.name)]) numOfMsgsWithZFN = 1 numOfMsgsWithFaults = 0 passes += int(msgCountOK(nodesSize, faultyNodes, actualMsgs, numOfMsgsWithZFN, numOfMsgsWithFaults)) assert passes >= len(nonPrimaryReplicas) - faultyNodes def primarySentsCorrectNumberOfPREPREPAREs(): """ 1. no of PRE-PREPARE sent by primary is 1 with or without fault in system but, when primary is faulty no of sent PRE_PREPARE will be zero and primary must be marked as malicious. """ actualMsgs = len([param for param in getAllArgs(primary, primary.doPrePrepare) if (param['reqDigest'].clientId, param['reqDigest'].reqId, param['reqDigest'].digest) == (propagated1.clientId, propagated1.reqId, propagated1.digest) ]) numOfMsgsWithZFN = 1 # TODO: Considering, Primary is not faulty and will always send # PRE-PREPARE. Write separate test for testing when Primary # is faulty assert msgCountOK(nodesSize, faultyNodes, actualMsgs, numOfMsgsWithZFN, numOfMsgsWithZFN) def nonPrimaryReceivesCorrectNumberOfPREPREPAREs(): """ 1. no of PRE-PREPARE received by non-primaries must be 1 with zero faults in system, and 0 faults in system. """ passes = 0 for npr in nonPrimaryReplicas: l4 = len([param for param in getAllArgs(npr, npr.addToPrePrepares) if (param['pp'].clientId, param['pp'].reqId, param['pp'].digest) == ( propagated1.clientId, propagated1.reqId, propagated1.digest)]) numOfMsgsWithZFN = 1 numOfMsgsWithFaults = 0 passes += msgCountOK(nodesSize, faultyNodes, l4, numOfMsgsWithZFN, numOfMsgsWithFaults) assert passes >= len(nonPrimaryReplicas) - faultyNodes primarySeesCorrectNumberOfPREPREPAREs() nonPrimarySeesCorrectNumberOfPREPREPAREs() primarySentsCorrectNumberOfPREPREPAREs() nonPrimaryReceivesCorrectNumberOfPREPREPAREs()