def testNonPrimarySendsAPrePrepare(looper, nodeSet, setup, propagated1): primaryReplica = getPrimaryReplica(nodeSet, instId) nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet, instId) firstNpr = nonPrimaryReplicas[0] remainingNpr = nonPrimaryReplicas[1:] def sendPrePrepareFromNonPrimary(replica): firstNpr.sendPrePrepare(propagated1.reqDigest) return PrePrepare( replica.instId, firstNpr.viewNo, firstNpr.prePrepareSeqNo, propagated1.clientId, propagated1.reqId, propagated1.digest) ppr = sendPrePrepareFromNonPrimary(firstNpr) def chk(): for r in (primaryReplica, *remainingNpr): recvdPps = recvdPrePrepare(r) assert len(recvdPps) == 1 assert recvdPps[0]['pp'] == ppr nodeSuspicions = len(getNodeSuspicions( r.node, Suspicions.PPR_FRM_NON_PRIMARY.code)) assert nodeSuspicions == 1 looper.run(eventually(chk, retryWait=.5, timeout=5)) looper.run(eventually(partial(checkViewNoForNodes, nodeSet, 1), retryWait=1, timeout=20))
def g(instId): allReplicas = getAllReplicas(nodeSet, instId) primaryReplica = getPrimaryReplica(nodeSet, instId) def replicasSeesCorrectNumOfCOMMITs(): """ num of commit messages must be = n when zero fault; n = num of nodes and greater than or equal to 2f + 1 with faults. """ passes = 0 numOfMsgsWithZFN = nodeCount numOfMsgsWithFault = (2 * f) + 1 key = (primaryReplica.viewNo, primaryReplica.prePrepareSeqNo) for r in allReplicas: if key in r.commits: rcvdCommitRqst = r.commits[key] assert rcvdCommitRqst[0] == prepared1.digest actualMsgsReceived = len(rcvdCommitRqst[1]) passes += int(msgCountOK(nodeCount, faultyNodes, actualMsgsReceived, numOfMsgsWithZFN, numOfMsgsWithFault)) assert passes >= len(allReplicas) - faultyNodes def replicasReceivesCorrectNumberOfCOMMITs(): """ num of commit messages seen by replica must be equal to n - 1; when zero fault and greater than or equal to 2f+1 with faults. """ passes = 0 numOfMsgsWithZFN = nodeCount - 1 numOfMsgsWithFault = 2 * f for r in allReplicas: args = getAllArgs(r, r.processCommit) actualMsgsReceived = len(args) passes += int(msgCountOK(nodeCount, faultyNodes, actualMsgsReceived, numOfMsgsWithZFN, numOfMsgsWithFault)) for arg in args: assert arg['commit'].viewNo == primaryReplica.viewNo and \ arg['commit'].ppSeqNo == primaryReplica.prePrepareSeqNo and \ arg['commit'].digest == prepared1.digest assert r.name != arg['sender'] assert passes >= len(allReplicas) - faultyNodes replicasReceivesCorrectNumberOfCOMMITs() replicasSeesCorrectNumOfCOMMITs()
def setup(nodeSet, up): def dontSendPrePrepareRequest(self, reqDigest: ReqDigest): logger.debug("EVIL: {} not sending pre-prepare message for request {}". format(self.name, reqDigest)) return pr = getPrimaryReplica(nodeSet, instId) evilMethod = types.MethodType(dontSendPrePrepareRequest, pr) pr.sendPrePrepare = evilMethod
def setup(nodeSet, up): primaryRep, nonPrimaryReps = getPrimaryReplica(nodeSet, 0), \ getNonPrimaryReplicas(nodeSet, 0) # The primary replica would send PRE-PREPARE messages with incorrect digest makeNodeFaulty(primaryRep.node, partial(send3PhaseMsgWithIncorrectDigest, msgType=PrePrepare)) return adict(primaryRep=primaryRep, nonPrimaryReps=nonPrimaryReps)
def sendPrepareFromPrimary(instId): primary = getPrimaryReplica(nodeSet, instId) preprepared1.viewNo = instId preprepared1.ppSeqNo = primary.prePrepareSeqNo primary.sendPrepare(preprepared1) for r in getNonPrimaryReplicas(nodeSet, instId): l = len([param for param in getAllArgs(r, r.processPrepare) if param['sender'] == primary.name]) assert l == 1 sendPrepareFromPrimary(0)
def setup(nodeSet, up): primaryRep, nonPrimaryReps = getPrimaryReplica(nodeSet, 0), \ getNonPrimaryReplicas(nodeSet, 0) # A non primary replica sends PREPARE messages with incorrect digest faultyRep = nonPrimaryReps[0] makeNodeFaulty(faultyRep.node, partial(send3PhaseMsgWithIncorrectDigest, msgType=Prepare, instId=0)) return adict(primaryRep=primaryRep, nonPrimaryReps=nonPrimaryReps, faultyRep=faultyRep)
def testReplicasRejectSamePrePrepareMsg(looper, nodeSet, client1): """ Replicas should not accept PRE-PREPARE for view "v" and prepare sequence number "n" if it has already accepted a request with view number "v" and sequence number "n" """ numOfNodes = 4 fValue = getMaxFailures(numOfNodes) request1 = sendRandomRequest(client1) result1 = looper.run( eventually(checkSufficientRepliesRecvd, client1.inBox, request1.reqId, fValue, retryWait=1, timeout=5)) logging.debug("request {} gives result {}".format(request1, result1)) primaryRepl = getPrimaryReplica(nodeSet) logging.debug("Primary Replica: {}".format(primaryRepl)) logging.debug( "Decrementing the primary replica's pre-prepare sequence number by one...") primaryRepl.prePrepareSeqNo -= 1 request2 = sendRandomRequest(client1) looper.run(eventually(checkPrePrepareReqSent, primaryRepl, request2, retryWait=1, timeout=10)) nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet) logging.debug("Non Primary Replicas: " + str(nonPrimaryReplicas)) prePrepareReq = PrePrepare( primaryRepl.instId, primaryRepl.viewNo, primaryRepl.prePrepareSeqNo, client1.clientId, request2.reqId, request2.digest) logging.debug("""Checking whether all the non primary replicas have received the pre-prepare request with same sequence number""") looper.run(eventually(checkPrePrepareReqRecvd, nonPrimaryReplicas, prePrepareReq, retryWait=1, timeout=10)) logging.debug("""Check that none of the non primary replicas didn't send any prepare message " in response to the pre-prepare message""") for npr in nonPrimaryReplicas: with pytest.raises(AssertionError): looper.run(eventually(checkPrepareReqSent, npr, client1.clientId, request2.reqId, retryWait=1, timeout=10))
def testPrimarySelectionAfterViewChange(looper, nodeSet, ready, primaryReplicas, viewChangeDone): """ Test that primary replica of a protocol instance shifts to a new node after a view change. """ # Primary replicas before view change prBeforeVC = primaryReplicas # Primary replicas after view change instanceCount = getNoInstances(nodeCount) prAfterVC = [getPrimaryReplica(nodeSet, i) for i in range(instanceCount)] # Primary replicas have moved to the next node for br, ar in zip(prBeforeVC, prAfterVC): assert ar.node.rank - br.node.rank == 1 checkProtocolInstanceSetup(looper, nodeSet, retryWait=1, timeout=5)
def setup(nodeSet, up): primaryRep, nonPrimaryReps = getPrimaryReplica(nodeSet, 0), \ getNonPrimaryReplicas(nodeSet, 0) # The primary replica would send 3 duplicate PRE-PREPARE requests to # non primary replicas makeNodeFaulty(primaryRep.node, partial(sendDuplicate3PhaseMsg, msgType=PrePrepare, count=3)) # The node of the primary replica above should not be blacklisted by any # other node since we are simulating multiple PRE-PREPARE messages and # want to check for a particular suspicion # for node in nodeSet: # if node != primaryRep.node: # node.whitelistNode(primaryRep.node.name, # Suspicions.DUPLICATE_PPR_SENT.code) return adict(primaryRep=primaryRep, nonPrimaryReps=nonPrimaryReps)
def testPrimarySendsAPrepareAndMarkedSuspicious(looper, nodeSet, preprepared1): def sendPrepareFromPrimary(instId): primary = getPrimaryReplica(nodeSet, instId) preprepared1.viewNo = instId preprepared1.ppSeqNo = primary.prePrepareSeqNo primary.sendPrepare(preprepared1) for r in getNonPrimaryReplicas(nodeSet, instId): l = len([param for param in getAllArgs(r, r.processPrepare) if param['sender'] == primary.name]) assert l == 1 sendPrepareFromPrimary(0) for node in nodeSet: if node in getNonPrimaryReplicas(nodeSet, 0): frm, reason, code = getAllArgs(node, Node.reportSuspiciousNode) assert frm == getPrimaryReplica(nodeSet, 0).node.name assert isinstance(reason, SuspiciousNode) assert len(getNodeSuspicions(node, Suspicions.PR_FRM_PRIMARY.code)) \ == 10
def replied1(looper, nodeSet, client1, committed1): for instId in range(getNoInstances(len(nodeSet))): primaryReplica = getPrimaryReplica(nodeSet, instId) looper.run(*[eventually(checkRequestReturnedToNode, node, client1.clientId, committed1.reqId, committed1.digest, instId, retryWait=1, timeout=30) for node in nodeSet]) looper.run(eventually( checkSufficientRepliesRecvd, client1.inBox, committed1.reqId, 2, retryWait=2, timeout=30)) return committed1
def setup(nodeSet, up): primaryRep, nonPrimaryReps = getPrimaryReplica(nodeSet, 0), \ getNonPrimaryReplicas(nodeSet, 0) # A non primary replica sends duplicate PREPARE requests to all other # replicas faultyRep = nonPrimaryReps[0] makeNodeFaulty(faultyRep.node, partial(sendDuplicate3PhaseMsg, msgType=Prepare, count=3, instId=0)) # The node of the primary replica above should not be blacklisted by any # other node since we are simulating multiple PREPARE messages and # want to check for a particular suspicion whitelistNode(faultyRep.node.name, [node for node in nodeSet if node != faultyRep.node], Suspicions.DUPLICATE_PR_SENT.code) return adict(primaryRep=primaryRep, nonPrimaryReps=nonPrimaryReps, faultyRep=faultyRep)
def g(instId): primary = getPrimaryReplica(nodeSet, instId) nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet, instId) def primarySeesCorrectNumberOfPREPREPAREs(): """ no of PRE-PREPARE as seen by processPrePrepare method for primary must be 0 with or without faults in system """ l1 = len([param for param in getAllArgs(primary, primary.processPrePrepare)]) assert l1 == 0 def nonPrimarySeesCorrectNumberOfPREPREPAREs(): """ 1. no of PRE-PREPARE as seen by processPrePrepare method for non-primaries must be 1; whn zero faulty nodes in system. 2. no of PRE-PREPARE as seen by processPrePrepare method for non-primaries must be greater than or equal to 0; with faults in system. """ expectedPrePrepareRequest = PrePrepare( instId, primary.viewNo, primary.prePrepareSeqNo, propagated1.clientId, propagated1.reqId, propagated1.digest) passes = 0 for npr in nonPrimaryReplicas: actualMsgs = len([param for param in getAllArgs(npr, npr.processPrePrepare) if (param['pp'], param['sender']) == ( expectedPrePrepareRequest, primary.name)]) numOfMsgsWithZFN = 1 numOfMsgsWithFaults = 0 passes += int(msgCountOK(nodesSize, faultyNodes, actualMsgs, numOfMsgsWithZFN, numOfMsgsWithFaults)) assert passes >= len(nonPrimaryReplicas) - faultyNodes def primarySentsCorrectNumberOfPREPREPAREs(): """ 1. no of PRE-PREPARE sent by primary is 1 with or without fault in system but, when primary is faulty no of sent PRE_PREPARE will be zero and primary must be marked as malicious. """ actualMsgs = len([param for param in getAllArgs(primary, primary.sendPrePrepare) if (param['reqDigest'].clientId, param['reqDigest'].reqId, param['reqDigest'].digest) == (propagated1.clientId, propagated1.reqId, propagated1.digest) ]) numOfMsgsWithZFN = 1 # TODO: Considering, Primary is not faulty and will always send # PRE-PREPARE. Write separate test for testing when Primary # is faulty assert msgCountOK(nodesSize, faultyNodes, actualMsgs, numOfMsgsWithZFN, numOfMsgsWithZFN) def nonPrimaryReceivesCorrectNumberOfPREPREPAREs(): """ 1. no of PRE-PREPARE received by non-primaries must be 1 with zero faults in system, and 0 faults in system. """ passes = 0 for npr in nonPrimaryReplicas: l4 = len([param for param in getAllArgs(npr, npr.addToPrePrepares) if (param['pp'].clientId, param['pp'].reqId, param['pp'].digest) == ( propagated1.clientId, propagated1.reqId, propagated1.digest)]) numOfMsgsWithZFN = 1 numOfMsgsWithFaults = 0 passes += msgCountOK(nodesSize, faultyNodes, l4, numOfMsgsWithZFN, numOfMsgsWithFaults) assert passes >= len(nonPrimaryReplicas) - faultyNodes primarySeesCorrectNumberOfPREPREPAREs() nonPrimarySeesCorrectNumberOfPREPREPAREs() primarySentsCorrectNumberOfPREPREPAREs() nonPrimaryReceivesCorrectNumberOfPREPREPAREs()
def g(instId): allReplicas = getAllReplicas(nodeSet, instId) primary = getPrimaryReplica(nodeSet, instId) nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet, instId) def primaryDontSendAnyPREPAREs(): """ 1. no of PREPARE sent by primary should be 0 """ for r in allReplicas: for param in getAllArgs(r, Replica.processPrepare): sender = param['sender'] assert sender != primary.name def allReplicasSeeCorrectNumberOfPREPAREs(): """ 1. no of PREPARE received by replicas must be n - 1; n = num of nodes without fault, and greater than or equal to 2f with faults. """ passes = 0 numOfMsgsWithZFN = nodeCount - 1 numOfMsgsWithFaults = 2 * f for replica in allReplicas: key = primary.viewNo, primary.prePrepareSeqNo if key in replica.prepares: actualMsgs = len(replica.prepares[key].voters) passes += int(msgCountOK(nodeCount, faultyNodes, actualMsgs, numOfMsgsWithZFN, numOfMsgsWithFaults)) assert passes >= len(allReplicas) - faultyNodes def primaryReceivesCorrectNumberOfPREPAREs(): """ num of PREPARE seen by primary replica is n - 1; n = num of nodes without fault, and greater than or equal to 2f with faults. """ actualMsgs = len([param for param in getAllArgs(primary, primary.processPrepare) if (param['prepare'].instId, param['prepare'].viewNo, param['prepare'].ppSeqNo) == (primary.instId, primary.viewNo, primary.prePrepareSeqNo) and param['sender'] != primary.name]) numOfMsgsWithZFN = nodeCount - 1 numOfMsgsWithFaults = 2 * f - 1 assert msgCountOK(nodeCount, faultyNodes, actualMsgs, numOfMsgsWithZFN, numOfMsgsWithFaults) # TODO what if the primary is faulty? def nonPrimaryReplicasReceiveCorrectNumberOfPREPAREs(): """ num of PREPARE seen by Non primary replica is n - 2 without faults and 2f - 1 with faults. """ passes = 0 numOfMsgsWithZFN = nodeCount - 2 numOfMsgsWithFaults = (2 * f) - 1 for npr in nonPrimaryReplicas: actualMsgs = len([param for param in getAllArgs( npr, npr.processPrepare) if (param['prepare'].instId, param['prepare'].viewNo, param['prepare'].ppSeqNo) == (primary.instId, primary.viewNo, primary.prePrepareSeqNo) ]) passes += int(msgCountOK(nodeCount, faultyNodes, actualMsgs, numOfMsgsWithZFN, numOfMsgsWithFaults)) assert passes >= len(nonPrimaryReplicas) - faultyNodes # TODO how do we know if one of the faulty nodes is a primary or # not? primaryDontSendAnyPREPAREs() allReplicasSeeCorrectNumberOfPREPAREs() primaryReceivesCorrectNumberOfPREPAREs() nonPrimaryReplicasReceiveCorrectNumberOfPREPAREs()
def primaryReplicas(nodeSet): instanceCount = getNoInstances(nodeCount) return [getPrimaryReplica(nodeSet, i) for i in range(instanceCount)]