def testNonPrimarySendsAPrePrepare(looper, nodeSet, setup, propagated1): primaryReplica = getPrimaryReplica(nodeSet, instId) nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet, instId) firstNpr = nonPrimaryReplicas[0] remainingNpr = nonPrimaryReplicas[1:] def sendPrePrepareFromNonPrimary(replica): firstNpr.sendPrePrepare(propagated1.reqDigest) return PrePrepare( replica.instId, firstNpr.viewNo, firstNpr.prePrepareSeqNo, propagated1.clientId, propagated1.reqId, propagated1.digest) ppr = sendPrePrepareFromNonPrimary(firstNpr) def chk(): for r in (primaryReplica, *remainingNpr): recvdPps = recvdPrePrepare(r) assert len(recvdPps) == 1 assert recvdPps[0]['pp'] == ppr nodeSuspicions = len(getNodeSuspicions( r.node, Suspicions.PPR_FRM_NON_PRIMARY.code)) assert nodeSuspicions == 1 looper.run(eventually(chk, retryWait=.5, timeout=5)) looper.run(eventually(partial(checkViewNoForNodes, nodeSet, 1), retryWait=1, timeout=20))
def testQueueingReqFromFutureView(delayedPerf, looper, nodeSet, up, client1): """ Test if every node queues 3 Phase requests(PRE-PREPARE, PREPARE and COMMIT) that come from a view which is greater than the current view """ f = getMaxFailures(nodeCount) # Delay processing of instance change on a node nodeA = nodeSet.Alpha nodeA.nodeIbStasher.delay(icDelay(60)) nonPrimReps = getNonPrimaryReplicas(nodeSet, 0) # Delay processing of PRE-PREPARE from all non primary replicas of master # so master's throughput falls and view changes ppDelayer = ppDelay(5, 0) for r in nonPrimReps: r.node.nodeIbStasher.delay(ppDelayer) sendReqsToNodesAndVerifySuffReplies(looper, client1, 4, timeout=5 * nodeCount) # Every node except Node A should have a view change for node in nodeSet: if node.name != nodeA.name: looper.run(eventually( partial(checkViewChangeInitiatedForNode, node, 0), retryWait=1, timeout=20)) # Node A's view should not have changed yet with pytest.raises(AssertionError): looper.run(eventually(partial( checkViewChangeInitiatedForNode, nodeA, 0), retryWait=1, timeout=20)) # NodeA should not have any pending 3 phase request for a later view for r in nodeA.replicas: # type: TestReplica assert len(r.threePhaseMsgsForLaterView) == 0 # Reset delays on incoming messages from all nodes for node in nodeSet: node.nodeIbStasher.nodelay(ppDelayer) # Send one more request sendRandomRequest(client1) def checkPending3PhaseReqs(): # Get all replicas that have their primary status decided reps = [rep for rep in nodeA.replicas if rep.isPrimary is not None] # Atleast one replica should have its primary status decided assert len(reps) > 0 for r in reps: # type: TestReplica logging.debug("primary status for replica {} is {}" .format(r, r.primaryNames)) assert len(r.threePhaseMsgsForLaterView) > 0 # NodeA should now have pending 3 phase request for a later view looper.run(eventually(checkPending3PhaseReqs, retryWait=1, timeout=30))
def setup(nodeSet, up): primaryRep, nonPrimaryReps = getPrimaryReplica(nodeSet, 0), \ getNonPrimaryReplicas(nodeSet, 0) # The primary replica would send PRE-PREPARE messages with incorrect digest makeNodeFaulty(primaryRep.node, partial(send3PhaseMsgWithIncorrectDigest, msgType=PrePrepare)) return adict(primaryRep=primaryRep, nonPrimaryReps=nonPrimaryReps)
def sendPrepareFromPrimary(instId): primary = getPrimaryReplica(nodeSet, instId) preprepared1.viewNo = instId preprepared1.ppSeqNo = primary.prePrepareSeqNo primary.sendPrepare(preprepared1) for r in getNonPrimaryReplicas(nodeSet, instId): l = len([param for param in getAllArgs(r, r.processPrepare) if param['sender'] == primary.name]) assert l == 1 sendPrepareFromPrimary(0)
def setup(nodeSet, up): primaryRep, nonPrimaryReps = getPrimaryReplica(nodeSet, 0), \ getNonPrimaryReplicas(nodeSet, 0) # A non primary replica sends PREPARE messages with incorrect digest faultyRep = nonPrimaryReps[0] makeNodeFaulty(faultyRep.node, partial(send3PhaseMsgWithIncorrectDigest, msgType=Prepare, instId=0)) return adict(primaryRep=primaryRep, nonPrimaryReps=nonPrimaryReps, faultyRep=faultyRep)
def testReplicasRejectSamePrePrepareMsg(looper, nodeSet, client1): """ Replicas should not accept PRE-PREPARE for view "v" and prepare sequence number "n" if it has already accepted a request with view number "v" and sequence number "n" """ numOfNodes = 4 fValue = getMaxFailures(numOfNodes) request1 = sendRandomRequest(client1) result1 = looper.run( eventually(checkSufficientRepliesRecvd, client1.inBox, request1.reqId, fValue, retryWait=1, timeout=5)) logging.debug("request {} gives result {}".format(request1, result1)) primaryRepl = getPrimaryReplica(nodeSet) logging.debug("Primary Replica: {}".format(primaryRepl)) logging.debug( "Decrementing the primary replica's pre-prepare sequence number by one...") primaryRepl.prePrepareSeqNo -= 1 request2 = sendRandomRequest(client1) looper.run(eventually(checkPrePrepareReqSent, primaryRepl, request2, retryWait=1, timeout=10)) nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet) logging.debug("Non Primary Replicas: " + str(nonPrimaryReplicas)) prePrepareReq = PrePrepare( primaryRepl.instId, primaryRepl.viewNo, primaryRepl.prePrepareSeqNo, client1.clientId, request2.reqId, request2.digest) logging.debug("""Checking whether all the non primary replicas have received the pre-prepare request with same sequence number""") looper.run(eventually(checkPrePrepareReqRecvd, nonPrimaryReplicas, prePrepareReq, retryWait=1, timeout=10)) logging.debug("""Check that none of the non primary replicas didn't send any prepare message " in response to the pre-prepare message""") for npr in nonPrimaryReplicas: with pytest.raises(AssertionError): looper.run(eventually(checkPrepareReqSent, npr, client1.clientId, request2.reqId, retryWait=1, timeout=10))
def viewChangeDone(nodeSet, looper, up, client1): """ Test that a view change is done when the performance of master goes down """ # Delay processing of PRE-PREPARE from all non primary replicas of master # so master's performance falls and view changes nonPrimReps = getNonPrimaryReplicas(nodeSet, 0) for r in nonPrimReps: r.node.nodeIbStasher.delay(ppDelay(10, 0)) sendReqsToNodesAndVerifySuffReplies(looper, client1, 4) looper.run(eventually(partial(checkViewNoForNodes, nodeSet, 1), retryWait=1, timeout=20))
def setup(nodeSet, up): primaryRep, nonPrimaryReps = getPrimaryReplica(nodeSet, 0), \ getNonPrimaryReplicas(nodeSet, 0) # The primary replica would send 3 duplicate PRE-PREPARE requests to # non primary replicas makeNodeFaulty(primaryRep.node, partial(sendDuplicate3PhaseMsg, msgType=PrePrepare, count=3)) # The node of the primary replica above should not be blacklisted by any # other node since we are simulating multiple PRE-PREPARE messages and # want to check for a particular suspicion # for node in nodeSet: # if node != primaryRep.node: # node.whitelistNode(primaryRep.node.name, # Suspicions.DUPLICATE_PPR_SENT.code) return adict(primaryRep=primaryRep, nonPrimaryReps=nonPrimaryReps)
def testPrimarySendsAPrepareAndMarkedSuspicious(looper, nodeSet, preprepared1): def sendPrepareFromPrimary(instId): primary = getPrimaryReplica(nodeSet, instId) preprepared1.viewNo = instId preprepared1.ppSeqNo = primary.prePrepareSeqNo primary.sendPrepare(preprepared1) for r in getNonPrimaryReplicas(nodeSet, instId): l = len([param for param in getAllArgs(r, r.processPrepare) if param['sender'] == primary.name]) assert l == 1 sendPrepareFromPrimary(0) for node in nodeSet: if node in getNonPrimaryReplicas(nodeSet, 0): frm, reason, code = getAllArgs(node, Node.reportSuspiciousNode) assert frm == getPrimaryReplica(nodeSet, 0).node.name assert isinstance(reason, SuspiciousNode) assert len(getNodeSuspicions(node, Suspicions.PR_FRM_PRIMARY.code)) \ == 10
def testElectionsAfterViewChange(delayedPerf, looper: Looper, nodeSet: TestNodeSet, up, client1): """ Test that a primary election does happen after a view change """ # Delay processing of PRE-PREPARE from all non primary replicas of master so master's throughput falls # and view changes nonPrimReps = getNonPrimaryReplicas(nodeSet, 0) for r in nonPrimReps: r.node.nodeIbStasher.delay(ppDelay(10, 0)) sendReqsToNodesAndVerifySuffReplies(looper, client1, 4) # Ensure view change happened for both node and its primary elector for node in nodeSet: looper.run(eventually(partial(checkViewChangeInitiatedForNode, node, 0), retryWait=1, timeout=20)) # Ensure elections are done again and pool is setup again with appropriate protocol instances and each # protocol instance is setup properly too checkProtocolInstanceSetup(looper, nodeSet, retryWait=1, timeout=30)
def setup(nodeSet, up): primaryRep, nonPrimaryReps = getPrimaryReplica(nodeSet, 0), \ getNonPrimaryReplicas(nodeSet, 0) # A non primary replica sends duplicate PREPARE requests to all other # replicas faultyRep = nonPrimaryReps[0] makeNodeFaulty(faultyRep.node, partial(sendDuplicate3PhaseMsg, msgType=Prepare, count=3, instId=0)) # The node of the primary replica above should not be blacklisted by any # other node since we are simulating multiple PREPARE messages and # want to check for a particular suspicion whitelistNode(faultyRep.node.name, [node for node in nodeSet if node != faultyRep.node], Suspicions.DUPLICATE_PR_SENT.code) return adict(primaryRep=primaryRep, nonPrimaryReps=nonPrimaryReps, faultyRep=faultyRep)
def testViewNotChanged(looper: Looper, nodeSet: TestNodeSet, up, client1): """ Test that a view change is not done when the performance of master does not go down """ """ Send multiple requests to the client and delay some requests by all backup instances to ensure master instance is always faster than backup instances and there is no view change """ # Delay PRE-PREPARE for all backup protocol instances so master performs # better for i in range(1, F + 1): nonPrimReps = getNonPrimaryReplicas(nodeSet, i) # type: Iterable[TestReplica] for r in nonPrimReps: r.node.nodeIbStasher.delay(ppDelay(10, i)) sendReqsToNodesAndVerifySuffReplies(looper, client1, 5) checkViewNoForNodes(nodeSet, 0)
def g(instId): primary = getPrimaryReplica(nodeSet, instId) nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet, instId) def primarySeesCorrectNumberOfPREPREPAREs(): """ no of PRE-PREPARE as seen by processPrePrepare method for primary must be 0 with or without faults in system """ l1 = len([param for param in getAllArgs(primary, primary.processPrePrepare)]) assert l1 == 0 def nonPrimarySeesCorrectNumberOfPREPREPAREs(): """ 1. no of PRE-PREPARE as seen by processPrePrepare method for non-primaries must be 1; whn zero faulty nodes in system. 2. no of PRE-PREPARE as seen by processPrePrepare method for non-primaries must be greater than or equal to 0; with faults in system. """ expectedPrePrepareRequest = PrePrepare( instId, primary.viewNo, primary.prePrepareSeqNo, propagated1.clientId, propagated1.reqId, propagated1.digest) passes = 0 for npr in nonPrimaryReplicas: actualMsgs = len([param for param in getAllArgs(npr, npr.processPrePrepare) if (param['pp'], param['sender']) == ( expectedPrePrepareRequest, primary.name)]) numOfMsgsWithZFN = 1 numOfMsgsWithFaults = 0 passes += int(msgCountOK(nodesSize, faultyNodes, actualMsgs, numOfMsgsWithZFN, numOfMsgsWithFaults)) assert passes >= len(nonPrimaryReplicas) - faultyNodes def primarySentsCorrectNumberOfPREPREPAREs(): """ 1. no of PRE-PREPARE sent by primary is 1 with or without fault in system but, when primary is faulty no of sent PRE_PREPARE will be zero and primary must be marked as malicious. """ actualMsgs = len([param for param in getAllArgs(primary, primary.sendPrePrepare) if (param['reqDigest'].clientId, param['reqDigest'].reqId, param['reqDigest'].digest) == (propagated1.clientId, propagated1.reqId, propagated1.digest) ]) numOfMsgsWithZFN = 1 # TODO: Considering, Primary is not faulty and will always send # PRE-PREPARE. Write separate test for testing when Primary # is faulty assert msgCountOK(nodesSize, faultyNodes, actualMsgs, numOfMsgsWithZFN, numOfMsgsWithZFN) def nonPrimaryReceivesCorrectNumberOfPREPREPAREs(): """ 1. no of PRE-PREPARE received by non-primaries must be 1 with zero faults in system, and 0 faults in system. """ passes = 0 for npr in nonPrimaryReplicas: l4 = len([param for param in getAllArgs(npr, npr.addToPrePrepares) if (param['pp'].clientId, param['pp'].reqId, param['pp'].digest) == ( propagated1.clientId, propagated1.reqId, propagated1.digest)]) numOfMsgsWithZFN = 1 numOfMsgsWithFaults = 0 passes += msgCountOK(nodesSize, faultyNodes, l4, numOfMsgsWithZFN, numOfMsgsWithFaults) assert passes >= len(nonPrimaryReplicas) - faultyNodes primarySeesCorrectNumberOfPREPREPAREs() nonPrimarySeesCorrectNumberOfPREPREPAREs() primarySentsCorrectNumberOfPREPREPAREs() nonPrimaryReceivesCorrectNumberOfPREPREPAREs()
def g(instId): allReplicas = getAllReplicas(nodeSet, instId) primary = getPrimaryReplica(nodeSet, instId) nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet, instId) def primaryDontSendAnyPREPAREs(): """ 1. no of PREPARE sent by primary should be 0 """ for r in allReplicas: for param in getAllArgs(r, Replica.processPrepare): sender = param['sender'] assert sender != primary.name def allReplicasSeeCorrectNumberOfPREPAREs(): """ 1. no of PREPARE received by replicas must be n - 1; n = num of nodes without fault, and greater than or equal to 2f with faults. """ passes = 0 numOfMsgsWithZFN = nodeCount - 1 numOfMsgsWithFaults = 2 * f for replica in allReplicas: key = primary.viewNo, primary.prePrepareSeqNo if key in replica.prepares: actualMsgs = len(replica.prepares[key].voters) passes += int(msgCountOK(nodeCount, faultyNodes, actualMsgs, numOfMsgsWithZFN, numOfMsgsWithFaults)) assert passes >= len(allReplicas) - faultyNodes def primaryReceivesCorrectNumberOfPREPAREs(): """ num of PREPARE seen by primary replica is n - 1; n = num of nodes without fault, and greater than or equal to 2f with faults. """ actualMsgs = len([param for param in getAllArgs(primary, primary.processPrepare) if (param['prepare'].instId, param['prepare'].viewNo, param['prepare'].ppSeqNo) == (primary.instId, primary.viewNo, primary.prePrepareSeqNo) and param['sender'] != primary.name]) numOfMsgsWithZFN = nodeCount - 1 numOfMsgsWithFaults = 2 * f - 1 assert msgCountOK(nodeCount, faultyNodes, actualMsgs, numOfMsgsWithZFN, numOfMsgsWithFaults) # TODO what if the primary is faulty? def nonPrimaryReplicasReceiveCorrectNumberOfPREPAREs(): """ num of PREPARE seen by Non primary replica is n - 2 without faults and 2f - 1 with faults. """ passes = 0 numOfMsgsWithZFN = nodeCount - 2 numOfMsgsWithFaults = (2 * f) - 1 for npr in nonPrimaryReplicas: actualMsgs = len([param for param in getAllArgs( npr, npr.processPrepare) if (param['prepare'].instId, param['prepare'].viewNo, param['prepare'].ppSeqNo) == (primary.instId, primary.viewNo, primary.prePrepareSeqNo) ]) passes += int(msgCountOK(nodeCount, faultyNodes, actualMsgs, numOfMsgsWithZFN, numOfMsgsWithFaults)) assert passes >= len(nonPrimaryReplicas) - faultyNodes # TODO how do we know if one of the faulty nodes is a primary or # not? primaryDontSendAnyPREPAREs() allReplicasSeeCorrectNumberOfPREPAREs() primaryReceivesCorrectNumberOfPREPAREs() nonPrimaryReplicasReceiveCorrectNumberOfPREPAREs()