def testPrimarySendsAPrepareAndMarkedSuspicious(looper, nodeSet, preprepared1):
    def sendPrepareFromPrimary(instId):
        primary = getPrimaryReplica(nodeSet, instId)
        viewNo, ppSeqNo = next(iter(primary.sentPrePrepares.keys()))
        prepare = Prepare(instId, viewNo, ppSeqNo, preprepared1.digest,
                          time.time())
        primary.doPrepare(prepare)

        def chk():
            for r in getNonPrimaryReplicas(nodeSet, instId):
                l = len([
                    param for param in getAllArgs(r, r.processPrepare)
                    if param['sender'] == primary.name
                ])
                assert l == 1

        looper.run(eventually(chk))

    sendPrepareFromPrimary(0)

    for node in nodeSet:
        if node in getNonPrimaryReplicas(nodeSet, 0):
            frm, reason, code = getAllArgs(node, TestNode.reportSuspiciousNode)
            assert frm == getPrimaryReplica(nodeSet, 0).node.name
            assert isinstance(reason, SuspiciousNode)
            assert len(getNodeSuspicions(node,
                                         Suspicions.PR_FRM_PRIMARY.code)) == 10
Beispiel #2
0
def testNonPrimarySendsAPrePrepare(looper, nodeSet, setup, propagated1):
    primaryReplica = getPrimaryReplica(nodeSet, instId)
    nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet, instId)
    firstNpr = nonPrimaryReplicas[0]
    remainingNpr = nonPrimaryReplicas[1:]

    def sendPrePrepareFromNonPrimary(replica):
        firstNpr.doPrePrepare(propagated1.reqDigest)

        return PrePrepare(
                replica.instId,
                firstNpr.viewNo,
                firstNpr.prePrepareSeqNo,
                propagated1.identifier,
                propagated1.reqId,
                propagated1.digest,
                time.time())

    ppr = sendPrePrepareFromNonPrimary(firstNpr)

    def chk():
        for r in (primaryReplica, *remainingNpr):
            recvdPps = recvdPrePrepare(r)
            assert len(recvdPps) == 1
            assert recvdPps[0]['pp'][:-1] == ppr[:-1]
            nodeSuspicions = len(getNodeSuspicions(
                r.node, Suspicions.PPR_FRM_NON_PRIMARY.code))
            assert nodeSuspicions == 1

    looper.run(eventually(chk,
                          retryWait=.5, timeout=5))
def setup(looper, startedNodes, up, wallet1, client1):
    # Get the master replica of the master protocol instance
    P = getPrimaryReplica(startedNodes)

    # Make `Delta` small enough so throughput check passes.
    for node in startedNodes:
        node.monitor.Delta = .001

    slowRequest = None

    # make P (primary replica on master) faulty, i.e., slow to send
    # PRE-PREPARE for a specific client request only
    def by65SpecificPrePrepare(msg):
        nonlocal slowRequest
        if isinstance(msg, PrePrepare) and slowRequest is None:
            slowRequest = getattr(msg, f.REQ_ID.nm)
            return 65

    P.outBoxTestStasher.delay(by65SpecificPrePrepare)

    sendReqsToNodesAndVerifySuffReplies(looper,
                                        wallet1,
                                        client1,
                                        numReqs=5,
                                        timeoutPerReq=80)

    return adict(nodes=startedNodes)
def testPrePrepareWithHighSeqNo(looper, nodeSet, propagated1):
    def chk():
        for r in getNonPrimaryReplicas(nodeSet, instId):
            nodeSuspicions = len(
                getNodeSuspicions(r.node, Suspicions.WRONG_PPSEQ_NO.code))
            assert nodeSuspicions == 1

    def checkPreprepare(replica, viewNo, ppSeqNo, req, numOfPrePrepares):
        assert (replica.prePrepares[viewNo, ppSeqNo][0]) == \
               (req.identifier, req.reqId, req.digest)

    primary = getPrimaryReplica(nodeSet, instId)
    nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet, instId)
    req = propagated1.reqDigest
    primary.doPrePrepare(req)
    for np in nonPrimaryReplicas:
        looper.run(
            eventually(checkPreprepare,
                       np,
                       primary.viewNo,
                       primary.lastPrePrepareSeqNo - 1,
                       req,
                       1,
                       retryWait=.5,
                       timeout=10))

    newReqDigest = ReqDigest(req.identifier, req.reqId + 1, req.digest)
    incorrectPrePrepareReq = PrePrepare(instId, primary.viewNo,
                                        primary.lastPrePrepareSeqNo + 2,
                                        *newReqDigest, time.time())
    primary.send(incorrectPrePrepareReq, TPCStat.PrePrepareSent)
    looper.run(eventually(chk, retryWait=1, timeout=50))
def testPrePrepareWithHighSeqNo(looper, nodeSet, propagated1):
    def chk():
        for r in getNonPrimaryReplicas(nodeSet, instId):
            nodeSuspicions = len(getNodeSuspicions(
                    r.node, Suspicions.WRONG_PPSEQ_NO.code))
            assert nodeSuspicions == 1

    def checkPreprepare(replica, viewNo, ppSeqNo, req, numOfPrePrepares):
        assert (replica.prePrepares[viewNo, ppSeqNo]) == (req.clientId, req.reqId, req.digest)

    primary = getPrimaryReplica(nodeSet, instId)
    nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet, instId)
    req = propagated1.reqDigest
    primary.doPrePrepare(req)
    for np in nonPrimaryReplicas:
        looper.run(
                eventually(checkPreprepare, np, primary.viewNo, primary.prePrepareSeqNo - 1,
                           req, 1,
                           retryWait=.5, timeout=10))

    newReqDigest = ReqDigest(req.clientId, req.reqId + 1, req.digest)
    incorrectPrePrepareReq = PrePrepare(instId,
                               primary.viewNo,
                               primary.prePrepareSeqNo + 2,
                               *newReqDigest)
    primary.send(incorrectPrePrepareReq,TPCStat.PrePrepareSent)
    looper.run(eventually(chk, retryWait=1, timeout=50))
Beispiel #6
0
def testReplicasRejectSamePrePrepareMsg(looper, nodeSet, client1, wallet1):
    """
    Replicas should not accept PRE-PREPARE for view "v" and prepare sequence
    number "n" if it has already accepted a request with view number "v" and
    sequence number "n"

    """
    numOfNodes = 4
    fValue = getMaxFailures(numOfNodes)
    request1 = sendRandomRequest(wallet1, client1)
    result1 = looper.run(
        eventually(checkSufficientRepliesRecvd,
                   client1.inBox,
                   request1.reqId,
                   fValue,
                   retryWait=1,
                   timeout=5))
    logger.debug("request {} gives result {}".format(request1, result1))
    primaryRepl = getPrimaryReplica(nodeSet)
    logger.debug("Primary Replica: {}".format(primaryRepl))
    logger.debug(
        "Decrementing the primary replica's pre-prepare sequence number by "
        "one...")
    primaryRepl.lastPrePrepareSeqNo -= 1
    request2 = sendRandomRequest(wallet1, client1)
    looper.run(
        eventually(checkPrePrepareReqSent,
                   primaryRepl,
                   request2,
                   retryWait=1,
                   timeout=10))

    nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet)
    logger.debug("Non Primary Replicas: " + str(nonPrimaryReplicas))
    prePrepareReq = PrePrepare(primaryRepl.instId, primaryRepl.viewNo,
                               primaryRepl.lastPrePrepareSeqNo,
                               wallet1.defaultId, request2.reqId,
                               request2.digest, time.time())

    logger.debug("""Checking whether all the non primary replicas have received
                the pre-prepare request with same sequence number""")
    looper.run(
        eventually(checkPrePrepareReqRecvd,
                   nonPrimaryReplicas,
                   prePrepareReq,
                   retryWait=1,
                   timeout=10))
    logger.debug("""Check that none of the non primary replicas didn't send
    any prepare message "
                             in response to the pre-prepare message""")
    for npr in nonPrimaryReplicas:
        with pytest.raises(AssertionError):
            looper.run(
                eventually(checkPrepareReqSent,
                           npr,
                           wallet1.defaultId,
                           request2.reqId,
                           retryWait=1,
                           timeout=10))
    def g(instId):
        allReplicas = getAllReplicas(nodeSet, instId)
        primaryReplica = getPrimaryReplica(nodeSet, instId)

        def replicasSeesCorrectNumOfCOMMITs():
            """
            num of commit messages must be = n when zero fault;
            n = num of nodes and greater than or equal to
            2f + 1 with faults.
            """
            passes = 0
            numOfMsgsWithZFN = nodeCount
            numOfMsgsWithFault = (2 * f) + 1

            key = (primaryReplica.viewNo, primaryReplica.prePrepareSeqNo)
            for r in allReplicas:
                if key in r.commits:
                    rcvdCommitRqst = r.commits[key]
                    assert rcvdCommitRqst[0] == prepared1.digest
                    actualMsgsReceived = len(rcvdCommitRqst[1])

                    passes += int(msgCountOK(nodeCount,
                                             faultyNodes,
                                             actualMsgsReceived,
                                             numOfMsgsWithZFN,
                                             numOfMsgsWithFault))

            assert passes >= len(allReplicas) - faultyNodes

        def replicasReceivesCorrectNumberOfCOMMITs():
            """
            num of commit messages seen by replica must be equal to n - 1;
            when zero fault and greater than or equal to
            2f+1 with faults.
            """
            passes = 0
            numOfMsgsWithZFN = nodeCount - 1
            numOfMsgsWithFault = 2 * f

            for r in allReplicas:
                args = getAllArgs(r, r.processCommit)
                actualMsgsReceived = len(args)

                passes += int(msgCountOK(nodeCount,
                                         faultyNodes,
                                         actualMsgsReceived,
                                         numOfMsgsWithZFN,
                                         numOfMsgsWithFault))

                for arg in args:
                    assert arg['commit'].viewNo == primaryReplica.viewNo and \
                           arg['commit'].ppSeqNo == primaryReplica.prePrepareSeqNo and \
                           arg['commit'].digest == prepared1.digest
                    assert r.name != arg['sender']

            assert passes >= len(allReplicas) - faultyNodes

        replicasReceivesCorrectNumberOfCOMMITs()
        replicasSeesCorrectNumOfCOMMITs()
Beispiel #8
0
def setup(nodeSet, up):
    def dontSendPrePrepareRequest(self, reqDigest: ReqDigest):
        logger.debug("EVIL: {} not sending pre-prepare message for request {}".
                     format(self.name, reqDigest))
        return

    pr = getPrimaryReplica(nodeSet, instId)
    evilMethod = types.MethodType(dontSendPrePrepareRequest, pr)
    pr.doPrePrepare = evilMethod
def testPrimaryForfeit(looper, nodeSet, up, client1, wallet1):
    """
    The primary of master protocol instance of the pool forfeits the primary
    status by triggering an election and not nominating itself
    """
    pr = getPrimaryReplica(nodeSet, instId=0)
    prNode = pr.node
    # TODO: Incomplete
    pass
def setup(nodeSet, up):
    primaryRep, nonPrimaryReps = getPrimaryReplica(nodeSet, 0), \
                                 getNonPrimaryReplicas(nodeSet, 0)

    # The primary replica would send PRE-PREPARE messages with incorrect digest
    makeNodeFaulty(primaryRep.node, partial(send3PhaseMsgWithIncorrectDigest,
                                            msgType=PrePrepare))

    return adict(primaryRep=primaryRep, nonPrimaryReps=nonPrimaryReps)
def setup(nodeSet, up):
    primaryRep = getPrimaryReplica(nodeSet, 0)
    nonPrimaryReps = getNonPrimaryReplicas(nodeSet, 0)

    faultyRep = nonPrimaryReps[0]
    makeNodeFaulty(faultyRep.node, partial(send3PhaseMsgWithIncorrectDigest,
                                           msgType=Commit, instId=0))

    return adict(primaryRep=primaryRep, nonPrimaryReps=nonPrimaryReps,
                 faultyRep=faultyRep)
def setup(nodeSet, up):
    primaryRep, nonPrimaryReps = getPrimaryReplica(nodeSet, 0), \
                                 getNonPrimaryReplicas(nodeSet, 0)

    # The primary replica would send PRE-PREPARE messages with incorrect digest
    makeNodeFaulty(
        primaryRep.node,
        partial(send3PhaseMsgWithIncorrectDigest, msgType=PrePrepare))

    return adict(primaryRep=primaryRep, nonPrimaryReps=nonPrimaryReps)
def testReplicasRejectSamePrePrepareMsg(looper, nodeSet, client1, wallet1):
    """
    Replicas should not accept PRE-PREPARE for view "v" and prepare sequence
    number "n" if it has already accepted a request with view number "v" and
    sequence number "n"

    """
    numOfNodes = 4
    fValue = getMaxFailures(numOfNodes)
    request1 = sendRandomRequest(wallet1, client1)
    result1 = looper.run(
        eventually(checkSufficientRepliesRecvd, client1.inBox,
                   request1.reqId, fValue,
                   retryWait=1, timeout=5))
    logger.debug("request {} gives result {}".format(request1, result1))
    primaryRepl = getPrimaryReplica(nodeSet)
    logger.debug("Primary Replica: {}".format(primaryRepl))
    logger.debug(
        "Decrementing the primary replica's pre-prepare sequence number by "
        "one...")
    primaryRepl.lastPrePrepareSeqNo -= 1
    request2 = sendRandomRequest(wallet1, client1)
    looper.run(eventually(checkPrePrepareReqSent, primaryRepl, request2,
                          retryWait=1, timeout=10))

    nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet)
    logger.debug("Non Primary Replicas: " + str(nonPrimaryReplicas))
    prePrepareReq = PrePrepare(
        primaryRepl.instId,
        primaryRepl.viewNo,
        primaryRepl.lastPrePrepareSeqNo,
        wallet1.defaultId,
        request2.reqId,
        request2.digest,
        time.time()
    )

    logger.debug("""Checking whether all the non primary replicas have received
                the pre-prepare request with same sequence number""")
    looper.run(eventually(checkPrePrepareReqRecvd,
                          nonPrimaryReplicas,
                          prePrepareReq,
                          retryWait=1,
                          timeout=10))
    logger.debug("""Check that none of the non primary replicas didn't send
    any prepare message "
                             in response to the pre-prepare message""")
    for npr in nonPrimaryReplicas:
        with pytest.raises(AssertionError):
            looper.run(eventually(checkPrepareReqSent,
                                  npr,
                                  wallet1.defaultId,
                                  request2.reqId,
                                  retryWait=1,
                                  timeout=10))
Beispiel #14
0
    def g(instId):
        allReplicas = getAllReplicas(nodeSet, instId)
        primaryReplica = getPrimaryReplica(nodeSet, instId)

        def replicasSeesCorrectNumOfCOMMITs():
            """
            num of commit messages must be = n when zero fault;
            n = num of nodes and greater than or equal to
            2f + 1 with faults.
            """
            passes = 0
            numOfMsgsWithZFN = nodeCount
            numOfMsgsWithFault = (2 * f) + 1

            key = (primaryReplica.viewNo, primaryReplica.prePrepareSeqNo)
            for r in allReplicas:
                if key in r.commits:
                    rcvdCommitRqst = r.commits[key]
                    assert rcvdCommitRqst[0] == prepared1.digest
                    actualMsgsReceived = len(rcvdCommitRqst[1])

                    passes += int(
                        msgCountOK(nodeCount, faultyNodes, actualMsgsReceived,
                                   numOfMsgsWithZFN, numOfMsgsWithFault))

            assert passes >= len(allReplicas) - faultyNodes

        def replicasReceivesCorrectNumberOfCOMMITs():
            """
            num of commit messages seen by replica must be equal to n - 1;
            when zero fault and greater than or equal to
            2f+1 with faults.
            """
            passes = 0
            numOfMsgsWithZFN = nodeCount - 1
            numOfMsgsWithFault = 2 * f

            for r in allReplicas:
                args = getAllArgs(r, r.processCommit)
                actualMsgsReceived = len(args)

                passes += int(
                    msgCountOK(nodeCount, faultyNodes, actualMsgsReceived,
                               numOfMsgsWithZFN, numOfMsgsWithFault))

                for arg in args:
                    assert arg['commit'].viewNo == primaryReplica.viewNo and \
                           arg['commit'].ppSeqNo == primaryReplica.prePrepareSeqNo and \
                           arg['commit'].digest == prepared1.digest
                    assert r.name != arg['sender']

            assert passes >= len(allReplicas) - faultyNodes

        replicasReceivesCorrectNumberOfCOMMITs()
        replicasSeesCorrectNumOfCOMMITs()
    def sendPrepareFromPrimary(instId):
        primary = getPrimaryReplica(nodeSet, instId)
        preprepared1.viewNo = instId
        preprepared1.ppSeqNo = primary.prePrepareSeqNo
        primary.doPrepare(preprepared1)
        for r in getNonPrimaryReplicas(nodeSet, instId):
            l = len([param for param in getAllArgs(r, r.processPrepare)
                     if param['sender'] == primary.name])
            assert l == 1

        sendPrepareFromPrimary(0)
Beispiel #16
0
def replied1(looper, nodeSet, client1, committed1):
    for instId in range(getNoInstances(len(nodeSet))):
        getPrimaryReplica(nodeSet, instId)

        looper.run(*[eventually(checkRequestReturnedToNode,
                                node,
                                client1.defaultIdentifier,
                                committed1.reqId,
                                committed1.digest,
                                instId,
                                retryWait=1, timeout=30)
                     for node in nodeSet])

        looper.run(eventually(
                checkSufficientRepliesRecvd,
                client1.inBox,
                committed1.reqId,
                2,
                retryWait=2,
                timeout=30))
    return committed1
Beispiel #17
0
def setup(nodeSet, up):
    primaryRep, nonPrimaryReps = getPrimaryReplica(nodeSet, 0), \
                                 getNonPrimaryReplicas(nodeSet, 0)

    # A non primary replica sends PREPARE messages with incorrect digest

    faultyRep = nonPrimaryReps[0]
    makeNodeFaulty(faultyRep.node, partial(send3PhaseMsgWithIncorrectDigest,
                                           msgType=Prepare, instId=0))

    return adict(primaryRep=primaryRep, nonPrimaryReps=nonPrimaryReps,
                 faultyRep=faultyRep)
Beispiel #18
0
def setup(nodeSet, up):
    primaryRep, nonPrimaryReps = getPrimaryReplica(nodeSet, 0), \
                                 getNonPrimaryReplicas(nodeSet, 0)

    # A non primary replica sends PREPARE messages with incorrect digest

    faultyRep = nonPrimaryReps[0]
    makeNodeFaulty(faultyRep.node, partial(send3PhaseMsgWithIncorrectDigest,
                                           msgType=Prepare, instId=0))

    return adict(primaryRep=primaryRep, nonPrimaryReps=nonPrimaryReps,
                 faultyRep=faultyRep)
Beispiel #19
0
def step1(looper, nodeSet, up, wallet1, client1):
    startedNodes = nodeSet
    """
    stand up a pool of nodes and send 5 requests to client
    """
    # the master instance has a primary replica, call it P
    P = getPrimaryReplica(startedNodes)

    requests = sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 5)
    # profile_this(sendReqsToNodesAndVerifySuffReplies, looper, client1, 5)

    return adict(P=P, nodes=startedNodes, requests=requests)
Beispiel #20
0
def setup(nodeSet, up):
    primaryRep = getPrimaryReplica(nodeSet, 0)
    nonPrimaryReps = getNonPrimaryReplicas(nodeSet, 0)

    faultyRep = nonPrimaryReps[0]
    makeNodeFaulty(
        faultyRep.node,
        partial(send3PhaseMsgWithIncorrectDigest, msgType=Commit, instId=0))

    return adict(primaryRep=primaryRep,
                 nonPrimaryReps=nonPrimaryReps,
                 faultyRep=faultyRep)
    def sendPrepareFromPrimary(instId):
        primary = getPrimaryReplica(nodeSet, instId)
        preprepared1.viewNo = instId
        preprepared1.ppSeqNo = primary.prePrepareSeqNo
        primary.doPrepare(preprepared1)
        for r in getNonPrimaryReplicas(nodeSet, instId):
            l = len([
                param for param in getAllArgs(r, r.processPrepare)
                if param['sender'] == primary.name
            ])
            assert l == 1

        sendPrepareFromPrimary(0)
def step1(looper, startedNodes, up, client1):
    """
    stand up a pool of nodes and send 5 requests to client
    """
    # the master instance has a primary replica, call it P
    P = getPrimaryReplica(startedNodes)

    requests = sendReqsToNodesAndVerifySuffReplies(looper, client1, 5)
    # profile_this(sendReqsToNodesAndVerifySuffReplies, looper, client1, 5)

    return adict(P=P,
                 nodes=startedNodes,
                 requests=requests)
def setup(nodeSet, up):
    primaryRep, nonPrimaryReps = getPrimaryReplica(nodeSet, 0), \
                                 getNonPrimaryReplicas(nodeSet, 0)

    # The primary replica would send 3 duplicate PRE-PREPARE requests to
    # non primary replicas
    makeNodeFaulty(primaryRep.node, partial(sendDuplicate3PhaseMsg,
                                            msgType=PrePrepare, count=3))

    # The node of the primary replica above should not be blacklisted by any
    # other node since we are simulating multiple PRE-PREPARE messages and
    # want to check for a particular suspicion

    return adict(primaryRep=primaryRep, nonPrimaryReps=nonPrimaryReps)
def setup(nodeSet, up):
    primaryRep, nonPrimaryReps = getPrimaryReplica(nodeSet, 0), \
                                 getNonPrimaryReplicas(nodeSet, 0)

    # The primary replica would send 3 duplicate PRE-PREPARE requests to
    # non primary replicas
    makeNodeFaulty(primaryRep.node, partial(sendDuplicate3PhaseMsg,
                                            msgType=PrePrepare, count=3))

    # The node of the primary replica above should not be blacklisted by any
    # other node since we are simulating multiple PRE-PREPARE messages and
    # want to check for a particular suspicion

    return adict(primaryRep=primaryRep, nonPrimaryReps=nonPrimaryReps)
Beispiel #25
0
def replied1(looper, nodeSet, client1, committed1, wallet1):
    for instId in range(getNoInstances(len(nodeSet))):
        getPrimaryReplica(nodeSet, instId)

        looper.run(*[
            eventually(checkRequestReturnedToNode,
                       node,
                       wallet1.defaultId,
                       committed1.reqId,
                       committed1.digest,
                       instId,
                       retryWait=1,
                       timeout=30) for node in nodeSet
        ])

        looper.run(
            eventually(checkSufficientRepliesRecvd,
                       client1.inBox,
                       committed1.reqId,
                       2,
                       retryWait=2,
                       timeout=30))
    return committed1
    def sendPrepareFromPrimary(instId):
        primary = getPrimaryReplica(nodeSet, instId)
        viewNo, ppSeqNo = next(iter(primary.sentPrePrepares.keys()))
        prepare = Prepare(instId, viewNo, ppSeqNo, preprepared1.digest,
                          time.time())
        primary.doPrepare(prepare)

        def chk():
            for r in getNonPrimaryReplicas(nodeSet, instId):
                l = len([
                    param for param in getAllArgs(r, r.processPrepare)
                    if param['sender'] == primary.name
                ])
                assert l == 1

        looper.run(eventually(chk))
def testPrimarySelectionAfterViewChange(looper, nodeSet, ready, primaryReplicas,
                                        viewChangeDone):
    """
    Test that primary replica of a protocol instance shifts to a new node after a view change.
    """

    # Primary replicas before view change
    prBeforeVC = primaryReplicas

    # Primary replicas after view change
    instanceCount = getNoInstances(nodeCount)
    prAfterVC = [getPrimaryReplica(nodeSet, i) for i in range(instanceCount)]

    # Primary replicas have moved to the next node
    for br, ar in zip(prBeforeVC, prAfterVC):
        assert ar.node.rank - br.node.rank == 1

    checkProtocolInstanceSetup(looper, nodeSet, retryWait=1, timeout=5)
def setup(nodeSet, up):
    primaryRep, nonPrimaryReps = getPrimaryReplica(nodeSet, 0), \
                                 getNonPrimaryReplicas(nodeSet, 0)

    faultyRep = nonPrimaryReps[0]
    makeNodeFaulty(faultyRep.node, partial(sendDuplicate3PhaseMsg,
                                           msgType=Commit, count=3,
                                           instId=0))

    # The node of the primary replica above should not be blacklisted by any
    # other node since we are simulating multiple COMMIT messages and
    # want to check for a particular suspicion

    whitelistNode(faultyRep.node.name,
                  [node for node in nodeSet if node != faultyRep.node],
                  Suspicions.DUPLICATE_CM_SENT.code)

    return adict(primaryRep=primaryRep, nonPrimaryReps=nonPrimaryReps,
                 faultyRep=faultyRep)
def setup(looper, startedNodes, up, client1):
    # Get the master replica of the master protocol instance
    P = getPrimaryReplica(startedNodes)

    # Make `Delta` small enough so throughput check passes.
    for node in startedNodes:
        node.monitor.Delta = .001

    # make P (primary replica on master) faulty, i.e., slow to send
    # PRE-PREPARE for a specific client request only
    def by65SpecificPrePrepare(msg):
        if isinstance(msg, PrePrepare) and getattr(msg, f.REQ_ID.nm) == 2:
            return 65

    P.outBoxTestStasher.delay(by65SpecificPrePrepare)

    sendReqsToNodesAndVerifySuffReplies(looper, client1, numReqs=5, timeout=80)

    return adict(nodes=startedNodes)
Beispiel #30
0
def setup(nodeSet, up):
    primaryRep, nonPrimaryReps = getPrimaryReplica(nodeSet, 0), \
                                 getNonPrimaryReplicas(nodeSet, 0)

    faultyRep = nonPrimaryReps[0]
    makeNodeFaulty(
        faultyRep.node,
        partial(sendDuplicate3PhaseMsg, msgType=Commit, count=3, instId=0))

    # The node of the primary replica above should not be blacklisted by any
    # other node since we are simulating multiple COMMIT messages and
    # want to check for a particular suspicion

    whitelistNode(faultyRep.node.name,
                  [node for node in nodeSet if node != faultyRep.node],
                  Suspicions.DUPLICATE_CM_SENT.code)

    return adict(primaryRep=primaryRep,
                 nonPrimaryReps=nonPrimaryReps,
                 faultyRep=faultyRep)
def testPrimarySendsAPrepareAndMarkedSuspicious(looper, nodeSet, preprepared1):
    def sendPrepareFromPrimary(instId):
        primary = getPrimaryReplica(nodeSet, instId)
        preprepared1.viewNo = instId
        preprepared1.ppSeqNo = primary.prePrepareSeqNo
        primary.doPrepare(preprepared1)
        for r in getNonPrimaryReplicas(nodeSet, instId):
            l = len([param for param in getAllArgs(r, r.processPrepare)
                     if param['sender'] == primary.name])
            assert l == 1

        sendPrepareFromPrimary(0)

    for node in nodeSet:
        if node in getNonPrimaryReplicas(nodeSet, 0):
            frm, reason, code = getAllArgs(node, Node.reportSuspiciousNode)
            assert frm == getPrimaryReplica(nodeSet, 0).node.name
            assert isinstance(reason, SuspiciousNode)
            assert len(getNodeSuspicions(node, Suspicions.PR_FRM_PRIMARY.code)) \
                   == 10
Beispiel #32
0
def testViewChangeCase1(nodeSet, looper, up, wallet1, client1, viewNo):
    """
    Node will change view even though it does not find the master to be degraded
    when a quorum of nodes agree that master performance degraded
    """

    # Delay processing of PRE-PREPARE from all non primary replicas of master
    # so master's performance falls and view changes
    delayNonPrimaries(nodeSet, 0, 10)

    pr = getPrimaryReplica(nodeSet, 0)
    relucatantNode = pr.node

    # Count sent instance changes of all nodes
    sentInstChanges = {}
    instChngMethodName = Node.sendInstanceChange.__name__
    for n in nodeSet:
        sentInstChanges[n.name] = n.spylog.count(instChngMethodName)

    # Node reluctant to change view, never says master is degraded
    relucatantNode.monitor.isMasterDegraded = types.MethodType(
        lambda x: False, relucatantNode.monitor)

    sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 4)

    # Check that view change happened for all nodes
    looper.run(
        eventually(partial(checkViewNoForNodes, nodeSet, viewNo + 1),
                   retryWait=1,
                   timeout=20))

    # All nodes except the reluctant node should have sent a view change and
    # thus must have called `sendInstanceChange`
    for n in nodeSet:
        if n.name != relucatantNode.name:
            assert n.spylog.count(instChngMethodName) > \
                   sentInstChanges.get(n.name, 0)
        else:
            assert n.spylog.count(instChngMethodName) == \
                   sentInstChanges.get(n.name, 0)
def testPrimarySendsAPrepareAndMarkedSuspicious(looper, nodeSet, preprepared1):
    def sendPrepareFromPrimary(instId):
        primary = getPrimaryReplica(nodeSet, instId)
        preprepared1.viewNo = instId
        preprepared1.ppSeqNo = primary.prePrepareSeqNo
        primary.doPrepare(preprepared1)
        for r in getNonPrimaryReplicas(nodeSet, instId):
            l = len([
                param for param in getAllArgs(r, r.processPrepare)
                if param['sender'] == primary.name
            ])
            assert l == 1

        sendPrepareFromPrimary(0)

    for node in nodeSet:
        if node in getNonPrimaryReplicas(nodeSet, 0):
            frm, reason, code = getAllArgs(node, Node.reportSuspiciousNode)
            assert frm == getPrimaryReplica(nodeSet, 0).node.name
            assert isinstance(reason, SuspiciousNode)
            assert len(getNodeSuspicions(node, Suspicions.PR_FRM_PRIMARY.code)) \
                   == 10
def step2(step1, looper):
    """
    Sends requests to client and check the ratio of throughput of master
    instance and backup instance must be greater than or equal to Delta and
    verify no view change takes place.
    """
    # record when Node.checkPerformance was last run
    lastPerfChecks = latestPerfChecks(step1.nodes)

    # wait for every node to run another checkPerformance
    newPerfChecks = waitForNextPerfCheck(looper, step1.nodes, lastPerfChecks)

    # verify all nodes say that P is performing OK, and that no view changes
    # have been done
    for n in step1.nodes:
        assert n.viewNo == 0

    # verify Primary is still the same
    assert getPrimaryReplica(step1.nodes) == step1.P

    step1.perfChecks = newPerfChecks
    return step1
Beispiel #35
0
def step2(step1, looper):
    """
    Sends requests to client and check the ratio of throughput of master
    instance and backup instance must be greater than or equal to Delta and
    verify no view change takes place.
    """
    # record when Node.checkPerformance was last run
    lastPerfChecks = latestPerfChecks(step1.nodes)

    # wait for every node to run another checkPerformance
    newPerfChecks = waitForNextPerfCheck(looper, step1.nodes, lastPerfChecks)

    # verify all nodes say that P is performing OK, and that no view changes
    # have been done
    for n in step1.nodes:
        assert n.viewNo == 0

    # verify Primary is still the same
    assert getPrimaryReplica(step1.nodes) == step1.P

    step1.perfChecks = newPerfChecks
    return step1
Beispiel #36
0
def testViewChangeCase1(nodeSet, looper, up, wallet1, client1, viewNo):
    """
    Node will change view even though it does not find the master to be degraded
    when a quorum of nodes agree that master performance degraded
    """

    # Delay processing of PRE-PREPARE from all non primary replicas of master
    # so master's performance falls and view changes
    delayNonPrimaries(nodeSet, 0, 10)

    pr = getPrimaryReplica(nodeSet, 0)
    relucatantNode = pr.node

    # Count sent instance changes of all nodes
    sentInstChanges = {}
    instChngMethodName = Node.sendInstanceChange.__name__
    for n in nodeSet:
        sentInstChanges[n.name] = n.spylog.count(instChngMethodName)

    # Node reluctant to change view, never says master is degraded
    relucatantNode.monitor.isMasterDegraded = types.MethodType(
        lambda x: False, relucatantNode.monitor)

    sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 4)

    # Check that view change happened for all nodes
    looper.run(eventually(partial(checkViewNoForNodes, nodeSet, viewNo + 1),
                          retryWait=1, timeout=20))

    # All nodes except the reluctant node should have sent a view change and
    # thus must have called `sendInstanceChange`
    for n in nodeSet:
        if n.name != relucatantNode.name:
            assert n.spylog.count(instChngMethodName) > \
                   sentInstChanges.get(n.name, 0)
        else:
            assert n.spylog.count(instChngMethodName) == \
                   sentInstChanges.get(n.name, 0)
Beispiel #37
0
def testOrderingCase2(looper, nodeSet, up, client1, wallet1):
    """
    Scenario -> A client sends requests, some nodes delay COMMITs to few
    specific nodes such some nodes achieve commit quorum later for those
    requests compared to other nodes. But all nodes `ORDER` request in the same
    order of ppSeqNos
    https://www.pivotaltracker.com/n/projects/1889887/stories/133655009
    """
    pr, replicas = getPrimaryReplica(nodeSet, instId=0), \
                   getNonPrimaryReplicas(nodeSet, instId=0)
    assert len(replicas) == 6

    rep0 = pr
    rep1 = replicas[0]
    rep2 = replicas[1]
    rep3 = replicas[2]
    rep4 = replicas[3]
    rep5 = replicas[4]
    rep6 = replicas[5]

    node0 = rep0.node
    node1 = rep1.node
    node2 = rep2.node
    node3 = rep3.node
    node4 = rep4.node
    node5 = rep5.node
    node6 = rep6.node

    ppSeqsToDelay = 5
    commitDelay = 3  # delay each COMMIT by this number of seconds
    delayedPpSeqNos = set()

    requestCount = 15
    requests = sendRandomRequests(wallet1, client1, requestCount)

    def specificCommits(wrappedMsg):
        nonlocal node3, node4, node5
        msg, sender = wrappedMsg
        if isinstance(msg, PrePrepare):
            if len(delayedPpSeqNos) < ppSeqsToDelay:
                delayedPpSeqNos.add(msg.ppSeqNo)
                logger.debug('ppSeqNo {} corresponding to request id {} would '
                             'be delayed'.format(msg.ppSeqNo, msg.reqId))
        if isinstance(msg, Commit) and msg.instId == 0 and \
            sender in (n.name for n in (node3, node4, node5)) and \
                msg.ppSeqNo in delayedPpSeqNos:
            return commitDelay

    for node in (node1, node2):
        logger.debug('{} would be delaying commits'.format(node))
        node.nodeIbStasher.delay(specificCommits)

    checkSufficientRepliesForRequests(looper, client1, requests)

    def ensureSlowNodesHaveAllTxns():
        nonlocal node1, node2
        for node in node1, node2:
            assert len(node.domainLedger) == requestCount

    looper.run(eventually(ensureSlowNodesHaveAllTxns, retryWait=1, timeout=15))

    checkAllLedgersEqual((n.domainLedger for n in (node0, node3, node4,
                                                   node5, node6)))

    for node in (node1, node2):
        for n in nodeSet:
            if n != node:
                checkLedgerEquality(node.domainLedger, n.domainLedger)

    checkAllLedgersEqual((n.domainLedger for n in nodeSet))
    def g(instId):
        allReplicas = getAllReplicas(nodeSet, instId)
        primary = getPrimaryReplica(nodeSet, instId)
        nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet, instId)

        def primaryDontSendAnyPREPAREs():
            """
            1. no of PREPARE sent by primary should be 0
            """
            for r in allReplicas:
                for param in getAllArgs(r, Replica.processPrepare):
                    sender = param['sender']
                    assert sender != primary.name

        def allReplicasSeeCorrectNumberOfPREPAREs():
            """
            1. no of PREPARE received by replicas must be n - 1;
            n = num of nodes without fault, and greater than or equal to
             2f with faults.
            """
            passes = 0
            numOfMsgsWithZFN = nodeCount - 1
            numOfMsgsWithFaults = 2 * f

            for replica in allReplicas:
                key = primary.viewNo, primary.lastPrePrepareSeqNo
                if key in replica.prepares:
                    actualMsgs = len(replica.prepares[key].voters)

                    passes += int(
                        msgCountOK(nodeCount, faultyNodes, actualMsgs,
                                   numOfMsgsWithZFN, numOfMsgsWithFaults))
            assert passes >= len(allReplicas) - faultyNodes

        def primaryReceivesCorrectNumberOfPREPAREs():
            """
            num of PREPARE seen by primary replica is n - 1;
                n = num of nodes without fault, and greater than or equal to
             2f with faults.
            """
            actualMsgs = len([
                param for param in getAllArgs(primary, primary.processPrepare)
                if (param['prepare'].instId, param['prepare'].viewNo,
                    param['prepare'].ppSeqNo) == (primary.instId,
                                                  primary.viewNo,
                                                  primary.lastPrePrepareSeqNo)
                and param['sender'] != primary.name
            ])

            numOfMsgsWithZFN = nodeCount - 1
            numOfMsgsWithFaults = 2 * f - 1

            assert msgCountOK(nodeCount, faultyNodes, actualMsgs,
                              numOfMsgsWithZFN, numOfMsgsWithFaults)
            # TODO what if the primary is faulty?

        def nonPrimaryReplicasReceiveCorrectNumberOfPREPAREs():
            """
            num of PREPARE seen by Non primary replica is n - 2 without
            faults and 2f - 1 with faults.
            """
            passes = 0
            numOfMsgsWithZFN = nodeCount - 2
            numOfMsgsWithFaults = (2 * f) - 1

            for npr in nonPrimaryReplicas:
                actualMsgs = len([
                    param for param in getAllArgs(npr, npr.processPrepare)
                    if (param['prepare'].instId, param['prepare'].viewNo,
                        param['prepare'].ppSeqNo) == (
                            primary.instId, primary.viewNo,
                            primary.lastPrePrepareSeqNo)
                ])

                passes += int(
                    msgCountOK(nodeCount, faultyNodes, actualMsgs,
                               numOfMsgsWithZFN, numOfMsgsWithFaults))

            assert passes >= len(nonPrimaryReplicas) - faultyNodes
            # TODO how do we know if one of the faulty nodes is a primary or
            # not?

        primaryDontSendAnyPREPAREs()
        allReplicasSeeCorrectNumberOfPREPAREs()
        primaryReceivesCorrectNumberOfPREPAREs()
        nonPrimaryReplicasReceiveCorrectNumberOfPREPAREs()
def testOrderingCase1(looper, nodeSet, up, client1, wallet1):
    """
    Scenario -> A client sends requests, some nodes delay COMMITs to few
    specific nodes such some nodes achieve commit quorum later for those
    requests compared to other nodes. But all nodes `ORDER` request in the same
    order of ppSeqNos
    https://www.pivotaltracker.com/n/projects/1889887/stories/133655009
    """
    pr, replicas = getPrimaryReplica(nodeSet, instId=0), \
                   getNonPrimaryReplicas(nodeSet, instId=0)
    assert len(replicas) == 6

    rep0 = pr
    rep1 = replicas[0]
    rep2 = replicas[1]
    rep3 = replicas[2]
    rep4 = replicas[3]
    rep5 = replicas[4]
    rep6 = replicas[5]

    node0 = rep0.node
    node1 = rep1.node
    node2 = rep2.node
    node3 = rep3.node
    node4 = rep4.node
    node5 = rep5.node
    node6 = rep6.node

    requests = sendRandomRequests(wallet1, client1, 15)

    ppSeqsToDelay = 5

    delayedPpSeqNos = set()

    def specificCommits(wrappedMsg):
        nonlocal node3, node4, node5
        msg, sender = wrappedMsg
        if isinstance(msg, PrePrepare):
            if len(delayedPpSeqNos) < ppSeqsToDelay:
                delayedPpSeqNos.add(msg.ppSeqNo)
                logger.debug('ppSeqNo {} corresponding to request id {} would '
                             'be delayed'.format(msg.ppSeqNo, msg.reqId))
        if isinstance(msg, Commit) and msg.instId == 0 and \
            sender in (n.name for n in (node3, node4, node5)) and \
                msg.ppSeqNo in delayedPpSeqNos:
            return 3

    for node in (node1, node2):
        logger.debug('{} would be delaying commits'.format(node))
        node.nodeIbStasher.delay(specificCommits)

    checkSufficientRepliesForRequests(looper, client1, requests)

    def ensureSlowNodesHaveAllTxns():
        nonlocal node1, node2
        for node in node1, node2:
            assert len(node.domainLedger) == 15

    looper.run(eventually(ensureSlowNodesHaveAllTxns, retryWait=1, timeout=15))

    checkAllLedgersEqual((n.domainLedger for n in (node0, node3, node4,
                                                   node5, node6)))

    for node in (node1, node2):
        for n in nodeSet:
            if n != node:
                checkLedgerEquality(node.domainLedger, n.domainLedger)

    checkAllLedgersEqual((n.domainLedger for n in nodeSet))
    def g(instId):
        primary = getPrimaryReplica(nodeSet, instId)
        nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet, instId)

        def primarySeesCorrectNumberOfPREPREPAREs():
            """
            no of PRE-PREPARE as seen by processPrePrepare
            method for primary must be 0 with or without faults in system
            """
            l1 = len([
                param
                for param in getAllArgs(primary, primary.processPrePrepare)
            ])
            assert l1 == 0

        def nonPrimarySeesCorrectNumberOfPREPREPAREs():
            """
            1. no of PRE-PREPARE as seen by processPrePrepare method for
            non-primaries must be 1; whn zero faulty nodes in system.

            2. no of PRE-PREPARE as seen by processPrePrepare method for
            non-primaries must be greater than or equal to 0;
            with faults in system.
            """
            expectedPrePrepareRequest = PrePrepare(instId, primary.viewNo,
                                                   primary.lastPrePrepareSeqNo,
                                                   propagated1.identifier,
                                                   propagated1.reqId,
                                                   propagated1.digest,
                                                   time.time())

            passes = 0
            for npr in nonPrimaryReplicas:
                actualMsgs = len([
                    param for param in getAllArgs(npr, npr.processPrePrepare)
                    if (param['pp'][:-1],
                        param['sender']) == (expectedPrePrepareRequest[:-1],
                                             primary.name)
                ])

                numOfMsgsWithZFN = 1
                numOfMsgsWithFaults = 0

                passes += int(
                    msgCountOK(nodesSize, faultyNodes, actualMsgs,
                               numOfMsgsWithZFN, numOfMsgsWithFaults))
            assert passes >= len(nonPrimaryReplicas) - faultyNodes

        def primarySentsCorrectNumberOfPREPREPAREs():
            """
            1. no of PRE-PREPARE sent by primary is 1 with or without
            fault in system but, when primary is faulty no of sent PRE_PREPARE
             will be zero and primary must be marked as malicious.
            """
            actualMsgs = len([
                param for param in getAllArgs(primary, primary.doPrePrepare)
                if (param['reqDigest'].identifier, param['reqDigest'].reqId,
                    param['reqDigest'].digest) == (propagated1.identifier,
                                                   propagated1.reqId,
                                                   propagated1.digest)
            ])

            numOfMsgsWithZFN = 1

            # TODO: Considering, Primary is not faulty and will always send
            # PRE-PREPARE. Write separate test for testing when Primary
            # is faulty
            assert msgCountOK(nodesSize, faultyNodes, actualMsgs,
                              numOfMsgsWithZFN, numOfMsgsWithZFN)

        def nonPrimaryReceivesCorrectNumberOfPREPREPAREs():
            """
            1. no of PRE-PREPARE received by non-primaries must be 1
            with zero faults in system, and 0 faults in system.
            """
            passes = 0
            for npr in nonPrimaryReplicas:
                l4 = len([
                    param for param in getAllArgs(npr, npr.addToPrePrepares)
                    if (param['pp'].identifier, param['pp'].reqId,
                        param['pp'].digest) == (propagated1.identifier,
                                                propagated1.reqId,
                                                propagated1.digest)
                ])

                numOfMsgsWithZFN = 1
                numOfMsgsWithFaults = 0

                passes += msgCountOK(nodesSize, faultyNodes, l4,
                                     numOfMsgsWithZFN, numOfMsgsWithFaults)

            assert passes >= len(nonPrimaryReplicas) - faultyNodes

        primarySeesCorrectNumberOfPREPREPAREs()
        nonPrimarySeesCorrectNumberOfPREPREPAREs()
        primarySentsCorrectNumberOfPREPREPAREs()
        nonPrimaryReceivesCorrectNumberOfPREPREPAREs()
Beispiel #41
0
    def g(instId):
        allReplicas = getAllReplicas(nodeSet, instId)
        primary = getPrimaryReplica(nodeSet, instId)
        nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet, instId)

        def primaryDontSendAnyPREPAREs():
            """
            1. no of PREPARE sent by primary should be 0
            """
            for r in allReplicas:
                for param in getAllArgs(r, Replica.processPrepare):
                    sender = param['sender']
                    assert sender != primary.name

        def allReplicasSeeCorrectNumberOfPREPAREs():
            """
            1. no of PREPARE received by replicas must be n - 1;
            n = num of nodes without fault, and greater than or equal to
             2f with faults.
            """
            passes = 0
            numOfMsgsWithZFN = nodeCount - 1
            numOfMsgsWithFaults = 2 * f

            for replica in allReplicas:
                key = primary.viewNo, primary.prePrepareSeqNo
                if key in replica.prepares:
                    actualMsgs = len(replica.prepares[key].voters)

                    passes += int(msgCountOK(nodeCount,
                                             faultyNodes,
                                             actualMsgs,
                                             numOfMsgsWithZFN,
                                             numOfMsgsWithFaults))
            assert passes >= len(allReplicas) - faultyNodes

        def primaryReceivesCorrectNumberOfPREPAREs():
            """
            num of PREPARE seen by primary replica is n - 1;
                n = num of nodes without fault, and greater than or equal to
             2f with faults.
            """
            actualMsgs = len([param for param in
                              getAllArgs(primary,
                                         primary.processPrepare)
                              if (param['prepare'].instId,
                                  param['prepare'].viewNo,
                                  param['prepare'].ppSeqNo) == (primary.instId,
                                                        primary.viewNo,
                                                        primary.prePrepareSeqNo) and
                              param['sender'] != primary.name])

            numOfMsgsWithZFN = nodeCount - 1
            numOfMsgsWithFaults = 2 * f - 1

            assert msgCountOK(nodeCount,
                              faultyNodes,
                              actualMsgs,
                              numOfMsgsWithZFN,
                              numOfMsgsWithFaults)
            # TODO what if the primary is faulty?

        def nonPrimaryReplicasReceiveCorrectNumberOfPREPAREs():
            """
            num of PREPARE seen by Non primary replica is n - 2 without
            faults and 2f - 1 with faults.
            """
            passes = 0
            numOfMsgsWithZFN = nodeCount - 2
            numOfMsgsWithFaults = (2 * f) - 1

            for npr in nonPrimaryReplicas:
                actualMsgs = len([param for param in
                                  getAllArgs(
                                          npr,
                                          npr.processPrepare)
                                  if (param['prepare'].instId,
                                      param['prepare'].viewNo,
                                      param['prepare'].ppSeqNo) == (primary.instId,
                                                            primary.viewNo,
                                                            primary.prePrepareSeqNo)
                                  ])

                passes += int(msgCountOK(nodeCount,
                                         faultyNodes,
                                         actualMsgs,
                                         numOfMsgsWithZFN,
                                         numOfMsgsWithFaults))

            assert passes >= len(nonPrimaryReplicas) - faultyNodes
            # TODO how do we know if one of the faulty nodes is a primary or
            # not?

        primaryDontSendAnyPREPAREs()
        allReplicasSeeCorrectNumberOfPREPAREs()
        primaryReceivesCorrectNumberOfPREPAREs()
        nonPrimaryReplicasReceiveCorrectNumberOfPREPAREs()
def primaryReplicas(nodeSet, up):
    instanceCount = getNoInstances(nodeCount)
    return [getPrimaryReplica(nodeSet, i) for i in range(instanceCount)]
def primaryReplicas(nodeSet, up):
    instanceCount = getNoInstances(nodeCount)
    return [getPrimaryReplica(nodeSet, i) for i in range(instanceCount)]
Beispiel #44
0
    def g(instId):
        primary = getPrimaryReplica(nodeSet, instId)
        nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet, instId)

        def primarySeesCorrectNumberOfPREPREPAREs():
            """
            no of PRE-PREPARE as seen by processPrePrepare
            method for primary must be 0 with or without faults in system
            """
            l1 = len([param for param in
                      getAllArgs(primary, primary.processPrePrepare)])
            assert l1 == 0

        def nonPrimarySeesCorrectNumberOfPREPREPAREs():
            """
            1. no of PRE-PREPARE as seen by processPrePrepare method for
            non-primaries must be 1; whn zero faulty nodes in system.

            2. no of PRE-PREPARE as seen by processPrePrepare method for
            non-primaries must be greater than or equal to 0;
            with faults in system.
            """
            expectedPrePrepareRequest = PrePrepare(
                    instId,
                    primary.viewNo,
                    primary.prePrepareSeqNo,
                    propagated1.clientId,
                    propagated1.reqId,
                    propagated1.digest)

            passes = 0
            for npr in nonPrimaryReplicas:
                actualMsgs = len([param for param in
                                  getAllArgs(npr, npr.processPrePrepare)
                                  if (param['pp'],
                                      param['sender']) == (
                                      expectedPrePrepareRequest,
                                      primary.name)])

                numOfMsgsWithZFN = 1
                numOfMsgsWithFaults = 0

                passes += int(msgCountOK(nodesSize,
                                         faultyNodes,
                                         actualMsgs,
                                         numOfMsgsWithZFN,
                                         numOfMsgsWithFaults))
            assert passes >= len(nonPrimaryReplicas) - faultyNodes

        def primarySentsCorrectNumberOfPREPREPAREs():
            """
            1. no of PRE-PREPARE sent by primary is 1 with or without
            fault in system but, when primary is faulty no of sent PRE_PREPARE
             will be zero and primary must be marked as malicious.
            """
            actualMsgs = len([param for param in
                              getAllArgs(primary, primary.doPrePrepare)
                              if (param['reqDigest'].clientId,
                                  param['reqDigest'].reqId,
                                  param['reqDigest'].digest) ==
                              (propagated1.clientId,
                               propagated1.reqId,
                               propagated1.digest)
                              ])

            numOfMsgsWithZFN = 1

            # TODO: Considering, Primary is not faulty and will always send
            # PRE-PREPARE. Write separate test for testing when Primary
            # is faulty
            assert msgCountOK(nodesSize,
                              faultyNodes,
                              actualMsgs,
                              numOfMsgsWithZFN,
                              numOfMsgsWithZFN)

        def nonPrimaryReceivesCorrectNumberOfPREPREPAREs():
            """
            1. no of PRE-PREPARE received by non-primaries must be 1
            with zero faults in system, and 0 faults in system.
            """
            passes = 0
            for npr in nonPrimaryReplicas:
                l4 = len([param for param in
                          getAllArgs(npr, npr.addToPrePrepares)
                          if (param['pp'].clientId,
                              param['pp'].reqId,
                              param['pp'].digest) == (
                              propagated1.clientId,
                              propagated1.reqId,
                              propagated1.digest)])

                numOfMsgsWithZFN = 1
                numOfMsgsWithFaults = 0

                passes += msgCountOK(nodesSize,
                                     faultyNodes,
                                     l4,
                                     numOfMsgsWithZFN,
                                     numOfMsgsWithFaults)

            assert passes >= len(nonPrimaryReplicas) - faultyNodes

        primarySeesCorrectNumberOfPREPREPAREs()
        nonPrimarySeesCorrectNumberOfPREPREPAREs()
        primarySentsCorrectNumberOfPREPREPAREs()
        nonPrimaryReceivesCorrectNumberOfPREPREPAREs()