def electContFixture(startedNodes: TestNodeSet):
    A, B, C, D = startedNodes.nodes.values()

    # Delaying nodeB' self nomination so nodeA's nomination can reach NodeC and NodeD
    B.delaySelfNomination(2)

    # For B delay nominate messages from A and C
    for node in [A, C]:
        B.nodeIbStasher.delay(delayerMsgTuple(10, Nomination, node.name))

    for node in [C, D]:
        B.nodeIbStasher.delay(delayerMsgTuple(5, Nomination, node.name))
        node.delaySelfNomination(10)
示例#2
0
def testTestNodeDelay(tdir_for_func):
    nodeNames = {"testA", "testB"}
    with TestNodeSet(names=nodeNames, tmpdir=tdir_for_func) as nodes:
        nodeA = nodes.getNode("testA")
        nodeB = nodes.getNode("testB")

        with Looper(nodes) as looper:
            for n in nodes:
                n.startKeySharing()

            logging.debug("connect")
            looper.run(checkNodesConnected(nodes))
            logging.debug("send one message, without delay")
            msg = randomMsg()
            looper.run(sendMsgAndCheck(nodes, nodeA, nodeB, msg, 1))
            logging.debug(
                    "set delay, then send another message and find that it doesn't arrive")
            msg = randomMsg()

            nodeB.nodeIbStasher.delay(delayerMsgTuple(4, type(msg), nodeA.name))

            with pytest.raises(AssertionError):
                looper.run(sendMsgAndCheck(nodes, nodeA, nodeB, msg, 3))
            logging.debug(
                    "but then find that it arrives after the delay duration has passed")
            looper.run(sendMsgAndCheck(nodes, nodeA, nodeB, msg, 2))
            logging.debug(
                    "reset the delay, and find another message comes quickly")
            nodeB.nodeIbStasher.resetDelays()
            msg = randomMsg()
            looper.run(sendMsgAndCheck(nodes, nodeA, nodeB, msg, 1))
示例#3
0
def testPrePrepareWhenPrimaryStatusIsUnknown(tdir_for_func):
    nodeNames = genNodeNames(4)
    nodeReg = genNodeReg(names=nodeNames)
    with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet:
        with Looper(nodeSet) as looper:
            prepareNodeSet(looper, nodeSet)

            nodeA, nodeB, nodeC, nodeD = tuple(
                addNodeBack(nodeSet, looper, nodeNames[i]) for i in range(0, 4))

            # Nodes C and D delays self nomination so A and B can become primaries
            nodeC.delaySelfNomination(30)
            nodeD.delaySelfNomination(30)

            # Node D delays receiving PRIMARY messages from all nodes so it will not know
            # whether it is primary or not

            # nodeD.nodestack.delay(delayer(20, PRIMARY))

            nodeD.nodeIbStasher.delay(delayerMsgTuple(20, Primary))

            checkPoolReady(looper=looper, nodes=nodeSet)

            client1 = setupClient(looper, nodeSet, tmpdir=tdir_for_func)
            request = sendRandomRequest(client1)

            # TODO Rethink this
            instNo = 0

            for i in range(3):
                node = nodeSet.getNode(nodeNames[i])
                # Nodes A, B and C should have received PROPAGATE request from Node D
                looper.run(
                    eventually(checkIfPropagateRecvdFromNode, node, nodeD,
                               request.clientId,
                               request.reqId, retryWait=1, timeout=10))

            # Node D should have 1 pending PRE-PREPARE request
            def assertOnePrePrepare():
                assert len(getPendingRequestsForReplica(nodeD.replicas[instNo],
                                                        PrePrepare)) == 1

            looper.run(eventually(assertOnePrePrepare, retryWait=1, timeout=10))

            # Node D should have 2 pending PREPARE requests(from node B and C)

            def assertTwoPrepare():
                assert len(getPendingRequestsForReplica(nodeD.replicas[instNo],
                                                        Prepare)) == 2

            looper.run(eventually(assertTwoPrepare, retryWait=1, timeout=10))

            # Node D should have no pending PRE-PREPARE, PREPARE or COMMIT requests
            for reqType in [PrePrepare, Prepare, Commit]:
                looper.run(eventually(lambda: assertLength(
                    getPendingRequestsForReplica(nodeD.replicas[instNo],
                                                 reqType),
                    0), retryWait=1, timeout=20))
def configNodeSet(nodeSet):
    A, B, C, D = nodeSet.nodes.values()
    # Nodes C and D delay Commit request from node A for protocol instance 0
    for n in [C, D]:
        n.nodeIbStasher.delay(delayerMsgTuple(30,
                                              Commit,
                                              senderFilter=A.name,
                                              instFilter=0))
    return nodeSet
def case2Setup(startedNodes: TestNodeSet):
    A, B, C, D = startedNodes.nodes.values()

    # Node B delays self nomination so A's nomination reaches everyone
    B.delaySelfNomination(5)
    # Node B delays nomination from all nodes
    B.nodeIbStasher.delay(delayerMsgTuple(5, Nomination))

    for node in [C, D]:
        # Also Nodes C and D are slow so they will not nominate for themselves
        # for long
        node.delaySelfNomination(25)

    # A, C and D should not blacklist B since we are trying to check if
    # multiple nominations from the same node have any impact on
    # the election
    for node in A, C, D:
        node.whitelistNode(B.name, Suspicions.DUPLICATE_NOM_SENT.code)
示例#6
0
def testRequestReturnToNodeWhenPrePrepareNotReceivedByOneNode(tdir_for_func):
    """Test no T-3"""
    nodeNames = genNodeNames(7)
    nodeReg = genNodeReg(names=nodeNames)
    with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet:
        with Looper(nodeSet) as looper:
            prepareNodeSet(looper, nodeSet)
            logging.debug("Add the seven nodes back in")
            # Every node except A delays self nomination so A can become  primary
            nodeA = addNodeBack(nodeSet, looper, nodeNames[0])
            for i in range(1, 7):
                node = addNodeBack(nodeSet, looper, nodeNames[i])
                node.delaySelfNomination(15)

            nodeB = nodeSet.getNode(nodeNames[1])
            # Node B delays PREPREPARE from node A(which would be the primary) for a long time.
            # TODO: Have a method to ignore a particular message from a node
            nodeB.nodeIbStasher.delay(
                delayerMsgTuple(120, PrePrepare, nodeA.name))

            # Ensure elections are done
            ensureElectionsDone(looper=looper, nodes=nodeSet, retryWait=1,
                                timeout=30)
            assert nodeA.hasPrimary

            instNo = nodeA.primaryReplicaNo
            client1 = setupClient(looper, nodeSet, tmpdir=tdir_for_func)
            req = sendRandomRequest(client1)

            # All nodes including B should return their ordered requests
            for node in nodeSet:
                looper.run(eventually(checkRequestReturnedToNode, node,
                                      client1.clientId, req.reqId,
                                      req.digest,
                                      instNo, retryWait=1, timeout=30))

            # Node B should not have received the PRE-PREPARE request yet
            replica = nodeB.replicas[instNo]  # type: Replica
            assert len(replica.prePrepares) == 0