Exemplo n.º 1
0
def testPrePrepareWhenPrimaryStatusIsUnknown(tdir_for_func):
    nodeNames = genNodeNames(4)
    nodeReg = genNodeReg(names=nodeNames)
    with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet:
        with Looper(nodeSet) as looper:
            prepareNodeSet(looper, nodeSet)

            nodeA, nodeB, nodeC, nodeD = tuple(
                addNodeBack(nodeSet, looper, nodeNames[i]) for i in range(0, 4))

            # Nodes C and D delays self nomination so A and B can become primaries
            nodeC.delaySelfNomination(30)
            nodeD.delaySelfNomination(30)

            # Node D delays receiving PRIMARY messages from all nodes so it will not know
            # whether it is primary or not

            # nodeD.nodestack.delay(delayer(20, PRIMARY))

            nodeD.nodeIbStasher.delay(delayerMsgTuple(20, Primary))

            checkPoolReady(looper=looper, nodes=nodeSet)

            client1 = setupClient(looper, nodeSet, tmpdir=tdir_for_func)
            request = sendRandomRequest(client1)

            # TODO Rethink this
            instNo = 0

            for i in range(3):
                node = nodeSet.getNode(nodeNames[i])
                # Nodes A, B and C should have received PROPAGATE request from Node D
                looper.run(
                    eventually(checkIfPropagateRecvdFromNode, node, nodeD,
                               request.clientId,
                               request.reqId, retryWait=1, timeout=10))

            # Node D should have 1 pending PRE-PREPARE request
            def assertOnePrePrepare():
                assert len(getPendingRequestsForReplica(nodeD.replicas[instNo],
                                                        PrePrepare)) == 1

            looper.run(eventually(assertOnePrePrepare, retryWait=1, timeout=10))

            # Node D should have 2 pending PREPARE requests(from node B and C)

            def assertTwoPrepare():
                assert len(getPendingRequestsForReplica(nodeD.replicas[instNo],
                                                        Prepare)) == 2

            looper.run(eventually(assertTwoPrepare, retryWait=1, timeout=10))

            # Node D should have no pending PRE-PREPARE, PREPARE or COMMIT requests
            for reqType in [PrePrepare, Prepare, Commit]:
                looper.run(eventually(lambda: assertLength(
                    getPendingRequestsForReplica(nodeD.replicas[instNo],
                                                 reqType),
                    0), retryWait=1, timeout=20))
def testPrimaryElectionWithAClearWinner(electContFixture, looper, keySharedNodes):
    """
    Primary selection (Sunny Day)
    A, B, C, D, E
    A, B, C, D startup. E is lagging.
    A sees the minimum number of nodes first, and then sends out a NOMINATE(A) message
    B, C, D all see the NOMINATE(A) message from A, and respond with NOMINATE(A) message to all other nodes

    A sees three other NOMINATE(A) votes (from B, C, D)
    A sees that A is the clear winner (2f+1 total), and sends PRIMARY(A) to all nodes

    B sees two more NOMINATE(A) votes (from C and D)
    B sees that A is the clear winner (2f+1 total), and sends PRIMARY(A) to all nodes

    C sees two more NOMINATE(A) votes (from B and D)
    C sees that A is the clear winner (2f+1 total), and sends PRIMARY(A) to all nodes

    D sees two more NOMINATE(A) votes (from B and C)
    D sees that A is the clear winner (2f+1 total), and sends PRIMARY(A) to all nodes

    A sees at least two other PRIMARY(A) votes (3 including it's own)
    selects A as primary

    B sees at least two other PRIMARY(A) votes (3 including it's own)
    selects A as primary

    C sees at least two other PRIMARY(A) votes (3 including it's own)
    selects A as primary

    D sees at least two other PRIMARY(A) votes (3 including it's own)
    selects A as primary
    """

    nodeSet = keySharedNodes
    A, B, C, D = nodeSet.nodes.values()
    nodesBCD = [B, C, D]

    # attempting to use a raet stack delay... not successful
    # nodeBPort = nodesBCD[0].stack.ha[1]
    # delayRef = RaetDelay(TrnsKind.alive, PcktKind.ack, nodeBPort)
    # nodeA.stack.delay(4, delayRef)

    checkPoolReady(looper, nodeSet)

    # Checking whether one of the replicas of Node A nominated itself
    looper.run(eventually(checkNomination, A, A.name, retryWait=1, timeout=10))

    for n in nodesBCD:
        # Checking whether Node B, C and D nominated Node A
        looper.run(eventually(checkNomination, n, A.name, retryWait=1,
                              timeout=10))

    checkProtocolInstanceSetup(looper=looper, nodes=nodeSet, retryWait=1,
                               timeout=10)
    assert A.hasPrimary
def testPrimaryElectionContested(electContFixture, looper, keySharedNodes):
    """
    Primary selection (Rainy Day)
    A, B, C, D, E
    A, B, C, D startup. E is lagging.
    A sees the minimum number of nodes, and then sends Nominate(A)
    At the same exact time, B sees the minimum number of nodes, and then sends out Nominate(B)
    A sees B sending Nominate(B), but it has already nominated itself, so it does nothing
    B sees A sending Nominate(A), but it has already nominated itself, so it does nothing
    C sees A sending Nominate(A), and sends Nominate(A)
    D sees A sending Nominate(A), and sends Nominate(A)
    All nodes see that B nominated B and A, C, and D all nominated A
    Because the votes for A exceeds the votes for B, all send out Primary(A)
    TODO's (see below)
    All see the others have sent Primary A, and then the nodes record who is the Primary.
    """

    # TODO what if not all send out Primary(A)?
    # TODO what if there are big delays in messages getting delivered?

    nodeSet = keySharedNodes
    A, B, C, D = nodeSet.nodes.values()

    checkPoolReady(looper, nodeSet)

    logging.debug("Check nomination")
    # Checking whether Node A nominated itself
    looper.run(eventually(checkNomination, A, A.name, retryWait=1, timeout=10))

    # Checking whether Node B nominated itself
    looper.run(eventually(checkNomination, B, B.name, retryWait=1, timeout=10))

    for n in [C, D]:
        # Checking whether Node C and Node D nominated Node A
        looper.run(eventually(checkNomination, n, A.name, retryWait=1, timeout=10))

    checkProtocolInstanceSetup(looper=looper, nodes=nodeSet,
                                                 retryWait=1, timeout=45)

    # Node D should not be primary
    assert not D.hasPrimary
    # A should have at least one primary
    assert A.hasPrimary
def testPrimaryElectionWithTie(electTieFixture, looper, keySharedNodes):
    """
    Primary selection (Rainy Day)
    A, B, C, D, E
    A, B, C, D startup. E is lagging.
    A sees the minimum number of nodes, and then sends Nominate(A)
    At the same exact time, B sees the minimum number of nodes, and then sends out Nominate(B)
    A sees B sending Nominate(B), but it has already nominated itself, so it does nothing
    B sees A sending Nominate(A), but it has already nominated itself, so it does nothing
    C sees A sending Nominate(A), and sends Nominate(A)
    D sees B sending Nominate(B), and sends Nominate(B)
    There's a split. C and A think A is the primary, B and D think B is the primary
    All nodes can see that there is a split. Each sends out Reelection([A,B])

    A and B both see Reelection([A,B]) from themselves as well as the other 3 (the number from others should be at least f+1),

    1. they wait a random amount of time (between 0 and 2 seconds),
    2. they each send out a Nominate(self)

    Voting is repeated until we have a good election.
    """

    # TODO optimize the sending messages in batches, for example, we don't
    #     send messages more often than 400 milliseconds. Once those 400
    #     millis have passed, we send the several queued messages in one
    #     batch.

    nodeSet = keySharedNodes
    A, B, C, D = nodeSet.nodes.values()

    checkPoolReady(looper, nodeSet.nodes.values())

    for node in nodeSet.nodes.values():
        for instId, replica in enumerate(node.elector.replicas):
            logging.debug("replica {} {} with votes {}".
                          format(replica.name, replica.instId,
                                 node.elector.nominations.get(instId, {})))

    logging.debug("Check nomination")
    # Checking whether Node A nominated itself
    looper.run(eventually(checkNomination, A, A.name, retryWait=1, timeout=10))

    # Checking whether Node B nominated itself
    looper.run(eventually(checkNomination, B, B.name, retryWait=1, timeout=10))

    # Checking whether Node C nominated Node A
    looper.run(eventually(checkNomination, C, A.name, retryWait=1, timeout=10))

    # Checking whether Node D nominated Node D
    looper.run(eventually(checkNomination, D, B.name, retryWait=1, timeout=10))

    # No node should be primary
    for node in nodeSet.nodes.values():
        assert node.hasPrimary is False

    for node in nodeSet.nodes.values():
        node.resetDelays()

    # TODO Check for spylog of `eatReelection` after putting spy on eaters too

    checkProtocolInstanceSetup(looper=looper, nodes=nodeSet, retryWait=1,
                               timeout=60)

    # TODO testPrimaryElectionWithTieButOneNodeDoesntKnowIt
    # TODO add E back in after election is complete, or even before it's
    # complete (E jumps in right in the middle of an election)
    #       When E joins, it needs to be able to ping the others for the
    #       current state (A could send to E the votes of the other nodes,
    #       so E would know if B is being maliciious), and the others respond,
    #       and E can make a determination based on those responses, even if
    #       one is malicious.

    # TODO We need to trap for the case when a node is being inconsistent. For example:
    """