Exemple #1
0
 def assertTwoPrepare():
     assert len(
         getPendingRequestsForReplica(nodeD.replicas[instNo],
                                      Prepare)) == 2
 def assert_msg_count(typ, count):
     assert len(
         getPendingRequestsForReplica(nodeD.replicas[instNo],
                                      typ)) == count
Exemple #3
0
def testPrePrepareWhenPrimaryStatusIsUnknown(tdir_for_func):
    nodeNames = genNodeNames(4)
    nodeReg = genNodeReg(names=nodeNames)
    with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet:
        with Looper(nodeSet) as looper:
            prepareNodeSet(looper, nodeSet)

            nodeA, nodeB, nodeC, nodeD = tuple(
                addNodeBack(nodeSet, looper, nodeNames[i])
                for i in range(0, 4))

            # Nodes C and D delays self nomination so A and B can become
            # primaries
            nodeC.delaySelfNomination(30)
            nodeD.delaySelfNomination(30)

            # Node D delays receiving PRIMARY messages from all nodes so it
            # will not know whether it is primary or not

            # nodeD.nodestack.delay(delayer(20, PRIMARY))

            nodeD.nodeIbStasher.delay(delayerMsgTuple(20, Primary))

            checkPoolReady(looper=looper, nodes=nodeSet)

            client1, wal = setupClient(looper, nodeSet, tmpdir=tdir_for_func)
            request = sendRandomRequest(wal, client1)

            # TODO Rethink this
            instNo = 0

            for i in range(3):
                node = nodeSet.getNode(nodeNames[i])
                # Nodes A, B and C should have received PROPAGATE request
                # from Node D
                looper.run(
                    eventually(checkIfPropagateRecvdFromNode,
                               node,
                               nodeD,
                               request.identifier,
                               request.reqId,
                               retryWait=1,
                               timeout=10))

            # Node D should have 1 pending PRE-PREPARE request
            def assertOnePrePrepare():
                assert len(
                    getPendingRequestsForReplica(nodeD.replicas[instNo],
                                                 PrePrepare)) == 1

            looper.run(eventually(assertOnePrePrepare, retryWait=1,
                                  timeout=10))

            # Node D should have 2 pending PREPARE requests(from node B and C)

            def assertTwoPrepare():
                assert len(
                    getPendingRequestsForReplica(nodeD.replicas[instNo],
                                                 Prepare)) == 2

            looper.run(eventually(assertTwoPrepare, retryWait=1, timeout=10))

            # Node D should have no pending PRE-PREPARE, PREPARE or COMMIT
            # requests
            for reqType in [PrePrepare, Prepare, Commit]:
                looper.run(
                    eventually(lambda: assertLength(
                        getPendingRequestsForReplica(nodeD.replicas[instNo],
                                                     reqType), 0),
                               retryWait=1,
                               timeout=20))
def testPrePrepareWhenPrimaryStatusIsUnknown(tdir_for_func):
    nodeNames = genNodeNames(4)
    nodeReg = genNodeReg(names=nodeNames)
    with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet:
        with Looper(nodeSet) as looper:
            prepareNodeSet(looper, nodeSet)

            nodeA, nodeB, nodeC, nodeD = tuple(
                addNodeBack(nodeSet, looper, nodeNames[i])
                for i in range(0, 4))

            # Since primary selection is round robin, A and B will be primaries

            # Nodes C and D delays self nomination so A and B can become
            # primaries
            # nodeC.delaySelfNomination(10)
            # nodeD.delaySelfNomination(10)

            # Node D delays receiving PRIMARY messages from all nodes so it
            # will not know whether it is primary or not

            # delayD = 5
            # nodeD.nodeIbStasher.delay(delayerMsgTuple(delayD, Primary))

            checkPoolReady(looper=looper, nodes=nodeSet)

            # client1, wal = setupClient(looper, nodeSet, tmpdir=tdir_for_func)
            # request = sendRandomRequest(wal, client1)

            # TODO Rethink this
            instNo = 0

            timeout = waits.expectedClientRequestPropagationTime(len(nodeSet))
            for i in range(3):
                node = nodeSet.getNode(nodeNames[i])
                # Nodes A, B and C should have received PROPAGATE request
                # from Node D
                looper.run(
                    eventually(checkIfPropagateRecvdFromNode,
                               node,
                               nodeD,
                               request.identifier,
                               request.reqId,
                               retryWait=1,
                               timeout=timeout))

            def assert_msg_count(typ, count):
                assert len(
                    getPendingRequestsForReplica(nodeD.replicas[instNo],
                                                 typ)) == count

            # Node D should have 1 pending PRE-PREPARE request
            timeout = waits.expectedPrePrepareTime(len(nodeSet))
            looper.run(
                eventually(assert_msg_count,
                           PrePrepare,
                           1,
                           retryWait=1,
                           timeout=timeout))

            # Node D should have 2 pending PREPARE requests(from node B and C)
            timeout = waits.expectedPrepareTime(len(nodeSet))
            looper.run(
                eventually(assert_msg_count,
                           Prepare,
                           2,
                           retryWait=1,
                           timeout=timeout))

            # Its been checked above that replica stashes 3 phase messages in
            # lack of primary, now avoid delay (fix the network)
            nodeD.nodeIbStasher.reset_delays_and_process_delayeds()

            # Node D should have no pending PRE-PREPARE, PREPARE or COMMIT
            # requests
            for reqType in [PrePrepare, Prepare, Commit]:
                looper.run(
                    eventually(lambda: assertLength(
                        getPendingRequestsForReplica(nodeD.replicas[instNo],
                                                     reqType), 0),
                               retryWait=1,
                               timeout=delayD))  # wait little more than delay
def testPrePrepareWhenPrimaryStatusIsUnknown(tdir_for_func):
    nodeNames = genNodeNames(4)
    nodeReg = genNodeReg(names=nodeNames)
    with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet:
        with Looper(nodeSet) as looper:
            prepareNodeSet(looper, nodeSet)

            nodeA, nodeB, nodeC, nodeD = tuple(
                addNodeBack(nodeSet, looper, nodeNames[i]) for i in range(0, 4))

            # Nodes C and D delays self nomination so A and B can become
            # primaries
            nodeC.delaySelfNomination(30)
            nodeD.delaySelfNomination(30)

            # Node D delays receiving PRIMARY messages from all nodes so it
            # will not know whether it is primary or not

            # nodeD.nodestack.delay(delayer(20, PRIMARY))

            nodeD.nodeIbStasher.delay(delayerMsgTuple(20, Primary))

            checkPoolReady(looper=looper, nodes=nodeSet)

            client1 = setupClient(looper, nodeSet, tmpdir=tdir_for_func)
            request = sendRandomRequest(client1)

            # TODO Rethink this
            instNo = 0

            for i in range(3):
                node = nodeSet.getNode(nodeNames[i])
                # Nodes A, B and C should have received PROPAGATE request
                # from Node D
                looper.run(
                    eventually(checkIfPropagateRecvdFromNode, node, nodeD,
                               request.identifier,
                               request.reqId, retryWait=1, timeout=10))

            # Node D should have 1 pending PRE-PREPARE request
            def assertOnePrePrepare():
                assert len(getPendingRequestsForReplica(nodeD.replicas[instNo],
                                                        PrePrepare)) == 1

            looper.run(eventually(assertOnePrePrepare, retryWait=1, timeout=10))

            # Node D should have 2 pending PREPARE requests(from node B and C)

            def assertTwoPrepare():
                assert len(getPendingRequestsForReplica(nodeD.replicas[instNo],
                                                        Prepare)) == 2

            looper.run(eventually(assertTwoPrepare, retryWait=1, timeout=10))

            # Node D should have no pending PRE-PREPARE, PREPARE or COMMIT
            # requests
            for reqType in [PrePrepare, Prepare, Commit]:
                looper.run(eventually(lambda: assertLength(
                    getPendingRequestsForReplica(nodeD.replicas[instNo],
                                                 reqType),
                    0), retryWait=1, timeout=20))
 def assertTwoPrepare():
     assert len(getPendingRequestsForReplica(nodeD.replicas[instNo],
                                             Prepare)) == 2
 def assert_msg_count(typ, count):
     assert len(getPendingRequestsForReplica(nodeD.replicas[instNo],
                                             typ)) == count
def testPrePrepareWhenPrimaryStatusIsUnknown(tdir_for_func):
    nodeNames = genNodeNames(4)
    nodeReg = genNodeReg(names=nodeNames)
    with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet:
        with Looper(nodeSet) as looper:
            prepareNodeSet(looper, nodeSet)

            nodeA, nodeB, nodeC, nodeD = tuple(
                addNodeBack(
                    nodeSet, looper, nodeNames[i]) for i in range(
                    0, 4))

            # Since primary selection is round robin, A and B will be primaries

            # Nodes C and D delays self nomination so A and B can become
            # primaries
            # nodeC.delaySelfNomination(10)
            # nodeD.delaySelfNomination(10)

            # Node D delays receiving PRIMARY messages from all nodes so it
            # will not know whether it is primary or not

            # delayD = 5
            # nodeD.nodeIbStasher.delay(delayerMsgTuple(delayD, Primary))

            checkPoolReady(looper=looper, nodes=nodeSet)

            # client1, wal = setupClient(looper, nodeSet, tmpdir=tdir_for_func)
            # request = sendRandomRequest(wal, client1)

            # TODO Rethink this
            instNo = 0

            timeout = waits.expectedClientRequestPropagationTime(len(nodeSet))
            for i in range(3):
                node = nodeSet.getNode(nodeNames[i])
                # Nodes A, B and C should have received PROPAGATE request
                # from Node D
                looper.run(
                    eventually(checkIfPropagateRecvdFromNode, node, nodeD,
                               request.identifier,
                               request.reqId, retryWait=1, timeout=timeout))

            def assert_msg_count(typ, count):
                assert len(getPendingRequestsForReplica(nodeD.replicas[instNo],
                                                        typ)) == count

            # Node D should have 1 pending PRE-PREPARE request
            timeout = waits.expectedPrePrepareTime(len(nodeSet))
            looper.run(eventually(assert_msg_count, PrePrepare, 1,
                                  retryWait=1, timeout=timeout))

            # Node D should have 2 pending PREPARE requests(from node B and C)
            timeout = waits.expectedPrepareTime(len(nodeSet))
            looper.run(eventually(assert_msg_count, Prepare, 2, retryWait=1,
                                  timeout=timeout))

            # Its been checked above that replica stashes 3 phase messages in
            # lack of primary, now avoid delay (fix the network)
            nodeD.nodeIbStasher.reset_delays_and_process_delayeds()

            # Node D should have no pending PRE-PREPARE, PREPARE or COMMIT
            # requests
            for reqType in [PrePrepare, Prepare, Commit]:
                looper.run(
                    eventually(
                        lambda: assertLength(
                            getPendingRequestsForReplica(
                                nodeD.replicas[instNo],
                                reqType),
                            0),
                        retryWait=1,
                        timeout=delayD))  # wait little more than delay