def testRequestReturnToNodeWhenPrePrepareNotReceivedByOneNode(tdir_for_func): """Test no T-3""" nodeNames = genNodeNames(7) nodeReg = genNodeReg(names=nodeNames) with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet: with Looper(nodeSet) as looper: prepareNodeSet(looper, nodeSet) logger.debug("Add the seven nodes back in") # Every node except A delays self nomination so A can become primary nodeA = addNodeBack(nodeSet, looper, nodeNames[0]) for i in range(1, 7): node = addNodeBack(nodeSet, looper, nodeNames[i]) node.delaySelfNomination(15) nodeB = nodeSet.getNode(nodeNames[1]) # Node B delays PREPREPARE from node A(which would be the primary) # for a long time. nodeB.nodeIbStasher.delay( delayerMsgTuple(120, PrePrepare, nodeA.name)) # Ensure elections are done ensureElectionsDone(looper=looper, nodes=nodeSet, retryWait=1, timeout=30) assert nodeA.hasPrimary instNo = nodeA.primaryReplicaNo client1, wallet1 = setupClient(looper, nodeSet, tmpdir=tdir_for_func) req = sendRandomRequest(wallet1, client1) # All nodes including B should return their ordered requests for node in nodeSet: looper.run( eventually(checkRequestReturnedToNode, node, wallet1.defaultId, req.reqId, instNo, retryWait=1, timeout=30)) # Node B should not have received the PRE-PREPARE request yet replica = nodeB.replicas[instNo] # type: Replica assert len(replica.prePrepares) == 0
def testSelfNominationDelay(tdir_for_func): nodeNames = ["testA", "testB", "testC", "testD"] with TestNodeSet(names=nodeNames, tmpdir=tdir_for_func) as nodeSet: with Looper(nodeSet) as looper: prepareNodeSet(looper, nodeSet) delay = 30 # Add node A nodeA = addNodeBack(nodeSet, looper, nodeNames[0]) nodeA.delaySelfNomination(delay) nodesBCD = [] for name in nodeNames[1:]: # nodesBCD.append(nodeSet.addNode(name, i+1, AutoMode.never)) nodesBCD.append(addNodeBack(nodeSet, looper, name)) # Ensuring that NodeA is started before any other node to demonstrate # that it is delaying self nomination looper.run( eventually(lambda: assertExp(nodeA.isReady()), retryWait=1, timeout=5)) # Elections should be done ensureElectionsDone(looper=looper, nodes=nodeSet, retryWait=1, timeout=10) # node A should not have any primary replica looper.run( eventually(lambda: assertExp(not nodeA.hasPrimary), retryWait=1, timeout=10)) # Make sure that after at the most 30 seconds, nodeA's # `startElection` is called looper.run( eventually(lambda: assertExp( len(nodeA.spylog.getAll(Node.decidePrimaries.__name__)) > 0 ), retryWait=1, timeout=30))
def testRequestReturnToNodeWhenPrePrepareNotReceivedByOneNode(tdir_for_func): """Test no T-3""" nodeNames = genNodeNames(7) nodeReg = genNodeReg(names=nodeNames) with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet: with Looper(nodeSet) as looper: prepareNodeSet(looper, nodeSet) logger.debug("Add the seven nodes back in") # Every node except A delays self nomination so A can become primary nodeA = addNodeBack(nodeSet, looper, nodeNames[0]) for i in range(1, 7): node = addNodeBack(nodeSet, looper, nodeNames[i]) node.delaySelfNomination(15) nodeB = nodeSet.getNode(nodeNames[1]) # Node B delays PREPREPARE from node A(which would be the primary) # for a long time. nodeB.nodeIbStasher.delay( delayerMsgTuple(120, PrePrepare, nodeA.name)) # Ensure elections are done ensureElectionsDone(looper=looper, nodes=nodeSet, retryWait=1, timeout=30) assert nodeA.hasPrimary instNo = nodeA.primaryReplicaNo client1, wallet1 = setupClient(looper, nodeSet, tmpdir=tdir_for_func) req = sendRandomRequest(wallet1, client1) # All nodes including B should return their ordered requests for node in nodeSet: looper.run(eventually(checkRequestReturnedToNode, node, wallet1.defaultId, req.reqId, instNo, retryWait=1, timeout=30)) # Node B should not have received the PRE-PREPARE request yet replica = nodeB.replicas[instNo] # type: Replica assert len(replica.prePrepares) == 0
def testPrePrepareWhenPrimaryStatusIsUnknown(tdir_for_func): nodeNames = genNodeNames(4) nodeReg = genNodeReg(names=nodeNames) with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet: with Looper(nodeSet) as looper: prepareNodeSet(looper, nodeSet) nodeA, nodeB, nodeC, nodeD = tuple( addNodeBack(nodeSet, looper, nodeNames[i]) for i in range(0, 4)) # Nodes C and D delays self nomination so A and B can become # primaries nodeC.delaySelfNomination(30) nodeD.delaySelfNomination(30) # Node D delays receiving PRIMARY messages from all nodes so it # will not know whether it is primary or not # nodeD.nodestack.delay(delayer(20, PRIMARY)) nodeD.nodeIbStasher.delay(delayerMsgTuple(20, Primary)) checkPoolReady(looper=looper, nodes=nodeSet) client1, wal = setupClient(looper, nodeSet, tmpdir=tdir_for_func) request = sendRandomRequest(wal, client1) # TODO Rethink this instNo = 0 for i in range(3): node = nodeSet.getNode(nodeNames[i]) # Nodes A, B and C should have received PROPAGATE request # from Node D looper.run( eventually(checkIfPropagateRecvdFromNode, node, nodeD, request.identifier, request.reqId, retryWait=1, timeout=10)) # Node D should have 1 pending PRE-PREPARE request def assertOnePrePrepare(): assert len( getPendingRequestsForReplica(nodeD.replicas[instNo], PrePrepare)) == 1 looper.run(eventually(assertOnePrePrepare, retryWait=1, timeout=10)) # Node D should have 2 pending PREPARE requests(from node B and C) def assertTwoPrepare(): assert len( getPendingRequestsForReplica(nodeD.replicas[instNo], Prepare)) == 2 looper.run(eventually(assertTwoPrepare, retryWait=1, timeout=10)) # Node D should have no pending PRE-PREPARE, PREPARE or COMMIT # requests for reqType in [PrePrepare, Prepare, Commit]: looper.run( eventually(lambda: assertLength( getPendingRequestsForReplica(nodeD.replicas[instNo], reqType), 0), retryWait=1, timeout=20))
def testPrePrepareWhenPrimaryStatusIsUnknown(tdir_for_func): nodeNames = genNodeNames(4) nodeReg = genNodeReg(names=nodeNames) with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet: with Looper(nodeSet) as looper: prepareNodeSet(looper, nodeSet) nodeA, nodeB, nodeC, nodeD = tuple( addNodeBack(nodeSet, looper, nodeNames[i]) for i in range(0, 4)) # Since primary selection is round robin, A and B will be primaries # Nodes C and D delays self nomination so A and B can become # primaries # nodeC.delaySelfNomination(10) # nodeD.delaySelfNomination(10) # Node D delays receiving PRIMARY messages from all nodes so it # will not know whether it is primary or not # delayD = 5 # nodeD.nodeIbStasher.delay(delayerMsgTuple(delayD, Primary)) checkPoolReady(looper=looper, nodes=nodeSet) # client1, wal = setupClient(looper, nodeSet, tmpdir=tdir_for_func) # request = sendRandomRequest(wal, client1) # TODO Rethink this instNo = 0 timeout = waits.expectedClientRequestPropagationTime(len(nodeSet)) for i in range(3): node = nodeSet.getNode(nodeNames[i]) # Nodes A, B and C should have received PROPAGATE request # from Node D looper.run( eventually(checkIfPropagateRecvdFromNode, node, nodeD, request.identifier, request.reqId, retryWait=1, timeout=timeout)) def assert_msg_count(typ, count): assert len( getPendingRequestsForReplica(nodeD.replicas[instNo], typ)) == count # Node D should have 1 pending PRE-PREPARE request timeout = waits.expectedPrePrepareTime(len(nodeSet)) looper.run( eventually(assert_msg_count, PrePrepare, 1, retryWait=1, timeout=timeout)) # Node D should have 2 pending PREPARE requests(from node B and C) timeout = waits.expectedPrepareTime(len(nodeSet)) looper.run( eventually(assert_msg_count, Prepare, 2, retryWait=1, timeout=timeout)) # Its been checked above that replica stashes 3 phase messages in # lack of primary, now avoid delay (fix the network) nodeD.nodeIbStasher.reset_delays_and_process_delayeds() # Node D should have no pending PRE-PREPARE, PREPARE or COMMIT # requests for reqType in [PrePrepare, Prepare, Commit]: looper.run( eventually(lambda: assertLength( getPendingRequestsForReplica(nodeD.replicas[instNo], reqType), 0), retryWait=1, timeout=delayD)) # wait little more than delay
def testPrePrepareWhenPrimaryStatusIsUnknown(tdir_for_func): nodeNames = genNodeNames(4) nodeReg = genNodeReg(names=nodeNames) with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet: with Looper(nodeSet) as looper: prepareNodeSet(looper, nodeSet) nodeA, nodeB, nodeC, nodeD = tuple( addNodeBack(nodeSet, looper, nodeNames[i]) for i in range(0, 4)) # Nodes C and D delays self nomination so A and B can become # primaries nodeC.delaySelfNomination(30) nodeD.delaySelfNomination(30) # Node D delays receiving PRIMARY messages from all nodes so it # will not know whether it is primary or not # nodeD.nodestack.delay(delayer(20, PRIMARY)) nodeD.nodeIbStasher.delay(delayerMsgTuple(20, Primary)) checkPoolReady(looper=looper, nodes=nodeSet) client1, wal = setupClient(looper, nodeSet, tmpdir=tdir_for_func) request = sendRandomRequest(wal, client1) # TODO Rethink this instNo = 0 for i in range(3): node = nodeSet.getNode(nodeNames[i]) # Nodes A, B and C should have received PROPAGATE request # from Node D looper.run( eventually(checkIfPropagateRecvdFromNode, node, nodeD, request.identifier, request.reqId, retryWait=1, timeout=10)) # Node D should have 1 pending PRE-PREPARE request def assertOnePrePrepare(): assert len(getPendingRequestsForReplica(nodeD.replicas[instNo], PrePrepare)) == 1 looper.run(eventually(assertOnePrePrepare, retryWait=1, timeout=10)) # Node D should have 2 pending PREPARE requests(from node B and C) def assertTwoPrepare(): assert len(getPendingRequestsForReplica(nodeD.replicas[instNo], Prepare)) == 2 looper.run(eventually(assertTwoPrepare, retryWait=1, timeout=10)) # Node D should have no pending PRE-PREPARE, PREPARE or COMMIT # requests for reqType in [PrePrepare, Prepare, Commit]: looper.run(eventually(lambda: assertLength( getPendingRequestsForReplica(nodeD.replicas[instNo], reqType), 0), retryWait=1, timeout=20))
def testPrePrepareWhenPrimaryStatusIsUnknown(tdir_for_func): nodeNames = genNodeNames(4) nodeReg = genNodeReg(names=nodeNames) with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet: with Looper(nodeSet) as looper: prepareNodeSet(looper, nodeSet) nodeA, nodeB, nodeC, nodeD = tuple( addNodeBack( nodeSet, looper, nodeNames[i]) for i in range( 0, 4)) # Since primary selection is round robin, A and B will be primaries # Nodes C and D delays self nomination so A and B can become # primaries # nodeC.delaySelfNomination(10) # nodeD.delaySelfNomination(10) # Node D delays receiving PRIMARY messages from all nodes so it # will not know whether it is primary or not # delayD = 5 # nodeD.nodeIbStasher.delay(delayerMsgTuple(delayD, Primary)) checkPoolReady(looper=looper, nodes=nodeSet) # client1, wal = setupClient(looper, nodeSet, tmpdir=tdir_for_func) # request = sendRandomRequest(wal, client1) # TODO Rethink this instNo = 0 timeout = waits.expectedClientRequestPropagationTime(len(nodeSet)) for i in range(3): node = nodeSet.getNode(nodeNames[i]) # Nodes A, B and C should have received PROPAGATE request # from Node D looper.run( eventually(checkIfPropagateRecvdFromNode, node, nodeD, request.identifier, request.reqId, retryWait=1, timeout=timeout)) def assert_msg_count(typ, count): assert len(getPendingRequestsForReplica(nodeD.replicas[instNo], typ)) == count # Node D should have 1 pending PRE-PREPARE request timeout = waits.expectedPrePrepareTime(len(nodeSet)) looper.run(eventually(assert_msg_count, PrePrepare, 1, retryWait=1, timeout=timeout)) # Node D should have 2 pending PREPARE requests(from node B and C) timeout = waits.expectedPrepareTime(len(nodeSet)) looper.run(eventually(assert_msg_count, Prepare, 2, retryWait=1, timeout=timeout)) # Its been checked above that replica stashes 3 phase messages in # lack of primary, now avoid delay (fix the network) nodeD.nodeIbStasher.reset_delays_and_process_delayeds() # Node D should have no pending PRE-PREPARE, PREPARE or COMMIT # requests for reqType in [PrePrepare, Prepare, Commit]: looper.run( eventually( lambda: assertLength( getPendingRequestsForReplica( nodeD.replicas[instNo], reqType), 0), retryWait=1, timeout=delayD)) # wait little more than delay