def testPrePrepareWhenPrimaryStatusIsUnknown(tdir_for_func): nodeNames = genNodeNames(4) nodeReg = genNodeReg(names=nodeNames) with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet: with Looper(nodeSet) as looper: prepareNodeSet(looper, nodeSet) nodeA, nodeB, nodeC, nodeD = tuple( addNodeBack(nodeSet, looper, nodeNames[i]) for i in range(0, 4)) # Nodes C and D delays self nomination so A and B can become primaries nodeC.delaySelfNomination(30) nodeD.delaySelfNomination(30) # Node D delays receiving PRIMARY messages from all nodes so it will not know # whether it is primary or not # nodeD.nodestack.delay(delayer(20, PRIMARY)) nodeD.nodeIbStasher.delay(delayerMsgTuple(20, Primary)) checkPoolReady(looper=looper, nodes=nodeSet) client1 = setupClient(looper, nodeSet, tmpdir=tdir_for_func) request = sendRandomRequest(client1) # TODO Rethink this instNo = 0 for i in range(3): node = nodeSet.getNode(nodeNames[i]) # Nodes A, B and C should have received PROPAGATE request from Node D looper.run( eventually(checkIfPropagateRecvdFromNode, node, nodeD, request.clientId, request.reqId, retryWait=1, timeout=10)) # Node D should have 1 pending PRE-PREPARE request def assertOnePrePrepare(): assert len(getPendingRequestsForReplica(nodeD.replicas[instNo], PrePrepare)) == 1 looper.run(eventually(assertOnePrePrepare, retryWait=1, timeout=10)) # Node D should have 2 pending PREPARE requests(from node B and C) def assertTwoPrepare(): assert len(getPendingRequestsForReplica(nodeD.replicas[instNo], Prepare)) == 2 looper.run(eventually(assertTwoPrepare, retryWait=1, timeout=10)) # Node D should have no pending PRE-PREPARE, PREPARE or COMMIT requests for reqType in [PrePrepare, Prepare, Commit]: looper.run(eventually(lambda: assertLength( getPendingRequestsForReplica(nodeD.replicas[instNo], reqType), 0), retryWait=1, timeout=20))
def testRequestReturnToNodeWhenPrePrepareNotReceivedByOneNode(tdir_for_func): """Test no T-3""" nodeNames = genNodeNames(7) nodeReg = genNodeReg(names=nodeNames) with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet: with Looper(nodeSet) as looper: prepareNodeSet(looper, nodeSet) logging.debug("Add the seven nodes back in") # Every node except A delays self nomination so A can become primary nodeA = addNodeBack(nodeSet, looper, nodeNames[0]) for i in range(1, 7): node = addNodeBack(nodeSet, looper, nodeNames[i]) node.delaySelfNomination(15) nodeB = nodeSet.getNode(nodeNames[1]) # Node B delays PREPREPARE from node A(which would be the primary) for a long time. # TODO: Have a method to ignore a particular message from a node nodeB.nodeIbStasher.delay( delayerMsgTuple(120, PrePrepare, nodeA.name)) # Ensure elections are done ensureElectionsDone(looper=looper, nodes=nodeSet, retryWait=1, timeout=30) assert nodeA.hasPrimary instNo = nodeA.primaryReplicaNo client1 = setupClient(looper, nodeSet, tmpdir=tdir_for_func) req = sendRandomRequest(client1) # All nodes including B should return their ordered requests for node in nodeSet: looper.run(eventually(checkRequestReturnedToNode, node, client1.clientId, req.reqId, req.digest, instNo, retryWait=1, timeout=30)) # Node B should not have received the PRE-PREPARE request yet replica = nodeB.replicas[instNo] # type: Replica assert len(replica.prePrepares) == 0