def testQueueingReqFromFutureView(delayedPerf, looper, nodeSet, up, client1): """ Test if every node queues 3 Phase requests(PRE-PREPARE, PREPARE and COMMIT) that come from a view which is greater than the current view """ f = getMaxFailures(nodeCount) # Delay processing of instance change on a node nodeA = nodeSet.Alpha nodeA.nodeIbStasher.delay(icDelay(60)) nonPrimReps = getNonPrimaryReplicas(nodeSet, 0) # Delay processing of PRE-PREPARE from all non primary replicas of master # so master's throughput falls and view changes ppDelayer = ppDelay(5, 0) for r in nonPrimReps: r.node.nodeIbStasher.delay(ppDelayer) sendReqsToNodesAndVerifySuffReplies(looper, client1, 4, timeout=5 * nodeCount) # Every node except Node A should have a view change for node in nodeSet: if node.name != nodeA.name: looper.run(eventually( partial(checkViewChangeInitiatedForNode, node, 0), retryWait=1, timeout=20)) # Node A's view should not have changed yet with pytest.raises(AssertionError): looper.run(eventually(partial( checkViewChangeInitiatedForNode, nodeA, 0), retryWait=1, timeout=20)) # NodeA should not have any pending 3 phase request for a later view for r in nodeA.replicas: # type: TestReplica assert len(r.threePhaseMsgsForLaterView) == 0 # Reset delays on incoming messages from all nodes for node in nodeSet: node.nodeIbStasher.nodelay(ppDelayer) # Send one more request sendRandomRequest(client1) def checkPending3PhaseReqs(): # Get all replicas that have their primary status decided reps = [rep for rep in nodeA.replicas if rep.isPrimary is not None] # Atleast one replica should have its primary status decided assert len(reps) > 0 for r in reps: # type: TestReplica logging.debug("primary status for replica {} is {}" .format(r, r.primaryNames)) assert len(r.threePhaseMsgsForLaterView) > 0 # NodeA should now have pending 3 phase request for a later view looper.run(eventually(checkPending3PhaseReqs, retryWait=1, timeout=30))
def testReplicasRejectSamePrePrepareMsg(looper, nodeSet, client1): """ Replicas should not accept PRE-PREPARE for view "v" and prepare sequence number "n" if it has already accepted a request with view number "v" and sequence number "n" """ numOfNodes = 4 fValue = getMaxFailures(numOfNodes) request1 = sendRandomRequest(client1) result1 = looper.run( eventually(checkSufficientRepliesRecvd, client1.inBox, request1.reqId, fValue, retryWait=1, timeout=5)) logging.debug("request {} gives result {}".format(request1, result1)) primaryRepl = getPrimaryReplica(nodeSet) logging.debug("Primary Replica: {}".format(primaryRepl)) logging.debug( "Decrementing the primary replica's pre-prepare sequence number by one...") primaryRepl.prePrepareSeqNo -= 1 request2 = sendRandomRequest(client1) looper.run(eventually(checkPrePrepareReqSent, primaryRepl, request2, retryWait=1, timeout=10)) nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet) logging.debug("Non Primary Replicas: " + str(nonPrimaryReplicas)) prePrepareReq = PrePrepare( primaryRepl.instId, primaryRepl.viewNo, primaryRepl.prePrepareSeqNo, client1.clientId, request2.reqId, request2.digest) logging.debug("""Checking whether all the non primary replicas have received the pre-prepare request with same sequence number""") looper.run(eventually(checkPrePrepareReqRecvd, nonPrimaryReplicas, prePrepareReq, retryWait=1, timeout=10)) logging.debug("""Check that none of the non primary replicas didn't send any prepare message " in response to the pre-prepare message""") for npr in nonPrimaryReplicas: with pytest.raises(AssertionError): looper.run(eventually(checkPrepareReqSent, npr, client1.clientId, request2.reqId, retryWait=1, timeout=10))
def testPrePrepareWhenPrimaryStatusIsUnknown(tdir_for_func): nodeNames = genNodeNames(4) nodeReg = genNodeReg(names=nodeNames) with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet: with Looper(nodeSet) as looper: prepareNodeSet(looper, nodeSet) nodeA, nodeB, nodeC, nodeD = tuple( addNodeBack(nodeSet, looper, nodeNames[i]) for i in range(0, 4)) # Nodes C and D delays self nomination so A and B can become primaries nodeC.delaySelfNomination(30) nodeD.delaySelfNomination(30) # Node D delays receiving PRIMARY messages from all nodes so it will not know # whether it is primary or not # nodeD.nodestack.delay(delayer(20, PRIMARY)) nodeD.nodeIbStasher.delay(delayerMsgTuple(20, Primary)) checkPoolReady(looper=looper, nodes=nodeSet) client1 = setupClient(looper, nodeSet, tmpdir=tdir_for_func) request = sendRandomRequest(client1) # TODO Rethink this instNo = 0 for i in range(3): node = nodeSet.getNode(nodeNames[i]) # Nodes A, B and C should have received PROPAGATE request from Node D looper.run( eventually(checkIfPropagateRecvdFromNode, node, nodeD, request.clientId, request.reqId, retryWait=1, timeout=10)) # Node D should have 1 pending PRE-PREPARE request def assertOnePrePrepare(): assert len(getPendingRequestsForReplica(nodeD.replicas[instNo], PrePrepare)) == 1 looper.run(eventually(assertOnePrePrepare, retryWait=1, timeout=10)) # Node D should have 2 pending PREPARE requests(from node B and C) def assertTwoPrepare(): assert len(getPendingRequestsForReplica(nodeD.replicas[instNo], Prepare)) == 2 looper.run(eventually(assertTwoPrepare, retryWait=1, timeout=10)) # Node D should have no pending PRE-PREPARE, PREPARE or COMMIT requests for reqType in [PrePrepare, Prepare, Commit]: looper.run(eventually(lambda: assertLength( getPendingRequestsForReplica(nodeD.replicas[instNo], reqType), 0), retryWait=1, timeout=20))
def testReplyWhenRepliesFromAllNodesAreSame(looper, client1): """ When there are not faulty nodes, the client must get a reply from all the nodes. """ request = sendRandomRequest(client1) looper.run( eventually(assertLength, client1.inBox, 2 * nodeCount * request.reqId, retryWait=.25, timeout=15)) checkResponseCorrectnessFromNodes(client1.inBox, request.reqId, F)
def x(): requests = [sendRandomRequest(client) for _ in range(10)] for request in requests: looper.run(eventually( checkSufficientRepliesRecvd, client.inBox, request.reqId, 3, retryWait=1, timeout=3 * len(nodeSet))) ss2 = snapshotStats(*nodeSet) diff = statsDiff(ss2, ss1) # TODO verify the correct number of expected PROPAGATE, PRE-PREPARE, PREPARE, and COMMITs occurred pprint(ss2) print("----------------------------------------------") pprint(diff)
def testThroughput(looper, nodeSet, client1): """ Checking if the throughput is being set """ client = client1 for i in range(5): req = sendRandomRequest(client) looper.run(eventually(checkSufficientRepliesRecvd, client.inBox, req.reqId, 1, retryWait=1, timeout=5)) for node in nodeSet: masterThroughput, avgBackupThroughput = node.monitor.getThroughputs(node.masterInst) logging.debug("Master throughput: {}. Avg. backup throughput: {}". format(masterThroughput, avgBackupThroughput)) assert masterThroughput > 0 assert avgBackupThroughput > 0
def testReqExecWhenReturnedByMaster(tdir_for_func): with TestNodeSet(count=4, tmpdir=tdir_for_func) as nodeSet: with Looper(nodeSet) as looper: for n in nodeSet: n.startKeySharing() client1 = setupNodesAndClient(looper, nodeSet, tmpdir=tdir_for_func) req = sendRandomRequest(client1) looper.run(eventually(checkSufficientRepliesRecvd, client1.inBox, req.reqId, 1, retryWait=1, timeout=15)) for node in nodeSet: entries = node.spylog.getAll( node.processOrdered.__name__) for entry in entries: arg = entry.params['ordered'] result = entry.result if arg.instId == node.masterInst: assert result else: assert result is None
def testAvgReqLatency(looper: Looper, nodeSet: TestNodeSet, client1): """ Checking if average latency is being set """ for i in range(5): req = sendRandomRequest(client1) looper.run(eventually(checkSufficientRepliesRecvd, client1.inBox, req.reqId, 1, retryWait=1, timeout=5)) for node in nodeSet: mLat = node.monitor.getAvgLatencyForClient(client1.clientId, node.masterInst) bLat = node.monitor.getAvgLatencyForClient(client1.clientId, *node.nonMasterInsts) logging.debug("Avg. master latency : {}. Avg. backup latency: {}". format(mLat, bLat)) assert mLat > 0 assert bLat > 0
def testRequestReturnToNodeWhenPrePrepareNotReceivedByOneNode(tdir_for_func): """Test no T-3""" nodeNames = genNodeNames(7) nodeReg = genNodeReg(names=nodeNames) with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet: with Looper(nodeSet) as looper: prepareNodeSet(looper, nodeSet) logging.debug("Add the seven nodes back in") # Every node except A delays self nomination so A can become primary nodeA = addNodeBack(nodeSet, looper, nodeNames[0]) for i in range(1, 7): node = addNodeBack(nodeSet, looper, nodeNames[i]) node.delaySelfNomination(15) nodeB = nodeSet.getNode(nodeNames[1]) # Node B delays PREPREPARE from node A(which would be the primary) for a long time. # TODO: Have a method to ignore a particular message from a node nodeB.nodeIbStasher.delay( delayerMsgTuple(120, PrePrepare, nodeA.name)) # Ensure elections are done ensureElectionsDone(looper=looper, nodes=nodeSet, retryWait=1, timeout=30) assert nodeA.hasPrimary instNo = nodeA.primaryReplicaNo client1 = setupClient(looper, nodeSet, tmpdir=tdir_for_func) req = sendRandomRequest(client1) # All nodes including B should return their ordered requests for node in nodeSet: looper.run(eventually(checkRequestReturnedToNode, node, client1.clientId, req.reqId, req.digest, instNo, retryWait=1, timeout=30)) # Node B should not have received the PRE-PREPARE request yet replica = nodeB.replicas[instNo] # type: Replica assert len(replica.prePrepares) == 0
def testReplyWhenRepliesFromExactlyFPlusOneNodesAreSame(looper, client1): """ When only :math:`2f+1` replies from the nodes are matching, the client would accept the reply """ request = sendRandomRequest(client1) # exactly f + 1 => (3) nodes have correct responses # modify some (numOfResponses of type REPLY - (f + 1)) => 4 responses to # have a different operations looper.run( eventually(assertLength, client1.inBox, 2 * nodeCount * request.reqId, retryWait=.25, timeout=15)) replies = (msg for msg, frm in client1.inBox if msg['op'] == REPLY and msg['reqId'] == request.reqId) # change two responses to something different for i in range(2): msg = next(replies) msg['result']['txnId'] = str(i) + "Some random id" checkResponseCorrectnessFromNodes(client1.inBox, request.reqId, F)