def testQueueingReqFromFutureView(delayedPerf, looper, nodeSet, up, client1):
"""
Test if every node queues 3 Phase requests(PRE-PREPARE, PREPARE and COMMIT)
that come from a view which is greater than the current view
"""
f = getMaxFailures(nodeCount)
# Delay processing of instance change on a node
nodeA = nodeSet.Alpha
nodeA.nodeIbStasher.delay(icDelay(60))
nonPrimReps = getNonPrimaryReplicas(nodeSet, 0)
# Delay processing of PRE-PREPARE from all non primary replicas of master
# so master's throughput falls and view changes
ppDelayer = ppDelay(5, 0)
for r in nonPrimReps:
r.node.nodeIbStasher.delay(ppDelayer)
sendReqsToNodesAndVerifySuffReplies(looper, client1, 4, timeout=5 * nodeCount)
# Every node except Node A should have a view change
for node in nodeSet:
if node.name != nodeA.name:
looper.run(eventually(
partial(checkViewChangeInitiatedForNode, node, 0),
retryWait=1,
timeout=20))
# Node A's view should not have changed yet
with pytest.raises(AssertionError):
looper.run(eventually(partial(
checkViewChangeInitiatedForNode, nodeA, 0),
retryWait=1,
timeout=20))
# NodeA should not have any pending 3 phase request for a later view
for r in nodeA.replicas: # type: TestReplica
assert len(r.threePhaseMsgsForLaterView) == 0
# Reset delays on incoming messages from all nodes
for node in nodeSet:
node.nodeIbStasher.nodelay(ppDelayer)
# Send one more request
sendRandomRequest(client1)
def checkPending3PhaseReqs():
# Get all replicas that have their primary status decided
reps = [rep for rep in nodeA.replicas if rep.isPrimary is not None]
# Atleast one replica should have its primary status decided
assert len(reps) > 0
for r in reps: # type: TestReplica
logging.debug("primary status for replica {} is {}"
.format(r, r.primaryNames))
assert len(r.threePhaseMsgsForLaterView) > 0
# NodeA should now have pending 3 phase request for a later view
looper.run(eventually(checkPending3PhaseReqs, retryWait=1, timeout=30))
def testReplicasRejectSamePrePrepareMsg(looper, nodeSet, client1):
"""
Replicas should not accept PRE-PREPARE for view "v" and prepare sequence
number "n" if it has already accepted a request with view number "v" and
sequence number "n"
"""
numOfNodes = 4
fValue = getMaxFailures(numOfNodes)
request1 = sendRandomRequest(client1)
result1 = looper.run(
eventually(checkSufficientRepliesRecvd, client1.inBox,
request1.reqId, fValue,
retryWait=1, timeout=5))
logging.debug("request {} gives result {}".format(request1, result1))
primaryRepl = getPrimaryReplica(nodeSet)
logging.debug("Primary Replica: {}".format(primaryRepl))
logging.debug(
"Decrementing the primary replica's pre-prepare sequence number by one...")
primaryRepl.prePrepareSeqNo -= 1
request2 = sendRandomRequest(client1)
looper.run(eventually(checkPrePrepareReqSent, primaryRepl, request2,
retryWait=1, timeout=10))
nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet)
logging.debug("Non Primary Replicas: " + str(nonPrimaryReplicas))
prePrepareReq = PrePrepare(
primaryRepl.instId,
primaryRepl.viewNo,
primaryRepl.prePrepareSeqNo,
client1.clientId,
request2.reqId,
request2.digest)
logging.debug("""Checking whether all the non primary replicas have received
the pre-prepare request with same sequence number""")
looper.run(eventually(checkPrePrepareReqRecvd,
nonPrimaryReplicas,
prePrepareReq,
retryWait=1,
timeout=10))
logging.debug("""Check that none of the non primary replicas didn't send any prepare message "
in response to the pre-prepare message""")
for npr in nonPrimaryReplicas:
with pytest.raises(AssertionError):
looper.run(eventually(checkPrepareReqSent,
npr,
client1.clientId,
request2.reqId,
retryWait=1,
timeout=10))
def testPrePrepareWhenPrimaryStatusIsUnknown(tdir_for_func):
nodeNames = genNodeNames(4)
nodeReg = genNodeReg(names=nodeNames)
with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet:
with Looper(nodeSet) as looper:
prepareNodeSet(looper, nodeSet)
nodeA, nodeB, nodeC, nodeD = tuple(
addNodeBack(nodeSet, looper, nodeNames[i]) for i in range(0, 4))
# Nodes C and D delays self nomination so A and B can become primaries
nodeC.delaySelfNomination(30)
nodeD.delaySelfNomination(30)
# Node D delays receiving PRIMARY messages from all nodes so it will not know
# whether it is primary or not
# nodeD.nodestack.delay(delayer(20, PRIMARY))
nodeD.nodeIbStasher.delay(delayerMsgTuple(20, Primary))
checkPoolReady(looper=looper, nodes=nodeSet)
client1 = setupClient(looper, nodeSet, tmpdir=tdir_for_func)
request = sendRandomRequest(client1)
# TODO Rethink this
instNo = 0
for i in range(3):
node = nodeSet.getNode(nodeNames[i])
# Nodes A, B and C should have received PROPAGATE request from Node D
looper.run(
eventually(checkIfPropagateRecvdFromNode, node, nodeD,
request.clientId,
request.reqId, retryWait=1, timeout=10))
# Node D should have 1 pending PRE-PREPARE request
def assertOnePrePrepare():
assert len(getPendingRequestsForReplica(nodeD.replicas[instNo],
PrePrepare)) == 1
looper.run(eventually(assertOnePrePrepare, retryWait=1, timeout=10))
# Node D should have 2 pending PREPARE requests(from node B and C)
def assertTwoPrepare():
assert len(getPendingRequestsForReplica(nodeD.replicas[instNo],
Prepare)) == 2
looper.run(eventually(assertTwoPrepare, retryWait=1, timeout=10))
# Node D should have no pending PRE-PREPARE, PREPARE or COMMIT requests
for reqType in [PrePrepare, Prepare, Commit]:
looper.run(eventually(lambda: assertLength(
getPendingRequestsForReplica(nodeD.replicas[instNo],
reqType),
0), retryWait=1, timeout=20))
def testReplyWhenRepliesFromAllNodesAreSame(looper, client1):
"""
When there are not faulty nodes, the client must get a reply from all the
nodes.
"""
request = sendRandomRequest(client1)
looper.run(
eventually(assertLength, client1.inBox,
2 * nodeCount * request.reqId,
retryWait=.25, timeout=15))
checkResponseCorrectnessFromNodes(client1.inBox, request.reqId, F)
def x():
requests = [sendRandomRequest(client) for _ in range(10)]
for request in requests:
looper.run(eventually(
checkSufficientRepliesRecvd, client.inBox,
request.reqId, 3,
retryWait=1, timeout=3 * len(nodeSet)))
ss2 = snapshotStats(*nodeSet)
diff = statsDiff(ss2, ss1)
# TODO verify the correct number of expected PROPAGATE, PRE-PREPARE, PREPARE, and COMMITs occurred
pprint(ss2)
print("----------------------------------------------")
pprint(diff)
def testThroughput(looper, nodeSet, client1):
"""
Checking if the throughput is being set
"""
client = client1
for i in range(5):
req = sendRandomRequest(client)
looper.run(eventually(checkSufficientRepliesRecvd,
client.inBox, req.reqId, 1,
retryWait=1, timeout=5))
for node in nodeSet:
masterThroughput, avgBackupThroughput = node.monitor.getThroughputs(node.masterInst)
logging.debug("Master throughput: {}. Avg. backup throughput: {}".
format(masterThroughput, avgBackupThroughput))
assert masterThroughput > 0
assert avgBackupThroughput > 0
def testReqExecWhenReturnedByMaster(tdir_for_func):
with TestNodeSet(count=4, tmpdir=tdir_for_func) as nodeSet:
with Looper(nodeSet) as looper:
for n in nodeSet:
n.startKeySharing()
client1 = setupNodesAndClient(looper, nodeSet, tmpdir=tdir_for_func)
req = sendRandomRequest(client1)
looper.run(eventually(checkSufficientRepliesRecvd, client1.inBox,
req.reqId, 1,
retryWait=1, timeout=15))
for node in nodeSet:
entries = node.spylog.getAll(
node.processOrdered.__name__)
for entry in entries:
arg = entry.params['ordered']
result = entry.result
if arg.instId == node.masterInst:
assert result
else:
assert result is None
def testAvgReqLatency(looper: Looper, nodeSet: TestNodeSet, client1):
"""
Checking if average latency is being set
"""
for i in range(5):
req = sendRandomRequest(client1)
looper.run(eventually(checkSufficientRepliesRecvd,
client1.inBox, req.reqId, 1,
retryWait=1, timeout=5))
for node in nodeSet:
mLat = node.monitor.getAvgLatencyForClient(client1.clientId,
node.masterInst)
bLat = node.monitor.getAvgLatencyForClient(client1.clientId,
*node.nonMasterInsts)
logging.debug("Avg. master latency : {}. Avg. backup latency: {}".
format(mLat, bLat))
assert mLat > 0
assert bLat > 0
def testRequestReturnToNodeWhenPrePrepareNotReceivedByOneNode(tdir_for_func):
"""Test no T-3"""
nodeNames = genNodeNames(7)
nodeReg = genNodeReg(names=nodeNames)
with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet:
with Looper(nodeSet) as looper:
prepareNodeSet(looper, nodeSet)
logging.debug("Add the seven nodes back in")
# Every node except A delays self nomination so A can become primary
nodeA = addNodeBack(nodeSet, looper, nodeNames[0])
for i in range(1, 7):
node = addNodeBack(nodeSet, looper, nodeNames[i])
node.delaySelfNomination(15)
nodeB = nodeSet.getNode(nodeNames[1])
# Node B delays PREPREPARE from node A(which would be the primary) for a long time.
# TODO: Have a method to ignore a particular message from a node
nodeB.nodeIbStasher.delay(
delayerMsgTuple(120, PrePrepare, nodeA.name))
# Ensure elections are done
ensureElectionsDone(looper=looper, nodes=nodeSet, retryWait=1,
timeout=30)
assert nodeA.hasPrimary
instNo = nodeA.primaryReplicaNo
client1 = setupClient(looper, nodeSet, tmpdir=tdir_for_func)
req = sendRandomRequest(client1)
# All nodes including B should return their ordered requests
for node in nodeSet:
looper.run(eventually(checkRequestReturnedToNode, node,
client1.clientId, req.reqId,
req.digest,
instNo, retryWait=1, timeout=30))
# Node B should not have received the PRE-PREPARE request yet
replica = nodeB.replicas[instNo] # type: Replica
assert len(replica.prePrepares) == 0
def testReplyWhenRepliesFromExactlyFPlusOneNodesAreSame(looper, client1):
"""
When only :math:`2f+1` replies from the nodes are matching, the client
would accept the reply
"""
request = sendRandomRequest(client1)
# exactly f + 1 => (3) nodes have correct responses
# modify some (numOfResponses of type REPLY - (f + 1)) => 4 responses to
# have a different operations
looper.run(
eventually(assertLength, client1.inBox,
2 * nodeCount * request.reqId,
retryWait=.25, timeout=15))
replies = (msg for msg, frm in client1.inBox
if msg['op'] == REPLY and
msg['reqId'] == request.reqId)
# change two responses to something different
for i in range(2):
msg = next(replies)
msg['result']['txnId'] = str(i) + "Some random id"
checkResponseCorrectnessFromNodes(client1.inBox, request.reqId, F)
