def testNonPrimarySendsAPrePrepare(looper, nodeSet, setup, propagated1):
primaryReplica = getPrimaryReplica(nodeSet, instId)
nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet, instId)
firstNpr = nonPrimaryReplicas[0]
remainingNpr = nonPrimaryReplicas[1:]
def sendPrePrepareFromNonPrimary(replica):
firstNpr.sendPrePrepare(propagated1.reqDigest)
return PrePrepare(
replica.instId,
firstNpr.viewNo,
firstNpr.prePrepareSeqNo,
propagated1.clientId,
propagated1.reqId,
propagated1.digest)
ppr = sendPrePrepareFromNonPrimary(firstNpr)
def chk():
for r in (primaryReplica, *remainingNpr):
recvdPps = recvdPrePrepare(r)
assert len(recvdPps) == 1
assert recvdPps[0]['pp'] == ppr
nodeSuspicions = len(getNodeSuspicions(
r.node, Suspicions.PPR_FRM_NON_PRIMARY.code))
assert nodeSuspicions == 1
looper.run(eventually(chk,
retryWait=.5, timeout=5))
looper.run(eventually(partial(checkViewNoForNodes, nodeSet, 1), retryWait=1,
timeout=20))
def testMultipleInstanceChangeMsgsMarkNodeAsSuspicious(looper, nodeSet, up):
maliciousNode = nodeSet.Alpha
for i in range(0, 5):
maliciousNode.send(InstanceChange(i))
def chk(instId):
for node in nodeSet:
if node.name != maliciousNode.name:
args = getAllArgs(node, Node.processInstanceChange)
assert len(args) == 5
for arg in args:
assert arg['frm'] == maliciousNode.name
for i in range(0, 5):
looper.run(eventually(chk, i, retryWait=1, timeout=20))
def g():
for node in nodeSet:
if node.name != maliciousNode.name:
frm, reason, code = getAllArgs(node, Node.reportSuspiciousNode)
assert frm == maliciousNode.name
assert isinstance(reason, SuspiciousNode)
assert len(getNodeSuspicions(node,
Suspicions.FREQUENT_INST_CHNG.code)) == 13
looper.run(eventually(g, retryWait=1, timeout=20))
def testQueueingReqFromFutureView(delayedPerf, looper, nodeSet, up, client1):
"""
Test if every node queues 3 Phase requests(PRE-PREPARE, PREPARE and COMMIT)
that come from a view which is greater than the current view
"""
f = getMaxFailures(nodeCount)
# Delay processing of instance change on a node
nodeA = nodeSet.Alpha
nodeA.nodeIbStasher.delay(icDelay(60))
nonPrimReps = getNonPrimaryReplicas(nodeSet, 0)
# Delay processing of PRE-PREPARE from all non primary replicas of master
# so master's throughput falls and view changes
ppDelayer = ppDelay(5, 0)
for r in nonPrimReps:
r.node.nodeIbStasher.delay(ppDelayer)
sendReqsToNodesAndVerifySuffReplies(looper, client1, 4, timeout=5 * nodeCount)
# Every node except Node A should have a view change
for node in nodeSet:
if node.name != nodeA.name:
looper.run(eventually(
partial(checkViewChangeInitiatedForNode, node, 0),
retryWait=1,
timeout=20))
# Node A's view should not have changed yet
with pytest.raises(AssertionError):
looper.run(eventually(partial(
checkViewChangeInitiatedForNode, nodeA, 0),
retryWait=1,
timeout=20))
# NodeA should not have any pending 3 phase request for a later view
for r in nodeA.replicas: # type: TestReplica
assert len(r.threePhaseMsgsForLaterView) == 0
# Reset delays on incoming messages from all nodes
for node in nodeSet:
node.nodeIbStasher.nodelay(ppDelayer)
# Send one more request
sendRandomRequest(client1)
def checkPending3PhaseReqs():
# Get all replicas that have their primary status decided
reps = [rep for rep in nodeA.replicas if rep.isPrimary is not None]
# Atleast one replica should have its primary status decided
assert len(reps) > 0
for r in reps: # type: TestReplica
logging.debug("primary status for replica {} is {}"
.format(r, r.primaryNames))
assert len(r.threePhaseMsgsForLaterView) > 0
# NodeA should now have pending 3 phase request for a later view
looper.run(eventually(checkPending3PhaseReqs, retryWait=1, timeout=30))
def testPrePrepareWhenPrimaryStatusIsUnknown(tdir_for_func):
nodeNames = genNodeNames(4)
nodeReg = genNodeReg(names=nodeNames)
with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet:
with Looper(nodeSet) as looper:
prepareNodeSet(looper, nodeSet)
nodeA, nodeB, nodeC, nodeD = tuple(
addNodeBack(nodeSet, looper, nodeNames[i]) for i in range(0, 4))
# Nodes C and D delays self nomination so A and B can become primaries
nodeC.delaySelfNomination(30)
nodeD.delaySelfNomination(30)
# Node D delays receiving PRIMARY messages from all nodes so it will not know
# whether it is primary or not
# nodeD.nodestack.delay(delayer(20, PRIMARY))
nodeD.nodeIbStasher.delay(delayerMsgTuple(20, Primary))
checkPoolReady(looper=looper, nodes=nodeSet)
client1 = setupClient(looper, nodeSet, tmpdir=tdir_for_func)
request = sendRandomRequest(client1)
# TODO Rethink this
instNo = 0
for i in range(3):
node = nodeSet.getNode(nodeNames[i])
# Nodes A, B and C should have received PROPAGATE request from Node D
looper.run(
eventually(checkIfPropagateRecvdFromNode, node, nodeD,
request.clientId,
request.reqId, retryWait=1, timeout=10))
# Node D should have 1 pending PRE-PREPARE request
def assertOnePrePrepare():
assert len(getPendingRequestsForReplica(nodeD.replicas[instNo],
PrePrepare)) == 1
looper.run(eventually(assertOnePrePrepare, retryWait=1, timeout=10))
# Node D should have 2 pending PREPARE requests(from node B and C)
def assertTwoPrepare():
assert len(getPendingRequestsForReplica(nodeD.replicas[instNo],
Prepare)) == 2
looper.run(eventually(assertTwoPrepare, retryWait=1, timeout=10))
# Node D should have no pending PRE-PREPARE, PREPARE or COMMIT requests
for reqType in [PrePrepare, Prepare, Commit]:
looper.run(eventually(lambda: assertLength(
getPendingRequestsForReplica(nodeD.replicas[instNo],
reqType),
0), retryWait=1, timeout=20))
def testPrimaryElectionWithAClearWinner(electContFixture, looper, keySharedNodes):
"""
Primary selection (Sunny Day)
A, B, C, D, E
A, B, C, D startup. E is lagging.
A sees the minimum number of nodes first, and then sends out a NOMINATE(A) message
B, C, D all see the NOMINATE(A) message from A, and respond with NOMINATE(A) message to all other nodes
A sees three other NOMINATE(A) votes (from B, C, D)
A sees that A is the clear winner (2f+1 total), and sends PRIMARY(A) to all nodes
B sees two more NOMINATE(A) votes (from C and D)
B sees that A is the clear winner (2f+1 total), and sends PRIMARY(A) to all nodes
C sees two more NOMINATE(A) votes (from B and D)
C sees that A is the clear winner (2f+1 total), and sends PRIMARY(A) to all nodes
D sees two more NOMINATE(A) votes (from B and C)
D sees that A is the clear winner (2f+1 total), and sends PRIMARY(A) to all nodes
A sees at least two other PRIMARY(A) votes (3 including it's own)
selects A as primary
B sees at least two other PRIMARY(A) votes (3 including it's own)
selects A as primary
C sees at least two other PRIMARY(A) votes (3 including it's own)
selects A as primary
D sees at least two other PRIMARY(A) votes (3 including it's own)
selects A as primary
"""
nodeSet = keySharedNodes
A, B, C, D = nodeSet.nodes.values()
nodesBCD = [B, C, D]
# attempting to use a raet stack delay... not successful
# nodeBPort = nodesBCD[0].stack.ha[1]
# delayRef = RaetDelay(TrnsKind.alive, PcktKind.ack, nodeBPort)
# nodeA.stack.delay(4, delayRef)
checkPoolReady(looper, nodeSet)
# Checking whether one of the replicas of Node A nominated itself
looper.run(eventually(checkNomination, A, A.name, retryWait=1, timeout=10))
for n in nodesBCD:
# Checking whether Node B, C and D nominated Node A
looper.run(eventually(checkNomination, n, A.name, retryWait=1,
timeout=10))
checkProtocolInstanceSetup(looper=looper, nodes=nodeSet, retryWait=1,
timeout=10)
assert A.hasPrimary
def testReplicasRejectSamePrePrepareMsg(looper, nodeSet, client1):
"""
Replicas should not accept PRE-PREPARE for view "v" and prepare sequence
number "n" if it has already accepted a request with view number "v" and
sequence number "n"
"""
numOfNodes = 4
fValue = getMaxFailures(numOfNodes)
request1 = sendRandomRequest(client1)
result1 = looper.run(
eventually(checkSufficientRepliesRecvd, client1.inBox,
request1.reqId, fValue,
retryWait=1, timeout=5))
logging.debug("request {} gives result {}".format(request1, result1))
primaryRepl = getPrimaryReplica(nodeSet)
logging.debug("Primary Replica: {}".format(primaryRepl))
logging.debug(
"Decrementing the primary replica's pre-prepare sequence number by one...")
primaryRepl.prePrepareSeqNo -= 1
request2 = sendRandomRequest(client1)
looper.run(eventually(checkPrePrepareReqSent, primaryRepl, request2,
retryWait=1, timeout=10))
nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet)
logging.debug("Non Primary Replicas: " + str(nonPrimaryReplicas))
prePrepareReq = PrePrepare(
primaryRepl.instId,
primaryRepl.viewNo,
primaryRepl.prePrepareSeqNo,
client1.clientId,
request2.reqId,
request2.digest)
logging.debug("""Checking whether all the non primary replicas have received
the pre-prepare request with same sequence number""")
looper.run(eventually(checkPrePrepareReqRecvd,
nonPrimaryReplicas,
prePrepareReq,
retryWait=1,
timeout=10))
logging.debug("""Check that none of the non primary replicas didn't send any prepare message "
in response to the pre-prepare message""")
for npr in nonPrimaryReplicas:
with pytest.raises(AssertionError):
looper.run(eventually(checkPrepareReqSent,
npr,
client1.clientId,
request2.reqId,
retryWait=1,
timeout=10))
def testPrimaryElectionCase2(case2Setup, looper, keySharedNodes):
"""
Case 2 - A node making nominations for a multiple other nodes. Consider 4
nodes A, B, C, and D. Lets say node B is malicious and nominates node C
to all nodes. Again node B nominates node D to all nodes.
"""
nodeSet = keySharedNodes
A, B, C, D = nodeSet.nodes.values()
looper.run(checkNodesConnected(nodeSet))
# Node B sends multiple NOMINATE msgs but only after A has nominated itself
looper.run(eventually(checkNomination, A, A.name, retryWait=.25, timeout=1))
instId = getSelfNominationByNode(A)
BRep = Replica.generateName(B.name, instId)
CRep = Replica.generateName(C.name, instId)
DRep = Replica.generateName(D.name, instId)
# Node B first sends NOMINATE msgs for Node C to all nodes
B.send(Nomination(CRep, instId, B.viewNo))
# Node B sends NOMINATE msgs for Node D to all nodes
B.send(Nomination(DRep, instId, B.viewNo))
# Ensure elections are done
ensureElectionsDone(looper=looper, nodes=nodeSet, retryWait=1, timeout=45)
# All nodes from node A, node C, node D(node B is malicious anyway so
# not considering it) should have nomination for node C from node B since
# node B first nominated node C
for node in [A, C, D]:
assert node.elector.nominations[instId][BRep] == CRep
def testOneNodeAltersAClientRequest(looper,
nodeSet,
setup,
evilAlpha,
sent1):
checkPropagated(looper, nodeSet, sent1, faultyNodes)
goodNodes = setup.goodNodes
def check():
for node in goodNodes:
# ensure the nodes are suspicious of Alpha
params = node.spylog.getLastParams(TestNode.reportSuspiciousNode)
frm = params["nodeName"]
reason = params["reason"]
assert frm == 'Alpha'
assert isinstance(reason, InvalidSignature)
# ensure Alpha's propagates were ignored by the other nodes
key = sent1.clientId, sent1.reqId
props = node.requests[key].propagates
assert 'Alpha' not in props
for good in goodNodes:
assert good.name in props
looper.run(eventually(check, retryWait=1, timeout=10))
def testPrimaryElectionCase4(case4Setup, looper):
"""
Case 4 - A node making multiple primary declarations for a particular node.
Consider 4 nodes A, B, C and D. Lets say node B is malicious and is
repeatedly declaring Node D as primary
"""
allNodes = case4Setup
A, B, C, D = allNodes
looper.run(checkNodesConnected(allNodes))
# Node B sends multiple declarations of node D's 0th protocol instance as
# primary to all nodes
for i in range(5):
B.send(Primary(D.name, 0, B.viewNo))
# No node from node A, node C, node D(node B is malicious anyway so not
# considering it) should have more than one primary declaration for node
# D since node D is slow. The one primary declaration for node D,
# that nodes A, C and D might have would be because of node B
def x():
primDecs = list(node.elector.primaryDeclarations[0].values())
assert primDecs.count(D.name) <= 1
for node in (A, C, D):
looper.run(eventually(x, retryWait=0.5, timeout=2))
ensureElectionsDone(looper=looper, nodes=allNodes, retryWait=1, timeout=45)
# Node D should not have any primary replica
assert not D.hasPrimary
def addNodeBackAndCheck(nodeIdx: int, expectedStatus: Status):
logging.info("Add back the {} node and see status of {}".
format(ordinal(nodeIdx + 1), expectedStatus))
addNodeBack(nodeSet, looper, nodeNames[nodeIdx])
looper.run(
eventually(checkNodeStatusRemotesAndF, expectedStatus,
nodeIdx,
retryWait=1, timeout=30))
def testDiscardInstChngMsgFrmPastView(nodeSet, looper, ensureView):
"""
Once a view change is done, any further INSTANCE_CHANGE messages for that
view must be discarded by the node.
"""
curViewNo = ensureView
# Send an instance change for an old instance message to all nodes
icMsg = InstanceChange(curViewNo - 1)
nodeSet.Alpha.send(icMsg)
# ensure every node but Alpha discards the invalid instance change request
looper.run(eventually(checkDiscardMsg, nodeSet, icMsg,
'less than its view no', nodeSet.Alpha, timeout=5))
# Check that that message is discarded.
looper.run(eventually(checkViewNoForNodes, nodeSet, timeout=3))
def sendReqsToNodesAndVerifySuffReplies(looper: Looper, client: TestClient,
numReqs: int, timeout: float = None):
nodeCount = len(client.nodeReg)
f = getMaxFailures(nodeCount)
requests = [sendRandomRequest(client) for i in range(numReqs)]
for request in requests:
looper.run(eventually(checkSufficientRepliesRecvd, client.inBox,
request.reqId, f,
retryWait=1, timeout=3 * nodeCount))
return requests
def setupNodesAndClientAndSendRandomReq(looper: Looper,
nodes: Sequence[TestNode], nodeReg=None,
tmpdir=None):
_client = setupNodesAndClient(looper, nodes, nodeReg, tmpdir)
request = sendRandomRequest(_client)
timeout = 3 * len(nodes)
looper.run(eventually(checkSufficientRepliesRecvd, _client.inBox,
request.reqId, 1,
retryWait=1, timeout=timeout))
return _client, request
def testPrimaryElectionContested(electContFixture, looper, keySharedNodes):
"""
Primary selection (Rainy Day)
A, B, C, D, E
A, B, C, D startup. E is lagging.
A sees the minimum number of nodes, and then sends Nominate(A)
At the same exact time, B sees the minimum number of nodes, and then sends out Nominate(B)
A sees B sending Nominate(B), but it has already nominated itself, so it does nothing
B sees A sending Nominate(A), but it has already nominated itself, so it does nothing
C sees A sending Nominate(A), and sends Nominate(A)
D sees A sending Nominate(A), and sends Nominate(A)
All nodes see that B nominated B and A, C, and D all nominated A
Because the votes for A exceeds the votes for B, all send out Primary(A)
TODO's (see below)
All see the others have sent Primary A, and then the nodes record who is the Primary.
"""
# TODO what if not all send out Primary(A)?
# TODO what if there are big delays in messages getting delivered?
nodeSet = keySharedNodes
A, B, C, D = nodeSet.nodes.values()
checkPoolReady(looper, nodeSet)
logging.debug("Check nomination")
# Checking whether Node A nominated itself
looper.run(eventually(checkNomination, A, A.name, retryWait=1, timeout=10))
# Checking whether Node B nominated itself
looper.run(eventually(checkNomination, B, B.name, retryWait=1, timeout=10))
for n in [C, D]:
# Checking whether Node C and Node D nominated Node A
looper.run(eventually(checkNomination, n, A.name, retryWait=1, timeout=10))
checkProtocolInstanceSetup(looper=looper, nodes=nodeSet,
retryWait=1, timeout=45)
# Node D should not be primary
assert not D.hasPrimary
# A should have at least one primary
assert A.hasPrimary
def testReplyWhenRepliesFromAllNodesAreSame(looper, client1):
"""
When there are not faulty nodes, the client must get a reply from all the
nodes.
"""
request = sendRandomRequest(client1)
looper.run(
eventually(assertLength, client1.inBox,
2 * nodeCount * request.reqId,
retryWait=.25, timeout=15))
checkResponseCorrectnessFromNodes(client1.inBox, request.reqId, F)
def testEveryNodeRepliesWithNoFaultyNodes(looper, client1, replied1):
"""
Every node will send a reply to the client when there are no faulty nodes in the system
"""
def chk():
receivedReplies = getRepliesFromClientInbox(client1.inBox,
replied1.reqId)
assert len(receivedReplies) == nodeCount
looper.run(eventually(chk))
def testBlacklistClient(setup, looper, nodeSet, up, client1, sent1):
"""
Client should be blacklisted by node on sending an unsigned request
"""
# Every node should blacklist the client
def chk():
for node in nodeSet:
assert node.isClientBlacklisted(client1.clientId)
looper.run(eventually(chk, retryWait=1, timeout=3))
def replied1(looper, nodeSet, client1, committed1):
for instId in range(getNoInstances(len(nodeSet))):
primaryReplica = getPrimaryReplica(nodeSet, instId)
looper.run(*[eventually(checkRequestReturnedToNode,
node,
client1.clientId,
committed1.reqId,
committed1.digest,
instId,
retryWait=1, timeout=30)
for node in nodeSet])
looper.run(eventually(
checkSufficientRepliesRecvd,
client1.inBox,
committed1.reqId,
2,
retryWait=2, timeout=30))
return committed1
def testDiscardInstChngMsgIfMasterDoesntSeePerformanceProblem(
nodeSet, looper, ensureView):
"""
A node that received an INSTANCE_CHANGE message must not send an
INSTANCE_CHANGE message if it doesn't observe too much difference in
performance between its replicas.
"""
curViewNo = ensureView
# Send an instance change message to all nodes
icMsg = InstanceChange(curViewNo)
nodeSet.Alpha.send(icMsg)
# ensure every node but Alpha discards the invalid instance change request
looper.run(eventually(checkDiscardMsg, nodeSet, icMsg,
'did not find the master to be slow', nodeSet.Alpha,
timeout=5))
# Check that that message is discarded.
looper.run(eventually(checkViewNoForNodes, nodeSet, timeout=3))
def checkEveryNodeHasAtMostOnePrimary(looper: Looper,
nodes: Sequence[TestNode],
retryWait: float = None,
timeout: float = None):
def checkAtMostOnePrim(node):
prims = [r for r in node.replicas if r.isPrimary]
assert len(prims) <= 1
for node in nodes:
looper.run(eventually(checkAtMostOnePrim,
node,
retryWait=retryWait,
timeout=timeout))
def testSelfNominationDelay(tdir_for_func):
nodeNames = ["testA", "testB", "testC", "testD"]
with TestNodeSet(names=nodeNames, tmpdir=tdir_for_func) as nodeSet:
with Looper(nodeSet) as looper:
prepareNodeSet(looper, nodeSet)
delay = 30
# Add node A
# nodeA = nodeSet.addNode(nodeNames[0], 0, AutoMode.never)
nodeA = addNodeBack(nodeSet, looper, nodeNames[0])
nodeA.delaySelfNomination(delay)
nodesBCD = []
for name in nodeNames[1:]:
# nodesBCD.append(nodeSet.addNode(name, i+1, AutoMode.never))
nodesBCD.append(addNodeBack(nodeSet, looper, name))
# Ensuring that NodeA is started before any other node to demonstrate that it is delaying
# self nomination
looper.run(
eventually(lambda: assertExp(nodeA.isReady()), retryWait=1,
timeout=5))
# Elections should be done
ensureElectionsDone(looper=looper, nodes=nodeSet, retryWait=1,
timeout=10)
# node A should not have any primary replica
looper.run(
eventually(lambda: assertExp(not nodeA.hasPrimary),
retryWait=1,
timeout=10))
# Make sure that after at the most 30 seconds, nodeA's `startElection` is called
looper.run(eventually(lambda: assertExp(
len(nodeA.spylog.getAll(
Node.decidePrimaries.__name__)) > 0),
retryWait=1, timeout=30))
def x():
requests = [sendRandomRequest(client) for _ in range(10)]
for request in requests:
looper.run(eventually(
checkSufficientRepliesRecvd, client.inBox,
request.reqId, 3,
retryWait=1, timeout=3 * len(nodeSet)))
ss2 = snapshotStats(*nodeSet)
diff = statsDiff(ss2, ss1)
# TODO verify the correct number of expected PROPAGATE, PRE-PREPARE, PREPARE, and COMMITs occurred
pprint(ss2)
print("----------------------------------------------")
pprint(diff)
def testDuplicateInstanceChangeMsgsMarkNodeAsSuspicious(looper, nodeSet, up):
maliciousNode = nodeSet.Alpha
maliciousNode.send(InstanceChange(0))
def chk(instId):
for node in nodeSet:
if node.name != maliciousNode.name:
param = getAllArgs(node, Node.processInstanceChange)
assert param[0]['instChg'].viewNo == instId
assert param[0]['frm'] == maliciousNode.name
looper.run(eventually(chk, 0, retryWait=1, timeout=20))
maliciousNode.send(InstanceChange(0))
def g():
for node in nodeSet:
if node.name != maliciousNode.name:
frm, reason, code = getAllArgs(node, Node.reportSuspiciousNode)
assert frm == maliciousNode.name
assert isinstance(reason, SuspiciousNode)
assert len(getNodeSuspicions(node,
Suspicions.DUPLICATE_INST_CHNG.code)) == 12
looper.run(eventually(g, retryWait=1, timeout=20))
def testPropagateRecvdAfterRequest(setup, looper, nodeSet, up, sent1):
A, B, C, D = nodeSet.nodes.values() # type: TestNode
def x():
# A should have received a request from the client
assert len(recvdRequest(A)) == 1
# A should not have received a PROPAGATE
assert len(recvdPropagate(A)) == 0
# A should have sent a PROPAGATE
assert len(sentPropagate(A)) == 1
looper.run(eventually(x, retryWait=.5, timeout=3))
def y():
# A should have received 3 PROPAGATEs
assert len(recvdPropagate(A)) == 3
# A should have total of 4 PROPAGATEs (3 from other nodes and 1 from
# itself)
key = sent1.clientId, sent1.reqId
assert len(A.requests[key].propagates) == 4
# A should still have sent only one PROPAGATE
assert len(sentPropagate(A)) == 1
looper.run(eventually(y, retryWait=.5, timeout=7))
def testReplyWhenRequestAlreadyExecuted(looper, nodeSet, client1, sent1):
"""
When a request has already been executed the previously executed reply
will be sent again to the client. An acknowledgement will not be sent
for a repeated request.
"""
# Since view no is always zero in the current setup
looper.run(eventually(checkSufficientRepliesRecvd,
client1.inBox,
sent1.reqId,
2,
retryWait=.25,
timeout=5))
orignalRquestResponsesLen = nodeCount * 2
duplicateRequestRepliesLen = nodeCount # for a duplicate request we need to send reply only not any ACK.
client1._enqueueIntoAllRemotes(sent1)
# Since view no is always zero in the current setup
looper.run(eventually(
lambda: assertLength([response for response in client1.inBox
if response[0]['reqId'] == sent1.reqId],
orignalRquestResponsesLen +
duplicateRequestRepliesLen),
retryWait=.25,
timeout=20))
def testPropagateRecvdBeforeRequest(setup, looper, nodeSet, up, sent1):
A, B, C, D = nodeSet.nodes.values()
def x():
# A should not have received a request from the client
assert len(recvdRequest(A)) == 0
# A should have received only one PROPAGATE
assert len(recvdPropagate(A)) == 1
# A should have sent only one PROPAGATE
assert len(sentPropagate(A)) == 1
looper.run(eventually(x, retryWait=.5, timeout=3))
def y():
# A should have received a request from the client
assert len(recvdRequest(A)) == 1
# A should still have sent only one PROPAGATE
assert len(sentPropagate(A)) == 1
looper.run(eventually(y, retryWait=.5, timeout=6))
# A should have forwarded the request
looper.run(eventually(assertLength, forwardedRequest(A), 1,
retryWait=.5, timeout=3))
def viewChangeDone(nodeSet, looper, up, client1):
"""
Test that a view change is done when the performance of master goes down
"""
# Delay processing of PRE-PREPARE from all non primary replicas of master
# so master's performance falls and view changes
nonPrimReps = getNonPrimaryReplicas(nodeSet, 0)
for r in nonPrimReps:
r.node.nodeIbStasher.delay(ppDelay(10, 0))
sendReqsToNodesAndVerifySuffReplies(looper, client1, 4)
looper.run(eventually(partial(checkViewNoForNodes, nodeSet, 1),
retryWait=1, timeout=20))
def testBlacklistNodeOnMultipleNominations(looper, keySharedNodes, ready):
"""
A node that sends multiple nominations must be blacklisted by other nodes
"""
nodeSet = keySharedNodes
A, B, C, D = nodeSet.nodes.values()
# B sends more than one nomination
B.send(Nomination(D.name, 0, B.viewNo))
B.send(Nomination(D.name, 0, B.viewNo))
# B should be blacklisted by A, C, D
def chk():
for node in A, C, D:
assert node.isNodeBlacklisted(B.name)
looper.run(eventually(chk, retryWait=1, timeout=3))
def testMsgFromInstanceDelay(configNodeSet, looper, prepared1):
A, B, C, D = configNodeSet.nodes.values()
def getCommits(node: TestNode, instId: int):
replica = node.replicas[instId] # type: Replica
return list(replica.commits.values())
def checkPresence():
for node in [C, D]:
commReqs = getCommits(node, 0)
assert len(commReqs) > 0
assert Replica.generateName(A.name, 0) not in commReqs[0][1]
commReqs = getCommits(node, 1)
assert len(commReqs) > 0
assert Replica.generateName(A.name, 1) in commReqs[0][1]
looper.run(eventually(checkPresence, retryWait=.5, timeout=10))
def checkEveryProtocolInstanceHasOnlyOnePrimary(looper: Looper,
nodes: Sequence[TestNode],
retryWait: float = None,
timeout: float = None):
coro = eventually(instances, nodes, retryWait=retryWait, timeout=timeout)
insts, timeConsumed = timeThis(looper.run, coro)
# TODO refactor this to just user eventuallyAll
newTimeout = timeout - timeConsumed if timeout is not None else None
for instId, replicas in insts.items():
logging.debug("Checking replicas in instance: {}".format(instId))
checkIfSameReplicaIPrimary(looper=looper,
replicas=replicas,
retryWait=retryWait,
timeout=newTimeout)
