def electContFixture(startedNodes: TestNodeSet):
A, B, C, D = startedNodes.nodes.values()
# Delaying nodeB' self nomination so nodeA's nomination can reach NodeC and NodeD
B.delaySelfNomination(2)
# For B delay nominate messages from A and C
for node in [A, C]:
B.nodeIbStasher.delay(delayerMsgTuple(10, Nomination, node.name))
for node in [C, D]:
B.nodeIbStasher.delay(delayerMsgTuple(5, Nomination, node.name))
node.delaySelfNomination(10)
def testTestNodeDelay(tdir_for_func):
nodeNames = {"testA", "testB"}
with TestNodeSet(names=nodeNames, tmpdir=tdir_for_func) as nodes:
nodeA = nodes.getNode("testA")
nodeB = nodes.getNode("testB")
with Looper(nodes) as looper:
for n in nodes:
n.startKeySharing()
logging.debug("connect")
looper.run(checkNodesConnected(nodes))
logging.debug("send one message, without delay")
msg = randomMsg()
looper.run(sendMsgAndCheck(nodes, nodeA, nodeB, msg, 1))
logging.debug(
"set delay, then send another message and find that it doesn't arrive")
msg = randomMsg()
nodeB.nodeIbStasher.delay(delayerMsgTuple(4, type(msg), nodeA.name))
with pytest.raises(AssertionError):
looper.run(sendMsgAndCheck(nodes, nodeA, nodeB, msg, 3))
logging.debug(
"but then find that it arrives after the delay duration has passed")
looper.run(sendMsgAndCheck(nodes, nodeA, nodeB, msg, 2))
logging.debug(
"reset the delay, and find another message comes quickly")
nodeB.nodeIbStasher.resetDelays()
msg = randomMsg()
looper.run(sendMsgAndCheck(nodes, nodeA, nodeB, msg, 1))
def testPrePrepareWhenPrimaryStatusIsUnknown(tdir_for_func):
nodeNames = genNodeNames(4)
nodeReg = genNodeReg(names=nodeNames)
with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet:
with Looper(nodeSet) as looper:
prepareNodeSet(looper, nodeSet)
nodeA, nodeB, nodeC, nodeD = tuple(
addNodeBack(nodeSet, looper, nodeNames[i]) for i in range(0, 4))
# Nodes C and D delays self nomination so A and B can become primaries
nodeC.delaySelfNomination(30)
nodeD.delaySelfNomination(30)
# Node D delays receiving PRIMARY messages from all nodes so it will not know
# whether it is primary or not
# nodeD.nodestack.delay(delayer(20, PRIMARY))
nodeD.nodeIbStasher.delay(delayerMsgTuple(20, Primary))
checkPoolReady(looper=looper, nodes=nodeSet)
client1 = setupClient(looper, nodeSet, tmpdir=tdir_for_func)
request = sendRandomRequest(client1)
# TODO Rethink this
instNo = 0
for i in range(3):
node = nodeSet.getNode(nodeNames[i])
# Nodes A, B and C should have received PROPAGATE request from Node D
looper.run(
eventually(checkIfPropagateRecvdFromNode, node, nodeD,
request.clientId,
request.reqId, retryWait=1, timeout=10))
# Node D should have 1 pending PRE-PREPARE request
def assertOnePrePrepare():
assert len(getPendingRequestsForReplica(nodeD.replicas[instNo],
PrePrepare)) == 1
looper.run(eventually(assertOnePrePrepare, retryWait=1, timeout=10))
# Node D should have 2 pending PREPARE requests(from node B and C)
def assertTwoPrepare():
assert len(getPendingRequestsForReplica(nodeD.replicas[instNo],
Prepare)) == 2
looper.run(eventually(assertTwoPrepare, retryWait=1, timeout=10))
# Node D should have no pending PRE-PREPARE, PREPARE or COMMIT requests
for reqType in [PrePrepare, Prepare, Commit]:
looper.run(eventually(lambda: assertLength(
getPendingRequestsForReplica(nodeD.replicas[instNo],
reqType),
0), retryWait=1, timeout=20))
def configNodeSet(nodeSet):
A, B, C, D = nodeSet.nodes.values()
# Nodes C and D delay Commit request from node A for protocol instance 0
for n in [C, D]:
n.nodeIbStasher.delay(delayerMsgTuple(30,
Commit,
senderFilter=A.name,
instFilter=0))
return nodeSet
def case2Setup(startedNodes: TestNodeSet):
A, B, C, D = startedNodes.nodes.values()
# Node B delays self nomination so A's nomination reaches everyone
B.delaySelfNomination(5)
# Node B delays nomination from all nodes
B.nodeIbStasher.delay(delayerMsgTuple(5, Nomination))
for node in [C, D]:
# Also Nodes C and D are slow so they will not nominate for themselves
# for long
node.delaySelfNomination(25)
# A, C and D should not blacklist B since we are trying to check if
# multiple nominations from the same node have any impact on
# the election
for node in A, C, D:
node.whitelistNode(B.name, Suspicions.DUPLICATE_NOM_SENT.code)
def testRequestReturnToNodeWhenPrePrepareNotReceivedByOneNode(tdir_for_func):
"""Test no T-3"""
nodeNames = genNodeNames(7)
nodeReg = genNodeReg(names=nodeNames)
with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet:
with Looper(nodeSet) as looper:
prepareNodeSet(looper, nodeSet)
logging.debug("Add the seven nodes back in")
# Every node except A delays self nomination so A can become primary
nodeA = addNodeBack(nodeSet, looper, nodeNames[0])
for i in range(1, 7):
node = addNodeBack(nodeSet, looper, nodeNames[i])
node.delaySelfNomination(15)
nodeB = nodeSet.getNode(nodeNames[1])
# Node B delays PREPREPARE from node A(which would be the primary) for a long time.
# TODO: Have a method to ignore a particular message from a node
nodeB.nodeIbStasher.delay(
delayerMsgTuple(120, PrePrepare, nodeA.name))
# Ensure elections are done
ensureElectionsDone(looper=looper, nodes=nodeSet, retryWait=1,
timeout=30)
assert nodeA.hasPrimary
instNo = nodeA.primaryReplicaNo
client1 = setupClient(looper, nodeSet, tmpdir=tdir_for_func)
req = sendRandomRequest(client1)
# All nodes including B should return their ordered requests
for node in nodeSet:
looper.run(eventually(checkRequestReturnedToNode, node,
client1.clientId, req.reqId,
req.digest,
instNo, retryWait=1, timeout=30))
# Node B should not have received the PRE-PREPARE request yet
replica = nodeB.replicas[instNo] # type: Replica
assert len(replica.prePrepares) == 0