def case5Setup(txnPoolNodeSet):
A, B, C, D = txnPoolNodeSet
# Node B delays self nomination so A's nomination reaches everyone
B.delaySelfNomination(30)
# Node B delays NOMINATE from Node A, B, C since it needs to send PRIMARY
# messages so it should not get any `NOMINATE` which might make it do
# Primary declarations much before we need it too
# A, C and D should not blacklist B since we are trying to check if
# multiple primary declarations from the same node have any impact on
# the election
for node in A, C, D:
node.whitelistNode(B.name, Suspicions.DUPLICATE_PRI_SENT.code)
for node in [A, C, D]:
B.nodeIbStasher.delay(delayerMsgTuple(delayOfElectionDone,
Nomination,
senderFilter=node.name,
instFilter=0))
for node in [C, D]:
# Nodes C and D delay NOMINATE from node A
node.nodeIbStasher.delay(delayerMsgTuple(5,
Nomination,
senderFilter=A.name,
instFilter=0))
# Also Nodes C and D are slow so they will not nominate themselves
node.delaySelfNomination(25)
def case5Setup(startedNodes: TestNodeSet):
A, B, C, D = startedNodes.nodes.values()
# Node B delays self nomination so A's nomination reaches everyone
B.delaySelfNomination(30)
# Node B delays NOMINATE from Node A, B, C since it needs to send PRIMARY
# messages so it should not get any `NOMINATE` which might make it do
# Primary declarations much before we need it too
# A, C and D should not blacklist B since we are trying to check if
# multiple primary declarations from the same node have any impact on
# the election
for node in A, C, D:
node.whitelistNode(B.name, Suspicions.DUPLICATE_PRI_SENT.code)
for node in [A, C, D]:
B.nodeIbStasher.delay(
delayerMsgTuple(delayOfElectionDone,
Nomination,
senderFilter=node.name,
instFilter=0))
for node in [C, D]:
# Nodes C and D delay NOMINATE from node A
node.nodeIbStasher.delay(
delayerMsgTuple(5, Nomination, senderFilter=A.name, instFilter=0))
# Also Nodes C and D are slow so they will not nominate themselves
node.delaySelfNomination(25)
def electContFixture(startedNodes: TestNodeSet):
A, B, C, D = startedNodes.nodes.values()
# Delaying nodeB' self nomination so nodeA's nomination can reach NodeC and NodeD
B.delaySelfNomination(2)
# For B delay nominate messages from A and C
for node in [A, C]:
B.nodeIbStasher.delay(delayerMsgTuple(10, Nomination, node.name))
for node in [C, D]:
B.nodeIbStasher.delay(delayerMsgTuple(5, Nomination, node.name))
node.delaySelfNomination(10)
def electContFixture(startedNodes: TestNodeSet):
A, B, C, D = startedNodes.nodes.values()
# Delaying nodeB' self nomination so nodeA's nomination can reach NodeC and NodeD
B.delaySelfNomination(2)
# For B delay nominate messages from A and C
for node in [A, C]:
B.nodeIbStasher.delay(delayerMsgTuple(10, Nomination, node.name))
for node in [C, D]:
B.nodeIbStasher.delay(delayerMsgTuple(5, Nomination, node.name))
node.delaySelfNomination(10)
def configNodeSet(txnPoolNodeSet):
A, B, C, D = txnPoolNodeSet
# Nodes C and D delay Commit request from node A for protocol instance 0
for n in [C, D]:
n.nodeIbStasher.delay(
delayerMsgTuple(30, Commit, senderFilter=A.name, instFilter=0))
return txnPoolNodeSet
def testTestNodeDelay(tdir_for_func):
nodeNames = {"testA", "testB"}
with TestNodeSet(names=nodeNames, tmpdir=tdir_for_func) as nodes:
nodeA = nodes.getNode("testA")
nodeB = nodes.getNode("testB")
with Looper(nodes) as looper:
for n in nodes:
n.startKeySharing()
logger.debug("connect")
looper.run(checkNodesConnected(nodes))
logger.debug("send one message, without delay")
msg = randomMsg()
looper.run(sendMsgAndCheck(nodes, nodeA, nodeB, msg, 1))
logger.debug("set delay, then send another message and find that "
"it doesn't arrive")
msg = randomMsg()
nodeB.nodeIbStasher.delay(delayerMsgTuple(6, type(msg),
nodeA.name))
with pytest.raises(AssertionError):
looper.run(sendMsgAndCheck(nodes, nodeA, nodeB, msg, 3))
logger.debug("but then find that it arrives after the delay "
"duration has passed")
looper.run(sendMsgAndCheck(nodes, nodeA, nodeB, msg, 4))
logger.debug(
"reset the delay, and find another message comes quickly")
nodeB.nodeIbStasher.resetDelays()
msg = randomMsg()
looper.run(sendMsgAndCheck(nodes, nodeA, nodeB, msg, 1))
def case1Setup(txnPoolNodeSet):
nodeB = txnPoolNodeSet[1]
# Node B delays self nomination so A's nomination reaches everyone
nodeB.delaySelfNomination(10)
# Node B delays nomination from all nodes
nodeB.nodeIbStasher.delay(delayerMsgTuple(delayOfNomination, Nomination))
# Add node C and node D
nodeC = txnPoolNodeSet[2]
nodeD = txnPoolNodeSet[3]
# Node C should not try to nominate itself until it gets NOMINATE from B
nodeC.delaySelfNomination(10)
# Node D should delay its self nomination for long as it is a slow node
# and should never win the
# primary election even if a node maliciously tries to make Node D primary
nodeD.delaySelfNomination(30)
# A, C and D should not blacklist B since we are trying to check if
# multiple nominations from the same node have any impact on
# the election
whitelistNode(nodeB.name,
[node for node in txnPoolNodeSet if node != nodeB],
Suspicions.DUPLICATE_NOM_SENT.code)
return txnPoolNodeSet
def case1Setup(startedNodes: TestNodeSet):
nodes = startedNodes
nodeNames = nodes.nodeNames
nodeB = nodes.getNode(nodeNames[1])
# Node B delays self nomination so A's nomination reaches everyone
nodeB.delaySelfNomination(10)
# Node B delays nomination from all nodes
nodeB.nodeIbStasher.delay(delayerMsgTuple(5, Nomination))
# Add node C and node D
nodeC = nodes.getNode(nodeNames[2])
nodeD = nodes.getNode(nodeNames[3])
# Node C should not try to nominate itself until it gets NOMINATE from B
nodeC.delaySelfNomination(10)
# Node D should delay its self nomination for long as it is a slow node
# and should never win the
# primary election even if a node maliciously tries to make Node D primary
nodeD.delaySelfNomination(30)
# A, C and D should not blacklist B since we are trying to check if
# multiple nominations from the same node have any impact on
# the election
whitelistNode(nodeB.name,
[node for node in nodes if node != nodeB],
Suspicions.DUPLICATE_NOM_SENT.code)
return nodes
def testTestNodeDelay(tdir_for_func):
nodeNames = {"testA", "testB"}
with TestNodeSet(names=nodeNames, tmpdir=tdir_for_func) as nodes:
nodeA = nodes.getNode("testA")
nodeB = nodes.getNode("testB")
with Looper(nodes) as looper:
looper.run(checkNodesConnected(nodes))
# send one message, without delay
looper.run(sendMessageAndCheckDelivery(nodes, nodeA, nodeB))
# set delay, then send another message
# and find that it doesn't arrive
delay = 5 * waits.expectedNodeToNodeMessageDeliveryTime()
nodeB.nodeIbStasher.delay(
delayerMsgTuple(delay, TestMsg, nodeA.name)
)
with pytest.raises(AssertionError):
looper.run(sendMessageAndCheckDelivery(nodes, nodeA, nodeB))
# but then find that it arrives after the delay
# duration has passed
timeout = waits.expectedNodeToNodeMessageDeliveryTime() + delay
looper.run(sendMessageAndCheckDelivery(nodes, nodeA, nodeB,
customTimeout=timeout))
# reset the delay, and find another message comes quickly
nodeB.nodeIbStasher.reset_delays_and_process_delayeds()
looper.run(sendMessageAndCheckDelivery(nodes, nodeA, nodeB))
def configNodeSet(nodeSet):
A, B, C, D = nodeSet.nodes.values()
# Nodes C and D delay Commit request from node A for protocol instance 0
for n in [C, D]:
n.nodeIbStasher.delay(delayerMsgTuple(30,
Commit,
senderFilter=A.name,
instFilter=0))
return nodeSet
def testRequestReturnToNodeWhenPrePrepareNotReceivedByOneNode(tdir_for_func):
"""Test no T-3"""
nodeNames = genNodeNames(7)
nodeReg = genNodeReg(names=nodeNames)
with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet:
with Looper(nodeSet) as looper:
prepareNodeSet(looper, nodeSet)
logger.debug("Add the seven nodes back in")
# Every node except A delays self nomination so A can become primary
nodeA = addNodeBack(nodeSet, looper, nodeNames[0])
for i in range(1, 7):
node = addNodeBack(nodeSet, looper, nodeNames[i])
node.delaySelfNomination(15)
nodeB = nodeSet.getNode(nodeNames[1])
# Node B delays PREPREPARE from node A(which would be the primary)
# for a long time.
nodeB.nodeIbStasher.delay(
delayerMsgTuple(120, PrePrepare, nodeA.name))
# Ensure elections are done
ensureElectionsDone(looper=looper,
nodes=nodeSet,
retryWait=1,
timeout=30)
assert nodeA.hasPrimary
instNo = nodeA.primaryReplicaNo
client1, wallet1 = setupClient(looper,
nodeSet,
tmpdir=tdir_for_func)
req = sendRandomRequest(wallet1, client1)
# All nodes including B should return their ordered requests
for node in nodeSet:
looper.run(
eventually(checkRequestReturnedToNode,
node,
wallet1.defaultId,
req.reqId,
instNo,
retryWait=1,
timeout=30))
# Node B should not have received the PRE-PREPARE request yet
replica = nodeB.replicas[instNo] # type: Replica
assert len(replica.prePrepares) == 0
def case2Setup(startedNodes: TestNodeSet):
A, B, C, D = startedNodes.nodes.values()
# Node B delays self nomination so A's nomination reaches everyone
B.delaySelfNomination(5)
# Node B delays nomination from all nodes
B.nodeIbStasher.delay(delayerMsgTuple(5, Nomination))
for node in [C, D]:
# Also Nodes C and D are slow so they will not nominate for themselves
# for long
node.delaySelfNomination(25)
# A, C and D should not blacklist B since we are trying to check if
# multiple nominations from the same node have any impact on
# the election
for node in A, C, D:
node.whitelistNode(B.name, Suspicions.DUPLICATE_NOM_SENT.code)
def case2Setup(startedNodes: TestNodeSet):
A, B, C, D = startedNodes.nodes.values()
# Node B delays self nomination so A's nomination reaches everyone
B.delaySelfNomination(5)
# Node B delays nomination from all nodes
B.nodeIbStasher.delay(delayerMsgTuple(5, Nomination))
for node in [C, D]:
# Also Nodes C and D are slow so they will not nominate for themselves
# for long
node.delaySelfNomination(25)
# A, C and D should not blacklist B since we are trying to check if
# multiple nominations from the same node have any impact on
# the election
for node in A, C, D:
node.whitelistNode(B.name, Suspicions.DUPLICATE_NOM_SENT.code)
def testRequestReturnToNodeWhenPrePrepareNotReceivedByOneNode(tdir_for_func):
"""Test no T-3"""
nodeNames = genNodeNames(7)
nodeReg = genNodeReg(names=nodeNames)
with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet:
with Looper(nodeSet) as looper:
prepareNodeSet(looper, nodeSet)
logger.debug("Add the seven nodes back in")
# Every node except A delays self nomination so A can become primary
nodeA = addNodeBack(nodeSet, looper, nodeNames[0])
for i in range(1, 7):
node = addNodeBack(nodeSet, looper, nodeNames[i])
node.delaySelfNomination(15)
nodeB = nodeSet.getNode(nodeNames[1])
# Node B delays PREPREPARE from node A(which would be the primary)
# for a long time.
nodeB.nodeIbStasher.delay(
delayerMsgTuple(120, PrePrepare, nodeA.name))
# Ensure elections are done
ensureElectionsDone(looper=looper, nodes=nodeSet, retryWait=1,
timeout=30)
assert nodeA.hasPrimary
instNo = nodeA.primaryReplicaNo
client1, wallet1 = setupClient(looper, nodeSet, tmpdir=tdir_for_func)
req = sendRandomRequest(wallet1, client1)
# All nodes including B should return their ordered requests
for node in nodeSet:
looper.run(eventually(checkRequestReturnedToNode, node,
wallet1.defaultId, req.reqId,
instNo, retryWait=1, timeout=30))
# Node B should not have received the PRE-PREPARE request yet
replica = nodeB.replicas[instNo] # type: Replica
assert len(replica.prePrepares) == 0
def testTestNodeDelay(looper, txnPoolNodeSet):
looper.run(checkNodesConnected(txnPoolNodeSet))
nodeA = txnPoolNodeSet[0]
nodeB = txnPoolNodeSet[1]
# send one message, without delay
looper.run(sendMessageAndCheckDelivery(nodeA, nodeB))
# set delay, then send another message
# and find that it doesn't arrive
delay = 5 * waits.expectedNodeToNodeMessageDeliveryTime()
nodeB.nodeIbStasher.delay(delayerMsgTuple(delay, TestMsg, nodeA.name))
with pytest.raises(AssertionError):
looper.run(sendMessageAndCheckDelivery(nodeA, nodeB))
# but then find that it arrives after the delay
# duration has passed
timeout = waits.expectedNodeToNodeMessageDeliveryTime() + delay
looper.run(sendMessageAndCheckDelivery(nodeA, nodeB,
customTimeout=timeout))
# reset the delay, and find another message comes quickly
nodeB.nodeIbStasher.reset_delays_and_process_delayeds()
looper.run(sendMessageAndCheckDelivery(nodeA, nodeB))
def testPrePrepareWhenPrimaryStatusIsUnknown(tdir_for_func):
nodeNames = genNodeNames(4)
nodeReg = genNodeReg(names=nodeNames)
with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet:
with Looper(nodeSet) as looper:
prepareNodeSet(looper, nodeSet)
nodeA, nodeB, nodeC, nodeD = tuple(
addNodeBack(nodeSet, looper, nodeNames[i])
for i in range(0, 4))
# Nodes C and D delays self nomination so A and B can become
# primaries
nodeC.delaySelfNomination(30)
nodeD.delaySelfNomination(30)
# Node D delays receiving PRIMARY messages from all nodes so it
# will not know whether it is primary or not
# nodeD.nodestack.delay(delayer(20, PRIMARY))
nodeD.nodeIbStasher.delay(delayerMsgTuple(20, Primary))
checkPoolReady(looper=looper, nodes=nodeSet)
client1, wal = setupClient(looper, nodeSet, tmpdir=tdir_for_func)
request = sendRandomRequest(wal, client1)
# TODO Rethink this
instNo = 0
for i in range(3):
node = nodeSet.getNode(nodeNames[i])
# Nodes A, B and C should have received PROPAGATE request
# from Node D
looper.run(
eventually(checkIfPropagateRecvdFromNode,
node,
nodeD,
request.identifier,
request.reqId,
retryWait=1,
timeout=10))
# Node D should have 1 pending PRE-PREPARE request
def assertOnePrePrepare():
assert len(
getPendingRequestsForReplica(nodeD.replicas[instNo],
PrePrepare)) == 1
looper.run(eventually(assertOnePrePrepare, retryWait=1,
timeout=10))
# Node D should have 2 pending PREPARE requests(from node B and C)
def assertTwoPrepare():
assert len(
getPendingRequestsForReplica(nodeD.replicas[instNo],
Prepare)) == 2
looper.run(eventually(assertTwoPrepare, retryWait=1, timeout=10))
# Node D should have no pending PRE-PREPARE, PREPARE or COMMIT
# requests
for reqType in [PrePrepare, Prepare, Commit]:
looper.run(
eventually(lambda: assertLength(
getPendingRequestsForReplica(nodeD.replicas[instNo],
reqType), 0),
retryWait=1,
timeout=20))
def testPrePrepareWhenPrimaryStatusIsUnknown(tdir_for_func):
nodeNames = genNodeNames(4)
nodeReg = genNodeReg(names=nodeNames)
with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet:
with Looper(nodeSet) as looper:
prepareNodeSet(looper, nodeSet)
nodeA, nodeB, nodeC, nodeD = tuple(
addNodeBack(nodeSet, looper, nodeNames[i]) for i in range(0, 4))
# Nodes C and D delays self nomination so A and B can become
# primaries
nodeC.delaySelfNomination(30)
nodeD.delaySelfNomination(30)
# Node D delays receiving PRIMARY messages from all nodes so it
# will not know whether it is primary or not
# nodeD.nodestack.delay(delayer(20, PRIMARY))
nodeD.nodeIbStasher.delay(delayerMsgTuple(20, Primary))
checkPoolReady(looper=looper, nodes=nodeSet)
client1, wal = setupClient(looper, nodeSet, tmpdir=tdir_for_func)
request = sendRandomRequest(wal, client1)
# TODO Rethink this
instNo = 0
for i in range(3):
node = nodeSet.getNode(nodeNames[i])
# Nodes A, B and C should have received PROPAGATE request
# from Node D
looper.run(
eventually(checkIfPropagateRecvdFromNode, node, nodeD,
request.identifier,
request.reqId, retryWait=1, timeout=10))
# Node D should have 1 pending PRE-PREPARE request
def assertOnePrePrepare():
assert len(getPendingRequestsForReplica(nodeD.replicas[instNo],
PrePrepare)) == 1
looper.run(eventually(assertOnePrePrepare, retryWait=1, timeout=10))
# Node D should have 2 pending PREPARE requests(from node B and C)
def assertTwoPrepare():
assert len(getPendingRequestsForReplica(nodeD.replicas[instNo],
Prepare)) == 2
looper.run(eventually(assertTwoPrepare, retryWait=1, timeout=10))
# Node D should have no pending PRE-PREPARE, PREPARE or COMMIT
# requests
for reqType in [PrePrepare, Prepare, Commit]:
looper.run(eventually(lambda: assertLength(
getPendingRequestsForReplica(nodeD.replicas[instNo],
reqType),
0), retryWait=1, timeout=20))
