def testPrimaryElectionCase2(case2Setup, looper, keySharedNodes):
"""
Case 2 - A node making nominations for a multiple other nodes. Consider 4
nodes A, B, C, and D. Lets say node B is malicious and nominates node C
to all nodes. Again node B nominates node D to all nodes.
"""
nodeSet = keySharedNodes
A, B, C, D = nodeSet.nodes.values()
looper.run(checkNodesConnected(nodeSet))
# Node B sends multiple NOMINATE msgs but only after A has nominated itself
looper.run(eventually(checkNomination, A, A.name, retryWait=.25, timeout=1))
instId = getSelfNominationByNode(A)
BRep = Replica.generateName(B.name, instId)
CRep = Replica.generateName(C.name, instId)
DRep = Replica.generateName(D.name, instId)
# Node B first sends NOMINATE msgs for Node C to all nodes
B.send(Nomination(CRep, instId, B.viewNo))
# Node B sends NOMINATE msgs for Node D to all nodes
B.send(Nomination(DRep, instId, B.viewNo))
# Ensure elections are done
ensureElectionsDone(looper=looper, nodes=nodeSet, retryWait=1, timeout=45)
# All nodes from node A, node C, node D(node B is malicious anyway so
# not considering it) should have nomination for node C from node B since
# node B first nominated node C
for node in [A, C, D]:
assert node.elector.nominations[instId][BRep] == CRep
def testPrimaryElectionCase4(case4Setup, looper):
"""
Case 4 - A node making multiple primary declarations for a particular node.
Consider 4 nodes A, B, C and D. Lets say node B is malicious and is
repeatedly declaring Node D as primary
"""
allNodes = case4Setup
A, B, C, D = allNodes
looper.run(checkNodesConnected(allNodes))
# Node B sends multiple declarations of node D's 0th protocol instance as
# primary to all nodes
for i in range(5):
B.send(Primary(D.name, 0, B.viewNo))
# No node from node A, node C, node D(node B is malicious anyway so not
# considering it) should have more than one primary declaration for node
# D since node D is slow. The one primary declaration for node D,
# that nodes A, C and D might have would be because of node B
def x():
primDecs = list(node.elector.primaryDeclarations[0].values())
assert primDecs.count(D.name) <= 1
for node in (A, C, D):
looper.run(eventually(x, retryWait=0.5, timeout=2))
ensureElectionsDone(looper=looper, nodes=allNodes, retryWait=1, timeout=45)
# Node D should not have any primary replica
assert not D.hasPrimary
def testAllNodesStartAtOnce(looper, tdir_for_func):
# Add Alpha
cli = Cli(looper, tdir_for_func, nodeReg, cliNodeReg)
# Add Alpha
cli.newNode('Alpha')
na = cli.nodes['Alpha'] # type: Node
# Add Beta
cli.newNode('Beta')
nb = cli.nodes['Beta'] # type: Node
# Add Gamma
cli.newNode('Gamma')
ng = cli.nodes['Gamma'] # type: Node
# Add Delta
cli.newNode('Delta')
nd = cli.nodes['Delta'] # type: Node
ensureElectionsDone(looper, [na, nb, ng, nd], retryWait=10, timeout=60)
# Add Client
cli.newClient('myclient')
# Send message to nodes
cli.sendMsg('myclient', '{"Me": "So cool"}')
client = cli.clients['myclient'] # type: Client
# Get all replies for node, corresponding
replies = [rep for rep in client.getRepliesFromAllNodes(1) if rep.reqId == 1]
assert len(replies) > 0
def testRequestReturnToNodeWhenPrePrepareNotReceivedByOneNode(tdir_for_func):
"""Test no T-3"""
nodeNames = genNodeNames(7)
nodeReg = genNodeReg(names=nodeNames)
with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet:
with Looper(nodeSet) as looper:
prepareNodeSet(looper, nodeSet)
logging.debug("Add the seven nodes back in")
# Every node except A delays self nomination so A can become primary
nodeA = addNodeBack(nodeSet, looper, nodeNames[0])
for i in range(1, 7):
node = addNodeBack(nodeSet, looper, nodeNames[i])
node.delaySelfNomination(15)
nodeB = nodeSet.getNode(nodeNames[1])
# Node B delays PREPREPARE from node A(which would be the primary) for a long time.
# TODO: Have a method to ignore a particular message from a node
nodeB.nodeIbStasher.delay(
delayerMsgTuple(120, PrePrepare, nodeA.name))
# Ensure elections are done
ensureElectionsDone(looper=looper, nodes=nodeSet, retryWait=1,
timeout=30)
assert nodeA.hasPrimary
instNo = nodeA.primaryReplicaNo
client1 = setupClient(looper, nodeSet, tmpdir=tdir_for_func)
req = sendRandomRequest(client1)
# All nodes including B should return their ordered requests
for node in nodeSet:
looper.run(eventually(checkRequestReturnedToNode, node,
client1.clientId, req.reqId,
req.digest,
instNo, retryWait=1, timeout=30))
# Node B should not have received the PRE-PREPARE request yet
replica = nodeB.replicas[instNo] # type: Replica
assert len(replica.prePrepares) == 0
def testSelfNominationDelay(tdir_for_func):
nodeNames = ["testA", "testB", "testC", "testD"]
with TestNodeSet(names=nodeNames, tmpdir=tdir_for_func) as nodeSet:
with Looper(nodeSet) as looper:
prepareNodeSet(looper, nodeSet)
delay = 30
# Add node A
# nodeA = nodeSet.addNode(nodeNames[0], 0, AutoMode.never)
nodeA = addNodeBack(nodeSet, looper, nodeNames[0])
nodeA.delaySelfNomination(delay)
nodesBCD = []
for name in nodeNames[1:]:
# nodesBCD.append(nodeSet.addNode(name, i+1, AutoMode.never))
nodesBCD.append(addNodeBack(nodeSet, looper, name))
# Ensuring that NodeA is started before any other node to demonstrate that it is delaying
# self nomination
looper.run(
eventually(lambda: assertExp(nodeA.isReady()), retryWait=1,
timeout=5))
# Elections should be done
ensureElectionsDone(looper=looper, nodes=nodeSet, retryWait=1,
timeout=10)
# node A should not have any primary replica
looper.run(
eventually(lambda: assertExp(not nodeA.hasPrimary),
retryWait=1,
timeout=10))
# Make sure that after at the most 30 seconds, nodeA's `startElection` is called
looper.run(eventually(lambda: assertExp(
len(nodeA.spylog.getAll(
Node.decidePrimaries.__name__)) > 0),
retryWait=1, timeout=30))
def up(looper, ready):
ensureElectionsDone(looper=looper, nodes=ready, retryWait=1, timeout=30)
