def testNodesConnectsWhenOneNodeIsLate():
with TemporaryDirectory() as td:
with Looper() as looper:
nodes = []
names = list(nodeReg.keys())
logger.debug("Node names: {}".format(names))
def create(name):
node = Node(name, nodeReg, basedirpath=td)
looper.add(node)
node.startKeySharing()
nodes.append(node)
for name in names[:3]:
create(name)
looper.run(checkNodesConnected(nodes))
# wait for the election to complete with the first three nodes
looper.runFor(10)
# create the fourth and see that it learns who the primaries are
# from the other nodes
create(names[3])
checkProtocolInstanceSetup(looper, nodes, timeout=10)
def testPrimaryElectionWithAClearWinner(electContFixture, looper, keySharedNodes):
"""
Primary selection (Sunny Day)
A, B, C, D, E
A, B, C, D startup. E is lagging.
A sees the minimum number of nodes first, and then sends out a NOMINATE(A) message
B, C, D all see the NOMINATE(A) message from A, and respond with NOMINATE(A) message to all other nodes
A sees three other NOMINATE(A) votes (from B, C, D)
A sees that A is the clear winner (2f+1 total), and sends PRIMARY(A) to all nodes
B sees two more NOMINATE(A) votes (from C and D)
B sees that A is the clear winner (2f+1 total), and sends PRIMARY(A) to all nodes
C sees two more NOMINATE(A) votes (from B and D)
C sees that A is the clear winner (2f+1 total), and sends PRIMARY(A) to all nodes
D sees two more NOMINATE(A) votes (from B and C)
D sees that A is the clear winner (2f+1 total), and sends PRIMARY(A) to all nodes
A sees at least two other PRIMARY(A) votes (3 including it's own)
selects A as primary
B sees at least two other PRIMARY(A) votes (3 including it's own)
selects A as primary
C sees at least two other PRIMARY(A) votes (3 including it's own)
selects A as primary
D sees at least two other PRIMARY(A) votes (3 including it's own)
selects A as primary
"""
nodeSet = keySharedNodes
A, B, C, D = nodeSet.nodes.values()
nodesBCD = [B, C, D]
# attempting to use a raet stack delay... not successful
# nodeBPort = nodesBCD[0].stack.ha[1]
# delayRef = RaetDelay(TrnsKind.alive, PcktKind.ack, nodeBPort)
# nodeA.stack.delay(4, delayRef)
checkPoolReady(looper, nodeSet)
# Checking whether one of the replicas of Node A nominated itself
looper.run(eventually(checkNomination, A, A.name, retryWait=1, timeout=10))
for n in nodesBCD:
# Checking whether Node B, C and D nominated Node A
looper.run(eventually(checkNomination, n, A.name, retryWait=1,
timeout=10))
checkProtocolInstanceSetup(looper=looper, nodes=nodeSet, retryWait=1,
timeout=10)
assert A.hasPrimary
def testPrimarySelectionAfterViewChange(looper, nodeSet, ready, primaryReplicas,
viewChangeDone):
"""
Test that primary replica of a protocol instance shifts to a new node after a view change.
"""
# Primary replicas before view change
prBeforeVC = primaryReplicas
# Primary replicas after view change
instanceCount = getNoInstances(nodeCount)
prAfterVC = [getPrimaryReplica(nodeSet, i) for i in range(instanceCount)]
# Primary replicas have moved to the next node
for br, ar in zip(prBeforeVC, prAfterVC):
assert ar.node.rank - br.node.rank == 1
checkProtocolInstanceSetup(looper, nodeSet, retryWait=1, timeout=5)
def testPrimaryElectionContested(electContFixture, looper, keySharedNodes):
"""
Primary selection (Rainy Day)
A, B, C, D, E
A, B, C, D startup. E is lagging.
A sees the minimum number of nodes, and then sends Nominate(A)
At the same exact time, B sees the minimum number of nodes, and then sends out Nominate(B)
A sees B sending Nominate(B), but it has already nominated itself, so it does nothing
B sees A sending Nominate(A), but it has already nominated itself, so it does nothing
C sees A sending Nominate(A), and sends Nominate(A)
D sees A sending Nominate(A), and sends Nominate(A)
All nodes see that B nominated B and A, C, and D all nominated A
Because the votes for A exceeds the votes for B, all send out Primary(A)
TODO's (see below)
All see the others have sent Primary A, and then the nodes record who is the Primary.
"""
# TODO what if not all send out Primary(A)?
# TODO what if there are big delays in messages getting delivered?
nodeSet = keySharedNodes
A, B, C, D = nodeSet.nodes.values()
checkPoolReady(looper, nodeSet)
logging.debug("Check nomination")
# Checking whether Node A nominated itself
looper.run(eventually(checkNomination, A, A.name, retryWait=1, timeout=10))
# Checking whether Node B nominated itself
looper.run(eventually(checkNomination, B, B.name, retryWait=1, timeout=10))
for n in [C, D]:
# Checking whether Node C and Node D nominated Node A
looper.run(eventually(checkNomination, n, A.name, retryWait=1, timeout=10))
checkProtocolInstanceSetup(looper=looper, nodes=nodeSet,
retryWait=1, timeout=45)
# Node D should not be primary
assert not D.hasPrimary
# A should have at least one primary
assert A.hasPrimary
def testElectionsAfterViewChange(delayedPerf, looper: Looper, nodeSet: TestNodeSet, up, client1):
"""
Test that a primary election does happen after a view change
"""
# Delay processing of PRE-PREPARE from all non primary replicas of master so master's throughput falls
# and view changes
nonPrimReps = getNonPrimaryReplicas(nodeSet, 0)
for r in nonPrimReps:
r.node.nodeIbStasher.delay(ppDelay(10, 0))
sendReqsToNodesAndVerifySuffReplies(looper, client1, 4)
# Ensure view change happened for both node and its primary elector
for node in nodeSet:
looper.run(eventually(partial(checkViewChangeInitiatedForNode, node, 0), retryWait=1, timeout=20))
# Ensure elections are done again and pool is setup again with appropriate protocol instances and each
# protocol instance is setup properly too
checkProtocolInstanceSetup(looper, nodeSet, retryWait=1, timeout=30)
def testPrimarySelectionAfterPoolReady(looper, nodeSet, ready):
"""
Once the pool is ready(node has connected to at least 3 other nodes), appropriate primary
replicas should be selected.
"""
def checkPrimaryPlacement():
# Node names sorted by rank
sortedNodeNames = sorted(nodeSet.nodes.values(),
key=operator.attrgetter("rank"))
for idx, node in enumerate(sortedNodeNames):
# For instance 0, the primary replica should be on the node with rank 0
if idx == 0:
primaryName = Replica.generateName(sortedNodeNames[idx], 0)
assert node.replicas[0].isPrimary
assert not node.replicas[1].isPrimary
assert not node.replicas[2].isPrimary
# For instance 1, the primary replica should be on the node with rank 1
if idx == 1:
primaryName = Replica.generateName(sortedNodeNames[idx], 1)
assert not node.replicas[0].isPrimary
assert node.replicas[1].isPrimary
assert not node.replicas[2].isPrimary
# For instance 2, the primary replica should be on the node with rank 2
if idx == 2:
primaryName = Replica.generateName(sortedNodeNames[idx], 2)
assert not node.replicas[0].isPrimary
assert not node.replicas[1].isPrimary
assert node.replicas[2].isPrimary
# Check if the primary is on the correct node
looper.run(eventually(checkPrimaryPlacement, retryWait=1, timeout=10))
# Check if every protocol instance has one and only one primary and any node
# has no more than one primary
checkProtocolInstanceSetup(looper, nodeSet, retryWait=1, timeout=5)
def testPrimaryElectionWithTie(electTieFixture, looper, keySharedNodes):
"""
Primary selection (Rainy Day)
A, B, C, D, E
A, B, C, D startup. E is lagging.
A sees the minimum number of nodes, and then sends Nominate(A)
At the same exact time, B sees the minimum number of nodes, and then sends out Nominate(B)
A sees B sending Nominate(B), but it has already nominated itself, so it does nothing
B sees A sending Nominate(A), but it has already nominated itself, so it does nothing
C sees A sending Nominate(A), and sends Nominate(A)
D sees B sending Nominate(B), and sends Nominate(B)
There's a split. C and A think A is the primary, B and D think B is the primary
All nodes can see that there is a split. Each sends out Reelection([A,B])
A and B both see Reelection([A,B]) from themselves as well as the other 3 (the number from others should be at least f+1),
1. they wait a random amount of time (between 0 and 2 seconds),
2. they each send out a Nominate(self)
Voting is repeated until we have a good election.
"""
# TODO optimize the sending messages in batches, for example, we don't
# send messages more often than 400 milliseconds. Once those 400
# millis have passed, we send the several queued messages in one
# batch.
nodeSet = keySharedNodes
A, B, C, D = nodeSet.nodes.values()
checkPoolReady(looper, nodeSet.nodes.values())
for node in nodeSet.nodes.values():
for instId, replica in enumerate(node.elector.replicas):
logging.debug("replica {} {} with votes {}".
format(replica.name, replica.instId,
node.elector.nominations.get(instId, {})))
logging.debug("Check nomination")
# Checking whether Node A nominated itself
looper.run(eventually(checkNomination, A, A.name, retryWait=1, timeout=10))
# Checking whether Node B nominated itself
looper.run(eventually(checkNomination, B, B.name, retryWait=1, timeout=10))
# Checking whether Node C nominated Node A
looper.run(eventually(checkNomination, C, A.name, retryWait=1, timeout=10))
# Checking whether Node D nominated Node D
looper.run(eventually(checkNomination, D, B.name, retryWait=1, timeout=10))
# No node should be primary
for node in nodeSet.nodes.values():
assert node.hasPrimary is False
for node in nodeSet.nodes.values():
node.resetDelays()
# TODO Check for spylog of `eatReelection` after putting spy on eaters too
checkProtocolInstanceSetup(looper=looper, nodes=nodeSet, retryWait=1,
timeout=60)
# TODO testPrimaryElectionWithTieButOneNodeDoesntKnowIt
# TODO add E back in after election is complete, or even before it's
# complete (E jumps in right in the middle of an election)
# When E joins, it needs to be able to ping the others for the
# current state (A could send to E the votes of the other nodes,
# so E would know if B is being maliciious), and the others respond,
# and E can make a determination based on those responses, even if
# one is malicious.
# TODO We need to trap for the case when a node is being inconsistent. For example:
"""
def pool(looper, nodeSet):
for n in nodeSet: # type: TestNode
n.startKeySharing()
looper.run(checkNodesConnected(nodeSet))
checkProtocolInstanceSetup(looper, nodeSet, timeout=5)
return adict(looper=looper, nodeset=nodeSet)
