def testNodesConnectsWhenOneNodeIsLate():
with TemporaryDirectory() as td:
with Looper() as looper:
nodes = []
names = list(nodeReg.keys())
logger.debug("Node names: {}".format(names))
def create(name):
node = Node(name, nodeReg, basedirpath=td)
looper.add(node)
node.startKeySharing()
nodes.append(node)
for name in names[:3]:
create(name)
looper.run(checkNodesConnected(nodes))
# wait for the election to complete with the first three nodes
looper.runFor(10)
# create the fourth and see that it learns who the primaries are
# from the other nodes
create(names[3])
checkProtocolInstanceSetup(looper, nodes, timeout=10)
def testPrimaryElectionWithTie(electTieFixture, looper, keySharedNodes):
"""
Primary selection (Rainy Day)
A, B, C, D, E
A, B, C, D startup. E is lagging.
A sees the minimum number of nodes, and then sends Nominate(A)
At the same exact time, B sees the minimum number of nodes, and then sends out Nominate(B)
A sees B sending Nominate(B), but it has already nominated itself, so it does nothing
B sees A sending Nominate(A), but it has already nominated itself, so it does nothing
C sees A sending Nominate(A), and sends Nominate(A)
D sees B sending Nominate(B), and sends Nominate(B)
There's a split. C and A think A is the primary, B and D think B is the primary
All nodes can see that there is a split. Each sends out Reelection([A,B])
A and B both see Reelection([A,B]) from themselves as well as the other 3 (the number from others should be at least f+1),
1. they wait a random amount of time (between 0 and 2 seconds),
2. they each send out a Nominate(self)
Voting is repeated until we have a good election.
"""
# TODO optimize the sending messages in batches, for example, we don't
# send messages more often than 400 milliseconds. Once those 400
# millis have passed, we send the several queued messages in one
# batch.
nodeSet = keySharedNodes
A, B, C, D = nodeSet.nodes.values()
checkPoolReady(looper, nodeSet.nodes.values())
for node in nodeSet.nodes.values():
for instId, replica in enumerate(node.elector.replicas):
logging.debug("replica {} {} with votes {}".
format(replica.name, replica.instId,
node.elector.nominations.get(instId, {})))
logging.debug("Check nomination")
# Checking whether Node A nominated itself
looper.run(eventually(checkNomination, A, A.name, retryWait=1, timeout=10))
# Checking whether Node B nominated itself
looper.run(eventually(checkNomination, B, B.name, retryWait=1, timeout=10))
# Checking whether Node C nominated Node A
looper.run(eventually(checkNomination, C, A.name, retryWait=1, timeout=10))
# Checking whether Node D nominated Node D
looper.run(eventually(checkNomination, D, B.name, retryWait=1, timeout=10))
# No node should be primary
for node in nodeSet.nodes.values():
assert node.hasPrimary is False
for node in nodeSet.nodes.values():
node.resetDelays()
checkProtocolInstanceSetup(looper=looper, nodes=nodeSet, retryWait=1,
timeout=60)
def testNodesConnectsWhenOneNodeIsLate(allPluginsPath, tdirAndLooper):
tdir, looper = tdirAndLooper
nodes = []
names = list(nodeReg.keys())
logger.debug("Node names: {}".format(names))
def create(name):
node = TestNode(name, nodeReg, basedirpath=tdir,
pluginPaths=allPluginsPath)
looper.add(node)
node.startKeySharing()
nodes.append(node)
for name in names[:3]:
create(name)
looper.run(checkNodesConnected(nodes))
# wait for the election to complete with the first three nodes
looper.runFor(10)
# create the fourth and see that it learns who the primaries are
# from the other nodes
create(names[3])
checkProtocolInstanceSetup(looper, nodes, timeout=10)
def testElectionsAfterViewChange(delayedPerf, looper: Looper,
nodeSet: TestNodeSet, up, client1):
"""
Test that a primary election does happen after a view change
"""
# Delay processing of PRE-PREPARE from all non primary replicas of master
# so master's throughput falls
# and view changes
nonPrimReps = getNonPrimaryReplicas(nodeSet, 0)
for r in nonPrimReps:
r.node.nodeIbStasher.delay(ppDelay(10, 0))
sendReqsToNodesAndVerifySuffReplies(looper, client1, 4)
# Ensure view change happened for both node and its primary elector
for node in nodeSet:
looper.run(
eventually(partial(checkViewChangeInitiatedForNode, node, 0),
retryWait=1,
timeout=20))
# Ensure elections are done again and pool is setup again with appropriate
# protocol instances and each protocol instance is setup properly too
checkProtocolInstanceSetup(looper, nodeSet, retryWait=1, timeout=30)
def testPrimaryElectionWithAClearWinner(electContFixture, looper,
keySharedNodes):
"""
Primary selection (Sunny Day)
A, B, C, D, E
A, B, C, D startup. E is lagging.
A sees the minimum number of nodes first, and then sends out a NOMINATE(A) message
B, C, D all see the NOMINATE(A) message from A, and respond with NOMINATE(A) message to all other nodes
A sees three other NOMINATE(A) votes (from B, C, D)
A sees that A is the clear winner (2f+1 total), and sends PRIMARY(A) to all nodes
B sees two more NOMINATE(A) votes (from C and D)
B sees that A is the clear winner (2f+1 total), and sends PRIMARY(A) to all nodes
C sees two more NOMINATE(A) votes (from B and D)
C sees that A is the clear winner (2f+1 total), and sends PRIMARY(A) to all nodes
D sees two more NOMINATE(A) votes (from B and C)
D sees that A is the clear winner (2f+1 total), and sends PRIMARY(A) to all nodes
A sees at least two other PRIMARY(A) votes (3 including it's own)
selects A as primary
B sees at least two other PRIMARY(A) votes (3 including it's own)
selects A as primary
C sees at least two other PRIMARY(A) votes (3 including it's own)
selects A as primary
D sees at least two other PRIMARY(A) votes (3 including it's own)
selects A as primary
"""
nodeSet = keySharedNodes
A, B, C, D = nodeSet.nodes.values()
nodesBCD = [B, C, D]
checkPoolReady(looper, nodeSet)
# Checking whether one of the replicas of Node A nominated itself
looper.run(eventually(checkNomination, A, A.name, retryWait=1, timeout=10))
for n in nodesBCD:
# Checking whether Node B, C and D nominated Node A
looper.run(
eventually(checkNomination, n, A.name, retryWait=1, timeout=10))
checkProtocolInstanceSetup(looper=looper,
nodes=nodeSet,
retryWait=1,
timeout=10)
assert A.hasPrimary
def testPrimaryElectionWithAClearWinner(electContFixture, looper, keySharedNodes):
"""
Primary selection (Sunny Day)
A, B, C, D, E
A, B, C, D startup. E is lagging.
A sees the minimum number of nodes first, and then sends out a NOMINATE(A) message
B, C, D all see the NOMINATE(A) message from A, and respond with NOMINATE(A) message to all other nodes
A sees three other NOMINATE(A) votes (from B, C, D)
A sees that A is the clear winner (2f+1 total), and sends PRIMARY(A) to all nodes
B sees two more NOMINATE(A) votes (from C and D)
B sees that A is the clear winner (2f+1 total), and sends PRIMARY(A) to all nodes
C sees two more NOMINATE(A) votes (from B and D)
C sees that A is the clear winner (2f+1 total), and sends PRIMARY(A) to all nodes
D sees two more NOMINATE(A) votes (from B and C)
D sees that A is the clear winner (2f+1 total), and sends PRIMARY(A) to all nodes
A sees at least two other PRIMARY(A) votes (3 including it's own)
selects A as primary
B sees at least two other PRIMARY(A) votes (3 including it's own)
selects A as primary
C sees at least two other PRIMARY(A) votes (3 including it's own)
selects A as primary
D sees at least two other PRIMARY(A) votes (3 including it's own)
selects A as primary
"""
nodeSet = keySharedNodes
A, B, C, D = nodeSet.nodes.values()
nodesBCD = [B, C, D]
checkPoolReady(looper, nodeSet)
# Checking whether one of the replicas of Node A nominated itself
looper.run(eventually(checkNomination, A, A.name, retryWait=1, timeout=10))
for n in nodesBCD:
# Checking whether Node B, C and D nominated Node A
looper.run(eventually(checkNomination, n, A.name, retryWait=1,
timeout=10))
checkProtocolInstanceSetup(looper=looper, nodes=nodeSet, retryWait=1,
timeout=10)
assert A.hasPrimary
def testPrimarySelectionAfterViewChange(looper, nodeSet, ready, primaryReplicas,
viewChangeDone):
"""
Test that primary replica of a protocol instance shifts to a new node after a view change.
"""
# Primary replicas before view change
prBeforeVC = primaryReplicas
# Primary replicas after view change
instanceCount = getNoInstances(nodeCount)
prAfterVC = [getPrimaryReplica(nodeSet, i) for i in range(instanceCount)]
# Primary replicas have moved to the next node
for br, ar in zip(prBeforeVC, prAfterVC):
assert ar.node.rank - br.node.rank == 1
checkProtocolInstanceSetup(looper, nodeSet, retryWait=1, timeout=5)
def testPrimaryElectionContested(electContFixture, looper, keySharedNodes):
"""
Primary selection (Rainy Day)
A, B, C, D, E
A, B, C, D startup. E is lagging.
A sees the minimum number of nodes, and then sends Nominate(A)
At the same exact time, B sees the minimum number of nodes, and then sends out Nominate(B)
A sees B sending Nominate(B), but it has already nominated itself, so it does nothing
B sees A sending Nominate(A), but it has already nominated itself, so it does nothing
C sees A sending Nominate(A), and sends Nominate(A)
D sees A sending Nominate(A), and sends Nominate(A)
All nodes see that B nominated B and A, C, and D all nominated A
Because the votes for A exceeds the votes for B, all send out Primary(A)
TODO's (see below)
All see the others have sent Primary A, and then the nodes record who is the Primary.
"""
nodeSet = keySharedNodes
A, B, C, D = nodeSet.nodes.values()
checkPoolReady(looper, nodeSet)
logger.debug("Check nomination")
# Checking whether Node A nominated itself
looper.run(eventually(checkNomination, A, A.name, retryWait=1, timeout=10))
# Checking whether Node B nominated itself
looper.run(eventually(checkNomination, B, B.name, retryWait=1, timeout=10))
for n in [C, D]:
# Checking whether Node C and Node D nominated Node A
looper.run(
eventually(checkNomination, n, A.name, retryWait=1, timeout=10))
checkProtocolInstanceSetup(looper=looper,
nodes=nodeSet,
retryWait=1,
timeout=45)
# Node D should not be primary
assert not D.hasPrimary
# A should have at least one primary
assert A.hasPrimary
def testPrimaryElectionContested(electContFixture, looper, keySharedNodes):
"""
Primary selection (Rainy Day)
A, B, C, D, E
A, B, C, D startup. E is lagging.
A sees the minimum number of nodes, and then sends Nominate(A)
At the same exact time, B sees the minimum number of nodes, and then sends out Nominate(B)
A sees B sending Nominate(B), but it has already nominated itself, so it does nothing
B sees A sending Nominate(A), but it has already nominated itself, so it does nothing
C sees A sending Nominate(A), and sends Nominate(A)
D sees A sending Nominate(A), and sends Nominate(A)
All nodes see that B nominated B and A, C, and D all nominated A
Because the votes for A exceeds the votes for B, all send out Primary(A)
TODO's (see below)
All see the others have sent Primary A, and then the nodes record who is the Primary.
"""
nodeSet = keySharedNodes
A, B, C, D = nodeSet.nodes.values()
checkPoolReady(looper, nodeSet)
logging.debug("Check nomination")
# Checking whether Node A nominated itself
looper.run(eventually(checkNomination, A, A.name, retryWait=1, timeout=10))
# Checking whether Node B nominated itself
looper.run(eventually(checkNomination, B, B.name, retryWait=1, timeout=10))
for n in [C, D]:
# Checking whether Node C and Node D nominated Node A
looper.run(eventually(checkNomination, n, A.name, retryWait=1, timeout=10))
checkProtocolInstanceSetup(looper=looper, nodes=nodeSet,
retryWait=1, timeout=45)
# Node D should not be primary
assert not D.hasPrimary
# A should have at least one primary
assert A.hasPrimary
def testElectionsAfterViewChange(delayedPerf, looper: Looper, nodeSet: TestNodeSet, up, client1):
"""
Test that a primary election does happen after a view change
"""
# Delay processing of PRE-PREPARE from all non primary replicas of master
# so master's throughput falls
# and view changes
nonPrimReps = getNonPrimaryReplicas(nodeSet, 0)
for r in nonPrimReps:
r.node.nodeIbStasher.delay(ppDelay(10, 0))
sendReqsToNodesAndVerifySuffReplies(looper, client1, 4)
# Ensure view change happened for both node and its primary elector
for node in nodeSet:
looper.run(eventually(partial(checkViewChangeInitiatedForNode, node, 0),
retryWait=1, timeout=20))
# Ensure elections are done again and pool is setup again with appropriate
# protocol instances and each protocol instance is setup properly too
checkProtocolInstanceSetup(looper, nodeSet, retryWait=1, timeout=30)
def testPrimarySelectionAfterPoolReady(looper, nodeSet, ready):
"""
Once the pool is ready(node has connected to at least 3 other nodes),
appropriate primary replicas should be selected.
"""
def checkPrimaryPlacement():
# Node names sorted by rank
sortedNodeNames = sorted(nodeSet.nodes.values(),
key=operator.attrgetter("rank"))
for idx, node in enumerate(sortedNodeNames):
# For instance 0, the primary replica should be on the node with rank 0
if idx == 0:
primaryName = Replica.generateName(sortedNodeNames[idx], 0)
assert node.replicas[0].isPrimary
assert not node.replicas[1].isPrimary
assert not node.replicas[2].isPrimary
# For instance 1, the primary replica should be on the node with rank 1
if idx == 1:
primaryName = Replica.generateName(sortedNodeNames[idx], 1)
assert not node.replicas[0].isPrimary
assert node.replicas[1].isPrimary
assert not node.replicas[2].isPrimary
# For instance 2, the primary replica should be on the node with rank 2
if idx == 2:
primaryName = Replica.generateName(sortedNodeNames[idx], 2)
assert not node.replicas[0].isPrimary
assert not node.replicas[1].isPrimary
assert node.replicas[2].isPrimary
# Check if the primary is on the correct node
looper.run(eventually(checkPrimaryPlacement, retryWait=1, timeout=10))
# Check if every protocol instance has one and only one primary and any node
# has no more than one primary
checkProtocolInstanceSetup(looper, nodeSet, retryWait=1, timeout=5)
def pool(looper, nodeSet):
for n in nodeSet: # type: TestNode
n.startKeySharing()
looper.run(checkNodesConnected(nodeSet))
checkProtocolInstanceSetup(looper, nodeSet, timeout=5)
return adict(looper=looper, nodeset=nodeSet)
def pool(looper, nodeSet):
for n in nodeSet: # type: TestNode
n.startKeySharing()
looper.run(checkNodesConnected(nodeSet))
checkProtocolInstanceSetup(looper, nodeSet, timeout=5)
return adict(looper=looper, nodeset=nodeSet)
def testPrimaryElectionWithTie(electTieFixture, looper, keySharedNodes):
"""
Primary selection (Rainy Day)
A, B, C, D, E
A, B, C, D startup. E is lagging.
A sees the minimum number of nodes, and then sends Nominate(A)
At the same exact time, B sees the minimum number of nodes, and then sends out Nominate(B)
A sees B sending Nominate(B), but it has already nominated itself, so it does nothing
B sees A sending Nominate(A), but it has already nominated itself, so it does nothing
C sees A sending Nominate(A), and sends Nominate(A)
D sees B sending Nominate(B), and sends Nominate(B)
There's a split. C and A think A is the primary, B and D think B is the primary
All nodes can see that there is a split. Each sends out Reelection([A,B])
A and B both see Reelection([A,B]) from themselves as well as the other 3 (the number from others should be at least f+1),
1. they wait a random amount of time (between 0 and 2 seconds),
2. they each send out a Nominate(self)
Voting is repeated until we have a good election.
"""
# TODO optimize the sending messages in batches, for example, we don't
# send messages more often than 400 milliseconds. Once those 400
# millis have passed, we send the several queued messages in one
# batch.
nodeSet = keySharedNodes
A, B, C, D = nodeSet.nodes.values()
checkPoolReady(looper, nodeSet.nodes.values())
for node in nodeSet.nodes.values():
for instId, replica in enumerate(node.elector.replicas):
logging.debug("replica {} {} with votes {}".format(
replica.name, replica.instId,
node.elector.nominations.get(instId, {})))
logging.debug("Check nomination")
# Checking whether Node A nominated itself
looper.run(eventually(checkNomination, A, A.name, retryWait=1, timeout=10))
# Checking whether Node B nominated itself
looper.run(eventually(checkNomination, B, B.name, retryWait=1, timeout=10))
# Checking whether Node C nominated Node A
looper.run(eventually(checkNomination, C, A.name, retryWait=1, timeout=10))
# Checking whether Node D nominated Node D
looper.run(eventually(checkNomination, D, B.name, retryWait=1, timeout=10))
# No node should be primary
for node in nodeSet.nodes.values():
assert node.hasPrimary is False
for node in nodeSet.nodes.values():
node.resetDelays()
checkProtocolInstanceSetup(looper=looper,
nodes=nodeSet,
retryWait=1,
timeout=60)
def testNodeDiscardMessageFromUnknownView(txnPoolNodeSet,
nodeSetWithNodeAddedAfterSomeTxns,
newNodeCaughtUp, tdirWithPoolTxns,
tconf, allPluginsPath):
"""
Node discards 3-phase and election messages from view nos that it does not
know of (view nos before it joined the pool)
:return:
"""
looper, nodeX, client, wallet, _, _ = nodeSetWithNodeAddedAfterSomeTxns
viewNo = nodeX.viewNo
# Delay processing of PRE-PREPARE from all non primary replicas of master
# so master's performance falls and view changes
delayNonPrimaries(txnPoolNodeSet, 0, 10)
sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, 4)
looper.run(
eventually(partial(checkViewNoForNodes, txnPoolNodeSet, viewNo + 1),
retryWait=1,
timeout=20))
newStewardName = "testClientSteward" + randomString(3)
nodeName = "Theta"
_, _, nodeTheta = addNewStewardAndNode(looper, client, wallet,
newStewardName, nodeName,
tdirWithPoolTxns, tconf,
allPluginsPath)
txnPoolNodeSet.append(nodeTheta)
looper.run(checkNodesConnected(txnPoolNodeSet))
looper.run(client.ensureConnectedToNodes())
looper.run(
eventually(checkNodeLedgersForEquality,
nodeTheta,
*txnPoolNodeSet[:-1],
retryWait=1,
timeout=5))
checkProtocolInstanceSetup(looper, txnPoolNodeSet, retryWait=1, timeout=10)
electMsg = Nomination(nodeX.name, 0, viewNo)
threePMsg = PrePrepare(0, viewNo, 10, wallet.defaultId,
wallet._getIdData().lastReqId + 1, "random digest",
time.time())
ridTheta = nodeX.nodestack.getRemote(nodeTheta.name).uid
nodeX.send(electMsg, ridTheta)
nodeX.send(threePMsg, ridTheta)
nodeX.send(electMsg, ridTheta)
looper.run(
eventually(checkDiscardMsg, [
nodeTheta,
],
electMsg,
'un-acceptable viewNo',
retryWait=1,
timeout=5))
nodeX.send(threePMsg, ridTheta)
looper.run(
eventually(checkDiscardMsg, [
nodeTheta,
],
threePMsg,
'un-acceptable viewNo',
retryWait=1,
timeout=5))