def testPrimaryElectionWithAClearWinner(electContFixture, looper,
keySharedNodes):
"""
Primary selection (Sunny Day)
A, B, C, D, E
A, B, C, D startup. E is lagging.
A sees the minimum number of nodes first, and then sends out a NOMINATE(A) message
B, C, D all see the NOMINATE(A) message from A, and respond with NOMINATE(A) message to all other nodes
A sees three other NOMINATE(A) votes (from B, C, D)
A sees that A is the clear winner (2f+1 total), and sends PRIMARY(A) to all nodes
B sees two more NOMINATE(A) votes (from C and D)
B sees that A is the clear winner (2f+1 total), and sends PRIMARY(A) to all nodes
C sees two more NOMINATE(A) votes (from B and D)
C sees that A is the clear winner (2f+1 total), and sends PRIMARY(A) to all nodes
D sees two more NOMINATE(A) votes (from B and C)
D sees that A is the clear winner (2f+1 total), and sends PRIMARY(A) to all nodes
A sees at least two other PRIMARY(A) votes (3 including it's own)
selects A as primary
B sees at least two other PRIMARY(A) votes (3 including it's own)
selects A as primary
C sees at least two other PRIMARY(A) votes (3 including it's own)
selects A as primary
D sees at least two other PRIMARY(A) votes (3 including it's own)
selects A as primary
"""
nodeSet = keySharedNodes
A, B, C, D = nodeSet.nodes.values()
nodesBCD = [B, C, D]
checkPoolReady(looper, nodeSet)
# Checking whether one of the replicas of Node A nominated itself
timeout = waits.expectedPoolNominationTimeout(len(nodeSet))
looper.run(
eventually(checkNomination, A, A.name, retryWait=1, timeout=timeout))
timeout = waits.expectedPoolNominationTimeout(len(nodeSet))
for n in nodesBCD:
# Checking whether Node B, C and D nominated Node A
looper.run(
eventually(checkNomination,
n,
A.name,
retryWait=1,
timeout=timeout))
checkProtocolInstanceSetup(looper=looper, nodes=nodeSet, retryWait=1)
assert A.hasPrimary
def testPrimaryElectionCase1(case1Setup, looper, keySharedNodes):
"""
Case 1 - A node making multiple nominations for a particular node. Consider
4 nodes A, B, C and D. Lets say node B is malicious and is repeatedly
nominating Node D
"""
nodes = keySharedNodes
nodeA, nodeB, nodeC, nodeD = [nodes.getNode(nm) for nm in nodes.nodeNames]
# Doesn't matter if nodes reach the ready state or not. Just start them
looper.run(checkNodesConnected(nodes))
# Node B sends multiple NOMINATE messages for Node D but only after A has
# nominated itself
timeout = waits.expectedPoolNominationTimeout(
nodeCount=len(keySharedNodes))
looper.run(
eventually(checkNomination,
nodeA,
nodeA.name,
retryWait=.25,
timeout=timeout))
instId = getSelfNominationByNode(nodeA)
for i in range(5):
# nodeB.send(Nomination(nodeD.name, instId, nodeB.viewNo))
nodeB.send(nominationByNode(nodeD.name, nodeB, instId))
nodeB.nodestack.flushOutBoxes()
# No node from node A, node C, node D(node B is malicious anyway so not
# considering it) should have more than one nomination for node D since
# node D is slow. The one nomination for D, that nodes A, C
# and D might have would be because of node B
for node in [nodeA, nodeC, nodeD]:
assert [n[0] for n in node.elector.nominations[instId].values()].count(
Replica.generateName(nodeD.name, instId)) \
<= 1
timeout = waits.expectedPoolElectionTimeout(nodeCount) + delayOfNomination
primaryReplicas = ensureElectionsDone(looper=looper,
nodes=nodes,
customTimeout=timeout)
for node in nodes:
logger.debug("{}'s nominations {}".format(node,
node.elector.nominations))
# Node D should not have any primary
assert not nodeD.hasPrimary
# A node other than Node D should not have any replica among the
# primary replicas
assert nodeD.name not in [pr.name for pr in primaryReplicas]
def testPrimaryElectionContested(electContFixture, looper, txnPoolNodeSet):
"""
Primary selection (Rainy Day)
A, B, C, D, E
A, B, C, D startup. E is lagging.
A sees the minimum number of nodes, and then sends Nominate(A)
At the same exact time, B sees the minimum number of nodes, and then sends out Nominate(B)
A sees B sending Nominate(B), but it has already nominated itself, so it does nothing
B sees A sending Nominate(A), but it has already nominated itself, so it does nothing
C sees A sending Nominate(A), and sends Nominate(A)
D sees A sending Nominate(A), and sends Nominate(A)
All nodes see that B nominated B and A, C, and D all nominated A
Because the votes for A exceeds the votes for B, all send out Primary(A)
TODO's (see below)
All see the others have sent Primary A, and then the nodes record who is the Primary.
"""
A, B, C, D = txnPoolNodeSet
checkPoolReady(looper, txnPoolNodeSet)
logger.debug("Check nomination")
timeout = waits.expectedPoolNominationTimeout(nodeCount)
# Checking whether Node A nominated itself
looper.run(
eventually(checkNomination, A, A.name, retryWait=1, timeout=timeout))
# Checking whether Node B nominated itself
looper.run(
eventually(checkNomination, B, B.name, retryWait=1, timeout=timeout))
for n in [C, D]:
# Checking whether Node C and Node D nominated Node A
looper.run(
eventually(checkNomination,
n,
A.name,
retryWait=1,
timeout=timeout))
checkProtocolInstanceSetup(looper=looper,
nodes=txnPoolNodeSet,
retryWait=1)
# Node D should not be primary
assert not D.hasPrimary
# A should have at least one primary
assert A.hasPrimary
def testBlacklistNodeOnMultipleNominations(looper, txnPoolNodeSet, ready):
"""
A node that sends multiple nominations must be blacklisted by other nodes
"""
A, B, C, D = txnPoolNodeSet
# B sends more than 2 nominations
for i in range(3):
B.send(Nomination(D.name, 0, B.viewNo))
# B should be blacklisted by A, C, D
def chk():
for node in A, C, D:
assert node.isNodeBlacklisted(B.name)
timeout = waits.expectedPoolNominationTimeout(len(txnPoolNodeSet))
looper.run(eventually(chk, retryWait=1, timeout=timeout))
def testBlacklistNodeOnMultipleNominations(looper, txnPoolNodeSet, ready):
"""
A node that sends multiple nominations must be blacklisted by other nodes
"""
A, B, C, D = txnPoolNodeSet
# B sends more than 2 nominations
for i in range(3):
B.send(Nomination(D.name, 0, B.viewNo))
# B should be blacklisted by A, C, D
def chk():
for node in A, C, D:
assert node.isNodeBlacklisted(B.name)
timeout = waits.expectedPoolNominationTimeout(len(txnPoolNodeSet))
looper.run(eventually(chk, retryWait=1, timeout=timeout))
def testPrimaryElectionCase1(case1Setup, looper, txnPoolNodeSet):
"""
Case 1 - A node making multiple nominations for a particular node. Consider
4 nodes A, B, C and D. Lets say node B is malicious and is repeatedly
nominating Node D
"""
nodeA, nodeB, nodeC, nodeD = txnPoolNodeSet
# Doesn't matter if nodes reach the ready state or not. Just start them
looper.run(checkNodesConnected(txnPoolNodeSet))
# Node B sends multiple NOMINATE messages for Node D but only after A has
# nominated itself
timeout = waits.expectedPoolNominationTimeout(
nodeCount=len(txnPoolNodeSet))
looper.run(eventually(checkNomination, nodeA, nodeA.name,
retryWait=.25, timeout=timeout))
instId = getSelfNominationByNode(nodeA)
for i in range(5):
# nodeB.send(Nomination(nodeD.name, instId, nodeB.viewNo))
nodeB.send(nominationByNode(nodeD.name, nodeB, instId))
nodeB.nodestack.flushOutBoxes()
# No node from node A, node C, node D(node B is malicious anyway so not
# considering it) should have more than one nomination for node D since
# node D is slow. The one nomination for D, that nodes A, C
# and D might have would be because of node B
for node in [nodeA, nodeC, nodeD]:
assert [n[0] for n in node.elector.nominations[instId].values()].count(
Replica.generateName(nodeD.name, instId)) \
<= 1
timeout = waits.expectedPoolElectionTimeout(nodeCount) + delayOfNomination
primaryReplicas = ensureElectionsDone(looper=looper,
nodes=txnPoolNodeSet, customTimeout=timeout)
for node in txnPoolNodeSet:
logger.debug(
"{}'s nominations {}".format(node, node.elector.nominations))
# Node D should not have any primary
assert not nodeD.hasPrimary
# A node other than Node D should not have any replica among the
# primary replicas
assert nodeD.name not in [pr.name for pr in primaryReplicas]
def testPrimaryElectionContested(electContFixture, looper, txnPoolNodeSet):
"""
Primary selection (Rainy Day)
A, B, C, D, E
A, B, C, D startup. E is lagging.
A sees the minimum number of nodes, and then sends Nominate(A)
At the same exact time, B sees the minimum number of nodes, and then sends out Nominate(B)
A sees B sending Nominate(B), but it has already nominated itself, so it does nothing
B sees A sending Nominate(A), but it has already nominated itself, so it does nothing
C sees A sending Nominate(A), and sends Nominate(A)
D sees A sending Nominate(A), and sends Nominate(A)
All nodes see that B nominated B and A, C, and D all nominated A
Because the votes for A exceeds the votes for B, all send out Primary(A)
TODO's (see below)
All see the others have sent Primary A, and then the nodes record who is the Primary.
"""
A, B, C, D = txnPoolNodeSet
checkPoolReady(looper, txnPoolNodeSet)
logger.debug("Check nomination")
timeout = waits.expectedPoolNominationTimeout(nodeCount)
# Checking whether Node A nominated itself
looper.run(eventually(checkNomination, A, A.name,
retryWait=1, timeout=timeout))
# Checking whether Node B nominated itself
looper.run(eventually(checkNomination, B, B.name,
retryWait=1, timeout=timeout))
for n in [C, D]:
# Checking whether Node C and Node D nominated Node A
looper.run(eventually(checkNomination, n, A.name,
retryWait=1, timeout=timeout))
checkProtocolInstanceSetup(looper=looper, nodes=txnPoolNodeSet, retryWait=1)
# Node D should not be primary
assert not D.hasPrimary
# A should have at least one primary
assert A.hasPrimary
def testBlacklistNodeOnMultiplePrimaryDeclarations(looper,
keySharedNodes,
ready):
"""
A node that sends multiple primary declarations must be blacklisted by
other nodes
"""
nodeSet = keySharedNodes
A, B, C, D = nodeSet.nodes.values()
# B sends more than 2 primary declarations
for i in range(3):
B.send(Primary(D.name, 0, B.viewNo))
# B should be blacklisted by A, C, D
def chk():
for node in A, C, D:
assert node.isNodeBlacklisted(B.name)
timeout = waits.expectedPoolNominationTimeout(len(nodeSet.nodes))
looper.run(eventually(chk, retryWait=1, timeout=timeout))
def testPrimaryElectionCase4(case4Setup, looper):
"""
Case 4 - A node making multiple primary declarations for a particular node.
Consider 4 nodes A, B, C and D. Lets say node B is malicious and is
repeatedly declaring Node D as primary
"""
allNodes = case4Setup
A, B, C, D = allNodes
looper.run(checkNodesConnected(allNodes))
# Node B sends multiple declarations of node D's 0th protocol instance as
# primary to all nodes
for i in range(5):
# B.send(Primary(D.name, 0, B.viewNo))
B.send(primaryByNode(D.name, B, 0))
# No node from node A, node C, node D(node B is malicious anyway so not
# considering it) should have more than one primary declaration for node
# D since node D is slow. The one primary declaration for node D,
# that nodes A, C and D might have would be because of node B
def x():
primDecs = [p[0] for p in node.elector.primaryDeclarations[0].values()]
assert primDecs.count(D.name) <= 1
# also have to take into account the catchup procedure
timeout = waits.expectedPoolNominationTimeout(len(allNodes)) + \
waits.expectedPoolCatchupTime(len(allNodes))
for node in (A, C, D):
looper.run(eventually(x, retryWait=.5, timeout=timeout))
timeout = waits.expectedPoolElectionTimeout(
len(allNodes)) + delaySelfNomination
ensureElectionsDone(looper=looper, nodes=allNodes, customTimeout=timeout)
# Node D should not have any primary replica
assert not D.hasPrimary
def testPrimaryElectionCase4(case4Setup, looper):
"""
Case 4 - A node making multiple primary declarations for a particular node.
Consider 4 nodes A, B, C and D. Lets say node B is malicious and is
repeatedly declaring Node D as primary
"""
allNodes = case4Setup
A, B, C, D = allNodes
looper.run(checkNodesConnected(allNodes))
# Node B sends multiple declarations of node D's 0th protocol instance as
# primary to all nodes
for i in range(5):
# B.send(Primary(D.name, 0, B.viewNo))
B.send(primaryByNode(D.name, B, 0))
# No node from node A, node C, node D(node B is malicious anyway so not
# considering it) should have more than one primary declaration for node
# D since node D is slow. The one primary declaration for node D,
# that nodes A, C and D might have would be because of node B
def x():
primDecs = [p[0] for p in node.elector.primaryDeclarations[0].values()]
assert primDecs.count(D.name) <= 1
# also have to take into account the catchup procedure
timeout = waits.expectedPoolNominationTimeout(len(allNodes)) + \
waits.expectedPoolCatchupTime(len(allNodes))
for node in (A, C, D):
looper.run(eventually(x, retryWait=.5, timeout=timeout))
timeout = waits.expectedPoolElectionTimeout(
len(allNodes)) + delaySelfNomination
ensureElectionsDone(looper=looper, nodes=allNodes, customTimeout=timeout)
# Node D should not have any primary replica
assert not D.hasPrimary
def testPrimaryElectionCase2(case2Setup, looper, keySharedNodes):
"""
Case 2 - A node making nominations for a multiple other nodes. Consider 4
nodes A, B, C, and D. Lets say node B is malicious and nominates node C
to all nodes. Again node B nominates node D to all nodes.
"""
nodeSet = keySharedNodes
A, B, C, D = nodeSet.nodes.values()
looper.run(checkNodesConnected(nodeSet))
# Node B sends multiple NOMINATE msgs but only after A has nominated itself
timeout = waits.expectedPoolNominationTimeout(len(nodeSet))
looper.run(
eventually(checkNomination, A, A.name, retryWait=.25, timeout=timeout))
instId = getSelfNominationByNode(A)
BRep = Replica.generateName(B.name, instId)
CRep = Replica.generateName(C.name, instId)
DRep = Replica.generateName(D.name, instId)
# Node B first sends NOMINATE msgs for Node C to all nodes
# B.send(Nomination(CRep, instId, B.viewNo))
B.send(nominationByNode(CRep, B, instId))
# Node B sends NOMINATE msgs for Node D to all nodes
# B.send(Nomination(DRep, instId, B.viewNo))
B.send(nominationByNode(DRep, B, instId))
# Ensure elections are done
ensureElectionsDone(looper=looper, nodes=nodeSet)
# All nodes from node A, node C, node D(node B is malicious anyway so
# not considering it) should have nomination for node C from node B since
# node B first nominated node C
for node in [A, C, D]:
assert node.elector.nominations[instId][BRep][0] == CRep
def testPrimaryElectionWithTie(electTieFixture, looper, keySharedNodes):
"""
Primary selection (Rainy Day)
A, B, C, D, E
A, B, C, D startup. E is lagging.
A sees the minimum number of nodes, and then sends Nominate(A)
At the same exact time, B sees the minimum number of nodes, and then sends out Nominate(B)
A sees B sending Nominate(B), but it has already nominated itself, so it does nothing
B sees A sending Nominate(A), but it has already nominated itself, so it does nothing
C sees A sending Nominate(A), and sends Nominate(A)
D sees B sending Nominate(B), and sends Nominate(B)
There's a split. C and A think A is the primary, B and D think B is the primary
All nodes can see that there is a split. Each sends out Reelection([A,B])
A and B both see Reelection([A,B]) from themselves as well as the other 3 (the number from others should be at least f+1),
1. they wait a random amount of time (between 0 and 2 seconds),
2. they each send out a Nominate(self)
Voting is repeated until we have a good election.
"""
# TODO optimize the sending messages in batches, for example, we don't
# send messages more often than 400 milliseconds. Once those 400
# millis have passed, we send the several queued messages in one
# batch.
nodeSet = keySharedNodes
A, B, C, D = nodeSet.nodes.values()
checkPoolReady(looper, nodeSet.nodes.values())
for node in nodeSet.nodes.values():
for instId, replica in enumerate(node.elector.replicas):
logger.debug("replica {} {} with votes {}".format(
replica.name, replica.instId,
node.elector.nominations.get(instId, {})))
nominationTimeout = waits.expectedPoolNominationTimeout(len(nodeSet))
logger.debug("Check nomination")
# Checking whether Node A nominated itself
looper.run(
eventually(checkNomination,
A,
A.name,
retryWait=1,
timeout=nominationTimeout))
# Checking whether Node B nominated itself
looper.run(
eventually(checkNomination,
B,
B.name,
retryWait=1,
timeout=nominationTimeout))
# Checking whether Node C nominated Node A
looper.run(
eventually(checkNomination,
C,
A.name,
retryWait=1,
timeout=nominationTimeout))
# Checking whether Node D nominated Node D
looper.run(
eventually(checkNomination,
D,
B.name,
retryWait=1,
timeout=nominationTimeout))
# No node should be primary
for node in nodeSet.nodes.values():
assert node.hasPrimary is False
for node in nodeSet.nodes.values():
node.resetDelays()
checkProtocolInstanceSetup(looper=looper, nodes=nodeSet, retryWait=1)
def testPrimaryElectionWithTie(electTieFixture, looper, txnPoolNodeSet):
"""
Primary selection (Rainy Day)
A, B, C, D, E
A, B, C, D startup. E is lagging.
A sees the minimum number of nodes, and then sends Nominate(A)
At the same exact time, B sees the minimum number of nodes, and then sends out Nominate(B)
A sees B sending Nominate(B), but it has already nominated itself, so it does nothing
B sees A sending Nominate(A), but it has already nominated itself, so it does nothing
C sees A sending Nominate(A), and sends Nominate(A)
D sees B sending Nominate(B), and sends Nominate(B)
There's a split. C and A think A is the primary, B and D think B is the primary
All nodes can see that there is a split. Each sends out Reelection([A,B])
A and B both see Reelection([A,B]) from themselves as well as the other 3 (the number from others should be at least f+1),
1. they wait a random amount of time (between 0 and 2 seconds),
2. they each send out a Nominate(self)
Voting is repeated until we have a good election.
"""
# TODO optimize the sending messages in batches, for example, we don't
# send messages more often than 400 milliseconds. Once those 400
# millis have passed, we send the several queued messages in one
# batch.
A, B, C, D = txnPoolNodeSet
checkPoolReady(looper, txnPoolNodeSet)
for node in txnPoolNodeSet:
for instId, replica in node.elector.replicas.items():
logger.debug("replica {} {} with votes {}".
format(replica.name, replica.instId,
node.elector.nominations.get(instId, {})))
nominationTimeout = waits.expectedPoolNominationTimeout(len(txnPoolNodeSet))
logger.debug("Check nomination")
# Checking whether Node A nominated itself
looper.run(eventually(checkNomination, A, A.name,
retryWait=1, timeout=nominationTimeout))
# Checking whether Node B nominated itself
looper.run(eventually(checkNomination, B, B.name,
retryWait=1, timeout=nominationTimeout))
# Checking whether Node C nominated Node A
looper.run(eventually(checkNomination, C, A.name,
retryWait=1, timeout=nominationTimeout))
# Checking whether Node D nominated Node D
looper.run(eventually(checkNomination, D, B.name,
retryWait=1, timeout=nominationTimeout))
# No node should be primary
for node in txnPoolNodeSet:
assert node.hasPrimary is False
for node in txnPoolNodeSet:
node.resetDelays()
checkProtocolInstanceSetup(looper=looper, nodes=txnPoolNodeSet, retryWait=1)
