def testPrimaryElectionWithTie(electTieFixture, looper, keySharedNodes):
"""
Primary selection (Rainy Day)
A, B, C, D, E
A, B, C, D startup. E is lagging.
A sees the minimum number of nodes, and then sends Nominate(A)
At the same exact time, B sees the minimum number of nodes, and then sends out Nominate(B)
A sees B sending Nominate(B), but it has already nominated itself, so it does nothing
B sees A sending Nominate(A), but it has already nominated itself, so it does nothing
C sees A sending Nominate(A), and sends Nominate(A)
D sees B sending Nominate(B), and sends Nominate(B)
There's a split. C and A think A is the primary, B and D think B is the primary
All nodes can see that there is a split. Each sends out Reelection([A,B])
A and B both see Reelection([A,B]) from themselves as well as the other 3 (the number from others should be at least f+1),
1. they wait a random amount of time (between 0 and 2 seconds),
2. they each send out a Nominate(self)
Voting is repeated until we have a good election.
"""
# TODO optimize the sending messages in batches, for example, we don't
# send messages more often than 400 milliseconds. Once those 400
# millis have passed, we send the several queued messages in one
# batch.
nodeSet = keySharedNodes
A, B, C, D = nodeSet.nodes.values()
checkPoolReady(looper, nodeSet.nodes.values())
for node in nodeSet.nodes.values():
for instId, replica in enumerate(node.elector.replicas):
logger.debug("replica {} {} with votes {}".
format(replica.name, replica.instId,
node.elector.nominations.get(instId, {})))
logger.debug("Check nomination")
# Checking whether Node A nominated itself
looper.run(eventually(checkNomination, A, A.name, retryWait=1, timeout=10))
# Checking whether Node B nominated itself
looper.run(eventually(checkNomination, B, B.name, retryWait=1, timeout=10))
# Checking whether Node C nominated Node A
looper.run(eventually(checkNomination, C, A.name, retryWait=1, timeout=10))
# Checking whether Node D nominated Node D
looper.run(eventually(checkNomination, D, B.name, retryWait=1, timeout=10))
# No node should be primary
for node in nodeSet.nodes.values():
assert node.hasPrimary is False
for node in nodeSet.nodes.values():
node.resetDelays()
checkProtocolInstanceSetup(looper=looper, nodes=nodeSet, retryWait=1,
timeout=60)
def testNodeNames(be, do, cli, validNodeNames):
"""
Test adding nodes with valid and invalid names. Also testing adding nodes
with duplicate names
"""
addNodes(be, do, cli, validNodeNames)
checkPoolReady(cli.looper, cli.nodes.values())
lastNodeName = validNodeNames[-1]
# Create a node with a name of an already created node
be(cli)
do("new node {}".format(lastNodeName), expect=["Node {} already exists.".format(lastNodeName)])
assert len(cli.nodes) == 4
# Create a node with invalid name
randName = randomString(10)
do("new node {}".format(randName), expect=["Invalid node name '{}'. ".format(randName)])
args = cli.printedTokens[-1]
token, _ = args['tokens'][0]
# An error token should be printed
assert isErrorToken(token)
# Count of cli.nodes should not change
assert len(cli.nodes) == len(validNodeNames)
# Node name should NOT be in cli.nodes
assert randName not in cli.nodes
def assertAllNodesCreated(cli, validNodeNames):
# Check if all nodes are connected
checkPoolReady(cli.looper, cli.nodes.values())
# Check if all nodes are added
assert len(cli.nodes) == len(validNodeNames)
assert set(cli.nodes.keys()) == set(cli.nodeReg.keys())
def testPrimaryElectionWithAClearWinner(electContFixture, looper,
keySharedNodes):
"""
Primary selection (Sunny Day)
A, B, C, D, E
A, B, C, D startup. E is lagging.
A sees the minimum number of nodes first, and then sends out a NOMINATE(A) message
B, C, D all see the NOMINATE(A) message from A, and respond with NOMINATE(A) message to all other nodes
A sees three other NOMINATE(A) votes (from B, C, D)
A sees that A is the clear winner (2f+1 total), and sends PRIMARY(A) to all nodes
B sees two more NOMINATE(A) votes (from C and D)
B sees that A is the clear winner (2f+1 total), and sends PRIMARY(A) to all nodes
C sees two more NOMINATE(A) votes (from B and D)
C sees that A is the clear winner (2f+1 total), and sends PRIMARY(A) to all nodes
D sees two more NOMINATE(A) votes (from B and C)
D sees that A is the clear winner (2f+1 total), and sends PRIMARY(A) to all nodes
A sees at least two other PRIMARY(A) votes (3 including it's own)
selects A as primary
B sees at least two other PRIMARY(A) votes (3 including it's own)
selects A as primary
C sees at least two other PRIMARY(A) votes (3 including it's own)
selects A as primary
D sees at least two other PRIMARY(A) votes (3 including it's own)
selects A as primary
"""
nodeSet = keySharedNodes
A, B, C, D = nodeSet.nodes.values()
nodesBCD = [B, C, D]
checkPoolReady(looper, nodeSet)
# Checking whether one of the replicas of Node A nominated itself
timeout = waits.expectedPoolNominationTimeout(len(nodeSet))
looper.run(
eventually(checkNomination, A, A.name, retryWait=1, timeout=timeout))
timeout = waits.expectedPoolNominationTimeout(len(nodeSet))
for n in nodesBCD:
# Checking whether Node B, C and D nominated Node A
looper.run(
eventually(checkNomination,
n,
A.name,
retryWait=1,
timeout=timeout))
checkProtocolInstanceSetup(looper=looper, nodes=nodeSet, retryWait=1)
assert A.hasPrimary
def testPrimaryElectionWithAClearWinner(electContFixture, looper, keySharedNodes):
"""
Primary selection (Sunny Day)
A, B, C, D, E
A, B, C, D startup. E is lagging.
A sees the minimum number of nodes first, and then sends out a NOMINATE(A) message
B, C, D all see the NOMINATE(A) message from A, and respond with NOMINATE(A) message to all other nodes
A sees three other NOMINATE(A) votes (from B, C, D)
A sees that A is the clear winner (2f+1 total), and sends PRIMARY(A) to all nodes
B sees two more NOMINATE(A) votes (from C and D)
B sees that A is the clear winner (2f+1 total), and sends PRIMARY(A) to all nodes
C sees two more NOMINATE(A) votes (from B and D)
C sees that A is the clear winner (2f+1 total), and sends PRIMARY(A) to all nodes
D sees two more NOMINATE(A) votes (from B and C)
D sees that A is the clear winner (2f+1 total), and sends PRIMARY(A) to all nodes
A sees at least two other PRIMARY(A) votes (3 including it's own)
selects A as primary
B sees at least two other PRIMARY(A) votes (3 including it's own)
selects A as primary
C sees at least two other PRIMARY(A) votes (3 including it's own)
selects A as primary
D sees at least two other PRIMARY(A) votes (3 including it's own)
selects A as primary
"""
nodeSet = keySharedNodes
A, B, C, D = nodeSet.nodes.values()
nodesBCD = [B, C, D]
checkPoolReady(looper, nodeSet)
# Checking whether one of the replicas of Node A nominated itself
looper.run(eventually(checkNomination, A, A.name, retryWait=1, timeout=10))
for n in nodesBCD:
# Checking whether Node B, C and D nominated Node A
looper.run(eventually(checkNomination, n, A.name, retryWait=1,
timeout=10))
checkProtocolInstanceSetup(looper=looper, nodes=nodeSet, retryWait=1,
timeout=10)
assert A.hasPrimary
def testPrimaryElectionContested(electContFixture, looper, txnPoolNodeSet):
"""
Primary selection (Rainy Day)
A, B, C, D, E
A, B, C, D startup. E is lagging.
A sees the minimum number of nodes, and then sends Nominate(A)
At the same exact time, B sees the minimum number of nodes, and then sends out Nominate(B)
A sees B sending Nominate(B), but it has already nominated itself, so it does nothing
B sees A sending Nominate(A), but it has already nominated itself, so it does nothing
C sees A sending Nominate(A), and sends Nominate(A)
D sees A sending Nominate(A), and sends Nominate(A)
All nodes see that B nominated B and A, C, and D all nominated A
Because the votes for A exceeds the votes for B, all send out Primary(A)
TODO's (see below)
All see the others have sent Primary A, and then the nodes record who is the Primary.
"""
A, B, C, D = txnPoolNodeSet
checkPoolReady(looper, txnPoolNodeSet)
logger.debug("Check nomination")
timeout = waits.expectedPoolNominationTimeout(nodeCount)
# Checking whether Node A nominated itself
looper.run(
eventually(checkNomination, A, A.name, retryWait=1, timeout=timeout))
# Checking whether Node B nominated itself
looper.run(
eventually(checkNomination, B, B.name, retryWait=1, timeout=timeout))
for n in [C, D]:
# Checking whether Node C and Node D nominated Node A
looper.run(
eventually(checkNomination,
n,
A.name,
retryWait=1,
timeout=timeout))
checkProtocolInstanceSetup(looper=looper,
nodes=txnPoolNodeSet,
retryWait=1)
# Node D should not be primary
assert not D.hasPrimary
# A should have at least one primary
assert A.hasPrimary
def testPrimaryElectionContested(electContFixture, looper, txnPoolNodeSet):
"""
Primary selection (Rainy Day)
A, B, C, D, E
A, B, C, D startup. E is lagging.
A sees the minimum number of nodes, and then sends Nominate(A)
At the same exact time, B sees the minimum number of nodes, and then sends out Nominate(B)
A sees B sending Nominate(B), but it has already nominated itself, so it does nothing
B sees A sending Nominate(A), but it has already nominated itself, so it does nothing
C sees A sending Nominate(A), and sends Nominate(A)
D sees A sending Nominate(A), and sends Nominate(A)
All nodes see that B nominated B and A, C, and D all nominated A
Because the votes for A exceeds the votes for B, all send out Primary(A)
TODO's (see below)
All see the others have sent Primary A, and then the nodes record who is the Primary.
"""
A, B, C, D = txnPoolNodeSet
checkPoolReady(looper, txnPoolNodeSet)
logger.debug("Check nomination")
timeout = waits.expectedPoolNominationTimeout(nodeCount)
# Checking whether Node A nominated itself
looper.run(eventually(checkNomination, A, A.name,
retryWait=1, timeout=timeout))
# Checking whether Node B nominated itself
looper.run(eventually(checkNomination, B, B.name,
retryWait=1, timeout=timeout))
for n in [C, D]:
# Checking whether Node C and Node D nominated Node A
looper.run(eventually(checkNomination, n, A.name,
retryWait=1, timeout=timeout))
checkProtocolInstanceSetup(looper=looper, nodes=txnPoolNodeSet, retryWait=1)
# Node D should not be primary
assert not D.hasPrimary
# A should have at least one primary
assert A.hasPrimary
def testNodeNames(cli, validNodeNames):
"""
Test adding nodes with valid and invalid names. Also testing adding nodes
with duplicate names
"""
# Add nodes with valid names
for i, nm in enumerate(validNodeNames):
cli.enterCmd("new node {}".format(nm))
# Count of cli.nodes should increase by 1
assert len(cli.nodes) == (i + 1)
checkNodeStarted(cli, nm)
checkPoolReady(cli.looper, cli.nodes.values())
# Create a node with a name of an already created node
cli.enterCmd("new node {}".format(nm))
msg = cli.lastPrintArgs['msg']
# Appropriate error msg should be printed
assert msg == "Node {} already exists.".format(nm)
# Count of cli.nodes should not change
assert len(cli.nodes) == 4
randName = randomString(10)
cli.enterCmd("new node {}".format(randName))
args = cli.printedTokens[-1]
token, msg = args['tokens'][0]
# An error token should be printed
assert isErrorToken(token)
# Appropriate error msg should be printed
assert msg == "Invalid node name '{}'. ".format(randName)
# Count of cli.nodes should not change
assert len(cli.nodes) == len(validNodeNames)
# Node name should be in cli.nodes
assert randName not in cli.nodes
def testPrePrepareWhenPrimaryStatusIsUnknown(tdir_for_func):
nodeNames = genNodeNames(4)
nodeReg = genNodeReg(names=nodeNames)
with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet:
with Looper(nodeSet) as looper:
prepareNodeSet(looper, nodeSet)
nodeA, nodeB, nodeC, nodeD = tuple(
addNodeBack(nodeSet, looper, nodeNames[i])
for i in range(0, 4))
# Nodes C and D delays self nomination so A and B can become
# primaries
nodeC.delaySelfNomination(30)
nodeD.delaySelfNomination(30)
# Node D delays receiving PRIMARY messages from all nodes so it
# will not know whether it is primary or not
# nodeD.nodestack.delay(delayer(20, PRIMARY))
nodeD.nodeIbStasher.delay(delayerMsgTuple(20, Primary))
checkPoolReady(looper=looper, nodes=nodeSet)
client1, wal = setupClient(looper, nodeSet, tmpdir=tdir_for_func)
request = sendRandomRequest(wal, client1)
# TODO Rethink this
instNo = 0
for i in range(3):
node = nodeSet.getNode(nodeNames[i])
# Nodes A, B and C should have received PROPAGATE request
# from Node D
looper.run(
eventually(checkIfPropagateRecvdFromNode,
node,
nodeD,
request.identifier,
request.reqId,
retryWait=1,
timeout=10))
# Node D should have 1 pending PRE-PREPARE request
def assertOnePrePrepare():
assert len(
getPendingRequestsForReplica(nodeD.replicas[instNo],
PrePrepare)) == 1
looper.run(eventually(assertOnePrePrepare, retryWait=1,
timeout=10))
# Node D should have 2 pending PREPARE requests(from node B and C)
def assertTwoPrepare():
assert len(
getPendingRequestsForReplica(nodeD.replicas[instNo],
Prepare)) == 2
looper.run(eventually(assertTwoPrepare, retryWait=1, timeout=10))
# Node D should have no pending PRE-PREPARE, PREPARE or COMMIT
# requests
for reqType in [PrePrepare, Prepare, Commit]:
looper.run(
eventually(lambda: assertLength(
getPendingRequestsForReplica(nodeD.replicas[instNo],
reqType), 0),
retryWait=1,
timeout=20))
def testPrimaryElectionWithTie(electTieFixture, looper, keySharedNodes):
"""
Primary selection (Rainy Day)
A, B, C, D, E
A, B, C, D startup. E is lagging.
A sees the minimum number of nodes, and then sends Nominate(A)
At the same exact time, B sees the minimum number of nodes, and then sends out Nominate(B)
A sees B sending Nominate(B), but it has already nominated itself, so it does nothing
B sees A sending Nominate(A), but it has already nominated itself, so it does nothing
C sees A sending Nominate(A), and sends Nominate(A)
D sees B sending Nominate(B), and sends Nominate(B)
There's a split. C and A think A is the primary, B and D think B is the primary
All nodes can see that there is a split. Each sends out Reelection([A,B])
A and B both see Reelection([A,B]) from themselves as well as the other 3 (the number from others should be at least f+1),
1. they wait a random amount of time (between 0 and 2 seconds),
2. they each send out a Nominate(self)
Voting is repeated until we have a good election.
"""
# TODO optimize the sending messages in batches, for example, we don't
# send messages more often than 400 milliseconds. Once those 400
# millis have passed, we send the several queued messages in one
# batch.
nodeSet = keySharedNodes
A, B, C, D = nodeSet.nodes.values()
checkPoolReady(looper, nodeSet.nodes.values())
for node in nodeSet.nodes.values():
for instId, replica in enumerate(node.elector.replicas):
logger.debug("replica {} {} with votes {}".format(
replica.name, replica.instId,
node.elector.nominations.get(instId, {})))
nominationTimeout = waits.expectedPoolNominationTimeout(len(nodeSet))
logger.debug("Check nomination")
# Checking whether Node A nominated itself
looper.run(
eventually(checkNomination,
A,
A.name,
retryWait=1,
timeout=nominationTimeout))
# Checking whether Node B nominated itself
looper.run(
eventually(checkNomination,
B,
B.name,
retryWait=1,
timeout=nominationTimeout))
# Checking whether Node C nominated Node A
looper.run(
eventually(checkNomination,
C,
A.name,
retryWait=1,
timeout=nominationTimeout))
# Checking whether Node D nominated Node D
looper.run(
eventually(checkNomination,
D,
B.name,
retryWait=1,
timeout=nominationTimeout))
# No node should be primary
for node in nodeSet.nodes.values():
assert node.hasPrimary is False
for node in nodeSet.nodes.values():
node.resetDelays()
checkProtocolInstanceSetup(looper=looper, nodes=nodeSet, retryWait=1)
def checkNodesReadyForRequest(looper: Looper, nodes: Sequence[TestNode],
timeout: int = 20):
checkPoolReady(looper, nodes, timeout)
def testPrePrepareWhenPrimaryStatusIsUnknown(tdir_for_func):
nodeNames = genNodeNames(4)
nodeReg = genNodeReg(names=nodeNames)
with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet:
with Looper(nodeSet) as looper:
prepareNodeSet(looper, nodeSet)
nodeA, nodeB, nodeC, nodeD = tuple(
addNodeBack(nodeSet, looper, nodeNames[i])
for i in range(0, 4))
# Since primary selection is round robin, A and B will be primaries
# Nodes C and D delays self nomination so A and B can become
# primaries
# nodeC.delaySelfNomination(10)
# nodeD.delaySelfNomination(10)
# Node D delays receiving PRIMARY messages from all nodes so it
# will not know whether it is primary or not
# delayD = 5
# nodeD.nodeIbStasher.delay(delayerMsgTuple(delayD, Primary))
checkPoolReady(looper=looper, nodes=nodeSet)
# client1, wal = setupClient(looper, nodeSet, tmpdir=tdir_for_func)
# request = sendRandomRequest(wal, client1)
# TODO Rethink this
instNo = 0
timeout = waits.expectedClientRequestPropagationTime(len(nodeSet))
for i in range(3):
node = nodeSet.getNode(nodeNames[i])
# Nodes A, B and C should have received PROPAGATE request
# from Node D
looper.run(
eventually(checkIfPropagateRecvdFromNode,
node,
nodeD,
request.identifier,
request.reqId,
retryWait=1,
timeout=timeout))
def assert_msg_count(typ, count):
assert len(
getPendingRequestsForReplica(nodeD.replicas[instNo],
typ)) == count
# Node D should have 1 pending PRE-PREPARE request
timeout = waits.expectedPrePrepareTime(len(nodeSet))
looper.run(
eventually(assert_msg_count,
PrePrepare,
1,
retryWait=1,
timeout=timeout))
# Node D should have 2 pending PREPARE requests(from node B and C)
timeout = waits.expectedPrepareTime(len(nodeSet))
looper.run(
eventually(assert_msg_count,
Prepare,
2,
retryWait=1,
timeout=timeout))
# Its been checked above that replica stashes 3 phase messages in
# lack of primary, now avoid delay (fix the network)
nodeD.nodeIbStasher.reset_delays_and_process_delayeds()
# Node D should have no pending PRE-PREPARE, PREPARE or COMMIT
# requests
for reqType in [PrePrepare, Prepare, Commit]:
looper.run(
eventually(lambda: assertLength(
getPendingRequestsForReplica(nodeD.replicas[instNo],
reqType), 0),
retryWait=1,
timeout=delayD)) # wait little more than delay
def testPrePrepareWhenPrimaryStatusIsUnknown(tdir_for_func):
nodeNames = genNodeNames(4)
nodeReg = genNodeReg(names=nodeNames)
with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet:
with Looper(nodeSet) as looper:
prepareNodeSet(looper, nodeSet)
nodeA, nodeB, nodeC, nodeD = tuple(
addNodeBack(nodeSet, looper, nodeNames[i]) for i in range(0, 4))
# Nodes C and D delays self nomination so A and B can become
# primaries
nodeC.delaySelfNomination(30)
nodeD.delaySelfNomination(30)
# Node D delays receiving PRIMARY messages from all nodes so it
# will not know whether it is primary or not
# nodeD.nodestack.delay(delayer(20, PRIMARY))
nodeD.nodeIbStasher.delay(delayerMsgTuple(20, Primary))
checkPoolReady(looper=looper, nodes=nodeSet)
client1, wal = setupClient(looper, nodeSet, tmpdir=tdir_for_func)
request = sendRandomRequest(wal, client1)
# TODO Rethink this
instNo = 0
for i in range(3):
node = nodeSet.getNode(nodeNames[i])
# Nodes A, B and C should have received PROPAGATE request
# from Node D
looper.run(
eventually(checkIfPropagateRecvdFromNode, node, nodeD,
request.identifier,
request.reqId, retryWait=1, timeout=10))
# Node D should have 1 pending PRE-PREPARE request
def assertOnePrePrepare():
assert len(getPendingRequestsForReplica(nodeD.replicas[instNo],
PrePrepare)) == 1
looper.run(eventually(assertOnePrePrepare, retryWait=1, timeout=10))
# Node D should have 2 pending PREPARE requests(from node B and C)
def assertTwoPrepare():
assert len(getPendingRequestsForReplica(nodeD.replicas[instNo],
Prepare)) == 2
looper.run(eventually(assertTwoPrepare, retryWait=1, timeout=10))
# Node D should have no pending PRE-PREPARE, PREPARE or COMMIT
# requests
for reqType in [PrePrepare, Prepare, Commit]:
looper.run(eventually(lambda: assertLength(
getPendingRequestsForReplica(nodeD.replicas[instNo],
reqType),
0), retryWait=1, timeout=20))
def testPrePrepareWhenPrimaryStatusIsUnknown(tdir_for_func):
nodeNames = genNodeNames(4)
nodeReg = genNodeReg(names=nodeNames)
with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet:
with Looper(nodeSet) as looper:
prepareNodeSet(looper, nodeSet)
nodeA, nodeB, nodeC, nodeD = tuple(
addNodeBack(
nodeSet, looper, nodeNames[i]) for i in range(
0, 4))
# Since primary selection is round robin, A and B will be primaries
# Nodes C and D delays self nomination so A and B can become
# primaries
# nodeC.delaySelfNomination(10)
# nodeD.delaySelfNomination(10)
# Node D delays receiving PRIMARY messages from all nodes so it
# will not know whether it is primary or not
# delayD = 5
# nodeD.nodeIbStasher.delay(delayerMsgTuple(delayD, Primary))
checkPoolReady(looper=looper, nodes=nodeSet)
# client1, wal = setupClient(looper, nodeSet, tmpdir=tdir_for_func)
# request = sendRandomRequest(wal, client1)
# TODO Rethink this
instNo = 0
timeout = waits.expectedClientRequestPropagationTime(len(nodeSet))
for i in range(3):
node = nodeSet.getNode(nodeNames[i])
# Nodes A, B and C should have received PROPAGATE request
# from Node D
looper.run(
eventually(checkIfPropagateRecvdFromNode, node, nodeD,
request.identifier,
request.reqId, retryWait=1, timeout=timeout))
def assert_msg_count(typ, count):
assert len(getPendingRequestsForReplica(nodeD.replicas[instNo],
typ)) == count
# Node D should have 1 pending PRE-PREPARE request
timeout = waits.expectedPrePrepareTime(len(nodeSet))
looper.run(eventually(assert_msg_count, PrePrepare, 1,
retryWait=1, timeout=timeout))
# Node D should have 2 pending PREPARE requests(from node B and C)
timeout = waits.expectedPrepareTime(len(nodeSet))
looper.run(eventually(assert_msg_count, Prepare, 2, retryWait=1,
timeout=timeout))
# Its been checked above that replica stashes 3 phase messages in
# lack of primary, now avoid delay (fix the network)
nodeD.nodeIbStasher.reset_delays_and_process_delayeds()
# Node D should have no pending PRE-PREPARE, PREPARE or COMMIT
# requests
for reqType in [PrePrepare, Prepare, Commit]:
looper.run(
eventually(
lambda: assertLength(
getPendingRequestsForReplica(
nodeD.replicas[instNo],
reqType),
0),
retryWait=1,
timeout=delayD)) # wait little more than delay