def testPrePrepareWhenPrimaryStatusIsUnknown(tdir_for_func): nodeNames = genNodeNames(4) nodeReg = genNodeReg(names=nodeNames) with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet: with Looper(nodeSet) as looper: prepareNodeSet(looper, nodeSet) nodeA, nodeB, nodeC, nodeD = tuple( addNodeBack(nodeSet, looper, nodeNames[i]) for i in range(0, 4)) # Nodes C and D delays self nomination so A and B can become primaries nodeC.delaySelfNomination(30) nodeD.delaySelfNomination(30) # Node D delays receiving PRIMARY messages from all nodes so it will not know # whether it is primary or not # nodeD.nodestack.delay(delayer(20, PRIMARY)) nodeD.nodeIbStasher.delay(delayerMsgTuple(20, Primary)) checkPoolReady(looper=looper, nodes=nodeSet) client1 = setupClient(looper, nodeSet, tmpdir=tdir_for_func) request = sendRandomRequest(client1) # TODO Rethink this instNo = 0 for i in range(3): node = nodeSet.getNode(nodeNames[i]) # Nodes A, B and C should have received PROPAGATE request from Node D looper.run( eventually(checkIfPropagateRecvdFromNode, node, nodeD, request.clientId, request.reqId, retryWait=1, timeout=10)) # Node D should have 1 pending PRE-PREPARE request def assertOnePrePrepare(): assert len(getPendingRequestsForReplica(nodeD.replicas[instNo], PrePrepare)) == 1 looper.run(eventually(assertOnePrePrepare, retryWait=1, timeout=10)) # Node D should have 2 pending PREPARE requests(from node B and C) def assertTwoPrepare(): assert len(getPendingRequestsForReplica(nodeD.replicas[instNo], Prepare)) == 2 looper.run(eventually(assertTwoPrepare, retryWait=1, timeout=10)) # Node D should have no pending PRE-PREPARE, PREPARE or COMMIT requests for reqType in [PrePrepare, Prepare, Commit]: looper.run(eventually(lambda: assertLength( getPendingRequestsForReplica(nodeD.replicas[instNo], reqType), 0), retryWait=1, timeout=20))
def testPrimaryElectionWithAClearWinner(electContFixture, looper, keySharedNodes): """ Primary selection (Sunny Day) A, B, C, D, E A, B, C, D startup. E is lagging. A sees the minimum number of nodes first, and then sends out a NOMINATE(A) message B, C, D all see the NOMINATE(A) message from A, and respond with NOMINATE(A) message to all other nodes A sees three other NOMINATE(A) votes (from B, C, D) A sees that A is the clear winner (2f+1 total), and sends PRIMARY(A) to all nodes B sees two more NOMINATE(A) votes (from C and D) B sees that A is the clear winner (2f+1 total), and sends PRIMARY(A) to all nodes C sees two more NOMINATE(A) votes (from B and D) C sees that A is the clear winner (2f+1 total), and sends PRIMARY(A) to all nodes D sees two more NOMINATE(A) votes (from B and C) D sees that A is the clear winner (2f+1 total), and sends PRIMARY(A) to all nodes A sees at least two other PRIMARY(A) votes (3 including it's own) selects A as primary B sees at least two other PRIMARY(A) votes (3 including it's own) selects A as primary C sees at least two other PRIMARY(A) votes (3 including it's own) selects A as primary D sees at least two other PRIMARY(A) votes (3 including it's own) selects A as primary """ nodeSet = keySharedNodes A, B, C, D = nodeSet.nodes.values() nodesBCD = [B, C, D] # attempting to use a raet stack delay... not successful # nodeBPort = nodesBCD[0].stack.ha[1] # delayRef = RaetDelay(TrnsKind.alive, PcktKind.ack, nodeBPort) # nodeA.stack.delay(4, delayRef) checkPoolReady(looper, nodeSet) # Checking whether one of the replicas of Node A nominated itself looper.run(eventually(checkNomination, A, A.name, retryWait=1, timeout=10)) for n in nodesBCD: # Checking whether Node B, C and D nominated Node A looper.run(eventually(checkNomination, n, A.name, retryWait=1, timeout=10)) checkProtocolInstanceSetup(looper=looper, nodes=nodeSet, retryWait=1, timeout=10) assert A.hasPrimary
def testPrimaryElectionContested(electContFixture, looper, keySharedNodes): """ Primary selection (Rainy Day) A, B, C, D, E A, B, C, D startup. E is lagging. A sees the minimum number of nodes, and then sends Nominate(A) At the same exact time, B sees the minimum number of nodes, and then sends out Nominate(B) A sees B sending Nominate(B), but it has already nominated itself, so it does nothing B sees A sending Nominate(A), but it has already nominated itself, so it does nothing C sees A sending Nominate(A), and sends Nominate(A) D sees A sending Nominate(A), and sends Nominate(A) All nodes see that B nominated B and A, C, and D all nominated A Because the votes for A exceeds the votes for B, all send out Primary(A) TODO's (see below) All see the others have sent Primary A, and then the nodes record who is the Primary. """ # TODO what if not all send out Primary(A)? # TODO what if there are big delays in messages getting delivered? nodeSet = keySharedNodes A, B, C, D = nodeSet.nodes.values() checkPoolReady(looper, nodeSet) logging.debug("Check nomination") # Checking whether Node A nominated itself looper.run(eventually(checkNomination, A, A.name, retryWait=1, timeout=10)) # Checking whether Node B nominated itself looper.run(eventually(checkNomination, B, B.name, retryWait=1, timeout=10)) for n in [C, D]: # Checking whether Node C and Node D nominated Node A looper.run(eventually(checkNomination, n, A.name, retryWait=1, timeout=10)) checkProtocolInstanceSetup(looper=looper, nodes=nodeSet, retryWait=1, timeout=45) # Node D should not be primary assert not D.hasPrimary # A should have at least one primary assert A.hasPrimary
def testPrimaryElectionWithTie(electTieFixture, looper, keySharedNodes): """ Primary selection (Rainy Day) A, B, C, D, E A, B, C, D startup. E is lagging. A sees the minimum number of nodes, and then sends Nominate(A) At the same exact time, B sees the minimum number of nodes, and then sends out Nominate(B) A sees B sending Nominate(B), but it has already nominated itself, so it does nothing B sees A sending Nominate(A), but it has already nominated itself, so it does nothing C sees A sending Nominate(A), and sends Nominate(A) D sees B sending Nominate(B), and sends Nominate(B) There's a split. C and A think A is the primary, B and D think B is the primary All nodes can see that there is a split. Each sends out Reelection([A,B]) A and B both see Reelection([A,B]) from themselves as well as the other 3 (the number from others should be at least f+1), 1. they wait a random amount of time (between 0 and 2 seconds), 2. they each send out a Nominate(self) Voting is repeated until we have a good election. """ # TODO optimize the sending messages in batches, for example, we don't # send messages more often than 400 milliseconds. Once those 400 # millis have passed, we send the several queued messages in one # batch. nodeSet = keySharedNodes A, B, C, D = nodeSet.nodes.values() checkPoolReady(looper, nodeSet.nodes.values()) for node in nodeSet.nodes.values(): for instId, replica in enumerate(node.elector.replicas): logging.debug("replica {} {} with votes {}". format(replica.name, replica.instId, node.elector.nominations.get(instId, {}))) logging.debug("Check nomination") # Checking whether Node A nominated itself looper.run(eventually(checkNomination, A, A.name, retryWait=1, timeout=10)) # Checking whether Node B nominated itself looper.run(eventually(checkNomination, B, B.name, retryWait=1, timeout=10)) # Checking whether Node C nominated Node A looper.run(eventually(checkNomination, C, A.name, retryWait=1, timeout=10)) # Checking whether Node D nominated Node D looper.run(eventually(checkNomination, D, B.name, retryWait=1, timeout=10)) # No node should be primary for node in nodeSet.nodes.values(): assert node.hasPrimary is False for node in nodeSet.nodes.values(): node.resetDelays() # TODO Check for spylog of `eatReelection` after putting spy on eaters too checkProtocolInstanceSetup(looper=looper, nodes=nodeSet, retryWait=1, timeout=60) # TODO testPrimaryElectionWithTieButOneNodeDoesntKnowIt # TODO add E back in after election is complete, or even before it's # complete (E jumps in right in the middle of an election) # When E joins, it needs to be able to ping the others for the # current state (A could send to E the votes of the other nodes, # so E would know if B is being maliciious), and the others respond, # and E can make a determination based on those responses, even if # one is malicious. # TODO We need to trap for the case when a node is being inconsistent. For example: """