def testNodesConnectsWhenOneNodeIsLate(): with TemporaryDirectory() as td: with Looper() as looper: nodes = [] names = list(nodeReg.keys()) logger.debug("Node names: {}".format(names)) def create(name): node = Node(name, nodeReg, basedirpath=td) looper.add(node) node.startKeySharing() nodes.append(node) for name in names[:3]: create(name) looper.run(checkNodesConnected(nodes)) # wait for the election to complete with the first three nodes looper.runFor(10) # create the fourth and see that it learns who the primaries are # from the other nodes create(names[3]) checkProtocolInstanceSetup(looper, nodes, timeout=10)
def testPrimaryElectionWithAClearWinner(electContFixture, looper, keySharedNodes): """ Primary selection (Sunny Day) A, B, C, D, E A, B, C, D startup. E is lagging. A sees the minimum number of nodes first, and then sends out a NOMINATE(A) message B, C, D all see the NOMINATE(A) message from A, and respond with NOMINATE(A) message to all other nodes A sees three other NOMINATE(A) votes (from B, C, D) A sees that A is the clear winner (2f+1 total), and sends PRIMARY(A) to all nodes B sees two more NOMINATE(A) votes (from C and D) B sees that A is the clear winner (2f+1 total), and sends PRIMARY(A) to all nodes C sees two more NOMINATE(A) votes (from B and D) C sees that A is the clear winner (2f+1 total), and sends PRIMARY(A) to all nodes D sees two more NOMINATE(A) votes (from B and C) D sees that A is the clear winner (2f+1 total), and sends PRIMARY(A) to all nodes A sees at least two other PRIMARY(A) votes (3 including it's own) selects A as primary B sees at least two other PRIMARY(A) votes (3 including it's own) selects A as primary C sees at least two other PRIMARY(A) votes (3 including it's own) selects A as primary D sees at least two other PRIMARY(A) votes (3 including it's own) selects A as primary """ nodeSet = keySharedNodes A, B, C, D = nodeSet.nodes.values() nodesBCD = [B, C, D] # attempting to use a raet stack delay... not successful # nodeBPort = nodesBCD[0].stack.ha[1] # delayRef = RaetDelay(TrnsKind.alive, PcktKind.ack, nodeBPort) # nodeA.stack.delay(4, delayRef) checkPoolReady(looper, nodeSet) # Checking whether one of the replicas of Node A nominated itself looper.run(eventually(checkNomination, A, A.name, retryWait=1, timeout=10)) for n in nodesBCD: # Checking whether Node B, C and D nominated Node A looper.run(eventually(checkNomination, n, A.name, retryWait=1, timeout=10)) checkProtocolInstanceSetup(looper=looper, nodes=nodeSet, retryWait=1, timeout=10) assert A.hasPrimary
def testPrimarySelectionAfterViewChange(looper, nodeSet, ready, primaryReplicas, viewChangeDone): """ Test that primary replica of a protocol instance shifts to a new node after a view change. """ # Primary replicas before view change prBeforeVC = primaryReplicas # Primary replicas after view change instanceCount = getNoInstances(nodeCount) prAfterVC = [getPrimaryReplica(nodeSet, i) for i in range(instanceCount)] # Primary replicas have moved to the next node for br, ar in zip(prBeforeVC, prAfterVC): assert ar.node.rank - br.node.rank == 1 checkProtocolInstanceSetup(looper, nodeSet, retryWait=1, timeout=5)
def testPrimaryElectionContested(electContFixture, looper, keySharedNodes): """ Primary selection (Rainy Day) A, B, C, D, E A, B, C, D startup. E is lagging. A sees the minimum number of nodes, and then sends Nominate(A) At the same exact time, B sees the minimum number of nodes, and then sends out Nominate(B) A sees B sending Nominate(B), but it has already nominated itself, so it does nothing B sees A sending Nominate(A), but it has already nominated itself, so it does nothing C sees A sending Nominate(A), and sends Nominate(A) D sees A sending Nominate(A), and sends Nominate(A) All nodes see that B nominated B and A, C, and D all nominated A Because the votes for A exceeds the votes for B, all send out Primary(A) TODO's (see below) All see the others have sent Primary A, and then the nodes record who is the Primary. """ # TODO what if not all send out Primary(A)? # TODO what if there are big delays in messages getting delivered? nodeSet = keySharedNodes A, B, C, D = nodeSet.nodes.values() checkPoolReady(looper, nodeSet) logging.debug("Check nomination") # Checking whether Node A nominated itself looper.run(eventually(checkNomination, A, A.name, retryWait=1, timeout=10)) # Checking whether Node B nominated itself looper.run(eventually(checkNomination, B, B.name, retryWait=1, timeout=10)) for n in [C, D]: # Checking whether Node C and Node D nominated Node A looper.run(eventually(checkNomination, n, A.name, retryWait=1, timeout=10)) checkProtocolInstanceSetup(looper=looper, nodes=nodeSet, retryWait=1, timeout=45) # Node D should not be primary assert not D.hasPrimary # A should have at least one primary assert A.hasPrimary
def testElectionsAfterViewChange(delayedPerf, looper: Looper, nodeSet: TestNodeSet, up, client1): """ Test that a primary election does happen after a view change """ # Delay processing of PRE-PREPARE from all non primary replicas of master so master's throughput falls # and view changes nonPrimReps = getNonPrimaryReplicas(nodeSet, 0) for r in nonPrimReps: r.node.nodeIbStasher.delay(ppDelay(10, 0)) sendReqsToNodesAndVerifySuffReplies(looper, client1, 4) # Ensure view change happened for both node and its primary elector for node in nodeSet: looper.run(eventually(partial(checkViewChangeInitiatedForNode, node, 0), retryWait=1, timeout=20)) # Ensure elections are done again and pool is setup again with appropriate protocol instances and each # protocol instance is setup properly too checkProtocolInstanceSetup(looper, nodeSet, retryWait=1, timeout=30)
def testPrimarySelectionAfterPoolReady(looper, nodeSet, ready): """ Once the pool is ready(node has connected to at least 3 other nodes), appropriate primary replicas should be selected. """ def checkPrimaryPlacement(): # Node names sorted by rank sortedNodeNames = sorted(nodeSet.nodes.values(), key=operator.attrgetter("rank")) for idx, node in enumerate(sortedNodeNames): # For instance 0, the primary replica should be on the node with rank 0 if idx == 0: primaryName = Replica.generateName(sortedNodeNames[idx], 0) assert node.replicas[0].isPrimary assert not node.replicas[1].isPrimary assert not node.replicas[2].isPrimary # For instance 1, the primary replica should be on the node with rank 1 if idx == 1: primaryName = Replica.generateName(sortedNodeNames[idx], 1) assert not node.replicas[0].isPrimary assert node.replicas[1].isPrimary assert not node.replicas[2].isPrimary # For instance 2, the primary replica should be on the node with rank 2 if idx == 2: primaryName = Replica.generateName(sortedNodeNames[idx], 2) assert not node.replicas[0].isPrimary assert not node.replicas[1].isPrimary assert node.replicas[2].isPrimary # Check if the primary is on the correct node looper.run(eventually(checkPrimaryPlacement, retryWait=1, timeout=10)) # Check if every protocol instance has one and only one primary and any node # has no more than one primary checkProtocolInstanceSetup(looper, nodeSet, retryWait=1, timeout=5)
def testPrimaryElectionWithTie(electTieFixture, looper, keySharedNodes): """ Primary selection (Rainy Day) A, B, C, D, E A, B, C, D startup. E is lagging. A sees the minimum number of nodes, and then sends Nominate(A) At the same exact time, B sees the minimum number of nodes, and then sends out Nominate(B) A sees B sending Nominate(B), but it has already nominated itself, so it does nothing B sees A sending Nominate(A), but it has already nominated itself, so it does nothing C sees A sending Nominate(A), and sends Nominate(A) D sees B sending Nominate(B), and sends Nominate(B) There's a split. C and A think A is the primary, B and D think B is the primary All nodes can see that there is a split. Each sends out Reelection([A,B]) A and B both see Reelection([A,B]) from themselves as well as the other 3 (the number from others should be at least f+1), 1. they wait a random amount of time (between 0 and 2 seconds), 2. they each send out a Nominate(self) Voting is repeated until we have a good election. """ # TODO optimize the sending messages in batches, for example, we don't # send messages more often than 400 milliseconds. Once those 400 # millis have passed, we send the several queued messages in one # batch. nodeSet = keySharedNodes A, B, C, D = nodeSet.nodes.values() checkPoolReady(looper, nodeSet.nodes.values()) for node in nodeSet.nodes.values(): for instId, replica in enumerate(node.elector.replicas): logging.debug("replica {} {} with votes {}". format(replica.name, replica.instId, node.elector.nominations.get(instId, {}))) logging.debug("Check nomination") # Checking whether Node A nominated itself looper.run(eventually(checkNomination, A, A.name, retryWait=1, timeout=10)) # Checking whether Node B nominated itself looper.run(eventually(checkNomination, B, B.name, retryWait=1, timeout=10)) # Checking whether Node C nominated Node A looper.run(eventually(checkNomination, C, A.name, retryWait=1, timeout=10)) # Checking whether Node D nominated Node D looper.run(eventually(checkNomination, D, B.name, retryWait=1, timeout=10)) # No node should be primary for node in nodeSet.nodes.values(): assert node.hasPrimary is False for node in nodeSet.nodes.values(): node.resetDelays() # TODO Check for spylog of `eatReelection` after putting spy on eaters too checkProtocolInstanceSetup(looper=looper, nodes=nodeSet, retryWait=1, timeout=60) # TODO testPrimaryElectionWithTieButOneNodeDoesntKnowIt # TODO add E back in after election is complete, or even before it's # complete (E jumps in right in the middle of an election) # When E joins, it needs to be able to ping the others for the # current state (A could send to E the votes of the other nodes, # so E would know if B is being maliciious), and the others respond, # and E can make a determination based on those responses, even if # one is malicious. # TODO We need to trap for the case when a node is being inconsistent. For example: """
def pool(looper, nodeSet): for n in nodeSet: # type: TestNode n.startKeySharing() looper.run(checkNodesConnected(nodeSet)) checkProtocolInstanceSetup(looper, nodeSet, timeout=5) return adict(looper=looper, nodeset=nodeSet)