def testTestNodeDelay(tdir_for_func): nodeNames = {"testA", "testB"} with TestNodeSet(names=nodeNames, tmpdir=tdir_for_func) as nodes: nodeA = nodes.getNode("testA") nodeB = nodes.getNode("testB") with Looper(nodes) as looper: for n in nodes: n.startKeySharing() logging.debug("connect") looper.run(checkNodesConnected(nodes)) logging.debug("send one message, without delay") msg = randomMsg() looper.run(sendMsgAndCheck(nodes, nodeA, nodeB, msg, 1)) logging.debug( "set delay, then send another message and find that it doesn't arrive") msg = randomMsg() nodeB.nodeIbStasher.delay(delayerMsgTuple(4, type(msg), nodeA.name)) with pytest.raises(AssertionError): looper.run(sendMsgAndCheck(nodes, nodeA, nodeB, msg, 3)) logging.debug( "but then find that it arrives after the delay duration has passed") looper.run(sendMsgAndCheck(nodes, nodeA, nodeB, msg, 2)) logging.debug( "reset the delay, and find another message comes quickly") nodeB.nodeIbStasher.resetDelays() msg = randomMsg() looper.run(sendMsgAndCheck(nodes, nodeA, nodeB, msg, 1))
def testNodesConnectsWhenOneNodeIsLate(): with TemporaryDirectory() as td: with Looper() as looper: nodes = [] names = list(nodeReg.keys()) logger.debug("Node names: {}".format(names)) def create(name): node = Node(name, nodeReg, basedirpath=td) looper.add(node) node.startKeySharing() nodes.append(node) for name in names[:3]: create(name) looper.run(checkNodesConnected(nodes)) # wait for the election to complete with the first three nodes looper.runFor(10) # create the fourth and see that it learns who the primaries are # from the other nodes create(names[3]) checkProtocolInstanceSetup(looper, nodes, timeout=10)
def testPrimaryElectionCase2(case2Setup, looper, keySharedNodes): """ Case 2 - A node making nominations for a multiple other nodes. Consider 4 nodes A, B, C, and D. Lets say node B is malicious and nominates node C to all nodes. Again node B nominates node D to all nodes. """ nodeSet = keySharedNodes A, B, C, D = nodeSet.nodes.values() looper.run(checkNodesConnected(nodeSet)) # Node B sends multiple NOMINATE msgs but only after A has nominated itself looper.run(eventually(checkNomination, A, A.name, retryWait=.25, timeout=1)) instId = getSelfNominationByNode(A) BRep = Replica.generateName(B.name, instId) CRep = Replica.generateName(C.name, instId) DRep = Replica.generateName(D.name, instId) # Node B first sends NOMINATE msgs for Node C to all nodes B.send(Nomination(CRep, instId, B.viewNo)) # Node B sends NOMINATE msgs for Node D to all nodes B.send(Nomination(DRep, instId, B.viewNo)) # Ensure elections are done ensureElectionsDone(looper=looper, nodes=nodeSet, retryWait=1, timeout=45) # All nodes from node A, node C, node D(node B is malicious anyway so # not considering it) should have nomination for node C from node B since # node B first nominated node C for node in [A, C, D]: assert node.elector.nominations[instId][BRep] == CRep
def testPrimaryElectionCase4(case4Setup, looper): """ Case 4 - A node making multiple primary declarations for a particular node. Consider 4 nodes A, B, C and D. Lets say node B is malicious and is repeatedly declaring Node D as primary """ allNodes = case4Setup A, B, C, D = allNodes looper.run(checkNodesConnected(allNodes)) # Node B sends multiple declarations of node D's 0th protocol instance as # primary to all nodes for i in range(5): B.send(Primary(D.name, 0, B.viewNo)) # No node from node A, node C, node D(node B is malicious anyway so not # considering it) should have more than one primary declaration for node # D since node D is slow. The one primary declaration for node D, # that nodes A, C and D might have would be because of node B def x(): primDecs = list(node.elector.primaryDeclarations[0].values()) assert primDecs.count(D.name) <= 1 for node in (A, C, D): looper.run(eventually(x, retryWait=0.5, timeout=2)) ensureElectionsDone(looper=looper, nodes=allNodes, retryWait=1, timeout=45) # Node D should not have any primary replica assert not D.hasPrimary
def testNodesConnectWhenTheyAllStartAtOnce(): with TemporaryDirectory() as td: with Looper() as looper: nodes = [] for name in nodeReg: node = Node(name, nodeReg, basedirpath=td) looper.add(node) node.startKeySharing() nodes.append(node) looper.run(checkNodesConnected(nodes))
def testKeyShareParty(tdir_for_func): """ connections to all nodes should be successfully established when key sharing is enabled. """ nodeReg = genNodeReg(5) logging.debug("-----sharing keys-----") with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet: with Looper(nodeSet) as looper: for n in nodeSet: n.startKeySharing() looper.run(checkNodesConnected(nodeSet)) logging.debug("-----key sharing done, connect after key sharing-----") with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet: with Looper(nodeSet) as loop: loop.run(checkNodesConnected(nodeSet), msgAll(nodeSet))
def testNodesComingUpAtDifferentTimes(): console = getConsole() console.reinit(flushy=True, verbosity=console.Wordage.verbose) with TemporaryDirectory() as td: print("temporary directory: {}".format(td)) with Looper() as looper: nodes = [] names = list(nodeReg.keys()) shuffle(names) waits = [randint(1, 10) for _ in names] rwaits = [randint(1, 10) for _ in names] for i, name in enumerate(names): node = Node(name, nodeReg, basedirpath=td) looper.add(node) node.startKeySharing() nodes.append(node) looper.runFor(waits[i]) looper.run(checkNodesConnected(nodes, overrideTimeout=10)) print("connects") print("node order: {}".format(names)) print("waits: {}".format(waits)) for n in nodes: n.stop() for i, n in enumerate(nodes): n.start() looper.runFor(rwaits[i]) looper.runFor(3) looper.run(checkNodesConnected(nodes, overrideTimeout=10)) print("reconnects") print("node order: {}".format(names)) print("rwaits: {}".format(rwaits))
def testNodeConnection(): console = getConsole() console.reinit(flushy=True, verbosity=console.Wordage.verbose) with TemporaryDirectory() as td: print("temporary directory: {}".format(td)) with Looper() as looper: names = ["Alpha", "Beta"] print(names) nrg = {n: nodeReg[n] for n in names} A, B = [Node(name, nrg, basedirpath=td) for name in names] looper.add(A) A.startKeySharing() looper.runFor(4) print("wait done") looper.add(B) B.startKeySharing() looper.runFor(4) looper.run(checkNodesConnected([A, B])) looper.stopall() A.start() looper.runFor(4) B.start() looper.run(checkNodesConnected([A, B]))
def testConnectWithoutKeySharingFails(tdir_for_func): """ attempts at connecting to nodes when key sharing is disabled must fail """ nodeNames = genNodeNames(5) with TestNodeSet(names=nodeNames, tmpdir=tdir_for_func, keyshare=False) as nodes: with Looper(nodes) as looper: try: looper.run( checkNodesConnected(nodes, RemoteState(None, None, None))) except RemoteNotFound: pass except KeyError as ex: assert [n for n in nodeNames if n == ex.args[0]] except Exception: raise
def pool(looper, nodeSet): for n in nodeSet: # type: TestNode n.startKeySharing() looper.run(checkNodesConnected(nodeSet)) checkProtocolInstanceSetup(looper, nodeSet, timeout=5) return adict(looper=looper, nodeset=nodeSet)
def ready(looper, keySharedNodes): looper.run(checkNodesConnected(keySharedNodes)) return keySharedNodes
def testProtocolInstanceCannotBecomeActiveWithLessThanFourServers( tdir_for_func): """ A protocol instance must have at least 4 nodes to come up. The status of the nodes will change from starting to started only after the addition of the fourth node to the system. """ nodeCount = 16 f = 5 minimumNodesToBeUp = 16 - f nodeNames = genNodeNames(nodeCount) with TestNodeSet(names=nodeNames, tmpdir=tdir_for_func) as nodeSet: with Looper(nodeSet) as looper: for n in nodeSet: n.startKeySharing() # helpers def genExpectedStates(connecteds: Iterable[str]): return { nn: CONNECTED if nn in connecteds else JOINED_NOT_ALLOWED for nn in nodeNames} def checkNodeStatusRemotesAndF(expectedStatus: Status, nodeIdx: int): for node in nodeSet.nodes.values(): checkNodeRemotes(node, genExpectedStates(nodeNames[:nodeIdx + 1])) assert node.status == expectedStatus def addNodeBackAndCheck(nodeIdx: int, expectedStatus: Status): logging.info("Add back the {} node and see status of {}". format(ordinal(nodeIdx + 1), expectedStatus)) addNodeBack(nodeSet, looper, nodeNames[nodeIdx]) looper.run( eventually(checkNodeStatusRemotesAndF, expectedStatus, nodeIdx, retryWait=1, timeout=30)) # tests logging.debug("Sharing keys") looper.run(checkNodesConnected(nodeSet)) logging.debug("Remove all the nodes") for n in nodeNames: looper.removeProdable(nodeSet.nodes[n]) nodeSet.removeNode(n, shouldClean=False) logging.debug("Add nodes back one at a time") for i in range(nodeCount): nodes = i + 1 if nodes < minimumNodesToBeUp: expectedStatus = Status.starting elif nodes < nodeCount: expectedStatus = Status.started_hungry else: expectedStatus = Status.started addNodeBackAndCheck(i, expectedStatus)