def testSerialFlooding(self): # Start a bootstrap node (status, self.bsNode, _observer) = yield TestUtils.startupBootstrapNode(self.myIP, 12345, 'localhost') self.assertTrue(status, 'Could not build bootstrap node') self.allNodes.append(self.bsNode) self.bsNode.addMessageObserver(self.messageReceived) # Start client nodes for i in range(numNodes): (status, node, observer) = yield TestUtils.startupClientNode(self.myIP, 12346+i, 'localhost', self.bsNode.nodeLocation) self.assertTrue(status, 'Could not startupClientNode') self.allNodes.append(node) observer = MyMessageObserver() node.addMessageObserver(observer.messageReceived) observer.node = node self.allObservers.append(observer) # Wait for flooding to reach all the nodes waiter = ConnectivityCounter() yield waiter.waitForConnectivity(numNodes, self.bsNode) # Does not count bsNode itself. # Now do the stress test status = yield self.doStressTest() # Now close it all down! yield self.allLeave() # Wait a second or two for network timeouts yield TestUtils.wait(9) defer.returnValue(True)
def buildNetwork(self): global startingPort # Create Bootstrap port = 12345 bootstrapNodeLocation = NodeLocation(None, self.myIP, port) self.allNodeObservers = [] self.allBootstrapObservers = [] self.allNodes = [] enclave = "theEnc" # Start a bootstrap node (status, self.bsNode, observer) = yield TestUtils.startupBootstrapNode(self.myIP, port, enclave) self.assertTrue(status, "Could not build bootstrap node") self.allBootstrapObservers.append(observer) # Add X nodes to each enclave for _ in range(numNodes): # Add a node to the enclave (status, node, observer) = yield TestUtils.startupClientNode( self.myIP, startingPort, enclave, bootstrapNodeLocation ) self.assertTrue(status, "Could not startupClientNode") startingPort += 1 self.allNodes.append(node) self.allNodeObservers.append(observer) # Wait for flooding to reach all the nodes waiter = ConnectivityCounter() yield waiter.waitForConnectivity(numNodes + 1, self.bsNode) # Does not bsNode . defer.returnValue(True)
def testSerialP2PSending(self): # Start a bootstrap node (status, self.bsNode, _observer) = yield TestUtils.startupBootstrapNode(self.myIP, 12345, 'localhost') self.assertTrue(status, 'Could not build bootstrap node') self.allNodes.append(self.bsNode) self.bsNode.addMessageObserver(self.messageReceived) # Start client nodes log.msg("Building nodes...") for i in range(numNodes): (status, node, observer) = yield TestUtils.startupClientNode(self.myIP, 12346+i, 'localhost', self.bsNode.nodeLocation) self.assertTrue(status, 'Could not startupClientNode') self.allNodes.append(node) # Wait for flooding to reach all the nodes waiter = ConnectivityCounter() yield waiter.waitForConnectivity(numNodes, self.bsNode) # Does not count bsNode itself. # Do the real test status = yield self.doStressTest() # Now close it all down! yield self.allLeave() # Wait a second or two yield TestUtils.wait(3+Config.CONNECTION_CACHE_DELAY) defer.returnValue(True)
def testDisconnectedBootstraps(self): '''Create a BS node and some clients. Create another bootstrap node and some clients (so we essentially have two rings). Verify, that the bootstrap nodes autodiscover each other and connect together ''' global startingPort # Create Bootstrap port = 12345 bootstrapNodeLocation = NodeLocation(None, self.myIP, port) bootstrapNodeLocation2 = NodeLocation(None, self.myIP, port+1) self.allNodes = [] self.allMetricsObservers = [] self.allTestObservers = [] # Build the BS node (status, bsClientAPI, bsNetworkAPI) = yield TestUtils.startNodeUsingAPI(bootstrapNodeLocation.ip, bootstrapNodeLocation.port, None, 'theEnclave', True, True) self.allMetricsObservers.append(MetricsMessageObserver(bsNetworkAPI.chordNode)) self.assertTrue(status, 'Could not build bootstrap node') # Build second BS node (status, bsClientAPI2, bsNetworkAPI2) = yield TestUtils.startNodeUsingAPI(bootstrapNodeLocation2.ip, bootstrapNodeLocation2.port, None, 'theEnclave', True, True) self.allMetricsObservers.append(MetricsMessageObserver(bsNetworkAPI2.chordNode)) self.assertTrue(status, 'Could not build bootstrap node 2') # Build the client node (status, clClientAPI, clNetworkAPI) = yield TestUtils.startNodeUsingAPI(self.myIP, port+2, bootstrapNodeLocation, 'theEnclave', False, False) self.allMetricsObservers.append(MetricsMessageObserver(clNetworkAPI.chordNode)) self.assertTrue(status, 'Could not build client node') # Build the client node (status, clClientAPI2, clNetworkAPI2) = yield TestUtils.startNodeUsingAPI(self.myIP, port+3, bootstrapNodeLocation2, 'theEnclave', False, False) self.allMetricsObservers.append(MetricsMessageObserver(clNetworkAPI2.chordNode)) self.assertTrue(status, 'Could not build client node') # Wait for flooding to reach all the nodes waiter = ConnectivityCounter() yield waiter.waitForConnectivity(3, clNetworkAPI.chordNode) # Does not count clNode itself. # Now shut everything down yield clNetworkAPI.disconnect() yield clNetworkAPI2.disconnect() yield bsNetworkAPI.disconnect() yield bsNetworkAPI2.disconnect() if Config.USE_CONNECTION_CACHE: yield TestUtils.waitForConnectionCache() else: yield TestUtils.wait(5) defer.returnValue(True)
def buildNetwork(self): global startingPort # Create two Bootstrap nodes port = 12345 self.allNodeObservers = [] self.allBootstrapObservers = [] self.allNodes = [] # Start a bootstrap node (status, self.bsNode1, observer) = yield TestUtils.startupBootstrapNode(self.myIP, port, allEnclaves[0]) self.assertTrue(status, 'Could not build bootstrap node') self.allBootstrapObservers.append(observer) # Start another bootstrap node (status, self.bsNode2, observer) = yield TestUtils.startupBootstrapNode(self.myIP, port+1, allEnclaves[1]) self.assertTrue(status, 'Could not build bootstrap node') self.allBootstrapObservers.append(observer) # Add some nodes to each bootstrap node for _ in range(numNodes): # Add a node to the enclave (status, node, observer) = yield TestUtils.startupClientNode(self.myIP, startingPort, allEnclaves[0], self.bsNode1.nodeLocation) self.assertTrue(status, 'Could not startupClientNode') startingPort += 1 self.allNodes.append(node) self.allNodeObservers.append(observer) for _ in range(numNodes): # Add a node to the enclave (status, node, observer) = yield TestUtils.startupClientNode(self.myIP, startingPort, allEnclaves[1], self.bsNode2.nodeLocation) self.assertTrue(status, 'Could not startupClientNode') startingPort += 1 self.allNodes.append(node) self.allNodeObservers.append(observer) # Wait for flooding to reach all the nodes waiter = ConnectivityCounter() yield waiter.waitForConnectivity(numNodes, self.bsNode1) # Does not count bsNode itself. yield waiter.waitForConnectivity(numNodes, self.bsNode2) # Does not count bsNode itself. defer.returnValue(True)
def buildNetwork(self): global startingPort # Create Bootstrap port = 12345 bootstrapNodeLocation = NodeLocation(None, self.myIP, port) self.allNodeObservers = [] self.allNetworkAPIs = [] self.allClientAPIs = [] # Start a bootstrap node log.msg("building BS node...") (status, clientAPI, networkAPI) = yield TestUtils.startNodeUsingAPI(self.myIP, port, None, 'theEnclave', False, True) self.assertTrue(status, 'Could not build bootstrap node') node = networkAPI.chordNode # This is very highly coupled -- bad MetricsMessageObserver(node) self.observer = TestMessageObserver(node, networkAPI) self.bsNode = node log.msg("building client nodes...") # Add X nodes to each enclave for _ in range(numNodes): # Add a node to the enclave (status, clientAPI, networkAPI) = yield TestUtils.startNodeUsingAPI(self.myIP, startingPort, bootstrapNodeLocation, 'theEnclave', False, False) self.assertTrue(status, 'Could not startupClientNode') node = networkAPI.chordNode # This is very highly coupled -- bad MetricsMessageObserver(node) observer = TestMessageObserver(node, networkAPI) startingPort += 1 self.allNetworkAPIs.append(networkAPI) self.allNodeObservers.append(observer) self.allClientAPIs.append(clientAPI) # Wait for flooding to reach all the nodes waiter = ConnectivityCounter() yield waiter.waitForConnectivity(numNodes+1, self.bsNode) # Does count bsNode itself. defer.returnValue(True)
def buildNetwork(self): global startingPort # Create Bootstrap port = 12345 bootstrapNodeLocation = NodeLocation(None, self.myIP, port) self.allNodeObservers = [] self.allBootstrapObservers = [] self.allNodes = [] # Start a bootstrap node log.msg("---> Startup Bootstrap Node", system="EnclaveP2P Test") (status, self.bsNode, observer) = yield TestUtils.startupBootstrapNode(self.myIP, port, allEnclaves[0]) self.assertTrue(status, 'Could not build bootstrap node') self.allBootstrapObservers.append(observer) # Join another enclave enableAutoDiscovery = True bootstrapNodeList = [ NodeLocation(None, self.myIP, port) ] status = yield self.bsNode.joinEnclave(bootstrapNodeList, enableAutoDiscovery, None, isBootstrapNode=True, enclaveStr=allEnclaves[1]) # Add X nodes to each enclave for enclave in allEnclaves: for _ in range(numNodes): log.msg("---> Add node to enclave", system="EnclaveP2P Test") # Add a node to the enclave (status, node, observer) = yield TestUtils.startupClientNode(self.myIP, startingPort, enclave, bootstrapNodeLocation) self.assertTrue(status, 'Could not startupClientNode') startingPort += 1 self.allNodes.append(node) self.allNodeObservers.append(observer) # Wait for flooding to reach all the nodes waiter = ConnectivityCounter() numberOfNodes = (len(allEnclaves) * numNodes) yield waiter.waitForConnectivity(numberOfNodes+1, self.bsNode) # Does not count bsNode itself. defer.returnValue(True)
def testNodeDisconnection(self): """Disconnect a node from the ring and do a few things. - Send a message to that node (verify error condition) - Send a flooding message -- wait until connectivity happens for all other nodes. """ # Disconnect a node nodeIndex = random.randint(0, numNodes - 1) # Node to disconnect nodeLoc = self.allNodes[nodeIndex].nodeLocation log.msg("testNodeDisconnection: Node is leaving [%s]...\n" % nodeLoc) ConnectionCache._printCache() yield self.allNodes[nodeIndex].leave() log.msg("testNodeDisconnection: Node left...\n") try: # Now wait for full connectivity to come back waiter = ConnectivityCounter() yield waiter.waitForConnectivity(numNodes, self.bsNode) # Does count bsNode. except error.ConnectionRefusedError: log.err("NodeDisconnect test got a connection refused error [2]") log.err() # Now wait for the disconnect to fully take hold yield TestUtils.wait(30) # Network timeout time print("DEBUG testNodeDisc \n\n\n\n") ConnectionCache._printCache() try: # P2P from bootstrap to node which left yield self.sendFailingP2PMsg(self.bsNode, self.allNodes[nodeIndex]) except error.ConnectionRefusedError: log.msg("\n\nNodeDisconnect test got a connection refused error [3], which is okay!\n\n") defer.returnValue(True)
def testEnclaveFlooding(self): """Build 2 enclaves and test flooding to each and all.""" global startingPort # Create Bootstrap port = 12345 bootstrapNodeLocation = NodeLocation(None, self.myIP, port) allNodes = [] allNodeObservers = [] allBootstrapObservers = [] self.allNodes = allNodes # Start a bootstrap node (status, bsNode, observer) = yield TestUtils.startupBootstrapNode(self.myIP, port, allEnclaves[0]) self.assertTrue(status, "Could not build bootstrap node") allBootstrapObservers.append(observer) self.bsNode = bsNode # Join another enclave bootstrapNodeList = None enableAutoDiscovery = True status = yield self.bsNode.joinEnclave( bootstrapNodeList, enableAutoDiscovery, None, isBootstrapNode=True, enclaveStr=allEnclaves[1] ) # Add X nodes to each enclave for enclave in allEnclaves: for _ in range(numNodes): # Add a node to the enclave (status, node, observer) = yield TestUtils.startupClientNode( self.myIP, startingPort, enclave, bootstrapNodeLocation ) self.assertTrue(status, "Could not startupClientNode") startingPort += 1 allNodes.append(node) allNodeObservers.append(observer) # Wait for flooding to reach all the nodes waiter = ConnectivityCounter() numberOfNodes = len(allEnclaves) * numNodes yield waiter.waitForConnectivity(numberOfNodes, bsNode) # Does not count bsNode itself. # Flood from bootstrap to all messageNum = random.randint(1, 9999999) status = yield TestUtils.sendFlood(bsNode, messageNum, "ALL") self.assertTrue(status, "sendFlood failed!") yield TestUtils.wait(3) # Wait for the messages to send # Check message counts status = TestUtils.didReceive(allNodeObservers, "ALL", messageNum, numberOfNodes) self.assertTrue(status, "All nodes did not receive message") # Flood from bootstrap to single enclave messageNum = random.randint(1, 9999999) status = yield TestUtils.sendFlood(bsNode, messageNum, allEnclaves[0]) self.assertTrue(status, "sendFlood failed!") yield TestUtils.wait(3) # Wait for the messages to send # Check message counts status = TestUtils.didReceive(allNodeObservers, allEnclaves[0], messageNum, numNodes) self.assertTrue(status, "Nodes in enclave %s didn't all recv message") status = TestUtils.didNotReceive(allNodeObservers, allEnclaves[1], messageNum, numNodes) self.assertTrue(status, "Some Nodes in enclave %s did recv message") log.msg("\n\n\n TEST ARE ALL DONE \n\n\n") yield TestUtils.wait(60) # Stop everything for node in self.allNodes: yield node.leave() yield self.bsNode.leave() # Wait for all network timeouts to finish yield TestUtils.wait(Config.CONNECTION_CACHE_DELAY + 3) # self.flushLoggedErrors() defer.returnValue(True)
def waitForConnectivity(self, _, numNodes): ''' Wait for numNodes to be connectable from the Bootstrap Node passed in. Returns a deferred which fires upon all nodes connecting. ''' waiter = ConnectivityCounter() return waiter.waitForConnectivity(numNodes, self.bsNetwork.chordNode)
def testNodeSendingToClass(self): class1 = ClassIDFactory.generateID(ClassIDFactory.WINDOWS, ClassIDFactory.LAPTOP, None) class2 = ClassIDFactory.generateID(ClassIDFactory.MAC, ClassIDFactory.DESKTOP, None) normalNodes = [] normalObservers = [] classNodes = [] classObservers = [] # Start a bootstrap node (status, self.bsNode, _observer) = yield TestUtils.startupBootstrapNode(self.myIP, 12345, 'localhost', class1) self.assertTrue(status, 'Could not build bootstrap node') # Start 10 client nodes not in the class for i in range(numNodes): (status, normalNode, observer) = yield TestUtils.startupClientNode(self.myIP, 12346+i, 'localhost', self.bsNode.nodeLocation, classID=class2) normalNodes.append(normalNode) normalObservers.append(observer) self.assertTrue(status, 'Could not startupClientNode') # Start a client node in the class for i in range(numClassNodes): (status, classNode, observer) = yield TestUtils.startupClientNode(self.myIP, 12346+numNodes+i, 'localhost', self.bsNode.nodeLocation, classID=class1) classNodes.append(classNode) classObservers.append(observer) self.assertTrue(status, 'Could not startupClientNode') # Wait for connectivity waiter = ConnectivityCounter() yield waiter.waitForConnectivity(numNodes+numClassNodes, self.bsNode) # Does not count bsNode itself. # Send a class query out msgNum = random.randint(0,1000000) classSpec = ClassIDFactory.generateSpec(computerType=ClassIDFactory.WINDOWS, hwType=None, userType=None) # Only Windows boxes. yield TestUtils.sendClassQuery(self.bsNode, classSpec, msgNum ) yield TestUtils.wait(3) enclaveID = 1512176389782608 print("DEBUG: BsNode ID: %s" % self.bsNode.nodeLocation.id) for o in classObservers: print("DEBUG: [%s] ClassNode: %s %s -> %s" % (o.getMessageCount(), o.chordNode.nodeLocation.id, o.chordNode.nodeLocation, o.chordNode.remote_getSuccessorLocation(enclaveID))) for o in normalObservers: print("DEBUG: [%s] NormalNode: %s %s -> %s" % (o.getMessageCount(), o.chordNode.nodeLocation.id, o.chordNode.nodeLocation, o.chordNode.remote_getSuccessorLocation(enclaveID))) # Verify that only one client node received it for o in normalObservers: self.assertTrue( o.getMessageCount() == 0, "Class message incorrectly went to a non-class node!") for o in classObservers: self.assertTrue( o.getMessageCount() == 1, "Node in class did not receive the class query! [Count:%d][ID:%s]" % (o.getMessageCount(), o.chordNode.nodeLocation.id)) # Everyone leave for n in classNodes: yield n.leave() for n in normalNodes: yield n.leave() yield self.bsNode.leave() # Wait for the connections to really all close if Config.USE_CONNECTION_CACHE: yield TestUtils.wait(Config.CONNECTION_CACHE_DELAY + 3) else: yield TestUtils.wait(3) defer.returnValue(True)