def testNodeStartup(self):
# Start a bootstrap node
(status, self.bsNode, _observer) = yield TestUtils.startupBootstrapNode(self.myIP, 12345, 'localhost')
self.assertTrue(status, 'Could not build bootstrap node')
# Start a client node
(status, self.normalNode, observer) = yield TestUtils.startupClientNode(self.myIP, 12346, 'localhost', self.bsNode.nodeLocation)
self.assertTrue(status, 'Could not startupClientNode')
# Are they connected together?
status = yield self.doQuickSendTest()
self.assertTrue(status, 'doQuickSendTest')
# Stop the nodes
yield self.normalNode.leave()
yield self.bsNode.leave()
# Wait for the connections to really all close
if Config.USE_CONNECTION_CACHE:
yield TestUtils.wait(Config.CONNECTION_CACHE_DELAY + 3)
else:
yield TestUtils.wait(3)
defer.returnValue(True)
def testSerialFlooding(self):
# Start a bootstrap node
(status, self.bsNode, _observer) = yield TestUtils.startupBootstrapNode(self.myIP, 12345, 'localhost')
self.assertTrue(status, 'Could not build bootstrap node')
self.allNodes.append(self.bsNode)
self.bsNode.addMessageObserver(self.messageReceived)
# Start client nodes
for i in range(numNodes):
(status, node, observer) = yield TestUtils.startupClientNode(self.myIP, 12346+i, 'localhost', self.bsNode.nodeLocation)
self.assertTrue(status, 'Could not startupClientNode')
self.allNodes.append(node)
observer = MyMessageObserver()
node.addMessageObserver(observer.messageReceived)
observer.node = node
self.allObservers.append(observer)
# Wait for flooding to reach all the nodes
waiter = ConnectivityCounter()
yield waiter.waitForConnectivity(numNodes, self.bsNode) # Does not count bsNode itself.
# Now do the stress test
status = yield self.doStressTest()
# Now close it all down!
yield self.allLeave()
# Wait a second or two for network timeouts
yield TestUtils.wait(9)
defer.returnValue(True)
def buildNetwork(self):
global startingPort
# Create Bootstrap
port = 12345
bootstrapNodeLocation = NodeLocation(None, self.myIP, port)
self.allNodeObservers = []
self.allBootstrapObservers = []
self.allNodes = []
enclave = "theEnc"
# Start a bootstrap node
(status, self.bsNode, observer) = yield TestUtils.startupBootstrapNode(self.myIP, port, enclave)
self.assertTrue(status, "Could not build bootstrap node")
self.allBootstrapObservers.append(observer)
# Add X nodes to each enclave
for _ in range(numNodes):
# Add a node to the enclave
(status, node, observer) = yield TestUtils.startupClientNode(
self.myIP, startingPort, enclave, bootstrapNodeLocation
)
self.assertTrue(status, "Could not startupClientNode")
startingPort += 1
self.allNodes.append(node)
self.allNodeObservers.append(observer)
# Wait for flooding to reach all the nodes
waiter = ConnectivityCounter()
yield waiter.waitForConnectivity(numNodes + 1, self.bsNode) # Does not bsNode .
defer.returnValue(True)
def testNodeStartup(self):
port = 12345
enclave = 'localhost'
bootstrapNodeLocation = NodeLocation(None, self.myIP, port)
# Start a bootstrap node
(status, bsNode, _) = yield TestUtils.startupBootstrapNode(self.myIP, port, enclave)
self.assertTrue(status, 'Could not build bootstrap node')
# Start a client node
(status, node, _) = yield TestUtils.startupClientNode(self.myIP, 12346, enclave, bootstrapNodeLocation)
self.assertTrue(status, 'Could not startupClientNode')
#yield TestUtils.wait(3)
# Are they connected together?
status = yield self.doConnectionTest(node)
self.assertTrue(status, 'doConnectionTest')
# Stop the nodes
yield bsNode.leave()
yield node.leave()
yield TestUtils.wait(Config.CONNECTION_CACHE_DELAY + 3)
defer.returnValue(True)
def testSerialP2PSending(self):
# Start a bootstrap node
(status, self.bsNode, _observer) = yield TestUtils.startupBootstrapNode(self.myIP, 12345, 'localhost')
self.assertTrue(status, 'Could not build bootstrap node')
self.allNodes.append(self.bsNode)
self.bsNode.addMessageObserver(self.messageReceived)
# Start client nodes
log.msg("Building nodes...")
for i in range(numNodes):
(status, node, observer) = yield TestUtils.startupClientNode(self.myIP, 12346+i, 'localhost', self.bsNode.nodeLocation)
self.assertTrue(status, 'Could not startupClientNode')
self.allNodes.append(node)
# Wait for flooding to reach all the nodes
waiter = ConnectivityCounter()
yield waiter.waitForConnectivity(numNodes, self.bsNode) # Does not count bsNode itself.
# Do the real test
status = yield self.doStressTest()
# Now close it all down!
yield self.allLeave()
# Wait a second or two
yield TestUtils.wait(3+Config.CONNECTION_CACHE_DELAY)
defer.returnValue(True)
def buildNetwork(self):
global startingPort
# Create two Bootstrap nodes
port = 12345
self.allNodeObservers = []
self.allBootstrapObservers = []
self.allNodes = []
# Start a bootstrap node
(status, self.bsNode1, observer) = yield TestUtils.startupBootstrapNode(self.myIP, port, allEnclaves[0])
self.assertTrue(status, 'Could not build bootstrap node')
self.allBootstrapObservers.append(observer)
# Start another bootstrap node
(status, self.bsNode2, observer) = yield TestUtils.startupBootstrapNode(self.myIP, port+1, allEnclaves[1])
self.assertTrue(status, 'Could not build bootstrap node')
self.allBootstrapObservers.append(observer)
# Add some nodes to each bootstrap node
for _ in range(numNodes):
# Add a node to the enclave
(status, node, observer) = yield TestUtils.startupClientNode(self.myIP, startingPort, allEnclaves[0], self.bsNode1.nodeLocation)
self.assertTrue(status, 'Could not startupClientNode')
startingPort += 1
self.allNodes.append(node)
self.allNodeObservers.append(observer)
for _ in range(numNodes):
# Add a node to the enclave
(status, node, observer) = yield TestUtils.startupClientNode(self.myIP, startingPort, allEnclaves[1], self.bsNode2.nodeLocation)
self.assertTrue(status, 'Could not startupClientNode')
startingPort += 1
self.allNodes.append(node)
self.allNodeObservers.append(observer)
# Wait for flooding to reach all the nodes
waiter = ConnectivityCounter()
yield waiter.waitForConnectivity(numNodes, self.bsNode1) # Does not count bsNode itself.
yield waiter.waitForConnectivity(numNodes, self.bsNode2) # Does not count bsNode itself.
defer.returnValue(True)
def buildNetwork(self):
global startingPort
# Create Bootstrap
port = 12345
bootstrapNodeLocation = NodeLocation(None, self.myIP, port)
self.allNodeObservers = []
self.allBootstrapObservers = []
self.allNodes = []
# Start a bootstrap node
log.msg("---> Startup Bootstrap Node", system="EnclaveP2P Test")
(status, self.bsNode, observer) = yield TestUtils.startupBootstrapNode(self.myIP, port, allEnclaves[0])
self.assertTrue(status, 'Could not build bootstrap node')
self.allBootstrapObservers.append(observer)
# Join another enclave
enableAutoDiscovery = True
bootstrapNodeList = [ NodeLocation(None, self.myIP, port) ]
status = yield self.bsNode.joinEnclave(bootstrapNodeList, enableAutoDiscovery, None, isBootstrapNode=True, enclaveStr=allEnclaves[1])
# Add X nodes to each enclave
for enclave in allEnclaves:
for _ in range(numNodes):
log.msg("---> Add node to enclave", system="EnclaveP2P Test")
# Add a node to the enclave
(status, node, observer) = yield TestUtils.startupClientNode(self.myIP, startingPort, enclave, bootstrapNodeLocation)
self.assertTrue(status, 'Could not startupClientNode')
startingPort += 1
self.allNodes.append(node)
self.allNodeObservers.append(observer)
# Wait for flooding to reach all the nodes
waiter = ConnectivityCounter()
numberOfNodes = (len(allEnclaves) * numNodes)
yield waiter.waitForConnectivity(numberOfNodes+1, self.bsNode) # Does not count bsNode itself.
defer.returnValue(True)
def testEnclaveBridge(self):
'''Create a node that can bridge between the enclaves
Send a flooding message from it to 1 enclave and verify only nodes in that one got it
'''
authenticationPayload = "This is a test"
# Build the bridge node
(status, bridgeNode, observer) = yield TestUtils.startupClientNode(self.myIP, 12500, allEnclaves[0], self.bsNode1.nodeLocation)
self.failUnless(status, 'testEnclave bridge could not join first bootstrap [%s]' % status)
# Join the other enclave also.
enableAutoDiscovery = True
bootstrapNodeList = [ self.bsNode2.nodeLocation ]
status = yield bridgeNode.joinEnclave(bootstrapNodeList, enableAutoDiscovery, authenticationPayload, False, None)
self.failUnless(status, 'testEnclave bridge could not join second bootstrap [%s]' % status)
self.bridgeNode = bridgeNode # Store a reference
# Send a flooding message to one enclave and verify only one got it.
messageNum = random.randint(1,9999999)
status = yield TestUtils.sendFlood(bridgeNode,messageNum, allEnclaves[0] )
self.assertTrue(status,"sendFlood failed!")
yield TestUtils.wait(3) # Wait for the messages to send
# Check message counts
status = TestUtils.didReceive(self.allNodeObservers, allEnclaves[0], messageNum, numNodes)
self.assertTrue(status,"Nodes in enclave %s didn't all recv message")
status = TestUtils.didNotReceive(self.allNodeObservers, allEnclaves[1], messageNum, numNodes)
self.assertTrue(status,"Some Nodes in enclave %s did recv message")
def testEnclaveFlooding(self):
"""Build 2 enclaves and test flooding to each and all."""
global startingPort
# Create Bootstrap
port = 12345
bootstrapNodeLocation = NodeLocation(None, self.myIP, port)
allNodes = []
allNodeObservers = []
allBootstrapObservers = []
self.allNodes = allNodes
# Start a bootstrap node
(status, bsNode, observer) = yield TestUtils.startupBootstrapNode(self.myIP, port, allEnclaves[0])
self.assertTrue(status, "Could not build bootstrap node")
allBootstrapObservers.append(observer)
self.bsNode = bsNode
# Join another enclave
bootstrapNodeList = None
enableAutoDiscovery = True
status = yield self.bsNode.joinEnclave(
bootstrapNodeList, enableAutoDiscovery, None, isBootstrapNode=True, enclaveStr=allEnclaves[1]
)
# Add X nodes to each enclave
for enclave in allEnclaves:
for _ in range(numNodes):
# Add a node to the enclave
(status, node, observer) = yield TestUtils.startupClientNode(
self.myIP, startingPort, enclave, bootstrapNodeLocation
)
self.assertTrue(status, "Could not startupClientNode")
startingPort += 1
allNodes.append(node)
allNodeObservers.append(observer)
# Wait for flooding to reach all the nodes
waiter = ConnectivityCounter()
numberOfNodes = len(allEnclaves) * numNodes
yield waiter.waitForConnectivity(numberOfNodes, bsNode) # Does not count bsNode itself.
# Flood from bootstrap to all
messageNum = random.randint(1, 9999999)
status = yield TestUtils.sendFlood(bsNode, messageNum, "ALL")
self.assertTrue(status, "sendFlood failed!")
yield TestUtils.wait(3) # Wait for the messages to send
# Check message counts
status = TestUtils.didReceive(allNodeObservers, "ALL", messageNum, numberOfNodes)
self.assertTrue(status, "All nodes did not receive message")
# Flood from bootstrap to single enclave
messageNum = random.randint(1, 9999999)
status = yield TestUtils.sendFlood(bsNode, messageNum, allEnclaves[0])
self.assertTrue(status, "sendFlood failed!")
yield TestUtils.wait(3) # Wait for the messages to send
# Check message counts
status = TestUtils.didReceive(allNodeObservers, allEnclaves[0], messageNum, numNodes)
self.assertTrue(status, "Nodes in enclave %s didn't all recv message")
status = TestUtils.didNotReceive(allNodeObservers, allEnclaves[1], messageNum, numNodes)
self.assertTrue(status, "Some Nodes in enclave %s did recv message")
log.msg("\n\n\n TEST ARE ALL DONE \n\n\n")
yield TestUtils.wait(60)
# Stop everything
for node in self.allNodes:
yield node.leave()
yield self.bsNode.leave()
# Wait for all network timeouts to finish
yield TestUtils.wait(Config.CONNECTION_CACHE_DELAY + 3)
# self.flushLoggedErrors()
defer.returnValue(True)
def testNodeIPChanging(self):
'''Try to create a node which needs an IP change, but try it with and without allowing
IP changes. This should test the allowFloatingIP capability.
'''
# Update this to speed up this test
Config.NETWORK_CONNECTION_TIMEOUT = 3
port = 12345
enclave = 'localhost'
bootstrapNodeLocation = NodeLocation(None, self.myIP, port)
# Start a bootstrap node
(status, bsNode, _) = yield TestUtils.startupBootstrapNode(self.myIP, port, enclave)
self.assertTrue(status, 'Could not build bootstrap node')
# Start a client node with a bad IP, and wait for it to change
allowFloatingIP = True
log.msg("\n\n Starting Client Node1 \n\n", system="testNodeIPChanging")
(status, node, _) = yield TestUtils.startupClientNode(self.myIP, 12346, enclave, bootstrapNodeLocation, allowFloatingIP)
self.assertTrue(status, 'Could not startupClientNode')
netObs1 = NetObserver()
node.addNetworkObserver(netObs1.setStatus)
# Are they connected together?
status = yield self.doConnectionTest(node)
self.assertTrue(status, 'doConnectionTest')
allowFloatingIP = False
log.msg("\n\n Starting Client Node2 \n\n", system="testNodeIPChanging")
(status, node2, _) = yield TestUtils.startupClientNode(self.myIP, 12347, enclave, bootstrapNodeLocation, allowFloatingIP)
self.assertTrue(status, 'Could not startupClientNode')
netObs2 = NetObserver()
node2.addNetworkObserver(netObs2.setStatus)
# Are they connected together?
status = yield self.doConnectionTest(node2)
self.assertTrue(status, 'doConnectionTest')
# Now, change the IP to a "bad IP" and wait.
badIP = "192.168.9.10"
node.nodeLocation.ip=badIP
node2.nodeLocation.ip=badIP
# Now we are waiting for the maint calls to detect the change, and
# Issue a new IP
yield TestUtils.wait(20)
# Check the current IP and verify it's not the bad ip
self.assertFalse(node.nodeLocation.ip == badIP, "Chord node did not update to a new IP, but should have!")
self.assertTrue(node2.nodeLocation.ip == badIP, "Chord node changed the IP, but wasn't allowed to float.")
# Verify the statusus
self.assertTrue(netObs1.inc == True, "network observer 1 failed incoming!")
self.assertTrue(netObs1.out == True, "network observer 1 failed outgoing!")
self.assertTrue(netObs2.inc == False, "network observer 2 failed incoming!")
self.assertTrue(netObs2.out == True, "network observer 2 failed outgoing!")
# Stop the nodes
yield bsNode.leave()
yield node.leave()
yield node2.leave()
yield TestUtils.wait(15)
# Update this to speed up this test
Config.NETWORK_CONNECTION_TIMEOUT = 30
defer.returnValue(True)
def testNodeSendingToClass(self):
class1 = ClassIDFactory.generateID(ClassIDFactory.WINDOWS, ClassIDFactory.LAPTOP, None)
class2 = ClassIDFactory.generateID(ClassIDFactory.MAC, ClassIDFactory.DESKTOP, None)
normalNodes = []
normalObservers = []
classNodes = []
classObservers = []
# Start a bootstrap node
(status, self.bsNode, _observer) = yield TestUtils.startupBootstrapNode(self.myIP, 12345, 'localhost', class1)
self.assertTrue(status, 'Could not build bootstrap node')
# Start 10 client nodes not in the class
for i in range(numNodes):
(status, normalNode, observer) = yield TestUtils.startupClientNode(self.myIP, 12346+i, 'localhost', self.bsNode.nodeLocation, classID=class2)
normalNodes.append(normalNode)
normalObservers.append(observer)
self.assertTrue(status, 'Could not startupClientNode')
# Start a client node in the class
for i in range(numClassNodes):
(status, classNode, observer) = yield TestUtils.startupClientNode(self.myIP, 12346+numNodes+i, 'localhost', self.bsNode.nodeLocation, classID=class1)
classNodes.append(classNode)
classObservers.append(observer)
self.assertTrue(status, 'Could not startupClientNode')
# Wait for connectivity
waiter = ConnectivityCounter()
yield waiter.waitForConnectivity(numNodes+numClassNodes, self.bsNode) # Does not count bsNode itself.
# Send a class query out
msgNum = random.randint(0,1000000)
classSpec = ClassIDFactory.generateSpec(computerType=ClassIDFactory.WINDOWS, hwType=None, userType=None) # Only Windows boxes.
yield TestUtils.sendClassQuery(self.bsNode, classSpec, msgNum )
yield TestUtils.wait(3)
enclaveID = 1512176389782608
print("DEBUG: BsNode ID: %s" % self.bsNode.nodeLocation.id)
for o in classObservers:
print("DEBUG: [%s] ClassNode: %s %s -> %s" % (o.getMessageCount(), o.chordNode.nodeLocation.id, o.chordNode.nodeLocation, o.chordNode.remote_getSuccessorLocation(enclaveID)))
for o in normalObservers:
print("DEBUG: [%s] NormalNode: %s %s -> %s" % (o.getMessageCount(), o.chordNode.nodeLocation.id, o.chordNode.nodeLocation, o.chordNode.remote_getSuccessorLocation(enclaveID)))
# Verify that only one client node received it
for o in normalObservers:
self.assertTrue( o.getMessageCount() == 0, "Class message incorrectly went to a non-class node!")
for o in classObservers:
self.assertTrue( o.getMessageCount() == 1, "Node in class did not receive the class query! [Count:%d][ID:%s]" % (o.getMessageCount(), o.chordNode.nodeLocation.id))
# Everyone leave
for n in classNodes:
yield n.leave()
for n in normalNodes:
yield n.leave()
yield self.bsNode.leave()
# Wait for the connections to really all close
if Config.USE_CONNECTION_CACHE:
yield TestUtils.wait(Config.CONNECTION_CACHE_DELAY + 3)
else:
yield TestUtils.wait(3)
defer.returnValue(True)
