def testNodeStartup(self):
# Start a bootstrap node
(status, self.bsNode, _observer) = yield TestUtils.startupBootstrapNode(self.myIP, 12345, 'localhost')
self.assertTrue(status, 'Could not build bootstrap node')
# Start a client node
(status, self.normalNode, observer) = yield TestUtils.startupClientNode(self.myIP, 12346, 'localhost', self.bsNode.nodeLocation)
self.assertTrue(status, 'Could not startupClientNode')
# Are they connected together?
status = yield self.doQuickSendTest()
self.assertTrue(status, 'doQuickSendTest')
# Stop the nodes
yield self.normalNode.leave()
yield self.bsNode.leave()
# Wait for the connections to really all close
if Config.USE_CONNECTION_CACHE:
yield TestUtils.wait(Config.CONNECTION_CACHE_DELAY + 3)
else:
yield TestUtils.wait(3)
defer.returnValue(True)
def testNodeStartup(self):
port = 12345
enclave = 'localhost'
bootstrapNodeLocation = NodeLocation(None, self.myIP, port)
# Start a bootstrap node
(status, bsNode, _) = yield TestUtils.startupBootstrapNode(self.myIP, port, enclave)
self.assertTrue(status, 'Could not build bootstrap node')
# Start a client node
(status, node, _) = yield TestUtils.startupClientNode(self.myIP, 12346, enclave, bootstrapNodeLocation)
self.assertTrue(status, 'Could not startupClientNode')
#yield TestUtils.wait(3)
# Are they connected together?
status = yield self.doConnectionTest(node)
self.assertTrue(status, 'doConnectionTest')
# Stop the nodes
yield bsNode.leave()
yield node.leave()
yield TestUtils.wait(Config.CONNECTION_CACHE_DELAY + 3)
defer.returnValue(True)
def getAggResponse(self):
'''Send a query and get an agg response from all the nodes.'''
bsNode = self.bsNetwork.chordNode
for aggNumber in range(1, numNodes + 5):
# Reset the message counter
obs = self.getObserver(bsNode, self.allTestObservers)
obs.resetMessageCount()
# Send the query asking for a response
messageNum = aggNumber # Just to keep it incrementing
yield TestUtils.sendFlood(bsNode,messageNum,'localhost', data="SEND_AGG_RESPONSE:%d" % aggNumber)
# Now check how many messages the bootstrap node got in return
for _ in range(10): # 10 seconds
recvCount = obs.getMessageCount()
if recvCount >= numNodes+1:
break
else:
yield TestUtils.wait(1)
self.failUnless(recvCount == numNodes+1, "getAggResponse didn't get the correct number of messages back.Expected[%d] Got[%d]" % (numNodes+1, recvCount))
defer.returnValue(True)
def waitForConnectivity(self, numToWaitFor, chordNode):
'''Wait till we can connect to all numNodes'''
self.node = chordNode
self.testCounter = 1
self.connectedNodeList = []
# Need to see the messages
self.node.addMessageObserver(self.messageReceived)
yield self.getNumConnected()
while len(self.connectedNodeList) < numToWaitFor:
log.msg("DEBUG: waiting for %d nodes. Got %d" % (numToWaitFor, len(self.connectedNodeList)), system="ConnectivityCounter")
self.testCounter += 1
yield self.getNumConnected()
yield TestUtils.wait(5) # Wait for messages to go around
log.msg("DEBUG: waiting for %d nodes. Got %d" % (numToWaitFor, len(self.connectedNodeList)), system="ConnectivityCounter")
# for n in self.connectedNodeList:
# print("DEBUG: Node:%s" % n)
# Don't care anymore
self.node.removeMessageObserver(self.messageReceived)
def waitForAllReceived(self):
'''Wait until all messages have been received by all the nodes.'''
numTries = 10
expectedNumMsg = numNodes*numMessages
for _ in range(numTries):
completedObservers = 0
incompleteObservers = 0
# Wait a sec
yield TestUtils.wait(1)
# Count them
for obs in self.allObservers:
numRx = obs.getNumReceived()
if numRx == expectedNumMsg:
completedObservers += 1
elif numRx > expectedNumMsg:
obs.printMessages()
raise Exception("Programming error... received more messages than possible! Got[%d] Expected[%d]" % (numRx,expectedNumMsg ))
else:
incompleteObservers += 1
print("waitForAllReceived: Complete:%d Incomplete:%d" % (completedObservers, incompleteObservers))
if incompleteObservers == 0:
defer.returnValue(True)
defer.returnValue(False)
def testSerialP2PSending(self):
# Start a bootstrap node
(status, self.bsNode, _observer) = yield TestUtils.startupBootstrapNode(self.myIP, 12345, 'localhost')
self.assertTrue(status, 'Could not build bootstrap node')
self.allNodes.append(self.bsNode)
self.bsNode.addMessageObserver(self.messageReceived)
# Start client nodes
log.msg("Building nodes...")
for i in range(numNodes):
(status, node, observer) = yield TestUtils.startupClientNode(self.myIP, 12346+i, 'localhost', self.bsNode.nodeLocation)
self.assertTrue(status, 'Could not startupClientNode')
self.allNodes.append(node)
# Wait for flooding to reach all the nodes
waiter = ConnectivityCounter()
yield waiter.waitForConnectivity(numNodes, self.bsNode) # Does not count bsNode itself.
# Do the real test
status = yield self.doStressTest()
# Now close it all down!
yield self.allLeave()
# Wait a second or two
yield TestUtils.wait(3+Config.CONNECTION_CACHE_DELAY)
defer.returnValue(True)
def testSerialFlooding(self):
# Start a bootstrap node
(status, self.bsNode, _observer) = yield TestUtils.startupBootstrapNode(self.myIP, 12345, 'localhost')
self.assertTrue(status, 'Could not build bootstrap node')
self.allNodes.append(self.bsNode)
self.bsNode.addMessageObserver(self.messageReceived)
# Start client nodes
for i in range(numNodes):
(status, node, observer) = yield TestUtils.startupClientNode(self.myIP, 12346+i, 'localhost', self.bsNode.nodeLocation)
self.assertTrue(status, 'Could not startupClientNode')
self.allNodes.append(node)
observer = MyMessageObserver()
node.addMessageObserver(observer.messageReceived)
observer.node = node
self.allObservers.append(observer)
# Wait for flooding to reach all the nodes
waiter = ConnectivityCounter()
yield waiter.waitForConnectivity(numNodes, self.bsNode) # Does not count bsNode itself.
# Now do the stress test
status = yield self.doStressTest()
# Now close it all down!
yield self.allLeave()
# Wait a second or two for network timeouts
yield TestUtils.wait(9)
defer.returnValue(True)
def waitForMessageCount(self, obs, numMessagesToWaitFor):
for _ in range(10):
if obs.getPingbackCount() >= numMessagesToWaitFor:
defer.returnValue(True)
else:
yield TestUtils.wait(1)
defer.returnValue(False)
def testAggregationMessageAPI(self):
'''
By the time this function is called, we should have a full network built and connected.
Tests to perform now:
1. Send P2P messages from BS Node to all nodes
2. Send P2P messages from all Nodes to BSNode
3. Send flooding from BSNode to all
4. Send query and return aggregation responses of different levels
Finally, do cleanup.
'''
try:
# Build a network
yield self.buildNetwork()
# 1. Send P2P messages from BS Node to all nodes
yield self.p2pFromBStoAll()
# 2. Send P2P messages from all Nodes to BSNode
yield self.p2pFromAlltoBS()
# 3. Send flooding from BSNode to all
yield self.floodFromBStoAll()
# 4. Send query and return aggregation responses of different levels
yield self.getAggResponse()
# Do the cleanup
yield TestUtils.wait(5)
# Shut it all down
for (_clientAPI, networkAPI) in self.allNodes:
yield networkAPI.disconnect()
yield self.bsNetwork.disconnect()
# Now wait for the network cache disconnect timeout
yield TestUtils.wait(Config.CONNECTION_CACHE_DELAY + 5)
except Exception, e:
log.err(e, "An error occurred in testAggregationMessageAPI")
def testDisconnectedBootstraps(self):
'''Create a BS node and some clients. Create another bootstrap node and some clients (so we essentially have two rings).
Verify, that the bootstrap nodes autodiscover each other and connect together
'''
global startingPort
# Create Bootstrap
port = 12345
bootstrapNodeLocation = NodeLocation(None, self.myIP, port)
bootstrapNodeLocation2 = NodeLocation(None, self.myIP, port+1)
self.allNodes = []
self.allMetricsObservers = []
self.allTestObservers = []
# Build the BS node
(status, bsClientAPI, bsNetworkAPI) = yield TestUtils.startNodeUsingAPI(bootstrapNodeLocation.ip, bootstrapNodeLocation.port, None, 'theEnclave', True, True)
self.allMetricsObservers.append(MetricsMessageObserver(bsNetworkAPI.chordNode))
self.assertTrue(status, 'Could not build bootstrap node')
# Build second BS node
(status, bsClientAPI2, bsNetworkAPI2) = yield TestUtils.startNodeUsingAPI(bootstrapNodeLocation2.ip, bootstrapNodeLocation2.port, None, 'theEnclave', True, True)
self.allMetricsObservers.append(MetricsMessageObserver(bsNetworkAPI2.chordNode))
self.assertTrue(status, 'Could not build bootstrap node 2')
# Build the client node
(status, clClientAPI, clNetworkAPI) = yield TestUtils.startNodeUsingAPI(self.myIP, port+2, bootstrapNodeLocation, 'theEnclave', False, False)
self.allMetricsObservers.append(MetricsMessageObserver(clNetworkAPI.chordNode))
self.assertTrue(status, 'Could not build client node')
# Build the client node
(status, clClientAPI2, clNetworkAPI2) = yield TestUtils.startNodeUsingAPI(self.myIP, port+3, bootstrapNodeLocation2, 'theEnclave', False, False)
self.allMetricsObservers.append(MetricsMessageObserver(clNetworkAPI2.chordNode))
self.assertTrue(status, 'Could not build client node')
# Wait for flooding to reach all the nodes
waiter = ConnectivityCounter()
yield waiter.waitForConnectivity(3, clNetworkAPI.chordNode) # Does not count clNode itself.
# Now shut everything down
yield clNetworkAPI.disconnect()
yield clNetworkAPI2.disconnect()
yield bsNetworkAPI.disconnect()
yield bsNetworkAPI2.disconnect()
if Config.USE_CONNECTION_CACHE:
yield TestUtils.waitForConnectionCache()
else:
yield TestUtils.wait(5)
defer.returnValue(True)
def testDefaultEnclaveAPI(self):
'''Create a node which should default to localhost as the enclave name. '''
global startingPort
# Create Bootstrap
port = 12345
bootstrapNodeLocation = NodeLocation(None, self.myIP, port)
self.allNodes = []
self.allMetricsObservers = []
self.allTestObservers = []
# Build the client and network objects
enclaveStr1 = None
self.bsClient = SampleClient(self.myIP, port, None)
self.bsNetwork = classChordNetworkChord(self.bsClient, port, self.myIP)
bsID = self.bsNetwork.generateNodeID(str(port), "WHO CARES") # Get the ID with the bits on it we need. Use "port" because it'll be uniq for tests
# Join the network
log.msg("---- Bootstrap Join 1 ---- ", system="testDefaultEnclaveAPI")
d = defer.Deferred()
callFunc = lambda x, payload: self.shouldSucceedCallback(x, payload, d, self.bsNetwork)
self.bsNetwork.start(callFunc, bsID, enclaveStr1, "authenticate:succeed", None, True, True)
yield d # Wait for join to succeed
# Now check that the node is part of localhost
enclave = yield self.bsNetwork.findEnclave(self.myIP, port)
self.assertEqual(enclave, "localhost", "testDefaultEnclaveAPI did not get localhost as enclave!")
# Now create a client node and have it ask the bootstrap node for the enclave
log.msg("---- Start Client 1 ---- ", system="testIPFindingAPI")
yield self.startClientNode(None, None, 12350+1, bootstrapNodeLocation, None)
# Check that it got enclave localhost
enclave = yield self.allNodes[0][1].findEnclave(self.myIP, 12350+1)
self.assertEqual(enclave, "localhost", "testDefaultEnclaveAPI: client did not get localhost as enclave!")
# Now shut everything down
for (_clientAPI, networkAPI) in self.allNodes:
yield networkAPI.disconnect()
yield self.bsNetwork.disconnect()
# Now wait for the network cache disconnect timeout
yield TestUtils.wait(Config.CONNECTION_CACHE_DELAY + 1)
defer.returnValue(True)
def testIPFindingAPI(self):
'''Create a node and ask the system to figure out it's IP from a bootstrap node. '''
global startingPort
# Create Bootstrap
port = 12345
bootstrapNodeLocation = NodeLocation(None, self.myIP, port)
self.allNodes = []
self.allMetricsObservers = []
self.allTestObservers = []
# Build the client and network objects
enclaveStr1 = "AnEnclave"
myIP = None # self.myIP
self.bsClient = SampleClient(myIP, port, None)
self.bsNetwork = classChordNetworkChord(self.bsClient, port, myIP)
bsID = self.bsNetwork.generateNodeID(str(port), "WHO CARES") # Get the ID with the bits on it we need. Use "port" because it'll be uniq for tests
# Join the network
log.msg("---- Bootstrap Join 1 ---- ", system="testDefaultEnclaveAPI")
d = defer.Deferred()
callFunc = lambda x, payload: self.shouldSucceedCallback(x, payload, d, self.bsNetwork)
self.bsNetwork.start(callFunc, bsID, enclaveStr1, "authenticate:succeed", None, True, False)
yield d # Wait for join to succeed
# Now create a client node which will get it's IP from the bootstrap node
log.msg("---- Start Client 1 ---- ", system="testIPFindingAPI")
yield self.startClientNode(None, None, 12350+1, bootstrapNodeLocation, enclaveStr1)
# Do a bit of verification
# Now shut everything down
for (_clientAPI, networkAPI) in self.allNodes:
rc = yield networkAPI.isConnected("ANY")
self.assertTrue(rc != False, "isConnected returned a False value in testIPFindingAPI")
# Now shut everything down
for (_clientAPI, networkAPI) in self.allNodes:
yield networkAPI.disconnect()
yield self.bsNetwork.disconnect()
# Now wait for the network cache disconnect timeout
yield TestUtils.wait(Config.CONNECTION_CACHE_DELAY + 1)
defer.returnValue(True)
def testAggregationMessage(self):
'''
Send flooding message and ask for aggregation response.
'''
print("testAggregationMessage method starting...")
counter = 1
try:
d = defer.Deferred()
messageNum = random.randint(1,9999999)
# Reset message count
self.observer.resetMessageCount()
self.observer.printDebug = True
aggMax = numNodes + 2
#aggNumber = 3
for aggNumber in range(1, aggMax+1):
#if aggNumber == aggNumber:
# Flood from bootstrap to all nodes
for i in range(numMessages):
#d.addCallback(self.sendFlood, self.bsNode,messageNum+i,'theEnclave', data="SEND_AGG_RESPONSE:%d" % aggNumber)
yield self.sendFlood(None, self.bsNode,messageNum+i,'theEnclave', data="SEND_AGG_RESPONSE:%d" % aggNumber)
print("sending message %d", counter)
counter += 1
d.callback(True)
yield d
# Wait a few seconds
yield TestUtils.wait(5)
print("testAggregationMessage check results...")
# Now check the results
yield self.checkResults(aggMax)
except Exception, e:
self.anError(e)
def doStressTest(self):
'''Randomly pick two nodes and send a message between them. Verify that it goes.'''
print("Running parallel stress test: %d p2p messages" % numMessages)
messageCounter = 0
while messageCounter < numMessages:
if messageCounter % 100 == 0:
print("Running test %d of %d" % (messageCounter, numMessages))
statusList = []
for _ in range(numMessagesInBlock):
messageCounter += 1
(srcNode, dstNode) = random.sample(self.allNodes, 2)
# Build the envelope
env = CopyEnvelope()
env['ttl'] = datetime.datetime.now() + datetime.timedelta(minutes=10)
env['source'] = srcNode.nodeLocation
env['type'] = 'p2p'
env['destination'] = dstNode.nodeLocation.id
env['msgID'] = random.getrandbits(128) # TODO: Something better here!
msgText = "Test number %d " % messageCounter
statusList.append(srcNode.sendSyncMessage(msgText, env))
# Now wait for all of them to complete
dl = defer.DeferredList(statusList)
results = yield dl # Wait for it
# Now check all the return codes
for (success, _) in results:
#print("DEBUG: doStressTest Result is %s" % success)
self.assertTrue(success, "doStressTest Message returned False!" )
# Wait a bit... just to ease up a smidge.
yield TestUtils.wait(0.1)
defer.returnValue(True)
def doTearDown(self):
'''Tear down the network created during setup.'''
log.msg("tearDown begins...")
# Stop everything
for networkAPI in self.allNetworkAPIs:
yield networkAPI.disconnect()
yield self.bsNode.leave()
# Wait for all network timeouts to finish
#self.flushLoggedErrors()
if Config.USE_CONNECTION_CACHE:
yield TestUtils.wait(Config.CONNECTION_CACHE_DELAY + 2)
else:
yield TestUtils.defWait(3)
defer.returnValue(True)
def testNodeAuth(self):
log.msg("---> Starting BootstrapNode")
yield self.startBootStrapNode()
log.msg("---> Creating Good Client")
yield self.startClientNode(self.myIP, 12350, self.bootstrapNodeLocation, self.enclaveStr)
log.msg("---> Creating Bad Client")
yield self.startBadClientNode(self.myIP, 12351, self.bootstrapNodeLocation, self.enclaveStr)
# Shut it all down
for (clientAPI, networkAPI) in self.allNodes:
yield networkAPI.disconnect()
yield self.bsNetwork.disconnect()
if Config.USE_CONNECTION_CACHE:
yield TestUtils.waitForConnectionCache()
else:
yield TestUtils.wait(2)
defer.returnValue(True)
def testAutoDiscoveryRebootstrap(self):
'''Try to start a bootstrap node and client with autodiscovery on.
Then start another bootstrap node.
Then kill first bootstrap node and have client try to re-bootstrap to
new bootstrap node.
'''
log.msg("---------------------- BEGIN testAutoDiscoveryRebootstrap -------------- ")
self.allNodes = []
self.allMetricsObservers = []
self.allTestObservers = []
# Create Bootstrap
port = 12345
enclaveStr = 'testclave'
bsNodeLocation = NodeLocation(None, self.myIP, port)
bootstrapNodeList = [ bsNodeLocation ]
log.msg("Starting bootstrap... \n\n")
(self.bsClient, self.bsNetwork, d) = self.startBootstrapNode(enclaveStr, self.myIP, port, "authenticate:succeed", bootstrapNodeList)
yield d
# Now try and join with autodiscovery
yield TestUtils.wait(1)
log.msg("Starting client... \n\n")
yield self.startClientNodeAutoDiscovery(None, self.myIP, port+1, enclaveStr, bootstrapNodeList)
# Now start a second bootstrap node
#NOT DONE YET
log.msg("Shutting down client... \n\n")
# Shut it all down
for (clientAPI, networkAPI) in self.allNodes:
yield networkAPI.disconnect()
yield self.bsNetwork.disconnect()
# Try and wait for connection cache to finish disconnects (just in case).
yield TestUtils.waitForConnectionCache()
def testNodeDisconnection(self):
"""Disconnect a node from the ring and do a few things.
- Send a message to that node (verify error condition)
- Send a flooding message -- wait until connectivity happens for all other nodes.
"""
# Disconnect a node
nodeIndex = random.randint(0, numNodes - 1) # Node to disconnect
nodeLoc = self.allNodes[nodeIndex].nodeLocation
log.msg("testNodeDisconnection: Node is leaving [%s]...\n" % nodeLoc)
ConnectionCache._printCache()
yield self.allNodes[nodeIndex].leave()
log.msg("testNodeDisconnection: Node left...\n")
try:
# Now wait for full connectivity to come back
waiter = ConnectivityCounter()
yield waiter.waitForConnectivity(numNodes, self.bsNode) # Does count bsNode.
except error.ConnectionRefusedError:
log.err("NodeDisconnect test got a connection refused error [2]")
log.err()
# Now wait for the disconnect to fully take hold
yield TestUtils.wait(30) # Network timeout time
print("DEBUG testNodeDisc \n\n\n\n")
ConnectionCache._printCache()
try:
# P2P from bootstrap to node which left
yield self.sendFailingP2PMsg(self.bsNode, self.allNodes[nodeIndex])
except error.ConnectionRefusedError:
log.msg("\n\nNodeDisconnect test got a connection refused error [3], which is okay!\n\n")
defer.returnValue(True)
def testStartAutoDiscoveryMultipleClients(self):
'''Try to start a bootstrap node and the many many clients all at once with autodiscovery on. '''
log.msg("---------------------- BEGIN testStartAutoDiscoveryMultipleClients -------------- ")
self.allNodes = []
self.allMetricsObservers = []
self.allTestObservers = []
numNodes = 15
# Create Bootstrap
port = 12345
enclaveStr = 'testclave'
bootstrapNodeList = [ NodeLocation(None, self.myIP, port) ]
(self.bsClient, self.bsNetwork, d) = self.startBootstrapNode(enclaveStr, self.myIP, port, "authenticate:succeed", bootstrapNodeList)
yield d
# Now try and join a bunch of nodes with autodiscovery
allDefs = [] # A list of deferreds
for counter in range(numNodes):
allDefs.append(self.startClientNodeAutoDiscovery(None, self.myIP, port+1+counter, enclaveStr, bootstrapNodeList))
dl = defer.DeferredList(allDefs)
yield dl # Wait for all deferred joins to complete
# Shut it all down
for (clientAPI, networkAPI) in self.allNodes:
yield networkAPI.disconnect()
yield self.bsNetwork.disconnect()
if Config.USE_CONNECTION_CACHE:
# Try and wait for connection cache to finish disconnects (just in case).
yield TestUtils.waitForConnectionCache()
else:
# Wait a few seconds for network timeouts
yield TestUtils.wait(5)
def testEnclaveBridge(self):
'''Create a node that can bridge between the enclaves
Send a flooding message from it to 1 enclave and verify only nodes in that one got it
'''
authenticationPayload = "This is a test"
# Build the bridge node
(status, bridgeNode, observer) = yield TestUtils.startupClientNode(self.myIP, 12500, allEnclaves[0], self.bsNode1.nodeLocation)
self.failUnless(status, 'testEnclave bridge could not join first bootstrap [%s]' % status)
# Join the other enclave also.
enableAutoDiscovery = True
bootstrapNodeList = [ self.bsNode2.nodeLocation ]
status = yield bridgeNode.joinEnclave(bootstrapNodeList, enableAutoDiscovery, authenticationPayload, False, None)
self.failUnless(status, 'testEnclave bridge could not join second bootstrap [%s]' % status)
self.bridgeNode = bridgeNode # Store a reference
# Send a flooding message to one enclave and verify only one got it.
messageNum = random.randint(1,9999999)
status = yield TestUtils.sendFlood(bridgeNode,messageNum, allEnclaves[0] )
self.assertTrue(status,"sendFlood failed!")
yield TestUtils.wait(3) # Wait for the messages to send
# Check message counts
status = TestUtils.didReceive(self.allNodeObservers, allEnclaves[0], messageNum, numNodes)
self.assertTrue(status,"Nodes in enclave %s didn't all recv message")
status = TestUtils.didNotReceive(self.allNodeObservers, allEnclaves[1], messageNum, numNodes)
self.assertTrue(status,"Some Nodes in enclave %s did recv message")
def testEnclaveFlooding(self):
"""Build 2 enclaves and test flooding to each and all."""
global startingPort
# Create Bootstrap
port = 12345
bootstrapNodeLocation = NodeLocation(None, self.myIP, port)
allNodes = []
allNodeObservers = []
allBootstrapObservers = []
self.allNodes = allNodes
# Start a bootstrap node
(status, bsNode, observer) = yield TestUtils.startupBootstrapNode(self.myIP, port, allEnclaves[0])
self.assertTrue(status, "Could not build bootstrap node")
allBootstrapObservers.append(observer)
self.bsNode = bsNode
# Join another enclave
bootstrapNodeList = None
enableAutoDiscovery = True
status = yield self.bsNode.joinEnclave(
bootstrapNodeList, enableAutoDiscovery, None, isBootstrapNode=True, enclaveStr=allEnclaves[1]
)
# Add X nodes to each enclave
for enclave in allEnclaves:
for _ in range(numNodes):
# Add a node to the enclave
(status, node, observer) = yield TestUtils.startupClientNode(
self.myIP, startingPort, enclave, bootstrapNodeLocation
)
self.assertTrue(status, "Could not startupClientNode")
startingPort += 1
allNodes.append(node)
allNodeObservers.append(observer)
# Wait for flooding to reach all the nodes
waiter = ConnectivityCounter()
numberOfNodes = len(allEnclaves) * numNodes
yield waiter.waitForConnectivity(numberOfNodes, bsNode) # Does not count bsNode itself.
# Flood from bootstrap to all
messageNum = random.randint(1, 9999999)
status = yield TestUtils.sendFlood(bsNode, messageNum, "ALL")
self.assertTrue(status, "sendFlood failed!")
yield TestUtils.wait(3) # Wait for the messages to send
# Check message counts
status = TestUtils.didReceive(allNodeObservers, "ALL", messageNum, numberOfNodes)
self.assertTrue(status, "All nodes did not receive message")
# Flood from bootstrap to single enclave
messageNum = random.randint(1, 9999999)
status = yield TestUtils.sendFlood(bsNode, messageNum, allEnclaves[0])
self.assertTrue(status, "sendFlood failed!")
yield TestUtils.wait(3) # Wait for the messages to send
# Check message counts
status = TestUtils.didReceive(allNodeObservers, allEnclaves[0], messageNum, numNodes)
self.assertTrue(status, "Nodes in enclave %s didn't all recv message")
status = TestUtils.didNotReceive(allNodeObservers, allEnclaves[1], messageNum, numNodes)
self.assertTrue(status, "Some Nodes in enclave %s did recv message")
log.msg("\n\n\n TEST ARE ALL DONE \n\n\n")
yield TestUtils.wait(60)
# Stop everything
for node in self.allNodes:
yield node.leave()
yield self.bsNode.leave()
# Wait for all network timeouts to finish
yield TestUtils.wait(Config.CONNECTION_CACHE_DELAY + 3)
# self.flushLoggedErrors()
defer.returnValue(True)
def testNodeSendingToClass(self):
class1 = ClassIDFactory.generateID(ClassIDFactory.WINDOWS, ClassIDFactory.LAPTOP, None)
class2 = ClassIDFactory.generateID(ClassIDFactory.MAC, ClassIDFactory.DESKTOP, None)
normalNodes = []
normalObservers = []
classNodes = []
classObservers = []
# Start a bootstrap node
(status, self.bsNode, _observer) = yield TestUtils.startupBootstrapNode(self.myIP, 12345, 'localhost', class1)
self.assertTrue(status, 'Could not build bootstrap node')
# Start 10 client nodes not in the class
for i in range(numNodes):
(status, normalNode, observer) = yield TestUtils.startupClientNode(self.myIP, 12346+i, 'localhost', self.bsNode.nodeLocation, classID=class2)
normalNodes.append(normalNode)
normalObservers.append(observer)
self.assertTrue(status, 'Could not startupClientNode')
# Start a client node in the class
for i in range(numClassNodes):
(status, classNode, observer) = yield TestUtils.startupClientNode(self.myIP, 12346+numNodes+i, 'localhost', self.bsNode.nodeLocation, classID=class1)
classNodes.append(classNode)
classObservers.append(observer)
self.assertTrue(status, 'Could not startupClientNode')
# Wait for connectivity
waiter = ConnectivityCounter()
yield waiter.waitForConnectivity(numNodes+numClassNodes, self.bsNode) # Does not count bsNode itself.
# Send a class query out
msgNum = random.randint(0,1000000)
classSpec = ClassIDFactory.generateSpec(computerType=ClassIDFactory.WINDOWS, hwType=None, userType=None) # Only Windows boxes.
yield TestUtils.sendClassQuery(self.bsNode, classSpec, msgNum )
yield TestUtils.wait(3)
enclaveID = 1512176389782608
print("DEBUG: BsNode ID: %s" % self.bsNode.nodeLocation.id)
for o in classObservers:
print("DEBUG: [%s] ClassNode: %s %s -> %s" % (o.getMessageCount(), o.chordNode.nodeLocation.id, o.chordNode.nodeLocation, o.chordNode.remote_getSuccessorLocation(enclaveID)))
for o in normalObservers:
print("DEBUG: [%s] NormalNode: %s %s -> %s" % (o.getMessageCount(), o.chordNode.nodeLocation.id, o.chordNode.nodeLocation, o.chordNode.remote_getSuccessorLocation(enclaveID)))
# Verify that only one client node received it
for o in normalObservers:
self.assertTrue( o.getMessageCount() == 0, "Class message incorrectly went to a non-class node!")
for o in classObservers:
self.assertTrue( o.getMessageCount() == 1, "Node in class did not receive the class query! [Count:%d][ID:%s]" % (o.getMessageCount(), o.chordNode.nodeLocation.id))
# Everyone leave
for n in classNodes:
yield n.leave()
for n in normalNodes:
yield n.leave()
yield self.bsNode.leave()
# Wait for the connections to really all close
if Config.USE_CONNECTION_CACHE:
yield TestUtils.wait(Config.CONNECTION_CACHE_DELAY + 3)
else:
yield TestUtils.wait(3)
defer.returnValue(True)
def testReBootstrap(self):
'''Create a BS node, then a client node, then kill the BS Node, wait, restart a BS node and
check for re-Bootstrap
'''
global startingPort
# Create Bootstrap
port = 12345
bootstrapNodeLocation = NodeLocation(None, self.myIP, port)
self.allNodes = []
self.allMetricsObservers = []
self.allTestObservers = []
# Build the BS node
log.msg("building BS node...")
(status, bsClientAPI, bsNetworkAPI) = yield TestUtils.startNodeUsingAPI(self.myIP, port, None, 'theEnclave', False, True)
self.assertTrue(status, 'Could not build bootstrap node')
# Build the client node
log.msg("building client node...")
(status, clClientAPI, clNetworkAPI) = yield TestUtils.startNodeUsingAPI(self.myIP, port+1, bootstrapNodeLocation, 'theEnclave', False, False)
self.assertTrue(status, 'Could not build client node')
# Check that the client is connected to something
connected = yield clNetworkAPI.isConnected()
self.assertTrue(connected, "Client did not connect to the bootstrap node in testReBootstrap!")
# Now kill the BS node
yield bsNetworkAPI.disconnect()
bsNetworkAPI = None
bsClientAPI = None
# Gotta wait for disconnect to really finish
yield TestUtils.waitForConnectionCache()
# Check that the client is connected to something
connected = yield clNetworkAPI.isConnected()
self.assertTrue(not connected, "Client remains connected to the bootstrap node in testReBootstrap after killing BS node!")
# Now startup another bootstrap node
log.msg("building BS node...")
(status, bsClientAPI, bsNetworkAPI) = yield TestUtils.startNodeUsingAPI(self.myIP, port, None, 'theEnclave', False, True)
self.assertTrue(status, 'Could not build second bootstrap node')
# Wait for it to connect or fail -- basically waiting for the
# maintenance call to run correctly.
for _ in range(10):
# Check that the client is connected to something
connected = yield clNetworkAPI.isConnected()
if connected:
break
yield TestUtils.wait(1)
self.assertTrue(connected, "Client could not re-bootstrap!")
# Now shut everything down
log.msg("\n\ntestReBootstrap: Shutting down now...\n\n")
yield clNetworkAPI.disconnect()
yield bsNetworkAPI.disconnect()
if Config.USE_CONNECTION_CACHE:
yield TestUtils.waitForConnectionCache()
else:
yield TestUtils.wait(5)
defer.returnValue(True)
def testNodeIPChanging(self):
'''Try to create a node which needs an IP change, but try it with and without allowing
IP changes. This should test the allowFloatingIP capability.
'''
# Update this to speed up this test
Config.NETWORK_CONNECTION_TIMEOUT = 3
port = 12345
enclave = 'localhost'
bootstrapNodeLocation = NodeLocation(None, self.myIP, port)
# Start a bootstrap node
(status, bsNode, _) = yield TestUtils.startupBootstrapNode(self.myIP, port, enclave)
self.assertTrue(status, 'Could not build bootstrap node')
# Start a client node with a bad IP, and wait for it to change
allowFloatingIP = True
log.msg("\n\n Starting Client Node1 \n\n", system="testNodeIPChanging")
(status, node, _) = yield TestUtils.startupClientNode(self.myIP, 12346, enclave, bootstrapNodeLocation, allowFloatingIP)
self.assertTrue(status, 'Could not startupClientNode')
netObs1 = NetObserver()
node.addNetworkObserver(netObs1.setStatus)
# Are they connected together?
status = yield self.doConnectionTest(node)
self.assertTrue(status, 'doConnectionTest')
allowFloatingIP = False
log.msg("\n\n Starting Client Node2 \n\n", system="testNodeIPChanging")
(status, node2, _) = yield TestUtils.startupClientNode(self.myIP, 12347, enclave, bootstrapNodeLocation, allowFloatingIP)
self.assertTrue(status, 'Could not startupClientNode')
netObs2 = NetObserver()
node2.addNetworkObserver(netObs2.setStatus)
# Are they connected together?
status = yield self.doConnectionTest(node2)
self.assertTrue(status, 'doConnectionTest')
# Now, change the IP to a "bad IP" and wait.
badIP = "192.168.9.10"
node.nodeLocation.ip=badIP
node2.nodeLocation.ip=badIP
# Now we are waiting for the maint calls to detect the change, and
# Issue a new IP
yield TestUtils.wait(20)
# Check the current IP and verify it's not the bad ip
self.assertFalse(node.nodeLocation.ip == badIP, "Chord node did not update to a new IP, but should have!")
self.assertTrue(node2.nodeLocation.ip == badIP, "Chord node changed the IP, but wasn't allowed to float.")
# Verify the statusus
self.assertTrue(netObs1.inc == True, "network observer 1 failed incoming!")
self.assertTrue(netObs1.out == True, "network observer 1 failed outgoing!")
self.assertTrue(netObs2.inc == False, "network observer 2 failed incoming!")
self.assertTrue(netObs2.out == True, "network observer 2 failed outgoing!")
# Stop the nodes
yield bsNode.leave()
yield node.leave()
yield node2.leave()
yield TestUtils.wait(15)
# Update this to speed up this test
Config.NETWORK_CONNECTION_TIMEOUT = 30
defer.returnValue(True)
def testJoinEnclaveAPI(self):
'''Create a node, then have it join another enclave... do this for both a bootstrap node, and
a client node.
'''
global startingPort
# Create Bootstrap
port = 12345
bootstrapNodeLocation = NodeLocation(None, self.myIP, port)
bootstrapNodeList = [ bootstrapNodeLocation ]
self.allNodes = []
self.allMetricsObservers = []
self.allTestObservers = []
# Build the client and network objects
enclaveStr1 = 'enclave1'
enclaveStr2 = 'enclave2'
self.bsClient = SampleClient(self.myIP, port, None)
self.bsNetwork = classChordNetworkChord(self.bsClient, port, self.myIP)
bsID = self.bsNetwork.generateNodeID(str(port), enclaveStr1) # Get the ID with the bits on it we need. Use "port" because it'll be uniq for tests
# Join the network
log.msg("---- Bootstrap Join 1 ---- ", system="testJoinEnclaveAPI")
d = defer.Deferred()
callFunc = lambda x, payload: self.shouldSucceedCallback(x, payload, d, self.bsNetwork)
self.bsNetwork.start(callFunc, bsID, enclaveStr1, "authenticate:succeed", None, True, True)
yield d # Wait for join to succeed
# Now have the bootstrap node join another enclave
log.msg("---- Bootstrap Join 2 ---- ", system="testJoinEnclaveAPI")
d = defer.Deferred()
callFunc2 = lambda x, payload: self.shouldSucceedCallback(x, payload, d, self.bsNetwork)
self.bsNetwork.joinEnclave([ (bootstrapNodeLocation.ip, bootstrapNodeLocation.port) ] , True, "authenticate:succeed", enclaveStr2, callFunc2, True)
yield d
# Now have a client try to join one of the enclaves
log.msg("---- Start Client 1 ---- ", system="testJoinEnclaveAPI")
yield self.startClientNode(None, self.myIP, 12350+1, bootstrapNodeLocation, enclaveStr1)
# Have another client join the other enclave
log.msg("---- Start Client 2 ---- ", system="testJoinEnclaveAPI")
yield self.startClientNode(None, self.myIP, 12350+2, bootstrapNodeLocation, enclaveStr2)
# Do a bit of verification
rc = yield self.bsNetwork.isConnected("ANY")
self.assertTrue(rc != False, "isConnected returned a False value instead of enclave ID")
rc = yield self.bsNetwork.isConnected(enclaveStr1)
self.assertTrue(rc != False, "isConnected returned a False value instead of enclave ID")
rc = yield self.bsNetwork.isConnected(enclaveStr2)
self.assertTrue(rc != False, "isConnected returned a False value instead of enclave ID")
rc = yield self.bsNetwork.isConnected("NOTREAL")
self.assertTrue(rc == False, "isConnected returned True value instead of False!")
# Now shut everything down
for (_clientAPI, networkAPI) in self.allNodes:
yield networkAPI.disconnect()
yield self.bsNetwork.disconnect()
# Now wait for the network cache disconnect timeout
yield TestUtils.wait(Config.CONNECTION_CACHE_DELAY + 1)
defer.returnValue(True)
