def testSerialFlooding(self):
# Start a bootstrap node
(status, self.bsNode, _observer) = yield TestUtils.startupBootstrapNode(self.myIP, 12345, 'localhost')
self.assertTrue(status, 'Could not build bootstrap node')
self.allNodes.append(self.bsNode)
self.bsNode.addMessageObserver(self.messageReceived)
# Start client nodes
for i in range(numNodes):
(status, node, observer) = yield TestUtils.startupClientNode(self.myIP, 12346+i, 'localhost', self.bsNode.nodeLocation)
self.assertTrue(status, 'Could not startupClientNode')
self.allNodes.append(node)
observer = MyMessageObserver()
node.addMessageObserver(observer.messageReceived)
observer.node = node
self.allObservers.append(observer)
# Wait for flooding to reach all the nodes
waiter = ConnectivityCounter()
yield waiter.waitForConnectivity(numNodes, self.bsNode) # Does not count bsNode itself.
# Now do the stress test
status = yield self.doStressTest()
# Now close it all down!
yield self.allLeave()
# Wait a second or two for network timeouts
yield TestUtils.wait(9)
defer.returnValue(True)
def buildNetwork(self):
global startingPort
# Create Bootstrap
port = 12345
bootstrapNodeLocation = NodeLocation(None, self.myIP, port)
self.allNodeObservers = []
self.allBootstrapObservers = []
self.allNodes = []
enclave = "theEnc"
# Start a bootstrap node
(status, self.bsNode, observer) = yield TestUtils.startupBootstrapNode(self.myIP, port, enclave)
self.assertTrue(status, "Could not build bootstrap node")
self.allBootstrapObservers.append(observer)
# Add X nodes to each enclave
for _ in range(numNodes):
# Add a node to the enclave
(status, node, observer) = yield TestUtils.startupClientNode(
self.myIP, startingPort, enclave, bootstrapNodeLocation
)
self.assertTrue(status, "Could not startupClientNode")
startingPort += 1
self.allNodes.append(node)
self.allNodeObservers.append(observer)
# Wait for flooding to reach all the nodes
waiter = ConnectivityCounter()
yield waiter.waitForConnectivity(numNodes + 1, self.bsNode) # Does not bsNode .
defer.returnValue(True)
def testSerialP2PSending(self):
# Start a bootstrap node
(status, self.bsNode, _observer) = yield TestUtils.startupBootstrapNode(self.myIP, 12345, 'localhost')
self.assertTrue(status, 'Could not build bootstrap node')
self.allNodes.append(self.bsNode)
self.bsNode.addMessageObserver(self.messageReceived)
# Start client nodes
log.msg("Building nodes...")
for i in range(numNodes):
(status, node, observer) = yield TestUtils.startupClientNode(self.myIP, 12346+i, 'localhost', self.bsNode.nodeLocation)
self.assertTrue(status, 'Could not startupClientNode')
self.allNodes.append(node)
# Wait for flooding to reach all the nodes
waiter = ConnectivityCounter()
yield waiter.waitForConnectivity(numNodes, self.bsNode) # Does not count bsNode itself.
# Do the real test
status = yield self.doStressTest()
# Now close it all down!
yield self.allLeave()
# Wait a second or two
yield TestUtils.wait(3+Config.CONNECTION_CACHE_DELAY)
defer.returnValue(True)
def testDisconnectedBootstraps(self):
'''Create a BS node and some clients. Create another bootstrap node and some clients (so we essentially have two rings).
Verify, that the bootstrap nodes autodiscover each other and connect together
'''
global startingPort
# Create Bootstrap
port = 12345
bootstrapNodeLocation = NodeLocation(None, self.myIP, port)
bootstrapNodeLocation2 = NodeLocation(None, self.myIP, port+1)
self.allNodes = []
self.allMetricsObservers = []
self.allTestObservers = []
# Build the BS node
(status, bsClientAPI, bsNetworkAPI) = yield TestUtils.startNodeUsingAPI(bootstrapNodeLocation.ip, bootstrapNodeLocation.port, None, 'theEnclave', True, True)
self.allMetricsObservers.append(MetricsMessageObserver(bsNetworkAPI.chordNode))
self.assertTrue(status, 'Could not build bootstrap node')
# Build second BS node
(status, bsClientAPI2, bsNetworkAPI2) = yield TestUtils.startNodeUsingAPI(bootstrapNodeLocation2.ip, bootstrapNodeLocation2.port, None, 'theEnclave', True, True)
self.allMetricsObservers.append(MetricsMessageObserver(bsNetworkAPI2.chordNode))
self.assertTrue(status, 'Could not build bootstrap node 2')
# Build the client node
(status, clClientAPI, clNetworkAPI) = yield TestUtils.startNodeUsingAPI(self.myIP, port+2, bootstrapNodeLocation, 'theEnclave', False, False)
self.allMetricsObservers.append(MetricsMessageObserver(clNetworkAPI.chordNode))
self.assertTrue(status, 'Could not build client node')
# Build the client node
(status, clClientAPI2, clNetworkAPI2) = yield TestUtils.startNodeUsingAPI(self.myIP, port+3, bootstrapNodeLocation2, 'theEnclave', False, False)
self.allMetricsObservers.append(MetricsMessageObserver(clNetworkAPI2.chordNode))
self.assertTrue(status, 'Could not build client node')
# Wait for flooding to reach all the nodes
waiter = ConnectivityCounter()
yield waiter.waitForConnectivity(3, clNetworkAPI.chordNode) # Does not count clNode itself.
# Now shut everything down
yield clNetworkAPI.disconnect()
yield clNetworkAPI2.disconnect()
yield bsNetworkAPI.disconnect()
yield bsNetworkAPI2.disconnect()
if Config.USE_CONNECTION_CACHE:
yield TestUtils.waitForConnectionCache()
else:
yield TestUtils.wait(5)
defer.returnValue(True)
def buildNetwork(self):
global startingPort
# Create two Bootstrap nodes
port = 12345
self.allNodeObservers = []
self.allBootstrapObservers = []
self.allNodes = []
# Start a bootstrap node
(status, self.bsNode1, observer) = yield TestUtils.startupBootstrapNode(self.myIP, port, allEnclaves[0])
self.assertTrue(status, 'Could not build bootstrap node')
self.allBootstrapObservers.append(observer)
# Start another bootstrap node
(status, self.bsNode2, observer) = yield TestUtils.startupBootstrapNode(self.myIP, port+1, allEnclaves[1])
self.assertTrue(status, 'Could not build bootstrap node')
self.allBootstrapObservers.append(observer)
# Add some nodes to each bootstrap node
for _ in range(numNodes):
# Add a node to the enclave
(status, node, observer) = yield TestUtils.startupClientNode(self.myIP, startingPort, allEnclaves[0], self.bsNode1.nodeLocation)
self.assertTrue(status, 'Could not startupClientNode')
startingPort += 1
self.allNodes.append(node)
self.allNodeObservers.append(observer)
for _ in range(numNodes):
# Add a node to the enclave
(status, node, observer) = yield TestUtils.startupClientNode(self.myIP, startingPort, allEnclaves[1], self.bsNode2.nodeLocation)
self.assertTrue(status, 'Could not startupClientNode')
startingPort += 1
self.allNodes.append(node)
self.allNodeObservers.append(observer)
# Wait for flooding to reach all the nodes
waiter = ConnectivityCounter()
yield waiter.waitForConnectivity(numNodes, self.bsNode1) # Does not count bsNode itself.
yield waiter.waitForConnectivity(numNodes, self.bsNode2) # Does not count bsNode itself.
defer.returnValue(True)
def buildNetwork(self):
global startingPort
# Create Bootstrap
port = 12345
bootstrapNodeLocation = NodeLocation(None, self.myIP, port)
self.allNodeObservers = []
self.allNetworkAPIs = []
self.allClientAPIs = []
# Start a bootstrap node
log.msg("building BS node...")
(status, clientAPI, networkAPI) = yield TestUtils.startNodeUsingAPI(self.myIP, port, None, 'theEnclave', False, True)
self.assertTrue(status, 'Could not build bootstrap node')
node = networkAPI.chordNode # This is very highly coupled -- bad
MetricsMessageObserver(node)
self.observer = TestMessageObserver(node, networkAPI)
self.bsNode = node
log.msg("building client nodes...")
# Add X nodes to each enclave
for _ in range(numNodes):
# Add a node to the enclave
(status, clientAPI, networkAPI) = yield TestUtils.startNodeUsingAPI(self.myIP, startingPort, bootstrapNodeLocation, 'theEnclave', False, False)
self.assertTrue(status, 'Could not startupClientNode')
node = networkAPI.chordNode # This is very highly coupled -- bad
MetricsMessageObserver(node)
observer = TestMessageObserver(node, networkAPI)
startingPort += 1
self.allNetworkAPIs.append(networkAPI)
self.allNodeObservers.append(observer)
self.allClientAPIs.append(clientAPI)
# Wait for flooding to reach all the nodes
waiter = ConnectivityCounter()
yield waiter.waitForConnectivity(numNodes+1, self.bsNode) # Does count bsNode itself.
defer.returnValue(True)
def buildNetwork(self):
global startingPort
# Create Bootstrap
port = 12345
bootstrapNodeLocation = NodeLocation(None, self.myIP, port)
self.allNodeObservers = []
self.allBootstrapObservers = []
self.allNodes = []
# Start a bootstrap node
log.msg("---> Startup Bootstrap Node", system="EnclaveP2P Test")
(status, self.bsNode, observer) = yield TestUtils.startupBootstrapNode(self.myIP, port, allEnclaves[0])
self.assertTrue(status, 'Could not build bootstrap node')
self.allBootstrapObservers.append(observer)
# Join another enclave
enableAutoDiscovery = True
bootstrapNodeList = [ NodeLocation(None, self.myIP, port) ]
status = yield self.bsNode.joinEnclave(bootstrapNodeList, enableAutoDiscovery, None, isBootstrapNode=True, enclaveStr=allEnclaves[1])
# Add X nodes to each enclave
for enclave in allEnclaves:
for _ in range(numNodes):
log.msg("---> Add node to enclave", system="EnclaveP2P Test")
# Add a node to the enclave
(status, node, observer) = yield TestUtils.startupClientNode(self.myIP, startingPort, enclave, bootstrapNodeLocation)
self.assertTrue(status, 'Could not startupClientNode')
startingPort += 1
self.allNodes.append(node)
self.allNodeObservers.append(observer)
# Wait for flooding to reach all the nodes
waiter = ConnectivityCounter()
numberOfNodes = (len(allEnclaves) * numNodes)
yield waiter.waitForConnectivity(numberOfNodes+1, self.bsNode) # Does not count bsNode itself.
defer.returnValue(True)
def testNodeDisconnection(self):
"""Disconnect a node from the ring and do a few things.
- Send a message to that node (verify error condition)
- Send a flooding message -- wait until connectivity happens for all other nodes.
"""
# Disconnect a node
nodeIndex = random.randint(0, numNodes - 1) # Node to disconnect
nodeLoc = self.allNodes[nodeIndex].nodeLocation
log.msg("testNodeDisconnection: Node is leaving [%s]...\n" % nodeLoc)
ConnectionCache._printCache()
yield self.allNodes[nodeIndex].leave()
log.msg("testNodeDisconnection: Node left...\n")
try:
# Now wait for full connectivity to come back
waiter = ConnectivityCounter()
yield waiter.waitForConnectivity(numNodes, self.bsNode) # Does count bsNode.
except error.ConnectionRefusedError:
log.err("NodeDisconnect test got a connection refused error [2]")
log.err()
# Now wait for the disconnect to fully take hold
yield TestUtils.wait(30) # Network timeout time
print("DEBUG testNodeDisc \n\n\n\n")
ConnectionCache._printCache()
try:
# P2P from bootstrap to node which left
yield self.sendFailingP2PMsg(self.bsNode, self.allNodes[nodeIndex])
except error.ConnectionRefusedError:
log.msg("\n\nNodeDisconnect test got a connection refused error [3], which is okay!\n\n")
defer.returnValue(True)
def testEnclaveFlooding(self):
"""Build 2 enclaves and test flooding to each and all."""
global startingPort
# Create Bootstrap
port = 12345
bootstrapNodeLocation = NodeLocation(None, self.myIP, port)
allNodes = []
allNodeObservers = []
allBootstrapObservers = []
self.allNodes = allNodes
# Start a bootstrap node
(status, bsNode, observer) = yield TestUtils.startupBootstrapNode(self.myIP, port, allEnclaves[0])
self.assertTrue(status, "Could not build bootstrap node")
allBootstrapObservers.append(observer)
self.bsNode = bsNode
# Join another enclave
bootstrapNodeList = None
enableAutoDiscovery = True
status = yield self.bsNode.joinEnclave(
bootstrapNodeList, enableAutoDiscovery, None, isBootstrapNode=True, enclaveStr=allEnclaves[1]
)
# Add X nodes to each enclave
for enclave in allEnclaves:
for _ in range(numNodes):
# Add a node to the enclave
(status, node, observer) = yield TestUtils.startupClientNode(
self.myIP, startingPort, enclave, bootstrapNodeLocation
)
self.assertTrue(status, "Could not startupClientNode")
startingPort += 1
allNodes.append(node)
allNodeObservers.append(observer)
# Wait for flooding to reach all the nodes
waiter = ConnectivityCounter()
numberOfNodes = len(allEnclaves) * numNodes
yield waiter.waitForConnectivity(numberOfNodes, bsNode) # Does not count bsNode itself.
# Flood from bootstrap to all
messageNum = random.randint(1, 9999999)
status = yield TestUtils.sendFlood(bsNode, messageNum, "ALL")
self.assertTrue(status, "sendFlood failed!")
yield TestUtils.wait(3) # Wait for the messages to send
# Check message counts
status = TestUtils.didReceive(allNodeObservers, "ALL", messageNum, numberOfNodes)
self.assertTrue(status, "All nodes did not receive message")
# Flood from bootstrap to single enclave
messageNum = random.randint(1, 9999999)
status = yield TestUtils.sendFlood(bsNode, messageNum, allEnclaves[0])
self.assertTrue(status, "sendFlood failed!")
yield TestUtils.wait(3) # Wait for the messages to send
# Check message counts
status = TestUtils.didReceive(allNodeObservers, allEnclaves[0], messageNum, numNodes)
self.assertTrue(status, "Nodes in enclave %s didn't all recv message")
status = TestUtils.didNotReceive(allNodeObservers, allEnclaves[1], messageNum, numNodes)
self.assertTrue(status, "Some Nodes in enclave %s did recv message")
log.msg("\n\n\n TEST ARE ALL DONE \n\n\n")
yield TestUtils.wait(60)
# Stop everything
for node in self.allNodes:
yield node.leave()
yield self.bsNode.leave()
# Wait for all network timeouts to finish
yield TestUtils.wait(Config.CONNECTION_CACHE_DELAY + 3)
# self.flushLoggedErrors()
defer.returnValue(True)
def waitForConnectivity(self, _, numNodes):
''' Wait for numNodes to be connectable from the Bootstrap Node passed in.
Returns a deferred which fires upon all nodes connecting.
'''
waiter = ConnectivityCounter()
return waiter.waitForConnectivity(numNodes, self.bsNetwork.chordNode)
def testNodeSendingToClass(self):
class1 = ClassIDFactory.generateID(ClassIDFactory.WINDOWS, ClassIDFactory.LAPTOP, None)
class2 = ClassIDFactory.generateID(ClassIDFactory.MAC, ClassIDFactory.DESKTOP, None)
normalNodes = []
normalObservers = []
classNodes = []
classObservers = []
# Start a bootstrap node
(status, self.bsNode, _observer) = yield TestUtils.startupBootstrapNode(self.myIP, 12345, 'localhost', class1)
self.assertTrue(status, 'Could not build bootstrap node')
# Start 10 client nodes not in the class
for i in range(numNodes):
(status, normalNode, observer) = yield TestUtils.startupClientNode(self.myIP, 12346+i, 'localhost', self.bsNode.nodeLocation, classID=class2)
normalNodes.append(normalNode)
normalObservers.append(observer)
self.assertTrue(status, 'Could not startupClientNode')
# Start a client node in the class
for i in range(numClassNodes):
(status, classNode, observer) = yield TestUtils.startupClientNode(self.myIP, 12346+numNodes+i, 'localhost', self.bsNode.nodeLocation, classID=class1)
classNodes.append(classNode)
classObservers.append(observer)
self.assertTrue(status, 'Could not startupClientNode')
# Wait for connectivity
waiter = ConnectivityCounter()
yield waiter.waitForConnectivity(numNodes+numClassNodes, self.bsNode) # Does not count bsNode itself.
# Send a class query out
msgNum = random.randint(0,1000000)
classSpec = ClassIDFactory.generateSpec(computerType=ClassIDFactory.WINDOWS, hwType=None, userType=None) # Only Windows boxes.
yield TestUtils.sendClassQuery(self.bsNode, classSpec, msgNum )
yield TestUtils.wait(3)
enclaveID = 1512176389782608
print("DEBUG: BsNode ID: %s" % self.bsNode.nodeLocation.id)
for o in classObservers:
print("DEBUG: [%s] ClassNode: %s %s -> %s" % (o.getMessageCount(), o.chordNode.nodeLocation.id, o.chordNode.nodeLocation, o.chordNode.remote_getSuccessorLocation(enclaveID)))
for o in normalObservers:
print("DEBUG: [%s] NormalNode: %s %s -> %s" % (o.getMessageCount(), o.chordNode.nodeLocation.id, o.chordNode.nodeLocation, o.chordNode.remote_getSuccessorLocation(enclaveID)))
# Verify that only one client node received it
for o in normalObservers:
self.assertTrue( o.getMessageCount() == 0, "Class message incorrectly went to a non-class node!")
for o in classObservers:
self.assertTrue( o.getMessageCount() == 1, "Node in class did not receive the class query! [Count:%d][ID:%s]" % (o.getMessageCount(), o.chordNode.nodeLocation.id))
# Everyone leave
for n in classNodes:
yield n.leave()
for n in normalNodes:
yield n.leave()
yield self.bsNode.leave()
# Wait for the connections to really all close
if Config.USE_CONNECTION_CACHE:
yield TestUtils.wait(Config.CONNECTION_CACHE_DELAY + 3)
else:
yield TestUtils.wait(3)
defer.returnValue(True)
