class IncomingDataThread(QueueProcessingThread): """ This threads will process the incoming network packets. """ def __init__(self, transferQueue, callbackObject): """ Initializes the thread's state Args: transferQueue: The incoming packages transferQueue callbackObject: The callback object that will process all the received packets. """ QueueProcessingThread.__init__(self, "Incoming data processing thread", transferQueue) self.__callbackObject = callbackObject self.__referenceCounter = MultithreadingCounter() def start(self): """ Starts the thread Args: None Returns: Nothing """ self.__referenceCounter.increment() if self.__referenceCounter.read() == 1: QueueProcessingThread.start(self) def stop(self, join): """ Asks this thread to terminate. Args: join: When True, the calling thread will wait the incoming data thread to terminate. When False, the calling thread will only ask this thread to terminate. Returns: Nothing """ self.__referenceCounter.decrement() if self.__referenceCounter.read() == 0: QueueProcessingThread.stop(self) if join: self.join() def run(self): QueueProcessingThread.run(self) def processElement(self, e): """ Processes a received packet Args: e: The packet to process Returns: Nothing """ self.__callbackObject.processPacket(e)
class IncomingDataThread(QueueProcessingThread): """ This threads will process the incoming network packets. """ def __init__(self, transferQueue, callbackObject): """ Initializes the thread's state Args: transferQueue: The incoming packages transferQueue callbackObject: The callback object that will process all the received packets. """ QueueProcessingThread.__init__(self, "Incoming data processing thread", transferQueue) self.__callbackObject = callbackObject self.__referenceCounter = MultithreadingCounter() def start(self): """ Starts the thread Args: None Returns: Nothing """ self.__referenceCounter.increment() if (self.__referenceCounter.read() == 1): QueueProcessingThread.start(self) def stop(self, join): """ Asks this thread to terminate. Args: join: When True, the calling thread will wait the incoming data thread to terminate. When False, the calling thread will only ask this thread to terminate. Returns: Nothing """ self.__referenceCounter.decrement() if self.__referenceCounter.read() == 0: QueueProcessingThread.stop(self) if join: self.join() def run(self): QueueProcessingThread.run(self) def processElement(self, e): """ Processes a received packet Args: e: The packet to process Returns: Nothing """ self.__callbackObject.processPacket(e)
class ImageRepositoryReactor(object): """ Image repository packet reactor """ def __init__(self, diskImagesDirectory): """ Initializes the reactor's state Args: diskImagesDirectory: the FTP server's root directory """ self.__diskImagesDirectory = diskImagesDirectory self.__slotCounter = MultithreadingCounter() self.__retrieveQueue = GenericThreadSafeList() self.__storeQueue = GenericThreadSafeList() self.__finish = False self.__networkManager = None self.__ftpServer = None def connectToDatabase(self, repositoryDBName, repositoryDBUser, repositoryDBPassword): """ Establishes the connection with the image repository's database. Args: repositoryDBName: a database name repositoryDBUser: an user name repositoryDBPassword: a password Returns: Nothing """ self.__dbConnector = ImageRepositoryDBConnector( repositoryDBUser, repositoryDBPassword, repositoryDBName) def startListenning(self, networkInterface, useSSL, certificatesDirectory, commandsListenningPort, ftpListenningPort, maxConnections, maxConnectionsPerIP, uploadBandwidthRatio, downloadBandwidthRatio, ftpUsername, ftpPasswordLength): """ Boots up the FTP server and creates the control connection. Args: networkInterface: the network interface that will be used by the FTP server useSSL: indicates if SSL encryption must be used in the control connection or not certificatesDirectory: the directory where the files server.crt and server.key are commandsListenningPort: the control connection's port ftpListenningPort: the FTP server listenning port maxConnections: maximum FTP connections maxConnectionsPerIP: maximum FTP connections per IP address uploadBandwidthRatio: maximum download bandwidth fraction downloadBandwidthRatio: maximum upload bandwidth fraction ftpUsername: the FTP user that the virtual machine servers will use ftpPasswordLength: the random FTP password length Returns: Nothing @attention: The FTP password will be randomly generated at every boot. """ try: self.__maxConnections = maxConnections self.__commandsListenningPort = commandsListenningPort self.__FTPListenningPort = ftpListenningPort self.__networkManager = NetworkManager(certificatesDirectory) self.__repositoryPacketHandler = ImageRepositoryPacketHandler( self.__networkManager) self.__commandsCallback = CommandsCallback( self.__networkManager, self.__repositoryPacketHandler, commandsListenningPort, self.__dbConnector, self.__retrieveQueue, self.__storeQueue, self.__diskImagesDirectory) self.__networkManager.startNetworkService() self.__networkManager.listenIn(commandsListenningPort, self.__commandsCallback, useSSL) dataCallback = FTPServerCallback(self.__slotCounter, self.__dbConnector) self.__ftpUsername = ftpUsername self.__ftpPassword = ChildProcessManager.runCommandInForeground( "openssl rand -base64 {0}".format(ftpPasswordLength), Exception) self.__ftpServer = ConfigurableFTPServer( "Image repository FTP Server") self.__ftpServer.startListenning(networkInterface, ftpListenningPort, maxConnections, maxConnectionsPerIP, dataCallback, downloadBandwidthRatio, uploadBandwidthRatio) self.__ftpServer.addUser(self.__ftpUsername, self.__ftpPassword, self.__diskImagesDirectory, "eramw") except Exception as e: print "Error: " + e.message self.__finish = True def stopListenning(self): """ Stops the FTP server and closes the control connection. Args: None Returns: Nothing """ if (self.__ftpServer != None): try: self.__ftpServer.stopListenning() except Exception: pass if (self.__networkManager != None): self.__networkManager.stopNetworkService() def initTransfers(self): """ Initializes the upload and download transfers Args: None Returns: Nothing """ store = False while not (self.__finish or self.__commandsCallback.haltReceived()): if (self.__slotCounter.read() == self.__maxConnections): # No free slots => sleep sleep(0.1) else: # There are slots => enable uploads and downloads in parallel self.__slotCounter.decrement() if (self.__retrieveQueue.isEmpty() and self.__storeQueue.isEmpty()): sleep(0.1) continue if (not self.__retrieveQueue.isEmpty() and self.__storeQueue.isEmpty()): queue = self.__retrieveQueue store = False elif (self.__retrieveQueue.isEmpty() and not self.__storeQueue.isEmpty()): queue = self.__storeQueue store = True else: if (store): queue = self.__retrieveQueue store = False else: queue = self.__storeQueue store = True (imageID, clientIP, clientPort) = queue.pop(0) imageData = self.__dbConnector.getImageData(imageID) if (imageData == None): if (store): packet_type = PACKET_T.STOR_ERROR else: packet_type = PACKET_T.RETR_ERROR p = self.__repositoryPacketHandler.createErrorPacket( packet_type, ERROR_DESC_T.IR_IMAGE_DELETED) self.__networkManager.sendPacket( '', self.__commandsListenningPort, p, clientIP, clientPort) else: compressedFilePath = imageData["compressedFilePath"] if (not "undefined" in compressedFilePath): serverDirectory = path.relpath( path.dirname(compressedFilePath), self.__diskImagesDirectory) compressedFileName = path.basename(compressedFilePath) else: serverDirectory = str(imageID) compressedFileName = "" serverDirectoryPath = path.join( self.__diskImagesDirectory, serverDirectory) if (path.exists(serverDirectoryPath)): # The directory exists, and can store shit => clean it up! ChildProcessManager.runCommandInForeground( "rm -rf " + serverDirectoryPath, Exception) mkdir(serverDirectoryPath) if (store): packet_type = PACKET_T.STOR_START else: packet_type = PACKET_T.RETR_START p = self.__repositoryPacketHandler.createTransferEnabledPacket( packet_type, imageID, self.__FTPListenningPort, self.__ftpUsername, self.__ftpPassword, serverDirectory, compressedFileName) self.__networkManager.sendPacket( '', self.__commandsListenningPort, p, clientIP, clientPort)
class Connection(object): """ This is the base class for all network connections. """ def __init__(self, useSSL, certificatesDirectory, port, transferQueue, incomingDataThread, callbackObject): """ Initializes the connection. Args: useSSL: if True, all the traffic will be protectd by SSLv4. If false, certificatesDirectory: the directory where the certificates are stored port: the port assigned to the connection. transferQueue: the incoming data transferQueue assigned to the connection incomingDataThread: the incoming data thread assigned to the connection callbackObject: the callback object assigned to the connection """ self._status = ConnectionStatus(CONNECTION_STATUS.OPENING) self._useSSL = useSSL self._certificatesDirectory = certificatesDirectory self._port = port self._factory = None self._queue = transferQueue self._incomingDataThread = incomingDataThread self._callback = callbackObject self._packagesToSend = MultithreadingCounter() self._deferred = None self._unexpectedlyClosed = False self._error = None @staticmethod def _prettyPrintTwistedError(uglyTwistedError): """ Puts an ugly twisted error in a human readable form Args: uglyTwistedError: the twisted error to pretty print Returns: A string containing the error message """ errorStr = str(uglyTwistedError) i = 0 while i < 3: (_head, _sep, errorMessage) = errorStr.partition(":") errorStr = errorMessage i += 1 (head, _sep, _errorMessage) = errorStr.partition(":") return head def establish(self, timeout=None): """ Tries to establish the network connection. Args: timeout: the timeout in seconds. This argument will only be used with client connections. Returns: True if the connection could be established. Otherwise, it will return false. """ self._factory = CygnusCloudProtocolFactory(self._queue) def getHost(self): """ Returns the server's hostname or IPv4 address Args: None Returns: The server's IP address. """ raise NotImplementedError def getPort(self): """ Returns the port assigned to this connection. Args: None Returns: The port assigned to this connection. """ return self._port def getQueue(self): """ Returns the incoming data queue assigned to this connection. Args: None Returns: The incoming data queue assigned to this connection """ return self._queue def getThread(self): """ Returns the incoming data processing thread assigned to this connection Args: None Returns: The incoming data processing thread assigned to this connection """ return self._incomingDataThread def getCallback(self): """ Returns the callback object assigned to this connection Args: None Returns: The callback object assigned to this connection. """ return self._callback def getStatus(self): """ Returns the connection status Args: None Returns: The connection status """ return self._status.get() def sendPacket(self, p, client_IP=None, client_port=None): """ Sends a packet though this connection. If the connection is closed, the packet will be discarded. Args: p: the packet to send client_IP: the client's IPv4 address client_port: the client's port @attention: The two last parameters will only be used with server connections. Returns: Nothing """ if (self._status.get() == CONNECTION_STATUS.READY or self._status.get() == CONNECTION_STATUS.CLOSING): self._factory.sendPacket(p, client_IP, client_port) self._packagesToSend.decrement() return True def registerPacket(self): """ Registers a packet to be sent through this connection. Args: None Returns: Nothing """ self._packagesToSend.increment() def isReady(self): """ Checks if the connection is ready to send data or not Args: None Returns: True if the connection is ready to send data, and false otherwise. """ return self._status.get() == CONNECTION_STATUS.READY def isServerConnection(self): """ Determines if this is a server connection or not. Args: None Returns: True if this connection is a server one, and false if it isn't. """ return self.__isServerConnection def refresh(self): """ Updates the connection's status Args: None Returns: Nothing """ raise NotImplementedError def isInErrorState(self): """ Checks if the connection is in an error state. Args: None Returns: True if the connection is in an error state, and False if it isn't. """ return self._status.get() == CONNECTION_STATUS.ERROR def wasUnexpectedlyClosed(self): """ Checks if the connection was unexpectedly closed. The connection may be closed unexpectedly in two different ways: 1) All the clients disconnect from a server. 2) A server disconnects from all its clients. Args: None Returns: True if the connection was unexpectedly closed, and False if it wasn't. """ return self._unexpectedlyClosed def getError(self): """ Returns the error message stored in this connection. """ return self._error def _setError(self, value): """ Modifies the error message stored in this connection. """ self._status.set(CONNECTION_STATUS.ERROR) self._error = value def close(self): """ Asks this connection to close Args: None Returns: Nothing """ self._status.set(CONNECTION_STATUS.CLOSING) def _freeTwistedResources(self): """ Destroys the twisted-related connection resources. Args: None Returns: Nothing """ self._factory.disconnect() def _close(self): """ Closes this connection. Args: None Returns: Nothing """ # Free the connection resources self._freeTwistedResources() self._incomingDataThread.stop(True) self._status.set(CONNECTION_STATUS.CLOSED)
class Connection(object): """ This is the base class for all network connections. """ def __init__(self, useSSL, certificatesDirectory, port, transferQueue, incomingDataThread, callbackObject): """ Initializes the connection. Args: useSSL: if True, all the traffic will be protectd by SSLv4. If false, certificatesDirectory: the directory where the certificates are stored port: the port assigned to the connection. transferQueue: the incoming data transferQueue assigned to the connection incomingDataThread: the incoming data thread assigned to the connection callbackObject: the callback object assigned to the connection """ self._status = ConnectionStatus(CONNECTION_STATUS.OPENING) self._useSSL = useSSL self._certificatesDirectory = certificatesDirectory self._port = port self._factory = None self._queue = transferQueue self._incomingDataThread = incomingDataThread self._callback = callbackObject self._packagesToSend = MultithreadingCounter() self._deferred = None self._unexpectedlyClosed = False self._error = None @staticmethod def _prettyPrintTwistedError(uglyTwistedError): """ Puts an ugly twisted error in a human readable form Args: uglyTwistedError: the twisted error to pretty print Returns: A string containing the error message """ errorStr = str(uglyTwistedError) i = 0 while i < 3 : (_head, _sep, errorMessage) = errorStr.partition(":") errorStr = errorMessage i += 1 (head, _sep, _errorMessage) = errorStr.partition(":") return head def establish(self, timeout=None): """ Tries to establish the network connection. Args: timeout: the timeout in seconds. This argument will only be used with client connections. Returns: True if the connection could be established. Otherwise, it will return false. """ self._factory = CygnusCloudProtocolFactory(self._queue) def getHost(self): """ Returns the server's hostname or IPv4 address Args: None Returns: The server's IP address. """ raise NotImplementedError def getPort(self): """ Returns the port assigned to this connection. Args: None Returns: The port assigned to this connection. """ return self._port def getQueue(self): """ Returns the incoming data queue assigned to this connection. Args: None Returns: The incoming data queue assigned to this connection """ return self._queue def getThread(self): """ Returns the incoming data processing thread assigned to this connection Args: None Returns: The incoming data processing thread assigned to this connection """ return self._incomingDataThread def getCallback(self): """ Returns the callback object assigned to this connection Args: None Returns: The callback object assigned to this connection. """ return self._callback def getStatus(self): """ Returns the connection status Args: None Returns: The connection status """ return self._status.get() def sendPacket(self, p, client_IP = None, client_port = None): """ Sends a packet though this connection. If the connection is closed, the packet will be discarded. Args: p: the packet to send client_IP: the client's IPv4 address client_port: the client's port @attention: The two last parameters will only be used with server connections. Returns: Nothing """ if (self._status.get() == CONNECTION_STATUS.READY or self._status.get() == CONNECTION_STATUS.CLOSING) : self._factory.sendPacket(p, client_IP, client_port) self._packagesToSend.decrement() return True def registerPacket(self): """ Registers a packet to be sent through this connection. Args: None Returns: Nothing """ self._packagesToSend.increment() def isReady(self): """ Checks if the connection is ready to send data or not Args: None Returns: True if the connection is ready to send data, and false otherwise. """ return self._status.get() == CONNECTION_STATUS.READY def isServerConnection(self): """ Determines if this is a server connection or not. Args: None Returns: True if this connection is a server one, and false if it isn't. """ return self.__isServerConnection def refresh(self): """ Updates the connection's status Args: None Returns: Nothing """ raise NotImplementedError def isInErrorState(self): """ Checks if the connection is in an error state. Args: None Returns: True if the connection is in an error state, and False if it isn't. """ return self._status.get() == CONNECTION_STATUS.ERROR def wasUnexpectedlyClosed(self): """ Checks if the connection was unexpectedly closed. The connection may be closed unexpectedly in two different ways: 1) All the clients disconnect from a server. 2) A server disconnects from all its clients. Args: None Returns: True if the connection was unexpectedly closed, and False if it wasn't. """ return self._unexpectedlyClosed def getError(self): """ Returns the error message stored in this connection. """ return self._error def _setError(self, value): """ Modifies the error message stored in this connection. """ self._status.set(CONNECTION_STATUS.ERROR) self._error = value def close(self): """ Asks this connection to close Args: None Returns: Nothing """ self._status.set(CONNECTION_STATUS.CLOSING) def _freeTwistedResources(self): """ Destroys the twisted-related connection resources. Args: None Returns: Nothing """ self._factory.disconnect() def _close(self): """ Closes this connection. Args: None Returns: Nothing """ # Free the connection resources self._freeTwistedResources() self._incomingDataThread.stop(True) self._status.set(CONNECTION_STATUS.CLOSED)
class ImageRepositoryReactor(object): """ Image repository packet reactor """ def __init__(self, diskImagesDirectory): """ Initializes the reactor's state Args: diskImagesDirectory: the FTP server's root directory """ self.__diskImagesDirectory = diskImagesDirectory self.__slotCounter = MultithreadingCounter() self.__retrieveQueue = GenericThreadSafeList() self.__storeQueue = GenericThreadSafeList() self.__finish = False self.__networkManager = None self.__ftpServer = None def connectToDatabase(self, repositoryDBName, repositoryDBUser, repositoryDBPassword): """ Establishes the connection with the image repository's database. Args: repositoryDBName: a database name repositoryDBUser: an user name repositoryDBPassword: a password Returns: Nothing """ self.__dbConnector = ImageRepositoryDBConnector(repositoryDBUser, repositoryDBPassword, repositoryDBName) def startListenning(self, networkInterface, useSSL, certificatesDirectory, commandsListenningPort, ftpListenningPort, maxConnections, maxConnectionsPerIP, uploadBandwidthRatio, downloadBandwidthRatio, ftpUsername, ftpPasswordLength): """ Boots up the FTP server and creates the control connection. Args: networkInterface: the network interface that will be used by the FTP server useSSL: indicates if SSL encryption must be used in the control connection or not certificatesDirectory: the directory where the files server.crt and server.key are commandsListenningPort: the control connection's port ftpListenningPort: the FTP server listenning port maxConnections: maximum FTP connections maxConnectionsPerIP: maximum FTP connections per IP address uploadBandwidthRatio: maximum download bandwidth fraction downloadBandwidthRatio: maximum upload bandwidth fraction ftpUsername: the FTP user that the virtual machine servers will use ftpPasswordLength: the random FTP password length Returns: Nothing @attention: The FTP password will be randomly generated at every boot. """ try : self.__maxConnections = maxConnections self.__commandsListenningPort = commandsListenningPort self.__FTPListenningPort = ftpListenningPort self.__networkManager = NetworkManager(certificatesDirectory) self.__repositoryPacketHandler = ImageRepositoryPacketHandler(self.__networkManager) self.__commandsCallback = CommandsCallback(self.__networkManager, self.__repositoryPacketHandler, commandsListenningPort, self.__dbConnector, self.__retrieveQueue, self.__storeQueue, self.__diskImagesDirectory) self.__networkManager.startNetworkService() self.__networkManager.listenIn(commandsListenningPort, self.__commandsCallback, useSSL) dataCallback = FTPServerCallback(self.__slotCounter, self.__dbConnector) self.__ftpUsername = ftpUsername self.__ftpPassword = ChildProcessManager.runCommandInForeground("openssl rand -base64 {0}".format(ftpPasswordLength), Exception) self.__ftpServer = ConfigurableFTPServer("Image repository FTP Server") self.__ftpServer.startListenning(networkInterface, ftpListenningPort, maxConnections, maxConnectionsPerIP, dataCallback, downloadBandwidthRatio, uploadBandwidthRatio) self.__ftpServer.addUser(self.__ftpUsername, self.__ftpPassword, self.__diskImagesDirectory, "eramw") except Exception as e: print "Error: " + e.message self.__finish = True def stopListenning(self): """ Stops the FTP server and closes the control connection. Args: None Returns: Nothing """ if (self.__ftpServer != None) : try : self.__ftpServer.stopListenning() except Exception : pass if (self.__networkManager != None) : self.__networkManager.stopNetworkService() def initTransfers(self): """ Initializes the upload and download transfers Args: None Returns: Nothing """ store = False while not (self.__finish or self.__commandsCallback.haltReceived()): if (self.__slotCounter.read() == self.__maxConnections) : # No free slots => sleep sleep(0.1) else : # There are slots => enable uploads and downloads in parallel self.__slotCounter.decrement() if (self.__retrieveQueue.isEmpty() and self.__storeQueue.isEmpty()) : sleep(0.1) continue if (not self.__retrieveQueue.isEmpty() and self.__storeQueue.isEmpty()) : queue = self.__retrieveQueue store = False elif (self.__retrieveQueue.isEmpty() and not self.__storeQueue.isEmpty()) : queue = self.__storeQueue store = True else : if (store) : queue = self.__retrieveQueue store = False else : queue = self.__storeQueue store = True (imageID, clientIP, clientPort) = queue.pop(0) imageData = self.__dbConnector.getImageData(imageID) if (imageData == None) : if (store) : packet_type = PACKET_T.STOR_ERROR else : packet_type = PACKET_T.RETR_ERROR p = self.__repositoryPacketHandler.createErrorPacket(packet_type, ERROR_DESC_T.IR_IMAGE_DELETED) self.__networkManager.sendPacket('', self.__commandsListenningPort, p, clientIP, clientPort) else : compressedFilePath = imageData["compressedFilePath"] if (not "undefined" in compressedFilePath) : serverDirectory = path.relpath(path.dirname(compressedFilePath), self.__diskImagesDirectory) compressedFileName = path.basename(compressedFilePath) else : serverDirectory = str(imageID) compressedFileName = "" serverDirectoryPath = path.join(self.__diskImagesDirectory, serverDirectory) if (path.exists(serverDirectoryPath)) : # The directory exists, and can store shit => clean it up! ChildProcessManager.runCommandInForeground("rm -rf " + serverDirectoryPath, Exception) mkdir(serverDirectoryPath) if (store) : packet_type = PACKET_T.STOR_START else : packet_type = PACKET_T.RETR_START p = self.__repositoryPacketHandler.createTransferEnabledPacket(packet_type, imageID, self.__FTPListenningPort, self.__ftpUsername, self.__ftpPassword, serverDirectory, compressedFileName) self.__networkManager.sendPacket('', self.__commandsListenningPort, p, clientIP, clientPort)