class IncomingDataThread(QueueProcessingThread):
"""
This threads will process the incoming network packets.
"""
def __init__(self, transferQueue, callbackObject):
"""
Initializes the thread's state
Args:
transferQueue: The incoming packages transferQueue
callbackObject: The callback object that will process
all the received packets.
"""
QueueProcessingThread.__init__(self, "Incoming data processing thread", transferQueue)
self.__callbackObject = callbackObject
self.__referenceCounter = MultithreadingCounter()
def start(self):
"""
Starts the thread
Args:
None
Returns:
Nothing
"""
self.__referenceCounter.increment()
if self.__referenceCounter.read() == 1:
QueueProcessingThread.start(self)
def stop(self, join):
"""
Asks this thread to terminate.
Args:
join: When True, the calling thread will wait the incoming data thread
to terminate. When False, the calling thread will only ask this thread to terminate.
Returns:
Nothing
"""
self.__referenceCounter.decrement()
if self.__referenceCounter.read() == 0:
QueueProcessingThread.stop(self)
if join:
self.join()
def run(self):
QueueProcessingThread.run(self)
def processElement(self, e):
"""
Processes a received packet
Args:
e: The packet to process
Returns:
Nothing
"""
self.__callbackObject.processPacket(e)
class IncomingDataThread(QueueProcessingThread):
"""
This threads will process the incoming network packets.
"""
def __init__(self, transferQueue, callbackObject):
"""
Initializes the thread's state
Args:
transferQueue: The incoming packages transferQueue
callbackObject: The callback object that will process
all the received packets.
"""
QueueProcessingThread.__init__(self, "Incoming data processing thread",
transferQueue)
self.__callbackObject = callbackObject
self.__referenceCounter = MultithreadingCounter()
def start(self):
"""
Starts the thread
Args:
None
Returns:
Nothing
"""
self.__referenceCounter.increment()
if (self.__referenceCounter.read() == 1):
QueueProcessingThread.start(self)
def stop(self, join):
"""
Asks this thread to terminate.
Args:
join: When True, the calling thread will wait the incoming data thread
to terminate. When False, the calling thread will only ask this thread to terminate.
Returns:
Nothing
"""
self.__referenceCounter.decrement()
if self.__referenceCounter.read() == 0:
QueueProcessingThread.stop(self)
if join:
self.join()
def run(self):
QueueProcessingThread.run(self)
def processElement(self, e):
"""
Processes a received packet
Args:
e: The packet to process
Returns:
Nothing
"""
self.__callbackObject.processPacket(e)
class ImageRepositoryReactor(object):
"""
Image repository packet reactor
"""
def __init__(self, diskImagesDirectory):
"""
Initializes the reactor's state
Args:
diskImagesDirectory: the FTP server's root directory
"""
self.__diskImagesDirectory = diskImagesDirectory
self.__slotCounter = MultithreadingCounter()
self.__retrieveQueue = GenericThreadSafeList()
self.__storeQueue = GenericThreadSafeList()
self.__finish = False
self.__networkManager = None
self.__ftpServer = None
def connectToDatabase(self, repositoryDBName, repositoryDBUser,
repositoryDBPassword):
"""
Establishes the connection with the image repository's database.
Args:
repositoryDBName: a database name
repositoryDBUser: an user name
repositoryDBPassword: a password
Returns:
Nothing
"""
self.__dbConnector = ImageRepositoryDBConnector(
repositoryDBUser, repositoryDBPassword, repositoryDBName)
def startListenning(self, networkInterface, useSSL, certificatesDirectory,
commandsListenningPort, ftpListenningPort,
maxConnections, maxConnectionsPerIP,
uploadBandwidthRatio, downloadBandwidthRatio,
ftpUsername, ftpPasswordLength):
"""
Boots up the FTP server and creates the control connection.
Args:
networkInterface: the network interface that will be used by the FTP server
useSSL: indicates if SSL encryption must be used in the control connection or not
certificatesDirectory: the directory where the files server.crt and server.key are
commandsListenningPort: the control connection's port
ftpListenningPort: the FTP server listenning port
maxConnections: maximum FTP connections
maxConnectionsPerIP: maximum FTP connections per IP address
uploadBandwidthRatio: maximum download bandwidth fraction
downloadBandwidthRatio: maximum upload bandwidth fraction
ftpUsername: the FTP user that the virtual machine servers will use
ftpPasswordLength: the random FTP password length
Returns:
Nothing
@attention: The FTP password will be randomly generated at every boot.
"""
try:
self.__maxConnections = maxConnections
self.__commandsListenningPort = commandsListenningPort
self.__FTPListenningPort = ftpListenningPort
self.__networkManager = NetworkManager(certificatesDirectory)
self.__repositoryPacketHandler = ImageRepositoryPacketHandler(
self.__networkManager)
self.__commandsCallback = CommandsCallback(
self.__networkManager, self.__repositoryPacketHandler,
commandsListenningPort, self.__dbConnector,
self.__retrieveQueue, self.__storeQueue,
self.__diskImagesDirectory)
self.__networkManager.startNetworkService()
self.__networkManager.listenIn(commandsListenningPort,
self.__commandsCallback, useSSL)
dataCallback = FTPServerCallback(self.__slotCounter,
self.__dbConnector)
self.__ftpUsername = ftpUsername
self.__ftpPassword = ChildProcessManager.runCommandInForeground(
"openssl rand -base64 {0}".format(ftpPasswordLength),
Exception)
self.__ftpServer = ConfigurableFTPServer(
"Image repository FTP Server")
self.__ftpServer.startListenning(networkInterface,
ftpListenningPort, maxConnections,
maxConnectionsPerIP, dataCallback,
downloadBandwidthRatio,
uploadBandwidthRatio)
self.__ftpServer.addUser(self.__ftpUsername, self.__ftpPassword,
self.__diskImagesDirectory, "eramw")
except Exception as e:
print "Error: " + e.message
self.__finish = True
def stopListenning(self):
"""
Stops the FTP server and closes the control connection.
Args:
None
Returns:
Nothing
"""
if (self.__ftpServer != None):
try:
self.__ftpServer.stopListenning()
except Exception:
pass
if (self.__networkManager != None):
self.__networkManager.stopNetworkService()
def initTransfers(self):
"""
Initializes the upload and download transfers
Args:
None
Returns:
Nothing
"""
store = False
while not (self.__finish or self.__commandsCallback.haltReceived()):
if (self.__slotCounter.read() == self.__maxConnections):
# No free slots => sleep
sleep(0.1)
else:
# There are slots => enable uploads and downloads in parallel
self.__slotCounter.decrement()
if (self.__retrieveQueue.isEmpty()
and self.__storeQueue.isEmpty()):
sleep(0.1)
continue
if (not self.__retrieveQueue.isEmpty()
and self.__storeQueue.isEmpty()):
queue = self.__retrieveQueue
store = False
elif (self.__retrieveQueue.isEmpty()
and not self.__storeQueue.isEmpty()):
queue = self.__storeQueue
store = True
else:
if (store):
queue = self.__retrieveQueue
store = False
else:
queue = self.__storeQueue
store = True
(imageID, clientIP, clientPort) = queue.pop(0)
imageData = self.__dbConnector.getImageData(imageID)
if (imageData == None):
if (store):
packet_type = PACKET_T.STOR_ERROR
else:
packet_type = PACKET_T.RETR_ERROR
p = self.__repositoryPacketHandler.createErrorPacket(
packet_type, ERROR_DESC_T.IR_IMAGE_DELETED)
self.__networkManager.sendPacket(
'', self.__commandsListenningPort, p, clientIP,
clientPort)
else:
compressedFilePath = imageData["compressedFilePath"]
if (not "undefined" in compressedFilePath):
serverDirectory = path.relpath(
path.dirname(compressedFilePath),
self.__diskImagesDirectory)
compressedFileName = path.basename(compressedFilePath)
else:
serverDirectory = str(imageID)
compressedFileName = ""
serverDirectoryPath = path.join(
self.__diskImagesDirectory, serverDirectory)
if (path.exists(serverDirectoryPath)):
# The directory exists, and can store shit => clean it up!
ChildProcessManager.runCommandInForeground(
"rm -rf " + serverDirectoryPath, Exception)
mkdir(serverDirectoryPath)
if (store):
packet_type = PACKET_T.STOR_START
else:
packet_type = PACKET_T.RETR_START
p = self.__repositoryPacketHandler.createTransferEnabledPacket(
packet_type, imageID, self.__FTPListenningPort,
self.__ftpUsername, self.__ftpPassword,
serverDirectory, compressedFileName)
self.__networkManager.sendPacket(
'', self.__commandsListenningPort, p, clientIP,
clientPort)
class Connection(object):
"""
This is the base class for all network connections.
"""
def __init__(self, useSSL, certificatesDirectory, port, transferQueue,
incomingDataThread, callbackObject):
"""
Initializes the connection.
Args:
useSSL: if True, all the traffic will be protectd by SSLv4. If false,
certificatesDirectory: the directory where the certificates are stored
port: the port assigned to the connection.
transferQueue: the incoming data transferQueue assigned to the connection
incomingDataThread: the incoming data thread assigned to the connection
callbackObject: the callback object assigned to the connection
"""
self._status = ConnectionStatus(CONNECTION_STATUS.OPENING)
self._useSSL = useSSL
self._certificatesDirectory = certificatesDirectory
self._port = port
self._factory = None
self._queue = transferQueue
self._incomingDataThread = incomingDataThread
self._callback = callbackObject
self._packagesToSend = MultithreadingCounter()
self._deferred = None
self._unexpectedlyClosed = False
self._error = None
@staticmethod
def _prettyPrintTwistedError(uglyTwistedError):
"""
Puts an ugly twisted error in a human readable form
Args:
uglyTwistedError: the twisted error to pretty print
Returns:
A string containing the error message
"""
errorStr = str(uglyTwistedError)
i = 0
while i < 3:
(_head, _sep, errorMessage) = errorStr.partition(":")
errorStr = errorMessage
i += 1
(head, _sep, _errorMessage) = errorStr.partition(":")
return head
def establish(self, timeout=None):
"""
Tries to establish the network connection.
Args:
timeout: the timeout in seconds. This argument will only be used with client
connections.
Returns:
True if the connection could be established. Otherwise, it will return false.
"""
self._factory = CygnusCloudProtocolFactory(self._queue)
def getHost(self):
"""
Returns the server's hostname or IPv4 address
Args:
None
Returns:
The server's IP address.
"""
raise NotImplementedError
def getPort(self):
"""
Returns the port assigned to this connection.
Args:
None
Returns:
The port assigned to this connection.
"""
return self._port
def getQueue(self):
"""
Returns the incoming data queue assigned to this connection.
Args:
None
Returns:
The incoming data queue assigned to this connection
"""
return self._queue
def getThread(self):
"""
Returns the incoming data processing thread assigned to this connection
Args:
None
Returns:
The incoming data processing thread assigned to this connection
"""
return self._incomingDataThread
def getCallback(self):
"""
Returns the callback object assigned to this connection
Args:
None
Returns:
The callback object assigned to this connection.
"""
return self._callback
def getStatus(self):
"""
Returns the connection status
Args:
None
Returns:
The connection status
"""
return self._status.get()
def sendPacket(self, p, client_IP=None, client_port=None):
"""
Sends a packet though this connection. If the connection is closed, the packet will be discarded.
Args:
p: the packet to send
client_IP: the client's IPv4 address
client_port: the client's port
@attention: The two last parameters will only be used with server connections.
Returns:
Nothing
"""
if (self._status.get() == CONNECTION_STATUS.READY
or self._status.get() == CONNECTION_STATUS.CLOSING):
self._factory.sendPacket(p, client_IP, client_port)
self._packagesToSend.decrement()
return True
def registerPacket(self):
"""
Registers a packet to be sent through this connection.
Args:
None
Returns:
Nothing
"""
self._packagesToSend.increment()
def isReady(self):
"""
Checks if the connection is ready to send data or not
Args:
None
Returns:
True if the connection is ready to send data, and false otherwise.
"""
return self._status.get() == CONNECTION_STATUS.READY
def isServerConnection(self):
"""
Determines if this is a server connection or not.
Args:
None
Returns:
True if this connection is a server one, and false if it isn't.
"""
return self.__isServerConnection
def refresh(self):
"""
Updates the connection's status
Args:
None
Returns:
Nothing
"""
raise NotImplementedError
def isInErrorState(self):
"""
Checks if the connection is in an error state.
Args:
None
Returns: True if the connection is in an error state, and False if it isn't.
"""
return self._status.get() == CONNECTION_STATUS.ERROR
def wasUnexpectedlyClosed(self):
"""
Checks if the connection was unexpectedly closed. The connection may be closed unexpectedly
in two different ways:
1) All the clients disconnect from a server.
2) A server disconnects from all its clients.
Args: None
Returns:
True if the connection was unexpectedly closed, and False if it wasn't.
"""
return self._unexpectedlyClosed
def getError(self):
"""
Returns the error message stored in this connection.
"""
return self._error
def _setError(self, value):
"""
Modifies the error message stored in this connection.
"""
self._status.set(CONNECTION_STATUS.ERROR)
self._error = value
def close(self):
"""
Asks this connection to close
Args:
None
Returns:
Nothing
"""
self._status.set(CONNECTION_STATUS.CLOSING)
def _freeTwistedResources(self):
"""
Destroys the twisted-related connection resources.
Args:
None
Returns:
Nothing
"""
self._factory.disconnect()
def _close(self):
"""
Closes this connection.
Args:
None
Returns:
Nothing
"""
# Free the connection resources
self._freeTwistedResources()
self._incomingDataThread.stop(True)
self._status.set(CONNECTION_STATUS.CLOSED)
class Connection(object):
"""
This is the base class for all network connections.
"""
def __init__(self, useSSL, certificatesDirectory, port, transferQueue, incomingDataThread, callbackObject):
"""
Initializes the connection.
Args:
useSSL: if True, all the traffic will be protectd by SSLv4. If false,
certificatesDirectory: the directory where the certificates are stored
port: the port assigned to the connection.
transferQueue: the incoming data transferQueue assigned to the connection
incomingDataThread: the incoming data thread assigned to the connection
callbackObject: the callback object assigned to the connection
"""
self._status = ConnectionStatus(CONNECTION_STATUS.OPENING)
self._useSSL = useSSL
self._certificatesDirectory = certificatesDirectory
self._port = port
self._factory = None
self._queue = transferQueue
self._incomingDataThread = incomingDataThread
self._callback = callbackObject
self._packagesToSend = MultithreadingCounter()
self._deferred = None
self._unexpectedlyClosed = False
self._error = None
@staticmethod
def _prettyPrintTwistedError(uglyTwistedError):
"""
Puts an ugly twisted error in a human readable form
Args:
uglyTwistedError: the twisted error to pretty print
Returns:
A string containing the error message
"""
errorStr = str(uglyTwistedError)
i = 0
while i < 3 :
(_head, _sep, errorMessage) = errorStr.partition(":")
errorStr = errorMessage
i += 1
(head, _sep, _errorMessage) = errorStr.partition(":")
return head
def establish(self, timeout=None):
"""
Tries to establish the network connection.
Args:
timeout: the timeout in seconds. This argument will only be used with client
connections.
Returns:
True if the connection could be established. Otherwise, it will return false.
"""
self._factory = CygnusCloudProtocolFactory(self._queue)
def getHost(self):
"""
Returns the server's hostname or IPv4 address
Args:
None
Returns:
The server's IP address.
"""
raise NotImplementedError
def getPort(self):
"""
Returns the port assigned to this connection.
Args:
None
Returns:
The port assigned to this connection.
"""
return self._port
def getQueue(self):
"""
Returns the incoming data queue assigned to this connection.
Args:
None
Returns:
The incoming data queue assigned to this connection
"""
return self._queue
def getThread(self):
"""
Returns the incoming data processing thread assigned to this connection
Args:
None
Returns:
The incoming data processing thread assigned to this connection
"""
return self._incomingDataThread
def getCallback(self):
"""
Returns the callback object assigned to this connection
Args:
None
Returns:
The callback object assigned to this connection.
"""
return self._callback
def getStatus(self):
"""
Returns the connection status
Args:
None
Returns:
The connection status
"""
return self._status.get()
def sendPacket(self, p, client_IP = None, client_port = None):
"""
Sends a packet though this connection. If the connection is closed, the packet will be discarded.
Args:
p: the packet to send
client_IP: the client's IPv4 address
client_port: the client's port
@attention: The two last parameters will only be used with server connections.
Returns:
Nothing
"""
if (self._status.get() == CONNECTION_STATUS.READY or self._status.get() == CONNECTION_STATUS.CLOSING) :
self._factory.sendPacket(p, client_IP, client_port)
self._packagesToSend.decrement()
return True
def registerPacket(self):
"""
Registers a packet to be sent through this connection.
Args:
None
Returns:
Nothing
"""
self._packagesToSend.increment()
def isReady(self):
"""
Checks if the connection is ready to send data or not
Args:
None
Returns:
True if the connection is ready to send data, and false otherwise.
"""
return self._status.get() == CONNECTION_STATUS.READY
def isServerConnection(self):
"""
Determines if this is a server connection or not.
Args:
None
Returns:
True if this connection is a server one, and false if it isn't.
"""
return self.__isServerConnection
def refresh(self):
"""
Updates the connection's status
Args:
None
Returns:
Nothing
"""
raise NotImplementedError
def isInErrorState(self):
"""
Checks if the connection is in an error state.
Args:
None
Returns: True if the connection is in an error state, and False if it isn't.
"""
return self._status.get() == CONNECTION_STATUS.ERROR
def wasUnexpectedlyClosed(self):
"""
Checks if the connection was unexpectedly closed. The connection may be closed unexpectedly
in two different ways:
1) All the clients disconnect from a server.
2) A server disconnects from all its clients.
Args: None
Returns:
True if the connection was unexpectedly closed, and False if it wasn't.
"""
return self._unexpectedlyClosed
def getError(self):
"""
Returns the error message stored in this connection.
"""
return self._error
def _setError(self, value):
"""
Modifies the error message stored in this connection.
"""
self._status.set(CONNECTION_STATUS.ERROR)
self._error = value
def close(self):
"""
Asks this connection to close
Args:
None
Returns:
Nothing
"""
self._status.set(CONNECTION_STATUS.CLOSING)
def _freeTwistedResources(self):
"""
Destroys the twisted-related connection resources.
Args:
None
Returns:
Nothing
"""
self._factory.disconnect()
def _close(self):
"""
Closes this connection.
Args:
None
Returns:
Nothing
"""
# Free the connection resources
self._freeTwistedResources()
self._incomingDataThread.stop(True)
self._status.set(CONNECTION_STATUS.CLOSED)
class ImageRepositoryReactor(object):
"""
Image repository packet reactor
"""
def __init__(self, diskImagesDirectory):
"""
Initializes the reactor's state
Args:
diskImagesDirectory: the FTP server's root directory
"""
self.__diskImagesDirectory = diskImagesDirectory
self.__slotCounter = MultithreadingCounter()
self.__retrieveQueue = GenericThreadSafeList()
self.__storeQueue = GenericThreadSafeList()
self.__finish = False
self.__networkManager = None
self.__ftpServer = None
def connectToDatabase(self, repositoryDBName, repositoryDBUser, repositoryDBPassword):
"""
Establishes the connection with the image repository's database.
Args:
repositoryDBName: a database name
repositoryDBUser: an user name
repositoryDBPassword: a password
Returns:
Nothing
"""
self.__dbConnector = ImageRepositoryDBConnector(repositoryDBUser, repositoryDBPassword, repositoryDBName)
def startListenning(self, networkInterface, useSSL, certificatesDirectory, commandsListenningPort, ftpListenningPort, maxConnections,
maxConnectionsPerIP, uploadBandwidthRatio, downloadBandwidthRatio, ftpUsername, ftpPasswordLength):
"""
Boots up the FTP server and creates the control connection.
Args:
networkInterface: the network interface that will be used by the FTP server
useSSL: indicates if SSL encryption must be used in the control connection or not
certificatesDirectory: the directory where the files server.crt and server.key are
commandsListenningPort: the control connection's port
ftpListenningPort: the FTP server listenning port
maxConnections: maximum FTP connections
maxConnectionsPerIP: maximum FTP connections per IP address
uploadBandwidthRatio: maximum download bandwidth fraction
downloadBandwidthRatio: maximum upload bandwidth fraction
ftpUsername: the FTP user that the virtual machine servers will use
ftpPasswordLength: the random FTP password length
Returns:
Nothing
@attention: The FTP password will be randomly generated at every boot.
"""
try :
self.__maxConnections = maxConnections
self.__commandsListenningPort = commandsListenningPort
self.__FTPListenningPort = ftpListenningPort
self.__networkManager = NetworkManager(certificatesDirectory)
self.__repositoryPacketHandler = ImageRepositoryPacketHandler(self.__networkManager)
self.__commandsCallback = CommandsCallback(self.__networkManager, self.__repositoryPacketHandler, commandsListenningPort, self.__dbConnector,
self.__retrieveQueue, self.__storeQueue, self.__diskImagesDirectory)
self.__networkManager.startNetworkService()
self.__networkManager.listenIn(commandsListenningPort, self.__commandsCallback, useSSL)
dataCallback = FTPServerCallback(self.__slotCounter, self.__dbConnector)
self.__ftpUsername = ftpUsername
self.__ftpPassword = ChildProcessManager.runCommandInForeground("openssl rand -base64 {0}".format(ftpPasswordLength), Exception)
self.__ftpServer = ConfigurableFTPServer("Image repository FTP Server")
self.__ftpServer.startListenning(networkInterface, ftpListenningPort, maxConnections, maxConnectionsPerIP,
dataCallback, downloadBandwidthRatio, uploadBandwidthRatio)
self.__ftpServer.addUser(self.__ftpUsername, self.__ftpPassword, self.__diskImagesDirectory, "eramw")
except Exception as e:
print "Error: " + e.message
self.__finish = True
def stopListenning(self):
"""
Stops the FTP server and closes the control connection.
Args:
None
Returns:
Nothing
"""
if (self.__ftpServer != None) :
try :
self.__ftpServer.stopListenning()
except Exception :
pass
if (self.__networkManager != None) :
self.__networkManager.stopNetworkService()
def initTransfers(self):
"""
Initializes the upload and download transfers
Args:
None
Returns:
Nothing
"""
store = False
while not (self.__finish or self.__commandsCallback.haltReceived()):
if (self.__slotCounter.read() == self.__maxConnections) :
# No free slots => sleep
sleep(0.1)
else :
# There are slots => enable uploads and downloads in parallel
self.__slotCounter.decrement()
if (self.__retrieveQueue.isEmpty() and self.__storeQueue.isEmpty()) :
sleep(0.1)
continue
if (not self.__retrieveQueue.isEmpty() and self.__storeQueue.isEmpty()) :
queue = self.__retrieveQueue
store = False
elif (self.__retrieveQueue.isEmpty() and not self.__storeQueue.isEmpty()) :
queue = self.__storeQueue
store = True
else :
if (store) :
queue = self.__retrieveQueue
store = False
else :
queue = self.__storeQueue
store = True
(imageID, clientIP, clientPort) = queue.pop(0)
imageData = self.__dbConnector.getImageData(imageID)
if (imageData == None) :
if (store) :
packet_type = PACKET_T.STOR_ERROR
else :
packet_type = PACKET_T.RETR_ERROR
p = self.__repositoryPacketHandler.createErrorPacket(packet_type, ERROR_DESC_T.IR_IMAGE_DELETED)
self.__networkManager.sendPacket('', self.__commandsListenningPort, p, clientIP, clientPort)
else :
compressedFilePath = imageData["compressedFilePath"]
if (not "undefined" in compressedFilePath) :
serverDirectory = path.relpath(path.dirname(compressedFilePath), self.__diskImagesDirectory)
compressedFileName = path.basename(compressedFilePath)
else :
serverDirectory = str(imageID)
compressedFileName = ""
serverDirectoryPath = path.join(self.__diskImagesDirectory, serverDirectory)
if (path.exists(serverDirectoryPath)) :
# The directory exists, and can store shit => clean it up!
ChildProcessManager.runCommandInForeground("rm -rf " + serverDirectoryPath, Exception)
mkdir(serverDirectoryPath)
if (store) :
packet_type = PACKET_T.STOR_START
else :
packet_type = PACKET_T.RETR_START
p = self.__repositoryPacketHandler.createTransferEnabledPacket(packet_type, imageID, self.__FTPListenningPort,
self.__ftpUsername, self.__ftpPassword, serverDirectory, compressedFileName)
self.__networkManager.sendPacket('', self.__commandsListenningPort, p, clientIP, clientPort)