class VMServerReactor(NetworkCallback):
"""
Virtual machine server packet reactor
"""
def __init__(self, configurationFileParser):
"""
Initializes the reactor's state, establishes the connection with the database
and creates the control connection
Args:
configurationFileParser: the virtual machine server's configuration file parser
"""
self.__finished = False
self.__emergencyStop = False
self.__fileTransferThread = None
self.__compressionThread = None
self.__networkManager = None
self.__parser = configurationFileParser
self.__domainHandler = None
self.__domainTimeout = 0
try :
self.__connectToDatabases("VMServerDB", self.__parser.getConfigurationParameter("databaseUserName"),
self.__parser.getConfigurationParameter("databasePassword"))
self.__startListenning()
except Exception as e:
print e.message
self.__emergencyStop = True
self.__finished = True
def __connectToDatabases(self, databaseName, user, password) :
"""
Establishes the connection with the database
Args:
databaseName: a database name
user: a SQL username
password: the user's password
Returns:
Nothing
"""
self.__commandsDBConnector = VMServerDBConnector(user, password, databaseName)
def __startListenning(self):
"""
Creates the control connection
Args:
None
Returns:
Nothing
"""
networkInterface = self.__parser.getConfigurationParameter("vncNetworkInterface")
listenningPort = self.__parser.getConfigurationParameter("listenningPort")
try :
self.__vncServerIP = get_ip_address(networkInterface)
except Exception :
raise Exception("Error: the network interface '{0}' is not ready. Exiting now".format(networkInterface))
self.__ftpTimeout = self.__parser.getConfigurationParameter("FTPTimeout")
self.__listenningPort = listenningPort
self.__networkManager = NetworkManager(self.__parser.getConfigurationParameter("certificatePath"))
self.__networkManager.startNetworkService()
self.__useSSL = self.__parser.getConfigurationParameter("useSSL")
self.__packetManager = VMServerPacketHandler(self.__networkManager)
self.__connectToDatabases("VMServerDB", self.__parser.getConfigurationParameter("databaseUserName"), self.__parser.getConfigurationParameter("databasePassword"))
self.__domainHandler = DomainHandler(self.__commandsDBConnector, self.__vncServerIP, self.__networkManager, self.__packetManager, self.__listenningPort,
self.__parser.getConfigurationParameter("useQEMUWebsockets"),
self.__parser.getConfigurationParameter("websockifyPath"),
self.__parser.getConfigurationParameter("configFilePath"),
self.__parser.getConfigurationParameter("sourceImagePath"), self.__parser.getConfigurationParameter("executionImagePath"),
self.__parser.getConfigurationParameter("passwordLength"))
self.__domainHandler.connectToLibvirt(self.__parser.getConfigurationParameter("vncNetworkInterface"),
self.__parser.getConfigurationParameter("vnName"), self.__parser.getConfigurationParameter("gatewayIP"),
self.__parser.getConfigurationParameter("netMask"), self.__parser.getConfigurationParameter("dhcpStartIP"),
self.__parser.getConfigurationParameter("dhcpEndIP"), self.__parser.getConfigurationParameter("createVirtualNetworkAsRoot"))
self.__domainHandler.doInitialCleanup()
self.__deleteTemporaryZipFiles()
self.__fileTransferThread = FileTransferThread(self.__networkManager, self.__listenningPort, self.__packetManager,
self.__parser.getConfigurationParameter("TransferDirectory"),
self.__parser.getConfigurationParameter("FTPTimeout"),
self.__parser.getConfigurationParameter("MaxTransferAttempts"), self.__commandsDBConnector, self.__useSSL)
self.__compressionThread = CompressionThread(self.__parser.getConfigurationParameter("TransferDirectory"), self.__parser.getConfigurationParameter("sourceImagePath"),
self.__parser.getConfigurationParameter("configFilePath"),
self.__commandsDBConnector, self.__domainHandler, self.__networkManager, self.__listenningPort, self.__packetManager)
self.__fileTransferThread.start()
self.__compressionThread.start()
self.__networkManager.listenIn(self.__listenningPort, self, self.__useSSL)
def __deleteTemporaryZipFiles(self):
"""
Deletes the temporary zip files
Args:
None
Returns:
Nothing
"""
transfer_dir_path = self.__parser.getConfigurationParameter("TransferDirectory")
for filePath in os.listdir(transfer_dir_path) :
fileName = os.path.splitext(filePath)[0]
if re.match("[^0-9]", fileName):
# The non-temporary zip files only have numbers on their names
os.remove(os.path.join(transfer_dir_path, filePath))
def shutdown(self):
"""
Shuts down the virtual machine server
Args:
None
Returns:
Nothing
"""
if (self.__emergencyStop) :
self.__domainTimeout = 0
self.__domainHandler.shutdown(self.__domainTimeout)
if (self.__fileTransferThread != None) :
self.__fileTransferThread.stop()
self.__fileTransferThread.join()
if (self.__compressionThread != None) :
self.__compressionThread.stop()
self.__compressionThread.join()
if (self.__networkManager != None) :
self.__networkManager.stopNetworkService() # Dejar de atender peticiones inmediatamente
sys.exit()
def processPacket(self, packet):
"""
Processes a packet sent from the cluster server
Args:
packet: the packet to process
Returns:
Nothing
"""
data = self.__packetManager.readPacket(packet)
if (data["packet_type"] == VM_SERVER_PACKET_T.CREATE_DOMAIN) :
self.__domainHandler.createDomain(data["MachineID"], data["UserID"], data["CommandID"])
elif (data["packet_type"] == VM_SERVER_PACKET_T.SERVER_STATUS_REQUEST) :
self.__sendStatusData()
elif (data["packet_type"] == VM_SERVER_PACKET_T.USER_FRIENDLY_SHUTDOWN) :
self.__domainTimeout = data["Timeout"]
self.__finished = True
elif (data["packet_type"] == VM_SERVER_PACKET_T.HALT) :
self.__domainTimeout = 0
self.__finished = True
elif (data['packet_type'] == VM_SERVER_PACKET_T.QUERY_ACTIVE_VM_DATA) :
self.__sendDomainsVNCConnectionData()
elif (data['packet_type'] == VM_SERVER_PACKET_T.DESTROY_DOMAIN) :
self.__domainHandler.destroyDomain(data["DomainID"])
elif (data['packet_type'] == VM_SERVER_PACKET_T.REBOOT_DOMAIN) :
self.__domainHandler.rebootDomain(data["DomainID"])
elif (data['packet_type'] == VM_SERVER_PACKET_T.QUERY_ACTIVE_DOMAIN_UIDS) :
self.__sendActiveDomainUIDs()
elif (data['packet_type'] == VM_SERVER_PACKET_T.IMAGE_EDITION) :
self.__processImageEditionPacket(data)
elif (data['packet_type'] == VM_SERVER_PACKET_T.DEPLOY_IMAGE) :
self.__processDeployImagePacket(data)
elif (data['packet_type'] == VM_SERVER_PACKET_T.DELETE_IMAGE) :
self.__processDeleteImagePacket(data)
def __processDeployImagePacket(self, data):
"""
Processes an image deployment packet
Args:
data: a dictionary containing the packet to process' data
Returns:
Nothing
"""
data.pop("packet_type")
data["Transfer_Type"] = TRANSFER_T.DEPLOY_IMAGE
self.__commandsDBConnector.addToTransferQueue(data)
def __processDeleteImagePacket(self, data):
isBootable = self.__commandsDBConnector.getBootableFlag(data["ImageID"])
if(isBootable):
osImagePath = os.path.join(self.__parser.getConfigurationParameter("sourceImagePath")
,self.__commandsDBConnector.getOSImagePath(data["ImageID"]))
definitionFilePath = os.path.join(self.__parser.getConfigurationParameter("configFilePath")
,self.__commandsDBConnector.getDefinitionFilePath(data["ImageID"]))
try :
self.__commandsDBConnector.deleteImage(data["ImageID"])
ChildProcessManager.runCommandInForeground("rm -rf " + os.path.dirname(osImagePath), VMServerException)
ChildProcessManager.runCommandInForeground("rm -rf " + os.path.dirname(definitionFilePath), VMServerException)
p = self.__packetManager.createConfirmationPacket(VM_SERVER_PACKET_T.IMAGE_DELETED, data["ImageID"], data["CommandID"])
except Exception:
p = self.__packetManager.createErrorPacket(VM_SERVER_PACKET_T.IMAGE_DELETION_ERROR, ERROR_DESC_T.VM_SRVR_INTERNAL_ERROR,
data["CommandID"])
else:
if (isBootable != None) :
errorCode = ERROR_DESC_T.VMSRVR_LOCKED_IMAGE
else :
errorCode = ERROR_DESC_T.VMSRVR_UNKNOWN_IMAGE
p = self.__packetManager.createErrorPacket(VM_SERVER_PACKET_T.IMAGE_DELETION_ERROR, errorCode,
data["CommandID"])
self.__networkManager.sendPacket('', self.__listenningPort, p)
def __processImageEditionPacket(self, data):
"""
Processes an image edition packet
Args:
data: a dictionary containing the packet to process' data
Returns:
Nothing
"""
data.pop("packet_type")
if (data["Modify"]) :
data["Transfer_Type"] = TRANSFER_T.EDIT_IMAGE
else :
data["Transfer_Type"] = TRANSFER_T.CREATE_IMAGE
data.pop("Modify")
self.__commandsDBConnector.addToTransferQueue(data)
def __sendDomainsVNCConnectionData(self):
'''
Sends the domains' VNC connection data to the cluster server
Args:
None
Returns:
Nothing
'''
vncConnectionData = self.__commandsDBConnector.getDomainsConnectionData()
segmentSize = 150
segmentCounter = 1
outgoingData = []
if (len(vncConnectionData) == 0):
segmentCounter = 0
segmentNumber = (len(vncConnectionData) / segmentSize)
if (len(vncConnectionData) % segmentSize != 0) :
segmentNumber += 1
sendLastSegment = True
else :
sendLastSegment = segmentNumber == 0
for connectionParameters in vncConnectionData :
outgoingData.append(connectionParameters)
if (len(outgoingData) >= segmentSize) :
# Flush
packet = self.__packetManager.createActiveVMsVNCDataPacket(self.__vncServerIP, segmentCounter, segmentNumber, outgoingData)
self.__networkManager.sendPacket('', self.__listenningPort, packet)
outgoingData = []
segmentCounter += 1
# Send the last segment
if (sendLastSegment) :
packet = self.__packetManager.createActiveVMsVNCDataPacket(self.__vncServerIP, segmentCounter, segmentNumber, outgoingData)
self.__networkManager.sendPacket('', self.__listenningPort, packet)
def __sendStatusData(self):
"""
Sends the server's status data to the cluster server
Args:
None
Returns:
Nothing
"""
info = self.__domainHandler.getLibvirtStatusData()
realCPUNumber = multiprocessing.cpu_count()
freeOutput = ChildProcessManager.runCommandInForeground("free -k", VMServerException)
# free's output contains the following lines and collumns:
# total used free shared buffers cached
# Mem: 4146708480 3939934208 206774272 0 224706560 1117671424
# -/+ buffers/cache: 2597556224 1549152256
# Swap: 2046816256 42455040 2004361216
#
# We must get the third line
availableMemory = int(freeOutput.split("\n")[1].split()[1])
freeMemory = int(freeOutput.split("\n")[2].split()[2])
freeStorageSpace, availableStorageSpace, freeTemporaryStorage, availableTemporaryStorage = self.__generateDiskStats()
packet = self.__packetManager.createVMServerStatusPacket(self.__vncServerIP, info["#domains"], freeMemory, availableMemory,
freeStorageSpace, availableStorageSpace,
freeTemporaryStorage, availableTemporaryStorage,
info["#vcpus"], realCPUNumber)
self.__networkManager.sendPacket('', self.__listenningPort, packet)
def __generateDiskStats(self):
"""
Generates the hard disk usage statistics
Args:
None
Returns:
the free and available storage and temporary space
"""
diskStats_storage = os.statvfs(self.__parser.getConfigurationParameter("sourceImagePath"))
diskStats_temporaryData = os.statvfs(self.__parser.getConfigurationParameter("executionImagePath"))
allocatedStorageSpace, allocatedTemporaryStorageSpace = self.__checkDiskImagesSpace()
freeStorageSpace = diskStats_storage.f_bfree * diskStats_storage.f_frsize / 1024 - allocatedStorageSpace
availableStorageSpace = diskStats_storage.f_blocks * diskStats_storage.f_frsize / 1024
freeTemporaryStorageSpace = diskStats_temporaryData.f_bfree * diskStats_temporaryData.f_frsize / 1024 - allocatedTemporaryStorageSpace
availableTemporaryStorageSpace = diskStats_temporaryData.f_blocks * diskStats_temporaryData.f_frsize / 1024
return freeStorageSpace, availableStorageSpace, freeTemporaryStorageSpace, availableTemporaryStorageSpace
def __checkDiskImagesSpace(self):
"""
Checks how much disk space must be allocated for the active virtual machines' disk images
Args:
None
Returns:
the storage and temporary storage space that must be allocated.
"""
activeDomainNames = self.__commandsDBConnector.getRegisteredDomainNames()
allocatedStorageSpace = 0
allocatedTemporaryStorageSpace = 0
for domainName in activeDomainNames:
imageID = self.__commandsDBConnector.getDomainImageID(domainName)
if (imageID == None) :
return 0, 0
dataImagePath = self.__commandsDBConnector.getDomainDataImagePath(domainName)
if (dataImagePath == None) :
return 0, 0
osImagePath = self.__commandsDBConnector.getDomainOSImagePath(domainName)
if (osImagePath == None):
return 0, 0
isEditedImage = self.__commandsDBConnector.getBootableFlag(imageID)
if (isEditedImage == None):
return 0, 0
dataSpace = self.__getAllocatedSpace(dataImagePath)
if (isEditedImage) :
osSpace = self.__getAllocatedSpace(osImagePath)
allocatedStorageSpace += osSpace + dataSpace
else :
allocatedTemporaryStorageSpace += dataSpace
return allocatedStorageSpace, allocatedTemporaryStorageSpace
def __getAllocatedSpace(self, imagePath):
"""
Returns the disk space that must be allocated for a disk image
Args:
imagePath: the disk image's path
Returns:
Nothing
"""
try :
output = ChildProcessManager.runCommandInForeground("qemu-img info " + imagePath, Exception)
lines = output.split("\n")
virtualSize = VMServerReactor.__extractImageSize(lines[2].split(":")[1].split("(")[0])
usedSpace = VMServerReactor.__extractImageSize(lines[3].split(":")[1])
return virtualSize - usedSpace
except Exception as e:
print e
return 0
@staticmethod
def __extractImageSize(string):
"""
Extracts an image size from a quemu-img command output.
Args:
string: the qemu-img command's image size
Returns:
the disk image size (in kilobytes)
"""
if ("G" in string) :
power = 2
elif ("M" in string):
power = 1
else:
power = 0
string = string.replace("G", "")
string = string.replace("M", "")
string = string.replace("K", "")
return int(ceil(float(string))) * 1024 ** power
def __sendActiveDomainUIDs(self):
"""
Sends domains' UUIDs to the cluster server
Args:
None
Returns:
Nothing
"""
activeDomainUIDs = self.__commandsDBConnector.getActiveDomainUIDs()
packet = self.__packetManager.createActiveDomainUIDsPacket(self.__vncServerIP, activeDomainUIDs)
self.__networkManager.sendPacket('', self.__listenningPort, packet)
def has_finished(self):
"""
Checks if the virtual machine server has been shut down or not
Args:
None
Returns:
Nothing
"""
return self.__finished
class ClusterServerReactor(WebPacketReactor, VMServerPacketReactor):
'''
These objects react to packages received from the website or from
a virtual machine server.
'''
def __init__(self, timeout):
"""
Initializes the reactor's state.
Args:
None
"""
self.__webCallback = WebCallback(self)
self.__finished = False
self.__timeout = timeout
def connectToDatabase(self, mysqlRootsPassword, dbName, dbUser, dbPassword, scriptPath):
"""
Establishes a connection with the cluster server database.
Args:
mysqlRootsPassword: MySQL root's password
dbName: the cluster server database's name
dbUser: the cluster server database's user name
dbPassword: the cluster server database's user password
scriptPath: the cluster server database's initialization script path
"""
configurator = DBConfigurator(mysqlRootsPassword)
configurator.runSQLScript(dbName, scriptPath)
configurator.addUser(dbUser, dbPassword, dbName, True)
self.__dbConnector = ClusterServerDatabaseConnector(dbUser, dbPassword, dbName)
self.__dbConnector.connect()
self.__dbConnector.resetVMServersStatus()
def startListenning(self, certificatePath, port):
"""
Starts the network service and creates a server connection.
Args:
certificatePath: the server.crt and server.key files path
port: the listenning port
Returns:
Nothing
"""
self.__loadBalancer = SimpleLoadBalancer(self.__dbConnector)
self.__networkManager = NetworkManager(certificatePath)
self.__webPort = port
self.__networkManager.startNetworkService()
self.__webPacketHandler = ClusterServerPacketHandler(self.__networkManager)
self.__vmServerPacketHandler = VMServerPacketHandler(self.__networkManager)
self.__networkManager.listenIn(port, self.__webCallback, True)
self.__vmServerCallback = VMServerCallback(self)
def processWebIncomingPacket(self, packet):
"""
Processes a packet received from the web server.
Args:
packet: the packet to process
Returns:
Nothing
"""
data = self.__webPacketHandler.readPacket(packet)
if (data["packet_type"] == WEB_PACKET_T.REGISTER_VM_SERVER) :
self.__registerVMServer(data)
elif (data["packet_type"] == WEB_PACKET_T.QUERY_VM_SERVERS_STATUS) :
self.__sendVMServerStatusData()
elif (data["packet_type"] == WEB_PACKET_T.UNREGISTER_OR_SHUTDOWN_VM_SERVER) :
self.__unregisterOrShutdownVMServer(data)
elif (data["packet_type"] == WEB_PACKET_T.BOOTUP_VM_SERVER) :
self.__bootUpVMServer(data)
elif (data["packet_type"] == WEB_PACKET_T.VM_BOOT_REQUEST):
self.__bootUpVM(data)
elif (data["packet_type"] == WEB_PACKET_T.HALT) :
self.__halt(data)
elif (data["packet_type"] == WEB_PACKET_T.QUERY_VM_DISTRIBUTION) :
self.__sendVMDistributionData()
elif (data["packet_type"] == WEB_PACKET_T.QUERY_ACTIVE_VM_DATA) :
self.__requestVNCConnectionData()
def __requestVNCConnectionData(self):
"""
Sends a VNC connection data request packet to all the active virtual machine servers
Args:
None
Returns:
Nothing
"""
# Create a VNC connection data packet
p = self.__vmServerPacketHandler.createVMServerDataRequestPacket(VMSRVR_PACKET_T.QUERY_ACTIVE_VM_DATA)
# Fetch the active virtual machine server's IP addresses and ports
connectionData = self.__dbConnector.getActiveVMServersConnectionData()
for cd in connectionData :
self.__networkManager.sendPacket(cd["ServerIP"], cd["ServerPort"], p)
def __halt(self, data):
"""
Shuts down the cluster (including the virtual machine servers).
Args:
haltVMServers: if True, all the active virtual machines will be destroyed and the virtual machine
servers will be shut down. If false, the virtual machine servers will wait until there are no
active virtual machines, and then they'll be shut down.
"""
vmServersConnectionData = self.__dbConnector.getActiveVMServersConnectionData()
if (vmServersConnectionData != None) :
args = dict()
args["Halt"] = data["HaltVMServers"]
args["Unregister"] = False
for connectionData in vmServersConnectionData :
args["ServerNameOrIPAddress"] = connectionData["ServerIP"]
self.__unregisterOrShutdownVMServer(args)
self.__finished = True
def __registerVMServer(self, data):
"""
Processes a virtual machine server registration packet.
Args:
data: the received virtual machine server registration packet.
Returns:
Nothing
"""
try :
# Check if the IP address is assigned to another virtual machine server
server_id = self.__dbConnector.getVMServerID(data["VMServerIP"])
if (server_id != None) :
raise Exception("The IP address " + data["VMServerIP"] + " is assigned to another VM server")
# Check if the name is assigned to another virtual machine server
server_id = self.__dbConnector.getVMServerID(data["VMServerName"])
if (server_id != None) :
raise Exception("The name " + data["VMServerName"] + " is assigned to another VM server")
# Establish a connection
self.__networkManager.connectTo(data["VMServerIP"], data["VMServerPort"],
20, self.__vmServerCallback, True, True)
while not self.__networkManager.isConnectionReady(data["VMServerIP"], data["VMServerPort"]) :
sleep(0.1)
# Register the server on the database
self.__dbConnector.registerVMServer(data["VMServerName"], data["VMServerIP"],
data["VMServerPort"])
# Command the virtual machine server to tell us its state
p = self.__vmServerPacketHandler.createVMServerDataRequestPacket(VMSRVR_PACKET_T.SERVER_STATUS_REQUEST)
self.__networkManager.sendPacket(data["VMServerIP"], data["VMServerPort"], p)
# Everything went fine
p = self.__webPacketHandler.createCommandExecutedPacket(data["CommandID"])
except Exception as e:
p = self.__webPacketHandler.createVMServerRegistrationErrorPacket(data["VMServerIP"],
data["VMServerPort"],
data["VMServerName"], str(e), data["CommandID"])
self.__networkManager.sendPacket('', self.__webPort, p)
def __unregisterOrShutdownVMServer(self, data):
"""
Processes a virtual machine server unregistration or shutdown packet.
Args:
key: the virtual machine server's IPv4 address or its name.
halt: if True, the virtual machine server will be shut down immediately. If false, it'll
wait until all the virtual machines are shut down, and then it will finally be shut down.
unregister: if True, the virtual machine server will be deleted from the cluster server's database.
"""
key = data["ServerNameOrIPAddress"]
halt = data["Halt"]
unregister = data["Unregister"]
if data.has_key("CommandID") :
useCommandID = True
commandID = data["CommandID"]
else :
useCommandID = False
# Shut down the server (if necessary)
serverId = self.__dbConnector.getVMServerID(key)
if (serverId != None) :
serverData = self.__dbConnector.getVMServerBasicData(serverId)
status = serverData["ServerStatus"]
if (status == SERVER_STATE_T.READY or status == SERVER_STATE_T.BOOTING) :
if not halt :
p = self.__vmServerPacketHandler.createVMServerShutdownPacket()
else :
p = self.__vmServerPacketHandler.createVMServerHaltPacket()
self.__networkManager.sendPacket(serverData["ServerIP"], serverData["ServerPort"], p)
# Close the network connection
self.__networkManager.closeConnection(serverData["ServerIP"], serverData["ServerPort"])
if (not unregister) :
self.__dbConnector.updateVMServerStatus(serverId, SERVER_STATE_T.SHUT_DOWN)
self.__dbConnector.deleteVMServerStatistics(serverId)
else :
# Update the virtual machine server's state
self.__dbConnector.deleteVMServer(key)
if (useCommandID) :
# Create a command executed packet and send it to the website
p = self.__webPacketHandler.createCommandExecutedPacket(commandID)
self.__networkManager.sendPacket('', self.__webPort, p)
else :
# Error
if (unregister) :
packet_type = WEB_PACKET_T.VM_SERVER_UNREGISTRATION_ERROR
else :
packet_type = WEB_PACKET_T.VM_SERVER_SHUTDOWN_ERROR
errorMessage = "The virtual machine server with name or IP address <<{0}>> is not registered".format(key)
p = self.__webPacketHandler.createVMServerGenericErrorPacket(packet_type, key, errorMessage, commandID)
self.__networkManager.sendPacket('', self.__webPort, p)
def __updateVMServerStatus(self, data):
"""
Processes a virtual machine server's status packet.
Args:
data: the received packet
Returns:
Nothing
"""
# Fetch the virtual machine server's ID
serverID = None
while (serverID == None) :
serverID = self.__dbConnector.getVMServerID(data["VMServerIP"])
if (serverID == None) :
sleep(0.1)
# Change its status
self.__dbConnector.updateVMServerStatus(serverID, SERVER_STATE_T.READY)
self.__dbConnector.setVMServerStatistics(serverID, data["ActiveDomains"])
def __bootUpVMServer(self, data):
"""
Processes a virtual machine server boot packet.
Args:
serverNameOrIPAddress: the virtual machine server's name or IPv4 address.
Returns:
Nothing
"""
try :
serverNameOrIPAddress = data["ServerNameOrIPAddress"]
serverId = self.__dbConnector.getVMServerID(serverNameOrIPAddress)
if (serverId == None) :
raise Exception("The virtual machine server is not registered")
serverData = self.__dbConnector.getVMServerBasicData(serverId)
# Connect to the virtual machine server
self.__networkManager.connectTo(serverData["ServerIP"], serverData["ServerPort"],
20, self.__vmServerCallback, True, True)
while not self.__networkManager.isConnectionReady(serverData["ServerIP"], serverData["ServerPort"]) :
sleep(0.1)
# Change its status
self.__dbConnector.updateVMServerStatus(serverId, SERVER_STATE_T.BOOTING)
# Send the status request
p = self.__vmServerPacketHandler.createVMServerDataRequestPacket(VMSRVR_PACKET_T.SERVER_STATUS_REQUEST)
self.__networkManager.sendPacket(serverData["ServerIP"], serverData["ServerPort"], p)
# Everything went fine
p = self.__webPacketHandler.createCommandExecutedPacket(data["CommandID"])
except Exception as e:
p = self.__webPacketHandler.createVMServerGenericErrorPacket(WEB_PACKET_T.VM_SERVER_BOOTUP_ERROR,
serverNameOrIPAddress, str(e), data["CommandID"])
self.__networkManager.sendPacket('', self.__webPort, p)
def __sendVMServerStatusData(self):
"""
Processes a virtual machine server data request packet.
Args:
None
Returns:
Nothing
"""
self.__sendStatusData(self.__dbConnector.getVMServerBasicData, self.__webPacketHandler.createVMServerStatusPacket)
def __sendVMDistributionData(self):
"""
Processes a virtual machine distribution data packet
Args:
None
Returns:
Nothing
"""
self.__sendStatusData(self.__dbConnector.getHostedImages, self.__webPacketHandler.createVMDistributionPacket)
def __sendStatusData(self, queryMethod, packetCreationMethod):
"""
Processes a data request packet.
Args:
queryMethod: the method that extracts the data from the database
packetCreationMethod: the method that creates the packet
Returns:
Nothing
"""
segmentSize = 5
outgoingData = []
serverIDs = self.__dbConnector.getVMServerIDs()
if (len(serverIDs) == 0) :
segmentCounter = 0
segmentNumber = 0
sendLastSegment = True
else :
segmentCounter = 1
segmentNumber = (len(serverIDs) / segmentSize)
if (len(serverIDs) % segmentSize != 0) :
segmentNumber += 1
sendLastSegment = True
else :
sendLastSegment = False
for serverID in serverIDs :
row = queryMethod(serverID)
if (isinstance(row, dict)) :
outgoingData.append(row)
else :
outgoingData += row
if (len(outgoingData) >= segmentSize) :
# Flush
packet = packetCreationMethod(segmentCounter, segmentNumber, outgoingData)
self.__networkManager.sendPacket('', self.__webPort, packet)
outgoingData = []
segmentCounter += 1
# Send the last segment
if (sendLastSegment) :
packet = packetCreationMethod(segmentCounter, segmentNumber, outgoingData)
self.__networkManager.sendPacket('', self.__webPort, packet)
def __bootUpVM(self, data):
"""
Processes a virtual machine boot request packet.
Args:
vmName: the virtual machine's ID
userID: the user's unique identifier
Returns:
Nothing
"""
vmID = data["VMID"]
userID = data["UserID"]
# Choose the virtual machine server that will host the image
(serverID, errorMessage) = self.__loadBalancer.assignVMServer(vmID)
if (errorMessage != None) :
# Something went wrong => warn the user
p = self.__webPacketHandler.createVMBootFailurePacket(vmID, errorMessage, data["CommandID"])
self.__networkManager.sendPacket('', self.__webPort, p)
else :
# Ask the virtual machine server to boot up the VM
p = self.__vmServerPacketHandler.createVMBootPacket(vmID, userID, data["CommandID"])
serverData = self.__dbConnector.getVMServerBasicData(serverID)
self.__networkManager.sendPacket(serverData["ServerIP"], serverData["ServerPort"], p)
# Register the boot command
self.__dbConnector.registerVMBootCommand(data["CommandID"], data["VMID"])
# Everything went fine
#p = self.__webPacketHandler.createCommandExecutedPacket(data["CommandID"])
#self.__networkManager.sendPacket('', self.__webPort, p)
def processVMServerIncomingPacket(self, packet):
"""
Processes a packet sent from a virtual machine server.
Args:
packet: the packet to process
Returns:
Nothing
"""
data = self.__vmServerPacketHandler.readPacket(packet)
if (data["packet_type"] == VMSRVR_PACKET_T.SERVER_STATUS) :
self.__updateVMServerStatus(data)
elif (data["packet_type"] == VMSRVR_PACKET_T.DOMAIN_CONNECTION_DATA) :
self.__sendVMConnectionData(data)
elif (data["packet_type"] == VMSRVR_PACKET_T.ACTIVE_VM_DATA) :
self.__sendVNCConnectionData(packet)
def processServerReconnectionData(self, ipAddress, reconnection_status) :
"""
Processes a reconnection status event
Args:
ipAddress: the connection's IPv4 address
port: the connection's port
reconnection_status: the reconnection process' status
Returns:
Nothing
"""
if (reconnection_status == RECONNECTION_T.RECONNECTING) :
status = SERVER_STATE_T.RECONNECTING
elif (reconnection_status == RECONNECTION_T.REESTABLISHED) :
status = SERVER_STATE_T.READY
else :
status = SERVER_STATE_T.CONNECTION_TIMED_OUT
serverID = self.__dbConnector.getVMServerID(ipAddress)
self.__dbConnector.updateVMServerStatus(serverID, status)
def __sendVNCConnectionData(self, packet):
"""
Processes a VNC connection data packet
Args:
packet: the packet to process
Returns:
Nothing
"""
p = self.__webPacketHandler.createActiveVMsDataPacket(packet)
self.__networkManager.sendPacket('', self.__webPort, p)
def __sendVMConnectionData(self, data):
"""
Processes a virtual machine connection data packet.
Args:
data: the packet to process' data.
Returns:
Nothing
"""
# Delete the boot command from the database
self.__dbConnector.removeVMBootCommand(data["CommandID"])
p = self.__webPacketHandler.createVMConnectionDataPacket(data["VNCServerIP"],
data["VNCServerPort"], data["VNCServerPassword"], data["CommandID"])
self.__networkManager.sendPacket('', self.__webPort, p)
def monitorVMBootCommands(self):
errorMessage = "Could not boot up virtual machine (timeout error)"
while not self.__finished :
data = self.__dbConnector.getOldVMBootCommandID(self.__timeout)
if (not self.__finished and data == None) :
# Be careful with this: if MySQL server receives too many queries per second,
# the reactor's database connection will be closed.
sleep(1)
else :
# Create a virtual machine boot error packet
p = self.__webPacketHandler.createVMBootFailurePacket(data[1], errorMessage, data[0])
self.__networkManager.sendPacket('', self.__webPort, p)
def hasFinished(self):
"""
Indicates if the cluster server has finished or not.
"""
return self.__finished
def shutdown(self):
"""
Shuts down the cluster server.
Args:
None
Returns:
Nothing
@attention: this method MUST NOT be called from a network thread. If you do so, the application will hang!
"""
self.__networkManager.stopNetworkService()
2. Ejecutar el clientReceptor.
3. Inmediatamente después ejecutar clientEmisor.
Módulos necesarios:
-serverWridge,clientEmisor,clientReceptor
'''
from network.manager.networkManager import NetworkManager
from network.tests.ConvCallback import ConvCallback
from time import sleep
if __name__ == "__main__" :
listP = []
listP1 = []
networkManager = NetworkManager()
networkManager.startNetworkService()
networkManager.listenIn(8080, ConvCallback(listP))
networkManager.listenIn(8081, ConvCallback(listP1))
print("The server is now ready")
#esperamos una respuesta
while(len(listP) == 0):
sleep(0.1)
# Extraemos la contestacion
ans = listP[0]
#Se la enviamos al otro cliente
p = networkManager.createPacket(0)
p.writeString(ans)
networkManager.sendPacket('', 8081, p)
#Esperamos al de regreso
while(len(listP1) == 0):
sleep(0.1)
# Extraemos la contestacion
# -*- coding: utf8 -*-
'''
A simple server test
@author: Luis Barrios Hernández
@version: 1.0
'''
from network.manager.networkManager import NetworkManager
from network.tests.dummyCallback import DummyCallback
from time import sleep
if __name__ == "__main__" :
networkManager = NetworkManager()
networkManager.startNetworkService()
networkManager.listenIn(8080, DummyCallback())
print("The server is now ready")
sleep(200)
networkManager.stopNetworkService()
class VMServerReactor(NetworkCallback):
"""
Virtual machine server packet reactor
"""
def __init__(self, configurationFileParser):
"""
Initializes the reactor's state, establishes the connection with the database
and creates the control connection
Args:
configurationFileParser: the virtual machine server's configuration file parser
"""
self.__finished = False
self.__emergencyStop = False
self.__fileTransferThread = None
self.__compressionThread = None
self.__networkManager = None
self.__parser = configurationFileParser
self.__domainHandler = None
self.__domainTimeout = 0
try:
self.__connectToDatabases(
"VMServerDB",
self.__parser.getConfigurationParameter("databaseUserName"),
self.__parser.getConfigurationParameter("databasePassword"))
self.__startListenning()
except Exception as e:
print e.message
self.__emergencyStop = True
self.__finished = True
def __connectToDatabases(self, databaseName, user, password):
"""
Establishes the connection with the database
Args:
databaseName: a database name
user: a SQL username
password: the user's password
Returns:
Nothing
"""
self.__commandsDBConnector = VMServerDBConnector(
user, password, databaseName)
def __startListenning(self):
"""
Creates the control connection
Args:
None
Returns:
Nothing
"""
networkInterface = self.__parser.getConfigurationParameter(
"vncNetworkInterface")
listenningPort = self.__parser.getConfigurationParameter(
"listenningPort")
try:
self.__vncServerIP = get_ip_address(networkInterface)
except Exception:
raise Exception(
"Error: the network interface '{0}' is not ready. Exiting now".
format(networkInterface))
self.__ftpTimeout = self.__parser.getConfigurationParameter(
"FTPTimeout")
self.__listenningPort = listenningPort
self.__networkManager = NetworkManager(
self.__parser.getConfigurationParameter("certificatePath"))
self.__networkManager.startNetworkService()
self.__useSSL = self.__parser.getConfigurationParameter("useSSL")
self.__packetManager = VMServerPacketHandler(self.__networkManager)
self.__connectToDatabases(
"VMServerDB",
self.__parser.getConfigurationParameter("databaseUserName"),
self.__parser.getConfigurationParameter("databasePassword"))
self.__domainHandler = DomainHandler(
self.__commandsDBConnector, self.__vncServerIP,
self.__networkManager, self.__packetManager, self.__listenningPort,
self.__parser.getConfigurationParameter("useQEMUWebsockets"),
self.__parser.getConfigurationParameter("websockifyPath"),
self.__parser.getConfigurationParameter("configFilePath"),
self.__parser.getConfigurationParameter("sourceImagePath"),
self.__parser.getConfigurationParameter("executionImagePath"),
self.__parser.getConfigurationParameter("passwordLength"))
self.__domainHandler.connectToLibvirt(
self.__parser.getConfigurationParameter("vncNetworkInterface"),
self.__parser.getConfigurationParameter("vnName"),
self.__parser.getConfigurationParameter("gatewayIP"),
self.__parser.getConfigurationParameter("netMask"),
self.__parser.getConfigurationParameter("dhcpStartIP"),
self.__parser.getConfigurationParameter("dhcpEndIP"),
self.__parser.getConfigurationParameter(
"createVirtualNetworkAsRoot"))
self.__domainHandler.doInitialCleanup()
self.__deleteTemporaryZipFiles()
self.__fileTransferThread = FileTransferThread(
self.__networkManager, self.__listenningPort, self.__packetManager,
self.__parser.getConfigurationParameter("TransferDirectory"),
self.__parser.getConfigurationParameter("FTPTimeout"),
self.__parser.getConfigurationParameter("MaxTransferAttempts"),
self.__commandsDBConnector, self.__useSSL)
self.__compressionThread = CompressionThread(
self.__parser.getConfigurationParameter("TransferDirectory"),
self.__parser.getConfigurationParameter("sourceImagePath"),
self.__parser.getConfigurationParameter("configFilePath"),
self.__commandsDBConnector, self.__domainHandler,
self.__networkManager, self.__listenningPort, self.__packetManager)
self.__fileTransferThread.start()
self.__compressionThread.start()
self.__networkManager.listenIn(self.__listenningPort, self,
self.__useSSL)
def __deleteTemporaryZipFiles(self):
"""
Deletes the temporary zip files
Args:
None
Returns:
Nothing
"""
transfer_dir_path = self.__parser.getConfigurationParameter(
"TransferDirectory")
for filePath in os.listdir(transfer_dir_path):
fileName = os.path.splitext(filePath)[0]
if re.match("[^0-9]", fileName):
# The non-temporary zip files only have numbers on their names
os.remove(os.path.join(transfer_dir_path, filePath))
def shutdown(self):
"""
Shuts down the virtual machine server
Args:
None
Returns:
Nothing
"""
if (self.__emergencyStop):
self.__domainTimeout = 0
self.__domainHandler.shutdown(self.__domainTimeout)
if (self.__fileTransferThread != None):
self.__fileTransferThread.stop()
self.__fileTransferThread.join()
if (self.__compressionThread != None):
self.__compressionThread.stop()
self.__compressionThread.join()
if (self.__networkManager != None):
self.__networkManager.stopNetworkService(
) # Dejar de atender peticiones inmediatamente
sys.exit()
def processPacket(self, packet):
"""
Processes a packet sent from the cluster server
Args:
packet: the packet to process
Returns:
Nothing
"""
data = self.__packetManager.readPacket(packet)
if (data["packet_type"] == VM_SERVER_PACKET_T.CREATE_DOMAIN):
self.__domainHandler.createDomain(data["MachineID"],
data["UserID"],
data["CommandID"])
elif (data["packet_type"] == VM_SERVER_PACKET_T.SERVER_STATUS_REQUEST):
self.__sendStatusData()
elif (data["packet_type"] == VM_SERVER_PACKET_T.USER_FRIENDLY_SHUTDOWN
):
self.__domainTimeout = data["Timeout"]
self.__finished = True
elif (data["packet_type"] == VM_SERVER_PACKET_T.HALT):
self.__domainTimeout = 0
self.__finished = True
elif (data['packet_type'] == VM_SERVER_PACKET_T.QUERY_ACTIVE_VM_DATA):
self.__sendDomainsVNCConnectionData()
elif (data['packet_type'] == VM_SERVER_PACKET_T.DESTROY_DOMAIN):
self.__domainHandler.destroyDomain(data["DomainID"])
elif (data['packet_type'] == VM_SERVER_PACKET_T.REBOOT_DOMAIN):
self.__domainHandler.rebootDomain(data["DomainID"])
elif (data['packet_type'] ==
VM_SERVER_PACKET_T.QUERY_ACTIVE_DOMAIN_UIDS):
self.__sendActiveDomainUIDs()
elif (data['packet_type'] == VM_SERVER_PACKET_T.IMAGE_EDITION):
self.__processImageEditionPacket(data)
elif (data['packet_type'] == VM_SERVER_PACKET_T.DEPLOY_IMAGE):
self.__processDeployImagePacket(data)
elif (data['packet_type'] == VM_SERVER_PACKET_T.DELETE_IMAGE):
self.__processDeleteImagePacket(data)
def __processDeployImagePacket(self, data):
"""
Processes an image deployment packet
Args:
data: a dictionary containing the packet to process' data
Returns:
Nothing
"""
data.pop("packet_type")
data["Transfer_Type"] = TRANSFER_T.DEPLOY_IMAGE
self.__commandsDBConnector.addToTransferQueue(data)
def __processDeleteImagePacket(self, data):
isBootable = self.__commandsDBConnector.getBootableFlag(
data["ImageID"])
if (isBootable):
osImagePath = os.path.join(
self.__parser.getConfigurationParameter("sourceImagePath"),
self.__commandsDBConnector.getOSImagePath(data["ImageID"]))
definitionFilePath = os.path.join(
self.__parser.getConfigurationParameter("configFilePath"),
self.__commandsDBConnector.getDefinitionFilePath(
data["ImageID"]))
try:
self.__commandsDBConnector.deleteImage(data["ImageID"])
ChildProcessManager.runCommandInForeground(
"rm -rf " + os.path.dirname(osImagePath),
VMServerException)
ChildProcessManager.runCommandInForeground(
"rm -rf " + os.path.dirname(definitionFilePath),
VMServerException)
p = self.__packetManager.createConfirmationPacket(
VM_SERVER_PACKET_T.IMAGE_DELETED, data["ImageID"],
data["CommandID"])
except Exception:
p = self.__packetManager.createErrorPacket(
VM_SERVER_PACKET_T.IMAGE_DELETION_ERROR,
ERROR_DESC_T.VM_SRVR_INTERNAL_ERROR, data["CommandID"])
else:
if (isBootable != None):
errorCode = ERROR_DESC_T.VMSRVR_LOCKED_IMAGE
else:
errorCode = ERROR_DESC_T.VMSRVR_UNKNOWN_IMAGE
p = self.__packetManager.createErrorPacket(
VM_SERVER_PACKET_T.IMAGE_DELETION_ERROR, errorCode,
data["CommandID"])
self.__networkManager.sendPacket('', self.__listenningPort, p)
def __processImageEditionPacket(self, data):
"""
Processes an image edition packet
Args:
data: a dictionary containing the packet to process' data
Returns:
Nothing
"""
data.pop("packet_type")
if (data["Modify"]):
data["Transfer_Type"] = TRANSFER_T.EDIT_IMAGE
else:
data["Transfer_Type"] = TRANSFER_T.CREATE_IMAGE
data.pop("Modify")
self.__commandsDBConnector.addToTransferQueue(data)
def __sendDomainsVNCConnectionData(self):
'''
Sends the domains' VNC connection data to the cluster server
Args:
None
Returns:
Nothing
'''
vncConnectionData = self.__commandsDBConnector.getDomainsConnectionData(
)
segmentSize = 150
segmentCounter = 1
outgoingData = []
if (len(vncConnectionData) == 0):
segmentCounter = 0
segmentNumber = (len(vncConnectionData) / segmentSize)
if (len(vncConnectionData) % segmentSize != 0):
segmentNumber += 1
sendLastSegment = True
else:
sendLastSegment = segmentNumber == 0
for connectionParameters in vncConnectionData:
outgoingData.append(connectionParameters)
if (len(outgoingData) >= segmentSize):
# Flush
packet = self.__packetManager.createActiveVMsVNCDataPacket(
self.__vncServerIP, segmentCounter, segmentNumber,
outgoingData)
self.__networkManager.sendPacket('', self.__listenningPort,
packet)
outgoingData = []
segmentCounter += 1
# Send the last segment
if (sendLastSegment):
packet = self.__packetManager.createActiveVMsVNCDataPacket(
self.__vncServerIP, segmentCounter, segmentNumber,
outgoingData)
self.__networkManager.sendPacket('', self.__listenningPort, packet)
def __sendStatusData(self):
"""
Sends the server's status data to the cluster server
Args:
None
Returns:
Nothing
"""
info = self.__domainHandler.getLibvirtStatusData()
realCPUNumber = multiprocessing.cpu_count()
freeOutput = ChildProcessManager.runCommandInForeground(
"free -k", VMServerException)
# free's output contains the following lines and collumns:
# total used free shared buffers cached
# Mem: 4146708480 3939934208 206774272 0 224706560 1117671424
# -/+ buffers/cache: 2597556224 1549152256
# Swap: 2046816256 42455040 2004361216
#
# We must get the third line
availableMemory = int(freeOutput.split("\n")[1].split()[1])
freeMemory = int(freeOutput.split("\n")[2].split()[2])
freeStorageSpace, availableStorageSpace, freeTemporaryStorage, availableTemporaryStorage = self.__generateDiskStats(
)
packet = self.__packetManager.createVMServerStatusPacket(
self.__vncServerIP, info["#domains"], freeMemory, availableMemory,
freeStorageSpace, availableStorageSpace, freeTemporaryStorage,
availableTemporaryStorage, info["#vcpus"], realCPUNumber)
self.__networkManager.sendPacket('', self.__listenningPort, packet)
def __generateDiskStats(self):
"""
Generates the hard disk usage statistics
Args:
None
Returns:
the free and available storage and temporary space
"""
diskStats_storage = os.statvfs(
self.__parser.getConfigurationParameter("sourceImagePath"))
diskStats_temporaryData = os.statvfs(
self.__parser.getConfigurationParameter("executionImagePath"))
allocatedStorageSpace, allocatedTemporaryStorageSpace = self.__checkDiskImagesSpace(
)
freeStorageSpace = diskStats_storage.f_bfree * diskStats_storage.f_frsize / 1024 - allocatedStorageSpace
availableStorageSpace = diskStats_storage.f_blocks * diskStats_storage.f_frsize / 1024
freeTemporaryStorageSpace = diskStats_temporaryData.f_bfree * diskStats_temporaryData.f_frsize / 1024 - allocatedTemporaryStorageSpace
availableTemporaryStorageSpace = diskStats_temporaryData.f_blocks * diskStats_temporaryData.f_frsize / 1024
return freeStorageSpace, availableStorageSpace, freeTemporaryStorageSpace, availableTemporaryStorageSpace
def __checkDiskImagesSpace(self):
"""
Checks how much disk space must be allocated for the active virtual machines' disk images
Args:
None
Returns:
the storage and temporary storage space that must be allocated.
"""
activeDomainNames = self.__commandsDBConnector.getRegisteredDomainNames(
)
allocatedStorageSpace = 0
allocatedTemporaryStorageSpace = 0
for domainName in activeDomainNames:
imageID = self.__commandsDBConnector.getDomainImageID(domainName)
if (imageID == None):
return 0, 0
dataImagePath = self.__commandsDBConnector.getDomainDataImagePath(
domainName)
if (dataImagePath == None):
return 0, 0
osImagePath = self.__commandsDBConnector.getDomainOSImagePath(
domainName)
if (osImagePath == None):
return 0, 0
isEditedImage = self.__commandsDBConnector.getBootableFlag(imageID)
if (isEditedImage == None):
return 0, 0
dataSpace = self.__getAllocatedSpace(dataImagePath)
if (isEditedImage):
osSpace = self.__getAllocatedSpace(osImagePath)
allocatedStorageSpace += osSpace + dataSpace
else:
allocatedTemporaryStorageSpace += dataSpace
return allocatedStorageSpace, allocatedTemporaryStorageSpace
def __getAllocatedSpace(self, imagePath):
"""
Returns the disk space that must be allocated for a disk image
Args:
imagePath: the disk image's path
Returns:
Nothing
"""
try:
output = ChildProcessManager.runCommandInForeground(
"qemu-img info " + imagePath, Exception)
lines = output.split("\n")
virtualSize = VMServerReactor.__extractImageSize(
lines[2].split(":")[1].split("(")[0])
usedSpace = VMServerReactor.__extractImageSize(
lines[3].split(":")[1])
return virtualSize - usedSpace
except Exception as e:
print e
return 0
@staticmethod
def __extractImageSize(string):
"""
Extracts an image size from a quemu-img command output.
Args:
string: the qemu-img command's image size
Returns:
the disk image size (in kilobytes)
"""
if ("G" in string):
power = 2
elif ("M" in string):
power = 1
else:
power = 0
string = string.replace("G", "")
string = string.replace("M", "")
string = string.replace("K", "")
return int(ceil(float(string))) * 1024**power
def __sendActiveDomainUIDs(self):
"""
Sends domains' UUIDs to the cluster server
Args:
None
Returns:
Nothing
"""
activeDomainUIDs = self.__commandsDBConnector.getActiveDomainUIDs()
packet = self.__packetManager.createActiveDomainUIDsPacket(
self.__vncServerIP, activeDomainUIDs)
self.__networkManager.sendPacket('', self.__listenningPort, packet)
def has_finished(self):
"""
Checks if the virtual machine server has been shut down or not
Args:
None
Returns:
Nothing
"""
return self.__finished
2. Ejecutar el clientReceptor.
3. Inmediatamente después ejecutar clientEmisor.
Módulos necesarios:
-serverWridge,clientEmisor,clientReceptor
'''
from network.manager.networkManager import NetworkManager
from network.tests.ConvCallback import ConvCallback
from time import sleep
if __name__ == "__main__":
listP = []
listP1 = []
networkManager = NetworkManager()
networkManager.startNetworkService()
networkManager.listenIn(8080, ConvCallback(listP))
networkManager.listenIn(8081, ConvCallback(listP1))
print("The server is now ready")
#esperamos una respuesta
while (len(listP) == 0):
sleep(0.1)
# Extraemos la contestacion
ans = listP[0]
#Se la enviamos al otro cliente
p = networkManager.createPacket(0)
p.writeString(ans)
networkManager.sendPacket('', 8081, p)
#Esperamos al de regreso
while (len(listP1) == 0):
sleep(0.1)
# Extraemos la contestacion
class ImageRepositoryReactor(object):
"""
Image repository packet reactor
"""
def __init__(self, diskImagesDirectory):
"""
Initializes the reactor's state
Args:
diskImagesDirectory: the FTP server's root directory
"""
self.__diskImagesDirectory = diskImagesDirectory
self.__slotCounter = MultithreadingCounter()
self.__retrieveQueue = GenericThreadSafeList()
self.__storeQueue = GenericThreadSafeList()
self.__finish = False
self.__networkManager = None
self.__ftpServer = None
def connectToDatabase(self, repositoryDBName, repositoryDBUser,
repositoryDBPassword):
"""
Establishes the connection with the image repository's database.
Args:
repositoryDBName: a database name
repositoryDBUser: an user name
repositoryDBPassword: a password
Returns:
Nothing
"""
self.__dbConnector = ImageRepositoryDBConnector(
repositoryDBUser, repositoryDBPassword, repositoryDBName)
def startListenning(self, networkInterface, useSSL, certificatesDirectory,
commandsListenningPort, ftpListenningPort,
maxConnections, maxConnectionsPerIP,
uploadBandwidthRatio, downloadBandwidthRatio,
ftpUsername, ftpPasswordLength):
"""
Boots up the FTP server and creates the control connection.
Args:
networkInterface: the network interface that will be used by the FTP server
useSSL: indicates if SSL encryption must be used in the control connection or not
certificatesDirectory: the directory where the files server.crt and server.key are
commandsListenningPort: the control connection's port
ftpListenningPort: the FTP server listenning port
maxConnections: maximum FTP connections
maxConnectionsPerIP: maximum FTP connections per IP address
uploadBandwidthRatio: maximum download bandwidth fraction
downloadBandwidthRatio: maximum upload bandwidth fraction
ftpUsername: the FTP user that the virtual machine servers will use
ftpPasswordLength: the random FTP password length
Returns:
Nothing
@attention: The FTP password will be randomly generated at every boot.
"""
try:
self.__maxConnections = maxConnections
self.__commandsListenningPort = commandsListenningPort
self.__FTPListenningPort = ftpListenningPort
self.__networkManager = NetworkManager(certificatesDirectory)
self.__repositoryPacketHandler = ImageRepositoryPacketHandler(
self.__networkManager)
self.__commandsCallback = CommandsCallback(
self.__networkManager, self.__repositoryPacketHandler,
commandsListenningPort, self.__dbConnector,
self.__retrieveQueue, self.__storeQueue,
self.__diskImagesDirectory)
self.__networkManager.startNetworkService()
self.__networkManager.listenIn(commandsListenningPort,
self.__commandsCallback, useSSL)
dataCallback = FTPServerCallback(self.__slotCounter,
self.__dbConnector)
self.__ftpUsername = ftpUsername
self.__ftpPassword = ChildProcessManager.runCommandInForeground(
"openssl rand -base64 {0}".format(ftpPasswordLength),
Exception)
self.__ftpServer = ConfigurableFTPServer(
"Image repository FTP Server")
self.__ftpServer.startListenning(networkInterface,
ftpListenningPort, maxConnections,
maxConnectionsPerIP, dataCallback,
downloadBandwidthRatio,
uploadBandwidthRatio)
self.__ftpServer.addUser(self.__ftpUsername, self.__ftpPassword,
self.__diskImagesDirectory, "eramw")
except Exception as e:
print "Error: " + e.message
self.__finish = True
def stopListenning(self):
"""
Stops the FTP server and closes the control connection.
Args:
None
Returns:
Nothing
"""
if (self.__ftpServer != None):
try:
self.__ftpServer.stopListenning()
except Exception:
pass
if (self.__networkManager != None):
self.__networkManager.stopNetworkService()
def initTransfers(self):
"""
Initializes the upload and download transfers
Args:
None
Returns:
Nothing
"""
store = False
while not (self.__finish or self.__commandsCallback.haltReceived()):
if (self.__slotCounter.read() == self.__maxConnections):
# No free slots => sleep
sleep(0.1)
else:
# There are slots => enable uploads and downloads in parallel
self.__slotCounter.decrement()
if (self.__retrieveQueue.isEmpty()
and self.__storeQueue.isEmpty()):
sleep(0.1)
continue
if (not self.__retrieveQueue.isEmpty()
and self.__storeQueue.isEmpty()):
queue = self.__retrieveQueue
store = False
elif (self.__retrieveQueue.isEmpty()
and not self.__storeQueue.isEmpty()):
queue = self.__storeQueue
store = True
else:
if (store):
queue = self.__retrieveQueue
store = False
else:
queue = self.__storeQueue
store = True
(imageID, clientIP, clientPort) = queue.pop(0)
imageData = self.__dbConnector.getImageData(imageID)
if (imageData == None):
if (store):
packet_type = PACKET_T.STOR_ERROR
else:
packet_type = PACKET_T.RETR_ERROR
p = self.__repositoryPacketHandler.createErrorPacket(
packet_type, ERROR_DESC_T.IR_IMAGE_DELETED)
self.__networkManager.sendPacket(
'', self.__commandsListenningPort, p, clientIP,
clientPort)
else:
compressedFilePath = imageData["compressedFilePath"]
if (not "undefined" in compressedFilePath):
serverDirectory = path.relpath(
path.dirname(compressedFilePath),
self.__diskImagesDirectory)
compressedFileName = path.basename(compressedFilePath)
else:
serverDirectory = str(imageID)
compressedFileName = ""
serverDirectoryPath = path.join(
self.__diskImagesDirectory, serverDirectory)
if (path.exists(serverDirectoryPath)):
# The directory exists, and can store shit => clean it up!
ChildProcessManager.runCommandInForeground(
"rm -rf " + serverDirectoryPath, Exception)
mkdir(serverDirectoryPath)
if (store):
packet_type = PACKET_T.STOR_START
else:
packet_type = PACKET_T.RETR_START
p = self.__repositoryPacketHandler.createTransferEnabledPacket(
packet_type, imageID, self.__FTPListenningPort,
self.__ftpUsername, self.__ftpPassword,
serverDirectory, compressedFileName)
self.__networkManager.sendPacket(
'', self.__commandsListenningPort, p, clientIP,
clientPort)
# -*- coding: utf8 -*-
'''
A simple server test
@author: Luis Barrios Hernández
@version: 1.0
'''
from network.manager.networkManager import NetworkManager
from network.tests.dummyCallback import DummyCallback
from time import sleep
if __name__ == "__main__":
networkManager = NetworkManager()
networkManager.startNetworkService()
networkManager.listenIn(8080, DummyCallback())
print("The server is now ready")
sleep(200)
networkManager.stopNetworkService()
class ClusterServerReactor(WebPacketReactor, VMServerPacketReactor):
'''
These objects react to packages received from the website or from
a virtual machine server.
'''
def __init__(self, timeout):
"""
Initializes the reactor's state.
Args:
None
"""
self.__webCallback = WebCallback(self)
self.__finished = False
self.__timeout = timeout
def connectToDatabase(self, mysqlRootsPassword, dbName, dbUser, dbPassword,
scriptPath):
"""
Establishes a connection with the cluster server database.
Args:
mysqlRootsPassword: MySQL root's password
dbName: the cluster server database's name
dbUser: the cluster server database's user name
dbPassword: the cluster server database's user password
scriptPath: the cluster server database's initialization script path
"""
configurator = DBConfigurator(mysqlRootsPassword)
configurator.runSQLScript(dbName, scriptPath)
configurator.addUser(dbUser, dbPassword, dbName, True)
self.__dbConnector = ClusterServerDatabaseConnector(
dbUser, dbPassword, dbName)
self.__dbConnector.connect()
self.__dbConnector.resetVMServersStatus()
def startListenning(self, certificatePath, port):
"""
Starts the network service and creates a server connection.
Args:
certificatePath: the server.crt and server.key files path
port: the listenning port
Returns:
Nothing
"""
self.__loadBalancer = SimpleLoadBalancer(self.__dbConnector)
self.__networkManager = NetworkManager(certificatePath)
self.__webPort = port
self.__networkManager.startNetworkService()
self.__webPacketHandler = ClusterServerPacketHandler(
self.__networkManager)
self.__vmServerPacketHandler = VMServerPacketHandler(
self.__networkManager)
self.__networkManager.listenIn(port, self.__webCallback, True)
self.__vmServerCallback = VMServerCallback(self)
def processWebIncomingPacket(self, packet):
"""
Processes a packet received from the web server.
Args:
packet: the packet to process
Returns:
Nothing
"""
data = self.__webPacketHandler.readPacket(packet)
if (data["packet_type"] == WEB_PACKET_T.REGISTER_VM_SERVER):
self.__registerVMServer(data)
elif (data["packet_type"] == WEB_PACKET_T.QUERY_VM_SERVERS_STATUS):
self.__sendVMServerStatusData()
elif (data["packet_type"] ==
WEB_PACKET_T.UNREGISTER_OR_SHUTDOWN_VM_SERVER):
self.__unregisterOrShutdownVMServer(data)
elif (data["packet_type"] == WEB_PACKET_T.BOOTUP_VM_SERVER):
self.__bootUpVMServer(data)
elif (data["packet_type"] == WEB_PACKET_T.VM_BOOT_REQUEST):
self.__bootUpVM(data)
elif (data["packet_type"] == WEB_PACKET_T.HALT):
self.__halt(data)
elif (data["packet_type"] == WEB_PACKET_T.QUERY_VM_DISTRIBUTION):
self.__sendVMDistributionData()
elif (data["packet_type"] == WEB_PACKET_T.QUERY_ACTIVE_VM_DATA):
self.__requestVNCConnectionData()
def __requestVNCConnectionData(self):
"""
Sends a VNC connection data request packet to all the active virtual machine servers
Args:
None
Returns:
Nothing
"""
# Create a VNC connection data packet
p = self.__vmServerPacketHandler.createVMServerDataRequestPacket(
VMSRVR_PACKET_T.QUERY_ACTIVE_VM_DATA)
# Fetch the active virtual machine server's IP addresses and ports
connectionData = self.__dbConnector.getActiveVMServersConnectionData()
for cd in connectionData:
self.__networkManager.sendPacket(cd["ServerIP"], cd["ServerPort"],
p)
def __halt(self, data):
"""
Shuts down the cluster (including the virtual machine servers).
Args:
haltVMServers: if True, all the active virtual machines will be destroyed and the virtual machine
servers will be shut down. If false, the virtual machine servers will wait until there are no
active virtual machines, and then they'll be shut down.
"""
vmServersConnectionData = self.__dbConnector.getActiveVMServersConnectionData(
)
if (vmServersConnectionData != None):
args = dict()
args["Halt"] = data["HaltVMServers"]
args["Unregister"] = False
for connectionData in vmServersConnectionData:
args["ServerNameOrIPAddress"] = connectionData["ServerIP"]
self.__unregisterOrShutdownVMServer(args)
self.__finished = True
def __registerVMServer(self, data):
"""
Processes a virtual machine server registration packet.
Args:
data: the received virtual machine server registration packet.
Returns:
Nothing
"""
try:
# Check if the IP address is assigned to another virtual machine server
server_id = self.__dbConnector.getVMServerID(data["VMServerIP"])
if (server_id != None):
raise Exception("The IP address " + data["VMServerIP"] +
" is assigned to another VM server")
# Check if the name is assigned to another virtual machine server
server_id = self.__dbConnector.getVMServerID(data["VMServerName"])
if (server_id != None):
raise Exception("The name " + data["VMServerName"] +
" is assigned to another VM server")
# Establish a connection
self.__networkManager.connectTo(data["VMServerIP"],
data["VMServerPort"], 20,
self.__vmServerCallback, True,
True)
while not self.__networkManager.isConnectionReady(
data["VMServerIP"], data["VMServerPort"]):
sleep(0.1)
# Register the server on the database
self.__dbConnector.registerVMServer(data["VMServerName"],
data["VMServerIP"],
data["VMServerPort"])
# Command the virtual machine server to tell us its state
p = self.__vmServerPacketHandler.createVMServerDataRequestPacket(
VMSRVR_PACKET_T.SERVER_STATUS_REQUEST)
self.__networkManager.sendPacket(data["VMServerIP"],
data["VMServerPort"], p)
# Everything went fine
p = self.__webPacketHandler.createCommandExecutedPacket(
data["CommandID"])
except Exception as e:
p = self.__webPacketHandler.createVMServerRegistrationErrorPacket(
data["VMServerIP"], data["VMServerPort"], data["VMServerName"],
str(e), data["CommandID"])
self.__networkManager.sendPacket('', self.__webPort, p)
def __unregisterOrShutdownVMServer(self, data):
"""
Processes a virtual machine server unregistration or shutdown packet.
Args:
key: the virtual machine server's IPv4 address or its name.
halt: if True, the virtual machine server will be shut down immediately. If false, it'll
wait until all the virtual machines are shut down, and then it will finally be shut down.
unregister: if True, the virtual machine server will be deleted from the cluster server's database.
"""
key = data["ServerNameOrIPAddress"]
halt = data["Halt"]
unregister = data["Unregister"]
if data.has_key("CommandID"):
useCommandID = True
commandID = data["CommandID"]
else:
useCommandID = False
# Shut down the server (if necessary)
serverId = self.__dbConnector.getVMServerID(key)
if (serverId != None):
serverData = self.__dbConnector.getVMServerBasicData(serverId)
status = serverData["ServerStatus"]
if (status == SERVER_STATE_T.READY
or status == SERVER_STATE_T.BOOTING):
if not halt:
p = self.__vmServerPacketHandler.createVMServerShutdownPacket(
)
else:
p = self.__vmServerPacketHandler.createVMServerHaltPacket()
self.__networkManager.sendPacket(serverData["ServerIP"],
serverData["ServerPort"], p)
# Close the network connection
self.__networkManager.closeConnection(serverData["ServerIP"],
serverData["ServerPort"])
if (not unregister):
self.__dbConnector.updateVMServerStatus(
serverId, SERVER_STATE_T.SHUT_DOWN)
self.__dbConnector.deleteVMServerStatistics(serverId)
else:
# Update the virtual machine server's state
self.__dbConnector.deleteVMServer(key)
if (useCommandID):
# Create a command executed packet and send it to the website
p = self.__webPacketHandler.createCommandExecutedPacket(
commandID)
self.__networkManager.sendPacket('', self.__webPort, p)
else:
# Error
if (unregister):
packet_type = WEB_PACKET_T.VM_SERVER_UNREGISTRATION_ERROR
else:
packet_type = WEB_PACKET_T.VM_SERVER_SHUTDOWN_ERROR
errorMessage = "The virtual machine server with name or IP address <<{0}>> is not registered".format(
key)
p = self.__webPacketHandler.createVMServerGenericErrorPacket(
packet_type, key, errorMessage, commandID)
self.__networkManager.sendPacket('', self.__webPort, p)
def __updateVMServerStatus(self, data):
"""
Processes a virtual machine server's status packet.
Args:
data: the received packet
Returns:
Nothing
"""
# Fetch the virtual machine server's ID
serverID = None
while (serverID == None):
serverID = self.__dbConnector.getVMServerID(data["VMServerIP"])
if (serverID == None):
sleep(0.1)
# Change its status
self.__dbConnector.updateVMServerStatus(serverID, SERVER_STATE_T.READY)
self.__dbConnector.setVMServerStatistics(serverID,
data["ActiveDomains"])
def __bootUpVMServer(self, data):
"""
Processes a virtual machine server boot packet.
Args:
serverNameOrIPAddress: the virtual machine server's name or IPv4 address.
Returns:
Nothing
"""
try:
serverNameOrIPAddress = data["ServerNameOrIPAddress"]
serverId = self.__dbConnector.getVMServerID(serverNameOrIPAddress)
if (serverId == None):
raise Exception("The virtual machine server is not registered")
serverData = self.__dbConnector.getVMServerBasicData(serverId)
# Connect to the virtual machine server
self.__networkManager.connectTo(serverData["ServerIP"],
serverData["ServerPort"], 20,
self.__vmServerCallback, True,
True)
while not self.__networkManager.isConnectionReady(
serverData["ServerIP"], serverData["ServerPort"]):
sleep(0.1)
# Change its status
self.__dbConnector.updateVMServerStatus(serverId,
SERVER_STATE_T.BOOTING)
# Send the status request
p = self.__vmServerPacketHandler.createVMServerDataRequestPacket(
VMSRVR_PACKET_T.SERVER_STATUS_REQUEST)
self.__networkManager.sendPacket(serverData["ServerIP"],
serverData["ServerPort"], p)
# Everything went fine
p = self.__webPacketHandler.createCommandExecutedPacket(
data["CommandID"])
except Exception as e:
p = self.__webPacketHandler.createVMServerGenericErrorPacket(
WEB_PACKET_T.VM_SERVER_BOOTUP_ERROR, serverNameOrIPAddress,
str(e), data["CommandID"])
self.__networkManager.sendPacket('', self.__webPort, p)
def __sendVMServerStatusData(self):
"""
Processes a virtual machine server data request packet.
Args:
None
Returns:
Nothing
"""
self.__sendStatusData(
self.__dbConnector.getVMServerBasicData,
self.__webPacketHandler.createVMServerStatusPacket)
def __sendVMDistributionData(self):
"""
Processes a virtual machine distribution data packet
Args:
None
Returns:
Nothing
"""
self.__sendStatusData(
self.__dbConnector.getHostedImages,
self.__webPacketHandler.createVMDistributionPacket)
def __sendStatusData(self, queryMethod, packetCreationMethod):
"""
Processes a data request packet.
Args:
queryMethod: the method that extracts the data from the database
packetCreationMethod: the method that creates the packet
Returns:
Nothing
"""
segmentSize = 5
outgoingData = []
serverIDs = self.__dbConnector.getVMServerIDs()
if (len(serverIDs) == 0):
segmentCounter = 0
segmentNumber = 0
sendLastSegment = True
else:
segmentCounter = 1
segmentNumber = (len(serverIDs) / segmentSize)
if (len(serverIDs) % segmentSize != 0):
segmentNumber += 1
sendLastSegment = True
else:
sendLastSegment = False
for serverID in serverIDs:
row = queryMethod(serverID)
if (isinstance(row, dict)):
outgoingData.append(row)
else:
outgoingData += row
if (len(outgoingData) >= segmentSize):
# Flush
packet = packetCreationMethod(segmentCounter, segmentNumber,
outgoingData)
self.__networkManager.sendPacket('', self.__webPort, packet)
outgoingData = []
segmentCounter += 1
# Send the last segment
if (sendLastSegment):
packet = packetCreationMethod(segmentCounter, segmentNumber,
outgoingData)
self.__networkManager.sendPacket('', self.__webPort, packet)
def __bootUpVM(self, data):
"""
Processes a virtual machine boot request packet.
Args:
vmName: the virtual machine's ID
userID: the user's unique identifier
Returns:
Nothing
"""
vmID = data["VMID"]
userID = data["UserID"]
# Choose the virtual machine server that will host the image
(serverID, errorMessage) = self.__loadBalancer.assignVMServer(vmID)
if (errorMessage != None):
# Something went wrong => warn the user
p = self.__webPacketHandler.createVMBootFailurePacket(
vmID, errorMessage, data["CommandID"])
self.__networkManager.sendPacket('', self.__webPort, p)
else:
# Ask the virtual machine server to boot up the VM
p = self.__vmServerPacketHandler.createVMBootPacket(
vmID, userID, data["CommandID"])
serverData = self.__dbConnector.getVMServerBasicData(serverID)
self.__networkManager.sendPacket(serverData["ServerIP"],
serverData["ServerPort"], p)
# Register the boot command
self.__dbConnector.registerVMBootCommand(data["CommandID"],
data["VMID"])
# Everything went fine
#p = self.__webPacketHandler.createCommandExecutedPacket(data["CommandID"])
#self.__networkManager.sendPacket('', self.__webPort, p)
def processVMServerIncomingPacket(self, packet):
"""
Processes a packet sent from a virtual machine server.
Args:
packet: the packet to process
Returns:
Nothing
"""
data = self.__vmServerPacketHandler.readPacket(packet)
if (data["packet_type"] == VMSRVR_PACKET_T.SERVER_STATUS):
self.__updateVMServerStatus(data)
elif (data["packet_type"] == VMSRVR_PACKET_T.DOMAIN_CONNECTION_DATA):
self.__sendVMConnectionData(data)
elif (data["packet_type"] == VMSRVR_PACKET_T.ACTIVE_VM_DATA):
self.__sendVNCConnectionData(packet)
def processServerReconnectionData(self, ipAddress, reconnection_status):
"""
Processes a reconnection status event
Args:
ipAddress: the connection's IPv4 address
port: the connection's port
reconnection_status: the reconnection process' status
Returns:
Nothing
"""
if (reconnection_status == RECONNECTION_T.RECONNECTING):
status = SERVER_STATE_T.RECONNECTING
elif (reconnection_status == RECONNECTION_T.REESTABLISHED):
status = SERVER_STATE_T.READY
else:
status = SERVER_STATE_T.CONNECTION_TIMED_OUT
serverID = self.__dbConnector.getVMServerID(ipAddress)
self.__dbConnector.updateVMServerStatus(serverID, status)
def __sendVNCConnectionData(self, packet):
"""
Processes a VNC connection data packet
Args:
packet: the packet to process
Returns:
Nothing
"""
p = self.__webPacketHandler.createActiveVMsDataPacket(packet)
self.__networkManager.sendPacket('', self.__webPort, p)
def __sendVMConnectionData(self, data):
"""
Processes a virtual machine connection data packet.
Args:
data: the packet to process' data.
Returns:
Nothing
"""
# Delete the boot command from the database
self.__dbConnector.removeVMBootCommand(data["CommandID"])
p = self.__webPacketHandler.createVMConnectionDataPacket(
data["VNCServerIP"], data["VNCServerPort"],
data["VNCServerPassword"], data["CommandID"])
self.__networkManager.sendPacket('', self.__webPort, p)
def monitorVMBootCommands(self):
errorMessage = "Could not boot up virtual machine (timeout error)"
while not self.__finished:
data = self.__dbConnector.getOldVMBootCommandID(self.__timeout)
if (not self.__finished and data == None):
# Be careful with this: if MySQL server receives too many queries per second,
# the reactor's database connection will be closed.
sleep(1)
else:
# Create a virtual machine boot error packet
p = self.__webPacketHandler.createVMBootFailurePacket(
data[1], errorMessage, data[0])
self.__networkManager.sendPacket('', self.__webPort, p)
def hasFinished(self):
"""
Indicates if the cluster server has finished or not.
"""
return self.__finished
def shutdown(self):
"""
Shuts down the cluster server.
Args:
None
Returns:
Nothing
@attention: this method MUST NOT be called from a network thread. If you do so, the application will hang!
"""
self.__networkManager.stopNetworkService()
class ClusterServerMainReactor(object):
'''
This class is associated with the cluster server main reactor.
'''
def __init__(self, loadBalancerSettings, averageCompressionRatio, timeout):
"""
Initializes the reactor's state
Args:
loadBalancerSettings: a list containing the load balancing algorithm settings
averageCompressionRation: the compression algorithm's average compression ratio
timeout: the virtual machine boot commands timeout
"""
self.__exit = False
self.__loadBalancerSettings = loadBalancerSettings
self.__averageCompressionRatio = averageCompressionRatio
self.__timeout = timeout
self.__statusMonitoringThread = None
def connectToDatabase(self, mysqlRootsPassword, dbName, dbUser, dbPassword,
scriptPath):
"""
Establishes a connection with the cluster server database
Args:
mysqlRootsPassword: the MySQL root user's password
dbName: a database name
dbUser: a MySQL user name
dbPassword: the user's password
scriptPath: the schema definition script
Returns:
Nothing
"""
configurator = DBConfigurator(mysqlRootsPassword)
configurator.runSQLScript(dbName, scriptPath, "root",
mysqlRootsPassword)
configurator.addUser(dbUser, dbPassword, dbName, True)
self.__dbConnector = ClusterServerDatabaseConnector(
dbUser, dbPassword, dbName)
self.__dbConnector.initializeVMServersStatus()
def startListenning(self, useSSL, certificatePath, listenningPort,
repositoryIP, repositoryPort,
vmServerStatusUpdateInterval):
"""
Creates the control connection
Args:
useSSL: indicates if SSL encryption must be used in the control connection or not
certificatePath: the directory where the server.crt and server.key files are
listenningPort: the control connection's port
repositoryIP: the image repository IP address
repositoryPort: the image repository's port
vmServerStatusUpdateInterval: the virtual machine server status database interval
Returns:
Nothing
"""
self.__networkManager = NetworkManager(certificatePath)
self.__networkManager.startNetworkService()
self.__listenningPort = listenningPort
self.__packetHandler = ClusterServerPacketHandler(
self.__networkManager)
self.__useSSL = useSSL
imageRepositoryPacketHandler = ImageRepositoryPacketHandler(
self.__networkManager)
vmServerPacketHandler = VMServerPacketHandler(self.__networkManager)
networkEventsReactor = NetworkEventsReactor(self.__dbConnector,
repositoryIP,
repositoryPort)
imageRepositoryPacketReactor = ImageRepositoryPacketReactor(
self.__dbConnector, self.__networkManager, listenningPort,
repositoryIP, repositoryPort, self.__packetHandler,
vmServerPacketHandler, imageRepositoryPacketHandler)
try:
imageRepositoryCallback = ImageRepositoryCallback(
imageRepositoryPacketReactor, networkEventsReactor)
self.__networkManager.connectTo(repositoryIP, repositoryPort, 10,
imageRepositoryCallback,
self.__useSSL, True)
self.__dbConnector.addImageRepository(repositoryIP, repositoryPort,
SERVER_STATE_T.READY)
except Exception as e:
print "Can't connect to the image repository: " + e.message
self.__exit = True
return
vmServerPacketReactor = VMServerPacketReactor(self.__dbConnector,
self.__networkManager,
listenningPort,
vmServerPacketHandler,
self.__packetHandler)
self.__endpointPacketReactor = EndpointPacketReactor(
self.__dbConnector, self.__networkManager, vmServerPacketHandler,
self.__packetHandler, imageRepositoryPacketHandler,
VMServerCallback(vmServerPacketReactor,
networkEventsReactor), listenningPort,
repositoryIP, repositoryPort, self.__loadBalancerSettings,
self.__averageCompressionRatio, self.__useSSL)
clusterEndpointCallback = ClusterEndpointCallback(
self.__endpointPacketReactor)
self.__networkManager.listenIn(listenningPort, clusterEndpointCallback,
self.__useSSL)
self.__statusMonitoringThread = ClusterStatusMonitoringThread(
vmServerStatusUpdateInterval, self.__dbConnector,
self.__networkManager, repositoryIP, repositoryPort,
vmServerPacketHandler, imageRepositoryPacketHandler)
self.__statusMonitoringThread.start()
def monitorVMBootCommands(self):
"""
Deletes the timed out virtual machine boot commands
Args:
None
Returns:
Nothing
"""
while not self.__exit and not self.__endpointPacketReactor.hasFinished(
):
data = self.__dbConnector.getOldVMBootCommandID(self.__timeout)
if (data == None):
sleep(1)
else:
self.__dbConnector.deleteActiveVMLocation(data[0])
p = self.__packetHandler.createErrorPacket(
CLUSTER_SERVER_PACKET_T.VM_BOOT_FAILURE,
ERROR_DESC_T.CLSRVR_VM_BOOT_TIMEOUT, data[0])
self.__networkManager.sendPacket('', self.__listenningPort, p)
def closeNetworkConnections(self):
"""
Closes the control connection
Args:
None
Returns:
Nothing
"""
if (self.__statusMonitoringThread != None):
self.__statusMonitoringThread.stop()
self.__networkManager.stopNetworkService()
class ClusterServerMainReactor(object):
'''
This class is associated with the cluster server main reactor.
'''
def __init__(self, loadBalancerSettings, averageCompressionRatio, timeout):
"""
Initializes the reactor's state
Args:
loadBalancerSettings: a list containing the load balancing algorithm settings
averageCompressionRation: the compression algorithm's average compression ratio
timeout: the virtual machine boot commands timeout
"""
self.__exit = False
self.__loadBalancerSettings = loadBalancerSettings
self.__averageCompressionRatio = averageCompressionRatio
self.__timeout = timeout
self.__statusMonitoringThread = None
def connectToDatabase(self, mysqlRootsPassword, dbName, dbUser, dbPassword, scriptPath):
"""
Establishes a connection with the cluster server database
Args:
mysqlRootsPassword: the MySQL root user's password
dbName: a database name
dbUser: a MySQL user name
dbPassword: the user's password
scriptPath: the schema definition script
Returns:
Nothing
"""
configurator = DBConfigurator(mysqlRootsPassword)
configurator.runSQLScript(dbName, scriptPath, "root", mysqlRootsPassword)
configurator.addUser(dbUser, dbPassword, dbName, True)
self.__dbConnector = ClusterServerDatabaseConnector(dbUser, dbPassword, dbName)
self.__dbConnector.initializeVMServersStatus()
def startListenning(self, useSSL, certificatePath, listenningPort, repositoryIP, repositoryPort, vmServerStatusUpdateInterval):
"""
Creates the control connection
Args:
useSSL: indicates if SSL encryption must be used in the control connection or not
certificatePath: the directory where the server.crt and server.key files are
listenningPort: the control connection's port
repositoryIP: the image repository IP address
repositoryPort: the image repository's port
vmServerStatusUpdateInterval: the virtual machine server status database interval
Returns:
Nothing
"""
self.__networkManager = NetworkManager(certificatePath)
self.__networkManager.startNetworkService()
self.__listenningPort = listenningPort
self.__packetHandler = ClusterServerPacketHandler(self.__networkManager)
self.__useSSL = useSSL
imageRepositoryPacketHandler = ImageRepositoryPacketHandler(self.__networkManager)
vmServerPacketHandler = VMServerPacketHandler(self.__networkManager)
networkEventsReactor = NetworkEventsReactor(self.__dbConnector, repositoryIP, repositoryPort)
imageRepositoryPacketReactor = ImageRepositoryPacketReactor(self.__dbConnector, self.__networkManager,
listenningPort, repositoryIP, repositoryPort,
self.__packetHandler, vmServerPacketHandler, imageRepositoryPacketHandler)
try :
imageRepositoryCallback = ImageRepositoryCallback(imageRepositoryPacketReactor, networkEventsReactor)
self.__networkManager.connectTo(repositoryIP, repositoryPort, 10, imageRepositoryCallback, self.__useSSL, True)
self.__dbConnector.addImageRepository(repositoryIP, repositoryPort, SERVER_STATE_T.READY)
except Exception as e:
print "Can't connect to the image repository: " + e.message
self.__exit = True
return
vmServerPacketReactor = VMServerPacketReactor(self.__dbConnector, self.__networkManager, listenningPort,
vmServerPacketHandler, self.__packetHandler)
self.__endpointPacketReactor = EndpointPacketReactor(self.__dbConnector, self.__networkManager,
vmServerPacketHandler, self.__packetHandler,
imageRepositoryPacketHandler,
VMServerCallback(vmServerPacketReactor, networkEventsReactor),
listenningPort, repositoryIP, repositoryPort, self.__loadBalancerSettings,
self.__averageCompressionRatio, self.__useSSL)
clusterEndpointCallback = ClusterEndpointCallback(self.__endpointPacketReactor)
self.__networkManager.listenIn(listenningPort, clusterEndpointCallback, self.__useSSL)
self.__statusMonitoringThread = ClusterStatusMonitoringThread(vmServerStatusUpdateInterval,
self.__dbConnector, self.__networkManager,
repositoryIP, repositoryPort,
vmServerPacketHandler, imageRepositoryPacketHandler)
self.__statusMonitoringThread.start()
def monitorVMBootCommands(self):
"""
Deletes the timed out virtual machine boot commands
Args:
None
Returns:
Nothing
"""
while not self.__exit and not self.__endpointPacketReactor.hasFinished():
data = self.__dbConnector.getOldVMBootCommandID(self.__timeout)
if (data == None) :
sleep(1)
else :
self.__dbConnector.deleteActiveVMLocation(data[0])
p = self.__packetHandler.createErrorPacket(CLUSTER_SERVER_PACKET_T.VM_BOOT_FAILURE, ERROR_DESC_T.CLSRVR_VM_BOOT_TIMEOUT, data[0])
self.__networkManager.sendPacket('', self.__listenningPort, p)
def closeNetworkConnections(self):
"""
Closes the control connection
Args:
None
Returns:
Nothing
"""
if (self.__statusMonitoringThread != None) :
self.__statusMonitoringThread.stop()
self.__networkManager.stopNetworkService()
class ImageRepositoryReactor(object):
"""
Image repository packet reactor
"""
def __init__(self, diskImagesDirectory):
"""
Initializes the reactor's state
Args:
diskImagesDirectory: the FTP server's root directory
"""
self.__diskImagesDirectory = diskImagesDirectory
self.__slotCounter = MultithreadingCounter()
self.__retrieveQueue = GenericThreadSafeList()
self.__storeQueue = GenericThreadSafeList()
self.__finish = False
self.__networkManager = None
self.__ftpServer = None
def connectToDatabase(self, repositoryDBName, repositoryDBUser, repositoryDBPassword):
"""
Establishes the connection with the image repository's database.
Args:
repositoryDBName: a database name
repositoryDBUser: an user name
repositoryDBPassword: a password
Returns:
Nothing
"""
self.__dbConnector = ImageRepositoryDBConnector(repositoryDBUser, repositoryDBPassword, repositoryDBName)
def startListenning(self, networkInterface, useSSL, certificatesDirectory, commandsListenningPort, ftpListenningPort, maxConnections,
maxConnectionsPerIP, uploadBandwidthRatio, downloadBandwidthRatio, ftpUsername, ftpPasswordLength):
"""
Boots up the FTP server and creates the control connection.
Args:
networkInterface: the network interface that will be used by the FTP server
useSSL: indicates if SSL encryption must be used in the control connection or not
certificatesDirectory: the directory where the files server.crt and server.key are
commandsListenningPort: the control connection's port
ftpListenningPort: the FTP server listenning port
maxConnections: maximum FTP connections
maxConnectionsPerIP: maximum FTP connections per IP address
uploadBandwidthRatio: maximum download bandwidth fraction
downloadBandwidthRatio: maximum upload bandwidth fraction
ftpUsername: the FTP user that the virtual machine servers will use
ftpPasswordLength: the random FTP password length
Returns:
Nothing
@attention: The FTP password will be randomly generated at every boot.
"""
try :
self.__maxConnections = maxConnections
self.__commandsListenningPort = commandsListenningPort
self.__FTPListenningPort = ftpListenningPort
self.__networkManager = NetworkManager(certificatesDirectory)
self.__repositoryPacketHandler = ImageRepositoryPacketHandler(self.__networkManager)
self.__commandsCallback = CommandsCallback(self.__networkManager, self.__repositoryPacketHandler, commandsListenningPort, self.__dbConnector,
self.__retrieveQueue, self.__storeQueue, self.__diskImagesDirectory)
self.__networkManager.startNetworkService()
self.__networkManager.listenIn(commandsListenningPort, self.__commandsCallback, useSSL)
dataCallback = FTPServerCallback(self.__slotCounter, self.__dbConnector)
self.__ftpUsername = ftpUsername
self.__ftpPassword = ChildProcessManager.runCommandInForeground("openssl rand -base64 {0}".format(ftpPasswordLength), Exception)
self.__ftpServer = ConfigurableFTPServer("Image repository FTP Server")
self.__ftpServer.startListenning(networkInterface, ftpListenningPort, maxConnections, maxConnectionsPerIP,
dataCallback, downloadBandwidthRatio, uploadBandwidthRatio)
self.__ftpServer.addUser(self.__ftpUsername, self.__ftpPassword, self.__diskImagesDirectory, "eramw")
except Exception as e:
print "Error: " + e.message
self.__finish = True
def stopListenning(self):
"""
Stops the FTP server and closes the control connection.
Args:
None
Returns:
Nothing
"""
if (self.__ftpServer != None) :
try :
self.__ftpServer.stopListenning()
except Exception :
pass
if (self.__networkManager != None) :
self.__networkManager.stopNetworkService()
def initTransfers(self):
"""
Initializes the upload and download transfers
Args:
None
Returns:
Nothing
"""
store = False
while not (self.__finish or self.__commandsCallback.haltReceived()):
if (self.__slotCounter.read() == self.__maxConnections) :
# No free slots => sleep
sleep(0.1)
else :
# There are slots => enable uploads and downloads in parallel
self.__slotCounter.decrement()
if (self.__retrieveQueue.isEmpty() and self.__storeQueue.isEmpty()) :
sleep(0.1)
continue
if (not self.__retrieveQueue.isEmpty() and self.__storeQueue.isEmpty()) :
queue = self.__retrieveQueue
store = False
elif (self.__retrieveQueue.isEmpty() and not self.__storeQueue.isEmpty()) :
queue = self.__storeQueue
store = True
else :
if (store) :
queue = self.__retrieveQueue
store = False
else :
queue = self.__storeQueue
store = True
(imageID, clientIP, clientPort) = queue.pop(0)
imageData = self.__dbConnector.getImageData(imageID)
if (imageData == None) :
if (store) :
packet_type = PACKET_T.STOR_ERROR
else :
packet_type = PACKET_T.RETR_ERROR
p = self.__repositoryPacketHandler.createErrorPacket(packet_type, ERROR_DESC_T.IR_IMAGE_DELETED)
self.__networkManager.sendPacket('', self.__commandsListenningPort, p, clientIP, clientPort)
else :
compressedFilePath = imageData["compressedFilePath"]
if (not "undefined" in compressedFilePath) :
serverDirectory = path.relpath(path.dirname(compressedFilePath), self.__diskImagesDirectory)
compressedFileName = path.basename(compressedFilePath)
else :
serverDirectory = str(imageID)
compressedFileName = ""
serverDirectoryPath = path.join(self.__diskImagesDirectory, serverDirectory)
if (path.exists(serverDirectoryPath)) :
# The directory exists, and can store shit => clean it up!
ChildProcessManager.runCommandInForeground("rm -rf " + serverDirectoryPath, Exception)
mkdir(serverDirectoryPath)
if (store) :
packet_type = PACKET_T.STOR_START
else :
packet_type = PACKET_T.RETR_START
p = self.__repositoryPacketHandler.createTransferEnabledPacket(packet_type, imageID, self.__FTPListenningPort,
self.__ftpUsername, self.__ftpPassword, serverDirectory, compressedFileName)
self.__networkManager.sendPacket('', self.__commandsListenningPort, p, clientIP, clientPort)
