def __init__(self): super(IMRegisterServerMoab, self).__init__() self.numparams = 5 # prefix,name,os,arch,machine self.prefix = "" self.name = "" self.operatingsystem = "" self.arch = "" self.machine = "" # load from config file self._registerConf = IMServerConf() self._registerConf.load_registerServerMoabConfig() self.port = self._registerConf.getMoabPort() self.moabInstallPath = self._registerConf.getMoabInstallPath() self.log_filename = self._registerConf.getLogMoab() # self.timeToRestartMoab = self._registerConf.getTimeToRestartMoab() #time that we wait to get the moab scheduler restarted (mschedctl -R) self.logLevel = self._registerConf.getLogLevelMoab() self._ca_certs = self._registerConf.getCaCertsMoab() self._certfile = self._registerConf.getCertFileMoab() self._keyfile = self._registerConf.getKeyFileMoab() print "\nReading Configuration file from " + self._registerConf.getConfigFile() + "\n" self.logger = self.setup_logger()
def __init__(self): super(IMRegisterServerIaaS, self).__init__() self.path = "" self.numparams = 9 #image path self.name = "" self.givenname = "" self.operatingsystem = "" self.version = "" self.arch = "" self.kernel = "" self.user = "" self.iaas = "" #load from config file self._registerConf = IMServerConf() self._registerConf.load_registerServerIaasConfig() self.port = self._registerConf.getIaasPort() self.http_server = self._registerConf.getHttpServerIaas() self.proc_max = self._registerConf.getProcMaxIaas() self.refresh_status = self._registerConf.getRefreshStatusIaas() self._nopasswdusers = self._registerConf.getNoPasswdUsersIaas() self.tempdir = self._registerConf.getTempDirIaas() self.log_filename = self._registerConf.getLogIaas() self.logLevel = self._registerConf.getLogLevelIaas() self._ca_certs = self._registerConf.getCaCertsIaas() self._certfile = self._registerConf.getCertFileIaas() self._keyfile = self._registerConf.getKeyFileIaas() print "\nReading Configuration file from " + self._registerConf.getConfigFile() + "\n" self.logger = self.setup_logger("") #Image repository Object verbose = False printLogStdout = False self._reposervice = IRServiceProxy(verbose, printLogStdout)
def __init__(self): super(IMGenerateServer, self).__init__() #********************* #Static Configuration. #********************* #this is to login in the VM. This MUST be root because IMGenerateScript needs this access. self.rootId = 'root' self.numparams = 13 #this is the user that requested the image self.user = "" self.os = "" self.version = "" self.arch = "" self.software = "" self.givenname = "" self.desc = "" self.getimg = False self.baseimage=False self.nocache=False self.size=1.5 #load configuration self._genConf = IMServerConf() self._genConf.load_generateServerConfig() self.port = self._genConf.getGenPort() self.proc_max = self._genConf.getProcMax() self.refresh_status = self._genConf.getRefreshStatus() self.wait_max = self._genConf.getWaitMax() self._nopasswdusers = self._genConf.getNoPasswdUsersGen() self.vmfile_centos = self._genConf.getVmFileCentos() self.vmfile_rhel = self._genConf.getVmFileRhel() self.vmfile_ubuntu = self._genConf.getVmFileUbuntu() self.vmfile_debian = self._genConf.getVmFileDebian() self.xmlrpcserver = self._genConf.getXmlRpcServer() self.bridge = self._genConf.getBridge() #self.serverdir = self._genConf.getServerDir() #if self.serverdir == None: self.serverdir = os.path.expanduser(os.path.dirname(__file__)) #Location of IMGenerateScript.py file, which should be the same that IMGenerateServer.py self.addrnfs = self._genConf.getAddrNfs() self.tempdirserver = self._genConf.getTempDirServerGen() self.tempdir = self._genConf.getTempDirGen() self.http_server = self._genConf.getHttpServerGen() #self.bcfg2_url = self._genConf.getBcfg2Url() #self.bcfg2_port = self._genConf.getBcgf2Port() self.oneauth = self._genConf.getOneUser() + ":" + self._genConf.getOnePass() self.log_filename = self._genConf.getLogGen() self.logLevel = self._genConf.getLogLevelGen() self.logger = self.setup_logger() self._ca_certs = self._genConf.getCaCertsGen() self._certfile = self._genConf.getCertFileGen() self._keyfile = self._genConf.getKeyFileGen() print "\nReading Configuration file from " + self._genConf.getConfigFile() + "\n" #Image repository Object verbose=False printLogStdout=False self._reposervice = IRServiceProxy(verbose,printLogStdout)
class IMGenerateServer(object): def __init__(self): super(IMGenerateServer, self).__init__() #********************* #Static Configuration. #********************* #this is to login in the VM. This MUST be root because IMGenerateScript needs this access. self.rootId = 'root' self.numparams = 13 #this is the user that requested the image self.user = "" self.os = "" self.version = "" self.arch = "" self.software = "" self.givenname = "" self.desc = "" self.getimg = False self.baseimage=False self.nocache=False self.size=1.5 #load configuration self._genConf = IMServerConf() self._genConf.load_generateServerConfig() self.port = self._genConf.getGenPort() self.proc_max = self._genConf.getProcMax() self.refresh_status = self._genConf.getRefreshStatus() self.wait_max = self._genConf.getWaitMax() self._nopasswdusers = self._genConf.getNoPasswdUsersGen() self.vmfile_centos = self._genConf.getVmFileCentos() self.vmfile_rhel = self._genConf.getVmFileRhel() self.vmfile_ubuntu = self._genConf.getVmFileUbuntu() self.vmfile_debian = self._genConf.getVmFileDebian() self.xmlrpcserver = self._genConf.getXmlRpcServer() self.bridge = self._genConf.getBridge() #self.serverdir = self._genConf.getServerDir() #if self.serverdir == None: self.serverdir = os.path.expanduser(os.path.dirname(__file__)) #Location of IMGenerateScript.py file, which should be the same that IMGenerateServer.py self.addrnfs = self._genConf.getAddrNfs() self.tempdirserver = self._genConf.getTempDirServerGen() self.tempdir = self._genConf.getTempDirGen() self.http_server = self._genConf.getHttpServerGen() #self.bcfg2_url = self._genConf.getBcfg2Url() #self.bcfg2_port = self._genConf.getBcgf2Port() self.oneauth = self._genConf.getOneUser() + ":" + self._genConf.getOnePass() self.log_filename = self._genConf.getLogGen() self.logLevel = self._genConf.getLogLevelGen() self.logger = self.setup_logger() self._ca_certs = self._genConf.getCaCertsGen() self._certfile = self._genConf.getCertFileGen() self._keyfile = self._genConf.getKeyFileGen() print "\nReading Configuration file from " + self._genConf.getConfigFile() + "\n" #Image repository Object verbose=False printLogStdout=False self._reposervice = IRServiceProxy(verbose,printLogStdout) def setup_logger(self): #Setup logging logger = logging.getLogger("GenerateServer") logger.setLevel(self.logLevel) formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") handler = logging.FileHandler(self.log_filename) handler.setLevel(self.logLevel) handler.setFormatter(formatter) logger.addHandler(handler) logger.propagate = False #Do not propagate to others return logger """ def get_adminpass(self, oneadmin): ############## #GET oneadmin password encoded in SHA1 ############## p = Popen('oneuser list', stdout=PIPE, shell=True) p1 = Popen('grep ' + oneadmin, stdin=p.stdout, stdout=PIPE, shell=True) p2 = Popen('cut -d\" \" -f13', stdin=p1.stdout, shell=True, stdout=PIPE) oneadminpass = p2.stdout.read().strip() return oneadmin + ":" + oneadminpass """ def start(self): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.bind(('', self.port)) sock.listen(1) #Maximum of system unaccepted connections. Maximum value depend of the system (usually 5) self.logger.info('Starting Server on port ' + str(self.port)) proc_list = [] total_count = 0 while True: if len(proc_list) == self.proc_max: full = True while full: for i in range(len(proc_list) - 1, -1, -1): #self.logger.debug(str(proc_list[i])) if not proc_list[i].is_alive(): #print "dead" proc_list.pop(i) full = False if full: time.sleep(self.refresh_status) total_count += 1 #channel, details = sock.accept() newsocket, fromaddr = sock.accept() connstream = 0 try: connstream = ssl.wrap_socket(newsocket, server_side=True, ca_certs=self._ca_certs, cert_reqs=ssl.CERT_REQUIRED, certfile=self._certfile, keyfile=self._keyfile, ssl_version=ssl.PROTOCOL_TLSv1) #print connstream proc_list.append(Process(target=self.generate, args=(connstream,fromaddr[0],))) proc_list[len(proc_list) - 1].start() except ssl.SSLError: self.logger.error("Unsuccessful connection attempt from: " + repr(fromaddr)) self.logger.info("Image Generation Request DONE") except socket.error: self.logger.error("Error with the socket connection") self.logger.info("Image Generation Request DONE") except: self.logger.error("Uncontrolled Error: " + str(sys.exc_info())) try: connstream.shutdown(socket.SHUT_RDWR) connstream.close() except: pass self.logger.info("Image Generation Request DONE") def auth(self, userCred): return FGAuth.auth(self.user, userCred) def checkUserStatus(self, userId, passwd, userIdB): """ return "Active", "NoActive", "NoUser"; also False in case the connection with the repo fails """ if not self._reposervice.connection(): msg = "ERROR: Connection with the Image Repository failed" self.logger.error(msg) return False else: self.logger.debug("Checking User Status") status= self._reposervice.getUserStatus(userId, passwd, userIdB) self._reposervice.disconnect() return status def checknopasswd(self, fromaddr): status = False if self.user in self._nopasswdusers: if fromaddr in self._nopasswdusers[self.user]: status = True return status def generate(self, channel, fromaddr): #this runs in a different proccess start_all = time.time() self.logger = logging.getLogger("GenerateServer." + str(os.getpid())) self.logger.info('Processing an image generation request') #receive the message data = channel.read(2048) self.logger.debug("received data: " + data) params = data.split('|') #params[0] is user #params[1] is operating system #params[2] is version #params[3] is arch #params[4] is software #params[5] is givenname #params[6] is the description #params[7] is to retrieve the image or to upload in the repo (true or false, respectively) #params[8] is the user password #params[9] is the type of password #params[10] is to generate only a base image #params[11] is to do not use a cached based image #params[12] is the size of the image. 1.5 GB minimum self.user = params[0].strip() self.os = params[1].strip() self.version = params[2].strip() self.arch = params[3].strip() self.software = params[4].strip() self.givenname = params[5].strip() self.desc = params[6].strip() self.getimg = eval(params[7].strip()) #boolean passwd = params[8].strip() passwdtype = params[9].strip() self.baseimage=eval(params[10].strip()) #boolean self.nocache=eval(params[11].strip()) #boolean self.size=eval(params[12].strip()) #double if len(params) != self.numparams: msg = "ERROR: incorrect message" self.errormsg(channel, msg) #break return retry = 0 maxretry = 3 endloop = False while (not endloop): if not self.checknopasswd(fromaddr): userCred = FGCredential(passwdtype, passwd) if self.auth(userCred): #check the status of the user in the image repository. #This contacts with image repository client to check its db. The user an password are OK because this was already checked. userstatus=self.checkUserStatus(self.user, passwd, self.user) if userstatus == "Active": channel.write("OK") elif userstatus == "NoActive": channel.write("NoActive") msg = "ERROR: The user " + self.user + " is not active" self.errormsg(channel, msg) return elif userstatus == "NoUser": channel.write("NoUser") msg = "ERROR: The user " + self.user + " does not exist" self.logger.error(msg) self.logger.info("Image Generation Request DONE") return else: channel.write("Could not connect with image repository server") msg = "ERROR: Could not connect with image repository server to verify the user status" self.logger.error(msg) self.logger.info("Image Generation Request DONE") return endloop = True else: retry += 1 if retry < maxretry: channel.write("TryAuthAgain") passwd = channel.read(2048) else: msg = "ERROR: authentication failed" endloop = True self.errormsg(channel, msg) return else: channel.write("OK") endloop = True baseimageuri=None if not self.nocache and not self.baseimage: #find a base image in the repository start = time.time() baseimageuri=self.findBaseImage(passwd) end = time.time() self.logger.info('TIME to retrieve and uncompress a base image from the repo:' + str(end - start)) if baseimageuri==None: self.genInVM(channel, passwd) else: self.genLocal(channel,passwd, baseimageuri) end_all = time.time() self.logger.info('TIME walltime image generate:' + str(end_all - start_all)) self.logger.info("Image Generation DONE") def genLocal(self, channel, passwd, baseimageuri): options = "-a " + self.arch + " -o " + self.os + " -v " + self.version + " -u " + self.user + " -t " + self.tempdirserver + " --size " + str(self.size) if type(self.givenname) is not NoneType: options += " -n " + self.givenname if type(self.desc) is not NoneType: options += " -e " + self.desc if type(self.software) is not NoneType: options += " -s " + self.software if baseimageuri != None: options += " -i " + baseimageuri if self.baseimage: options += " -b " options += " -c " + self.http_server + " -l " + self.tempdirserver + "/"+ str(os.getpid()) +"_gen_cache.log" cmdexec = '/usr/bin/python ' + self.serverdir + "/IMGenerateScript.py " + options #cmdexec = self.serverdir + "/image/management/IMGenerateScript.py " + options self.logger.info(cmdexec) start = time.time() cmd = "sudo " + cmdexec self.logger.debug(cmd) p = Popen(cmd.split(' '), stdout=PIPE, stderr=PIPE) std = p.communicate() end = time.time() self.logger.info('TIME generate image:' + str(end - start)) if p.returncode != 0: self.logger.error('Command: ' + cmd + ' failed, status: ' + str(p.returncode) + ' --- ' + std[1]) status = std[0].strip() #it contains error or filename self.logger.debug("status returned generate script|"+status+"|") if os.path.isfile(self.tempdirserver + "/" + status + ".img") and os.path.isfile(self.tempdirserver + "/" + status + ".manifest.xml"): self.compressUploadSend(status, channel, passwd) else: msg = "ERROR: " + str(std[1]) self.errormsg(channel, msg) def genInVM(self, channel, passwd): vmfile = "" vmID = 0 destroyed = False if self.os == "ubuntu": vmfile = self.vmfile_ubuntu elif self.os == "debian": vmfile = self.vmfile_debian elif self.os == "rhel": vmfile = self.vmfile_rhel elif self.os == "centos": vmfile = self.vmfile_centos[self.version] # ---Start xmlrpc client to opennebula server------------- try: server = xmlrpclib.ServerProxy(self.xmlrpcserver) except: self.logger.error("Error connection with OpenNebula " + str(sys.exc_info())) #print "Error connecting with OpenNebula " + str(sys.exc_info()) return ########### #BOOT VM## ########## start = time.time() output = self.boot_VM(server, vmfile) end = time.time() self.logger.info('TIME boot VM:' + str(end - start)) vmaddr = output[0] vmID = output[1] ##### if vmaddr != "fail": self.logger.info("The VM deployed is in " + vmaddr) self.logger.info("Mount scratch directory in the VM") cmd = "ssh -q -oBatchMode=yes " + self.rootId + "@" + vmaddr cmdmount = " mount -t nfs " + self.addrnfs + ":" + self.tempdirserver + " " + self.tempdir self.logger.info(cmd + cmdmount) stat = os.system(cmd + cmdmount) if (stat == 0): self.logger.info("Sending IMGenerateScript.py to the VM") cmdscp = "scp -q -oBatchMode=yes " + self.serverdir + '/IMGenerateScript.py ' + self.rootId + "@" + vmaddr + ":/root/" #cmdscp = "scp -q -oBatchMode=yes " + self.serverdir + '/image/management/IMGenerateScript.py ' + self.rootId + "@" + vmaddr + ":/root/" self.logger.info(cmdscp) stat = os.system(cmdscp) if (stat != 0): msg = "ERROR: sending IMGenerateScript.py to the VM. Exit status " + str(stat) self.errormsg(channel, msg) else: options = "-a " + self.arch + " -o " + self.os + " -v " + self.version + " -u " + self.user + " -t " + self.tempdir + " --size " + str(self.size) if type(self.givenname) is not NoneType: options += " -n " + self.givenname if type(self.desc) is not NoneType: options += " -e " + self.desc if type(self.software) is not NoneType: options += " -s " + self.software if self.baseimage: options += " -b " options += " -c " + self.http_server #+ " -b " + self.bcfg2_url + " -p " + str(self.bcfg2_port) cmdexec = " -q '/usr/bin/python /root/IMGenerateScript.py " + options + " '" self.logger.info(cmdexec) start = time.time() uid = self._rExec(self.rootId, cmdexec, vmaddr) self.logger.debug("Returned from script execution" +str(uid)) end = time.time() self.logger.info('TIME generate image:' + str(end - start)) self.logger.info("copying fg-image-generate.log to scrach partition " + self.tempdirserver + "/" + str(vmID) + "_gen.log") cmdscp = "scp -q -oBatchMode=yes " + self.rootId + "@" + vmaddr + ":/root/fg-image-generate.log " + self.tempdirserver + "/" + str(vmID) + "_gen.log" os.system(cmdscp) status = uid[0].strip() #it contains error or filename if "error" in uid: msg = "ERROR: " + str(uid[1]) self.errormsg(channel, msg) else: #stat = 0 #while stat != 0 and : self.logger.info("Umount scratch directory in the VM") cmd = "ssh -q -oBatchMode=yes " + self.rootId + "@" + vmaddr cmdmount = " umount " + self.tempdir + " 2>/dev/null" #stat = os.system(cmd + cmdmount) #self.logger.debug("exit status " + str(stat)) #if stat != 0: # time.sleep(2) start = time.time() #umount the image max_retry = 15 retry_done = 0 umounted = False #Done making changes to root fs while not umounted: self.logger.debug(cmd + cmdmount) stat = os.system(cmd + cmdmount) if stat == 0: umounted = True elif retry_done == max_retry: self.logger.debug("exit status " + str(stat)) umounted = True self.logger.error("Problems to umount the image. Exit status " + str(stat)) else: retry_done += 1 time.sleep(5) end = time.time() self.logger.info('TIME umount image:' + str(end - start)) #destroy VM self.logger.info("Destroy VM") server.one.vm.action(self.oneauth, "finalize", vmID) destroyed = True self.compressUploadSend(status, channel, passwd) else: msg = "ERROR: booting VM" self.errormsg(channel, msg) #destroy VM if not destroyed and vmID != -1: self.logger.info("Destroy VM") try: server.one.vm.action(self.oneauth, "finalize", vmID) except: msg = "ERROR: finalizing VM" self.errormsg(channel, msg) def compressUploadSend(self, status, channel, passwd): self.logger.debug("Generating tgz with image and manifest files") self.logger.debug("tar cfz " + self.tempdirserver + "/" + status + ".tgz -C " + self.tempdirserver + \ " " + status + ".manifest.xml " + status + ".img") start = time.time() out = os.system("tar cfz " + self.tempdirserver + "/" + status + ".tgz -C " + self.tempdirserver + \ " " + status + ".manifest.xml " + status + ".img") end = time.time() self.logger.info('TIME tgz image:' + str(end - start)) os.system("rm -f " + self.tempdirserver + "" + status + ".manifest.xml " + self.tempdirserver + \ "" + status + ".img") if out != 0: msg = "ERROR: generating compressed file with the image and manifest" self.errormsg(channel, msg) return if self.getimg: #send back the url where the image is channel.write(self.tempdirserver + "" + status + ".tgz") self.logger.info("Waiting until the client retrieve the image") channel.read() #we can include a loop to retry if the client has problems getting the image channel.shutdown(socket.SHUT_RDWR) channel.close() else: status_repo = "" error_repo = False #send back the ID of the image in the repository try: #connect with the server if not self._reposervice.connection(): msg = "ERROR: Connection with the Image Repository failed" self.errormsg(channel, msg) else: self.logger.info("Storing image " + self.tempdirserver + "/" + status + ".tgz" + " in the repository") start = time.time() if self.baseimage: status_repo = self._reposervice.put(self.user, passwd, self.user, self.tempdirserver + "" + status + ".tgz", "os=" + \ self.os + "_" + self.version + "&arch=" + self.arch + "&description=" + \ self.desc + "&tag=" + status +",BaseImage") else: status_repo = self._reposervice.put(self.user, passwd, self.user, self.tempdirserver + "" + status + ".tgz", "os=" + \ self.os + "_" + self.version + "&arch=" + self.arch + "&description=" + \ self.desc + "&tag=" + status) end = time.time() self.logger.info('TIME upload image to the repo:' + str(end - start)) success=True if (re.search('^ERROR', status_repo)): self.errormsg(channel, status_repo) else: channel.write(str(status_repo)) channel.shutdown(socket.SHUT_RDWR) channel.close() self._reposervice.disconnect() except: msg = "ERROR: uploading image to the repository. " + str(sys.exc_info()) self.errormsg(channel, msg) #Remove file from server because the client or the repository already finished os.system("rm -f " + self.tempdirserver + "" + status + ".tgz") def findBaseImage(self, passwd): baseimageuri=None try: #connect with the server if not self._reposervice.connection(): msg = "ERROR: Connection with the Image Repository failed" self.logger.error(msg) else: self.logger.info("Find Base Image os=" + self.os + ",version=" +self.version +",arch="+self.arch + " in the repository") imgsList=None imgsList = self._reposervice.query(self.user, passwd, self.user, "* where os=" + \ self.os + "_" + self.version + ",arch=" + self.arch + ",tag=BaseImage,imgStatus=available") self._reposervice.disconnect() #the server disconnect even without this. #if there are images, I try to find one that is owned by me. if not I take the first one. If I own several, I take the first one baseImageId="" if(imgsList != None): try: imgs = eval(imgsList) for key in imgs.keys(): imageproperties=imgs[key].split(", ") tempbaseImageId=imageproperties[0].split("=")[1] #imgid if imageproperties[3].split("=")[1].strip() == self.user: #owner baseImageId=tempbaseImageId break elif imageproperties[8].split("=")[1].strip() == 'public': #permission if baseImageId=="": baseImageId=tempbaseImageId except: self.logger.error("findbaseimage:Server replied: " + str(imgsList)) self.logger.error("findbaseimage: Error interpreting the list of images from Image Repository" + str(sys.exc_info())) if baseImageId != "":#retrieve the image if not self._reposervice.connection(): msg = "ERROR: Connection with the Image Repository failed" self.logger.error(msg) else: #create a random dir to download image random.seed() exists=True while exists: randid = str(random.getrandbits(64)) path = self.tempdirserver + "/" + randid if not os.path.isdir(path): exists=False os.system("mkdir " + path) baseimageuri = self._reposervice.get(self.user, passwd, self.user, "img", baseImageId, path) self._reposervice.disconnect() #the server disconnect even without this. else: self.logger.debug("No Base Image found") else: self.logger.info("No Base Image found") except: msg = "ERROR: searching for a base image in the repository. " + str(sys.exc_info()) self.logger.error(msg) #extract the image. If we do that from the VM is slower due to NFS. if baseimageuri != None: #change the URI to the one that the VM is going to see baseimageuri = self.process_baseimage(baseimageuri, randid) self.logger.debug("Inside the VM, the image will be in: "+ str(baseimageuri)) return baseimageuri def process_baseimage(self, baseimageuri, randid): pathimg=None realnameimg = "" self.logger.info('untar file with image and manifest') cmd = "tar xvfz " + baseimageuri + " -C " + os.path.dirname(baseimageuri) self.logger.debug(cmd) p = Popen(cmd.split(' '), stdout=PIPE, stderr=PIPE) std = p.communicate() stat = 0 if len(std[0]) > 0: realnameimg = std[0].split("\n")[0].strip().split(".")[0] if p.returncode != 0: self.logger.error('Command: ' + cmd + ' failed, status: ' + str(p.returncode) + ' --- ' + std[1]) stat = 1 cmd = 'rm -f ' + baseimageuri os.system(cmd) if (stat != 0): msg = "ERROR: Extracting Base Image: " +str(sys.exc_info()) self.logger.error(msg) cmd = "rm -rf " + os.path.basename(baseimageuri) os.system(cmd) else: pathimg=self.tempdirserver+"/"+randid+"/"+realnameimg + ".img" return pathimg def errormsg(self, channel, msg): self.logger.error(msg) try: channel.write(msg) channel.shutdown(socket.SHUT_RDWR) channel.close() except: self.logger.debug("In errormsg: " + str(sys.exc_info())) self.logger.info("Image Generation DONE") def boot_VM(self, server, vmfile): """ It will boot a VM using XMLRPC API for OpenNebula from lib/ruby/OpenNebula/VirtualMachine.rb index start in 0 VM_STATE=%w{INIT PENDING HOLD ACTIVE STOPPED SUSPENDED DONE FAILED} LCM_STATE=%w{LCM_INIT PROLOG BOOT RUNNING MIGRATE SAVE_STOP SAVE_SUSPEND SAVE_MIGRATE PROLOG_MIGRATE PROLOG_RESUME EPILOG_STOP EPILOG SHUTDOWN CANCEL FAILURE CLEANUP UNKNOWN} """ vmaddr = "" fail = False #print vmfile #-----read template into string ------------------------- #s=open('./share/examples/ubuntu_context.one','r').read() s = open(os.path.expanduser(vmfile), 'r').read() #self.logger.debug("Vm template:\n"+s) #-----Start VM------------------------------------------- try: vm = server.one.vm.allocate(self.oneauth, s) except: msg = "ERROR: Trying to allocate a VM" self.logger.error(msg) return ["fail", -1] # -1 dont try to finalize VM because it did not started #print self.oneauth #print vm if vm[0]: self.logger.debug("VM ID: " + str(vm[1])) #monitor VM booted = False maxretry = self.wait_max / 5 #time that the VM has to change from penn to runn retry = 0 while not booted and retry < maxretry: #eventually the VM has to boot or fail try: #-------Get Info about VM ------------------------------- vminfo = server.one.vm.info(self.oneauth, vm[1]) #print vminfo[1] manifest = parseString(vminfo[1]) #VM_status (init=0, pend=1, act=3, fail=7) vm_status = manifest.getElementsByTagName('STATE')[0].firstChild.nodeValue.strip() if vm_status == "3": #running #LCM_status (prol=1,boot=2,runn=3, fail=14, unk=16) lcm_status = manifest.getElementsByTagName('LCM_STATE')[0].firstChild.nodeValue.strip() if lcm_status == "3": #if vm_status is 3, this will be 3 too. booted = True elif vm_status == "7": #fail self.logger.error("Fail to deploy VM " + str(vm[1])) booted = True fail = True vmaddr = "fail" elif vm_status == "6": #done self.logger.error("The status of the VM " + str(vm[1]) + " is DONE") booted = True fail = True vmaddr = "fail" else: retry += 1 time.sleep(5) except: pass if retry >= maxretry: self.logger.error("The VM " + str(vm[1]) + " did not change to runn status. Please verify that the status of the OpenNebula hosts " "or increase the wait time in the configuration file (max_wait) \n") vmaddr = "fail" fail = True if not fail: #get IP nics = manifest.getElementsByTagName('NIC') for i in range(len(nics)): if(nics[i].childNodes[0].firstChild.nodeValue.strip() == self.bridge): vmaddr = nics[i].childNodes[1].firstChild.nodeValue.strip() if vmaddr.strip() != "": self.logger.debug("IP of the VM " + str(vm[1]) + " is " + str(vmaddr)) access = False maxretry = 240 #this says that we wait 20 minutes maximum to allow the VM get online. #this also prevent to get here forever if the ssh key was not injected propertly. retry = 0 self.logger.debug("Waiting to have access to VM") while not access and retry < maxretry: cmd = "ssh -q -oBatchMode=yes root@" + vmaddr + " uname" p = Popen(cmd, shell=True, stdout=PIPE) status = os.waitpid(p.pid, 0)[1] #print status if status == 0: access = True self.logger.debug("The VM " + str(vm[1]) + " with ip " + str(vmaddr) + "is accessible") else: retry += 1 time.sleep(5) if retry >= maxretry: self.logger.error("Could not get access to the VM " + str(vm[1]) + " with ip " + str(vmaddr) + "\n" "Please verify the OpenNebula templates to make sure that the public ssh key to be injected is accessible to the oneadmin user. \n" "Also verify that the VM has ssh server and is active on boot.") vmaddr = "fail" else: self.logger.error("Could not determine the IP of the VM " + str(vm[1]) + " for the bridge " + self.bridge) vmaddr = "fail" else: vmaddr = "fail" return [vmaddr, vm[1]] ############################################################ # _rExec ############################################################ def _rExec(self, userId, cmdexec, vmaddr): #TODO: do we want to use the .format statement from python to make code more readable? #Set up random string random.seed() randid = str(random.getrandbits(32)) cmdssh = "ssh -oBatchMode=yes " + userId + "@" + vmaddr tmpFile = "/tmp/" + str(time.time()) + str(randid) #print tmpFile cmdexec = cmdexec + " > " + tmpFile cmd = cmdssh + cmdexec self.logger.info(str(cmd)) stat = os.system(cmd) if (str(stat) != "0"): #print stat self.logger.info(str(stat)) f = open(tmpFile, "r") outputs = f.readlines() f.close() os.system("rm -f " + tmpFile) return outputs
class IMRegisterServerMoab(object): def __init__(self): super(IMRegisterServerMoab, self).__init__() self.numparams = 5 # prefix,name,os,arch,machine self.prefix = "" self.name = "" self.operatingsystem = "" self.arch = "" self.machine = "" # load from config file self._registerConf = IMServerConf() self._registerConf.load_registerServerMoabConfig() self.port = self._registerConf.getMoabPort() self.moabInstallPath = self._registerConf.getMoabInstallPath() self.log_filename = self._registerConf.getLogMoab() # self.timeToRestartMoab = self._registerConf.getTimeToRestartMoab() #time that we wait to get the moab scheduler restarted (mschedctl -R) self.logLevel = self._registerConf.getLogLevelMoab() self._ca_certs = self._registerConf.getCaCertsMoab() self._certfile = self._registerConf.getCertFileMoab() self._keyfile = self._registerConf.getKeyFileMoab() print "\nReading Configuration file from " + self._registerConf.getConfigFile() + "\n" self.logger = self.setup_logger() def setup_logger(self): # Setup logging logger = logging.getLogger("RegisterMoab") logger.setLevel(self.logLevel) formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") handler = logging.FileHandler(self.log_filename) handler.setLevel(self.logLevel) handler.setFormatter(formatter) logger.addHandler(handler) logger.propagate = False # Do not propagate to others return logger def start(self): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.bind(("", self.port)) sock.listen(1) self.logger.info("Starting Server on port " + str(self.port)) while True: newsocket, fromaddr = sock.accept() connstream = 0 try: connstream = ssl.wrap_socket( newsocket, server_side=True, ca_certs=self._ca_certs, cert_reqs=ssl.CERT_REQUIRED, certfile=self._certfile, keyfile=self._keyfile, ssl_version=ssl.PROTOCOL_TLSv1, ) self.process_client(connstream) except ssl.SSLError: self.logger.error("Unsuccessful connection attempt from: " + repr(fromaddr)) finally: if connstream is ssl.SSLSocket: connstream.shutdown(socket.SHUT_RDWR) connstream.close() def process_client(self, connstream): self.logger.info("Accepted new connection") # receive the message data = connstream.read(2048) params = data.split(",") # params[0] is prefix # params[1] is name # params[2] is operating system # params[3] is arch # params[4] is machine (india, minicluster..) self.prefix = params[0] self.name = params[1] self.operatingsystem = params[2] self.arch = params[3] self.machine = params[4] if len(params) != self.numparams: msg = "ERROR: incorrect message" self.errormsg(connstream, msg) return if self.prefix == "list": moabimageslist = "" f = open(self.moabInstallPath + "/tools/msm/images.txt") for i in f: if not re.search("^#", i): moabimageslist += i.split()[0] + "," moabimageslist = moabimageslist.rstrip(",") self.logger.debug("Moab image list: " + moabimageslist) connstream.write(moabimageslist) connstream.shutdown(socket.SHUT_RDWR) connstream.close() self.logger.info("Image Register Moab (list) DONE") elif self.prefix == "infosites": self.logger.debug("Information Moab Site: " + str(self.machine)) connstream.write("True") connstream.shutdown(socket.SHUT_RDWR) connstream.close() self.logger.info("Image Register Moab (info sites) DONE") elif self.prefix == "remove": self.logger.debug("Remove image from Moab") cmd = "sudo sed -i '/^" + self.name + "/d' /opt/moab/tools/msm/images.txt" self.logger.info(cmd) status = os.system(cmd) if status != 0: msg = "ERROR: removing image from image.txt file" self.logger.debug(msg) self.errormsg(connstream, msg) return else: connstream.write("OK") connstream.shutdown(socket.SHUT_RDWR) connstream.close() cmd = "sudo " + self.moabInstallPath + "/bin/mschedctl -R" self.runCmd(cmd) self.logger.info("Image Register Moab DONE") else: moabstring = "" if self.machine == "minicluster": moabstring = ( 'echo "' + self.prefix + self.operatingsystem + "" + self.name + " " + self.arch + " " + self.prefix + self.operatingsystem + "" + self.name + ' compute netboot" | sudo tee -a ' + self.moabInstallPath + "/tools/msm/images.txt > /dev/null" ) # moabstring = 'echo \"' + prefix + operatingsystem + '' + name + ' ' + arch + ' boottarget ' + prefix + operatingsystem + '' + name + ' netboot\" >> ' + moabInstallPath + '/tools/msm/images.txt' elif self.machine == "india": moabstring = ( 'echo "' + self.prefix + self.operatingsystem + "" + self.name + " " + self.arch + " boottarget " + self.prefix + self.operatingsystem + "" + self.name + ' netboot" | sudo tee -a ' + self.moabInstallPath + "/tools/msm/images.txt > /dev/null" ) # This message inster the line in the images.txt file self.logger.debug(moabstring) status = os.system(moabstring) if status != 0: msg = "ERROR: including image name in image.txt file" self.logger.debug(msg) self.errormsg(connstream, msg) return else: connstream.write("OK") connstream.shutdown(socket.SHUT_RDWR) connstream.close() cmd = "sudo " + self.moabInstallPath + "/bin/mschedctl -R" status = self.runCmd(cmd) self.logger.info("Image Register Moab DONE") """ if not os.path.isfile('/tmp/image-register-fork.lock'): os.system('touch /tmp/image-register-fork.lock') child_pid = os.fork() if child_pid == 0: self.logger.debug("Child Process: PID# %s" % os.getpid()) time.sleep(self.timeToRestartMoab) cmd = 'mschedctl -R' status = self.runCmd(cmd) os.system('rm -f /tmp/image-register-fork.lock') else: self.logger.debug("Parent Process: PID# %s" % os.getpid()) """ def errormsg(self, connstream, msg): self.logger.error(msg) try: connstream.write(msg) connstream.shutdown(socket.SHUT_RDWR) connstream.close() except: self.logger.debug("In errormsg: " + str(sys.exc_info())) self.logger.info("Image Register Moab DONE") def runCmd(self, cmd): cmdLog = logging.getLogger("RegisterMoab.exec") cmdLog.debug(cmd) p = Popen(cmd.split(" "), stdout=PIPE, stderr=PIPE) std = p.communicate() status = 0 if len(std[0]) > 0: cmdLog.debug("stdout: " + std[0]) cmdLog.debug("stderr: " + std[1]) if p.returncode != 0: cmdLog.error("Command: " + cmd + " failed, status: " + str(p.returncode) + " --- " + std[1]) status = 1 # sys.exit(p.returncode) return status
class IMRegisterServerIaaS(object): def __init__(self): super(IMRegisterServerIaaS, self).__init__() self.path = "" self.numparams = 9 #image path self.name = "" self.givenname = "" self.operatingsystem = "" self.version = "" self.arch = "" self.kernel = "" self.user = "" self.iaas = "" #load from config file self._registerConf = IMServerConf() self._registerConf.load_registerServerIaasConfig() self.port = self._registerConf.getIaasPort() self.http_server = self._registerConf.getHttpServerIaas() self.proc_max = self._registerConf.getProcMaxIaas() self.refresh_status = self._registerConf.getRefreshStatusIaas() self._nopasswdusers = self._registerConf.getNoPasswdUsersIaas() self.tempdir = self._registerConf.getTempDirIaas() self.log_filename = self._registerConf.getLogIaas() self.logLevel = self._registerConf.getLogLevelIaas() self._ca_certs = self._registerConf.getCaCertsIaas() self._certfile = self._registerConf.getCertFileIaas() self._keyfile = self._registerConf.getKeyFileIaas() print "\nReading Configuration file from " + self._registerConf.getConfigFile() + "\n" self.logger = self.setup_logger("") #Image repository Object verbose = False printLogStdout = False self._reposervice = IRServiceProxy(verbose, printLogStdout) def setup_logger(self, extra): #Setup logging logger = logging.getLogger("RegisterIaaS" + extra) logger.setLevel(self.logLevel) formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") handler = logging.FileHandler(self.log_filename) handler.setLevel(self.logLevel) handler.setFormatter(formatter) logger.addHandler(handler) logger.propagate = False #Do not propagate to others return logger def start(self): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.bind(('', self.port)) sock.listen(1) self.logger.info('Starting Server on port ' + str(self.port)) proc_list = [] total_count = 0 while True: if len(proc_list) == self.proc_max: full = True while full: for i in range(len(proc_list) - 1, -1, -1): #self.logger.debug(str(proc_list[i])) if not proc_list[i].is_alive(): #print "dead" proc_list.pop(i) full = False if full: time.sleep(self.refresh_status) total_count += 1 #channel, details = sock.accept() newsocket, fromaddr = sock.accept() connstream = 0 try: connstream = ssl.wrap_socket(newsocket, server_side=True, ca_certs=self._ca_certs, cert_reqs=ssl.CERT_REQUIRED, certfile=self._certfile, keyfile=self._keyfile, ssl_version=ssl.PROTOCOL_TLSv1) #print connstream proc_list.append(Process(target=self.process_client, args=(connstream, fromaddr[0]))) proc_list[len(proc_list) - 1].start() except ssl.SSLError: self.logger.error("Unsuccessful connection attempt from: " + repr(fromaddr)) self.logger.info("IaaS register server Request DONE") except socket.error: self.logger.error("Error with the socket connection") self.logger.info("IaaS register server Request DONE") except: self.logger.error("Uncontrolled Error: " + str(sys.exc_info())) if type(connstream) is ssl.SSLSocket: connstream.shutdown(socket.SHUT_RDWR) connstream.close() self.logger.info("IaaS register server Request DONE") def auth(self, userCred): return FGAuth.auth(self.user, userCred) def checkUserStatus(self, userId, passwd, userIdB): """ return "Active", "NoActive", "NoUser"; also False in case the connection with the repo fails """ if not self._reposervice.connection(): msg = "ERROR: Connection with the Image Repository failed" self.logger.error(msg) return False else: self.logger.debug("Checking User Status") status = self._reposervice.getUserStatus(userId, passwd, userIdB) self._reposervice.disconnect() return status def checknopasswd(self, fromaddr): status = False if self.user in self._nopasswdusers: if fromaddr in self._nopasswdusers[self.user]: status = True return status def checkKernel(self): status = False if (self.iaas == "euca"): status = self.kernel in self._euca_auth_kernels elif (self.iaas == "nimbus"): status = self.kernel in self._nimbus_auth_kernels elif (self.iaas == "openstack"): status = self.kernel in self._openstack_auth_kernels elif (self.iaas == "opennebula"): status = self.kernel in self._opennebula_auth_kernels return status def checkIaasAvail(self): status = False if (self.iaas == "euca"): if not self.default_euca_kernel == "": status = True elif (self.iaas == "nimbus"): if not self.default_nimbus_kernel == "": status = True elif (self.iaas == "openstack"): if not self.default_openstack_kernel == "": status = True elif (self.iaas == "opennebula"): if not self.default_opennebula_kernel == "": status = True return status def loadIaasConfig(self, iaasSite): self._registerConf.loadIaasSiteConfig(iaasSite) self.default_euca_kernel = self._registerConf.getDefaultEucaKernel() self.default_nimbus_kernel = self._registerConf.getDefaultNimbusKernel() self.default_openstack_kernel = self._registerConf.getDefaultOpenstackKernel() self.default_opennebula_kernel = self._registerConf.getDefaultOpennebulaKernel() self._euca_auth_kernels = self._registerConf.getEucaAuthKernels() self._nimbus_auth_kernels = self._registerConf.getNimbusAuthKernels() self._openstack_auth_kernels = self._registerConf.getOpenstackAuthKernels() self._opennebula_auth_kernels = self._registerConf.getOpennebulaAuthKernels() def process_client(self, connstream, fromaddr): start_all = time.time() self.logger = self.setup_logger("." + str(os.getpid())) self.logger.info('Accepted new connection') #receive the message data = connstream.read(2048) self.logger.debug("msg received: " + data) params = data.split(',') #print data #params[0] is image ID or image path or "kernels" #params[1] is the source of the image (repo,disk). #params[2] is the machine name #params[3] is the iaas cloud #params[4] is the kernel #params[5] is the user #params[6] is the user password #params[7] is the type of password #params[8] is the ldap configure or not imgID = params[0].strip() imgSource = params[1].strip() machinename = params[2].strip() self.iaas = params[3].strip() self.kernel = params[4].strip() self.user = params[5].strip() passwd = params[6].strip() passwdtype = params[7].strip() ldap = False try: ldap = eval(params[8].strip()) except: self.logger.warning("Ldap configure set to False in except") ldap = False if len(params) != self.numparams: msg = "ERROR: incorrect message" self.errormsg(connstream, msg) return retry = 0 maxretry = 3 endloop = False while (not endloop): if not self.checknopasswd(fromaddr): userCred = FGCredential(passwdtype, passwd) if (self.auth(userCred)):# contact directly with LDAP #check the status of the user in the image repository. #This contacts with image repository client to check its db. The user an password are OK because this was already checked. userstatus = self.checkUserStatus(self.user, passwd, self.user) if userstatus == "Active": connstream.write("OK") elif userstatus == "NoActive": connstream.write("NoActive") msg = "ERROR: The user " + self.user + " is not active" self.errormsg(connstream, msg) return elif userstatus == "NoUser": connstream.write("NoUser") msg = "ERROR: The user " + self.user + " does not exist" self.logger.error(msg) self.logger.info("IaaS register server Request DONE") return else: connstream.write("Could not connect with image repository server") msg = "ERROR: Could not connect with image repository server to verify the user status" self.logger.error(msg) self.logger.info("IaaS register server Request DONE") return endloop = True else: retry += 1 if retry < maxretry: connstream.write("TryAuthAgain") passwd = connstream.read(2048) else: msg = "ERROR: authentication failed" endloop = True self.errormsg(connstream, msg) return else: connstream.write("OK") endloop = True if imgID == "infosites": infosites=self._registerConf.listIaasSites() self.logger.debug("Information Cloud Sites: " + str(infosites)) connstream.write(str(infosites)) connstream.shutdown(socket.SHUT_RDWR) connstream.close() self.logger.info("Image Register Request (info sites) DONE") return #description and infrastructures without kernel details else: #Load Configuration of the Site. This indicates the IaaS infrastructures available in the site and kernel configurations. output=self.loadIaasConfig(machinename) if output == "ERROR": msg = "ERROR: The specified site " + machinename + ". Please use the option --listsites to list available sites and its services. \n" self.errormsg(connstream, msg) return if imgID == "kernels": kernelslist = {} if (self.iaas == "euca"): kernelslist["Default"] = self.default_euca_kernel kernelslist["Authorized"] = self._euca_auth_kernels elif (self.iaas == "nimbus"): kernelslist["Default"] = self.default_nimbus_kernel kernelslist["Authorized"] = self._nimbus_auth_kernels elif (self.iaas == "openstack"): kernelslist["Default"] = self.default_openstack_kernel kernelslist["Authorized"] = self._openstack_auth_kernels elif (self.iaas == "opennebula"): kernelslist["Default"] = self.default_opennebula_kernel kernelslist["Authorized"] = self._opennebula_auth_kernels self.logger.debug(self.iaas + " default kernels list: " + str(kernelslist["Default"])) self.logger.debug(self.iaas + " kernels list: " + str(kernelslist["Authorized"])) connstream.write(str(kernelslist)) connstream.shutdown(socket.SHUT_RDWR) connstream.close() self.logger.info("Image Register Request (kernel list " + self.iaas + ") DONE") return #check if the infrastructure required is available elif not self.checkIaasAvail(): msg = "ERROR: The specified infrastructure " + self.iaas + " is not available on " + machinename + ". Please use the option --listservices to list available sites and its services. \n" self.errormsg(connstream, msg) return #verify kernel is authorized if self.kernel != "None": if not self.checkKernel(): msg = "ERROR: The specified kernel ("+self.kernel+") is not available. Authorized kernels for " + self.iaas + " are: " + str(eval("self._" + self.iaas + "_auth_kernels")) self.errormsg(connstream, msg) return else: self.logger.debug("The kernel " + self.kernel + " is valid") #create a unique directory auxdir = str(randrange(999999999999999999999999)) localtempdir = self.tempdir + "/" + auxdir + "_0" while os.path.isdir(localtempdir): auxdir = str(randrange(999999999999999999999999)) localtempdir = self.tempdir + "/" + auxdir + "_0" cmd = 'mkdir -p ' + localtempdir self.runCmd(cmd) self.runCmd("chmod 777 " + localtempdir) start = time.time() if imgSource == "repo": #GET IMAGE from repo if not self._reposervice.connection(): msg = "ERROR: Connection with the Image Repository failed" self.errormsg(connstream, msg) return else: self.logger.info("Retrieving image from repository") image = self._reposervice.get(self.user, passwd, self.user, "img", imgID, localtempdir) if image == None: msg = "ERROR: Cannot get access to the image with imgId " + str(imgID) self.errormsg(connstream, msg) self._reposervice.disconnect() self.runCmd("rm -rf " + localtempdir) return else: self._reposervice.disconnect() else: connstream.write(localtempdir) status = connstream.read(1024) status = status.split(',') if len(status) == 2: image = localtempdir + '/' + status[1].strip() if status[0].strip() != 'OK': msg = "ERROR: Receiving image from client: " + str(status) self.errormsg(connstream, msg) return else: msg = "ERROR: Message received from client is incorrect: " + str(status) self.errormsg(connstream, msg) return end = time.time() self.logger.info('TIME retrieve image from repo or client:' + str(end - start)) if not os.path.isfile(image): msg = "ERROR: file " + image + " not found" self.errormsg(connstream, msg) return start = time.time() #extracts image/manifest, read manifest if not self.handle_image(image, localtempdir, connstream): return end = time.time() self.logger.info('TIME untar image: ' + str(end - start)) #self.preprocess() start = time.time() stat = 0 if (self.iaas == "euca"): stat = self.euca_method(localtempdir, ldap) elif (self.iaas == "nimbus"): stat = self.nimbus_method(localtempdir, ldap) elif (self.iaas == "opennebula"): stat = self.opennebula_method(localtempdir, ldap) elif (self.iaas == "openstack"): stat = self.openstack_method(localtempdir, ldap) end = time.time() self.logger.info('TIME customize image for specific IaaS framework:' + str(end - start)) start = time.time() #umount the image max_retry = 5 retry_done = 0 umounted = False #Done making changes to root fs while not umounted: status = self.runCmd('sudo umount ' + localtempdir + '/temp') if status == 0: umounted = True elif retry_done == max_retry: umounted = True self.logger.error("Problems to umount the image") else: retry_done += 1 time.sleep(2) end = time.time() self.logger.info('TIME umount image:' + str(end - start)) status = self.runCmd("mv -f " + localtempdir + '/' + self.name + '.img ' + localtempdir + '/' + self.operatingsystem + self.version + self.name + '.img') kerneltouse=eval("self._" + self.iaas + "_auth_kernels") eki=kerneltouse[self.kernel][0] eri=kerneltouse[self.kernel][1] connstream.write(localtempdir + '/' + self.operatingsystem + self.version + self.name + '.img,' + eki + "," + eri + "," + self.operatingsystem) start = time.time() #wait until client retrieve img self.logger.info("Wait until client get the image") connstream.read() end = time.time() self.logger.info('TIME wait until client get image:' + str(end - start)) #remove image cmd = 'rm -rf ' + localtempdir status = self.runCmd(cmd) try: connstream.shutdown(socket.SHUT_RDWR) connstream.close() except: self.logger.error("ERROR: " + str(sys.exc_info())) end_all = time.time() self.logger.info('TIME walltime image register IaaS:' + str(end_all - start_all)) self.logger.info("Image Register Request DONE") def configure_ldap(self, localtempdir): start = time.time() if self.operatingsystem == "centos": self.runCmd('sudo chroot ' + localtempdir + '/temp/ yum -y install fuse-sshfs') self.logger.info('Installing LDAP packages') if (self.version == "5"): self.runCmd('sudo chroot ' + localtempdir + '/temp/ yum -y install openldap-clients nss_ldap') self.runCmd('sudo wget ' + self.http_server + '/ldap/nsswitch.conf -O ' + localtempdir + '/temp/etc/nsswitch.conf') elif (self.version == "6"): self.runCmd('sudo chroot ' + localtempdir + '/temp/ yum -y install openldap-clients nss-pam-ldapd sssd') self.runCmd('sudo wget ' + self.http_server + '/ldap/nsswitch.conf_centos6 -O ' + localtempdir + '/temp/etc/nsswitch.conf') self.runCmd('sudo wget ' + self.http_server + '/ldap/sssd.conf_centos6 -O ' + localtempdir + '/temp/etc/sssd/sssd.conf') self.runCmd('sudo chmod 600 ' + localtempdir + '/temp/etc/sssd/sssd.conf') self.runCmd('sudo chroot ' + localtempdir + '/temp/ chkconfig sssd on') self.logger.info('Configuring LDAP access') self.runCmd('sudo mkdir -p ' + localtempdir + '/temp/etc/openldap/cacerts ' + localtempdir + '/temp/N/u') self.runCmd('sudo wget ' + self.http_server + '/ldap/cacerts/12d3b66a.0 -O ' + localtempdir + '/temp/etc/openldap/cacerts/12d3b66a.0') self.runCmd('sudo wget ' + self.http_server + '/ldap/cacerts/cacert.pem -O ' + localtempdir + '/temp/etc/openldap/cacerts/cacert.pem') self.runCmd('sudo wget ' + self.http_server + '/ldap/ldap.conf -O ' + localtempdir + '/temp/etc/ldap.conf') self.runCmd('sudo wget ' + self.http_server + '/ldap/openldap/ldap.conf -O ' + localtempdir + '/temp/etc/openldap/ldap.conf') os.system('sudo sed -i \'s/enforcing/disabled/g\' ' + localtempdir + '/temp/etc/selinux/config') #self.runCmd('sudo wget ' + self.http_server + '/ldap/sshd_centos' + self.version + ' -O ' + localtempdir + '/temp/usr/sbin/sshd') #os.system('echo "UseLPK yes" | sudo tee -a ' + localtempdir + '/temp/etc/ssh/sshd_config > /dev/null') #os.system('echo "LpkLdapConf /etc/ldap.conf" | sudo tee -a ' + localtempdir + '/temp/etc/ssh/sshd_config > /dev/null') #self.runCmd('sudo chroot ' + localtempdir + '/temp/ yum -y install fuse-sshfs') elif self.operatingsystem == "ubuntu": #services will install, but not start f = open(localtempdir + '/_policy-rc.d', 'w') f.write("#!/bin/sh" + '\n' + "exit 101" + '\n') f.close() self.runCmd('sudo mv -f ' + localtempdir + '/_policy-rc.d ' + localtempdir + '/temp/usr/sbin/policy-rc.d') self.runCmd('sudo chmod +x ' + localtempdir + '/temp/usr/sbin/policy-rc.d') self.runCmd('sudo chroot ' + localtempdir + '/temp/ apt-get -y install sshfs') #try this other way #chroot maverick-vm /bin/bash -c 'DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes install linux-image-server' #env DEBIAN_FRONTEND="noninteractive" chroot /tmp/javi3789716749 /bin/bash -c 'apt-get --force-yes -y install ldap-utils libpam-ldap libpam-ldap libnss-ldap nss-updatedb libnss-db' self.logger.info('Configuring LDAP access') self.runCmd('sudo wget ' + self.http_server + '/ldap/nsswitch.conf -O ' + localtempdir + '/temp/etc/nsswitch.conf') self.runCmd('sudo mkdir -p ' + localtempdir + '/temp/etc/ldap/cacerts ' + localtempdir + '/temp/N/u') self.runCmd('sudo wget ' + self.http_server + '/ldap/cacerts/12d3b66a.0 -O ' + localtempdir + '/temp/etc/ldap/cacerts/12d3b66a.0') self.runCmd('sudo wget ' + self.http_server + '/ldap/cacerts/cacert.pem -O ' + localtempdir + '/temp/etc/ldap/cacerts/cacert.pem') self.runCmd('sudo wget ' + self.http_server + '/ldap/ldap.conf -O ' + localtempdir + '/temp/etc/ldap.conf') self.runCmd('sudo wget ' + self.http_server + '/ldap/openldap/ldap.conf -O ' + localtempdir + '/temp/etc/ldap/ldap.conf') os.system('sudo sed -i \'s/openldap/ldap/g\' ' + localtempdir + '/temp/etc/ldap/ldap.conf') os.system('sudo sed -i \'s/openldap/ldap/g\' ' + localtempdir + '/temp/etc/ldap.conf') self.logger.info('Installing LDAP packages') f = open(localtempdir + '/_ldap.install', 'w') f.write("#!/bin/bash" + '\n' + "export DEBIAN_FRONTEND=noninteractive" + '\n' + 'apt-get ' + \ '-y install ldap-utils libnss-ldapd nss-updatedb libnss-db') f.close() self.runCmd("sudo mv -f " + localtempdir + '/_ldap.install ' + localtempdir + '/temp/tmp/ldap.install') os.system('sudo chmod +x ' + localtempdir + '/temp/tmp/ldap.install') self.runCmd('sudo chroot ' + localtempdir + '/temp /tmp/ldap.install') #I think this is not needed #self.runCmd('wget '+ self.http_server +'/ldap/sshd_ubuntu -O ' + localtempdir + '/temp/usr/sbin/sshd') #os.system('echo "UseLPK yes" | sudo tee -a ' + localtempdir + '/temp/etc/ssh/sshd_config > /dev/null') #os.system('echo "LpkLdapConf /etc/ldap.conf" | sudo tee -a ' + localtempdir + '/temp/etc/ssh/sshd_config > /dev/null') self.runCmd('sudo mv -f ' + localtempdir + '/temp/usr/sbin/policy-rc.d ' + localtempdir + '/_policy-rc.d') self.runCmd('rm -f ' + localtempdir + '/_policy-rc.d') end = time.time() self.logger.info('TIME configure LDAP (this is included in the TIME customize image for specific IaaS framework):' + str(end - start)) def euca_method(self, localtempdir, ldap): stat = 0 #Select kernel version #This is not yet supported as we get always the same kernel self.logger.debug("kernel: " + self.kernel) if self.kernel == "None": self.kernel = self.default_euca_kernel #Inject the kernel self.logger.info('Retrieving kernel ' + self.kernel) stat = self.runCmd('wget ' + self.http_server + 'kernel/' + self.kernel + '.modules.tar.gz -O ' + localtempdir + '/' + self.kernel + '.modules.tar.gz') if stat == 0: self.runCmd('sudo tar xfz ' + localtempdir + '/' + self.kernel + '.modules.tar.gz --directory ' + localtempdir + '/temp/lib/modules/') self.logger.info('Injected kernel ' + self.kernel) # Setup fstab fstab = ''' # Default fstab /dev/sda1 / ext3 defaults,errors=remount-ro 0 0 /dev/sda2 /tmp ext3 defaults 0 0 /dev/sda3 swap swap defaults 0 0 proc /proc proc defaults 0 0 devpts /dev/pts devpts gid=5,mode=620 0 0 ''' f = open(localtempdir + '/fstab', 'w') f.write(fstab) f.close() self.runCmd('sudo mv -f ' + localtempdir + '/fstab ' + localtempdir + '/temp/etc/fstab') self.runCmd('sudo chown root:root ' + localtempdir + '/temp/etc/fstab') self.logger.info('fstab Injected') if self.operatingsystem == "centos": os.system('sudo sed -i \'s/enforcing/disabled/g\' ' + localtempdir + '/temp/etc/selinux/config') if ldap: self.configure_ldap(localtempdir) return stat def nimbus_method(self, localtempdir, ldap): #Select kernel version #This is not yet supported as we get always the same kernel stat = 0 self.logger.debug("kernel: " + self.kernel) if self.kernel == "None": self.kernel = self.default_nimbus_kernel #Inject the kernel self.logger.info('Retrieving kernel ' + self.kernel) stat = self.runCmd('wget ' + self.http_server + 'kernel/' + self.kernel + '.modules.tar.gz -O ' + localtempdir + '/' + self.kernel + '.modules.tar.gz') if stat == 0: self.runCmd('sudo tar xfz ' + localtempdir + '/' + self.kernel + '.modules.tar.gz --directory ' + localtempdir + '/temp/lib/modules/') self.logger.info('Injected kernel ' + self.kernel) # Setup fstab fstab = ''' # Default fstab /dev/sda1 / ext3 defaults,errors=remount-ro 0 0 proc /proc proc defaults 0 0 devpts /dev/pts devpts gid=5,mode=620 0 0 ''' f = open(localtempdir + '/fstab', 'w') f.write(fstab) f.close() self.runCmd('sudo mv -f ' + localtempdir + '/fstab ' + localtempdir + '/temp/etc/fstab') self.runCmd('sudo chown root:root ' + localtempdir + '/temp/etc/fstab') self.logger.info('fstab Injected') if self.operatingsystem == "centos": os.system('sudo sed -i \'s/enforcing/disabled/g\' ' + localtempdir + '/temp/etc/selinux/config') os.system('sudo mkdir -p ' + localtempdir + "/temp/root/.ssh") if ldap: self.configure_ldap(localtempdir) return stat def openstack_method(self, localtempdir, ldap): #Select kernel version #This is not yet supported as we get always the same kernel self.logger.debug("kernel: " + self.kernel) if self.kernel == "None": self.kernel = self.default_openstack_kernel #Inject the kernel self.logger.info('Retrieving kernel ' + self.kernel) self.runCmd('wget ' + self.http_server + 'kernel/' + self.kernel + '.modules.tar.gz -O ' + localtempdir + '/' + self.kernel + '.modules.tar.gz') self.runCmd('sudo tar xfz ' + localtempdir + '/' + self.kernel + '.modules.tar.gz --directory ' + localtempdir + '/temp/lib/modules/') self.logger.info('Injected kernel ' + self.kernel) # Setup fstab fstab = ''' # Default fstab /dev/sda1 / ext3 defaults,errors=remount-ro 0 0 proc /proc proc defaults 0 0 devpts /dev/pts devpts gid=5,mode=620 0 0 ''' #install sshfs #we need to mount only home directory of user using sshfs. The mount an directory creation can be done before executing the job. we need to inject ssh pub/priv keys f = open(localtempdir + '/fstab', 'w') f.write(fstab) f.close() self.runCmd('sudo mv -f ' + localtempdir + '/fstab ' + localtempdir + '/temp/etc/fstab') self.runCmd('sudo chown root:root ' + localtempdir + '/temp/etc/fstab') self.logger.info('fstab Injected') #install curl just in case if self.operatingsystem == "ubuntu": self.runCmd('sudo chroot ' + localtempdir + '/temp/ apt-get -y install curl cloud-init') cloud_cfg = ''' cloud_type: auto user: root disable_root: 0 preserve_hostname: False ''' f = open(localtempdir + '/cloud.cfg', 'w') f.write(cloud_cfg) f.close() self.runCmd('sudo mv -f ' + localtempdir + '/cloud.cfg ' + localtempdir + '/temp/etc/cloud/cloud.cfg') self.runCmd('sudo chown root:root ' + localtempdir + '/temp/etc/cloud/cloud.cfg') elif self.operatingsystem == "centos": #customize rc.local rc_local = ''' route del -net 169.254.0.0 netmask 255.255.0.0 dev eth0 # load pci hotplug for dynamic disk attach in KVM (for EBS) depmod -a modprobe acpiphp # simple attempt to get the user ssh key using the meta-data service mkdir -p /root/.ssh echo >> /root/.ssh/authorized_keys curl -m 10 -s http://169.254.169.254/latest/meta-data/public-keys/0/openssh-key | grep 'ssh-rsa' >> /root/.ssh/authorized_keys echo "AUTHORIZED_KEYS:" echo "************************" cat /root/.ssh/authorized_keys echo "************************" ''' f_org = open(localtempdir + '/temp/etc/rc.local', 'r') f = open(localtempdir + '/rc.local', 'w') write_remain = False for line in f_org: if (re.search('^#', line) or write_remain): f.write(line) else: f.write(rc_local) write_remain = True f.close() f_org.close() self.runCmd('sudo mv -f ' + localtempdir + '/rc.local ' + localtempdir + '/temp/etc/rc.local') self.runCmd('sudo chroot ' + localtempdir + '/temp/ cp -f /etc/rc.local /etc/rc3.d/../') self.runCmd('sudo chown root:root ' + localtempdir + '/temp/etc/rc.local') self.runCmd('sudo chmod 755 ' + localtempdir + '/temp/etc/rc.local') cmd = 'echo "NOZEROCONF=yes" | sudo tee -a ' + localtempdir + '/temp/etc/sysconfig/network > /dev/null' self.logger.debug(cmd) os.system(cmd) self.runCmd('sudo chroot ' + localtempdir + '/temp/ yum -y install curl') if ldap: self.configure_ldap(localtempdir) def opennebula_method(self, localtempdir, ldap): #Select kernel version #This is not yet supported as we get always the same kernel self.logger.debug("kernel: " + self.kernel) #download vmcontext.sh self.runCmd('sudo wget ' + self.http_server + "/opennebula/" + self.operatingsystem + '/vmcontext.sh -O ' + localtempdir + '/temp/etc/init.d/vmcontext.sh') self.runCmd('sudo chmod +x ' + localtempdir + '/temp/etc/init.d/vmcontext.sh') self.runCmd('sudo chown root:root ' + localtempdir + '/temp/etc/rc.local') device = "sda" rc_local = "" if self.operatingsystem == "ubuntu": #setup vmcontext.sh self.runCmd("sudo sudo chroot " + localtempdir + "/temp ln -s /etc/init.d/vmcontext.sh /etc/rc2.d/S01vmcontext.sh") device = "sda" rc_local = "mount -t iso9660 /dev/sr0 /mnt \n" #delete persisten network rules self.runCmd("sudo rm -f " + localtempdir + "/temp/etc/udev/rules.d/70-persistent-net.rules") if self.kernel == "None": self.kernel = self.default_opennebula_kernel elif self.operatingsystem == "centos": #setup vmcontext.sh self.runCmd("sudo chroot " + localtempdir + "/temp chkconfig --add vmcontext.sh") if self.version == "5": device = "hda" rc_local = "mount -t iso9660 /dev/hdc /mnt \n" elif self.version == "6": device = "sda" rc_local = "mount -t iso9660 /dev/sr0 /mnt \n" #in centos 6 is sr0 self.runCmd("sudo rm -f " + localtempdir + "/temp/etc/udev/rules.d/70-persistent-net.rules") os.system('sudo sed -i \'s/enforcing/disabled/g\' ' + localtempdir + '/temp/etc/selinux/config') if self.kernel == "None": self.kernel = self.default_opennebula_kernel #Inject the kernel self.logger.info('Retrieving kernel ' + self.kernel) self.runCmd('wget ' + self.http_server + 'kernel/' + self.kernel + '.modules.tar.gz -O ' + localtempdir + '/' + self.kernel + '.modules.tar.gz') self.runCmd('sudo tar xfz ' + localtempdir + '/' + self.kernel + '.modules.tar.gz --directory ' + localtempdir + '/temp/lib/modules/') self.logger.info('Injected kernel ' + self.kernel) #customize rc.local rc_local += "if [ -f /mnt/context.sh ]; then \n" rc_local += " . /mnt/init.sh \n" rc_local += "fi \n" rc_local += "umount /mnt \n\n" f_org = open(localtempdir + '/temp/etc/rc.local', 'r') f = open(localtempdir + '/rc.local', 'w') write_remain = False for line in f_org: if (re.search('^#', line) or write_remain): f.write(line) else: f.write(rc_local) write_remain = True f.close() f_org.close() self.runCmd('sudo mv -f ' + localtempdir + '/rc.local ' + localtempdir + '/temp/etc/rc.local') self.runCmd('sudo chown root:root ' + localtempdir + '/temp/etc/rc.local') self.runCmd('sudo chmod 755 ' + localtempdir + '/temp/etc/rc.local') #in centos /etc/rc.local is a symbolink link of /etc/rc.d/rc.local/ os.system('sudo chroot ' + localtempdir + '/temp/ cp -f /etc/rc.local /etc/rc3.d/../') # Setup fstab fstab = "# Default fstab \n " fstab += "/dev/" + device + " / ext3 defaults,errors=remount-ro 0 0 \n" fstab += "proc /proc proc defaults 0 0 \n" fstab += "devpts /dev/pts devpts gid=5,mode=620 0 0 \n" #if ldap: #this is for india # fstab+="149.165.146.145:/users /N/u nfs rw,rsize=1048576,wsize=1048576,intr,nosuid" f = open(localtempdir + '/fstab', 'w') f.write(fstab) f.close() self.runCmd('sudo mv -f ' + localtempdir + '/fstab ' + localtempdir + '/temp/etc/fstab') self.runCmd('sudo chown root:root ' + localtempdir + '/temp/etc/fstab') self.logger.info('fstab Injected') def handle_image(self, image, localtempdir, connstream): #print image success = True """ urlparts = image.split("/") #print urlparts self.logger.debug("urls parts: " + str(urlparts)) if len(urlparts) == 1: nameimg = urlparts[0].split(".")[0] elif len(urlparts) == 2: nameimg = urlparts[1].split(".")[0] else: nameimg = urlparts[len(urlparts) - 1].split(".")[0] self.logger.debug("image name " + nameimg) localtempdir = self.tempdir + "/" + nameimg + "_0" if os.path.isfile(localtempdir): exists = True i = 0 while (exists): aux = fulldestpath + "_" + i.__str__() if os.path.isfile(aux): i += 1 else: exists = False localtempdir = aux + "/" cmd = 'mkdir -p ' + localtempdir self.runCmd(cmd) """ realnameimg = "" self.logger.info('untar file with image and manifest') cmd = "tar xvfz " + image + " -C " + localtempdir self.logger.debug(cmd) p = Popen(cmd.split(' '), stdout=PIPE, stderr=PIPE) std = p.communicate() stat = 0 if len(std[0]) > 0: realnameimg = std[0].split("\n")[0].strip().split(".")[0] if p.returncode != 0: self.logger.error('Command: ' + cmd + ' failed, status: ' + str(p.returncode) + ' --- ' + std[1]) stat = 1 cmd = 'rm -f ' + image status = self.runCmd(cmd) if (stat != 0): msg = "ERROR: the files were not extracted" self.errormsg(connstream, msg) return False self.manifestname = realnameimg + ".manifest.xml" manifestfile = open(localtempdir + "/" + self.manifestname, 'r') manifest = parse(manifestfile) self.name = "" self.givenname = "" self.operatingsystem = "" self.version = "" self.arch = "" self.name = manifest.getElementsByTagName('name')[0].firstChild.nodeValue.strip() self.givenname = manifest.getElementsByTagName('givenname') self.operatingsystem = manifest.getElementsByTagName('os')[0].firstChild.nodeValue.strip() self.version = manifest.getElementsByTagName('version')[0].firstChild.nodeValue.strip() self.arch = manifest.getElementsByTagName('arch')[0].firstChild.nodeValue.strip() #kernel = manifest.getElementsByTagName('kernel')[0].firstChild.nodeValue.strip() self.logger.debug(self.name + " " + self.operatingsystem + " " + self.version + " " + self.arch) #create rootimg and temp directories cmd = 'mkdir -p ' + localtempdir + '/temp' status = self.runCmd(cmd) if status != 0: msg = "ERROR: creating temp directory inside " + localtempdir self.errormsg(connstream, msg) return False #mount image to extract files cmd = 'sudo mount -o loop ' + localtempdir + '/' + self.name + '.img ' + localtempdir + '/temp' status = self.runCmd(cmd) if status != 0: msg = "ERROR: mounting image" self.errormsg(connstream, msg) return False #Mount proc and pts #runCmd('mount -t proc proc '+localtempdir + '/temp/proc') #runCmd('mount -t devpts devpts '+localtempdir + '/temp/dev/pts') return True def errormsg(self, connstream, msg): self.logger.error(msg) try: connstream.write(msg) connstream.shutdown(socket.SHUT_RDWR) connstream.close() except: self.logger.debug("In errormsg: " + str(sys.exc_info())) self.logger.info("Image Register Request DONE") def runCmd(self, cmd): cmdLog = logging.getLogger('RegisterIaaS.' + str(os.getpid()) + '.exec') cmdLog.debug(cmd) p = Popen(cmd.split(' '), stdout=PIPE, stderr=PIPE) std = p.communicate() status = 0 if len(std[0]) > 0: cmdLog.debug('stdout: ' + std[0]) cmdLog.debug('stderr: ' + std[1]) if p.returncode != 0: cmdLog.error('Command: ' + cmd + ' failed, status: ' + str(p.returncode) + ' --- ' + std[1]) status = 1 #sys.exit(p.returncode) return status