def main(): parser = argparse.ArgumentParser(prog="fg-repo", formatter_class=argparse.RawDescriptionHelpFormatter, description="FutureGrid Image Repository Help ", epilog=textwrap.dedent(extra_help())) parser.add_argument('-u', '--user', dest='user', required=True, metavar='user', help='FutureGrid User name') parser.add_argument('-d', '--debug', dest='debug', action="store_true", help='Print logs in the screen for debug') group = parser.add_mutually_exclusive_group(required=True) group.add_argument('-q', '--list', dest='list', nargs='?', default='', metavar='AttributeString', help='Get list of images that meet the criteria.') group.add_argument('-g', '--get', dest='get', metavar='imgId', help='Get an image') group.add_argument('-p', '--put', dest='put', nargs='+', metavar=('imgFile', 'AttributeString'), help='Upload/Register an image') group.add_argument('-m', '--modify', dest='modify', nargs=2, metavar=('imgId', 'AttributeString'), help='Update image metadata') group.add_argument('-r', '--remove', dest='remove', metavar='imgId', nargs="+", help='Delete images from the Repository') group.add_argument('-s', '--setpermission', dest='setpermission', nargs=2, metavar=('imgId', 'permissionString'), help='Set Access permission') group.add_argument('--useradd', dest='useradd', metavar='userId', help='Add a new user to the repository') group.add_argument('--userdel', dest='userdel', metavar='userId', help='Delete an user from the repository') group.add_argument('--userlist', dest='userlist', action="store_true", help='List of users') group.add_argument('--setuserquota', dest='setuserquota', nargs=2, metavar=('userId', 'quotaExpresion'), help='Modify User Quota') group.add_argument('--setuserrole', dest='setuserrole', nargs=2, metavar=('userId', 'role'), help='Modify User Role') group.add_argument('--setuserstatus', dest='setuserstatus', nargs=2, metavar=('userId', 'status'), help='Modify User Status') group.add_argument('--histimg', dest='histimg', nargs='?', metavar='imgId', default='None', help='Get usage info of an Image') group.add_argument('--histuser', dest='histuser', nargs='?', metavar='userId', default='None', help='Get usage info of an User') parser.add_argument('--nopasswd', dest='nopasswd', action="store_true", default=False, help='If this option is used, the password is not requested. This is intended for systems daemons like Inca') args = parser.parse_args() #print sys.argv #print args if len(sys.argv) == 3: print "\nERROR: You need to select and additional option to indicate the operation that you want to do. \n" parser.print_help() verbose = True #This is to print the logs in the screen service = IRServiceProxy(verbose, args.debug) if args.nopasswd == False: print "Please insert the password for the user " + args.user + "" m = hashlib.md5() m.update(getpass()) passwd = m.hexdigest() else: passwd = "None" used_args = sys.argv[1:] print "Your request is in the queue to be processed" #connect with the server if not service.connection(): print "ERROR: Connection with the server failed" sys.exit(1) if ('-q' in used_args or '--list' in used_args): if (args.list == None): imgsList = service.query(args.user, passwd, args.user, "*") else: imgsList = service.query(args.user, passwd, args.user, args.list) #dict wrapped into a list, convert it first #print imgsList if(imgsList != None): try: imgs = eval(imgsList) print str(len(imgs)) + " items found" for key in imgs.keys(): print imgs[key] except: print "Server replied: " + str(imgsList) print "list: Error:" + str(sys.exc_info()) + "\n" service._log.error("list: Error interpreting the list of images from Image Repository" + str(sys.exc_info())) else: print "No list of images returned" elif ('-g' in used_args or '--get' in used_args): #if (args.get[0] == 'img' or args.get[0] == 'uri'): img1 = service.get(args.user, passwd, args.user, "img", args.get, "./") if img1: print "The image " + args.get + " is located in " + img1 else: print "Cannot get access to the image with imgId = " + args.get #else: # print "Error: First argument of -g/--get must be 'img' or 'uri'" elif ('-p' in used_args or '--put' in used_args): status = "" ok = False if (len(args.put) == 2): status = service.put(args.user, passwd, args.user, args.put[0], args.put[1]) ok = True elif (len(args.put) == 1): status = service.put(args.user, passwd, args.user, args.put[0], "") ok = True else: args.print_help() if(ok): if (re.search('^ERROR', status)): print 'The image has not been uploaded. Exit error: ' + status else: print "The image has been uploaded and registered with id " + str(status) elif ('-m' in used_args or '--modify' in used_args): success = service.updateItem(args.user, passwd, args.user, args.modify[0], args.modify[1]) if (success == "True"): print "The item was successfully updated" else: print "Error in the update. Please verify that you are the owner and the attribute string" elif ('-r' in used_args or '--remove' in used_args): output = service.remove(args.user, passwd, args.user, args.remove) if ( output == "True"): print "All images have been removed." else: print "Some images have NOT been removed. Images with imgIds= " + str(output) + " have NOT been removed. Please verify the imgIds and if you are the owner" elif('-s' in used_args or '--setpermission' in used_args): status = service.setPermission(args.user, passwd, args.user, args.setpermission[0], args.setpermission[1]) if(status == "True"): print "Permission of img " + args.setpermission[0] + " updated" else: print "The permission have not been changed. " + status #This commands only can be used by users with Admin Role. elif ('--useradd' in used_args): #args[0] is the username. It MUST be the same that the system user status = service.userAdd(args.user, passwd, args.user, args.useradd) if(status == "True"): print "User created successfully." print "Remember that you still need to activate this user (see --setuserstatus command)\n" else: print "The user has not been created. \n" + \ "Please verify that you are admin and that the username does not exist \n" elif ('--userdel' in used_args): status = service.userDel(args.user, passwd, args.user, args.userdel) if(status == "True"): print "User deleted successfully." else: print "The user has not been deleted. \n" + \ "Please verify that you are admin and that the username \"" + args.userdel + "\" exists \n" elif ('--userlist' in used_args): userList = service.userList(args.user, passwd, args.user) #print userList if(userList != None): try: imgs = eval(userList) print str(len(imgs)) + " users found" for key in imgs.keys(): print imgs[key] except: print "Server replied: " + str(userList) print "userlist: Error:" + str(sys.exc_info()[0]) + "\n" service._log.error("userlist: Error interpreting the list of users from Image Repository" + str(sys.exc_info()[0])) else: print "No list of user returned. \n" + \ "Please verify that you are admin \n" #LOOK into how to use math ops, maybe we need to put it between "" elif ('--setuserquota' in used_args): status = service.setUserQuota(args.user, passwd, args.user, args.setuserquota[0], args.setuserquota[1]) if(status == "True"): print "Quota changed successfully." else: print "The user quota has not been changed. \n" + \ "Please verify that you are admin and that the username \"" + args.setuserquota[0] + "\" exists \n" elif ('--setuserrole' in used_args): status = service.setUserRole(args.user, passwd, args.user, args.setuserrole[0], args.setuserrole[1]) if(status == "True"): print "Role changed successfully." else: print "The user role has not been changed. " + status + "\n"+\ "Please verify that you are admin and that the username \"" + args.setuserrole[0] + "\" exists \n" elif ('--setuserstatus' in used_args): status = service.setUserStatus(args.user, passwd, args.user, args.setuserstatus[0], args.setuserstatus[1]) if(status == "True"): print "Status changed successfully." else: print "The user status has not been changed. " + status + "\n"+\ "Please verify that you are admin and that the username \""+args.setuserstatus[0]+"\"exists \n" #these are again for eveyone elif ('--histimg' in used_args): if( args.histimg != None ): imgsList = service.histImg(args.user, passwd, args.user, args.histimg) else: imgsList = service.histImg(args.user, passwd, args.user, "None") if imgsList == None: print "ERROR: Not image record found" else: try: imgs = eval(imgsList) for key in imgs.keys(): print imgs[key] except: print "Server replied: " + str(imgsList) print "histimg: Error:" + str(sys.exc_info()) + "\n" service._log.error("histimg: Error interpreting the list of images from Image Repository" + str(sys.exc_info()[0])) elif('--histuser' in used_args): if( args.histuser != None): userList = service.histUser(args.user, passwd, args.user, args.histuser) else: userList = service.histUser(args.user, passwd, args.user, "None") if userList == None: print "ERROR: Not user found" else: try: users = eval(userList) for key in users.keys(): print users[key] except: print "Server replied: " + str(userList) print "histuser: Error:" + str(sys.exc_info()) + "\n" service._log.error("histuser: Error interpreting the list of users from Image Repository" + str(sys.exc_info()))
class fgShellRepo(Cmd): ############################################################ # ############################################################ def __init__(self): print "Init Repo" verbose = True printLogStdout = False self._service = IRServiceProxy(verbose, printLogStdout) ############################################################ # test ############################################################ """ @options([make_option('-q', '--quick', help='Make things fast'), make_option('-s', '--slow', type=int, help='Make things slow')]) def do_repotest(self, args, opts): arg = ''.join(args) print opts.quick print opts.slow self._log.error("SHElll test in fgshell repo") """ ############################################################ # hist img ############################################################ def do_repohistimg(self, args): args = self.getArgs(args) #connect with the server if not self._service.connection(): print "ERROR: Connection with the server failed" return if(len(args) == 1): imgsList = self._service.histImg(self.user, self.passwd, self.user, args[0]) else: imgsList = self._service.histImg(self.user, self.passwd, self.user, "None") if imgsList == None: print "ERROR: Not image record found" else: try: imgs = eval(imgsList) for key in imgs.keys(): print imgs[key] except: print "Server replied: " + str(imgsList) print "histimg: Error:" + str(sys.exc_info()) + "\n" self._log.error("do_repohistimg: Error interpreting the list of images from Image Repository" + str(sys.exc_info()[0])) def help_repohistimg(self): msg = "Image Repository histimg command: Return information about the " + \ " image historical usage. \n" self.print_man("histimg [imgId]", msg) def do_repohistuser(self, args): args = self.getArgs(args) #connect with the server if not self._service.connection(): print "ERROR: Connection with the server failed" return if(len(args) == 1): userList = self._service.histUser(self.user, self.passwd, self.user, args[0]) else: userList = self._service.histUser(self.user, self.passwd, self.user, "None") if userList == None: print "ERROR: Not user found" else: try: users = eval(userList) for key in users.keys(): print users[key] except: print "Server replied: " + str(userList) + "\n" print "histuser: Error:" + str(sys.exc_info()) + "\n" self._log.error("do_repohistuser: Error interpreting the list of users from Image Repository" + str(sys.exc_info())) def help_repohistuser(self): msg = "Image Repository histuser command: Return information about the " + \ "user historical usage." self.print_man("histuser [userId]", msg) ################### #user ################### def do_repouser(self, args): argslist = args.split("-")[1:] prefix = '' sys.argv=[''] for i in range(len(argslist)): if argslist[i] == "": prefix = '-' else: newlist = argslist[i].split(" ") sys.argv += [prefix+'-'+newlist[0]] newlist = newlist [1:] if len(newlist) > 0: sys.argv += newlist #sys.argv += [prefix+'-'+argslist[i]] prefix = '' #TODO: GVL: maybe do some reformating to smaller line length parser = argparse.ArgumentParser(prog="repouser", formatter_class=argparse.RawDescriptionHelpFormatter, description="User command ") group1 = parser.add_mutually_exclusive_group() group1.add_argument('-a', '--add', dest='usertoAdd', metavar='userId', help='Add new user to the Image Repository') group1.add_argument('-d', '--del', dest='usertoDel', metavar='userId', help='Delete an user to the Image Repository') group1.add_argument('-l', '--list', dest='list', action="store_true", help='List users from Image Repository') group1.add_argument('-m', '--modify', dest='modify', nargs=3, metavar=('userId', 'quota/role/status', 'value'), help='Modify quota, role or status of an user') args = parser.parse_args() used_args = sys.argv[1:] if len(used_args) == 0: parser.print_help() return if ('-a' in used_args or '--add' in used_args): self.repouseradd(args.usertoAdd) if ('-d' in used_args or '--del' in used_args): self.repouserdel(args.usertoDel) if ('-l' in used_args or '--list' in used_args): self.repouserlist("") if ('-m' in used_args or '--modify' in used_args): if args.modify[1].strip() == "quota": print args.modify[0] print args.modify[2] self.reposetuserquota(args.modify[0],args.modify[2]) elif args.modify[1].strip() == "role": if not args.modify[2] in IRTypes.IRUser.Role: print "ERROR: third positional parameter must be one of these: " + str(IRTypes.IRUser.Role) else: self.reposetuserrole(args.modify[0],args.modify[2]) elif args.modify[1].strip() == "status": if not args.modify[2] in IRTypes.IRUser.Status: print "ERROR: third positional parameter must be one of these: " + str(IRTypes.IRUser.Status) else: self.reposetuserstatus(args.modify[0],args.modify[2]) else: print "ERROR: second positional parameter must be quota, role or status" def help_repouser(self): msg = "Repo user command: Manage Image Repository Users " self.print_man("user ", msg) eval("self.do_repouser(\"-h\")") ############################################################ # user add ############################################################ def repouseradd(self, args): '''Image Repository useradd command: Add new user (only Admin user can execut it).''' args = self.getArgs(args) if (len(args) == 1): #connect with the server if not self._service.connection(): print "ERROR: Connection with the server failed" return status = self._service.userAdd(self.user, self.passwd, self.user, args[0]) if(status == "True"): print "User created successfully." print "Remember that you still need to activate this user (see user -m command)\n" else: print "The user has not been created. \n" + \ "Please verify that you are admin and that the username does not exist \n" else: self.help_repouseradd() ############################################################ # user del ############################################################ def repouserdel(self, args): '''Image Repository userdel command: Remove a user (only Admin user can execut it).''' args = self.getArgs(args) if (len(args) == 1): #connect with the server if not self._service.connection(): print "ERROR: Connection with the server failed" return status = self._service.userDel(self.user, self.passwd, self.user, args[0]) if(status == "True"): print "User deleted successfully." else: print "The user has not been deleted. \n" + \ "Please verify that you are admin and that the username exists \n" else: self.help_repouserdel() ############################################################ # userlist ############################################################ def repouserlist(self, args): '''Image Repository userlist command: Get list of users''' #connect with the server if not self._service.connection(): print "ERROR: Connection with the server failed" return userList = self._service.userList(self.user, self.passwd, self.user) if(userList.strip() != None): try: imgs = eval(userList) print str(len(imgs)) + " users found" for key in imgs.keys(): print imgs[key] except: print "Server replied: " + str(userList) print "do_repouserlist: Error:" + str(sys.exc_info()[0]) + "\n" self._log.error("user -l: Error interpreting the list of users from Image Repository" + str(sys.exc_info()[0])) else: print "No list of user returned. \n" + \ "Please verify that you are admin \n" ############################################################ # setuserquota ############################################################ def reposetuserquota(self, userId, value): '''Image Repository setuserquota command: Establish disk space available for users (this is given in bytes). Quota argument allow math expressions like 4*1024''' #connect with the server if not self._service.connection(): print "ERROR: Connection with the server failed" return status = self._service.setUserQuota(self.user, self.passwd, self.user, userId, value) if(status == "True"): print "User quota changed successfully." else: print "The user quota has not been changed. \n" + \ "Please verify that you are admin and that the username exists \n" ############################################################ # userrole ############################################################ def reposetuserrole(self, userId, value): #connect with the server if not self._service.connection(): print "ERROR: Connection with the server failed" return status = self._service.setUserRole(self.user, self.passwd, self.user, userId, value) if(status == "True"): print "User role has been changed successfully." else: print "The user role has not been changed. " + status + "\n"\ "Please verify that you are admin and that the username exists \n" ############################################################ # userstatus ############################################################ def reposetuserstatus(self, userId, value): #connect with the server if not self._service.connection(): print "ERROR: Connection with the server failed" return status = self._service.setUserStatus(self.user, self.passwd, self.user, userId, value) if(status == "True"): print "User status has been changed successfully." else: print "The user status has not been changed. " + status + "\n"\ "Please verify that you are admin and that the username exists \n" ############################################################ # list ############################################################ def do_repolist(self, args): '''Image Repository list command: Get list of images that meet the criteria. If not argument provided it get all images. queryString can "be: * ; * where field=XX, field2=YY; field1,field2 where field3=XX''' #connect with the server if not self._service.connection(): print "ERROR: Connection with the server failed" return if (args.strip() == ""): imgsList = self._service.query(self.user, self.passwd, self.user, "*") else: imgsList = self._service.query(self.user, self.passwd, self.user, args) if(imgsList != None): try: imgs = eval(imgsList) print str(len(imgs)) + " items found" for key in imgs.keys(): print imgs[key] except: print "Server replied: " + str(imgsList) print "list: Error:" + str(sys.exc_info()) + "\n" self._log.error("list: Error interpreting the list of images from Image Repository" + str(sys.exc_info())) else: print "No list of images returned" def help_repolist(self): '''Help message for the repouserlist command''' self.print_man("list [queryString] ", self.do_repolist.__doc__) ############################################################ # modify ############################################################ def do_repomodify(self, args): args = self.getArgs(args) second = "" if(len(args) > 1): for i in range(1, len(args)): second += args[i] + " " #second = second.replace("&", "|") if (len(args) >= 2): #connect with the server if not self._service.connection(): print "ERROR: Connection with the server failed" return status = self._service.updateItem(self.user, self.passwd, self.user, args[0], second) if(status == "True"): print "The metadata of img " + args[0] + " has been updated" else: print "Error in the update. Please verify that you are the owner or that you introduced the correct arguments" else: self.help_repomodify() def help_repomodify(self): msg = "Image Repository modify command: Modify image metadata. Example " + \ "of all values of attributeString (you do not need to provide all of " + \ "them): vmtype=xen & imgtype=opennebula & os=linux & arch=x86_64 & " + \ "description=my image & tag=tag1,tag2 & permission=public & " + \ "imgStatus=available. Some attributes are controlled:" self.print_man("modify <imgId> <Metadata>", msg) first = True # TODO: GVL: maybe this calls for a function? # GVL: _print_first_occurance(self,"vmtype= ", IRTypes.ImgMeta.VmType) for line in textwrap.wrap("vmtype= " + str(IRTypes.ImgMeta.VmType), 64): if first: print " %s" % (line) first = False else: print " %s" % (line) # GVL: _print_first_occurance(self,"imgtype= " + IRTypes.ImgMeta.ImgType) first = True for line in textwrap.wrap("imgtype= " + str(IRTypes.ImgMeta.ImgType), 64): if first: print " %s" % (line) first = False else: print " %s" % (line) # GVL: _print_first_occurance(self, "imgStatus", IRTypes.ImgMeta.ImgStatus) first = True for line in textwrap.wrap("imgStatus= " + str(IRTypes.ImgMeta.ImgStatus), 64): if first: print " %s" % (line) first = False else: print " %s" % (line) # GVL: _print_first_occurance(self, "Permission", IRTypes.ImgMeta.Permission) first = True for line in textwrap.wrap("Permission= " + str(IRTypes.ImgMeta.Permission), 64): if first: print " %s" % (line) first = False else: print " %s" % (line) # GVL: #def _print_first_occurance(self, label, type): # " label is in Permission, imgStatus, imagetype, vmtype" # " type is in IRTypes.ImgMeta.Permission" # first = True # for line in textwrap.wrap(label + "= " + str(type), 64): # if first: # print " %s" % (line) # first = False # else: # print " %s" % (line) ############################################################ # permission ############################################################ def do_reposetpermission(self, args): args = self.getArgs(args) if (len(args) == 2): #connect with the server if not self._service.connection(): print "ERROR: Connection with the server failed" return status = self._service.setPermission(self.user, self.passwd, self.user, args[0], args[1]) if(status == "True"): print "Permission of img " + args[0] + " updated" else: print "The permission have not been changed. " + status else: self.help_reposetpermission() def help_reposetpermission(self): msg = "Image Repository setPermission command: Change image permission." + \ " Permission= " + str(IRTypes.ImgMeta.Permission) self.print_man("setpermission <imgId> <permission>", msg) ############################################################ # get ############################################################ def do_repoget(self, args): '''Image Repository get command: Get an image or only the URI by id.''' args = self.getArgs(args) if (len(args) == 1): #connect with the server if not self._service.connection(): print "ERROR: Connection with the server failed" return imgstatus = self._service.get(self.user, self.passwd, self.user, "img", args[0], "./") if imgstatus: print "The image " + args[0] + " is located in " +imgstatus else: print "Cannot get access to the image with imgId = " + args[0] else: self.help_repoget() def help_repoget(self): '''Help message for the repoget command''' self.print_man("get <imgId>", self.do_repoget.__doc__) ############################################################ # put ############################################################ def do_repoput(self, args): args = self.getArgs(args) second = "" if(len(args) > 1): for i in range(1, len(args)): second += args[i] + " " #second = second.replace("&", "|") status = "" ok = False if (len(args) > 1): #connect with the server if not self._service.connection(): print "ERROR: Connection with the server failed" return status = self._service.put(self.user, self.passwd, self.user, args[0], second) ok = True elif (len(args) == 1): #connect with the server if not self._service.connection(): print "ERROR: Connection with the server failed" return status = self._service.put(self.user, self.passwd, self.user, args[0], "") ok = True else: self.help_repoput() if(ok): if (re.search('^ERROR', status)): print 'The image has not been uploaded. Exit error: ' + status else: print "The image has been uploaded and registered with id " + str(status) def help_repoput(self): msg = "Image Repository put command: Upload a new image and its " + \ "metadata as an attributeString. If no attributeString provided some default values are " + \ "assigned. Example of all values of attributeString (you do not need to " + \ "provide all of them): vmtype=xen & imgtype=opennebula & os=linux & " + \ "arch=x86_64 & description=my image & tag=tag1,tag2 & permission=public & " + \ "imgStatus=available. Some attributes are controlled:" self.print_man("put <imgFile> [attributeString]", msg) # TODO: GVL: If labels would be same capitalizatiion, one could # simplify the code for finding the first occurance via eval. low priority. self.type_print ("vmtype= ", IRTypes.ImgMeta.VmType) self.type_print ("imgtype= ", IRTypes.ImgMeta.ImgType) self.type_print ("imgStatus= ",IRTypes.ImgMeta.ImgStatus) self.type_print ("Permission=" ,IRTypes.ImgMeta.Permission) # GVL: ups ;-) I see Javier had the same idea ;-) I suggest to use this in the help_repomodify ... def type_print (self, label, irtype): first = True for line in textwrap.wrap(label + str(irtype), 64): if first: print " %s" % (line) first = False else: print " %s" % (line) ############################################################ # remove ############################################################ def do_reporemove(self, args): '''The Image Repository remove command: Remove images from the repository.''' args = self.getArgs(args) if (len(args) >= 1): #connect with the server if not self._service.connection(): print "ERROR: Connection with the server failed" return output=self._service.remove(self.user, self.passwd, self.user, args) if ( output == "True"): print "All images have been removed." else: print "Some images have NOT been removed. Images with imgIds= " + str(output) + " have NOT been removed. Please verify the imgIds and if you are the owner" else: self.help_reporemove() def help_reporemove(self): '''Help message for the repremove command''' self.print_man("remove <list of imgId>", self.do_reporemove.__doc__)
class IMGenerateServer(object): def __init__(self): super(IMGenerateServer, self).__init__() #********************* #Static Configuration. #********************* #this is to login in the VM. This MUST be root because IMGenerateScript needs this access. self.rootId = 'root' self.numparams = 13 #this is the user that requested the image self.user = "" self.os = "" self.version = "" self.arch = "" self.software = "" self.givenname = "" self.desc = "" self.getimg = False self.baseimage=False self.nocache=False self.size=1.5 #load configuration self._genConf = IMServerConf() self._genConf.load_generateServerConfig() self.port = self._genConf.getGenPort() self.proc_max = self._genConf.getProcMax() self.refresh_status = self._genConf.getRefreshStatus() self.wait_max = self._genConf.getWaitMax() self._nopasswdusers = self._genConf.getNoPasswdUsersGen() self.vmfile_centos = self._genConf.getVmFileCentos() self.vmfile_rhel = self._genConf.getVmFileRhel() self.vmfile_ubuntu = self._genConf.getVmFileUbuntu() self.vmfile_debian = self._genConf.getVmFileDebian() self.xmlrpcserver = self._genConf.getXmlRpcServer() self.bridge = self._genConf.getBridge() #self.serverdir = self._genConf.getServerDir() #if self.serverdir == None: self.serverdir = os.path.expanduser(os.path.dirname(__file__)) #Location of IMGenerateScript.py file, which should be the same that IMGenerateServer.py self.addrnfs = self._genConf.getAddrNfs() self.tempdirserver = self._genConf.getTempDirServerGen() self.tempdir = self._genConf.getTempDirGen() self.http_server = self._genConf.getHttpServerGen() #self.bcfg2_url = self._genConf.getBcfg2Url() #self.bcfg2_port = self._genConf.getBcgf2Port() self.oneauth = self._genConf.getOneUser() + ":" + self._genConf.getOnePass() self.log_filename = self._genConf.getLogGen() self.logLevel = self._genConf.getLogLevelGen() self.logger = self.setup_logger() self._ca_certs = self._genConf.getCaCertsGen() self._certfile = self._genConf.getCertFileGen() self._keyfile = self._genConf.getKeyFileGen() print "\nReading Configuration file from " + self._genConf.getConfigFile() + "\n" #Image repository Object verbose=False printLogStdout=False self._reposervice = IRServiceProxy(verbose,printLogStdout) def setup_logger(self): #Setup logging logger = logging.getLogger("GenerateServer") logger.setLevel(self.logLevel) formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") handler = logging.FileHandler(self.log_filename) handler.setLevel(self.logLevel) handler.setFormatter(formatter) logger.addHandler(handler) logger.propagate = False #Do not propagate to others return logger """ def get_adminpass(self, oneadmin): ############## #GET oneadmin password encoded in SHA1 ############## p = Popen('oneuser list', stdout=PIPE, shell=True) p1 = Popen('grep ' + oneadmin, stdin=p.stdout, stdout=PIPE, shell=True) p2 = Popen('cut -d\" \" -f13', stdin=p1.stdout, shell=True, stdout=PIPE) oneadminpass = p2.stdout.read().strip() return oneadmin + ":" + oneadminpass """ def start(self): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.bind(('', self.port)) sock.listen(1) #Maximum of system unaccepted connections. Maximum value depend of the system (usually 5) self.logger.info('Starting Server on port ' + str(self.port)) proc_list = [] total_count = 0 while True: if len(proc_list) == self.proc_max: full = True while full: for i in range(len(proc_list) - 1, -1, -1): #self.logger.debug(str(proc_list[i])) if not proc_list[i].is_alive(): #print "dead" proc_list.pop(i) full = False if full: time.sleep(self.refresh_status) total_count += 1 #channel, details = sock.accept() newsocket, fromaddr = sock.accept() connstream = 0 try: connstream = ssl.wrap_socket(newsocket, server_side=True, ca_certs=self._ca_certs, cert_reqs=ssl.CERT_REQUIRED, certfile=self._certfile, keyfile=self._keyfile, ssl_version=ssl.PROTOCOL_TLSv1) #print connstream proc_list.append(Process(target=self.generate, args=(connstream,fromaddr[0],))) proc_list[len(proc_list) - 1].start() except ssl.SSLError: self.logger.error("Unsuccessful connection attempt from: " + repr(fromaddr)) self.logger.info("Image Generation Request DONE") except socket.error: self.logger.error("Error with the socket connection") self.logger.info("Image Generation Request DONE") except: self.logger.error("Uncontrolled Error: " + str(sys.exc_info())) try: connstream.shutdown(socket.SHUT_RDWR) connstream.close() except: pass self.logger.info("Image Generation Request DONE") def auth(self, userCred): return FGAuth.auth(self.user, userCred) def checkUserStatus(self, userId, passwd, userIdB): """ return "Active", "NoActive", "NoUser"; also False in case the connection with the repo fails """ if not self._reposervice.connection(): msg = "ERROR: Connection with the Image Repository failed" self.logger.error(msg) return False else: self.logger.debug("Checking User Status") status= self._reposervice.getUserStatus(userId, passwd, userIdB) self._reposervice.disconnect() return status def checknopasswd(self, fromaddr): status = False if self.user in self._nopasswdusers: if fromaddr in self._nopasswdusers[self.user]: status = True return status def generate(self, channel, fromaddr): #this runs in a different proccess start_all = time.time() self.logger = logging.getLogger("GenerateServer." + str(os.getpid())) self.logger.info('Processing an image generation request') #receive the message data = channel.read(2048) self.logger.debug("received data: " + data) params = data.split('|') #params[0] is user #params[1] is operating system #params[2] is version #params[3] is arch #params[4] is software #params[5] is givenname #params[6] is the description #params[7] is to retrieve the image or to upload in the repo (true or false, respectively) #params[8] is the user password #params[9] is the type of password #params[10] is to generate only a base image #params[11] is to do not use a cached based image #params[12] is the size of the image. 1.5 GB minimum self.user = params[0].strip() self.os = params[1].strip() self.version = params[2].strip() self.arch = params[3].strip() self.software = params[4].strip() self.givenname = params[5].strip() self.desc = params[6].strip() self.getimg = eval(params[7].strip()) #boolean passwd = params[8].strip() passwdtype = params[9].strip() self.baseimage=eval(params[10].strip()) #boolean self.nocache=eval(params[11].strip()) #boolean self.size=eval(params[12].strip()) #double if len(params) != self.numparams: msg = "ERROR: incorrect message" self.errormsg(channel, msg) #break return retry = 0 maxretry = 3 endloop = False while (not endloop): if not self.checknopasswd(fromaddr): userCred = FGCredential(passwdtype, passwd) if self.auth(userCred): #check the status of the user in the image repository. #This contacts with image repository client to check its db. The user an password are OK because this was already checked. userstatus=self.checkUserStatus(self.user, passwd, self.user) if userstatus == "Active": channel.write("OK") elif userstatus == "NoActive": channel.write("NoActive") msg = "ERROR: The user " + self.user + " is not active" self.errormsg(channel, msg) return elif userstatus == "NoUser": channel.write("NoUser") msg = "ERROR: The user " + self.user + " does not exist" self.logger.error(msg) self.logger.info("Image Generation Request DONE") return else: channel.write("Could not connect with image repository server") msg = "ERROR: Could not connect with image repository server to verify the user status" self.logger.error(msg) self.logger.info("Image Generation Request DONE") return endloop = True else: retry += 1 if retry < maxretry: channel.write("TryAuthAgain") passwd = channel.read(2048) else: msg = "ERROR: authentication failed" endloop = True self.errormsg(channel, msg) return else: channel.write("OK") endloop = True baseimageuri=None if not self.nocache and not self.baseimage: #find a base image in the repository start = time.time() baseimageuri=self.findBaseImage(passwd) end = time.time() self.logger.info('TIME to retrieve and uncompress a base image from the repo:' + str(end - start)) if baseimageuri==None: self.genInVM(channel, passwd) else: self.genLocal(channel,passwd, baseimageuri) end_all = time.time() self.logger.info('TIME walltime image generate:' + str(end_all - start_all)) self.logger.info("Image Generation DONE") def genLocal(self, channel, passwd, baseimageuri): options = "-a " + self.arch + " -o " + self.os + " -v " + self.version + " -u " + self.user + " -t " + self.tempdirserver + " --size " + str(self.size) if type(self.givenname) is not NoneType: options += " -n " + self.givenname if type(self.desc) is not NoneType: options += " -e " + self.desc if type(self.software) is not NoneType: options += " -s " + self.software if baseimageuri != None: options += " -i " + baseimageuri if self.baseimage: options += " -b " options += " -c " + self.http_server + " -l " + self.tempdirserver + "/"+ str(os.getpid()) +"_gen_cache.log" cmdexec = '/usr/bin/python ' + self.serverdir + "/IMGenerateScript.py " + options #cmdexec = self.serverdir + "/image/management/IMGenerateScript.py " + options self.logger.info(cmdexec) start = time.time() cmd = "sudo " + cmdexec self.logger.debug(cmd) p = Popen(cmd.split(' '), stdout=PIPE, stderr=PIPE) std = p.communicate() end = time.time() self.logger.info('TIME generate image:' + str(end - start)) if p.returncode != 0: self.logger.error('Command: ' + cmd + ' failed, status: ' + str(p.returncode) + ' --- ' + std[1]) status = std[0].strip() #it contains error or filename self.logger.debug("status returned generate script|"+status+"|") if os.path.isfile(self.tempdirserver + "/" + status + ".img") and os.path.isfile(self.tempdirserver + "/" + status + ".manifest.xml"): self.compressUploadSend(status, channel, passwd) else: msg = "ERROR: " + str(std[1]) self.errormsg(channel, msg) def genInVM(self, channel, passwd): vmfile = "" vmID = 0 destroyed = False if self.os == "ubuntu": vmfile = self.vmfile_ubuntu elif self.os == "debian": vmfile = self.vmfile_debian elif self.os == "rhel": vmfile = self.vmfile_rhel elif self.os == "centos": vmfile = self.vmfile_centos[self.version] # ---Start xmlrpc client to opennebula server------------- try: server = xmlrpclib.ServerProxy(self.xmlrpcserver) except: self.logger.error("Error connection with OpenNebula " + str(sys.exc_info())) #print "Error connecting with OpenNebula " + str(sys.exc_info()) return ########### #BOOT VM## ########## start = time.time() output = self.boot_VM(server, vmfile) end = time.time() self.logger.info('TIME boot VM:' + str(end - start)) vmaddr = output[0] vmID = output[1] ##### if vmaddr != "fail": self.logger.info("The VM deployed is in " + vmaddr) self.logger.info("Mount scratch directory in the VM") cmd = "ssh -q -oBatchMode=yes " + self.rootId + "@" + vmaddr cmdmount = " mount -t nfs " + self.addrnfs + ":" + self.tempdirserver + " " + self.tempdir self.logger.info(cmd + cmdmount) stat = os.system(cmd + cmdmount) if (stat == 0): self.logger.info("Sending IMGenerateScript.py to the VM") cmdscp = "scp -q -oBatchMode=yes " + self.serverdir + '/IMGenerateScript.py ' + self.rootId + "@" + vmaddr + ":/root/" #cmdscp = "scp -q -oBatchMode=yes " + self.serverdir + '/image/management/IMGenerateScript.py ' + self.rootId + "@" + vmaddr + ":/root/" self.logger.info(cmdscp) stat = os.system(cmdscp) if (stat != 0): msg = "ERROR: sending IMGenerateScript.py to the VM. Exit status " + str(stat) self.errormsg(channel, msg) else: options = "-a " + self.arch + " -o " + self.os + " -v " + self.version + " -u " + self.user + " -t " + self.tempdir + " --size " + str(self.size) if type(self.givenname) is not NoneType: options += " -n " + self.givenname if type(self.desc) is not NoneType: options += " -e " + self.desc if type(self.software) is not NoneType: options += " -s " + self.software if self.baseimage: options += " -b " options += " -c " + self.http_server #+ " -b " + self.bcfg2_url + " -p " + str(self.bcfg2_port) cmdexec = " -q '/usr/bin/python /root/IMGenerateScript.py " + options + " '" self.logger.info(cmdexec) start = time.time() uid = self._rExec(self.rootId, cmdexec, vmaddr) self.logger.debug("Returned from script execution" +str(uid)) end = time.time() self.logger.info('TIME generate image:' + str(end - start)) self.logger.info("copying fg-image-generate.log to scrach partition " + self.tempdirserver + "/" + str(vmID) + "_gen.log") cmdscp = "scp -q -oBatchMode=yes " + self.rootId + "@" + vmaddr + ":/root/fg-image-generate.log " + self.tempdirserver + "/" + str(vmID) + "_gen.log" os.system(cmdscp) status = uid[0].strip() #it contains error or filename if "error" in uid: msg = "ERROR: " + str(uid[1]) self.errormsg(channel, msg) else: #stat = 0 #while stat != 0 and : self.logger.info("Umount scratch directory in the VM") cmd = "ssh -q -oBatchMode=yes " + self.rootId + "@" + vmaddr cmdmount = " umount " + self.tempdir + " 2>/dev/null" #stat = os.system(cmd + cmdmount) #self.logger.debug("exit status " + str(stat)) #if stat != 0: # time.sleep(2) start = time.time() #umount the image max_retry = 15 retry_done = 0 umounted = False #Done making changes to root fs while not umounted: self.logger.debug(cmd + cmdmount) stat = os.system(cmd + cmdmount) if stat == 0: umounted = True elif retry_done == max_retry: self.logger.debug("exit status " + str(stat)) umounted = True self.logger.error("Problems to umount the image. Exit status " + str(stat)) else: retry_done += 1 time.sleep(5) end = time.time() self.logger.info('TIME umount image:' + str(end - start)) #destroy VM self.logger.info("Destroy VM") server.one.vm.action(self.oneauth, "finalize", vmID) destroyed = True self.compressUploadSend(status, channel, passwd) else: msg = "ERROR: booting VM" self.errormsg(channel, msg) #destroy VM if not destroyed and vmID != -1: self.logger.info("Destroy VM") try: server.one.vm.action(self.oneauth, "finalize", vmID) except: msg = "ERROR: finalizing VM" self.errormsg(channel, msg) def compressUploadSend(self, status, channel, passwd): self.logger.debug("Generating tgz with image and manifest files") self.logger.debug("tar cfz " + self.tempdirserver + "/" + status + ".tgz -C " + self.tempdirserver + \ " " + status + ".manifest.xml " + status + ".img") start = time.time() out = os.system("tar cfz " + self.tempdirserver + "/" + status + ".tgz -C " + self.tempdirserver + \ " " + status + ".manifest.xml " + status + ".img") end = time.time() self.logger.info('TIME tgz image:' + str(end - start)) os.system("rm -f " + self.tempdirserver + "" + status + ".manifest.xml " + self.tempdirserver + \ "" + status + ".img") if out != 0: msg = "ERROR: generating compressed file with the image and manifest" self.errormsg(channel, msg) return if self.getimg: #send back the url where the image is channel.write(self.tempdirserver + "" + status + ".tgz") self.logger.info("Waiting until the client retrieve the image") channel.read() #we can include a loop to retry if the client has problems getting the image channel.shutdown(socket.SHUT_RDWR) channel.close() else: status_repo = "" error_repo = False #send back the ID of the image in the repository try: #connect with the server if not self._reposervice.connection(): msg = "ERROR: Connection with the Image Repository failed" self.errormsg(channel, msg) else: self.logger.info("Storing image " + self.tempdirserver + "/" + status + ".tgz" + " in the repository") start = time.time() if self.baseimage: status_repo = self._reposervice.put(self.user, passwd, self.user, self.tempdirserver + "" + status + ".tgz", "os=" + \ self.os + "_" + self.version + "&arch=" + self.arch + "&description=" + \ self.desc + "&tag=" + status +",BaseImage") else: status_repo = self._reposervice.put(self.user, passwd, self.user, self.tempdirserver + "" + status + ".tgz", "os=" + \ self.os + "_" + self.version + "&arch=" + self.arch + "&description=" + \ self.desc + "&tag=" + status) end = time.time() self.logger.info('TIME upload image to the repo:' + str(end - start)) success=True if (re.search('^ERROR', status_repo)): self.errormsg(channel, status_repo) else: channel.write(str(status_repo)) channel.shutdown(socket.SHUT_RDWR) channel.close() self._reposervice.disconnect() except: msg = "ERROR: uploading image to the repository. " + str(sys.exc_info()) self.errormsg(channel, msg) #Remove file from server because the client or the repository already finished os.system("rm -f " + self.tempdirserver + "" + status + ".tgz") def findBaseImage(self, passwd): baseimageuri=None try: #connect with the server if not self._reposervice.connection(): msg = "ERROR: Connection with the Image Repository failed" self.logger.error(msg) else: self.logger.info("Find Base Image os=" + self.os + ",version=" +self.version +",arch="+self.arch + " in the repository") imgsList=None imgsList = self._reposervice.query(self.user, passwd, self.user, "* where os=" + \ self.os + "_" + self.version + ",arch=" + self.arch + ",tag=BaseImage,imgStatus=available") self._reposervice.disconnect() #the server disconnect even without this. #if there are images, I try to find one that is owned by me. if not I take the first one. If I own several, I take the first one baseImageId="" if(imgsList != None): try: imgs = eval(imgsList) for key in imgs.keys(): imageproperties=imgs[key].split(", ") tempbaseImageId=imageproperties[0].split("=")[1] #imgid if imageproperties[3].split("=")[1].strip() == self.user: #owner baseImageId=tempbaseImageId break elif imageproperties[8].split("=")[1].strip() == 'public': #permission if baseImageId=="": baseImageId=tempbaseImageId except: self.logger.error("findbaseimage:Server replied: " + str(imgsList)) self.logger.error("findbaseimage: Error interpreting the list of images from Image Repository" + str(sys.exc_info())) if baseImageId != "":#retrieve the image if not self._reposervice.connection(): msg = "ERROR: Connection with the Image Repository failed" self.logger.error(msg) else: #create a random dir to download image random.seed() exists=True while exists: randid = str(random.getrandbits(64)) path = self.tempdirserver + "/" + randid if not os.path.isdir(path): exists=False os.system("mkdir " + path) baseimageuri = self._reposervice.get(self.user, passwd, self.user, "img", baseImageId, path) self._reposervice.disconnect() #the server disconnect even without this. else: self.logger.debug("No Base Image found") else: self.logger.info("No Base Image found") except: msg = "ERROR: searching for a base image in the repository. " + str(sys.exc_info()) self.logger.error(msg) #extract the image. If we do that from the VM is slower due to NFS. if baseimageuri != None: #change the URI to the one that the VM is going to see baseimageuri = self.process_baseimage(baseimageuri, randid) self.logger.debug("Inside the VM, the image will be in: "+ str(baseimageuri)) return baseimageuri def process_baseimage(self, baseimageuri, randid): pathimg=None realnameimg = "" self.logger.info('untar file with image and manifest') cmd = "tar xvfz " + baseimageuri + " -C " + os.path.dirname(baseimageuri) self.logger.debug(cmd) p = Popen(cmd.split(' '), stdout=PIPE, stderr=PIPE) std = p.communicate() stat = 0 if len(std[0]) > 0: realnameimg = std[0].split("\n")[0].strip().split(".")[0] if p.returncode != 0: self.logger.error('Command: ' + cmd + ' failed, status: ' + str(p.returncode) + ' --- ' + std[1]) stat = 1 cmd = 'rm -f ' + baseimageuri os.system(cmd) if (stat != 0): msg = "ERROR: Extracting Base Image: " +str(sys.exc_info()) self.logger.error(msg) cmd = "rm -rf " + os.path.basename(baseimageuri) os.system(cmd) else: pathimg=self.tempdirserver+"/"+randid+"/"+realnameimg + ".img" return pathimg def errormsg(self, channel, msg): self.logger.error(msg) try: channel.write(msg) channel.shutdown(socket.SHUT_RDWR) channel.close() except: self.logger.debug("In errormsg: " + str(sys.exc_info())) self.logger.info("Image Generation DONE") def boot_VM(self, server, vmfile): """ It will boot a VM using XMLRPC API for OpenNebula from lib/ruby/OpenNebula/VirtualMachine.rb index start in 0 VM_STATE=%w{INIT PENDING HOLD ACTIVE STOPPED SUSPENDED DONE FAILED} LCM_STATE=%w{LCM_INIT PROLOG BOOT RUNNING MIGRATE SAVE_STOP SAVE_SUSPEND SAVE_MIGRATE PROLOG_MIGRATE PROLOG_RESUME EPILOG_STOP EPILOG SHUTDOWN CANCEL FAILURE CLEANUP UNKNOWN} """ vmaddr = "" fail = False #print vmfile #-----read template into string ------------------------- #s=open('./share/examples/ubuntu_context.one','r').read() s = open(os.path.expanduser(vmfile), 'r').read() #self.logger.debug("Vm template:\n"+s) #-----Start VM------------------------------------------- try: vm = server.one.vm.allocate(self.oneauth, s) except: msg = "ERROR: Trying to allocate a VM" self.logger.error(msg) return ["fail", -1] # -1 dont try to finalize VM because it did not started #print self.oneauth #print vm if vm[0]: self.logger.debug("VM ID: " + str(vm[1])) #monitor VM booted = False maxretry = self.wait_max / 5 #time that the VM has to change from penn to runn retry = 0 while not booted and retry < maxretry: #eventually the VM has to boot or fail try: #-------Get Info about VM ------------------------------- vminfo = server.one.vm.info(self.oneauth, vm[1]) #print vminfo[1] manifest = parseString(vminfo[1]) #VM_status (init=0, pend=1, act=3, fail=7) vm_status = manifest.getElementsByTagName('STATE')[0].firstChild.nodeValue.strip() if vm_status == "3": #running #LCM_status (prol=1,boot=2,runn=3, fail=14, unk=16) lcm_status = manifest.getElementsByTagName('LCM_STATE')[0].firstChild.nodeValue.strip() if lcm_status == "3": #if vm_status is 3, this will be 3 too. booted = True elif vm_status == "7": #fail self.logger.error("Fail to deploy VM " + str(vm[1])) booted = True fail = True vmaddr = "fail" elif vm_status == "6": #done self.logger.error("The status of the VM " + str(vm[1]) + " is DONE") booted = True fail = True vmaddr = "fail" else: retry += 1 time.sleep(5) except: pass if retry >= maxretry: self.logger.error("The VM " + str(vm[1]) + " did not change to runn status. Please verify that the status of the OpenNebula hosts " "or increase the wait time in the configuration file (max_wait) \n") vmaddr = "fail" fail = True if not fail: #get IP nics = manifest.getElementsByTagName('NIC') for i in range(len(nics)): if(nics[i].childNodes[0].firstChild.nodeValue.strip() == self.bridge): vmaddr = nics[i].childNodes[1].firstChild.nodeValue.strip() if vmaddr.strip() != "": self.logger.debug("IP of the VM " + str(vm[1]) + " is " + str(vmaddr)) access = False maxretry = 240 #this says that we wait 20 minutes maximum to allow the VM get online. #this also prevent to get here forever if the ssh key was not injected propertly. retry = 0 self.logger.debug("Waiting to have access to VM") while not access and retry < maxretry: cmd = "ssh -q -oBatchMode=yes root@" + vmaddr + " uname" p = Popen(cmd, shell=True, stdout=PIPE) status = os.waitpid(p.pid, 0)[1] #print status if status == 0: access = True self.logger.debug("The VM " + str(vm[1]) + " with ip " + str(vmaddr) + "is accessible") else: retry += 1 time.sleep(5) if retry >= maxretry: self.logger.error("Could not get access to the VM " + str(vm[1]) + " with ip " + str(vmaddr) + "\n" "Please verify the OpenNebula templates to make sure that the public ssh key to be injected is accessible to the oneadmin user. \n" "Also verify that the VM has ssh server and is active on boot.") vmaddr = "fail" else: self.logger.error("Could not determine the IP of the VM " + str(vm[1]) + " for the bridge " + self.bridge) vmaddr = "fail" else: vmaddr = "fail" return [vmaddr, vm[1]] ############################################################ # _rExec ############################################################ def _rExec(self, userId, cmdexec, vmaddr): #TODO: do we want to use the .format statement from python to make code more readable? #Set up random string random.seed() randid = str(random.getrandbits(32)) cmdssh = "ssh -oBatchMode=yes " + userId + "@" + vmaddr tmpFile = "/tmp/" + str(time.time()) + str(randid) #print tmpFile cmdexec = cmdexec + " > " + tmpFile cmd = cmdssh + cmdexec self.logger.info(str(cmd)) stat = os.system(cmd) if (str(stat) != "0"): #print stat self.logger.info(str(stat)) f = open(tmpFile, "r") outputs = f.readlines() f.close() os.system("rm -f " + tmpFile) return outputs