def __init__(self):
super(IMRegisterServerIaaS, self).__init__()
self.path = ""
self.numparams = 9 #image path
self.name = ""
self.givenname = ""
self.operatingsystem = ""
self.version = ""
self.arch = ""
self.kernel = ""
self.user = ""
self.iaas = ""
#load from config file
self._registerConf = IMServerConf()
self._registerConf.load_registerServerIaasConfig()
self.port = self._registerConf.getIaasPort()
self.http_server = self._registerConf.getHttpServerIaas()
self.proc_max = self._registerConf.getProcMaxIaas()
self.refresh_status = self._registerConf.getRefreshStatusIaas()
self._nopasswdusers = self._registerConf.getNoPasswdUsersIaas()
self.tempdir = self._registerConf.getTempDirIaas()
self.log_filename = self._registerConf.getLogIaas()
self.logLevel = self._registerConf.getLogLevelIaas()
self._ca_certs = self._registerConf.getCaCertsIaas()
self._certfile = self._registerConf.getCertFileIaas()
self._keyfile = self._registerConf.getKeyFileIaas()
print "\nReading Configuration file from " + self._registerConf.getConfigFile() + "\n"
self.logger = self.setup_logger("")
#Image repository Object
verbose = False
printLogStdout = False
self._reposervice = IRServiceProxy(verbose, printLogStdout)
def __init__(self):
print "Init Repo"
verbose = True
printLogStdout = False
self._service = IRServiceProxy(verbose, printLogStdout)
class fgShellRepo(Cmd):
############################################################
#
############################################################
def __init__(self):
print "Init Repo"
verbose = True
printLogStdout = False
self._service = IRServiceProxy(verbose, printLogStdout)
############################################################
# test
############################################################
"""
@options([make_option('-q', '--quick', help='Make things fast'),
make_option('-s', '--slow', type=int, help='Make things slow')])
def do_repotest(self, args, opts):
arg = ''.join(args)
print opts.quick
print opts.slow
self._log.error("SHElll test in fgshell repo")
"""
############################################################
# hist img
############################################################
def do_repohistimg(self, args):
args = self.getArgs(args)
#connect with the server
if not self._service.connection():
print "ERROR: Connection with the server failed"
return
if(len(args) == 1):
imgsList = self._service.histImg(self.user, self.passwd, self.user, args[0])
else:
imgsList = self._service.histImg(self.user, self.passwd, self.user, "None")
if imgsList == None:
print "ERROR: Not image record found"
else:
try:
imgs = eval(imgsList)
for key in imgs.keys():
print imgs[key]
except:
print "Server replied: " + str(imgsList)
print "histimg: Error:" + str(sys.exc_info()) + "\n"
self._log.error("do_repohistimg: Error interpreting the list of images from Image Repository" + str(sys.exc_info()[0]))
def help_repohistimg(self):
msg = "Image Repository histimg command: Return information about the " + \
" image historical usage. \n"
self.print_man("histimg [imgId]", msg)
def do_repohistuser(self, args):
args = self.getArgs(args)
#connect with the server
if not self._service.connection():
print "ERROR: Connection with the server failed"
return
if(len(args) == 1):
userList = self._service.histUser(self.user, self.passwd, self.user, args[0])
else:
userList = self._service.histUser(self.user, self.passwd, self.user, "None")
if userList == None:
print "ERROR: Not user found"
else:
try:
users = eval(userList)
for key in users.keys():
print users[key]
except:
print "Server replied: " + str(userList) + "\n"
print "histuser: Error:" + str(sys.exc_info()) + "\n"
self._log.error("do_repohistuser: Error interpreting the list of users from Image Repository" + str(sys.exc_info()))
def help_repohistuser(self):
msg = "Image Repository histuser command: Return information about the " + \
"user historical usage."
self.print_man("histuser [userId]", msg)
###################
#user
###################
def do_repouser(self, args):
argslist = args.split("-")[1:]
prefix = ''
sys.argv=['']
for i in range(len(argslist)):
if argslist[i] == "":
prefix = '-'
else:
newlist = argslist[i].split(" ")
sys.argv += [prefix+'-'+newlist[0]]
newlist = newlist [1:]
if len(newlist) > 0:
sys.argv += newlist
#sys.argv += [prefix+'-'+argslist[i]]
prefix = ''
#TODO: GVL: maybe do some reformating to smaller line length
parser = argparse.ArgumentParser(prog="repouser", formatter_class=argparse.RawDescriptionHelpFormatter,
description="User command ")
group1 = parser.add_mutually_exclusive_group()
group1.add_argument('-a', '--add', dest='usertoAdd', metavar='userId', help='Add new user to the Image Repository')
group1.add_argument('-d', '--del', dest='usertoDel', metavar='userId', help='Delete an user to the Image Repository')
group1.add_argument('-l', '--list', dest='list', action="store_true", help='List users from Image Repository')
group1.add_argument('-m', '--modify', dest='modify', nargs=3, metavar=('userId', 'quota/role/status', 'value'), help='Modify quota, role or status of an user')
args = parser.parse_args()
used_args = sys.argv[1:]
if len(used_args) == 0:
parser.print_help()
return
if ('-a' in used_args or '--add' in used_args):
self.repouseradd(args.usertoAdd)
if ('-d' in used_args or '--del' in used_args):
self.repouserdel(args.usertoDel)
if ('-l' in used_args or '--list' in used_args):
self.repouserlist("")
if ('-m' in used_args or '--modify' in used_args):
if args.modify[1].strip() == "quota":
print args.modify[0]
print args.modify[2]
self.reposetuserquota(args.modify[0],args.modify[2])
elif args.modify[1].strip() == "role":
if not args.modify[2] in IRTypes.IRUser.Role:
print "ERROR: third positional parameter must be one of these: " + str(IRTypes.IRUser.Role)
else:
self.reposetuserrole(args.modify[0],args.modify[2])
elif args.modify[1].strip() == "status":
if not args.modify[2] in IRTypes.IRUser.Status:
print "ERROR: third positional parameter must be one of these: " + str(IRTypes.IRUser.Status)
else:
self.reposetuserstatus(args.modify[0],args.modify[2])
else:
print "ERROR: second positional parameter must be quota, role or status"
def help_repouser(self):
msg = "Repo user command: Manage Image Repository Users "
self.print_man("user ", msg)
eval("self.do_repouser(\"-h\")")
############################################################
# user add
############################################################
def repouseradd(self, args):
'''Image Repository useradd command: Add new user (only Admin
user can execut it).'''
args = self.getArgs(args)
if (len(args) == 1):
#connect with the server
if not self._service.connection():
print "ERROR: Connection with the server failed"
return
status = self._service.userAdd(self.user, self.passwd, self.user, args[0])
if(status == "True"):
print "User created successfully."
print "Remember that you still need to activate this user (see user -m command)\n"
else:
print "The user has not been created. \n" + \
"Please verify that you are admin and that the username does not exist \n"
else:
self.help_repouseradd()
############################################################
# user del
############################################################
def repouserdel(self, args):
'''Image Repository userdel command: Remove a user (only Admin
user can execut it).'''
args = self.getArgs(args)
if (len(args) == 1):
#connect with the server
if not self._service.connection():
print "ERROR: Connection with the server failed"
return
status = self._service.userDel(self.user, self.passwd, self.user, args[0])
if(status == "True"):
print "User deleted successfully."
else:
print "The user has not been deleted. \n" + \
"Please verify that you are admin and that the username exists \n"
else:
self.help_repouserdel()
############################################################
# userlist
############################################################
def repouserlist(self, args):
'''Image Repository userlist command: Get list of users'''
#connect with the server
if not self._service.connection():
print "ERROR: Connection with the server failed"
return
userList = self._service.userList(self.user, self.passwd, self.user)
if(userList.strip() != None):
try:
imgs = eval(userList)
print str(len(imgs)) + " users found"
for key in imgs.keys():
print imgs[key]
except:
print "Server replied: " + str(userList)
print "do_repouserlist: Error:" + str(sys.exc_info()[0]) + "\n"
self._log.error("user -l: Error interpreting the list of users from Image Repository" + str(sys.exc_info()[0]))
else:
print "No list of user returned. \n" + \
"Please verify that you are admin \n"
############################################################
# setuserquota
############################################################
def reposetuserquota(self, userId, value):
'''Image Repository setuserquota command: Establish disk space
available for users (this is given in bytes). Quota argument
allow math expressions like 4*1024'''
#connect with the server
if not self._service.connection():
print "ERROR: Connection with the server failed"
return
status = self._service.setUserQuota(self.user, self.passwd, self.user, userId, value)
if(status == "True"):
print "User quota changed successfully."
else:
print "The user quota has not been changed. \n" + \
"Please verify that you are admin and that the username exists \n"
############################################################
# userrole
############################################################
def reposetuserrole(self, userId, value):
#connect with the server
if not self._service.connection():
print "ERROR: Connection with the server failed"
return
status = self._service.setUserRole(self.user, self.passwd, self.user, userId, value)
if(status == "True"):
print "User role has been changed successfully."
else:
print "The user role has not been changed. " + status + "\n"\
"Please verify that you are admin and that the username exists \n"
############################################################
# userstatus
############################################################
def reposetuserstatus(self, userId, value):
#connect with the server
if not self._service.connection():
print "ERROR: Connection with the server failed"
return
status = self._service.setUserStatus(self.user, self.passwd, self.user, userId, value)
if(status == "True"):
print "User status has been changed successfully."
else:
print "The user status has not been changed. " + status + "\n"\
"Please verify that you are admin and that the username exists \n"
############################################################
# list
############################################################
def do_repolist(self, args):
'''Image Repository list command: Get list of images that meet
the criteria. If not argument provided it get all
images. queryString can "be: * ; * where field=XX, field2=YY;
field1,field2 where field3=XX'''
#connect with the server
if not self._service.connection():
print "ERROR: Connection with the server failed"
return
if (args.strip() == ""):
imgsList = self._service.query(self.user, self.passwd, self.user, "*")
else:
imgsList = self._service.query(self.user, self.passwd, self.user, args)
if(imgsList != None):
try:
imgs = eval(imgsList)
print str(len(imgs)) + " items found"
for key in imgs.keys():
print imgs[key]
except:
print "Server replied: " + str(imgsList)
print "list: Error:" + str(sys.exc_info()) + "\n"
self._log.error("list: Error interpreting the list of images from Image Repository" + str(sys.exc_info()))
else:
print "No list of images returned"
def help_repolist(self):
'''Help message for the repouserlist command'''
self.print_man("list [queryString] ", self.do_repolist.__doc__)
############################################################
# modify
############################################################
def do_repomodify(self, args):
args = self.getArgs(args)
second = ""
if(len(args) > 1):
for i in range(1, len(args)):
second += args[i] + " "
#second = second.replace("&", "|")
if (len(args) >= 2):
#connect with the server
if not self._service.connection():
print "ERROR: Connection with the server failed"
return
status = self._service.updateItem(self.user, self.passwd, self.user, args[0], second)
if(status == "True"):
print "The metadata of img " + args[0] + " has been updated"
else:
print "Error in the update. Please verify that you are the owner or that you introduced the correct arguments"
else:
self.help_repomodify()
def help_repomodify(self):
msg = "Image Repository modify command: Modify image metadata. Example " + \
"of all values of attributeString (you do not need to provide all of " + \
"them): vmtype=xen & imgtype=opennebula & os=linux & arch=x86_64 & " + \
"description=my image & tag=tag1,tag2 & permission=public & " + \
"imgStatus=available. Some attributes are controlled:"
self.print_man("modify <imgId> <Metadata>", msg)
first = True
# TODO: GVL: maybe this calls for a function?
# GVL: _print_first_occurance(self,"vmtype= ", IRTypes.ImgMeta.VmType)
for line in textwrap.wrap("vmtype= " + str(IRTypes.ImgMeta.VmType), 64):
if first:
print " %s" % (line)
first = False
else:
print " %s" % (line)
# GVL: _print_first_occurance(self,"imgtype= " + IRTypes.ImgMeta.ImgType)
first = True
for line in textwrap.wrap("imgtype= " + str(IRTypes.ImgMeta.ImgType), 64):
if first:
print " %s" % (line)
first = False
else:
print " %s" % (line)
# GVL: _print_first_occurance(self, "imgStatus", IRTypes.ImgMeta.ImgStatus)
first = True
for line in textwrap.wrap("imgStatus= " + str(IRTypes.ImgMeta.ImgStatus), 64):
if first:
print " %s" % (line)
first = False
else:
print " %s" % (line)
# GVL: _print_first_occurance(self, "Permission", IRTypes.ImgMeta.Permission)
first = True
for line in textwrap.wrap("Permission= " + str(IRTypes.ImgMeta.Permission), 64):
if first:
print " %s" % (line)
first = False
else:
print " %s" % (line)
# GVL:
#def _print_first_occurance(self, label, type):
# " label is in Permission, imgStatus, imagetype, vmtype"
# " type is in IRTypes.ImgMeta.Permission"
# first = True
# for line in textwrap.wrap(label + "= " + str(type), 64):
# if first:
# print " %s" % (line)
# first = False
# else:
# print " %s" % (line)
############################################################
# permission
############################################################
def do_reposetpermission(self, args):
args = self.getArgs(args)
if (len(args) == 2):
#connect with the server
if not self._service.connection():
print "ERROR: Connection with the server failed"
return
status = self._service.setPermission(self.user, self.passwd, self.user, args[0], args[1])
if(status == "True"):
print "Permission of img " + args[0] + " updated"
else:
print "The permission have not been changed. " + status
else:
self.help_reposetpermission()
def help_reposetpermission(self):
msg = "Image Repository setPermission command: Change image permission." + \
" Permission= " + str(IRTypes.ImgMeta.Permission)
self.print_man("setpermission <imgId> <permission>", msg)
############################################################
# get
############################################################
def do_repoget(self, args):
'''Image Repository get command: Get an image or only the URI by id.'''
args = self.getArgs(args)
if (len(args) == 1):
#connect with the server
if not self._service.connection():
print "ERROR: Connection with the server failed"
return
imgstatus = self._service.get(self.user, self.passwd, self.user, "img", args[0], "./")
if imgstatus:
print "The image " + args[0] + " is located in " +imgstatus
else:
print "Cannot get access to the image with imgId = " + args[0]
else:
self.help_repoget()
def help_repoget(self):
'''Help message for the repoget command'''
self.print_man("get <imgId>", self.do_repoget.__doc__)
############################################################
# put
############################################################
def do_repoput(self, args):
args = self.getArgs(args)
second = ""
if(len(args) > 1):
for i in range(1, len(args)):
second += args[i] + " "
#second = second.replace("&", "|")
status = ""
ok = False
if (len(args) > 1):
#connect with the server
if not self._service.connection():
print "ERROR: Connection with the server failed"
return
status = self._service.put(self.user, self.passwd, self.user, args[0], second)
ok = True
elif (len(args) == 1):
#connect with the server
if not self._service.connection():
print "ERROR: Connection with the server failed"
return
status = self._service.put(self.user, self.passwd, self.user, args[0], "")
ok = True
else:
self.help_repoput()
if(ok):
if (re.search('^ERROR', status)):
print 'The image has not been uploaded. Exit error: ' + status
else:
print "The image has been uploaded and registered with id " + str(status)
def help_repoput(self):
msg = "Image Repository put command: Upload a new image and its " + \
"metadata as an attributeString. If no attributeString provided some default values are " + \
"assigned. Example of all values of attributeString (you do not need to " + \
"provide all of them): vmtype=xen & imgtype=opennebula & os=linux & " + \
"arch=x86_64 & description=my image & tag=tag1,tag2 & permission=public & " + \
"imgStatus=available. Some attributes are controlled:"
self.print_man("put <imgFile> [attributeString]", msg)
# TODO: GVL: If labels would be same capitalizatiion, one could
# simplify the code for finding the first occurance via eval. low priority.
self.type_print ("vmtype= ", IRTypes.ImgMeta.VmType)
self.type_print ("imgtype= ", IRTypes.ImgMeta.ImgType)
self.type_print ("imgStatus= ",IRTypes.ImgMeta.ImgStatus)
self.type_print ("Permission=" ,IRTypes.ImgMeta.Permission)
# GVL: ups ;-) I see Javier had the same idea ;-) I suggest to use this in the help_repomodify ...
def type_print (self, label, irtype):
first = True
for line in textwrap.wrap(label + str(irtype), 64):
if first:
print " %s" % (line)
first = False
else:
print " %s" % (line)
############################################################
# remove
############################################################
def do_reporemove(self, args):
'''The Image Repository remove command: Remove images from
the repository.'''
args = self.getArgs(args)
if (len(args) >= 1):
#connect with the server
if not self._service.connection():
print "ERROR: Connection with the server failed"
return
output=self._service.remove(self.user, self.passwd, self.user, args)
if ( output == "True"):
print "All images have been removed."
else:
print "Some images have NOT been removed. Images with imgIds= " + str(output) + " have NOT been removed. Please verify the imgIds and if you are the owner"
else:
self.help_reporemove()
def help_reporemove(self):
'''Help message for the repremove command'''
self.print_man("remove <list of imgId>", self.do_reporemove.__doc__)
def main():
parser = argparse.ArgumentParser(prog="fg-repo", formatter_class=argparse.RawDescriptionHelpFormatter,
description="FutureGrid Image Repository Help ",
epilog=textwrap.dedent(extra_help()))
parser.add_argument('-u', '--user', dest='user', required=True, metavar='user', help='FutureGrid User name')
parser.add_argument('-d', '--debug', dest='debug', action="store_true", help='Print logs in the screen for debug')
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument('-q', '--list', dest='list', nargs='?', default='', metavar='AttributeString',
help='Get list of images that meet the criteria.')
group.add_argument('-g', '--get', dest='get', metavar='imgId',
help='Get an image')
group.add_argument('-p', '--put', dest='put', nargs='+', metavar=('imgFile', 'AttributeString'), help='Upload/Register an image')
group.add_argument('-m', '--modify', dest='modify', nargs=2, metavar=('imgId', 'AttributeString'), help='Update image metadata')
group.add_argument('-r', '--remove', dest='remove', metavar='imgId', nargs="+", help='Delete images from the Repository')
group.add_argument('-s', '--setpermission', dest='setpermission', nargs=2, metavar=('imgId', 'permissionString'),
help='Set Access permission')
group.add_argument('--useradd', dest='useradd', metavar='userId', help='Add a new user to the repository')
group.add_argument('--userdel', dest='userdel', metavar='userId', help='Delete an user from the repository')
group.add_argument('--userlist', dest='userlist', action="store_true", help='List of users')
group.add_argument('--setuserquota', dest='setuserquota', nargs=2, metavar=('userId', 'quotaExpresion'), help='Modify User Quota')
group.add_argument('--setuserrole', dest='setuserrole', nargs=2, metavar=('userId', 'role'), help='Modify User Role')
group.add_argument('--setuserstatus', dest='setuserstatus', nargs=2, metavar=('userId', 'status'), help='Modify User Status')
group.add_argument('--histimg', dest='histimg', nargs='?', metavar='imgId', default='None', help='Get usage info of an Image')
group.add_argument('--histuser', dest='histuser', nargs='?', metavar='userId', default='None', help='Get usage info of an User')
parser.add_argument('--nopasswd', dest='nopasswd', action="store_true", default=False, help='If this option is used, the password is not requested. This is intended for systems daemons like Inca')
args = parser.parse_args()
#print sys.argv
#print args
if len(sys.argv) == 3:
print "\nERROR: You need to select and additional option to indicate the operation that you want to do. \n"
parser.print_help()
verbose = True #This is to print the logs in the screen
service = IRServiceProxy(verbose, args.debug)
if args.nopasswd == False:
print "Please insert the password for the user " + args.user + ""
m = hashlib.md5()
m.update(getpass())
passwd = m.hexdigest()
else:
passwd = "None"
used_args = sys.argv[1:]
print "Your request is in the queue to be processed"
#connect with the server
if not service.connection():
print "ERROR: Connection with the server failed"
sys.exit(1)
if ('-q' in used_args or '--list' in used_args):
if (args.list == None):
imgsList = service.query(args.user, passwd, args.user, "*")
else:
imgsList = service.query(args.user, passwd, args.user, args.list)
#dict wrapped into a list, convert it first
#print imgsList
if(imgsList != None):
try:
imgs = eval(imgsList)
print str(len(imgs)) + " items found"
for key in imgs.keys():
print imgs[key]
except:
print "Server replied: " + str(imgsList)
print "list: Error:" + str(sys.exc_info()) + "\n"
service._log.error("list: Error interpreting the list of images from Image Repository" + str(sys.exc_info()))
else:
print "No list of images returned"
elif ('-g' in used_args or '--get' in used_args):
#if (args.get[0] == 'img' or args.get[0] == 'uri'):
img1 = service.get(args.user, passwd, args.user, "img", args.get, "./")
if img1:
print "The image " + args.get + " is located in " + img1
else:
print "Cannot get access to the image with imgId = " + args.get
#else:
# print "Error: First argument of -g/--get must be 'img' or 'uri'"
elif ('-p' in used_args or '--put' in used_args):
status = ""
ok = False
if (len(args.put) == 2):
status = service.put(args.user, passwd, args.user, args.put[0], args.put[1])
ok = True
elif (len(args.put) == 1):
status = service.put(args.user, passwd, args.user, args.put[0], "")
ok = True
else:
args.print_help()
if(ok):
if (re.search('^ERROR', status)):
print 'The image has not been uploaded. Exit error: ' + status
else:
print "The image has been uploaded and registered with id " + str(status)
elif ('-m' in used_args or '--modify' in used_args):
success = service.updateItem(args.user, passwd, args.user, args.modify[0], args.modify[1])
if (success == "True"):
print "The item was successfully updated"
else:
print "Error in the update. Please verify that you are the owner and the attribute string"
elif ('-r' in used_args or '--remove' in used_args):
output = service.remove(args.user, passwd, args.user, args.remove)
if ( output == "True"):
print "All images have been removed."
else:
print "Some images have NOT been removed. Images with imgIds= " + str(output) + " have NOT been removed. Please verify the imgIds and if you are the owner"
elif('-s' in used_args or '--setpermission' in used_args):
status = service.setPermission(args.user, passwd, args.user, args.setpermission[0], args.setpermission[1])
if(status == "True"):
print "Permission of img " + args.setpermission[0] + " updated"
else:
print "The permission have not been changed. " + status
#This commands only can be used by users with Admin Role.
elif ('--useradd' in used_args): #args[0] is the username. It MUST be the same that the system user
status = service.userAdd(args.user, passwd, args.user, args.useradd)
if(status == "True"):
print "User created successfully."
print "Remember that you still need to activate this user (see --setuserstatus command)\n"
else:
print "The user has not been created. \n" + \
"Please verify that you are admin and that the username does not exist \n"
elif ('--userdel' in used_args):
status = service.userDel(args.user, passwd, args.user, args.userdel)
if(status == "True"):
print "User deleted successfully."
else:
print "The user has not been deleted. \n" + \
"Please verify that you are admin and that the username \"" + args.userdel + "\" exists \n"
elif ('--userlist' in used_args):
userList = service.userList(args.user, passwd, args.user)
#print userList
if(userList != None):
try:
imgs = eval(userList)
print str(len(imgs)) + " users found"
for key in imgs.keys():
print imgs[key]
except:
print "Server replied: " + str(userList)
print "userlist: Error:" + str(sys.exc_info()[0]) + "\n"
service._log.error("userlist: Error interpreting the list of users from Image Repository" + str(sys.exc_info()[0]))
else:
print "No list of user returned. \n" + \
"Please verify that you are admin \n"
#LOOK into how to use math ops, maybe we need to put it between ""
elif ('--setuserquota' in used_args):
status = service.setUserQuota(args.user, passwd, args.user, args.setuserquota[0], args.setuserquota[1])
if(status == "True"):
print "Quota changed successfully."
else:
print "The user quota has not been changed. \n" + \
"Please verify that you are admin and that the username \"" + args.setuserquota[0] + "\" exists \n"
elif ('--setuserrole' in used_args):
status = service.setUserRole(args.user, passwd, args.user, args.setuserrole[0], args.setuserrole[1])
if(status == "True"):
print "Role changed successfully."
else:
print "The user role has not been changed. " + status + "\n"+\
"Please verify that you are admin and that the username \"" + args.setuserrole[0] + "\" exists \n"
elif ('--setuserstatus' in used_args):
status = service.setUserStatus(args.user, passwd, args.user, args.setuserstatus[0], args.setuserstatus[1])
if(status == "True"):
print "Status changed successfully."
else:
print "The user status has not been changed. " + status + "\n"+\
"Please verify that you are admin and that the username \""+args.setuserstatus[0]+"\"exists \n"
#these are again for eveyone
elif ('--histimg' in used_args):
if( args.histimg != None ):
imgsList = service.histImg(args.user, passwd, args.user, args.histimg)
else:
imgsList = service.histImg(args.user, passwd, args.user, "None")
if imgsList == None:
print "ERROR: Not image record found"
else:
try:
imgs = eval(imgsList)
for key in imgs.keys():
print imgs[key]
except:
print "Server replied: " + str(imgsList)
print "histimg: Error:" + str(sys.exc_info()) + "\n"
service._log.error("histimg: Error interpreting the list of images from Image Repository" + str(sys.exc_info()[0]))
elif('--histuser' in used_args):
if( args.histuser != None):
userList = service.histUser(args.user, passwd, args.user, args.histuser)
else:
userList = service.histUser(args.user, passwd, args.user, "None")
if userList == None:
print "ERROR: Not user found"
else:
try:
users = eval(userList)
for key in users.keys():
print users[key]
except:
print "Server replied: " + str(userList)
print "histuser: Error:" + str(sys.exc_info()) + "\n"
service._log.error("histuser: Error interpreting the list of users from Image Repository" + str(sys.exc_info()))
def __init__(self):
super(IMGenerateServer, self).__init__()
#*********************
#Static Configuration.
#*********************
#this is to login in the VM. This MUST be root because IMGenerateScript needs this access.
self.rootId = 'root'
self.numparams = 13
#this is the user that requested the image
self.user = ""
self.os = ""
self.version = ""
self.arch = ""
self.software = ""
self.givenname = ""
self.desc = ""
self.getimg = False
self.baseimage=False
self.nocache=False
self.size=1.5
#load configuration
self._genConf = IMServerConf()
self._genConf.load_generateServerConfig()
self.port = self._genConf.getGenPort()
self.proc_max = self._genConf.getProcMax()
self.refresh_status = self._genConf.getRefreshStatus()
self.wait_max = self._genConf.getWaitMax()
self._nopasswdusers = self._genConf.getNoPasswdUsersGen()
self.vmfile_centos = self._genConf.getVmFileCentos()
self.vmfile_rhel = self._genConf.getVmFileRhel()
self.vmfile_ubuntu = self._genConf.getVmFileUbuntu()
self.vmfile_debian = self._genConf.getVmFileDebian()
self.xmlrpcserver = self._genConf.getXmlRpcServer()
self.bridge = self._genConf.getBridge()
#self.serverdir = self._genConf.getServerDir()
#if self.serverdir == None:
self.serverdir = os.path.expanduser(os.path.dirname(__file__)) #Location of IMGenerateScript.py file, which should be the same that IMGenerateServer.py
self.addrnfs = self._genConf.getAddrNfs()
self.tempdirserver = self._genConf.getTempDirServerGen()
self.tempdir = self._genConf.getTempDirGen()
self.http_server = self._genConf.getHttpServerGen()
#self.bcfg2_url = self._genConf.getBcfg2Url()
#self.bcfg2_port = self._genConf.getBcgf2Port()
self.oneauth = self._genConf.getOneUser() + ":" + self._genConf.getOnePass()
self.log_filename = self._genConf.getLogGen()
self.logLevel = self._genConf.getLogLevelGen()
self.logger = self.setup_logger()
self._ca_certs = self._genConf.getCaCertsGen()
self._certfile = self._genConf.getCertFileGen()
self._keyfile = self._genConf.getKeyFileGen()
print "\nReading Configuration file from " + self._genConf.getConfigFile() + "\n"
#Image repository Object
verbose=False
printLogStdout=False
self._reposervice = IRServiceProxy(verbose,printLogStdout)
class IMGenerateServer(object):
def __init__(self):
super(IMGenerateServer, self).__init__()
#*********************
#Static Configuration.
#*********************
#this is to login in the VM. This MUST be root because IMGenerateScript needs this access.
self.rootId = 'root'
self.numparams = 13
#this is the user that requested the image
self.user = ""
self.os = ""
self.version = ""
self.arch = ""
self.software = ""
self.givenname = ""
self.desc = ""
self.getimg = False
self.baseimage=False
self.nocache=False
self.size=1.5
#load configuration
self._genConf = IMServerConf()
self._genConf.load_generateServerConfig()
self.port = self._genConf.getGenPort()
self.proc_max = self._genConf.getProcMax()
self.refresh_status = self._genConf.getRefreshStatus()
self.wait_max = self._genConf.getWaitMax()
self._nopasswdusers = self._genConf.getNoPasswdUsersGen()
self.vmfile_centos = self._genConf.getVmFileCentos()
self.vmfile_rhel = self._genConf.getVmFileRhel()
self.vmfile_ubuntu = self._genConf.getVmFileUbuntu()
self.vmfile_debian = self._genConf.getVmFileDebian()
self.xmlrpcserver = self._genConf.getXmlRpcServer()
self.bridge = self._genConf.getBridge()
#self.serverdir = self._genConf.getServerDir()
#if self.serverdir == None:
self.serverdir = os.path.expanduser(os.path.dirname(__file__)) #Location of IMGenerateScript.py file, which should be the same that IMGenerateServer.py
self.addrnfs = self._genConf.getAddrNfs()
self.tempdirserver = self._genConf.getTempDirServerGen()
self.tempdir = self._genConf.getTempDirGen()
self.http_server = self._genConf.getHttpServerGen()
#self.bcfg2_url = self._genConf.getBcfg2Url()
#self.bcfg2_port = self._genConf.getBcgf2Port()
self.oneauth = self._genConf.getOneUser() + ":" + self._genConf.getOnePass()
self.log_filename = self._genConf.getLogGen()
self.logLevel = self._genConf.getLogLevelGen()
self.logger = self.setup_logger()
self._ca_certs = self._genConf.getCaCertsGen()
self._certfile = self._genConf.getCertFileGen()
self._keyfile = self._genConf.getKeyFileGen()
print "\nReading Configuration file from " + self._genConf.getConfigFile() + "\n"
#Image repository Object
verbose=False
printLogStdout=False
self._reposervice = IRServiceProxy(verbose,printLogStdout)
def setup_logger(self):
#Setup logging
logger = logging.getLogger("GenerateServer")
logger.setLevel(self.logLevel)
formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
handler = logging.FileHandler(self.log_filename)
handler.setLevel(self.logLevel)
handler.setFormatter(formatter)
logger.addHandler(handler)
logger.propagate = False #Do not propagate to others
return logger
"""
def get_adminpass(self, oneadmin):
##############
#GET oneadmin password encoded in SHA1
##############
p = Popen('oneuser list', stdout=PIPE, shell=True)
p1 = Popen('grep ' + oneadmin, stdin=p.stdout, stdout=PIPE, shell=True)
p2 = Popen('cut -d\" \" -f13', stdin=p1.stdout, shell=True, stdout=PIPE)
oneadminpass = p2.stdout.read().strip()
return oneadmin + ":" + oneadminpass
"""
def start(self):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind(('', self.port))
sock.listen(1) #Maximum of system unaccepted connections. Maximum value depend of the system (usually 5)
self.logger.info('Starting Server on port ' + str(self.port))
proc_list = []
total_count = 0
while True:
if len(proc_list) == self.proc_max:
full = True
while full:
for i in range(len(proc_list) - 1, -1, -1):
#self.logger.debug(str(proc_list[i]))
if not proc_list[i].is_alive():
#print "dead"
proc_list.pop(i)
full = False
if full:
time.sleep(self.refresh_status)
total_count += 1
#channel, details = sock.accept()
newsocket, fromaddr = sock.accept()
connstream = 0
try:
connstream = ssl.wrap_socket(newsocket,
server_side=True,
ca_certs=self._ca_certs,
cert_reqs=ssl.CERT_REQUIRED,
certfile=self._certfile,
keyfile=self._keyfile,
ssl_version=ssl.PROTOCOL_TLSv1)
#print connstream
proc_list.append(Process(target=self.generate, args=(connstream,fromaddr[0],)))
proc_list[len(proc_list) - 1].start()
except ssl.SSLError:
self.logger.error("Unsuccessful connection attempt from: " + repr(fromaddr))
self.logger.info("Image Generation Request DONE")
except socket.error:
self.logger.error("Error with the socket connection")
self.logger.info("Image Generation Request DONE")
except:
self.logger.error("Uncontrolled Error: " + str(sys.exc_info()))
try:
connstream.shutdown(socket.SHUT_RDWR)
connstream.close()
except:
pass
self.logger.info("Image Generation Request DONE")
def auth(self, userCred):
return FGAuth.auth(self.user, userCred)
def checkUserStatus(self, userId, passwd, userIdB):
"""
return "Active", "NoActive", "NoUser"; also False in case the connection with the repo fails
"""
if not self._reposervice.connection():
msg = "ERROR: Connection with the Image Repository failed"
self.logger.error(msg)
return False
else:
self.logger.debug("Checking User Status")
status= self._reposervice.getUserStatus(userId, passwd, userIdB)
self._reposervice.disconnect()
return status
def checknopasswd(self, fromaddr):
status = False
if self.user in self._nopasswdusers:
if fromaddr in self._nopasswdusers[self.user]:
status = True
return status
def generate(self, channel, fromaddr):
#this runs in a different proccess
start_all = time.time()
self.logger = logging.getLogger("GenerateServer." + str(os.getpid()))
self.logger.info('Processing an image generation request')
#receive the message
data = channel.read(2048)
self.logger.debug("received data: " + data)
params = data.split('|')
#params[0] is user
#params[1] is operating system
#params[2] is version
#params[3] is arch
#params[4] is software
#params[5] is givenname
#params[6] is the description
#params[7] is to retrieve the image or to upload in the repo (true or false, respectively)
#params[8] is the user password
#params[9] is the type of password
#params[10] is to generate only a base image
#params[11] is to do not use a cached based image
#params[12] is the size of the image. 1.5 GB minimum
self.user = params[0].strip()
self.os = params[1].strip()
self.version = params[2].strip()
self.arch = params[3].strip()
self.software = params[4].strip()
self.givenname = params[5].strip()
self.desc = params[6].strip()
self.getimg = eval(params[7].strip()) #boolean
passwd = params[8].strip()
passwdtype = params[9].strip()
self.baseimage=eval(params[10].strip()) #boolean
self.nocache=eval(params[11].strip()) #boolean
self.size=eval(params[12].strip()) #double
if len(params) != self.numparams:
msg = "ERROR: incorrect message"
self.errormsg(channel, msg)
#break
return
retry = 0
maxretry = 3
endloop = False
while (not endloop):
if not self.checknopasswd(fromaddr):
userCred = FGCredential(passwdtype, passwd)
if self.auth(userCred):
#check the status of the user in the image repository.
#This contacts with image repository client to check its db. The user an password are OK because this was already checked.
userstatus=self.checkUserStatus(self.user, passwd, self.user)
if userstatus == "Active":
channel.write("OK")
elif userstatus == "NoActive":
channel.write("NoActive")
msg = "ERROR: The user " + self.user + " is not active"
self.errormsg(channel, msg)
return
elif userstatus == "NoUser":
channel.write("NoUser")
msg = "ERROR: The user " + self.user + " does not exist"
self.logger.error(msg)
self.logger.info("Image Generation Request DONE")
return
else:
channel.write("Could not connect with image repository server")
msg = "ERROR: Could not connect with image repository server to verify the user status"
self.logger.error(msg)
self.logger.info("Image Generation Request DONE")
return
endloop = True
else:
retry += 1
if retry < maxretry:
channel.write("TryAuthAgain")
passwd = channel.read(2048)
else:
msg = "ERROR: authentication failed"
endloop = True
self.errormsg(channel, msg)
return
else:
channel.write("OK")
endloop = True
baseimageuri=None
if not self.nocache and not self.baseimage:
#find a base image in the repository
start = time.time()
baseimageuri=self.findBaseImage(passwd)
end = time.time()
self.logger.info('TIME to retrieve and uncompress a base image from the repo:' + str(end - start))
if baseimageuri==None:
self.genInVM(channel, passwd)
else:
self.genLocal(channel,passwd, baseimageuri)
end_all = time.time()
self.logger.info('TIME walltime image generate:' + str(end_all - start_all))
self.logger.info("Image Generation DONE")
def genLocal(self, channel, passwd, baseimageuri):
options = "-a " + self.arch + " -o " + self.os + " -v " + self.version + " -u " + self.user + " -t " + self.tempdirserver + " --size " + str(self.size)
if type(self.givenname) is not NoneType:
options += " -n " + self.givenname
if type(self.desc) is not NoneType:
options += " -e " + self.desc
if type(self.software) is not NoneType:
options += " -s " + self.software
if baseimageuri != None:
options += " -i " + baseimageuri
if self.baseimage:
options += " -b "
options += " -c " + self.http_server + " -l " + self.tempdirserver + "/"+ str(os.getpid()) +"_gen_cache.log"
cmdexec = '/usr/bin/python ' + self.serverdir + "/IMGenerateScript.py " + options
#cmdexec = self.serverdir + "/image/management/IMGenerateScript.py " + options
self.logger.info(cmdexec)
start = time.time()
cmd = "sudo " + cmdexec
self.logger.debug(cmd)
p = Popen(cmd.split(' '), stdout=PIPE, stderr=PIPE)
std = p.communicate()
end = time.time()
self.logger.info('TIME generate image:' + str(end - start))
if p.returncode != 0:
self.logger.error('Command: ' + cmd + ' failed, status: ' + str(p.returncode) + ' --- ' + std[1])
status = std[0].strip() #it contains error or filename
self.logger.debug("status returned generate script|"+status+"|")
if os.path.isfile(self.tempdirserver + "/" + status + ".img") and os.path.isfile(self.tempdirserver + "/" + status + ".manifest.xml"):
self.compressUploadSend(status, channel, passwd)
else:
msg = "ERROR: " + str(std[1])
self.errormsg(channel, msg)
def genInVM(self, channel, passwd):
vmfile = ""
vmID = 0
destroyed = False
if self.os == "ubuntu":
vmfile = self.vmfile_ubuntu
elif self.os == "debian":
vmfile = self.vmfile_debian
elif self.os == "rhel":
vmfile = self.vmfile_rhel
elif self.os == "centos":
vmfile = self.vmfile_centos[self.version]
# ---Start xmlrpc client to opennebula server-------------
try:
server = xmlrpclib.ServerProxy(self.xmlrpcserver)
except:
self.logger.error("Error connection with OpenNebula " + str(sys.exc_info()))
#print "Error connecting with OpenNebula " + str(sys.exc_info())
return
###########
#BOOT VM##
##########
start = time.time()
output = self.boot_VM(server, vmfile)
end = time.time()
self.logger.info('TIME boot VM:' + str(end - start))
vmaddr = output[0]
vmID = output[1]
#####
if vmaddr != "fail":
self.logger.info("The VM deployed is in " + vmaddr)
self.logger.info("Mount scratch directory in the VM")
cmd = "ssh -q -oBatchMode=yes " + self.rootId + "@" + vmaddr
cmdmount = " mount -t nfs " + self.addrnfs + ":" + self.tempdirserver + " " + self.tempdir
self.logger.info(cmd + cmdmount)
stat = os.system(cmd + cmdmount)
if (stat == 0):
self.logger.info("Sending IMGenerateScript.py to the VM")
cmdscp = "scp -q -oBatchMode=yes " + self.serverdir + '/IMGenerateScript.py ' + self.rootId + "@" + vmaddr + ":/root/"
#cmdscp = "scp -q -oBatchMode=yes " + self.serverdir + '/image/management/IMGenerateScript.py ' + self.rootId + "@" + vmaddr + ":/root/"
self.logger.info(cmdscp)
stat = os.system(cmdscp)
if (stat != 0):
msg = "ERROR: sending IMGenerateScript.py to the VM. Exit status " + str(stat)
self.errormsg(channel, msg)
else:
options = "-a " + self.arch + " -o " + self.os + " -v " + self.version + " -u " + self.user + " -t " + self.tempdir + " --size " + str(self.size)
if type(self.givenname) is not NoneType:
options += " -n " + self.givenname
if type(self.desc) is not NoneType:
options += " -e " + self.desc
if type(self.software) is not NoneType:
options += " -s " + self.software
if self.baseimage:
options += " -b "
options += " -c " + self.http_server #+ " -b " + self.bcfg2_url + " -p " + str(self.bcfg2_port)
cmdexec = " -q '/usr/bin/python /root/IMGenerateScript.py " + options + " '"
self.logger.info(cmdexec)
start = time.time()
uid = self._rExec(self.rootId, cmdexec, vmaddr)
self.logger.debug("Returned from script execution" +str(uid))
end = time.time()
self.logger.info('TIME generate image:' + str(end - start))
self.logger.info("copying fg-image-generate.log to scrach partition " + self.tempdirserver + "/" + str(vmID) + "_gen.log")
cmdscp = "scp -q -oBatchMode=yes " + self.rootId + "@" + vmaddr + ":/root/fg-image-generate.log " + self.tempdirserver + "/" + str(vmID) + "_gen.log"
os.system(cmdscp)
status = uid[0].strip() #it contains error or filename
if "error" in uid:
msg = "ERROR: " + str(uid[1])
self.errormsg(channel, msg)
else:
#stat = 0
#while stat != 0 and :
self.logger.info("Umount scratch directory in the VM")
cmd = "ssh -q -oBatchMode=yes " + self.rootId + "@" + vmaddr
cmdmount = " umount " + self.tempdir + " 2>/dev/null"
#stat = os.system(cmd + cmdmount)
#self.logger.debug("exit status " + str(stat))
#if stat != 0:
# time.sleep(2)
start = time.time()
#umount the image
max_retry = 15
retry_done = 0
umounted = False
#Done making changes to root fs
while not umounted:
self.logger.debug(cmd + cmdmount)
stat = os.system(cmd + cmdmount)
if stat == 0:
umounted = True
elif retry_done == max_retry:
self.logger.debug("exit status " + str(stat))
umounted = True
self.logger.error("Problems to umount the image. Exit status " + str(stat))
else:
retry_done += 1
time.sleep(5)
end = time.time()
self.logger.info('TIME umount image:' + str(end - start))
#destroy VM
self.logger.info("Destroy VM")
server.one.vm.action(self.oneauth, "finalize", vmID)
destroyed = True
self.compressUploadSend(status, channel, passwd)
else:
msg = "ERROR: booting VM"
self.errormsg(channel, msg)
#destroy VM
if not destroyed and vmID != -1:
self.logger.info("Destroy VM")
try:
server.one.vm.action(self.oneauth, "finalize", vmID)
except:
msg = "ERROR: finalizing VM"
self.errormsg(channel, msg)
def compressUploadSend(self, status, channel, passwd):
self.logger.debug("Generating tgz with image and manifest files")
self.logger.debug("tar cfz " + self.tempdirserver + "/" + status + ".tgz -C " + self.tempdirserver + \
" " + status + ".manifest.xml " + status + ".img")
start = time.time()
out = os.system("tar cfz " + self.tempdirserver + "/" + status + ".tgz -C " + self.tempdirserver + \
" " + status + ".manifest.xml " + status + ".img")
end = time.time()
self.logger.info('TIME tgz image:' + str(end - start))
os.system("rm -f " + self.tempdirserver + "" + status + ".manifest.xml " + self.tempdirserver + \
"" + status + ".img")
if out != 0:
msg = "ERROR: generating compressed file with the image and manifest"
self.errormsg(channel, msg)
return
if self.getimg:
#send back the url where the image is
channel.write(self.tempdirserver + "" + status + ".tgz")
self.logger.info("Waiting until the client retrieve the image")
channel.read()
#we can include a loop to retry if the client has problems getting the image
channel.shutdown(socket.SHUT_RDWR)
channel.close()
else:
status_repo = ""
error_repo = False
#send back the ID of the image in the repository
try:
#connect with the server
if not self._reposervice.connection():
msg = "ERROR: Connection with the Image Repository failed"
self.errormsg(channel, msg)
else:
self.logger.info("Storing image " + self.tempdirserver + "/" + status + ".tgz" + " in the repository")
start = time.time()
if self.baseimage:
status_repo = self._reposervice.put(self.user, passwd, self.user, self.tempdirserver + "" + status + ".tgz", "os=" + \
self.os + "_" + self.version + "&arch=" + self.arch + "&description=" + \
self.desc + "&tag=" + status +",BaseImage")
else:
status_repo = self._reposervice.put(self.user, passwd, self.user, self.tempdirserver + "" + status + ".tgz", "os=" + \
self.os + "_" + self.version + "&arch=" + self.arch + "&description=" + \
self.desc + "&tag=" + status)
end = time.time()
self.logger.info('TIME upload image to the repo:' + str(end - start))
success=True
if (re.search('^ERROR', status_repo)):
self.errormsg(channel, status_repo)
else:
channel.write(str(status_repo))
channel.shutdown(socket.SHUT_RDWR)
channel.close()
self._reposervice.disconnect()
except:
msg = "ERROR: uploading image to the repository. " + str(sys.exc_info())
self.errormsg(channel, msg)
#Remove file from server because the client or the repository already finished
os.system("rm -f " + self.tempdirserver + "" + status + ".tgz")
def findBaseImage(self, passwd):
baseimageuri=None
try:
#connect with the server
if not self._reposervice.connection():
msg = "ERROR: Connection with the Image Repository failed"
self.logger.error(msg)
else:
self.logger.info("Find Base Image os=" + self.os + ",version=" +self.version +",arch="+self.arch + " in the repository")
imgsList=None
imgsList = self._reposervice.query(self.user, passwd, self.user, "* where os=" + \
self.os + "_" + self.version + ",arch=" + self.arch + ",tag=BaseImage,imgStatus=available")
self._reposervice.disconnect() #the server disconnect even without this.
#if there are images, I try to find one that is owned by me. if not I take the first one. If I own several, I take the first one
baseImageId=""
if(imgsList != None):
try:
imgs = eval(imgsList)
for key in imgs.keys():
imageproperties=imgs[key].split(", ")
tempbaseImageId=imageproperties[0].split("=")[1] #imgid
if imageproperties[3].split("=")[1].strip() == self.user: #owner
baseImageId=tempbaseImageId
break
elif imageproperties[8].split("=")[1].strip() == 'public': #permission
if baseImageId=="":
baseImageId=tempbaseImageId
except:
self.logger.error("findbaseimage:Server replied: " + str(imgsList))
self.logger.error("findbaseimage: Error interpreting the list of images from Image Repository" + str(sys.exc_info()))
if baseImageId != "":#retrieve the image
if not self._reposervice.connection():
msg = "ERROR: Connection with the Image Repository failed"
self.logger.error(msg)
else:
#create a random dir to download image
random.seed()
exists=True
while exists:
randid = str(random.getrandbits(64))
path = self.tempdirserver + "/" + randid
if not os.path.isdir(path):
exists=False
os.system("mkdir " + path)
baseimageuri = self._reposervice.get(self.user, passwd, self.user, "img", baseImageId, path)
self._reposervice.disconnect() #the server disconnect even without this.
else:
self.logger.debug("No Base Image found")
else:
self.logger.info("No Base Image found")
except:
msg = "ERROR: searching for a base image in the repository. " + str(sys.exc_info())
self.logger.error(msg)
#extract the image. If we do that from the VM is slower due to NFS.
if baseimageuri != None: #change the URI to the one that the VM is going to see
baseimageuri = self.process_baseimage(baseimageuri, randid)
self.logger.debug("Inside the VM, the image will be in: "+ str(baseimageuri))
return baseimageuri
def process_baseimage(self, baseimageuri, randid):
pathimg=None
realnameimg = ""
self.logger.info('untar file with image and manifest')
cmd = "tar xvfz " + baseimageuri + " -C " + os.path.dirname(baseimageuri)
self.logger.debug(cmd)
p = Popen(cmd.split(' '), stdout=PIPE, stderr=PIPE)
std = p.communicate()
stat = 0
if len(std[0]) > 0:
realnameimg = std[0].split("\n")[0].strip().split(".")[0]
if p.returncode != 0:
self.logger.error('Command: ' + cmd + ' failed, status: ' + str(p.returncode) + ' --- ' + std[1])
stat = 1
cmd = 'rm -f ' + baseimageuri
os.system(cmd)
if (stat != 0):
msg = "ERROR: Extracting Base Image: " +str(sys.exc_info())
self.logger.error(msg)
cmd = "rm -rf " + os.path.basename(baseimageuri)
os.system(cmd)
else:
pathimg=self.tempdirserver+"/"+randid+"/"+realnameimg + ".img"
return pathimg
def errormsg(self, channel, msg):
self.logger.error(msg)
try:
channel.write(msg)
channel.shutdown(socket.SHUT_RDWR)
channel.close()
except:
self.logger.debug("In errormsg: " + str(sys.exc_info()))
self.logger.info("Image Generation DONE")
def boot_VM(self, server, vmfile):
"""
It will boot a VM using XMLRPC API for OpenNebula
from lib/ruby/OpenNebula/VirtualMachine.rb
index start in 0
VM_STATE=%w{INIT PENDING HOLD ACTIVE STOPPED SUSPENDED DONE FAILED}
LCM_STATE=%w{LCM_INIT PROLOG BOOT RUNNING MIGRATE SAVE_STOP SAVE_SUSPEND
SAVE_MIGRATE PROLOG_MIGRATE PROLOG_RESUME EPILOG_STOP EPILOG
SHUTDOWN CANCEL FAILURE CLEANUP UNKNOWN}
"""
vmaddr = ""
fail = False
#print vmfile
#-----read template into string -------------------------
#s=open('./share/examples/ubuntu_context.one','r').read()
s = open(os.path.expanduser(vmfile), 'r').read()
#self.logger.debug("Vm template:\n"+s)
#-----Start VM-------------------------------------------
try:
vm = server.one.vm.allocate(self.oneauth, s)
except:
msg = "ERROR: Trying to allocate a VM"
self.logger.error(msg)
return ["fail", -1] # -1 dont try to finalize VM because it did not started
#print self.oneauth
#print vm
if vm[0]:
self.logger.debug("VM ID: " + str(vm[1]))
#monitor VM
booted = False
maxretry = self.wait_max / 5 #time that the VM has to change from penn to runn
retry = 0
while not booted and retry < maxretry: #eventually the VM has to boot or fail
try:
#-------Get Info about VM -------------------------------
vminfo = server.one.vm.info(self.oneauth, vm[1])
#print vminfo[1]
manifest = parseString(vminfo[1])
#VM_status (init=0, pend=1, act=3, fail=7)
vm_status = manifest.getElementsByTagName('STATE')[0].firstChild.nodeValue.strip()
if vm_status == "3": #running
#LCM_status (prol=1,boot=2,runn=3, fail=14, unk=16)
lcm_status = manifest.getElementsByTagName('LCM_STATE')[0].firstChild.nodeValue.strip()
if lcm_status == "3": #if vm_status is 3, this will be 3 too.
booted = True
elif vm_status == "7": #fail
self.logger.error("Fail to deploy VM " + str(vm[1]))
booted = True
fail = True
vmaddr = "fail"
elif vm_status == "6": #done
self.logger.error("The status of the VM " + str(vm[1]) + " is DONE")
booted = True
fail = True
vmaddr = "fail"
else:
retry += 1
time.sleep(5)
except:
pass
if retry >= maxretry:
self.logger.error("The VM " + str(vm[1]) + " did not change to runn status. Please verify that the status of the OpenNebula hosts "
"or increase the wait time in the configuration file (max_wait) \n")
vmaddr = "fail"
fail = True
if not fail:
#get IP
nics = manifest.getElementsByTagName('NIC')
for i in range(len(nics)):
if(nics[i].childNodes[0].firstChild.nodeValue.strip() == self.bridge):
vmaddr = nics[i].childNodes[1].firstChild.nodeValue.strip()
if vmaddr.strip() != "":
self.logger.debug("IP of the VM " + str(vm[1]) + " is " + str(vmaddr))
access = False
maxretry = 240 #this says that we wait 20 minutes maximum to allow the VM get online.
#this also prevent to get here forever if the ssh key was not injected propertly.
retry = 0
self.logger.debug("Waiting to have access to VM")
while not access and retry < maxretry:
cmd = "ssh -q -oBatchMode=yes root@" + vmaddr + " uname"
p = Popen(cmd, shell=True, stdout=PIPE)
status = os.waitpid(p.pid, 0)[1]
#print status
if status == 0:
access = True
self.logger.debug("The VM " + str(vm[1]) + " with ip " + str(vmaddr) + "is accessible")
else:
retry += 1
time.sleep(5)
if retry >= maxretry:
self.logger.error("Could not get access to the VM " + str(vm[1]) + " with ip " + str(vmaddr) + "\n"
"Please verify the OpenNebula templates to make sure that the public ssh key to be injected is accessible to the oneadmin user. \n"
"Also verify that the VM has ssh server and is active on boot.")
vmaddr = "fail"
else:
self.logger.error("Could not determine the IP of the VM " + str(vm[1]) + " for the bridge " + self.bridge)
vmaddr = "fail"
else:
vmaddr = "fail"
return [vmaddr, vm[1]]
############################################################
# _rExec
############################################################
def _rExec(self, userId, cmdexec, vmaddr):
#TODO: do we want to use the .format statement from python to make code more readable?
#Set up random string
random.seed()
randid = str(random.getrandbits(32))
cmdssh = "ssh -oBatchMode=yes " + userId + "@" + vmaddr
tmpFile = "/tmp/" + str(time.time()) + str(randid)
#print tmpFile
cmdexec = cmdexec + " > " + tmpFile
cmd = cmdssh + cmdexec
self.logger.info(str(cmd))
stat = os.system(cmd)
if (str(stat) != "0"):
#print stat
self.logger.info(str(stat))
f = open(tmpFile, "r")
outputs = f.readlines()
f.close()
os.system("rm -f " + tmpFile)
return outputs
class IMRegisterServerIaaS(object):
def __init__(self):
super(IMRegisterServerIaaS, self).__init__()
self.path = ""
self.numparams = 9 #image path
self.name = ""
self.givenname = ""
self.operatingsystem = ""
self.version = ""
self.arch = ""
self.kernel = ""
self.user = ""
self.iaas = ""
#load from config file
self._registerConf = IMServerConf()
self._registerConf.load_registerServerIaasConfig()
self.port = self._registerConf.getIaasPort()
self.http_server = self._registerConf.getHttpServerIaas()
self.proc_max = self._registerConf.getProcMaxIaas()
self.refresh_status = self._registerConf.getRefreshStatusIaas()
self._nopasswdusers = self._registerConf.getNoPasswdUsersIaas()
self.tempdir = self._registerConf.getTempDirIaas()
self.log_filename = self._registerConf.getLogIaas()
self.logLevel = self._registerConf.getLogLevelIaas()
self._ca_certs = self._registerConf.getCaCertsIaas()
self._certfile = self._registerConf.getCertFileIaas()
self._keyfile = self._registerConf.getKeyFileIaas()
print "\nReading Configuration file from " + self._registerConf.getConfigFile() + "\n"
self.logger = self.setup_logger("")
#Image repository Object
verbose = False
printLogStdout = False
self._reposervice = IRServiceProxy(verbose, printLogStdout)
def setup_logger(self, extra):
#Setup logging
logger = logging.getLogger("RegisterIaaS" + extra)
logger.setLevel(self.logLevel)
formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
handler = logging.FileHandler(self.log_filename)
handler.setLevel(self.logLevel)
handler.setFormatter(formatter)
logger.addHandler(handler)
logger.propagate = False #Do not propagate to others
return logger
def start(self):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind(('', self.port))
sock.listen(1)
self.logger.info('Starting Server on port ' + str(self.port))
proc_list = []
total_count = 0
while True:
if len(proc_list) == self.proc_max:
full = True
while full:
for i in range(len(proc_list) - 1, -1, -1):
#self.logger.debug(str(proc_list[i]))
if not proc_list[i].is_alive():
#print "dead"
proc_list.pop(i)
full = False
if full:
time.sleep(self.refresh_status)
total_count += 1
#channel, details = sock.accept()
newsocket, fromaddr = sock.accept()
connstream = 0
try:
connstream = ssl.wrap_socket(newsocket,
server_side=True,
ca_certs=self._ca_certs,
cert_reqs=ssl.CERT_REQUIRED,
certfile=self._certfile,
keyfile=self._keyfile,
ssl_version=ssl.PROTOCOL_TLSv1)
#print connstream
proc_list.append(Process(target=self.process_client, args=(connstream, fromaddr[0])))
proc_list[len(proc_list) - 1].start()
except ssl.SSLError:
self.logger.error("Unsuccessful connection attempt from: " + repr(fromaddr))
self.logger.info("IaaS register server Request DONE")
except socket.error:
self.logger.error("Error with the socket connection")
self.logger.info("IaaS register server Request DONE")
except:
self.logger.error("Uncontrolled Error: " + str(sys.exc_info()))
if type(connstream) is ssl.SSLSocket:
connstream.shutdown(socket.SHUT_RDWR)
connstream.close()
self.logger.info("IaaS register server Request DONE")
def auth(self, userCred):
return FGAuth.auth(self.user, userCred)
def checkUserStatus(self, userId, passwd, userIdB):
"""
return "Active", "NoActive", "NoUser"; also False in case the connection with the repo fails
"""
if not self._reposervice.connection():
msg = "ERROR: Connection with the Image Repository failed"
self.logger.error(msg)
return False
else:
self.logger.debug("Checking User Status")
status = self._reposervice.getUserStatus(userId, passwd, userIdB)
self._reposervice.disconnect()
return status
def checknopasswd(self, fromaddr):
status = False
if self.user in self._nopasswdusers:
if fromaddr in self._nopasswdusers[self.user]:
status = True
return status
def checkKernel(self):
status = False
if (self.iaas == "euca"):
status = self.kernel in self._euca_auth_kernels
elif (self.iaas == "nimbus"):
status = self.kernel in self._nimbus_auth_kernels
elif (self.iaas == "openstack"):
status = self.kernel in self._openstack_auth_kernels
elif (self.iaas == "opennebula"):
status = self.kernel in self._opennebula_auth_kernels
return status
def checkIaasAvail(self):
status = False
if (self.iaas == "euca"):
if not self.default_euca_kernel == "":
status = True
elif (self.iaas == "nimbus"):
if not self.default_nimbus_kernel == "":
status = True
elif (self.iaas == "openstack"):
if not self.default_openstack_kernel == "":
status = True
elif (self.iaas == "opennebula"):
if not self.default_opennebula_kernel == "":
status = True
return status
def loadIaasConfig(self, iaasSite):
self._registerConf.loadIaasSiteConfig(iaasSite)
self.default_euca_kernel = self._registerConf.getDefaultEucaKernel()
self.default_nimbus_kernel = self._registerConf.getDefaultNimbusKernel()
self.default_openstack_kernel = self._registerConf.getDefaultOpenstackKernel()
self.default_opennebula_kernel = self._registerConf.getDefaultOpennebulaKernel()
self._euca_auth_kernels = self._registerConf.getEucaAuthKernels()
self._nimbus_auth_kernels = self._registerConf.getNimbusAuthKernels()
self._openstack_auth_kernels = self._registerConf.getOpenstackAuthKernels()
self._opennebula_auth_kernels = self._registerConf.getOpennebulaAuthKernels()
def process_client(self, connstream, fromaddr):
start_all = time.time()
self.logger = self.setup_logger("." + str(os.getpid()))
self.logger.info('Accepted new connection')
#receive the message
data = connstream.read(2048)
self.logger.debug("msg received: " + data)
params = data.split(',')
#print data
#params[0] is image ID or image path or "kernels"
#params[1] is the source of the image (repo,disk).
#params[2] is the machine name
#params[3] is the iaas cloud
#params[4] is the kernel
#params[5] is the user
#params[6] is the user password
#params[7] is the type of password
#params[8] is the ldap configure or not
imgID = params[0].strip()
imgSource = params[1].strip()
machinename = params[2].strip()
self.iaas = params[3].strip()
self.kernel = params[4].strip()
self.user = params[5].strip()
passwd = params[6].strip()
passwdtype = params[7].strip()
ldap = False
try:
ldap = eval(params[8].strip())
except:
self.logger.warning("Ldap configure set to False in except")
ldap = False
if len(params) != self.numparams:
msg = "ERROR: incorrect message"
self.errormsg(connstream, msg)
return
retry = 0
maxretry = 3
endloop = False
while (not endloop):
if not self.checknopasswd(fromaddr):
userCred = FGCredential(passwdtype, passwd)
if (self.auth(userCred)):# contact directly with LDAP
#check the status of the user in the image repository.
#This contacts with image repository client to check its db. The user an password are OK because this was already checked.
userstatus = self.checkUserStatus(self.user, passwd, self.user)
if userstatus == "Active":
connstream.write("OK")
elif userstatus == "NoActive":
connstream.write("NoActive")
msg = "ERROR: The user " + self.user + " is not active"
self.errormsg(connstream, msg)
return
elif userstatus == "NoUser":
connstream.write("NoUser")
msg = "ERROR: The user " + self.user + " does not exist"
self.logger.error(msg)
self.logger.info("IaaS register server Request DONE")
return
else:
connstream.write("Could not connect with image repository server")
msg = "ERROR: Could not connect with image repository server to verify the user status"
self.logger.error(msg)
self.logger.info("IaaS register server Request DONE")
return
endloop = True
else:
retry += 1
if retry < maxretry:
connstream.write("TryAuthAgain")
passwd = connstream.read(2048)
else:
msg = "ERROR: authentication failed"
endloop = True
self.errormsg(connstream, msg)
return
else:
connstream.write("OK")
endloop = True
if imgID == "infosites":
infosites=self._registerConf.listIaasSites()
self.logger.debug("Information Cloud Sites: " + str(infosites))
connstream.write(str(infosites))
connstream.shutdown(socket.SHUT_RDWR)
connstream.close()
self.logger.info("Image Register Request (info sites) DONE")
return
#description and infrastructures without kernel details
else:
#Load Configuration of the Site. This indicates the IaaS infrastructures available in the site and kernel configurations.
output=self.loadIaasConfig(machinename)
if output == "ERROR":
msg = "ERROR: The specified site " + machinename + ". Please use the option --listsites to list available sites and its services. \n"
self.errormsg(connstream, msg)
return
if imgID == "kernels":
kernelslist = {}
if (self.iaas == "euca"):
kernelslist["Default"] = self.default_euca_kernel
kernelslist["Authorized"] = self._euca_auth_kernels
elif (self.iaas == "nimbus"):
kernelslist["Default"] = self.default_nimbus_kernel
kernelslist["Authorized"] = self._nimbus_auth_kernels
elif (self.iaas == "openstack"):
kernelslist["Default"] = self.default_openstack_kernel
kernelslist["Authorized"] = self._openstack_auth_kernels
elif (self.iaas == "opennebula"):
kernelslist["Default"] = self.default_opennebula_kernel
kernelslist["Authorized"] = self._opennebula_auth_kernels
self.logger.debug(self.iaas + " default kernels list: " + str(kernelslist["Default"]))
self.logger.debug(self.iaas + " kernels list: " + str(kernelslist["Authorized"]))
connstream.write(str(kernelslist))
connstream.shutdown(socket.SHUT_RDWR)
connstream.close()
self.logger.info("Image Register Request (kernel list " + self.iaas + ") DONE")
return
#check if the infrastructure required is available
elif not self.checkIaasAvail():
msg = "ERROR: The specified infrastructure " + self.iaas + " is not available on " + machinename + ". Please use the option --listservices to list available sites and its services. \n"
self.errormsg(connstream, msg)
return
#verify kernel is authorized
if self.kernel != "None":
if not self.checkKernel():
msg = "ERROR: The specified kernel ("+self.kernel+") is not available. Authorized kernels for " + self.iaas + " are: " + str(eval("self._" + self.iaas + "_auth_kernels"))
self.errormsg(connstream, msg)
return
else:
self.logger.debug("The kernel " + self.kernel + " is valid")
#create a unique directory
auxdir = str(randrange(999999999999999999999999))
localtempdir = self.tempdir + "/" + auxdir + "_0"
while os.path.isdir(localtempdir):
auxdir = str(randrange(999999999999999999999999))
localtempdir = self.tempdir + "/" + auxdir + "_0"
cmd = 'mkdir -p ' + localtempdir
self.runCmd(cmd)
self.runCmd("chmod 777 " + localtempdir)
start = time.time()
if imgSource == "repo":
#GET IMAGE from repo
if not self._reposervice.connection():
msg = "ERROR: Connection with the Image Repository failed"
self.errormsg(connstream, msg)
return
else:
self.logger.info("Retrieving image from repository")
image = self._reposervice.get(self.user, passwd, self.user, "img", imgID, localtempdir)
if image == None:
msg = "ERROR: Cannot get access to the image with imgId " + str(imgID)
self.errormsg(connstream, msg)
self._reposervice.disconnect()
self.runCmd("rm -rf " + localtempdir)
return
else:
self._reposervice.disconnect()
else:
connstream.write(localtempdir)
status = connstream.read(1024)
status = status.split(',')
if len(status) == 2:
image = localtempdir + '/' + status[1].strip()
if status[0].strip() != 'OK':
msg = "ERROR: Receiving image from client: " + str(status)
self.errormsg(connstream, msg)
return
else:
msg = "ERROR: Message received from client is incorrect: " + str(status)
self.errormsg(connstream, msg)
return
end = time.time()
self.logger.info('TIME retrieve image from repo or client:' + str(end - start))
if not os.path.isfile(image):
msg = "ERROR: file " + image + " not found"
self.errormsg(connstream, msg)
return
start = time.time()
#extracts image/manifest, read manifest
if not self.handle_image(image, localtempdir, connstream):
return
end = time.time()
self.logger.info('TIME untar image: ' + str(end - start))
#self.preprocess()
start = time.time()
stat = 0
if (self.iaas == "euca"):
stat = self.euca_method(localtempdir, ldap)
elif (self.iaas == "nimbus"):
stat = self.nimbus_method(localtempdir, ldap)
elif (self.iaas == "opennebula"):
stat = self.opennebula_method(localtempdir, ldap)
elif (self.iaas == "openstack"):
stat = self.openstack_method(localtempdir, ldap)
end = time.time()
self.logger.info('TIME customize image for specific IaaS framework:' + str(end - start))
start = time.time()
#umount the image
max_retry = 5
retry_done = 0
umounted = False
#Done making changes to root fs
while not umounted:
status = self.runCmd('sudo umount ' + localtempdir + '/temp')
if status == 0:
umounted = True
elif retry_done == max_retry:
umounted = True
self.logger.error("Problems to umount the image")
else:
retry_done += 1
time.sleep(2)
end = time.time()
self.logger.info('TIME umount image:' + str(end - start))
status = self.runCmd("mv -f " + localtempdir + '/' + self.name + '.img ' + localtempdir + '/' + self.operatingsystem + self.version + self.name + '.img')
kerneltouse=eval("self._" + self.iaas + "_auth_kernels")
eki=kerneltouse[self.kernel][0]
eri=kerneltouse[self.kernel][1]
connstream.write(localtempdir + '/' + self.operatingsystem + self.version + self.name + '.img,' + eki + "," + eri + "," + self.operatingsystem)
start = time.time()
#wait until client retrieve img
self.logger.info("Wait until client get the image")
connstream.read()
end = time.time()
self.logger.info('TIME wait until client get image:' + str(end - start))
#remove image
cmd = 'rm -rf ' + localtempdir
status = self.runCmd(cmd)
try:
connstream.shutdown(socket.SHUT_RDWR)
connstream.close()
except:
self.logger.error("ERROR: " + str(sys.exc_info()))
end_all = time.time()
self.logger.info('TIME walltime image register IaaS:' + str(end_all - start_all))
self.logger.info("Image Register Request DONE")
def configure_ldap(self, localtempdir):
start = time.time()
if self.operatingsystem == "centos":
self.runCmd('sudo chroot ' + localtempdir + '/temp/ yum -y install fuse-sshfs')
self.logger.info('Installing LDAP packages')
if (self.version == "5"):
self.runCmd('sudo chroot ' + localtempdir + '/temp/ yum -y install openldap-clients nss_ldap')
self.runCmd('sudo wget ' + self.http_server + '/ldap/nsswitch.conf -O ' + localtempdir + '/temp/etc/nsswitch.conf')
elif (self.version == "6"):
self.runCmd('sudo chroot ' + localtempdir + '/temp/ yum -y install openldap-clients nss-pam-ldapd sssd')
self.runCmd('sudo wget ' + self.http_server + '/ldap/nsswitch.conf_centos6 -O ' + localtempdir + '/temp/etc/nsswitch.conf')
self.runCmd('sudo wget ' + self.http_server + '/ldap/sssd.conf_centos6 -O ' + localtempdir + '/temp/etc/sssd/sssd.conf')
self.runCmd('sudo chmod 600 ' + localtempdir + '/temp/etc/sssd/sssd.conf')
self.runCmd('sudo chroot ' + localtempdir + '/temp/ chkconfig sssd on')
self.logger.info('Configuring LDAP access')
self.runCmd('sudo mkdir -p ' + localtempdir + '/temp/etc/openldap/cacerts ' + localtempdir + '/temp/N/u')
self.runCmd('sudo wget ' + self.http_server + '/ldap/cacerts/12d3b66a.0 -O ' + localtempdir + '/temp/etc/openldap/cacerts/12d3b66a.0')
self.runCmd('sudo wget ' + self.http_server + '/ldap/cacerts/cacert.pem -O ' + localtempdir + '/temp/etc/openldap/cacerts/cacert.pem')
self.runCmd('sudo wget ' + self.http_server + '/ldap/ldap.conf -O ' + localtempdir + '/temp/etc/ldap.conf')
self.runCmd('sudo wget ' + self.http_server + '/ldap/openldap/ldap.conf -O ' + localtempdir + '/temp/etc/openldap/ldap.conf')
os.system('sudo sed -i \'s/enforcing/disabled/g\' ' + localtempdir + '/temp/etc/selinux/config')
#self.runCmd('sudo wget ' + self.http_server + '/ldap/sshd_centos' + self.version + ' -O ' + localtempdir + '/temp/usr/sbin/sshd')
#os.system('echo "UseLPK yes" | sudo tee -a ' + localtempdir + '/temp/etc/ssh/sshd_config > /dev/null')
#os.system('echo "LpkLdapConf /etc/ldap.conf" | sudo tee -a ' + localtempdir + '/temp/etc/ssh/sshd_config > /dev/null')
#self.runCmd('sudo chroot ' + localtempdir + '/temp/ yum -y install fuse-sshfs')
elif self.operatingsystem == "ubuntu":
#services will install, but not start
f = open(localtempdir + '/_policy-rc.d', 'w')
f.write("#!/bin/sh" + '\n' + "exit 101" + '\n')
f.close()
self.runCmd('sudo mv -f ' + localtempdir + '/_policy-rc.d ' + localtempdir + '/temp/usr/sbin/policy-rc.d')
self.runCmd('sudo chmod +x ' + localtempdir + '/temp/usr/sbin/policy-rc.d')
self.runCmd('sudo chroot ' + localtempdir + '/temp/ apt-get -y install sshfs')
#try this other way
#chroot maverick-vm /bin/bash -c 'DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes install linux-image-server'
#env DEBIAN_FRONTEND="noninteractive" chroot /tmp/javi3789716749 /bin/bash -c 'apt-get --force-yes -y install ldap-utils libpam-ldap libpam-ldap libnss-ldap nss-updatedb libnss-db'
self.logger.info('Configuring LDAP access')
self.runCmd('sudo wget ' + self.http_server + '/ldap/nsswitch.conf -O ' + localtempdir + '/temp/etc/nsswitch.conf')
self.runCmd('sudo mkdir -p ' + localtempdir + '/temp/etc/ldap/cacerts ' + localtempdir + '/temp/N/u')
self.runCmd('sudo wget ' + self.http_server + '/ldap/cacerts/12d3b66a.0 -O ' + localtempdir + '/temp/etc/ldap/cacerts/12d3b66a.0')
self.runCmd('sudo wget ' + self.http_server + '/ldap/cacerts/cacert.pem -O ' + localtempdir + '/temp/etc/ldap/cacerts/cacert.pem')
self.runCmd('sudo wget ' + self.http_server + '/ldap/ldap.conf -O ' + localtempdir + '/temp/etc/ldap.conf')
self.runCmd('sudo wget ' + self.http_server + '/ldap/openldap/ldap.conf -O ' + localtempdir + '/temp/etc/ldap/ldap.conf')
os.system('sudo sed -i \'s/openldap/ldap/g\' ' + localtempdir + '/temp/etc/ldap/ldap.conf')
os.system('sudo sed -i \'s/openldap/ldap/g\' ' + localtempdir + '/temp/etc/ldap.conf')
self.logger.info('Installing LDAP packages')
f = open(localtempdir + '/_ldap.install', 'w')
f.write("#!/bin/bash" + '\n' + "export DEBIAN_FRONTEND=noninteractive" + '\n' + 'apt-get ' + \
'-y install ldap-utils libnss-ldapd nss-updatedb libnss-db')
f.close()
self.runCmd("sudo mv -f " + localtempdir + '/_ldap.install ' + localtempdir + '/temp/tmp/ldap.install')
os.system('sudo chmod +x ' + localtempdir + '/temp/tmp/ldap.install')
self.runCmd('sudo chroot ' + localtempdir + '/temp /tmp/ldap.install')
#I think this is not needed
#self.runCmd('wget '+ self.http_server +'/ldap/sshd_ubuntu -O ' + localtempdir + '/temp/usr/sbin/sshd')
#os.system('echo "UseLPK yes" | sudo tee -a ' + localtempdir + '/temp/etc/ssh/sshd_config > /dev/null')
#os.system('echo "LpkLdapConf /etc/ldap.conf" | sudo tee -a ' + localtempdir + '/temp/etc/ssh/sshd_config > /dev/null')
self.runCmd('sudo mv -f ' + localtempdir + '/temp/usr/sbin/policy-rc.d ' + localtempdir + '/_policy-rc.d')
self.runCmd('rm -f ' + localtempdir + '/_policy-rc.d')
end = time.time()
self.logger.info('TIME configure LDAP (this is included in the TIME customize image for specific IaaS framework):' + str(end - start))
def euca_method(self, localtempdir, ldap):
stat = 0
#Select kernel version
#This is not yet supported as we get always the same kernel
self.logger.debug("kernel: " + self.kernel)
if self.kernel == "None":
self.kernel = self.default_euca_kernel
#Inject the kernel
self.logger.info('Retrieving kernel ' + self.kernel)
stat = self.runCmd('wget ' + self.http_server + 'kernel/' + self.kernel + '.modules.tar.gz -O ' + localtempdir + '/' + self.kernel + '.modules.tar.gz')
if stat == 0:
self.runCmd('sudo tar xfz ' + localtempdir + '/' + self.kernel + '.modules.tar.gz --directory ' + localtempdir + '/temp/lib/modules/')
self.logger.info('Injected kernel ' + self.kernel)
# Setup fstab
fstab = '''
# Default fstab
/dev/sda1 / ext3 defaults,errors=remount-ro 0 0
/dev/sda2 /tmp ext3 defaults 0 0
/dev/sda3 swap swap defaults 0 0
proc /proc proc defaults 0 0
devpts /dev/pts devpts gid=5,mode=620 0 0
'''
f = open(localtempdir + '/fstab', 'w')
f.write(fstab)
f.close()
self.runCmd('sudo mv -f ' + localtempdir + '/fstab ' + localtempdir + '/temp/etc/fstab')
self.runCmd('sudo chown root:root ' + localtempdir + '/temp/etc/fstab')
self.logger.info('fstab Injected')
if self.operatingsystem == "centos":
os.system('sudo sed -i \'s/enforcing/disabled/g\' ' + localtempdir + '/temp/etc/selinux/config')
if ldap:
self.configure_ldap(localtempdir)
return stat
def nimbus_method(self, localtempdir, ldap):
#Select kernel version
#This is not yet supported as we get always the same kernel
stat = 0
self.logger.debug("kernel: " + self.kernel)
if self.kernel == "None":
self.kernel = self.default_nimbus_kernel
#Inject the kernel
self.logger.info('Retrieving kernel ' + self.kernel)
stat = self.runCmd('wget ' + self.http_server + 'kernel/' + self.kernel + '.modules.tar.gz -O ' + localtempdir + '/' + self.kernel + '.modules.tar.gz')
if stat == 0:
self.runCmd('sudo tar xfz ' + localtempdir + '/' + self.kernel + '.modules.tar.gz --directory ' + localtempdir + '/temp/lib/modules/')
self.logger.info('Injected kernel ' + self.kernel)
# Setup fstab
fstab = '''
# Default fstab
/dev/sda1 / ext3 defaults,errors=remount-ro 0 0
proc /proc proc defaults 0 0
devpts /dev/pts devpts gid=5,mode=620 0 0
'''
f = open(localtempdir + '/fstab', 'w')
f.write(fstab)
f.close()
self.runCmd('sudo mv -f ' + localtempdir + '/fstab ' + localtempdir + '/temp/etc/fstab')
self.runCmd('sudo chown root:root ' + localtempdir + '/temp/etc/fstab')
self.logger.info('fstab Injected')
if self.operatingsystem == "centos":
os.system('sudo sed -i \'s/enforcing/disabled/g\' ' + localtempdir + '/temp/etc/selinux/config')
os.system('sudo mkdir -p ' + localtempdir + "/temp/root/.ssh")
if ldap:
self.configure_ldap(localtempdir)
return stat
def openstack_method(self, localtempdir, ldap):
#Select kernel version
#This is not yet supported as we get always the same kernel
self.logger.debug("kernel: " + self.kernel)
if self.kernel == "None":
self.kernel = self.default_openstack_kernel
#Inject the kernel
self.logger.info('Retrieving kernel ' + self.kernel)
self.runCmd('wget ' + self.http_server + 'kernel/' + self.kernel + '.modules.tar.gz -O ' + localtempdir + '/' + self.kernel + '.modules.tar.gz')
self.runCmd('sudo tar xfz ' + localtempdir + '/' + self.kernel + '.modules.tar.gz --directory ' + localtempdir + '/temp/lib/modules/')
self.logger.info('Injected kernel ' + self.kernel)
# Setup fstab
fstab = '''
# Default fstab
/dev/sda1 / ext3 defaults,errors=remount-ro 0 0
proc /proc proc defaults 0 0
devpts /dev/pts devpts gid=5,mode=620 0 0
'''
#install sshfs
#we need to mount only home directory of user using sshfs. The mount an directory creation can be done before executing the job. we need to inject ssh pub/priv keys
f = open(localtempdir + '/fstab', 'w')
f.write(fstab)
f.close()
self.runCmd('sudo mv -f ' + localtempdir + '/fstab ' + localtempdir + '/temp/etc/fstab')
self.runCmd('sudo chown root:root ' + localtempdir + '/temp/etc/fstab')
self.logger.info('fstab Injected')
#install curl just in case
if self.operatingsystem == "ubuntu":
self.runCmd('sudo chroot ' + localtempdir + '/temp/ apt-get -y install curl cloud-init')
cloud_cfg = '''
cloud_type: auto
user: root
disable_root: 0
preserve_hostname: False
'''
f = open(localtempdir + '/cloud.cfg', 'w')
f.write(cloud_cfg)
f.close()
self.runCmd('sudo mv -f ' + localtempdir + '/cloud.cfg ' + localtempdir + '/temp/etc/cloud/cloud.cfg')
self.runCmd('sudo chown root:root ' + localtempdir + '/temp/etc/cloud/cloud.cfg')
elif self.operatingsystem == "centos":
#customize rc.local
rc_local = '''
route del -net 169.254.0.0 netmask 255.255.0.0 dev eth0
# load pci hotplug for dynamic disk attach in KVM (for EBS)
depmod -a
modprobe acpiphp
# simple attempt to get the user ssh key using the meta-data service
mkdir -p /root/.ssh
echo >> /root/.ssh/authorized_keys
curl -m 10 -s http://169.254.169.254/latest/meta-data/public-keys/0/openssh-key | grep 'ssh-rsa' >> /root/.ssh/authorized_keys
echo "AUTHORIZED_KEYS:"
echo "************************"
cat /root/.ssh/authorized_keys
echo "************************"
'''
f_org = open(localtempdir + '/temp/etc/rc.local', 'r')
f = open(localtempdir + '/rc.local', 'w')
write_remain = False
for line in f_org:
if (re.search('^#', line) or write_remain):
f.write(line)
else:
f.write(rc_local)
write_remain = True
f.close()
f_org.close()
self.runCmd('sudo mv -f ' + localtempdir + '/rc.local ' + localtempdir + '/temp/etc/rc.local')
self.runCmd('sudo chroot ' + localtempdir + '/temp/ cp -f /etc/rc.local /etc/rc3.d/../')
self.runCmd('sudo chown root:root ' + localtempdir + '/temp/etc/rc.local')
self.runCmd('sudo chmod 755 ' + localtempdir + '/temp/etc/rc.local')
cmd = 'echo "NOZEROCONF=yes" | sudo tee -a ' + localtempdir + '/temp/etc/sysconfig/network > /dev/null'
self.logger.debug(cmd)
os.system(cmd)
self.runCmd('sudo chroot ' + localtempdir + '/temp/ yum -y install curl')
if ldap:
self.configure_ldap(localtempdir)
def opennebula_method(self, localtempdir, ldap):
#Select kernel version
#This is not yet supported as we get always the same kernel
self.logger.debug("kernel: " + self.kernel)
#download vmcontext.sh
self.runCmd('sudo wget ' + self.http_server + "/opennebula/" + self.operatingsystem + '/vmcontext.sh -O ' + localtempdir + '/temp/etc/init.d/vmcontext.sh')
self.runCmd('sudo chmod +x ' + localtempdir + '/temp/etc/init.d/vmcontext.sh')
self.runCmd('sudo chown root:root ' + localtempdir + '/temp/etc/rc.local')
device = "sda"
rc_local = ""
if self.operatingsystem == "ubuntu":
#setup vmcontext.sh
self.runCmd("sudo sudo chroot " + localtempdir + "/temp ln -s /etc/init.d/vmcontext.sh /etc/rc2.d/S01vmcontext.sh")
device = "sda"
rc_local = "mount -t iso9660 /dev/sr0 /mnt \n"
#delete persisten network rules
self.runCmd("sudo rm -f " + localtempdir + "/temp/etc/udev/rules.d/70-persistent-net.rules")
if self.kernel == "None":
self.kernel = self.default_opennebula_kernel
elif self.operatingsystem == "centos":
#setup vmcontext.sh
self.runCmd("sudo chroot " + localtempdir + "/temp chkconfig --add vmcontext.sh")
if self.version == "5":
device = "hda"
rc_local = "mount -t iso9660 /dev/hdc /mnt \n"
elif self.version == "6":
device = "sda"
rc_local = "mount -t iso9660 /dev/sr0 /mnt \n" #in centos 6 is sr0
self.runCmd("sudo rm -f " + localtempdir + "/temp/etc/udev/rules.d/70-persistent-net.rules")
os.system('sudo sed -i \'s/enforcing/disabled/g\' ' + localtempdir + '/temp/etc/selinux/config')
if self.kernel == "None":
self.kernel = self.default_opennebula_kernel
#Inject the kernel
self.logger.info('Retrieving kernel ' + self.kernel)
self.runCmd('wget ' + self.http_server + 'kernel/' + self.kernel + '.modules.tar.gz -O ' + localtempdir + '/' + self.kernel + '.modules.tar.gz')
self.runCmd('sudo tar xfz ' + localtempdir + '/' + self.kernel + '.modules.tar.gz --directory ' + localtempdir + '/temp/lib/modules/')
self.logger.info('Injected kernel ' + self.kernel)
#customize rc.local
rc_local += "if [ -f /mnt/context.sh ]; then \n"
rc_local += " . /mnt/init.sh \n"
rc_local += "fi \n"
rc_local += "umount /mnt \n\n"
f_org = open(localtempdir + '/temp/etc/rc.local', 'r')
f = open(localtempdir + '/rc.local', 'w')
write_remain = False
for line in f_org:
if (re.search('^#', line) or write_remain):
f.write(line)
else:
f.write(rc_local)
write_remain = True
f.close()
f_org.close()
self.runCmd('sudo mv -f ' + localtempdir + '/rc.local ' + localtempdir + '/temp/etc/rc.local')
self.runCmd('sudo chown root:root ' + localtempdir + '/temp/etc/rc.local')
self.runCmd('sudo chmod 755 ' + localtempdir + '/temp/etc/rc.local')
#in centos /etc/rc.local is a symbolink link of /etc/rc.d/rc.local/
os.system('sudo chroot ' + localtempdir + '/temp/ cp -f /etc/rc.local /etc/rc3.d/../')
# Setup fstab
fstab = "# Default fstab \n "
fstab += "/dev/" + device + " / ext3 defaults,errors=remount-ro 0 0 \n"
fstab += "proc /proc proc defaults 0 0 \n"
fstab += "devpts /dev/pts devpts gid=5,mode=620 0 0 \n"
#if ldap: #this is for india
# fstab+="149.165.146.145:/users /N/u nfs rw,rsize=1048576,wsize=1048576,intr,nosuid"
f = open(localtempdir + '/fstab', 'w')
f.write(fstab)
f.close()
self.runCmd('sudo mv -f ' + localtempdir + '/fstab ' + localtempdir + '/temp/etc/fstab')
self.runCmd('sudo chown root:root ' + localtempdir + '/temp/etc/fstab')
self.logger.info('fstab Injected')
def handle_image(self, image, localtempdir, connstream):
#print image
success = True
"""
urlparts = image.split("/")
#print urlparts
self.logger.debug("urls parts: " + str(urlparts))
if len(urlparts) == 1:
nameimg = urlparts[0].split(".")[0]
elif len(urlparts) == 2:
nameimg = urlparts[1].split(".")[0]
else:
nameimg = urlparts[len(urlparts) - 1].split(".")[0]
self.logger.debug("image name " + nameimg)
localtempdir = self.tempdir + "/" + nameimg + "_0"
if os.path.isfile(localtempdir):
exists = True
i = 0
while (exists):
aux = fulldestpath + "_" + i.__str__()
if os.path.isfile(aux):
i += 1
else:
exists = False
localtempdir = aux + "/"
cmd = 'mkdir -p ' + localtempdir
self.runCmd(cmd)
"""
realnameimg = ""
self.logger.info('untar file with image and manifest')
cmd = "tar xvfz " + image + " -C " + localtempdir
self.logger.debug(cmd)
p = Popen(cmd.split(' '), stdout=PIPE, stderr=PIPE)
std = p.communicate()
stat = 0
if len(std[0]) > 0:
realnameimg = std[0].split("\n")[0].strip().split(".")[0]
if p.returncode != 0:
self.logger.error('Command: ' + cmd + ' failed, status: ' + str(p.returncode) + ' --- ' + std[1])
stat = 1
cmd = 'rm -f ' + image
status = self.runCmd(cmd)
if (stat != 0):
msg = "ERROR: the files were not extracted"
self.errormsg(connstream, msg)
return False
self.manifestname = realnameimg + ".manifest.xml"
manifestfile = open(localtempdir + "/" + self.manifestname, 'r')
manifest = parse(manifestfile)
self.name = ""
self.givenname = ""
self.operatingsystem = ""
self.version = ""
self.arch = ""
self.name = manifest.getElementsByTagName('name')[0].firstChild.nodeValue.strip()
self.givenname = manifest.getElementsByTagName('givenname')
self.operatingsystem = manifest.getElementsByTagName('os')[0].firstChild.nodeValue.strip()
self.version = manifest.getElementsByTagName('version')[0].firstChild.nodeValue.strip()
self.arch = manifest.getElementsByTagName('arch')[0].firstChild.nodeValue.strip()
#kernel = manifest.getElementsByTagName('kernel')[0].firstChild.nodeValue.strip()
self.logger.debug(self.name + " " + self.operatingsystem + " " + self.version + " " + self.arch)
#create rootimg and temp directories
cmd = 'mkdir -p ' + localtempdir + '/temp'
status = self.runCmd(cmd)
if status != 0:
msg = "ERROR: creating temp directory inside " + localtempdir
self.errormsg(connstream, msg)
return False
#mount image to extract files
cmd = 'sudo mount -o loop ' + localtempdir + '/' + self.name + '.img ' + localtempdir + '/temp'
status = self.runCmd(cmd)
if status != 0:
msg = "ERROR: mounting image"
self.errormsg(connstream, msg)
return False
#Mount proc and pts
#runCmd('mount -t proc proc '+localtempdir + '/temp/proc')
#runCmd('mount -t devpts devpts '+localtempdir + '/temp/dev/pts')
return True
def errormsg(self, connstream, msg):
self.logger.error(msg)
try:
connstream.write(msg)
connstream.shutdown(socket.SHUT_RDWR)
connstream.close()
except:
self.logger.debug("In errormsg: " + str(sys.exc_info()))
self.logger.info("Image Register Request DONE")
def runCmd(self, cmd):
cmdLog = logging.getLogger('RegisterIaaS.' + str(os.getpid()) + '.exec')
cmdLog.debug(cmd)
p = Popen(cmd.split(' '), stdout=PIPE, stderr=PIPE)
std = p.communicate()
status = 0
if len(std[0]) > 0:
cmdLog.debug('stdout: ' + std[0])
cmdLog.debug('stderr: ' + std[1])
if p.returncode != 0:
cmdLog.error('Command: ' + cmd + ' failed, status: ' + str(p.returncode) + ' --- ' + std[1])
status = 1
#sys.exit(p.returncode)
return status
