def check_register_key(self, username, cloudname, keyname, keycontent): self.mongo.activate(cm_user_id=username, names=[cloudname]) cloudmanager = self.mongo.clouds[username][cloudname]['manager'] keynamenew = _keyname_sanitation(username, keyname) keysRegistered = cloudmanager.keypair_list() registered = False # Openstack & Eucalyptus if 'keypairs' in keysRegistered: keypairsRegistered = keysRegistered["keypairs"] for akeypair in keypairsRegistered: if keynamenew == akeypair['keypair']['name']: registered = True break else: if keynamenew in keysRegistered: registered = True if not registered: cloudmanager.keypair_add(keynamenew, keycontent) log.info("Automatically registered the default key <%s> for user <%s>" % ( keyname, username))
def do_launcher(self, args, arguments): """ :: Usage: launcher start MENU launcher stop STACK_NAME launcher list launcher show STACK_NAME launcher menu [--column=COLUMN] [--format=FORMAT] launcher import [FILEPATH] [--force] launcher export FILEPATH launcher help | -h An orchestration tool with Chef Cookbooks Arguments: MENU Name of a cookbook STACK_NAME Name of a launcher FILEPATH Filepath COLUMN column name to display FORMAT display format (json, table) help Prints this message Options: -v verbose mode """ log.info(arguments) self.cm_mongo = cm_mongo() self.cm_config = cm_config() self.cm_user = cm_user() self._id = "t_stacks" if arguments["help"] or arguments["-h"]: print(self.do_launcher.__doc__) elif arguments['show'] and arguments['STACK_NAME']: print("NOT IMPLEMENTED") return elif arguments['menu']: userid = self.cm_config.username() launchers = self.cm_mongo.launcher_get(userid) if launchers.count() == 0: Console.warning( "no launcher in database, please import launcher first" "(launcher import [FILEPATH] [--force])") return else: d = {} for launcher in launchers: d[launcher['cm_launcher']] = launcher if "_id" in d[launcher['cm_launcher']]: del d[launcher['cm_launcher']]['_id'] columns = None if arguments['--column']: if arguments['--column'] != "all": columns = [ x.strip() for x in arguments['--column'].split(',') ] else: columns = ['name', 'description'] if arguments['--format']: if arguments['--format'] not in ['table', 'json', 'csv']: Console.error("please select printing format ", "among table, json and csv") return else: p_format = arguments['--format'] else: p_format = None shell_commands_dict_output(userid, d, print_format=p_format, firstheader="launcher", header=columns # vertical_table=True ) elif arguments['list']: userid = self.cm_config.username() self.cm_mongo.activate(userid) self.cm_mongo.refresh(cm_user_id=userid, types=[self._id]) stacks = self.cm_mongo.stacks(cm_user_id=userid) launchers = self.filter_launcher(stacks, { "search": "contain", "key": "stack_name", "value": "launcher" }) log.debug(launchers) d = {} for k0, v0 in launchers.iteritems(): for k1, v1 in launchers[k0].iteritems(): d[v1['id']] = v1 columns = [ 'stack_name', 'description', 'stack_status', 'creation_time', 'cm_cloud' ] if arguments['--column'] and arguments['--column'] != "all": columns = [x.strip() for x in arguments['--column'].split(',')] if arguments['--format']: if arguments['--format'] not in ['table', 'json', 'csv']: Console.error( "please select printing format among table, json and csv" ) return else: p_format = arguments['--format'] else: p_format = None shell_commands_dict_output(userid, d, print_format=p_format, firstheader="launcher_id", header=columns # vertical_table=True ) elif arguments['start'] and arguments['MENU']: userid = self.cm_config.username() def_cloud = self.get_cloud_name(userid) self.cm_mongo.activate(userid) config_launcher = cm_config_launcher() userinfo = self.cm_user.info(userid) if "key" in userinfo["defaults"]: key = userinfo["defaults"]["key"] elif len(userinfo["keys"]["keylist"].keys()) > 0: key = userinfo["keys"]["keylist"].keys()[0] if key: keycontent = userinfo["keys"]["keylist"][key] if keycontent.startswith('key '): keycontent = keycontent[4:] cm_keys_mongo(userid).check_register_key( userid, def_cloud, key, keycontent) keynamenew = _keyname_sanitation(userid, key) else: Console.warning("No sshkey found. Please Upload one") cookbook = arguments['MENU'] s_name = "launcher-{0}-{1}-{2}".format(userid, cookbook, get_rand_string()) dummy = "123456789" # doing nothing. just for test try: t_url = \ config_launcher['cloudmesh']['launcher']['default']['template'] except: # If key is missing (KeyError), new cloudmesh_launcher.yaml # needs to be copied to ~/.cloudmesh t_url = \ "https://raw.githubusercontent.com/cloudmesh/cloudmesh/master/heat-templates/centos6/launcher/launcher.yaml" param = { 'KeyName': keynamenew, 'Cookbook': cookbook, 'dummy': dummy } # test for openmpi, hadoop if cookbook[:6] == "hadoop" or cookbook[:7] == "openmpi": privatekey, publickey = generate_keypair() try: t_url = \ config_launcher['cloudmesh']['launcher']['recipes'][cookbook]['template'] except: # If key is missing (KeyError), new cloudmesh_launcher.yaml # needs to be copied to ~/.cloudmesh t_url = \ ("https://raw.githubusercontent.com/cloudmesh/cloudmesh/master/heat-templates/ubuntu-14.04/" + str(cookbook) + "-cluster/" + str(cookbook) + "-cluster.yaml") param = { 'KeyName': keynamenew, 'PublicKeyString': publickey, 'PrivateKeyString': privatekey } if cookbook[:9] == "hadoop2.7": param["UserName"] = userid log.debug(def_cloud, userid, s_name, t_url, param) res = self.cm_mongo.stack_create(cloud=def_cloud, cm_user_id=userid, servername=s_name, template_url=t_url, parameters=param) log.debug(res) if 'error' in res: print(res['error']['message']) return res elif arguments['stop'] and arguments['STACK_NAME']: userid = self.cm_config.username() def_cloud = self.get_cloud_name(userid) s_id = arguments['STACK_NAME'] self.cm_mongo.activate(userid) res = self.cm_mongo.stack_delete(cloud=def_cloud, cm_user_id=userid, server=s_id) log.debug(res) return res elif arguments['import']: filepath = config_file("/cloudmesh_launcher.yaml") if arguments['FILEPATH']: filepath = arguments['FILEPATH'] try: filename = path_expand(filepath) fileconfig = ConfigDict(filename=filename) except Exception, err: Console.error( "error while loading '{0}', please check".format(filepath)) print(traceback.format_exc()) print(sys.exc_info()[0]) return try: recipes_dict = fileconfig.get("cloudmesh", "launcher", "recipies") except: Console.error("error while loading recipies from the file") # print recipes_dict userid = self.cm_config.username() launcher_names = [] launchers = self.cm_mongo.launcher_get(userid) for launcher in launchers: launcher_names.append(launcher['cm_launcher'].encode("ascii")) for key in recipes_dict: if key in launcher_names: if arguments['--force']: self.cm_mongo.launcher_remove(userid, key) self.cm_mongo.launcher_import(recipes_dict[key], key, userid) print("launcher '{0}' overwritten.".format(key)) else: print("ERROR: launcher '{0}' exists, " "please remove it first, or use " "'--force' when adding".format(key)) else: self.cm_mongo.launcher_import(recipes_dict[key], key, userid) print("launcher '{0}' added.".format(key))
def do_launcher(self, args, arguments): """ Usage: launcher start MENU launcher stop STACK_NAME launcher list launcher show STACK_NAME launcher menu [--column=COLUMN] [--format=FORMAT] launcher import [FILEPATH] [--force] launcher export FILEPATH launcher help | -h An orchestration tool with Chef Cookbooks Arguments: MENU Name of a cookbook STACK_NAME Name of a launcher FILEPATH Filepath COLUMN column name to display FORMAT display format (json, table) help Prints this message Options: -v verbose mode """ log.info(arguments) self.cm_mongo = cm_mongo() self.cm_config = cm_config() self.user = cm_user() if arguments["help"] or arguments["-h"]: print (self.do_launcher.__doc__) elif arguments['show'] and arguments['STACK_NAME']: print ("NOT IMPLEMENTED") return elif arguments['menu']: userid = self.cm_config.username() launchers = self.cm_mongo.launcher_get(userid) if launchers.count() == 0: Console.warning("no launcher in database, please import launcher first" "(launcher import [FILEPATH] [--force])") return else: d = {} for launcher in launchers: d[launcher['cm_launcher']] = launcher if "_id" in d[launcher['cm_launcher']]: del d[launcher['cm_launcher']]['_id'] columns = None if arguments['--column']: if arguments['--column'] != "all": columns = [x.strip() for x in arguments['--column'].split(',')] else: columns = ['name', 'description'] if arguments['--format']: if arguments['--format'] not in ['table', 'json', 'csv']: Console.error("please select printing format ", "among table, json and csv") return else: p_format = arguments['--format'] else: p_format = None shell_commands_dict_output(userid, d, print_format=p_format, firstheader="launcher", header=columns # vertical_table=True ) elif arguments['list']: userid = self.cm_config.username() self.cm_mongo.activate(userid) self.cm_mongo.refresh(cm_user_id=userid, types=[self._id]) stacks = self.cm_mongo.stacks(cm_user_id=userid) launchers = self.filter_launcher( stacks, {"search": "contain", "key": "stack_name", "value": "launcher"} ) log.debug(launchers) d = {} for k0, v0 in launchers.iteritems(): for k1, v1 in launchers[k0].iteritems(): d[v1['id']] = v1 columns = ['stack_name', 'description', 'stack_status', 'creation_time', 'cm_cloud'] if arguments['--column'] and arguments['--column'] != "all": columns = [x.strip() for x in arguments['--column'].split(',')] if arguments['--format']: if arguments['--format'] not in ['table', 'json', 'csv']: Console.error("please select printing format among table, json and csv") return else: p_format = arguments['--format'] else: p_format = None shell_commands_dict_output(userid, d, print_format=p_format, firstheader="launcher_id", header=columns # vertical_table=True ) elif arguments['start'] and arguments['MENU']: userid = self.cm_config.username() def_cloud = self.get_cloud_name(userid) self.cm_mongo.activate(userid) userinfo = self.user.info(userid) if "key" in userinfo["defaults"]: key = userinfo["defaults"]["key"] elif len(userinfo["keys"]["keylist"].keys()) > 0: key = userinfo["keys"]["keylist"].keys()[0] if key: keycontent = userinfo["keys"]["keylist"][key] if keycontent.startswith('key '): keycontent = keycontent[4:] cm_keys_mongo(userid).check_register_key(userid, def_cloud, key, keycontent) keynamenew = _keyname_sanitation(userid, key) else: Console.warning("No sshkey found. Please Upload one") cookbook = arguments['MENU'] s_name = "launcher-{0}-{1}-{2}".format(userid, cookbook, get_rand_string()) dummy = "123456789" # doing nothing. just for test t_url = \ "https://raw.githubusercontent.com/cloudmesh/cloudmesh/master/heat-templates/centos6/launcher/launcher.yaml" param = {'KeyName': keynamenew, 'Cookbook': cookbook, 'dummy': dummy} log.debug(def_cloud, userid, s_name, t_url, param) res = self.cm_mongo.stack_create(cloud=def_cloud, cm_user_id=userid, servername=s_name, template_url=t_url, parameters=param) log.debug(res) if 'error' in res: print (res['error']['message']) return res elif arguments['stop'] and arguments['STACK_NAME']: userid = self.cm_config.username() def_cloud = self.get_cloud_name(userid) s_id = arguments['STACK_NAME'] self.cm_mongo.activate(userid) res = self.cm_mongo.stack_delete(cloud=def_cloud, cm_user_id=userid, server=s_id) log.debug(res) return res elif arguments['import']: filepath = config_file("/cloudmesh_launcher.yaml") if arguments['FILEPATH']: filepath = arguments['FILEPATH'] try: filename = path_expand(filepath) fileconfig = ConfigDict(filename=filename) except Exception, err: Console.error( "error while loading '{0}', please check".format(filepath)) print (traceback.format_exc()) print (sys.exc_info()[0]) return try: recipes_dict = fileconfig.get("cloudmesh", "launcher", "recipies") except: Console.error("error while loading recipies from the file") # print recipes_dict userid = self.cm_config.username() launcher_names = [] launchers = self.cm_mongo.launcher_get(userid) for launcher in launchers: launcher_names.append(launcher['cm_launcher'].encode("ascii")) for key in recipes_dict: if key in launcher_names: if arguments['--force']: self.cm_mongo.launcher_remove(userid, key) self.cm_mongo.launcher_import( recipes_dict[key], key, userid) print ("launcher '{0}' overwritten.".format(key)) else: print ("ERROR: launcher '{0}' exists, " "please remove it first, or use " "'--force' when adding".format(key)) else: self.cm_mongo.launcher_import( recipes_dict[key], key, userid) print ("launcher '{0}' added.".format(key))
def start_vm(username, cloudname, count=1, flavorname=None, flavorid=None, imagename=None, imageid=None, groupname=None, servername=None): ''' create a vm of a cloud of a user will check flavor, image existence if provided user can specify groupname which will be written in metadata, servername which will replace prefix+index as the vm name it's better to check cloud active status before use this function :param username: string :param cloudname: string :param count: number of vms to start :return False if error TODO: what if fail, how to acknowledge it; no return now as using celery input key missing security group ''' # Changed scope of this import - hyungro lee 12/01/2014 from cloudmesh.experiment.group_usage import add_vm_to_group_while_creating mongo = cm_mongo() userobj = cm_user() cloudobj = CloudManage() mongo.activate(cm_user_id=username, names=[cloudname]) userinfo = userobj.info(username) key = None vm_image_id = None vm_flavor_id = None error = '' # ------------------------- # refresh server flavor or image to_refresh = ["servers"] if flavorname is not None or flavorid is not None: to_refresh.append("flavors") if imagename is not None or imageid is not None: to_refresh.append("images") if to_refresh != []: mongo.refresh(username, names=[cloudname], types=to_refresh) # ------------------------- # get exist VM names list, to prevent names duplicate serverdata = mongo.servers( clouds=[cloudname], cm_user_id=username)[cloudname] servers_names_list = [] for k,v in serverdata.iteritems(): servers_names_list.append(v['name']) # ------------------------- # flavor handler if flavorname is not None or flavorid is not None: flavordata = mongo.flavors( clouds=[cloudname], cm_user_id=username)[cloudname] same_name_count = 0 for k, v in flavordata.iteritems(): if flavorname is not None: if flavorname == v['name']: vm_flavor_id = k same_name_count = same_name_count + 1 else: if flavorid == k: vm_flavor_id = k break if vm_flavor_id is None: error = error + "The flavor you provide doesn't exist. " if same_name_count > 1: error = error + "There are more than one flavor with the name you provide" \ "please use flavorid instead or select one by command cloud" \ "set flavor [CLOUD]. " else: try: vm_flavor_id = userinfo["defaults"]["flavors"][cloudname] except: pass if vm_flavor_id in [None, 'none']: error = error + \ "Please specify a default flavor(command: cloud set flavor [CLOUD]). " # ------------------------- # image handler if imagename is not None or imageid is not None: imagedata = mongo.images( clouds=[cloudname], cm_user_id=username)[cloudname] same_name_count = 0 for k, v in imagedata.iteritems(): if imagename is not None: if imagename == v['name']: vm_image_id = k same_name_count = same_name_count + 1 else: if imageid == k: vm_image_id = k break if vm_image_id is None: error = error + "The image you provide doesn't exist. " if same_name_count > 1: error = error + "There are more than one image with the name you provide" \ "please use imageid instead or select one by command cloud" \ "set image [CLOUD]. " else: try: vm_image_id = userinfo["defaults"]["images"][cloudname] except: pass if vm_image_id in [None, 'none']: error = error + \ "Please specify a default image(command: cloud set flavor [CLOUD]). " # ------------------------- # key handler if "key" in userinfo["defaults"]: key = userinfo["defaults"]["key"] elif len(userinfo["keys"]["keylist"].keys()) > 0: key = userinfo["keys"]["keylist"].keys()[0] if key: keycontent = userinfo["keys"]["keylist"][key] if keycontent.startswith('key '): keycontent = keycontent[4:] cm_keys_mongo(username).check_register_key(username, cloudname, key, keycontent) keynamenew = _keyname_sanitation(username, key) else: error = error + \ "No sshkey found. Please Upload one" # ------------------------- if error != '': Console.error(error) return False # ------------------------- metadata = {'cm_owner': username} if groupname: metadata['cm_group'] = groupname tmpnamefl = cloudobj.get_flavors( cloudname=cloudname, getone=True, id=vm_flavor_id)['name'] tmpnameim = cloudobj.get_images( cloudname=cloudname, getone=True, id=vm_image_id)['name'] while count > 0: userinfo = userobj.info(username) if servername: prefix = '' index = '' givenvmname = servername tmpnameser = servername else: prefix = userinfo["defaults"]["prefix"] index = userinfo["defaults"]["index"] givenvmname = None tmpnameser = prefix + '_' + str(index) # ------------------------ # do not allow server name duplicate if tmpnameser in servers_names_list: Console.error("vm name '{0}' exists, please use other names or delete it first".format( tmpnameser)) if not servername: userobj.set_default_attribute(username, "index", int(index) + 1) count = count - 1 continue # ------------------------ # vm start procedure banner("Starting vm->{0} on cloud->{1} using image->{2}, flavor->{3}, key->{4}" .format(tmpnameser, cloudname, tmpnameim, tmpnamefl, keynamenew)) useQueue = False if useQueue: result = mongo.vm_create_queue(cloudname, prefix, index, vm_flavor_id, vm_image_id, keynamenew, meta=metadata, cm_user_id=username, givenvmname=givenvmname) print("job status:", result.state) else: result = mongo.vm_create(cloudname, prefix, index, vm_flavor_id, vm_image_id, keynamenew, meta=metadata, cm_user_id=username, givenvmname=givenvmname) if "server" in result and "adminPass" in result["server"]: result["server"]["adminPass"] = "******" pprint(result) # ------------------------ # add it to the group in database if groupname provided if groupname: try: add_vm_to_group_while_creating(username, groupname, tmpnameser) except Exception, err: Console.error(str(err)) return # ------------------------ # increase index if it is used if not servername: userobj.set_default_attribute(username, "index", int(index) + 1) # ------------------------ servers_names_list.append(tmpnameser) count = count - 1
def do_cluster(self, args, arguments): """ Usage: cluster start CLUSTER_NAME cluster list cluster login CLUSTER_NAME cluster stop CLUSTER_NAME cluster create --count=<count> --group=<group> [--ln=<LoginName>] [--cloud=<CloudName>] [--image=<imgName>|--imageid=<imgId>] [--flavor=<flavorName>|--flavorid=<flavorId>] [--force] Description: Cluster Management cluster create --count=<count> --group=<group> --ln=<LoginName> [options...] <count> specify amount of VMs in the cluster <group> specify a group name of the cluster, make sure it's unique Start a cluster of VMs, and each of them can log into all others. CAUTION: you sould do some default setting before using this command: 1. select cloud to work on, e.g. cloud select india 2. activate the cloud, e.g. cloud on india 3. set the default key to start VMs, e.g. key default [NAME] 4. set the start name of VMs, which is prefix and index, e.g. label --prefix=test --id=1 5. set image of VMs, e.g. default image 6. set flavor of VMs, e.g. default flavor Also, please make sure the group name of the cluster is unique Options: --ln=<LoginName> give a login name for the VMs, e.g. ubuntu --cloud=<CloudName> give a cloud to work on --flavor=<flavorName> give the name of the flavor --flavorid=<flavorId> give the id of the flavor --image=<imgName> give the name of the image --imageid=<imgId> give the id of the image --force if a group exists and there are VMs in it, the program will ask user to proceed or not, use this flag to respond yes as default(if there are VMs in the group before creating this cluster, the program will include the exist VMs into the cluster) """ #pprint(arguments) self.cm_config = cm_config() self.cm_mongo = cm_mongo() self.user = cm_user() # ----------------------------- # TODO:: # add VMs to cluster # ----------------------------- if arguments['start'] and arguments['CLUSTER_NAME']: '''Starts a cluster''' # Initialize default variables. e.g. userid, default cloud and # default keypair userid = self.cm_config.username() def_cloud = self.get_cloud_name(userid) self.cm_mongo.activate(userid) userinfo = self.user.info(userid) if "key" in userinfo["defaults"]: key = userinfo["defaults"]["key"] elif len(userinfo["keys"]["keylist"].keys()) > 0: key = userinfo["keys"]["keylist"].keys()[0] if key: keycontent = userinfo["keys"]["keylist"][key] if keycontent.startswith('key '): keycontent = keycontent[4:] cm_keys_mongo(userid).check_register_key(userid, def_cloud, key, keycontent) keynamenew = _keyname_sanitation(userid, key) else: Console.warning("No sshkey found. Please Upload one") return clustername = arguments['CLUSTER_NAME'] s_name = "cluster-{0}-{1}-{2}".format(userid, clustername, get_rand_string()) # TEMP FOR HADOOP CLUSTER if clustername != "hadoop": Console.warning('hadoop is only available cluster') return # 1. keypair for the communication between master and worker nodes privatekey, publickey = generate_keypair() t_url = \ "https://raw.githubusercontent.com/cloudmesh/cloudmesh/dev1.3/heat-templates/ubuntu-14.04/hadoop-cluster/hadoop-cluster.yaml" param = {'KeyName': keynamenew, 'PublicKeyString': publickey, 'PrivateKeyString': privatekey} log.debug(def_cloud, userid, s_name, t_url, param, publickey, privatekey) res = self.cm_mongo.stack_create(cloud=def_cloud, cm_user_id=userid, servername=s_name, template_url=t_url, parameters=param) log.debug(res) if 'error' in res: print (res['error']['message']) return res elif arguments['list']: userid = self.cm_config.username() self.cm_mongo.activate(userid) self.cm_mongo.refresh(cm_user_id=userid, types=[self._id]) stacks = self.cm_mongo.stacks(cm_user_id=userid) launchers = self.filter_launcher( stacks, {"search": "contain", "key": "stack_name", "value": "launcher"} ) log.debug(launchers) d = {} for k0, v0 in launchers.iteritems(): for k1, v1 in launchers[k0].iteritems(): d[v1['id']] = v1 columns = ['stack_name', 'description', 'stack_status', 'creation_time', 'cm_cloud'] if arguments['--column'] and arguments['--column'] != "all": columns = [x.strip() for x in arguments['--column'].split(',')] if arguments['--format']: if arguments['--format'] not in ['table', 'json', 'csv']: Console.error("please select printing format among table, json and csv") return else: p_format = arguments['--format'] else: p_format = None shell_commands_dict_output(d, print_format=p_format, firstheader="launcher_id", header=columns # vertical_table=True ) elif arguments['login']: Console.error("Not implemented") return elif arguments['stop'] and arguments['CLUSTER_NAME']: userid = self.cm_config.username() def_cloud = self.get_cloud_name(userid) c_id = arguments['CLUSTER_NAME'] self.cm_mongo.activate(userid) res = self.cm_mongo.stack_delete(cloud=def_cloud, cm_user_id=userid, server=c_id) log.debug(res) return res elif arguments['create']: try: config = cm_config() except: Console.error("Failed to load the cloudmesh yaml file") return username = config['cloudmesh']['profile']['username'] cloudname = arguments['--cloud'] or CloudManage().get_selected_cloud(username) temp_dir_name = ".temp_cluster_create_" + username + "_0" while os.path.isdir("./{0}".format(temp_dir_name)): temp_dir_name = temp_dir_name[:-1] + str(int(temp_dir_name[-1]) + 1) dir_name = temp_dir_name #NumOfVM = None GroupName = None vm_login_name = "ubuntu" temp_key_name = "sshkey_temp" _key = "-i ./{0}/{1}".format(dir_name, temp_key_name) StrictHostKeyChecking = "-o StrictHostKeyChecking=no" res = None to_print = [] ''' try: NumOfVM = abs(int(argument['--count'])) except: Console.error("<count> must be an integer") return ''' if arguments['--group'] == '': Console.error("<group> cannot be empty") return else: GroupName = arguments['--group'] if arguments['--ln']: if arguments['--ln'] == '': Console.error("<LoginName> cannot be empty") return else: vm_login_name = arguments['--ln'] if not arguments['--force']: # Moved the import inside of this function # If this import goes up to the top, monodb connection will be # estabilished. Due to that reason, this import stays here # Hyungro Lee 12/01/2014 from cloudmesh.experiment.group import GroupManagement GroupManage = GroupManagement(username) groups_list = GroupManage.get_groups_names_list() if GroupName in groups_list: vms_in_group_list = GroupManage.list_items_of_group(GroupName, _type="VM")["VM"] if len(vms_in_group_list) != 0: if yn_choice("The group you provide exists and it has VMs in it, " + \ "do you want to proceed? (if you choose yes, these exist " +\ "VMs will be included in the cluster, this could also " +\ "rewrite the key on the exist VMs)", default='n', tries=3): pass else: return # start VMs print ("starting VMs...") arguments_temp = arguments arguments_temp['start'] = True arguments_temp['--name'] = None vmclass = VMcommand(arguments_temp) res = vmclass.call_procedure() if res == False: return def string_to_dict(s): h = s.find("{") t = s.rfind("}") return json.loads(s[h:t+1]) def check_public_ip_existence(d): temp = d['addresses']['private'] for item in temp: if item["OS-EXT-IPS:type"] == "floating": return True return False def get_ip(d, kind="floating"): # kind is either floating or fixed temp = d['addresses']['private'] for item in temp: if item["OS-EXT-IPS:type"] == kind: return item['addr']#.encode('ascii') return "FAIL: doesn't exist" # check all VMs are active command_refresh = "vm list --refresh --group={0} --format=json".format(GroupName) def _help0(d): for k, v in d.iteritems(): if v['status'] != 'ACTIVE': return False return True proceed = False repeat_index = 1 while proceed != True: if repeat_index > 10: Console.warning("Please check the network") return print ("checking({0})...".format(repeat_index)) time.sleep(5) res = str(cm(command_refresh)) res = string_to_dict(res) if _help0(res): proceed = True else: repeat_index = repeat_index + 1 continue # assign ip to all VMs print ("assigning public ips...") for k, v in res.iteritems(): if not check_public_ip_existence(v): cm("vm ip assign --id={0}".format(k.encode('ascii'))) def _help(d): for k, v in d.iteritems(): if check_public_ip_existence(v) != True: return False return True # make sure all VMs have been assigned a public ip proceed = False repeat_index = 1 while proceed != True: if repeat_index > 10: Console.warning("Please check the network") return print ("checking({0})...".format(repeat_index)) time.sleep(5) res = str(cm(command_refresh)) res = string_to_dict(res) if _help(res): proceed = True else: repeat_index = repeat_index + 1 continue # ------------------------- # key handler userinfo = cm_user().info(username) key = None if "key" in userinfo["defaults"]: key = userinfo["defaults"]["key"] elif len(userinfo["keys"]["keylist"].keys()) > 0: key = userinfo["keys"]["keylist"].keys()[0] Console.warning("default key is not set, trying to use a key in the database...") if key: keycontent = userinfo["keys"]["keylist"][key] if keycontent.startswith('key '): keycontent = keycontent[4:] cm_keys_mongo(username).check_register_key(username, cloudname, key, keycontent) else: Console.error("No sshkey found. Please Upload one") return # ------------------------- # generate ssh keys for VMs and prepare two files: authorized_keys and hosts print ("generating ssh keys...") os.popen("mkdir {0}".format(dir_name)) fa = open("./{0}/authorized_keys_temp".format(dir_name), "w") fh = open("./{0}/hosts_temp".format(dir_name), "w") fk = open("./{0}/{1}".format(dir_name, temp_key_name), "w") fk.write(keycontent) fk.close() os.popen("chmod 644 ./{0}/{1}".format(dir_name, temp_key_name)) for k, v in res.iteritems(): address_floating = get_ip(v) address_fixed = get_ip(v, kind="fixed") vm_name = v['name']#.encode('ascii') to_print.append("{0} {1}, {2}".format(vm_name, address_floating, address_fixed)) fh.write(address_floating + " " + vm_name + "\n" + address_fixed + " " + vm_name + "-i\n") os.popen("ssh {2} {3} {0}@{1} \"ssh-keygen -t rsa -N '' -f ~/.ssh/id_rsa\""\ .format(vm_login_name,address_floating, _key, StrictHostKeyChecking)) temp = os.popen("ssh {2} {3} {0}@{1} \"cat ~/.ssh/id_rsa.pub\""\ .format(vm_login_name, address_floating, _key, StrictHostKeyChecking)).read() fa.write(temp) fa.close() fh.close() # copy the files to VMs print ("copying the files...") os.popen("mkdir ./{0}/oops".format(dir_name)) for k, v in res.iteritems(): address_floating = get_ip(v) os.popen("scp {2} {3} {0}@{1}:~/.ssh/authorized_keys ./{4}/"\ .format(vm_login_name,address_floating, _key, StrictHostKeyChecking, dir_name)) os.popen("cat ./{0}/authorized_keys_temp >> ./{0}/authorized_keys"\ .format(dir_name)) os.popen("scp {2} {3} ./{4}/authorized_keys {0}@{1}:~"\ .format(vm_login_name,address_floating, _key, StrictHostKeyChecking, dir_name)) os.popen("ssh {2} {3} {0}@{1} \"sudo mv authorized_keys ~/.ssh/\""\ .format(vm_login_name,address_floating, _key, StrictHostKeyChecking)) os.popen("rm ./{0}/authorized_keys".format(dir_name)) os.popen("cp ./{0}/hosts_temp ./{0}/oops/".format(dir_name)) os.popen("mv ./{0}/oops/hosts_temp ./{0}/oops/hosts".format(dir_name)) fh0 = open("./{0}/oops/hosts".format(dir_name), "a") os.popen("scp {2} {3} {0}@{1}:/etc/hosts ./{4}/"\ .format(vm_login_name,address_floating, _key, StrictHostKeyChecking, dir_name)) with open("./{0}/hosts".format(dir_name)) as f0: content = f0.readlines() for item in content: fh0.write(item + "\n") fh0.close() os.popen("scp {2} {3} ./{4}/oops/hosts {0}@{1}:~"\ .format(vm_login_name,address_floating, _key, StrictHostKeyChecking, dir_name)) os.popen("ssh {2} {3} {0}@{1} \"sudo mv hosts /etc/\""\ .format(vm_login_name,address_floating, _key, StrictHostKeyChecking)) os.popen("rm ./{0}/oops/hosts".format(dir_name)) print ("finishing...") os.popen("rm -rf {0}".format(dir_name)) print ("DONE.") print ("cluster group: ", GroupName) for item in to_print: print (item) print ("(host name for private ips will have -i at the end of VM name, e.g. testVM -> testVM-i)")
def do_launcher(self, args, arguments): """ :: Usage: launcher start MENU launcher stop STACK_NAME launcher list launcher show STACK_NAME launcher menu [--column=COLUMN] [--format=FORMAT] launcher import [FILEPATH] [--force] launcher export FILEPATH launcher help | -h An orchestration tool with Chef Cookbooks Arguments: MENU Name of a cookbook STACK_NAME Name of a launcher FILEPATH Filepath COLUMN column name to display FORMAT display format (json, table) help Prints this message Options: -v verbose mode """ log.info(arguments) self.cm_mongo = cm_mongo() self.cm_config = cm_config() self.cm_user = cm_user() self._id = "t_stacks" if arguments["help"] or arguments["-h"]: print(self.do_launcher.__doc__) elif arguments["show"] and arguments["STACK_NAME"]: print("NOT IMPLEMENTED") return elif arguments["menu"]: userid = self.cm_config.username() launchers = self.cm_mongo.launcher_get(userid) if launchers.count() == 0: Console.warning( "no launcher in database, please import launcher first" "(launcher import [FILEPATH] [--force])" ) return else: d = {} for launcher in launchers: d[launcher["cm_launcher"]] = launcher if "_id" in d[launcher["cm_launcher"]]: del d[launcher["cm_launcher"]]["_id"] columns = None if arguments["--column"]: if arguments["--column"] != "all": columns = [x.strip() for x in arguments["--column"].split(",")] else: columns = ["name", "description"] if arguments["--format"]: if arguments["--format"] not in ["table", "json", "csv"]: Console.error("please select printing format ", "among table, json and csv") return else: p_format = arguments["--format"] else: p_format = None shell_commands_dict_output( userid, d, print_format=p_format, firstheader="launcher", header=columns # vertical_table=True ) elif arguments["list"]: userid = self.cm_config.username() self.cm_mongo.activate(userid) self.cm_mongo.refresh(cm_user_id=userid, types=[self._id]) stacks = self.cm_mongo.stacks(cm_user_id=userid) launchers = self.filter_launcher(stacks, {"search": "contain", "key": "stack_name", "value": "launcher"}) log.debug(launchers) d = {} for k0, v0 in launchers.iteritems(): for k1, v1 in launchers[k0].iteritems(): d[v1["id"]] = v1 columns = ["stack_name", "description", "stack_status", "creation_time", "cm_cloud"] if arguments["--column"] and arguments["--column"] != "all": columns = [x.strip() for x in arguments["--column"].split(",")] if arguments["--format"]: if arguments["--format"] not in ["table", "json", "csv"]: Console.error("please select printing format among table, json and csv") return else: p_format = arguments["--format"] else: p_format = None shell_commands_dict_output( userid, d, print_format=p_format, firstheader="launcher_id", header=columns # vertical_table=True ) elif arguments["start"] and arguments["MENU"]: userid = self.cm_config.username() def_cloud = self.get_cloud_name(userid) self.cm_mongo.activate(userid) config_launcher = cm_config_launcher() userinfo = self.cm_user.info(userid) if "key" in userinfo["defaults"]: key = userinfo["defaults"]["key"] elif len(userinfo["keys"]["keylist"].keys()) > 0: key = userinfo["keys"]["keylist"].keys()[0] if key: keycontent = userinfo["keys"]["keylist"][key] if keycontent.startswith("key "): keycontent = keycontent[4:] cm_keys_mongo(userid).check_register_key(userid, def_cloud, key, keycontent) keynamenew = _keyname_sanitation(userid, key) else: Console.warning("No sshkey found. Please Upload one") cookbook = arguments["MENU"] s_name = "launcher-{0}-{1}-{2}".format(userid, cookbook, get_rand_string()) dummy = "123456789" # doing nothing. just for test try: t_url = config_launcher["cloudmesh"]["launcher"]["default"]["template"] except: # If key is missing (KeyError), new cloudmesh_launcher.yaml # needs to be copied to ~/.cloudmesh t_url = "https://raw.githubusercontent.com/cloudmesh/cloudmesh/master/heat-templates/centos6/launcher/launcher.yaml" param = {"KeyName": keynamenew, "Cookbook": cookbook, "dummy": dummy} # test for openmpi, hadoop if cookbook[:6] == "hadoop" or cookbook[:7] == "openmpi": privatekey, publickey = generate_keypair() try: t_url = config_launcher["cloudmesh"]["launcher"]["recipes"][cookbook]["template"] except: # If key is missing (KeyError), new cloudmesh_launcher.yaml # needs to be copied to ~/.cloudmesh t_url = ( "https://raw.githubusercontent.com/cloudmesh/cloudmesh/master/heat-templates/ubuntu-14.04/" + str(cookbook) + "-cluster/" + str(cookbook) + "-cluster.yaml" ) param = {"KeyName": keynamenew, "PublicKeyString": publickey, "PrivateKeyString": privatekey} if cookbook[:9] == "hadoop2.7": param["UserName"] = userid log.debug(def_cloud, userid, s_name, t_url, param) res = self.cm_mongo.stack_create( cloud=def_cloud, cm_user_id=userid, servername=s_name, template_url=t_url, parameters=param ) log.debug(res) if "error" in res: print(res["error"]["message"]) return res elif arguments["stop"] and arguments["STACK_NAME"]: userid = self.cm_config.username() def_cloud = self.get_cloud_name(userid) s_id = arguments["STACK_NAME"] self.cm_mongo.activate(userid) res = self.cm_mongo.stack_delete(cloud=def_cloud, cm_user_id=userid, server=s_id) log.debug(res) return res elif arguments["import"]: filepath = config_file("/cloudmesh_launcher.yaml") if arguments["FILEPATH"]: filepath = arguments["FILEPATH"] try: filename = path_expand(filepath) fileconfig = ConfigDict(filename=filename) except Exception, err: Console.error("error while loading '{0}', please check".format(filepath)) print(traceback.format_exc()) print(sys.exc_info()[0]) return try: recipes_dict = fileconfig.get("cloudmesh", "launcher", "recipies") except: Console.error("error while loading recipies from the file") # print recipes_dict userid = self.cm_config.username() launcher_names = [] launchers = self.cm_mongo.launcher_get(userid) for launcher in launchers: launcher_names.append(launcher["cm_launcher"].encode("ascii")) for key in recipes_dict: if key in launcher_names: if arguments["--force"]: self.cm_mongo.launcher_remove(userid, key) self.cm_mongo.launcher_import(recipes_dict[key], key, userid) print("launcher '{0}' overwritten.".format(key)) else: print( "ERROR: launcher '{0}' exists, " "please remove it first, or use " "'--force' when adding".format(key) ) else: self.cm_mongo.launcher_import(recipes_dict[key], key, userid) print("launcher '{0}' added.".format(key))