示例#1
0
 def _load_keys_mongo(self):
     #try:
     from cloudmesh.config.cm_config import cm_config
     config = cm_config()
     self.user = config.get('cloudmesh.profile.username')
             
     if self.echo:
         log.info("Reading keys information from -> mongo")
     self.keys_mongo = cm_keys_mongo(self.user)
     self.keys_loaded_mongo = True
示例#2
0
def deletekey(name):
    #
    #  BUG cm_keys reads from config file
    #
    keys = cm_keys_mongo(g.user.id)

    try:
        keys.delete(name)
        # delete from mongo directly
        # no seperate write back needed
        # keys.write()
    except:
        print "Error: deleting the key %s" % name
    return redirect("/keys/")
示例#3
0
 def _default_key(self):
     key_store = cm_keys_mongo(self.username)
     # print key_store.names()
     # no name provided, will get current default key
     if not self.arguments["VALUE"]:
         defaultkey = key_store.default()
         print("Current default key is: {0}".format(defaultkey))
     # name provided, check if it exists in the db
     elif self.arguments["VALUE"] in key_store.names():
         key_store.setdefault(self.arguments["VALUE"])
         # Update mongo db defaults with new default key
         print('The default key was successfully set to: ', self.arguments['VALUE'])
     else:
         print("ERROR: Specified key is not registered.")
     return
示例#4
0
            
            # -------------------------
            # key handler
            userinfo = userobj.info(username)
            key = None
            if "key" in userinfo["defaults"]:
                key = userinfo["defaults"]["key"]
            elif len(userinfo["keys"]["keylist"].keys()) > 0:
                key = userinfo["keys"]["keylist"].keys()[0]
                Console.warning("default key is not set, trying to use a key in the database...")
        
            if key:
                keycontent = userinfo["keys"]["keylist"][key]
                if keycontent.startswith('key '):
                    keycontent = keycontent[4:]
                cm_keys_mongo(username).check_register_key(username, cloudname, key, keycontent)
            else:
                Console.error("No sshkey found. Please Upload one")
                return
            # -------------------------
            
            
            # generate ssh keys for VMs and prepare two files: authorized_keys and hosts
            print ("generating ssh keys...")
            os.popen("mkdir {0}".format(dir_name))

            fa = open("./{0}/authorized_keys_temp".format(dir_name), "w")
            fh = open("./{0}/hosts_temp".format(dir_name), "w")
            fk = open("./{0}/{1}".format(dir_name, temp_key_name), "w")
            
            fk.write(keycontent)
    def do_launcher(self, args, arguments):
        """
        Usage:
            launcher start MENU
            launcher stop STACK_NAME
            launcher list
            launcher show STACK_NAME
            launcher menu [--column=COLUMN] [--format=FORMAT]
            launcher import [FILEPATH] [--force]
            launcher export FILEPATH
            launcher help | -h

        An orchestration tool with Chef Cookbooks

        Arguments:

          MENU           Name of a cookbook
          STACK_NAME     Name of a launcher
          FILEPATH       Filepath
          COLUMN         column name to display
          FORMAT         display format (json, table)
          help           Prints this message

        Options:

           -v       verbose mode

        """
        log.info(arguments)
        self.cm_mongo = cm_mongo()
        self.cm_config = cm_config()
        self.user = cm_user()

        if arguments["help"] or arguments["-h"]:
            print (self.do_launcher.__doc__)
        elif arguments['show'] and arguments['STACK_NAME']:
            print ("NOT IMPLEMENTED")
            return
        elif arguments['menu']:
            userid = self.cm_config.username()
            launchers = self.cm_mongo.launcher_get(userid)

            if launchers.count() == 0:
                Console.warning("no launcher in database, please import launcher first"
                                "(launcher import [FILEPATH] [--force])")
                return
            else:
                d = {}
                for launcher in launchers:
                    d[launcher['cm_launcher']] = launcher
                    if "_id" in d[launcher['cm_launcher']]:
                        del d[launcher['cm_launcher']]['_id']

            columns = None
            if arguments['--column']:
                if arguments['--column'] != "all":
                    columns = [x.strip() for x in arguments['--column'].split(',')]
            else:
                columns = ['name', 'description']

            if arguments['--format']:
                if arguments['--format'] not in ['table', 'json', 'csv']:
                    Console.error("please select printing format ",
                                  "among table, json and csv")
                    return
                else:
                    p_format = arguments['--format']
            else:
                p_format = None

            shell_commands_dict_output(userid,
                                       d,
                                       print_format=p_format,
                                       firstheader="launcher",
                                       header=columns
                                       # vertical_table=True
                                       )

        elif arguments['list']:
            userid = self.cm_config.username()
            self.cm_mongo.activate(userid)
            self.cm_mongo.refresh(cm_user_id=userid, types=[self._id])
            stacks = self.cm_mongo.stacks(cm_user_id=userid)
            launchers = self.filter_launcher(
                stacks,
                {"search": "contain",
                 "key": "stack_name",
                 "value": "launcher"}
                )
            log.debug(launchers)

            d = {}
            for k0, v0 in launchers.iteritems():
                for k1, v1 in launchers[k0].iteritems():
                    d[v1['id']] = v1
            columns = ['stack_name', 'description', 'stack_status',
                       'creation_time', 'cm_cloud']
            if arguments['--column'] and arguments['--column'] != "all":
                columns = [x.strip() for x in arguments['--column'].split(',')]

            if arguments['--format']:
                if arguments['--format'] not in ['table', 'json', 'csv']:
                    Console.error("please select printing format among table, json and csv")
                    return
                else:
                    p_format = arguments['--format']
            else:
                p_format = None

            shell_commands_dict_output(userid,
                                       d,
                                       print_format=p_format,
                                       firstheader="launcher_id",
                                       header=columns
                                       # vertical_table=True
                                       )

        elif arguments['start'] and arguments['MENU']:
            userid = self.cm_config.username()
            def_cloud = self.get_cloud_name(userid)
            self.cm_mongo.activate(userid)
            
            userinfo = self.user.info(userid)
            if "key" in userinfo["defaults"]:
                key = userinfo["defaults"]["key"]
            elif len(userinfo["keys"]["keylist"].keys()) > 0:
                key = userinfo["keys"]["keylist"].keys()[0]
        
            if key:
                keycontent = userinfo["keys"]["keylist"][key]
                if keycontent.startswith('key '):
                    keycontent = keycontent[4:]
                cm_keys_mongo(userid).check_register_key(userid, def_cloud, key, keycontent)
                keynamenew = _keyname_sanitation(userid, key)
            else:
                Console.warning("No sshkey found. Please Upload one")
            
            cookbook = arguments['MENU']
            s_name = "launcher-{0}-{1}-{2}".format(userid, cookbook, get_rand_string())
            dummy = "123456789"  # doing nothing. just for test
            t_url = \
            "https://raw.githubusercontent.com/cloudmesh/cloudmesh/master/heat-templates/centos6/launcher/launcher.yaml"
            param = {'KeyName': keynamenew,
                     'Cookbook': cookbook,
                     'dummy': dummy}
            log.debug(def_cloud, userid, s_name, t_url, param)
            res = self.cm_mongo.stack_create(cloud=def_cloud, cm_user_id=userid,
                                             servername=s_name,
                                             template_url=t_url,
                                             parameters=param)
            log.debug(res)
            if 'error' in res:
                print (res['error']['message'])
            return res

        elif arguments['stop'] and arguments['STACK_NAME']:
            userid = self.cm_config.username()
            def_cloud = self.get_cloud_name(userid)
            s_id = arguments['STACK_NAME']
            self.cm_mongo.activate(userid)
            res = self.cm_mongo.stack_delete(cloud=def_cloud,
                                             cm_user_id=userid,
                                             server=s_id)
            log.debug(res)
            return res

        elif arguments['import']:
            filepath = config_file("/cloudmesh_launcher.yaml")

            if arguments['FILEPATH']:
                filepath = arguments['FILEPATH']
            try:
                filename = path_expand(filepath)
                fileconfig = ConfigDict(filename=filename)
            except Exception, err:
                Console.error(
                    "error while loading '{0}', please check".format(filepath))
                print (traceback.format_exc())
                print (sys.exc_info()[0])
                return
            try:
                recipes_dict = fileconfig.get("cloudmesh", "launcher", "recipies")
            except:
                Console.error("error while loading recipies from the file")

            # print recipes_dict
            userid = self.cm_config.username()
            launcher_names = []
            launchers = self.cm_mongo.launcher_get(userid)
            for launcher in launchers:
                launcher_names.append(launcher['cm_launcher'].encode("ascii"))

            for key in recipes_dict:
                if key in launcher_names:
                    if arguments['--force']:
                        self.cm_mongo.launcher_remove(userid, key)
                        self.cm_mongo.launcher_import(
                            recipes_dict[key], key, userid)
                        print ("launcher '{0}' overwritten.".format(key))
                    else:
                        print ("ERROR: launcher '{0}' exists, "
                               "please remove it first, or use "
                               "'--force' when adding".format(key))
                else:
                    self.cm_mongo.launcher_import(
                        recipes_dict[key], key, userid)
                    print ("launcher '{0}' added.".format(key))
示例#6
0
    def do_key(self, args, arguments):
        """
        Usage:
               key -h|--help
               key list [--source=SOURCE] [--dir=DIR] [--format=FORMAT]
               key add [--keyname=KEYNAME] FILENAME
               key default [KEYNAME]
               key delete KEYNAME

        Manages the keys

        Arguments:

          SOURCE         mongo, yaml, ssh
          KEYNAME        The name of a key
          FORMAT         The format of the output (table, json, yaml)
          FILENAME       The filename with full path in which the key is located

        Options:

           --dir=DIR            the directory with keys [default: ~/.ssh]
           --format=FORMAT      the format of the output [default: table]
           --source=SOURCE      the source for the keys [default: mongo]
           --keyname=KEYNAME    the name of the keys

        Description:


        key list --source=ssh  [--dir=DIR] [--format=FORMAT]

           lists all keys in the directory. If the directory is not
           specified the defualt will be ~/.ssh

        key list --source=yaml  [--dir=DIR] [--format=FORMAT]

           lists all keys in cloudmesh.yaml file in the specified directory.
            dir is by default ~/.cloudmesh

        key list [--format=FORMAT]

            list the keys in mongo

        key add [--keyname=keyname] FILENAME

            adds the key specifid by the filename to mongodb


        key list

             Prints list of keys. NAME of the key can be specified

        key default [NAME]

             Used to set a key from the key-list as the default key if NAME
             is given. Otherwise print the current default key

        key delete NAME

             deletes a key. In yaml mode it can delete only key that
             are not saved in mongo

        """
        # print arguments

        def _find_keys(directory):
            return [file for file in listdir(expanduser(directory))
                    if file.lower().endswith(".pub")]

        #
        # DIR (OK)
        #

        directory = path_expand(arguments["--dir"])
        source = arguments["--source"]

        if source not in ["ssh", "yaml", "mongo"]:
            print("ERROR: source is not defined")
            return

        #
        # PRINT DICT (OK)
        #

        def _print_dict(d, header=None):
            print_format = arguments['--format']
            if print_format == "json":
                return json.dumps(d, indent=4)
            elif print_format == "yaml":
                return yaml.dump(d, default_flow_style=False)
            else:
                return two_column_table(keys, header)

        #
        # PRINT SYSTEM (OK)
        #

        if arguments["list"] and source == "ssh":

            files = _find_keys(directory)

            keys = {}
            for key in files:
                keys[key] = directory + "/" + key
            print(_print_dict(keys, header=["Key", "Location"]))

            return

        #
        # PRINT YAML (OK)
        #

        if arguments["list"] and source == "yaml":

            key_store = cm_keys_yaml()
            keynames = key_store.names()
            keys = {}
            for key in keynames:
                keys[key] = get_fingerprint(key_store[key])

            print(_print_dict(keys, header=["Key", "Fingerprint"]))

            return

        #
        # FROM HERE ON BROKEN
        #

        if arguments["list"] and source == "mongo":

            username = cm_config().username()
            key_store = cm_keys_mongo(username)
            keynames = key_store.names()

            keys = {}
            for key in keynames:

                keys[key] = key_store[key]
                if keytype(keys[key]) == 'string':
                    keys[key] = get_fingerprint(keys[key])

            print(_print_dict(keys, header=["Key", "Fingerprint"]))

            return

        # self._load_keys_from_yaml()
        # self._load_mongo()
        # self._load_keys_mongo()

        if arguments["add"] and arguments["FILENAME"]:

            username = cm_config().username()
            key_store = cm_keys_mongo(username)
            filename = arguments["FILENAME"]

            # print filename
            # file existence check moved to the util function
            # if file does not exist it will return None
            keystring = get_key_from_file(filename)
            if keystring:
                keystring = keystring.strip()
                keysegments = key_parse(keystring)
                # key valid?
                if key_validate("string", keystring):
                    if arguments["--keyname"]:
                        keyname = arguments["--keyname"]
                    else:
                        # get keyname from the comment field of key
                        keyname = keysegments[2]
                        # in case a comment field is really missing
                        # use a timestamp for uniqueness
                        if keyname is None:
                            tstamp = datetime.now().strftime("%Y%m%d%H%M%S")
                            keyname = tstamp
                        # sanitize the key name for mongo use
                        else:
                            keyname = re.sub(r'@|-|\.', '_', keysegments[2])
                    key_store[keyname] = keystring
                else:
                    print("ERROR: INVALID key. Please verify!")
            else:
                print("ERROR: INVALID filename provided!")

            return

            """
            def func():
                if arguments["KEY"]:
                    key_store.__setitem__(
                        arguments["NAME"], arguments["KEY"])
                else:
                    files = _find_keys(directory)

                    result = menu_return_num(
                        title="Select a public key", menu_list=files, tries=3)
                    if result == 'q':
                        return
                    else:
                        key_store.__setitem__(arguments["NAME"],
                                                  "{0}/{1}".format(directory,files[result]))

            if arguments["NAME"] in key_store.names():
                if yn_choice("key {0} exists, update?"
                             .format(arguments["NAME"]), default='n'):
                    print "Updating key {0} ...".format(arguments["NAME"])
                    func()
                else:
                    return
            else:
                print "adding key {0} ...".format(arguments["NAME"])
                func()
            """

        if arguments["default"]:
            username = cm_config().username()
            key_store = cm_keys_mongo(username)
            # print key_store.names()
            # no name provided, will get current default key
            if not arguments["KEYNAME"]:
                defaultkey = key_store.default()
                print("Current default key is: {0}".format(defaultkey))
            # name provided, check if it exists in the db
            elif arguments["KEYNAME"] in key_store.names():
                key_store.setdefault(arguments["KEYNAME"])
                # Update mongo db defaults with new default key
                print('The default key was successfully set to: ', arguments['KEYNAME'])
            else:
                print("ERROR: Specified key is not registered.")
            return

        if arguments["delete"]:
            username = cm_config().username()
            key_store = cm_keys_mongo(username)
            print("Attempting to delete key: {0}".format(arguments["KEYNAME"]))
            if self.use_yaml:

                print("WARNING: This will only remove the keys that"
                      "have not been written to the databse already when "
                      "'keys save' is"
                      "called. If your key is already in the database, "
                      "you should use mongo mode\n")

            key_store.delete(arguments["KEYNAME"])
            return

        # deprecating...
        """
示例#7
0
def start_vm(username,
             cloudname,
             count=1,
             flavorname=None,
             flavorid=None,
             imagename=None,
             imageid=None,
             groupname=None,
             servername=None):
    '''
    create a vm of a cloud of a user
    will check flavor, image existence if provided
    user can specify groupname which will be written in metadata, servername which
    will replace prefix+index as the vm name
    it's better to check cloud active status before use this function
    :param username: string
    :param cloudname: string
    :param count: number of vms to start
    :return False if error

    TODO: what if fail, how to acknowledge it; no return now as using celery
          input key
          missing security group
    '''
    # Changed scope of this import - hyungro lee 12/01/2014
    from cloudmesh.experiment.group_usage import add_vm_to_group_while_creating

    mongo = cm_mongo()
    userobj = cm_user()
    cloudobj = CloudManage()
    mongo.activate(cm_user_id=username, names=[cloudname])
    userinfo = userobj.info(username)
    key = None
    vm_image_id = None
    vm_flavor_id = None

    error = ''

    # -------------------------
    # refresh server flavor or image
    to_refresh = ["servers"]

    if flavorname is not None or flavorid is not None:
        to_refresh.append("flavors")
    if imagename is not None or imageid is not None:
        to_refresh.append("images")
    if to_refresh != []:
        mongo.refresh(username, names=[cloudname], types=to_refresh)
        
    # -------------------------
    # get exist VM names list, to prevent names duplicate
    serverdata = mongo.servers(
                        clouds=[cloudname], cm_user_id=username)[cloudname]
    servers_names_list = []
    for k,v in serverdata.iteritems():
        servers_names_list.append(v['name'])

    # -------------------------
    # flavor handler
    if flavorname is not None or flavorid is not None:
        flavordata = mongo.flavors(
            clouds=[cloudname], cm_user_id=username)[cloudname]
        same_name_count = 0
        for k, v in flavordata.iteritems():
            if flavorname is not None:
                if flavorname == v['name']:
                    vm_flavor_id = k
                    same_name_count = same_name_count + 1
            else:
                if flavorid == k:
                    vm_flavor_id = k
                    break
        if vm_flavor_id is None:
            error = error + "The flavor you provide doesn't exist. "
        if same_name_count > 1:
            error = error + "There are more than one flavor with the name you provide" \
                            "please use flavorid instead or select one by command cloud" \
                            "set flavor [CLOUD]. "
    else:
        try:
            vm_flavor_id = userinfo["defaults"]["flavors"][cloudname]
        except:
            pass
        if vm_flavor_id in [None, 'none']:
            error = error + \
                "Please specify a default flavor(command: cloud set flavor [CLOUD]). "
    # -------------------------
    # image handler
    if imagename is not None or imageid is not None:
        imagedata = mongo.images(
            clouds=[cloudname], cm_user_id=username)[cloudname]
        same_name_count = 0
        for k, v in imagedata.iteritems():
            if imagename is not None:
                if imagename == v['name']:
                    vm_image_id = k
                    same_name_count = same_name_count + 1
            else:
                if imageid == k:
                    vm_image_id = k
                    break
        if vm_image_id is None:
            error = error + "The image you provide doesn't exist. "
        if same_name_count > 1:
            error = error + "There are more than one image with the name you provide" \
                            "please use imageid instead or select one by command cloud" \
                            "set image [CLOUD]. "
    else:
        try:
            vm_image_id = userinfo["defaults"]["images"][cloudname]
        except:
            pass
        if vm_image_id in [None, 'none']:
            error = error + \
                "Please specify a default image(command: cloud set flavor [CLOUD]). "

    # -------------------------
    # key handler
    if "key" in userinfo["defaults"]:
        key = userinfo["defaults"]["key"]
    elif len(userinfo["keys"]["keylist"].keys()) > 0:
        key = userinfo["keys"]["keylist"].keys()[0]

    if key:
        keycontent = userinfo["keys"]["keylist"][key]
        if keycontent.startswith('key '):
            keycontent = keycontent[4:]
        cm_keys_mongo(username).check_register_key(username, cloudname, key, keycontent)
        keynamenew = _keyname_sanitation(username, key)
    else:
        error = error + \
            "No sshkey found. Please Upload one"
    # -------------------------

    if error != '':
        Console.error(error)
        return False
    # -------------------------
    metadata = {'cm_owner': username}
    if groupname:
        metadata['cm_group'] = groupname

    tmpnamefl = cloudobj.get_flavors(
        cloudname=cloudname, getone=True, id=vm_flavor_id)['name']
    tmpnameim = cloudobj.get_images(
        cloudname=cloudname, getone=True, id=vm_image_id)['name']

    while count > 0:
        userinfo = userobj.info(username)
        if servername:
            prefix = ''
            index = ''
            givenvmname = servername
            tmpnameser = servername
        else:
            prefix = userinfo["defaults"]["prefix"]
            index = userinfo["defaults"]["index"]
            givenvmname = None
            tmpnameser = prefix + '_' + str(index)
            
        # ------------------------
        # do not allow server name duplicate
        if tmpnameser in servers_names_list:
            Console.error("vm name '{0}' exists, please use other names or delete it first".format(
                                                                                tmpnameser))
            if not servername:
                userobj.set_default_attribute(username, "index", int(index) + 1)
            count = count - 1
            continue
        # ------------------------
        # vm start procedure

        banner("Starting vm->{0} on cloud->{1} using image->{2}, flavor->{3}, key->{4}"
               .format(tmpnameser, cloudname, tmpnameim, tmpnamefl, keynamenew))
        
        useQueue = False
        if useQueue:
            result = mongo.vm_create_queue(cloudname,
                                            prefix,
                                            index,
                                            vm_flavor_id,
                                            vm_image_id,
                                            keynamenew,
                                            meta=metadata,
                                            cm_user_id=username,
                                            givenvmname=givenvmname)
            print("job status:", result.state)
        else:
            result = mongo.vm_create(cloudname,
                                        prefix,
                                        index,
                                        vm_flavor_id,
                                        vm_image_id,
                                        keynamenew,
                                        meta=metadata,
                                        cm_user_id=username,
                                        givenvmname=givenvmname)
            if "server" in result and "adminPass" in result["server"]:
                result["server"]["adminPass"] = "******"
            pprint(result)
        
        # ------------------------
        # add it to the group in database if groupname provided
        if groupname:
            try:
                add_vm_to_group_while_creating(username, groupname, tmpnameser)
            except Exception, err:
                Console.error(str(err))
                return
            
        # ------------------------
        # increase index if it is used
        if not servername:
            userobj.set_default_attribute(username, "index", int(index) + 1)
        # ------------------------
        servers_names_list.append(tmpnameser)

        count = count - 1
    def do_cluster(self, args, arguments):
        """
        Usage:
            cluster start CLUSTER_NAME
            cluster list
            cluster login CLUSTER_NAME
            cluster stop CLUSTER_NAME
            cluster create --count=<count>
                           --group=<group>
                           [--ln=<LoginName>]
                           [--cloud=<CloudName>]
                           [--image=<imgName>|--imageid=<imgId>]
                           [--flavor=<flavorName>|--flavorid=<flavorId>]
                           [--force]
                                    
        Description:
            Cluster Management
            
            cluster create --count=<count> --group=<group> --ln=<LoginName> [options...]
            <count>            specify amount of VMs in the cluster
            <group>            specify a group name of the cluster, make sure it's unique
                Start a cluster of VMs, and each of them can log into all others.
                CAUTION: you sould do some default setting before using this command:
                1. select cloud to work on, e.g. cloud select india
                2. activate the cloud, e.g. cloud on india
                3. set the default key to start VMs, e.g. key default [NAME]
                4. set the start name of VMs, which is prefix and index, e.g. label --prefix=test --id=1
                5. set image of VMs, e.g. default image
                6. set flavor of VMs, e.g. default flavor
                Also, please make sure the group name of the cluster is unique
                
        Options:
            --ln=<LoginName>           give a login name for the VMs, e.g. ubuntu
            --cloud=<CloudName>        give a cloud to work on
            --flavor=<flavorName>      give the name of the flavor
            --flavorid=<flavorId>      give the id of the flavor
            --image=<imgName>          give the name of the image
            --imageid=<imgId>          give the id of the image
            --force                    if a group exists and there are VMs in it, the program will
                                       ask user to proceed or not, use this flag to respond yes as 
                                       default(if there are VMs in the group before creating this 
                                       cluster, the program will include the exist VMs into the cluster)


        """
        #pprint(arguments)
        self.cm_config = cm_config()
        self.cm_mongo = cm_mongo()
        self.user = cm_user()
       
        # -----------------------------
        # TODO::
        # add VMs to cluster
        # -----------------------------
       
        if arguments['start'] and arguments['CLUSTER_NAME']:
            '''Starts a cluster'''

            # Initialize default variables. e.g. userid, default cloud and
            # default keypair
            userid = self.cm_config.username()
            def_cloud = self.get_cloud_name(userid)
            self.cm_mongo.activate(userid)
            
            userinfo = self.user.info(userid)
            if "key" in userinfo["defaults"]:
                key = userinfo["defaults"]["key"]
            elif len(userinfo["keys"]["keylist"].keys()) > 0:
                key = userinfo["keys"]["keylist"].keys()[0]
        
            if key:
                keycontent = userinfo["keys"]["keylist"][key]
                if keycontent.startswith('key '):
                    keycontent = keycontent[4:]
                cm_keys_mongo(userid).check_register_key(userid, def_cloud, key, keycontent)
                keynamenew = _keyname_sanitation(userid, key)
            else:
                Console.warning("No sshkey found. Please Upload one")
                return
            
            clustername = arguments['CLUSTER_NAME']
            s_name = "cluster-{0}-{1}-{2}".format(userid, clustername, get_rand_string())
            # TEMP FOR HADOOP CLUSTER
            if clustername != "hadoop":
                Console.warning('hadoop is only available cluster')
                return
            
            # 1. keypair for the communication between master and worker nodes
            privatekey, publickey = generate_keypair()
            t_url = \
            "https://raw.githubusercontent.com/cloudmesh/cloudmesh/dev1.3/heat-templates/ubuntu-14.04/hadoop-cluster/hadoop-cluster.yaml"
            param = {'KeyName': keynamenew,
                     'PublicKeyString': publickey,
                     'PrivateKeyString': privatekey}
            log.debug(def_cloud, userid, s_name, t_url, param, publickey,
                      privatekey)
            res = self.cm_mongo.stack_create(cloud=def_cloud, cm_user_id=userid,
                                             servername=s_name,
                                             template_url=t_url,
                                             parameters=param)
            log.debug(res)
            if 'error' in res:
                print (res['error']['message'])
            return res

        elif arguments['list']:
            userid = self.cm_config.username()
            self.cm_mongo.activate(userid)
            self.cm_mongo.refresh(cm_user_id=userid, types=[self._id])
            stacks = self.cm_mongo.stacks(cm_user_id=userid)
            launchers = self.filter_launcher(
                stacks,
                {"search": "contain",
                 "key": "stack_name",
                 "value": "launcher"}
                )
            log.debug(launchers)

            d = {}
            for k0, v0 in launchers.iteritems():
                for k1, v1 in launchers[k0].iteritems():
                    d[v1['id']] = v1
            columns = ['stack_name', 'description', 'stack_status',
                       'creation_time', 'cm_cloud']
            if arguments['--column'] and arguments['--column'] != "all":
                columns = [x.strip() for x in arguments['--column'].split(',')]

            if arguments['--format']:
                if arguments['--format'] not in ['table', 'json', 'csv']:
                    Console.error("please select printing format among table, json and csv")
                    return
                else:
                    p_format = arguments['--format']
            else:
                p_format = None

            shell_commands_dict_output(d,
                                       print_format=p_format,
                                       firstheader="launcher_id",
                                       header=columns
                                       # vertical_table=True
                                       )

        elif arguments['login']:
            Console.error("Not implemented")
            return

        elif arguments['stop'] and arguments['CLUSTER_NAME']:
            userid = self.cm_config.username()
            def_cloud = self.get_cloud_name(userid)
            c_id = arguments['CLUSTER_NAME']
            self.cm_mongo.activate(userid)
            res = self.cm_mongo.stack_delete(cloud=def_cloud,
                                             cm_user_id=userid,
                                             server=c_id)
            log.debug(res)
            return res

        elif arguments['create']:
            try:
                config = cm_config()
            except:
                Console.error("Failed to load the cloudmesh yaml file")
                return
            username = config['cloudmesh']['profile']['username']
            cloudname = arguments['--cloud'] or CloudManage().get_selected_cloud(username)
            temp_dir_name = ".temp_cluster_create_" + username + "_0"
            while os.path.isdir("./{0}".format(temp_dir_name)):
                temp_dir_name = temp_dir_name[:-1] + str(int(temp_dir_name[-1]) + 1)
            dir_name = temp_dir_name
            
            #NumOfVM = None
            GroupName = None

            vm_login_name = "ubuntu"

            
            temp_key_name = "sshkey_temp"
            _key = "-i ./{0}/{1}".format(dir_name, temp_key_name)
            StrictHostKeyChecking = "-o StrictHostKeyChecking=no"
            
            res = None
            to_print = []
            '''
            try:
                NumOfVM = abs(int(argument['--count']))
            except:
                Console.error("<count> must be an integer")
                return
            '''
            if arguments['--group'] == '':
                Console.error("<group> cannot be empty")
                return
            else:
                GroupName = arguments['--group']
            
            if arguments['--ln']:
                if arguments['--ln'] == '':
                    Console.error("<LoginName> cannot be empty")
                    return
                else:
                    vm_login_name = arguments['--ln']
            
            if not arguments['--force']:
                # Moved the import inside of this function
                # If this import goes up to the top, monodb connection will be
                # estabilished. Due to that reason, this import stays here
                # Hyungro Lee 12/01/2014
                from cloudmesh.experiment.group import GroupManagement
                GroupManage = GroupManagement(username)
                groups_list = GroupManage.get_groups_names_list()
                if GroupName in groups_list:
                    vms_in_group_list = GroupManage.list_items_of_group(GroupName, _type="VM")["VM"]
                    if len(vms_in_group_list) != 0:
                        if yn_choice("The group you provide exists and it has VMs in it, " + \
                                     "do you want to proceed? (if you choose yes, these exist " +\
                                     "VMs will be included in the cluster, this could also " +\
                                     "rewrite the key on the exist VMs)",
                                     default='n',
                                     tries=3):
                            pass
                        else:
                            return
                
            # start VMs 
            print ("starting VMs...")
            arguments_temp = arguments
            arguments_temp['start'] = True
            arguments_temp['--name'] = None
            
            vmclass = VMcommand(arguments_temp)
            res = vmclass.call_procedure()
            if res == False: return
        
            def string_to_dict(s):
                h = s.find("{")
                t = s.rfind("}")
                return json.loads(s[h:t+1])
            
            def check_public_ip_existence(d):
                temp = d['addresses']['private']
                for item in temp:
                    if item["OS-EXT-IPS:type"] == "floating":
                        return True
                return False
            
            def get_ip(d, kind="floating"): # kind is either floating or fixed
                temp = d['addresses']['private']
                for item in temp:
                    if item["OS-EXT-IPS:type"] == kind:
                        return item['addr']#.encode('ascii')
                return "FAIL: doesn't exist"
            
            # check all VMs are active
            command_refresh = "vm list --refresh --group={0} --format=json".format(GroupName)
            def _help0(d):
                for k, v in d.iteritems():
                    if v['status'] != 'ACTIVE':
                        return False
                return True
                
            proceed = False
            repeat_index = 1
            while proceed != True:
                if repeat_index > 10:
                    Console.warning("Please check the network")
                    return
                print ("checking({0})...".format(repeat_index))
                time.sleep(5)
                res = str(cm(command_refresh))
                res = string_to_dict(res)
                if _help0(res):
                    proceed = True
                else:
                    repeat_index = repeat_index + 1
                    continue
            
            # assign ip to all VMs
            print ("assigning public ips...")
            for k, v in res.iteritems():
                if not check_public_ip_existence(v):
                    cm("vm ip assign --id={0}".format(k.encode('ascii')))
            
            def _help(d):
                for k, v in d.iteritems():
                    if check_public_ip_existence(v) != True:
                        return False
                return True
            
            # make sure all VMs have been assigned a public ip
            proceed = False
            repeat_index = 1
            while proceed != True:
                if repeat_index > 10:
                    Console.warning("Please check the network")
                    return
                print ("checking({0})...".format(repeat_index))
                time.sleep(5)
                res = str(cm(command_refresh))
                res = string_to_dict(res)
                if _help(res):
                    proceed = True
                else:
                    repeat_index = repeat_index + 1
                    continue
                
            
            # -------------------------
            # key handler
            userinfo = cm_user().info(username)
            key = None
            if "key" in userinfo["defaults"]:
                key = userinfo["defaults"]["key"]
            elif len(userinfo["keys"]["keylist"].keys()) > 0:
                key = userinfo["keys"]["keylist"].keys()[0]
                Console.warning("default key is not set, trying to use a key in the database...")
        
            if key:
                keycontent = userinfo["keys"]["keylist"][key]
                if keycontent.startswith('key '):
                    keycontent = keycontent[4:]
                cm_keys_mongo(username).check_register_key(username, cloudname, key, keycontent)
            else:
                Console.error("No sshkey found. Please Upload one")
                return
            # -------------------------
            
            
            # generate ssh keys for VMs and prepare two files: authorized_keys and hosts
            print ("generating ssh keys...")
            os.popen("mkdir {0}".format(dir_name))

            fa = open("./{0}/authorized_keys_temp".format(dir_name), "w")
            fh = open("./{0}/hosts_temp".format(dir_name), "w")
            fk = open("./{0}/{1}".format(dir_name, temp_key_name), "w")
            
            fk.write(keycontent)
            fk.close()
            os.popen("chmod 644 ./{0}/{1}".format(dir_name, temp_key_name))
            
            for k, v in res.iteritems():
                address_floating = get_ip(v)
                address_fixed = get_ip(v, kind="fixed")
                vm_name = v['name']#.encode('ascii')
                to_print.append("{0} {1}, {2}".format(vm_name, address_floating, address_fixed))
                fh.write(address_floating + "  " + vm_name + "\n"
                         + address_fixed + "  " + vm_name + "-i\n")
                os.popen("ssh {2} {3} {0}@{1} \"ssh-keygen -t rsa -N '' -f ~/.ssh/id_rsa\""\
                         .format(vm_login_name,address_floating, _key, StrictHostKeyChecking))
                temp = os.popen("ssh {2} {3} {0}@{1} \"cat ~/.ssh/id_rsa.pub\""\
                                .format(vm_login_name, address_floating, _key, StrictHostKeyChecking)).read()
                fa.write(temp)
                
            fa.close()
            fh.close()
            
            # copy the files to VMs
            print ("copying the files...")
            os.popen("mkdir ./{0}/oops".format(dir_name))
            for k, v in res.iteritems():
                address_floating = get_ip(v)
                os.popen("scp {2} {3} {0}@{1}:~/.ssh/authorized_keys ./{4}/"\
                         .format(vm_login_name,address_floating, _key, StrictHostKeyChecking, dir_name))
                os.popen("cat ./{0}/authorized_keys_temp >> ./{0}/authorized_keys"\
                         .format(dir_name))
                os.popen("scp {2} {3} ./{4}/authorized_keys {0}@{1}:~"\
                         .format(vm_login_name,address_floating, _key, StrictHostKeyChecking, dir_name))
                os.popen("ssh {2} {3} {0}@{1} \"sudo mv authorized_keys ~/.ssh/\""\
                         .format(vm_login_name,address_floating, _key, StrictHostKeyChecking))
                os.popen("rm ./{0}/authorized_keys".format(dir_name))
                
                os.popen("cp ./{0}/hosts_temp ./{0}/oops/".format(dir_name))
                os.popen("mv ./{0}/oops/hosts_temp ./{0}/oops/hosts".format(dir_name))
                fh0 = open("./{0}/oops/hosts".format(dir_name), "a")
                os.popen("scp {2} {3} {0}@{1}:/etc/hosts ./{4}/"\
                         .format(vm_login_name,address_floating, _key, StrictHostKeyChecking, dir_name))
                with open("./{0}/hosts".format(dir_name)) as f0:
                    content = f0.readlines()
                for item in content:
                    fh0.write(item + "\n")
                fh0.close()
                os.popen("scp {2} {3} ./{4}/oops/hosts {0}@{1}:~"\
                         .format(vm_login_name,address_floating, _key, StrictHostKeyChecking, dir_name))
                os.popen("ssh {2} {3} {0}@{1} \"sudo mv hosts /etc/\""\
                         .format(vm_login_name,address_floating, _key, StrictHostKeyChecking))
                os.popen("rm ./{0}/oops/hosts".format(dir_name))
            
            
            print ("finishing...")
            os.popen("rm -rf {0}".format(dir_name))
            print ("DONE.")
            
            print ("cluster group: ", GroupName)
            for item in to_print:
                print (item)
            print ("(host name for private ips will have -i at the end of VM name, e.g. testVM -> testVM-i)")
示例#9
0
    def do_launcher(self, args, arguments):
        """
        ::

          Usage:
                launcher start MENU
                launcher stop STACK_NAME
                launcher list
                launcher show STACK_NAME
                launcher menu [--column=COLUMN] [--format=FORMAT]
                launcher import [FILEPATH] [--force]
                launcher export FILEPATH
                launcher help | -h

            An orchestration tool with Chef Cookbooks

            Arguments:

              MENU           Name of a cookbook
              STACK_NAME     Name of a launcher
              FILEPATH       Filepath
              COLUMN         column name to display
              FORMAT         display format (json, table)
              help           Prints this message

            Options:

               -v       verbose mode

        """
        log.info(arguments)
        self.cm_mongo = cm_mongo()
        self.cm_config = cm_config()
        self.cm_user = cm_user()
        self._id = "t_stacks"

        if arguments["help"] or arguments["-h"]:
            print(self.do_launcher.__doc__)
        elif arguments["show"] and arguments["STACK_NAME"]:
            print("NOT IMPLEMENTED")
            return
        elif arguments["menu"]:
            userid = self.cm_config.username()
            launchers = self.cm_mongo.launcher_get(userid)

            if launchers.count() == 0:
                Console.warning(
                    "no launcher in database, please import launcher first" "(launcher import [FILEPATH] [--force])"
                )
                return
            else:
                d = {}
                for launcher in launchers:
                    d[launcher["cm_launcher"]] = launcher
                    if "_id" in d[launcher["cm_launcher"]]:
                        del d[launcher["cm_launcher"]]["_id"]

            columns = None
            if arguments["--column"]:
                if arguments["--column"] != "all":
                    columns = [x.strip() for x in arguments["--column"].split(",")]
            else:
                columns = ["name", "description"]

            if arguments["--format"]:
                if arguments["--format"] not in ["table", "json", "csv"]:
                    Console.error("please select printing format ", "among table, json and csv")
                    return
                else:
                    p_format = arguments["--format"]
            else:
                p_format = None

            shell_commands_dict_output(
                userid,
                d,
                print_format=p_format,
                firstheader="launcher",
                header=columns
                # vertical_table=True
            )

        elif arguments["list"]:
            userid = self.cm_config.username()
            self.cm_mongo.activate(userid)
            self.cm_mongo.refresh(cm_user_id=userid, types=[self._id])
            stacks = self.cm_mongo.stacks(cm_user_id=userid)
            launchers = self.filter_launcher(stacks, {"search": "contain", "key": "stack_name", "value": "launcher"})
            log.debug(launchers)

            d = {}
            for k0, v0 in launchers.iteritems():
                for k1, v1 in launchers[k0].iteritems():
                    d[v1["id"]] = v1
            columns = ["stack_name", "description", "stack_status", "creation_time", "cm_cloud"]
            if arguments["--column"] and arguments["--column"] != "all":
                columns = [x.strip() for x in arguments["--column"].split(",")]

            if arguments["--format"]:
                if arguments["--format"] not in ["table", "json", "csv"]:
                    Console.error("please select printing format among table, json and csv")
                    return
                else:
                    p_format = arguments["--format"]
            else:
                p_format = None

            shell_commands_dict_output(
                userid,
                d,
                print_format=p_format,
                firstheader="launcher_id",
                header=columns
                # vertical_table=True
            )

        elif arguments["start"] and arguments["MENU"]:
            userid = self.cm_config.username()
            def_cloud = self.get_cloud_name(userid)
            self.cm_mongo.activate(userid)
            config_launcher = cm_config_launcher()

            userinfo = self.cm_user.info(userid)
            if "key" in userinfo["defaults"]:
                key = userinfo["defaults"]["key"]
            elif len(userinfo["keys"]["keylist"].keys()) > 0:
                key = userinfo["keys"]["keylist"].keys()[0]

            if key:
                keycontent = userinfo["keys"]["keylist"][key]
                if keycontent.startswith("key "):
                    keycontent = keycontent[4:]
                cm_keys_mongo(userid).check_register_key(userid, def_cloud, key, keycontent)
                keynamenew = _keyname_sanitation(userid, key)
            else:
                Console.warning("No sshkey found. Please Upload one")

            cookbook = arguments["MENU"]
            s_name = "launcher-{0}-{1}-{2}".format(userid, cookbook, get_rand_string())
            dummy = "123456789"  # doing nothing. just for test
            try:
                t_url = config_launcher["cloudmesh"]["launcher"]["default"]["template"]
            except:
                # If key is missing (KeyError), new cloudmesh_launcher.yaml
                # needs to be copied to ~/.cloudmesh
                t_url = "https://raw.githubusercontent.com/cloudmesh/cloudmesh/master/heat-templates/centos6/launcher/launcher.yaml"

            param = {"KeyName": keynamenew, "Cookbook": cookbook, "dummy": dummy}
            # test for openmpi, hadoop
            if cookbook[:6] == "hadoop" or cookbook[:7] == "openmpi":
                privatekey, publickey = generate_keypair()
                try:
                    t_url = config_launcher["cloudmesh"]["launcher"]["recipes"][cookbook]["template"]
                except:
                    # If key is missing (KeyError), new cloudmesh_launcher.yaml
                    # needs to be copied to ~/.cloudmesh
                    t_url = (
                        "https://raw.githubusercontent.com/cloudmesh/cloudmesh/master/heat-templates/ubuntu-14.04/"
                        + str(cookbook)
                        + "-cluster/"
                        + str(cookbook)
                        + "-cluster.yaml"
                    )
                param = {"KeyName": keynamenew, "PublicKeyString": publickey, "PrivateKeyString": privatekey}
                if cookbook[:9] == "hadoop2.7":
                    param["UserName"] = userid

            log.debug(def_cloud, userid, s_name, t_url, param)
            res = self.cm_mongo.stack_create(
                cloud=def_cloud, cm_user_id=userid, servername=s_name, template_url=t_url, parameters=param
            )
            log.debug(res)
            if "error" in res:
                print(res["error"]["message"])
            return res

        elif arguments["stop"] and arguments["STACK_NAME"]:
            userid = self.cm_config.username()
            def_cloud = self.get_cloud_name(userid)
            s_id = arguments["STACK_NAME"]
            self.cm_mongo.activate(userid)
            res = self.cm_mongo.stack_delete(cloud=def_cloud, cm_user_id=userid, server=s_id)
            log.debug(res)
            return res

        elif arguments["import"]:
            filepath = config_file("/cloudmesh_launcher.yaml")

            if arguments["FILEPATH"]:
                filepath = arguments["FILEPATH"]
            try:
                filename = path_expand(filepath)
                fileconfig = ConfigDict(filename=filename)
            except Exception, err:
                Console.error("error while loading '{0}', please check".format(filepath))
                print(traceback.format_exc())
                print(sys.exc_info()[0])
                return
            try:
                recipes_dict = fileconfig.get("cloudmesh", "launcher", "recipies")
            except:
                Console.error("error while loading recipies from the file")

            # print recipes_dict
            userid = self.cm_config.username()
            launcher_names = []
            launchers = self.cm_mongo.launcher_get(userid)
            for launcher in launchers:
                launcher_names.append(launcher["cm_launcher"].encode("ascii"))

            for key in recipes_dict:
                if key in launcher_names:
                    if arguments["--force"]:
                        self.cm_mongo.launcher_remove(userid, key)
                        self.cm_mongo.launcher_import(recipes_dict[key], key, userid)
                        print("launcher '{0}' overwritten.".format(key))
                    else:
                        print(
                            "ERROR: launcher '{0}' exists, "
                            "please remove it first, or use "
                            "'--force' when adding".format(key)
                        )
                else:
                    self.cm_mongo.launcher_import(recipes_dict[key], key, userid)
                    print("launcher '{0}' added.".format(key))