def remove_vm_from_group_while_deleting(username, vmname):
    '''
    remove a vm from the group, assume there is no duplicate vm names, so
    this function will simplt remove all vms with name vmname from group
    database
    '''
    GroupManage = GroupManagement(username)
    res = GroupManage.get_same_items_from_all_groups("VM", vmname)
    for item in res:
        item.delete()
def add_vm_to_group_while_creating(username, groupname, vmname):
    '''
    add a vm to a group while the vm is being created, if the group doesn't exist, 
    this will create one
    '''
    GroupManage = GroupManagement(username)
    groups_list = GroupManage.get_groups_names_list()
    if groupname not in groups_list:
        GroupManage.create_group(groupname)
    GroupManage.add_item_to_group(groupname, "VM", vmname)
def add_item_to_group(username, groupname, _type, value, refresh=False):
    '''
    this is a more spicific function to add items to groups, e.g. using command group add item
    NAME VM VALUE, check whether vm exists before add it to group
    '''
    GroupManage = GroupManagement(username)
    # if type is VM, check whether the VM exists, in this case, value is the VM name
    if _type.upper() == "VM":
        mongo = cm_mongo()
        if refresh:
            mongo.activate(cm_user_id=username)
            mongo.refresh(cm_user_id=username, types=['servers'])
        servers_dict = mongo.servers(cm_user_id=username)
        _vm_exists = False
        for k, v in servers_dict.iteritems():
            for k0, v0 in v.iteritems():
                if 'name' in v0 and v0['name'] == value:
                    _vm_exists = True
                    break
            if _vm_exists:
                break
        if _vm_exists:
            GroupManage.add_item_to_group(groupname, _type, value)
        else:
            raise Exception("VM '{0}' doesn't exist".format(value))
    else:
        GroupManage.add_item_to_group(groupname, _type, value)
Exemple #4
0
                    Console.error("<LoginName> cannot be empty")
                    return
                else:
                    vm_login_name = arguments['--ln']
            # Moved the import inside of this function
            # If this import goes up to the top, monodb connection will be
            # estabilished. Due to that reason, this import stays here
            # Hyungro Lee 12/01/2014
            # 
            # we have modified how the mongonenigne connects and 
            # it's safe to import any class definition now at the beginning of file
            # Fugang 02/06/2015
            #
            # from cloudmesh.experiment.group import GroupManagement
            # 
            GroupManage = GroupManagement(username)
            groups_list = GroupManage.get_groups_names_list()
            vms_in_group_list = {}
            if GroupName in groups_list:
                vms_in_group_list = GroupManage.list_items_of_group(GroupName, _type="VM")["VM"]

            if not arguments['--force'] and len(vms_in_group_list) != 0:
                if yn_choice("The group you provide exists and it has VMs in it, " + \
                             "do you want to proceed? (if you choose yes, these exist " +\
                             "VMs will be included in the cluster, this could also " +\
                             "rewrite the key on the exist VMs)",
                             default='n',
                             tries=3):
                    pass
                else:
                    return
def get_vms_look_for(username,
                     cloudname,
                     servername=None,
                     serverid=None,
                     groupname=None,
                     prefix=None,
                     hostls=None,
                     getAll=False,
                     refresh=False): 
    '''
    work as a filter to find the VMs you are looking for. Input the seaching conditions,
    and returns a list of server ids that meet the condition
    
    you cannot provide servername and serverid at the same time
    you cannot provide prefix and hostlist at the same time
    param hostls:: e.g. sample[1-3,18] => ['sample1', 'sample2', 'sample3', 'sample18']
    param refresh:: refresh before filtering
    param getAll:: if True, the function consider all VMs are selected before filtering.
                   if False, then none are selected before filtering
    '''
    # GroupManagement automatically loads mongodb connection.
    # To avoid creating db connection by import, we moved this import inside of
    # the function. This way allows us to use GroupManagement class only in this
    # function. Other functions in this file won't load db connection by import.
    # Hyungro Lee - 12/1/2014
    from cloudmesh.experiment.group import GroupManagement
    # input checking 
    if servername and serverid:
        Console.error("you cannot provide servername and serverid at the same time")
        return False
    if prefix and hostls:
        Console.error("you cannot provide prefix and hostlist at the same time")
        return False
    if hostls:
        try:
            hostls_list = hostlist.expand_hostlist(hostls)
        except:
            Console.error("please check your hostlist input, right format e.g. sample[1-9,18]")
            return False
    
    # get server data
    try:
        mongo = cm_mongo()
    except:
        Console.error("There is a problem with the mongo server")
        return False
    
    if refresh:
        mongo.activate(cm_user_id=username, names=[cloudname])
        mongo.refresh(cm_user_id=username,
                      names=[cloudname],
                      types=['servers'])
        
    if groupname:
        vms_in_group_list = []
        GroupManage = GroupManagement(username)
        groups_list = GroupManage.get_groups_names_list()
        if groupname not in groups_list:
            return []
        else:
            vms_in_group_list = GroupManage.list_items_of_group(groupname, _type="VM")["VM"]
        
    servers_dict = mongo.servers(
                clouds=[cloudname], cm_user_id=username)[cloudname]
                
    # search for qualified vms for each critera
    res_d = {}
    if servername:
        res_d['servername'] = []
    if serverid:
        res_d['serverid'] = []
    if groupname:
        res_d['groupname'] = []
    if prefix:
        res_d['prefix'] = []
    if hostls:
        res_d['hostls'] = []
    if getAll:
        res_d['getAll'] = []
    
    for k, v in servers_dict.iteritems():
        if servername and servername == v['name']:
            res_d['servername'].append(k)
        if serverid and serverid == k:
            res_d['serverid'].append(k)
        if groupname:
            if v['name'] in vms_in_group_list:
                res_d['groupname'].append(k)
        if prefix:
            nametemp = server_name_analyzer(v['name'])
        if prefix and prefix == nametemp[0]:
            res_d['prefix'].append(k)
        if hostls and v['name'] in hostls_list:
            res_d['hostls'].append(k)
        if getAll and v['cm_cloud'] == cloudname:
            res_d['getAll'].append(k)
    # -------------------------
    # intersect the results
    ls = res_d.values()
    l = len(ls)
    if l == 0:
        res = []
    elif l == 1:
        res = ls[0]
    else:
        res = ls[0]
        del ls[0]
        for i in ls:
            res = set(res) & set(i)
        res = list(res)
    
    return res
Exemple #6
0
 def __init__(self):
     self.config = cm_config()
     self.mongodb = cm_mongo()
     self.username = self.config.username()
     self.GroupManage = GroupManagement(self.username)
     pass
Exemple #7
0
class Clusters(object):
    def __init__(self):
        self.config = cm_config()
        self.mongodb = cm_mongo()
        self.username = self.config.username()
        self.GroupManage = GroupManagement(self.username)
        pass

    def defaults(self, default_dict):
        """setes image, cloud, flavor, key, username, ...
        
        TODO:: cloud select
               cloud on
               key default
               label
               default image
               default flavor
        """
        pass

    def list_clusters(self):
        """ returns the list of the clusters in json format. This includes the status"""
        groups = self.GroupManage.get_groups()
        res = {}
        for group in groups:
            if self.check_group_is_a_cluster(group):
                res[group.name] = {}
                vms_info = self.vms(group.name)[group.name]
                res[group.name]['num_of_nodes'] = len(vms_info)
                num_of_active_nodes = 0
                for key, value in vms_info.iteritems():
                    if 'status' in value and value['status'].upper() == 'ACTIVE':
                        num_of_active_nodes = num_of_active_nodes + 1
                res[group.name]['num_of_active_nodes'] = num_of_active_nodes

        return res

    def group(self, name):
        """
        returns the group belonging to the cluster according to cluster name
        TODO:: this is similar as function vms
        """
        return None

    def ips(self, name):
        """
        returns a list of ips belonging to the named cluster
        """
        return None

    def names(self, name):
        """
        returns a lsit of names of the vms belonging to the cluster
        
        TODO:: group show name vm --format=json
        """
        return None

    def vms(self, name, refresh=True):
        """
        returns in json the information about the vms of the named cluster
        this includes cluster name and its detailed VMs information
        """
        group_ = self.GroupManage.get_groups(groupname=name)
        if self.check_group_is_a_cluster(group_):
            VM_name_list = self.GroupManage.list_items_of_group(
                name,
                _type="VM")["VM"]
            if refresh:
                self.mongodb.activate(cm_user_id=self.username)
                self.mongodb.refresh(cm_user_id=self.username, types=['servers'])

            VM_dict = self.mongodb.servers(cm_user_id=self.username)
            res = {}
            res[name] = {}
            for cloud, value_0 in VM_dict.iteritems():
                for id_, value_1 in value_0.iteritems():
                    if value_1['name'] in VM_name_list:
                        res[name][value_1['name']] = value_1
                        if '_id' in res[name][value_1['name']]:
                            del res[name][value_1['name']]['_id']
            return res
        else:
            raise Exception("group '{0}' is not a cluster group".format(name))

    def delete(self, name, grouponly=False):
        """
        deletes the named cluster
        
        ::param grouponly: in default, when deleting a cluster, the group object along
                           with its VMs will be deleted, if grouponly=True, then the 
                           VMs will be preserved
        """
        from cloudmesh.experiment.group_usage import remove_vm_from_group_while_deleting

        vms = self.vms(name)[name]
        if not grouponly:
            clouds = []
            for vmname, value in vms.iteritems():
                cloudname = value['cm_cloud']
                id_ = value['id']
                banner("Deleting vm->{0} on cloud->{1}".format(vmname, cloudname))
                result = self.mongodb.vm_delete(cloudname, id_, self.username)
                print(result)
                if cloudname not in clouds:
                    clouds.append(cloudname)
                remove_vm_from_group_while_deleting(self.username, vmname)
            time.sleep(5)
            for cloud in clouds:
                self.mongodb.release_unused_public_ips(cloud, self.username)
            self.mongodb.refresh(cm_user_id=self.username, types=['servers'])
        banner("Deleting group->{0}".format(name))
        self.GroupManage.delete_group(name)

    def info(self, name):
        """returns a simplified information about the cluster in json format"""
        """each vm contains the ips and the name, as well as the status of the vm"""
        return None

    def create_with_existence_check(self, name, n):
        """
        creates a cluster with the given name and the number of vms.
        If the cluster exists an exception is thrown. Returns the json of the cluster.
        """
        pass
        # raise ClusterExistsError('Cluster ' + name + ' exists')

    def create(self, name, n):
        """
        creates a cluster with the given name and the number of vms.
        
        NOTICE: this function doesn't check the existence of a cluster(a 
        cluster us identified by group name), if the cluster already exists,
        this function will add number n VMs to the cluster instead of create 
        a new one
        """
        pass

    def check_group_is_a_cluster(self, groupobj):
        """
        a cluster group object shoud have a 'cluster' tag in its tags
        
        ::param groupobj: group object
        return True if a group contains the tag, otherwise return False
        """
        if "cluster" in groupobj.tags:
            return True
        else:
            return False
Exemple #8
0
 def __init__(self):
     self.config = cm_config()
     self.mongodb = cm_mongo()
     self.username = self.config.username()
     self.GroupManage = GroupManagement(self.username)
     pass
Exemple #9
0
class Clusters(object):
    def __init__(self):
        self.config = cm_config()
        self.mongodb = cm_mongo()
        self.username = self.config.username()
        self.GroupManage = GroupManagement(self.username)
        pass

    def defaults(self, default_dict):
        """setes image, cloud, flavor, key, username, ...
        
        TODO:: cloud select
               cloud on
               key default
               label
               default image
               default flavor
        """
        pass

    def list_clusters(self):
        """ returns the list of the clusters in json format. This includes the status"""
        groups = self.GroupManage.get_groups()
        res = {}
        for group in groups:
            if self.check_group_is_a_cluster(group):
                res[group.name] = {}
                vms_info = self.vms(group.name)[group.name]
                res[group.name]['num_of_nodes'] = len(vms_info)
                num_of_active_nodes = 0
                for key, value in vms_info.iteritems():
                    if 'status' in value and value['status'].upper(
                    ) == 'ACTIVE':
                        num_of_active_nodes = num_of_active_nodes + 1
                res[group.name]['num_of_active_nodes'] = num_of_active_nodes

        return res

    def group(self, name):
        """
        returns the group belonging to the cluster according to cluster name
        TODO:: this is similar as function vms
        """
        return None

    def ips(self, name):
        """
        returns a list of ips belonging to the named cluster
        """
        return None

    def names(self, name):
        """
        returns a lsit of names of the vms belonging to the cluster
        
        TODO:: group show name vm --format=json
        """
        return None

    def vms(self, name, refresh=True):
        """
        returns in json the information about the vms of the named cluster
        this includes cluster name and its detailed VMs information
        """
        group_ = self.GroupManage.get_groups(groupname=name)
        if self.check_group_is_a_cluster(group_):
            VM_name_list = self.GroupManage.list_items_of_group(
                name, _type="VM")["VM"]
            if refresh:
                self.mongodb.activate(cm_user_id=self.username)
                self.mongodb.refresh(cm_user_id=self.username,
                                     types=['servers'])

            VM_dict = self.mongodb.servers(cm_user_id=self.username)
            res = {}
            res[name] = {}
            for cloud, value_0 in VM_dict.iteritems():
                for id_, value_1 in value_0.iteritems():
                    if value_1['name'] in VM_name_list:
                        res[name][value_1['name']] = value_1
                        if '_id' in res[name][value_1['name']]:
                            del res[name][value_1['name']]['_id']
            return res
        else:
            raise Exception("group '{0}' is not a cluster group".format(name))

    def delete(self, name, grouponly=False):
        """
        deletes the named cluster
        
        ::param grouponly: in default, when deleting a cluster, the group object along
                           with its VMs will be deleted, if grouponly=True, then the 
                           VMs will be preserved
        """
        from cloudmesh.experiment.group_usage import remove_vm_from_group_while_deleting

        vms = self.vms(name)[name]
        if not grouponly:
            clouds = []
            for vmname, value in vms.iteritems():
                cloudname = value['cm_cloud']
                id_ = value['id']
                banner("Deleting vm->{0} on cloud->{1}".format(
                    vmname, cloudname))
                result = self.mongodb.vm_delete(cloudname, id_, self.username)
                print(result)
                if cloudname not in clouds:
                    clouds.append(cloudname)
                remove_vm_from_group_while_deleting(self.username, vmname)
            time.sleep(5)
            for cloud in clouds:
                self.mongodb.release_unused_public_ips(cloud, self.username)
            self.mongodb.refresh(cm_user_id=self.username, types=['servers'])
        banner("Deleting group->{0}".format(name))
        self.GroupManage.delete_group(name)

    def info(self, name):
        """returns a simplified information about the cluster in json format"""
        """each vm contains the ips and the name, as well as the status of the vm"""
        return None

    def create_with_existence_check(self, name, n):
        """
        creates a cluster with the given name and the number of vms.
        If the cluster exists an exception is thrown. Returns the json of the cluster.
        """
        pass
        # raise ClusterExistsError('Cluster ' + name + ' exists')

    def create(self, name, n):
        """
        creates a cluster with the given name and the number of vms.
        
        NOTICE: this function doesn't check the existence of a cluster(a 
        cluster us identified by group name), if the cluster already exists,
        this function will add number n VMs to the cluster instead of create 
        a new one
        """
        pass

    def check_group_is_a_cluster(self, groupobj):
        """
        a cluster group object shoud have a 'cluster' tag in its tags
        
        ::param groupobj: group object
        return True if a group contains the tag, otherwise return False
        """
        if "cluster" in groupobj.tags:
            return True
        else:
            return False
def shell_command_experiment_group(arguments):
    """
    Usage:
        group list [--format=FORMAT]
        group create NAME
        group remove NAME
        group add item NAME TYPE VALUE
        group remove item NAME TYPE VALUE
        group show NAME [TYPE] [--format=FORMAT]

    Arguments:

        NAME    name of the group
        TYPE    type of the item in the group, e.g. vm 
        VALUE   value of item to add, e.g. vm name

    Options:

        -v               verbose mode
        --format=FORMAT  output format: table, json, csv

    Description:

       group list           lists the groups
       group create         creates a new group
       group remove         removes a group
       group add item       addes an item of a type to a group
       group remove item    removes an item of a type from a group
       group show           lists items of a group
       
    Examples:
        group add item sample vm samplevm
            add vm named samplevm to group sample
            
        group show sample vm --format=json
            list all VMs of group sample in json format

    Example:

       group create experiment_1
       vm start
       last = vm label
       group add experiment_1 vm last

       group create experiment_2
       vm start
       last = vm info label  # prints the vm label /prefix + number
       ipno = vm info ip # prints the ip of the last vm
       ipno = vm info ip gvonlasz_1  # get ip of vm with label gvonlasz_1

       group add expermiment_2 ip ipno

       groups are just tuples

       i can have multiple Kinds in the tuple

    mongoengine

    class groupObject

        def add (... name, kind, attribute ...)
        def printer ( ... kind, printfunction, name...)
        def getter ( .... kind, name)
        
    def getter ( .... kind, name ...)

       if kind == "vm":
          vm = get vm from mongo
          return vm
       elif kind = "image"
          iamge = get image from mongo
          return iamge
       ....

    def vmPrinter ( .... vm ...)

       print vm.ip
       print vm.name
       ....

    def imagePrinter ( .... image ...)

       print image.size
       print image.name
       ....

       

    g = groupObject()
    g.printer("vm", cmPrinter)
    g.printer("image", imagePrinter)

    
       
       
    """
    # Changed the scope of this import.
    from cloudmesh.experiment.group import GroupManagement
    from cloudmesh.experiment.group_usage import add_item_to_group

    name = arguments["NAME"]
    type = arguments["TYPE"]
    value = arguments["VALUE"]

    config = cm_config()
    username = config.username()
    # print username
    user = cm_user()
    
    GroupManage = GroupManagement(username)

    '''
    if arguments["info"]:

        print("Default experiment group:", user.get_defaults(username)["group"])

    '''
    if arguments["list"]:
        try:
            res = GroupManage.get_groups_names_list()
        except Exception, err:
            Console.error(str(err))
            return
        d = {}
        d["groups"] = res
        
        if arguments['--format']:
            p_format = arguments['--format']
        else:
            p_format = None
            
        shell_commands_dict_output(username,
                                   d,
                                   print_format=p_format,
                                   table_format="key_list",
                                   indexed=True)
Exemple #11
0
def get_vms_look_for(username,
                     cloudname,
                     servername=None,
                     serverid=None,
                     groupname=None,
                     prefix=None,
                     hostls=None,
                     getAll=False,
                     refresh=False):
    '''
    work as a filter to find the VMs you are looking for. Input the seaching conditions,
    and returns a list of server ids that meet the condition
    
    you cannot provide servername and serverid at the same time
    you cannot provide prefix and hostlist at the same time
    param hostls:: e.g. sample[1-3,18] => ['sample1', 'sample2', 'sample3', 'sample18']
    param refresh:: refresh before filtering
    param getAll:: if True, the function consider all VMs are selected before filtering.
                   if False, then none are selected before filtering
    '''
    # GroupManagement automatically loads mongodb connection.
    # To avoid creating db connection by import, we moved this import inside of
    # the function. This way allows us to use GroupManagement class only in this
    # function. Other functions in this file won't load db connection by import.
    # Hyungro Lee - 12/1/2014
    from cloudmesh.experiment.group import GroupManagement
    # input checking
    if servername and serverid:
        Console.error(
            "you cannot provide servername and serverid at the same time")
        return False
    if prefix and hostls:
        Console.error(
            "you cannot provide prefix and hostlist at the same time")
        return False
    if hostls:
        try:
            hostls_list = hostlist.expand_hostlist(hostls)
        except:
            Console.error(
                "please check your hostlist input, right format e.g. sample[1-9,18]"
            )
            return False

    # get server data
    try:
        mongo = cm_mongo()
    except:
        Console.error("There is a problem with the mongo server")
        return False

    if refresh:
        mongo.activate(cm_user_id=username, names=[cloudname])
        mongo.refresh(cm_user_id=username,
                      names=[cloudname],
                      types=['servers'])

    if groupname:
        vms_in_group_list = []
        GroupManage = GroupManagement(username)
        groups_list = GroupManage.get_groups_names_list()
        if groupname not in groups_list:
            return []
        else:
            vms_in_group_list = GroupManage.list_items_of_group(
                groupname, _type="VM")["VM"]

    servers_dict = mongo.servers(clouds=[cloudname],
                                 cm_user_id=username)[cloudname]

    # search for qualified vms for each critera
    res_d = {}
    if servername:
        res_d['servername'] = []
    if serverid:
        res_d['serverid'] = []
    if groupname:
        res_d['groupname'] = []
    if prefix:
        res_d['prefix'] = []
    if hostls:
        res_d['hostls'] = []
    if getAll:
        res_d['getAll'] = []

    for k, v in servers_dict.iteritems():
        if servername and servername == v['name']:
            res_d['servername'].append(k)
        if serverid and serverid == k:
            res_d['serverid'].append(k)
        if groupname:
            if v['name'] in vms_in_group_list:
                res_d['groupname'].append(k)
        if prefix:
            nametemp = server_name_analyzer(v['name'])
        if prefix and prefix == nametemp[0]:
            res_d['prefix'].append(k)
        if hostls and v['name'] in hostls_list:
            res_d['hostls'].append(k)
        if getAll and v['cm_cloud'] == cloudname:
            res_d['getAll'].append(k)
    # -------------------------
    # intersect the results
    ls = res_d.values()
    l = len(ls)
    if l == 0:
        res = []
    elif l == 1:
        res = ls[0]
    else:
        res = ls[0]
        del ls[0]
        for i in ls:
            res = set(res) & set(i)
        res = list(res)

    return res
Exemple #12
0
def shell_command_experiment_group(arguments):
    """
    ::

      Usage:
          group list [--format=FORMAT]
          group create NAME
          group remove NAME
          group add item NAME TYPE VALUE
          group remove item NAME TYPE VALUE
          group show NAME [TYPE] [--format=FORMAT]

      Arguments:

          NAME    name of the group
          TYPE    type of the item in the group, e.g. vm 
          VALUE   value of item to add, e.g. vm name

      Options:

          -v               verbose mode
          --format=FORMAT  output format: table, json, csv

      Description:

         group list           lists the groups
         group create         creates a new group
         group remove         removes a group
         group add item       addes an item of a type to a group
         group remove item    removes an item of a type from a group
         group show           lists items of a group

      Examples:
          group add item sample vm samplevm
              add vm named samplevm to group sample

          group show sample vm --format=json
              list all VMs of group sample in json format
    """

    """
      Example:

         group create experiment_1
         vm start
         last = vm label
         group add experiment_1 vm last

         group create experiment_2
         vm start
         last = vm info label  # prints the vm label /prefix + number
         ipno = vm info ip # prints the ip of the last vm
         ipno = vm info ip gvonlasz_1  # get ip of vm with label gvonlasz_1

         group add expermiment_2 ip ipno

         groups are just tuples

         i can have multiple Kinds in the tuple

      mongoengine

      class groupObject

          def add (... name, kind, attribute ...)
          def printer ( ... kind, printfunction, name...)
          def getter ( .... kind, name)

      def getter ( .... kind, name ...)

         if kind == "vm":
            vm = get vm from mongo
            return vm
         elif kind = "image"
            iamge = get image from mongo
            return iamge
         ....

      def vmPrinter ( .... vm ...)

         print vm.ip
         print vm.name
         ....

      def imagePrinter ( .... image ...)

         print image.size
         print image.name
         ....



      g = groupObject()
      g.printer("vm", cmPrinter)
      g.printer("image", imagePrinter)



       
    """
    # Changed the scope of this import.
    from cloudmesh.experiment.group import GroupManagement
    from cloudmesh.experiment.group_usage import add_item_to_group

    name = arguments["NAME"]
    type_ = arguments["TYPE"]
    value = arguments["VALUE"]

    config = cm_config()
    username = config.username()
    # print username
    user = cm_user()

    GroupManage = GroupManagement(username)

    '''
    if arguments["info"]:

        print("Default experiment group:", user.get_defaults(username)["group"])

    '''
    if arguments["list"]:
        try:
            res = GroupManage.get_groups_names_list()
        except Exception, err:
            Console.error(str(err))
            return
        d = {}
        d["groups"] = res

        if arguments['--format']:
            p_format = arguments['--format']
        else:
            p_format = None

        shell_commands_dict_output(username,
                                   d,
                                   print_format=p_format,
                                   table_format="key_list",
                                   indexed=True)
                    Console.error("<LoginName> cannot be empty")
                    return
                else:
                    vm_login_name = arguments['--ln']
            # Moved the import inside of this function
            # If this import goes up to the top, monodb connection will be
            # estabilished. Due to that reason, this import stays here
            # Hyungro Lee 12/01/2014
            #
            # we have modified how the mongonenigne connects and
            # it's safe to import any class definition now at the beginning of file
            # Fugang 02/06/2015
            #
            # from cloudmesh.experiment.group import GroupManagement
            #
            GroupManage = GroupManagement(username)
            groups_list = GroupManage.get_groups_names_list()
            vms_in_group_list = {}
            if GroupName in groups_list:
                vms_in_group_list = GroupManage.list_items_of_group(
                    GroupName, _type="VM")["VM"]

            if not arguments['--force'] and len(vms_in_group_list) != 0:
                if yn_choice("The group you provide exists and it has VMs in it, " + \
                             "do you want to proceed? (if you choose yes, these exist " +\
                             "VMs will be included in the cluster, this could also " +\
                             "rewrite the key on the exist VMs)",
                             default='n',
                             tries=3):
                    pass
                else:
Exemple #14
0
    def do_cluster(self, args, arguments):
        """
        Usage:
            cluster start CLUSTER_NAME
            cluster list
            cluster login CLUSTER_NAME
            cluster stop CLUSTER_NAME
            cluster create --count=<count>
                           --group=<group>
                           [--ln=<LoginName>]
                           [--cloud=<CloudName>]
                           [--image=<imgName>|--imageid=<imgId>]
                           [--flavor=<flavorName>|--flavorid=<flavorId>]
                           [--force]
                                    
        Description:
            Cluster Management
            
            cluster create --count=<count> --group=<group> --ln=<LoginName> [options...]
            <count>            specify amount of VMs in the cluster
            <group>            specify a group name of the cluster, make sure it's unique
                Start a cluster of VMs, and each of them can log into all others.
                CAUTION: you sould do some default setting before using this command:
                1. select cloud to work on, e.g. cloud select india
                2. activate the cloud, e.g. cloud on india
                3. set the default key to start VMs, e.g. key default [NAME]
                4. set the start name of VMs, which is prefix and index, e.g. label --prefix=test --id=1
                5. set image of VMs, e.g. default image
                6. set flavor of VMs, e.g. default flavor
                Also, please make sure the group name of the cluster is unique
                
        Options:
            --ln=<LoginName>           give a login name for the VMs, e.g. ubuntu
            --cloud=<CloudName>        give a cloud to work on
            --flavor=<flavorName>      give the name of the flavor
            --flavorid=<flavorId>      give the id of the flavor
            --image=<imgName>          give the name of the image
            --imageid=<imgId>          give the id of the image
            --force                    if a group exists and there are VMs in it, the program will
                                       ask user to proceed or not, use this flag to respond yes as 
                                       default(if there are VMs in the group before creating this 
                                       cluster, the program will include the exist VMs into the cluster)


        """
        #pprint(arguments)
        self.cm_config = cm_config()
        self.cm_mongo = cm_mongo()
        self.user = cm_user()
       
        # -----------------------------
        # TODO::
        # add VMs to cluster
        # -----------------------------
       
        if arguments['start'] and arguments['CLUSTER_NAME']:
            '''Starts a cluster'''

            # Initialize default variables. e.g. userid, default cloud and
            # default keypair
            userid = self.cm_config.username()
            def_cloud = self.get_cloud_name(userid)
            self.cm_mongo.activate(userid)
            
            userinfo = self.user.info(userid)
            if "key" in userinfo["defaults"]:
                key = userinfo["defaults"]["key"]
            elif len(userinfo["keys"]["keylist"].keys()) > 0:
                key = userinfo["keys"]["keylist"].keys()[0]
        
            if key:
                keycontent = userinfo["keys"]["keylist"][key]
                if keycontent.startswith('key '):
                    keycontent = keycontent[4:]
                cm_keys_mongo(userid).check_register_key(userid, def_cloud, key, keycontent)
                keynamenew = _keyname_sanitation(userid, key)
            else:
                Console.warning("No sshkey found. Please Upload one")
                return
            
            clustername = arguments['CLUSTER_NAME']
            s_name = "cluster-{0}-{1}-{2}".format(userid, clustername, get_rand_string())
            # TEMP FOR HADOOP CLUSTER
            if clustername != "hadoop":
                Console.warning('hadoop is only available cluster')
                return
            
            # 1. keypair for the communication between master and worker nodes
            privatekey, publickey = generate_keypair()
            t_url = \
            "https://raw.githubusercontent.com/cloudmesh/cloudmesh/dev1.3/heat-templates/ubuntu-14.04/hadoop-cluster/hadoop-cluster.yaml"
            param = {'KeyName': keynamenew,
                     'PublicKeyString': publickey,
                     'PrivateKeyString': privatekey}
            log.debug(def_cloud, userid, s_name, t_url, param, publickey,
                      privatekey)
            res = self.cm_mongo.stack_create(cloud=def_cloud, cm_user_id=userid,
                                             servername=s_name,
                                             template_url=t_url,
                                             parameters=param)
            log.debug(res)
            if 'error' in res:
                print (res['error']['message'])
            return res

        elif arguments['list']:
            userid = self.cm_config.username()
            self.cm_mongo.activate(userid)
            self.cm_mongo.refresh(cm_user_id=userid, types=[self._id])
            stacks = self.cm_mongo.stacks(cm_user_id=userid)
            launchers = self.filter_launcher(
                stacks,
                {"search": "contain",
                 "key": "stack_name",
                 "value": "launcher"}
                )
            log.debug(launchers)

            d = {}
            for k0, v0 in launchers.iteritems():
                for k1, v1 in launchers[k0].iteritems():
                    d[v1['id']] = v1
            columns = ['stack_name', 'description', 'stack_status',
                       'creation_time', 'cm_cloud']
            if arguments['--column'] and arguments['--column'] != "all":
                columns = [x.strip() for x in arguments['--column'].split(',')]

            if arguments['--format']:
                if arguments['--format'] not in ['table', 'json', 'csv']:
                    Console.error("please select printing format among table, json and csv")
                    return
                else:
                    p_format = arguments['--format']
            else:
                p_format = None

            shell_commands_dict_output(d,
                                       print_format=p_format,
                                       firstheader="launcher_id",
                                       header=columns
                                       # vertical_table=True
                                       )

        elif arguments['login']:
            Console.error("Not implemented")
            return

        elif arguments['stop'] and arguments['CLUSTER_NAME']:
            userid = self.cm_config.username()
            def_cloud = self.get_cloud_name(userid)
            c_id = arguments['CLUSTER_NAME']
            self.cm_mongo.activate(userid)
            res = self.cm_mongo.stack_delete(cloud=def_cloud,
                                             cm_user_id=userid,
                                             server=c_id)
            log.debug(res)
            return res

        elif arguments['create']:
            try:
                config = cm_config()
            except:
                Console.error("Failed to load the cloudmesh yaml file")
                return
            username = config['cloudmesh']['profile']['username']
            cloudname = arguments['--cloud'] or CloudManage().get_selected_cloud(username)
            temp_dir_name = ".temp_cluster_create_" + username + "_0"
            while os.path.isdir("./{0}".format(temp_dir_name)):
                temp_dir_name = temp_dir_name[:-1] + str(int(temp_dir_name[-1]) + 1)
            dir_name = temp_dir_name
            
            #NumOfVM = None
            GroupName = None

            vm_login_name = "ubuntu"

            
            temp_key_name = "sshkey_temp"
            _key = "-i ./{0}/{1}".format(dir_name, temp_key_name)
            StrictHostKeyChecking = "-o StrictHostKeyChecking=no"
            
            res = None
            to_print = []
            '''
            try:
                NumOfVM = abs(int(argument['--count']))
            except:
                Console.error("<count> must be an integer")
                return
            '''
            if arguments['--group'] == '':
                Console.error("<group> cannot be empty")
                return
            else:
                GroupName = arguments['--group']
            
            if arguments['--ln']:
                if arguments['--ln'] == '':
                    Console.error("<LoginName> cannot be empty")
                    return
                else:
                    vm_login_name = arguments['--ln']
            
            if not arguments['--force']:
                # Moved the import inside of this function
                # If this import goes up to the top, monodb connection will be
                # estabilished. Due to that reason, this import stays here
                # Hyungro Lee 12/01/2014
                from cloudmesh.experiment.group import GroupManagement
                GroupManage = GroupManagement(username)
                groups_list = GroupManage.get_groups_names_list()
                if GroupName in groups_list:
                    vms_in_group_list = GroupManage.list_items_of_group(GroupName, _type="VM")["VM"]
                    if len(vms_in_group_list) != 0:
                        if yn_choice("The group you provide exists and it has VMs in it, " + \
                                     "do you want to proceed? (if you choose yes, these exist " +\
                                     "VMs will be included in the cluster, this could also " +\
                                     "rewrite the key on the exist VMs)",
                                     default='n',
                                     tries=3):
                            pass
                        else:
                            return
                
            # start VMs 
            print ("starting VMs...")
            arguments_temp = arguments
            arguments_temp['start'] = True
            arguments_temp['--name'] = None
            
            vmclass = VMcommand(arguments_temp)
            res = vmclass.call_procedure()
            if res == False: return
        
            def string_to_dict(s):
                h = s.find("{")
                t = s.rfind("}")
                return json.loads(s[h:t+1])
            
            def check_public_ip_existence(d):
                temp = d['addresses']['private']
                for item in temp:
                    if item["OS-EXT-IPS:type"] == "floating":
                        return True
                return False
            
            def get_ip(d, kind="floating"): # kind is either floating or fixed
                temp = d['addresses']['private']
                for item in temp:
                    if item["OS-EXT-IPS:type"] == kind:
                        return item['addr']#.encode('ascii')
                return "FAIL: doesn't exist"
            
            # check all VMs are active
            command_refresh = "vm list --refresh --group={0} --format=json".format(GroupName)
            def _help0(d):
                for k, v in d.iteritems():
                    if v['status'] != 'ACTIVE':
                        return False
                return True
                
            proceed = False
            repeat_index = 1
            while proceed != True:
                if repeat_index > 10:
                    Console.warning("Please check the network")
                    return
                print ("checking({0})...".format(repeat_index))
                time.sleep(5)
                res = str(cm(command_refresh))
                res = string_to_dict(res)
                if _help0(res):
                    proceed = True
                else:
                    repeat_index = repeat_index + 1
                    continue
            
            # assign ip to all VMs
            print ("assigning public ips...")
            for k, v in res.iteritems():
                if not check_public_ip_existence(v):
                    cm("vm ip assign --id={0}".format(k.encode('ascii')))
            
            def _help(d):
                for k, v in d.iteritems():
                    if check_public_ip_existence(v) != True:
                        return False
                return True
            
            # make sure all VMs have been assigned a public ip
            proceed = False
            repeat_index = 1
            while proceed != True:
                if repeat_index > 10:
                    Console.warning("Please check the network")
                    return
                print ("checking({0})...".format(repeat_index))
                time.sleep(5)
                res = str(cm(command_refresh))
                res = string_to_dict(res)
                if _help(res):
                    proceed = True
                else:
                    repeat_index = repeat_index + 1
                    continue
                
            
            # -------------------------
            # key handler
            userinfo = cm_user().info(username)
            key = None
            if "key" in userinfo["defaults"]:
                key = userinfo["defaults"]["key"]
            elif len(userinfo["keys"]["keylist"].keys()) > 0:
                key = userinfo["keys"]["keylist"].keys()[0]
                Console.warning("default key is not set, trying to use a key in the database...")
        
            if key:
                keycontent = userinfo["keys"]["keylist"][key]
                if keycontent.startswith('key '):
                    keycontent = keycontent[4:]
                cm_keys_mongo(username).check_register_key(username, cloudname, key, keycontent)
            else:
                Console.error("No sshkey found. Please Upload one")
                return
            # -------------------------
            
            
            # generate ssh keys for VMs and prepare two files: authorized_keys and hosts
            print ("generating ssh keys...")
            os.popen("mkdir {0}".format(dir_name))

            fa = open("./{0}/authorized_keys_temp".format(dir_name), "w")
            fh = open("./{0}/hosts_temp".format(dir_name), "w")
            fk = open("./{0}/{1}".format(dir_name, temp_key_name), "w")
            
            fk.write(keycontent)
            fk.close()
            os.popen("chmod 644 ./{0}/{1}".format(dir_name, temp_key_name))
            
            for k, v in res.iteritems():
                address_floating = get_ip(v)
                address_fixed = get_ip(v, kind="fixed")
                vm_name = v['name']#.encode('ascii')
                to_print.append("{0} {1}, {2}".format(vm_name, address_floating, address_fixed))
                fh.write(address_floating + "  " + vm_name + "\n"
                         + address_fixed + "  " + vm_name + "-i\n")
                os.popen("ssh {2} {3} {0}@{1} \"ssh-keygen -t rsa -N '' -f ~/.ssh/id_rsa\""\
                         .format(vm_login_name,address_floating, _key, StrictHostKeyChecking))
                temp = os.popen("ssh {2} {3} {0}@{1} \"cat ~/.ssh/id_rsa.pub\""\
                                .format(vm_login_name, address_floating, _key, StrictHostKeyChecking)).read()
                fa.write(temp)
                
            fa.close()
            fh.close()
            
            # copy the files to VMs
            print ("copying the files...")
            os.popen("mkdir ./{0}/oops".format(dir_name))
            for k, v in res.iteritems():
                address_floating = get_ip(v)
                os.popen("scp {2} {3} {0}@{1}:~/.ssh/authorized_keys ./{4}/"\
                         .format(vm_login_name,address_floating, _key, StrictHostKeyChecking, dir_name))
                os.popen("cat ./{0}/authorized_keys_temp >> ./{0}/authorized_keys"\
                         .format(dir_name))
                os.popen("scp {2} {3} ./{4}/authorized_keys {0}@{1}:~"\
                         .format(vm_login_name,address_floating, _key, StrictHostKeyChecking, dir_name))
                os.popen("ssh {2} {3} {0}@{1} \"sudo mv authorized_keys ~/.ssh/\""\
                         .format(vm_login_name,address_floating, _key, StrictHostKeyChecking))
                os.popen("rm ./{0}/authorized_keys".format(dir_name))
                
                os.popen("cp ./{0}/hosts_temp ./{0}/oops/".format(dir_name))
                os.popen("mv ./{0}/oops/hosts_temp ./{0}/oops/hosts".format(dir_name))
                fh0 = open("./{0}/oops/hosts".format(dir_name), "a")
                os.popen("scp {2} {3} {0}@{1}:/etc/hosts ./{4}/"\
                         .format(vm_login_name,address_floating, _key, StrictHostKeyChecking, dir_name))
                with open("./{0}/hosts".format(dir_name)) as f0:
                    content = f0.readlines()
                for item in content:
                    fh0.write(item + "\n")
                fh0.close()
                os.popen("scp {2} {3} ./{4}/oops/hosts {0}@{1}:~"\
                         .format(vm_login_name,address_floating, _key, StrictHostKeyChecking, dir_name))
                os.popen("ssh {2} {3} {0}@{1} \"sudo mv hosts /etc/\""\
                         .format(vm_login_name,address_floating, _key, StrictHostKeyChecking))
                os.popen("rm ./{0}/oops/hosts".format(dir_name))
            
            
            print ("finishing...")
            os.popen("rm -rf {0}".format(dir_name))
            print ("DONE.")
            
            print ("cluster group: ", GroupName)
            for item in to_print:
                print (item)
            print ("(host name for private ips will have -i at the end of VM name, e.g. testVM -> testVM-i)")