def remove_vm_from_group_while_deleting(username, vmname):
'''
remove a vm from the group, assume there is no duplicate vm names, so
this function will simplt remove all vms with name vmname from group
database
'''
GroupManage = GroupManagement(username)
res = GroupManage.get_same_items_from_all_groups("VM", vmname)
for item in res:
item.delete()
def add_vm_to_group_while_creating(username, groupname, vmname):
'''
add a vm to a group while the vm is being created, if the group doesn't exist,
this will create one
'''
GroupManage = GroupManagement(username)
groups_list = GroupManage.get_groups_names_list()
if groupname not in groups_list:
GroupManage.create_group(groupname)
GroupManage.add_item_to_group(groupname, "VM", vmname)
def add_item_to_group(username, groupname, _type, value, refresh=False):
'''
this is a more spicific function to add items to groups, e.g. using command group add item
NAME VM VALUE, check whether vm exists before add it to group
'''
GroupManage = GroupManagement(username)
# if type is VM, check whether the VM exists, in this case, value is the VM name
if _type.upper() == "VM":
mongo = cm_mongo()
if refresh:
mongo.activate(cm_user_id=username)
mongo.refresh(cm_user_id=username, types=['servers'])
servers_dict = mongo.servers(cm_user_id=username)
_vm_exists = False
for k, v in servers_dict.iteritems():
for k0, v0 in v.iteritems():
if 'name' in v0 and v0['name'] == value:
_vm_exists = True
break
if _vm_exists:
break
if _vm_exists:
GroupManage.add_item_to_group(groupname, _type, value)
else:
raise Exception("VM '{0}' doesn't exist".format(value))
else:
GroupManage.add_item_to_group(groupname, _type, value)
Console.error("<LoginName> cannot be empty")
return
else:
vm_login_name = arguments['--ln']
# Moved the import inside of this function
# If this import goes up to the top, monodb connection will be
# estabilished. Due to that reason, this import stays here
# Hyungro Lee 12/01/2014
#
# we have modified how the mongonenigne connects and
# it's safe to import any class definition now at the beginning of file
# Fugang 02/06/2015
#
# from cloudmesh.experiment.group import GroupManagement
#
GroupManage = GroupManagement(username)
groups_list = GroupManage.get_groups_names_list()
vms_in_group_list = {}
if GroupName in groups_list:
vms_in_group_list = GroupManage.list_items_of_group(GroupName, _type="VM")["VM"]
if not arguments['--force'] and len(vms_in_group_list) != 0:
if yn_choice("The group you provide exists and it has VMs in it, " + \
"do you want to proceed? (if you choose yes, these exist " +\
"VMs will be included in the cluster, this could also " +\
"rewrite the key on the exist VMs)",
default='n',
tries=3):
pass
else:
return
def get_vms_look_for(username,
cloudname,
servername=None,
serverid=None,
groupname=None,
prefix=None,
hostls=None,
getAll=False,
refresh=False):
'''
work as a filter to find the VMs you are looking for. Input the seaching conditions,
and returns a list of server ids that meet the condition
you cannot provide servername and serverid at the same time
you cannot provide prefix and hostlist at the same time
param hostls:: e.g. sample[1-3,18] => ['sample1', 'sample2', 'sample3', 'sample18']
param refresh:: refresh before filtering
param getAll:: if True, the function consider all VMs are selected before filtering.
if False, then none are selected before filtering
'''
# GroupManagement automatically loads mongodb connection.
# To avoid creating db connection by import, we moved this import inside of
# the function. This way allows us to use GroupManagement class only in this
# function. Other functions in this file won't load db connection by import.
# Hyungro Lee - 12/1/2014
from cloudmesh.experiment.group import GroupManagement
# input checking
if servername and serverid:
Console.error("you cannot provide servername and serverid at the same time")
return False
if prefix and hostls:
Console.error("you cannot provide prefix and hostlist at the same time")
return False
if hostls:
try:
hostls_list = hostlist.expand_hostlist(hostls)
except:
Console.error("please check your hostlist input, right format e.g. sample[1-9,18]")
return False
# get server data
try:
mongo = cm_mongo()
except:
Console.error("There is a problem with the mongo server")
return False
if refresh:
mongo.activate(cm_user_id=username, names=[cloudname])
mongo.refresh(cm_user_id=username,
names=[cloudname],
types=['servers'])
if groupname:
vms_in_group_list = []
GroupManage = GroupManagement(username)
groups_list = GroupManage.get_groups_names_list()
if groupname not in groups_list:
return []
else:
vms_in_group_list = GroupManage.list_items_of_group(groupname, _type="VM")["VM"]
servers_dict = mongo.servers(
clouds=[cloudname], cm_user_id=username)[cloudname]
# search for qualified vms for each critera
res_d = {}
if servername:
res_d['servername'] = []
if serverid:
res_d['serverid'] = []
if groupname:
res_d['groupname'] = []
if prefix:
res_d['prefix'] = []
if hostls:
res_d['hostls'] = []
if getAll:
res_d['getAll'] = []
for k, v in servers_dict.iteritems():
if servername and servername == v['name']:
res_d['servername'].append(k)
if serverid and serverid == k:
res_d['serverid'].append(k)
if groupname:
if v['name'] in vms_in_group_list:
res_d['groupname'].append(k)
if prefix:
nametemp = server_name_analyzer(v['name'])
if prefix and prefix == nametemp[0]:
res_d['prefix'].append(k)
if hostls and v['name'] in hostls_list:
res_d['hostls'].append(k)
if getAll and v['cm_cloud'] == cloudname:
res_d['getAll'].append(k)
# -------------------------
# intersect the results
ls = res_d.values()
l = len(ls)
if l == 0:
res = []
elif l == 1:
res = ls[0]
else:
res = ls[0]
del ls[0]
for i in ls:
res = set(res) & set(i)
res = list(res)
return res
def __init__(self):
self.config = cm_config()
self.mongodb = cm_mongo()
self.username = self.config.username()
self.GroupManage = GroupManagement(self.username)
pass
class Clusters(object):
def __init__(self):
self.config = cm_config()
self.mongodb = cm_mongo()
self.username = self.config.username()
self.GroupManage = GroupManagement(self.username)
pass
def defaults(self, default_dict):
"""setes image, cloud, flavor, key, username, ...
TODO:: cloud select
cloud on
key default
label
default image
default flavor
"""
pass
def list_clusters(self):
""" returns the list of the clusters in json format. This includes the status"""
groups = self.GroupManage.get_groups()
res = {}
for group in groups:
if self.check_group_is_a_cluster(group):
res[group.name] = {}
vms_info = self.vms(group.name)[group.name]
res[group.name]['num_of_nodes'] = len(vms_info)
num_of_active_nodes = 0
for key, value in vms_info.iteritems():
if 'status' in value and value['status'].upper() == 'ACTIVE':
num_of_active_nodes = num_of_active_nodes + 1
res[group.name]['num_of_active_nodes'] = num_of_active_nodes
return res
def group(self, name):
"""
returns the group belonging to the cluster according to cluster name
TODO:: this is similar as function vms
"""
return None
def ips(self, name):
"""
returns a list of ips belonging to the named cluster
"""
return None
def names(self, name):
"""
returns a lsit of names of the vms belonging to the cluster
TODO:: group show name vm --format=json
"""
return None
def vms(self, name, refresh=True):
"""
returns in json the information about the vms of the named cluster
this includes cluster name and its detailed VMs information
"""
group_ = self.GroupManage.get_groups(groupname=name)
if self.check_group_is_a_cluster(group_):
VM_name_list = self.GroupManage.list_items_of_group(
name,
_type="VM")["VM"]
if refresh:
self.mongodb.activate(cm_user_id=self.username)
self.mongodb.refresh(cm_user_id=self.username, types=['servers'])
VM_dict = self.mongodb.servers(cm_user_id=self.username)
res = {}
res[name] = {}
for cloud, value_0 in VM_dict.iteritems():
for id_, value_1 in value_0.iteritems():
if value_1['name'] in VM_name_list:
res[name][value_1['name']] = value_1
if '_id' in res[name][value_1['name']]:
del res[name][value_1['name']]['_id']
return res
else:
raise Exception("group '{0}' is not a cluster group".format(name))
def delete(self, name, grouponly=False):
"""
deletes the named cluster
::param grouponly: in default, when deleting a cluster, the group object along
with its VMs will be deleted, if grouponly=True, then the
VMs will be preserved
"""
from cloudmesh.experiment.group_usage import remove_vm_from_group_while_deleting
vms = self.vms(name)[name]
if not grouponly:
clouds = []
for vmname, value in vms.iteritems():
cloudname = value['cm_cloud']
id_ = value['id']
banner("Deleting vm->{0} on cloud->{1}".format(vmname, cloudname))
result = self.mongodb.vm_delete(cloudname, id_, self.username)
print(result)
if cloudname not in clouds:
clouds.append(cloudname)
remove_vm_from_group_while_deleting(self.username, vmname)
time.sleep(5)
for cloud in clouds:
self.mongodb.release_unused_public_ips(cloud, self.username)
self.mongodb.refresh(cm_user_id=self.username, types=['servers'])
banner("Deleting group->{0}".format(name))
self.GroupManage.delete_group(name)
def info(self, name):
"""returns a simplified information about the cluster in json format"""
"""each vm contains the ips and the name, as well as the status of the vm"""
return None
def create_with_existence_check(self, name, n):
"""
creates a cluster with the given name and the number of vms.
If the cluster exists an exception is thrown. Returns the json of the cluster.
"""
pass
# raise ClusterExistsError('Cluster ' + name + ' exists')
def create(self, name, n):
"""
creates a cluster with the given name and the number of vms.
NOTICE: this function doesn't check the existence of a cluster(a
cluster us identified by group name), if the cluster already exists,
this function will add number n VMs to the cluster instead of create
a new one
"""
pass
def check_group_is_a_cluster(self, groupobj):
"""
a cluster group object shoud have a 'cluster' tag in its tags
::param groupobj: group object
return True if a group contains the tag, otherwise return False
"""
if "cluster" in groupobj.tags:
return True
else:
return False
def __init__(self):
self.config = cm_config()
self.mongodb = cm_mongo()
self.username = self.config.username()
self.GroupManage = GroupManagement(self.username)
pass
class Clusters(object):
def __init__(self):
self.config = cm_config()
self.mongodb = cm_mongo()
self.username = self.config.username()
self.GroupManage = GroupManagement(self.username)
pass
def defaults(self, default_dict):
"""setes image, cloud, flavor, key, username, ...
TODO:: cloud select
cloud on
key default
label
default image
default flavor
"""
pass
def list_clusters(self):
""" returns the list of the clusters in json format. This includes the status"""
groups = self.GroupManage.get_groups()
res = {}
for group in groups:
if self.check_group_is_a_cluster(group):
res[group.name] = {}
vms_info = self.vms(group.name)[group.name]
res[group.name]['num_of_nodes'] = len(vms_info)
num_of_active_nodes = 0
for key, value in vms_info.iteritems():
if 'status' in value and value['status'].upper(
) == 'ACTIVE':
num_of_active_nodes = num_of_active_nodes + 1
res[group.name]['num_of_active_nodes'] = num_of_active_nodes
return res
def group(self, name):
"""
returns the group belonging to the cluster according to cluster name
TODO:: this is similar as function vms
"""
return None
def ips(self, name):
"""
returns a list of ips belonging to the named cluster
"""
return None
def names(self, name):
"""
returns a lsit of names of the vms belonging to the cluster
TODO:: group show name vm --format=json
"""
return None
def vms(self, name, refresh=True):
"""
returns in json the information about the vms of the named cluster
this includes cluster name and its detailed VMs information
"""
group_ = self.GroupManage.get_groups(groupname=name)
if self.check_group_is_a_cluster(group_):
VM_name_list = self.GroupManage.list_items_of_group(
name, _type="VM")["VM"]
if refresh:
self.mongodb.activate(cm_user_id=self.username)
self.mongodb.refresh(cm_user_id=self.username,
types=['servers'])
VM_dict = self.mongodb.servers(cm_user_id=self.username)
res = {}
res[name] = {}
for cloud, value_0 in VM_dict.iteritems():
for id_, value_1 in value_0.iteritems():
if value_1['name'] in VM_name_list:
res[name][value_1['name']] = value_1
if '_id' in res[name][value_1['name']]:
del res[name][value_1['name']]['_id']
return res
else:
raise Exception("group '{0}' is not a cluster group".format(name))
def delete(self, name, grouponly=False):
"""
deletes the named cluster
::param grouponly: in default, when deleting a cluster, the group object along
with its VMs will be deleted, if grouponly=True, then the
VMs will be preserved
"""
from cloudmesh.experiment.group_usage import remove_vm_from_group_while_deleting
vms = self.vms(name)[name]
if not grouponly:
clouds = []
for vmname, value in vms.iteritems():
cloudname = value['cm_cloud']
id_ = value['id']
banner("Deleting vm->{0} on cloud->{1}".format(
vmname, cloudname))
result = self.mongodb.vm_delete(cloudname, id_, self.username)
print(result)
if cloudname not in clouds:
clouds.append(cloudname)
remove_vm_from_group_while_deleting(self.username, vmname)
time.sleep(5)
for cloud in clouds:
self.mongodb.release_unused_public_ips(cloud, self.username)
self.mongodb.refresh(cm_user_id=self.username, types=['servers'])
banner("Deleting group->{0}".format(name))
self.GroupManage.delete_group(name)
def info(self, name):
"""returns a simplified information about the cluster in json format"""
"""each vm contains the ips and the name, as well as the status of the vm"""
return None
def create_with_existence_check(self, name, n):
"""
creates a cluster with the given name and the number of vms.
If the cluster exists an exception is thrown. Returns the json of the cluster.
"""
pass
# raise ClusterExistsError('Cluster ' + name + ' exists')
def create(self, name, n):
"""
creates a cluster with the given name and the number of vms.
NOTICE: this function doesn't check the existence of a cluster(a
cluster us identified by group name), if the cluster already exists,
this function will add number n VMs to the cluster instead of create
a new one
"""
pass
def check_group_is_a_cluster(self, groupobj):
"""
a cluster group object shoud have a 'cluster' tag in its tags
::param groupobj: group object
return True if a group contains the tag, otherwise return False
"""
if "cluster" in groupobj.tags:
return True
else:
return False
def shell_command_experiment_group(arguments):
"""
Usage:
group list [--format=FORMAT]
group create NAME
group remove NAME
group add item NAME TYPE VALUE
group remove item NAME TYPE VALUE
group show NAME [TYPE] [--format=FORMAT]
Arguments:
NAME name of the group
TYPE type of the item in the group, e.g. vm
VALUE value of item to add, e.g. vm name
Options:
-v verbose mode
--format=FORMAT output format: table, json, csv
Description:
group list lists the groups
group create creates a new group
group remove removes a group
group add item addes an item of a type to a group
group remove item removes an item of a type from a group
group show lists items of a group
Examples:
group add item sample vm samplevm
add vm named samplevm to group sample
group show sample vm --format=json
list all VMs of group sample in json format
Example:
group create experiment_1
vm start
last = vm label
group add experiment_1 vm last
group create experiment_2
vm start
last = vm info label # prints the vm label /prefix + number
ipno = vm info ip # prints the ip of the last vm
ipno = vm info ip gvonlasz_1 # get ip of vm with label gvonlasz_1
group add expermiment_2 ip ipno
groups are just tuples
i can have multiple Kinds in the tuple
mongoengine
class groupObject
def add (... name, kind, attribute ...)
def printer ( ... kind, printfunction, name...)
def getter ( .... kind, name)
def getter ( .... kind, name ...)
if kind == "vm":
vm = get vm from mongo
return vm
elif kind = "image"
iamge = get image from mongo
return iamge
....
def vmPrinter ( .... vm ...)
print vm.ip
print vm.name
....
def imagePrinter ( .... image ...)
print image.size
print image.name
....
g = groupObject()
g.printer("vm", cmPrinter)
g.printer("image", imagePrinter)
"""
# Changed the scope of this import.
from cloudmesh.experiment.group import GroupManagement
from cloudmesh.experiment.group_usage import add_item_to_group
name = arguments["NAME"]
type = arguments["TYPE"]
value = arguments["VALUE"]
config = cm_config()
username = config.username()
# print username
user = cm_user()
GroupManage = GroupManagement(username)
'''
if arguments["info"]:
print("Default experiment group:", user.get_defaults(username)["group"])
'''
if arguments["list"]:
try:
res = GroupManage.get_groups_names_list()
except Exception, err:
Console.error(str(err))
return
d = {}
d["groups"] = res
if arguments['--format']:
p_format = arguments['--format']
else:
p_format = None
shell_commands_dict_output(username,
d,
print_format=p_format,
table_format="key_list",
indexed=True)
def get_vms_look_for(username,
cloudname,
servername=None,
serverid=None,
groupname=None,
prefix=None,
hostls=None,
getAll=False,
refresh=False):
'''
work as a filter to find the VMs you are looking for. Input the seaching conditions,
and returns a list of server ids that meet the condition
you cannot provide servername and serverid at the same time
you cannot provide prefix and hostlist at the same time
param hostls:: e.g. sample[1-3,18] => ['sample1', 'sample2', 'sample3', 'sample18']
param refresh:: refresh before filtering
param getAll:: if True, the function consider all VMs are selected before filtering.
if False, then none are selected before filtering
'''
# GroupManagement automatically loads mongodb connection.
# To avoid creating db connection by import, we moved this import inside of
# the function. This way allows us to use GroupManagement class only in this
# function. Other functions in this file won't load db connection by import.
# Hyungro Lee - 12/1/2014
from cloudmesh.experiment.group import GroupManagement
# input checking
if servername and serverid:
Console.error(
"you cannot provide servername and serverid at the same time")
return False
if prefix and hostls:
Console.error(
"you cannot provide prefix and hostlist at the same time")
return False
if hostls:
try:
hostls_list = hostlist.expand_hostlist(hostls)
except:
Console.error(
"please check your hostlist input, right format e.g. sample[1-9,18]"
)
return False
# get server data
try:
mongo = cm_mongo()
except:
Console.error("There is a problem with the mongo server")
return False
if refresh:
mongo.activate(cm_user_id=username, names=[cloudname])
mongo.refresh(cm_user_id=username,
names=[cloudname],
types=['servers'])
if groupname:
vms_in_group_list = []
GroupManage = GroupManagement(username)
groups_list = GroupManage.get_groups_names_list()
if groupname not in groups_list:
return []
else:
vms_in_group_list = GroupManage.list_items_of_group(
groupname, _type="VM")["VM"]
servers_dict = mongo.servers(clouds=[cloudname],
cm_user_id=username)[cloudname]
# search for qualified vms for each critera
res_d = {}
if servername:
res_d['servername'] = []
if serverid:
res_d['serverid'] = []
if groupname:
res_d['groupname'] = []
if prefix:
res_d['prefix'] = []
if hostls:
res_d['hostls'] = []
if getAll:
res_d['getAll'] = []
for k, v in servers_dict.iteritems():
if servername and servername == v['name']:
res_d['servername'].append(k)
if serverid and serverid == k:
res_d['serverid'].append(k)
if groupname:
if v['name'] in vms_in_group_list:
res_d['groupname'].append(k)
if prefix:
nametemp = server_name_analyzer(v['name'])
if prefix and prefix == nametemp[0]:
res_d['prefix'].append(k)
if hostls and v['name'] in hostls_list:
res_d['hostls'].append(k)
if getAll and v['cm_cloud'] == cloudname:
res_d['getAll'].append(k)
# -------------------------
# intersect the results
ls = res_d.values()
l = len(ls)
if l == 0:
res = []
elif l == 1:
res = ls[0]
else:
res = ls[0]
del ls[0]
for i in ls:
res = set(res) & set(i)
res = list(res)
return res
def shell_command_experiment_group(arguments):
"""
::
Usage:
group list [--format=FORMAT]
group create NAME
group remove NAME
group add item NAME TYPE VALUE
group remove item NAME TYPE VALUE
group show NAME [TYPE] [--format=FORMAT]
Arguments:
NAME name of the group
TYPE type of the item in the group, e.g. vm
VALUE value of item to add, e.g. vm name
Options:
-v verbose mode
--format=FORMAT output format: table, json, csv
Description:
group list lists the groups
group create creates a new group
group remove removes a group
group add item addes an item of a type to a group
group remove item removes an item of a type from a group
group show lists items of a group
Examples:
group add item sample vm samplevm
add vm named samplevm to group sample
group show sample vm --format=json
list all VMs of group sample in json format
"""
"""
Example:
group create experiment_1
vm start
last = vm label
group add experiment_1 vm last
group create experiment_2
vm start
last = vm info label # prints the vm label /prefix + number
ipno = vm info ip # prints the ip of the last vm
ipno = vm info ip gvonlasz_1 # get ip of vm with label gvonlasz_1
group add expermiment_2 ip ipno
groups are just tuples
i can have multiple Kinds in the tuple
mongoengine
class groupObject
def add (... name, kind, attribute ...)
def printer ( ... kind, printfunction, name...)
def getter ( .... kind, name)
def getter ( .... kind, name ...)
if kind == "vm":
vm = get vm from mongo
return vm
elif kind = "image"
iamge = get image from mongo
return iamge
....
def vmPrinter ( .... vm ...)
print vm.ip
print vm.name
....
def imagePrinter ( .... image ...)
print image.size
print image.name
....
g = groupObject()
g.printer("vm", cmPrinter)
g.printer("image", imagePrinter)
"""
# Changed the scope of this import.
from cloudmesh.experiment.group import GroupManagement
from cloudmesh.experiment.group_usage import add_item_to_group
name = arguments["NAME"]
type_ = arguments["TYPE"]
value = arguments["VALUE"]
config = cm_config()
username = config.username()
# print username
user = cm_user()
GroupManage = GroupManagement(username)
'''
if arguments["info"]:
print("Default experiment group:", user.get_defaults(username)["group"])
'''
if arguments["list"]:
try:
res = GroupManage.get_groups_names_list()
except Exception, err:
Console.error(str(err))
return
d = {}
d["groups"] = res
if arguments['--format']:
p_format = arguments['--format']
else:
p_format = None
shell_commands_dict_output(username,
d,
print_format=p_format,
table_format="key_list",
indexed=True)
Console.error("<LoginName> cannot be empty")
return
else:
vm_login_name = arguments['--ln']
# Moved the import inside of this function
# If this import goes up to the top, monodb connection will be
# estabilished. Due to that reason, this import stays here
# Hyungro Lee 12/01/2014
#
# we have modified how the mongonenigne connects and
# it's safe to import any class definition now at the beginning of file
# Fugang 02/06/2015
#
# from cloudmesh.experiment.group import GroupManagement
#
GroupManage = GroupManagement(username)
groups_list = GroupManage.get_groups_names_list()
vms_in_group_list = {}
if GroupName in groups_list:
vms_in_group_list = GroupManage.list_items_of_group(
GroupName, _type="VM")["VM"]
if not arguments['--force'] and len(vms_in_group_list) != 0:
if yn_choice("The group you provide exists and it has VMs in it, " + \
"do you want to proceed? (if you choose yes, these exist " +\
"VMs will be included in the cluster, this could also " +\
"rewrite the key on the exist VMs)",
default='n',
tries=3):
pass
else:
def do_cluster(self, args, arguments):
"""
Usage:
cluster start CLUSTER_NAME
cluster list
cluster login CLUSTER_NAME
cluster stop CLUSTER_NAME
cluster create --count=<count>
--group=<group>
[--ln=<LoginName>]
[--cloud=<CloudName>]
[--image=<imgName>|--imageid=<imgId>]
[--flavor=<flavorName>|--flavorid=<flavorId>]
[--force]
Description:
Cluster Management
cluster create --count=<count> --group=<group> --ln=<LoginName> [options...]
<count> specify amount of VMs in the cluster
<group> specify a group name of the cluster, make sure it's unique
Start a cluster of VMs, and each of them can log into all others.
CAUTION: you sould do some default setting before using this command:
1. select cloud to work on, e.g. cloud select india
2. activate the cloud, e.g. cloud on india
3. set the default key to start VMs, e.g. key default [NAME]
4. set the start name of VMs, which is prefix and index, e.g. label --prefix=test --id=1
5. set image of VMs, e.g. default image
6. set flavor of VMs, e.g. default flavor
Also, please make sure the group name of the cluster is unique
Options:
--ln=<LoginName> give a login name for the VMs, e.g. ubuntu
--cloud=<CloudName> give a cloud to work on
--flavor=<flavorName> give the name of the flavor
--flavorid=<flavorId> give the id of the flavor
--image=<imgName> give the name of the image
--imageid=<imgId> give the id of the image
--force if a group exists and there are VMs in it, the program will
ask user to proceed or not, use this flag to respond yes as
default(if there are VMs in the group before creating this
cluster, the program will include the exist VMs into the cluster)
"""
#pprint(arguments)
self.cm_config = cm_config()
self.cm_mongo = cm_mongo()
self.user = cm_user()
# -----------------------------
# TODO::
# add VMs to cluster
# -----------------------------
if arguments['start'] and arguments['CLUSTER_NAME']:
'''Starts a cluster'''
# Initialize default variables. e.g. userid, default cloud and
# default keypair
userid = self.cm_config.username()
def_cloud = self.get_cloud_name(userid)
self.cm_mongo.activate(userid)
userinfo = self.user.info(userid)
if "key" in userinfo["defaults"]:
key = userinfo["defaults"]["key"]
elif len(userinfo["keys"]["keylist"].keys()) > 0:
key = userinfo["keys"]["keylist"].keys()[0]
if key:
keycontent = userinfo["keys"]["keylist"][key]
if keycontent.startswith('key '):
keycontent = keycontent[4:]
cm_keys_mongo(userid).check_register_key(userid, def_cloud, key, keycontent)
keynamenew = _keyname_sanitation(userid, key)
else:
Console.warning("No sshkey found. Please Upload one")
return
clustername = arguments['CLUSTER_NAME']
s_name = "cluster-{0}-{1}-{2}".format(userid, clustername, get_rand_string())
# TEMP FOR HADOOP CLUSTER
if clustername != "hadoop":
Console.warning('hadoop is only available cluster')
return
# 1. keypair for the communication between master and worker nodes
privatekey, publickey = generate_keypair()
t_url = \
"https://raw.githubusercontent.com/cloudmesh/cloudmesh/dev1.3/heat-templates/ubuntu-14.04/hadoop-cluster/hadoop-cluster.yaml"
param = {'KeyName': keynamenew,
'PublicKeyString': publickey,
'PrivateKeyString': privatekey}
log.debug(def_cloud, userid, s_name, t_url, param, publickey,
privatekey)
res = self.cm_mongo.stack_create(cloud=def_cloud, cm_user_id=userid,
servername=s_name,
template_url=t_url,
parameters=param)
log.debug(res)
if 'error' in res:
print (res['error']['message'])
return res
elif arguments['list']:
userid = self.cm_config.username()
self.cm_mongo.activate(userid)
self.cm_mongo.refresh(cm_user_id=userid, types=[self._id])
stacks = self.cm_mongo.stacks(cm_user_id=userid)
launchers = self.filter_launcher(
stacks,
{"search": "contain",
"key": "stack_name",
"value": "launcher"}
)
log.debug(launchers)
d = {}
for k0, v0 in launchers.iteritems():
for k1, v1 in launchers[k0].iteritems():
d[v1['id']] = v1
columns = ['stack_name', 'description', 'stack_status',
'creation_time', 'cm_cloud']
if arguments['--column'] and arguments['--column'] != "all":
columns = [x.strip() for x in arguments['--column'].split(',')]
if arguments['--format']:
if arguments['--format'] not in ['table', 'json', 'csv']:
Console.error("please select printing format among table, json and csv")
return
else:
p_format = arguments['--format']
else:
p_format = None
shell_commands_dict_output(d,
print_format=p_format,
firstheader="launcher_id",
header=columns
# vertical_table=True
)
elif arguments['login']:
Console.error("Not implemented")
return
elif arguments['stop'] and arguments['CLUSTER_NAME']:
userid = self.cm_config.username()
def_cloud = self.get_cloud_name(userid)
c_id = arguments['CLUSTER_NAME']
self.cm_mongo.activate(userid)
res = self.cm_mongo.stack_delete(cloud=def_cloud,
cm_user_id=userid,
server=c_id)
log.debug(res)
return res
elif arguments['create']:
try:
config = cm_config()
except:
Console.error("Failed to load the cloudmesh yaml file")
return
username = config['cloudmesh']['profile']['username']
cloudname = arguments['--cloud'] or CloudManage().get_selected_cloud(username)
temp_dir_name = ".temp_cluster_create_" + username + "_0"
while os.path.isdir("./{0}".format(temp_dir_name)):
temp_dir_name = temp_dir_name[:-1] + str(int(temp_dir_name[-1]) + 1)
dir_name = temp_dir_name
#NumOfVM = None
GroupName = None
vm_login_name = "ubuntu"
temp_key_name = "sshkey_temp"
_key = "-i ./{0}/{1}".format(dir_name, temp_key_name)
StrictHostKeyChecking = "-o StrictHostKeyChecking=no"
res = None
to_print = []
'''
try:
NumOfVM = abs(int(argument['--count']))
except:
Console.error("<count> must be an integer")
return
'''
if arguments['--group'] == '':
Console.error("<group> cannot be empty")
return
else:
GroupName = arguments['--group']
if arguments['--ln']:
if arguments['--ln'] == '':
Console.error("<LoginName> cannot be empty")
return
else:
vm_login_name = arguments['--ln']
if not arguments['--force']:
# Moved the import inside of this function
# If this import goes up to the top, monodb connection will be
# estabilished. Due to that reason, this import stays here
# Hyungro Lee 12/01/2014
from cloudmesh.experiment.group import GroupManagement
GroupManage = GroupManagement(username)
groups_list = GroupManage.get_groups_names_list()
if GroupName in groups_list:
vms_in_group_list = GroupManage.list_items_of_group(GroupName, _type="VM")["VM"]
if len(vms_in_group_list) != 0:
if yn_choice("The group you provide exists and it has VMs in it, " + \
"do you want to proceed? (if you choose yes, these exist " +\
"VMs will be included in the cluster, this could also " +\
"rewrite the key on the exist VMs)",
default='n',
tries=3):
pass
else:
return
# start VMs
print ("starting VMs...")
arguments_temp = arguments
arguments_temp['start'] = True
arguments_temp['--name'] = None
vmclass = VMcommand(arguments_temp)
res = vmclass.call_procedure()
if res == False: return
def string_to_dict(s):
h = s.find("{")
t = s.rfind("}")
return json.loads(s[h:t+1])
def check_public_ip_existence(d):
temp = d['addresses']['private']
for item in temp:
if item["OS-EXT-IPS:type"] == "floating":
return True
return False
def get_ip(d, kind="floating"): # kind is either floating or fixed
temp = d['addresses']['private']
for item in temp:
if item["OS-EXT-IPS:type"] == kind:
return item['addr']#.encode('ascii')
return "FAIL: doesn't exist"
# check all VMs are active
command_refresh = "vm list --refresh --group={0} --format=json".format(GroupName)
def _help0(d):
for k, v in d.iteritems():
if v['status'] != 'ACTIVE':
return False
return True
proceed = False
repeat_index = 1
while proceed != True:
if repeat_index > 10:
Console.warning("Please check the network")
return
print ("checking({0})...".format(repeat_index))
time.sleep(5)
res = str(cm(command_refresh))
res = string_to_dict(res)
if _help0(res):
proceed = True
else:
repeat_index = repeat_index + 1
continue
# assign ip to all VMs
print ("assigning public ips...")
for k, v in res.iteritems():
if not check_public_ip_existence(v):
cm("vm ip assign --id={0}".format(k.encode('ascii')))
def _help(d):
for k, v in d.iteritems():
if check_public_ip_existence(v) != True:
return False
return True
# make sure all VMs have been assigned a public ip
proceed = False
repeat_index = 1
while proceed != True:
if repeat_index > 10:
Console.warning("Please check the network")
return
print ("checking({0})...".format(repeat_index))
time.sleep(5)
res = str(cm(command_refresh))
res = string_to_dict(res)
if _help(res):
proceed = True
else:
repeat_index = repeat_index + 1
continue
# -------------------------
# key handler
userinfo = cm_user().info(username)
key = None
if "key" in userinfo["defaults"]:
key = userinfo["defaults"]["key"]
elif len(userinfo["keys"]["keylist"].keys()) > 0:
key = userinfo["keys"]["keylist"].keys()[0]
Console.warning("default key is not set, trying to use a key in the database...")
if key:
keycontent = userinfo["keys"]["keylist"][key]
if keycontent.startswith('key '):
keycontent = keycontent[4:]
cm_keys_mongo(username).check_register_key(username, cloudname, key, keycontent)
else:
Console.error("No sshkey found. Please Upload one")
return
# -------------------------
# generate ssh keys for VMs and prepare two files: authorized_keys and hosts
print ("generating ssh keys...")
os.popen("mkdir {0}".format(dir_name))
fa = open("./{0}/authorized_keys_temp".format(dir_name), "w")
fh = open("./{0}/hosts_temp".format(dir_name), "w")
fk = open("./{0}/{1}".format(dir_name, temp_key_name), "w")
fk.write(keycontent)
fk.close()
os.popen("chmod 644 ./{0}/{1}".format(dir_name, temp_key_name))
for k, v in res.iteritems():
address_floating = get_ip(v)
address_fixed = get_ip(v, kind="fixed")
vm_name = v['name']#.encode('ascii')
to_print.append("{0} {1}, {2}".format(vm_name, address_floating, address_fixed))
fh.write(address_floating + " " + vm_name + "\n"
+ address_fixed + " " + vm_name + "-i\n")
os.popen("ssh {2} {3} {0}@{1} \"ssh-keygen -t rsa -N '' -f ~/.ssh/id_rsa\""\
.format(vm_login_name,address_floating, _key, StrictHostKeyChecking))
temp = os.popen("ssh {2} {3} {0}@{1} \"cat ~/.ssh/id_rsa.pub\""\
.format(vm_login_name, address_floating, _key, StrictHostKeyChecking)).read()
fa.write(temp)
fa.close()
fh.close()
# copy the files to VMs
print ("copying the files...")
os.popen("mkdir ./{0}/oops".format(dir_name))
for k, v in res.iteritems():
address_floating = get_ip(v)
os.popen("scp {2} {3} {0}@{1}:~/.ssh/authorized_keys ./{4}/"\
.format(vm_login_name,address_floating, _key, StrictHostKeyChecking, dir_name))
os.popen("cat ./{0}/authorized_keys_temp >> ./{0}/authorized_keys"\
.format(dir_name))
os.popen("scp {2} {3} ./{4}/authorized_keys {0}@{1}:~"\
.format(vm_login_name,address_floating, _key, StrictHostKeyChecking, dir_name))
os.popen("ssh {2} {3} {0}@{1} \"sudo mv authorized_keys ~/.ssh/\""\
.format(vm_login_name,address_floating, _key, StrictHostKeyChecking))
os.popen("rm ./{0}/authorized_keys".format(dir_name))
os.popen("cp ./{0}/hosts_temp ./{0}/oops/".format(dir_name))
os.popen("mv ./{0}/oops/hosts_temp ./{0}/oops/hosts".format(dir_name))
fh0 = open("./{0}/oops/hosts".format(dir_name), "a")
os.popen("scp {2} {3} {0}@{1}:/etc/hosts ./{4}/"\
.format(vm_login_name,address_floating, _key, StrictHostKeyChecking, dir_name))
with open("./{0}/hosts".format(dir_name)) as f0:
content = f0.readlines()
for item in content:
fh0.write(item + "\n")
fh0.close()
os.popen("scp {2} {3} ./{4}/oops/hosts {0}@{1}:~"\
.format(vm_login_name,address_floating, _key, StrictHostKeyChecking, dir_name))
os.popen("ssh {2} {3} {0}@{1} \"sudo mv hosts /etc/\""\
.format(vm_login_name,address_floating, _key, StrictHostKeyChecking))
os.popen("rm ./{0}/oops/hosts".format(dir_name))
print ("finishing...")
os.popen("rm -rf {0}".format(dir_name))
print ("DONE.")
print ("cluster group: ", GroupName)
for item in to_print:
print (item)
print ("(host name for private ips will have -i at the end of VM name, e.g. testVM -> testVM-i)")