def setup_inventory(self):
banner("Read Dicts")
self.sample_user = ConfigDict(filename=config_file("/me.yaml"))
self.portalname = self.sample_user.get("portalname")
print("PORTALNAME", self.portalname)
print("SAMPLE USER", self.sample_user)
banner("create user from template, duplicates cm init generate me")
t = cm_template(config_file("/etc/cloudmesh.yaml"))
pprint(set(t.variables()))
self.config = t.replace(kind="dict", values=self.sample_user)
print(type(self.config))
print(self.config)
#
# BUG?
#
self.idp = cm_userLDAP()
self.idp.connect("fg-ldap", "ldap")
self.idp.refresh()
ldap_info = self.idp.get(self.portalname)
print(ldap_info)
print(type(self.config))
self.config['cloudmesh']['projects'] = ldap_info['projects']
self.config['cloudmesh']['keys'] = ldap_info['keys']
try:
self.config['cloudmesh']['projects']['deafult'] = ldap_info[
'projects']['active'][0]
except:
print("ERROR: you have no projects")
def createVM(N):
username = cloudmesh.load().username()
print username
mesh = cloudmesh.mesh("mongo")
mesh.activate(username)
cloudmesh.shell("cloud on india")
flavor = mesh.flavor('india', 'm1.large')
image=mesh.image('india','futuregrid/ubuntu-14.04')
#vm_ip=[]
for i in range(0,N):
result = mesh.start(cloud='india',
cm_user_id=username,
flavor=flavor,
image=image)
server = result['server']['id']
ip=mesh.assign_public_ip('india', server, username)
print ip
#vm_ip.append(ip)
try:
result = mesh.wait(ipaddr=ip, command="ls -al", interval=10, retry=5)
cloudmesh.banner("INSTALLING MONGO TO THE VM "+str(ip))
result=mesh.ssh_execute(username='******',ipaddr=ip,command='ls -al',pkey=None)
print result
script ="scp filescript.sh ubuntu@"+ip+":/home/ubuntu/"
print script
os.system(script)
result=mesh.ssh_execute(username='******',ipaddr=ip,command='bash -s < ./filescript.sh',pkey=None)
print result
result=mesh.ssh_execute(username='******',ipaddr=ip,command='ls -al',pkey=None)
print result
except:
print "Authentication failed when connecting to %s" % ip
sys.exit(1)
print "All VMs Mongo Installed.. "
def test_gregor(self):
banner("ME")
id = ConfigDict(filename=config_file("/me.yaml")).get("portalname")
user = cm_user()
result = user.info(id)
pprint(result)
pass
def Parallel(execution_array, f, **kwargs):
task = {}
for element in execution_array:
print "submitting -> {0}".format(element)
task[element] = f.apply_async(args=(element,),
kwargs=kwargs,
expires=10)
banner("tasks", c=".")
pprint(task)
result = {}
for element in execution_array:
print "getting -> {0}".format(element), str(task[element])
result[element] = task[element].get(propagate=False, no_ack=False)
banner("info")
print "INFO", task[element].info
banner("result")
print "RESULT", task[element].result
banner("backend")
print "BACKEND", task[element].backend
# print "OOOO", result[element]
return result
def shell_command_user(arguments):
"""
Usage:
user list
user info [ID]
user id
Administrative command to lists the users from LDAP
Arguments:
list list the users
ID list the user with the given ID
Options:
-v verbose mode
"""
user = cm_user()
if (arguments["info"]):
id = arguments["ID"]
if id is None:
config = cm_config()
id = config.username()
banner("User Information in Mongo for user: {0}".format(id))
user = cm_user()
result = user.info(id)
pprint(result)
elif (arguments["list"]):
user = cm_user()
list_of_users = user.list_users()
pprint(list_of_users)
print()
print("=========================")
num = len(list_of_users)
print(str(num) + " users listed")
elif arguments['id']:
config = cm_config()
print(config.username())
else:
print("WRONG PARAMETERS")
return
def test_mongo_credential(self):
banner("USER")
pprint(self.user.info("gvonlasz"))
username = "******"
cloudname = "dummy"
password = "******"
tennant = "fg1"
name = username
self.user.set_credential(
username, cloudname, {
"OS_USERNAME": name,
"OS_PASSWORD": password,
"OS_TENANT_NAME": tennant,
"CM_CLOUD_TYPE": "openstack"
})
cred = self.user.get_credential(username, cloudname)
banner("credential")
print(cred)
banner("credentials")
pprint(self.user.get_credentials(username))
banner("find")
result = self.user.userdb_passwd.find({})
for r in result:
pprint(r)
def shell_command_user(arguments):
"""
Usage:
user list
user info [ID]
user id
Administrative command to lists the users from LDAP
Arguments:
list list the users
ID list the user with the given ID
Options:
-v verbose mode
"""
user = cm_user()
if (arguments["info"]):
id = arguments["ID"]
if id is None:
config = cm_config()
id = config.username()
banner("User Information in Mongo for user: {0}".format(id))
user = cm_user()
result = user.info(id)
pprint(result)
elif (arguments["list"]):
user = cm_user()
list_of_users = user.list_users()
pprint(list_of_users)
print()
print("=========================")
num = len(list_of_users)
print(str(num) + " users listed")
elif arguments['id']:
config = cm_config()
print(config.username())
else:
print("WRONG PARAMETERS")
return
def refresh_qstat(self, host):
'''
obtains a refreshed qstat data set from the specified host. The result is written into the mongo db.
:param host: The host on which to execute qstat
'''
time_now = datetime.now()
data = dict(self.hosts[host].qstat(refresh=True))
self.db_qstat.remove({"cm_host": host, "cm_kind": "qstat"}, safe=True)
for name in data:
banner(name)
for job in data[name]:
entry = data[name][job]
id = "{0}-{1}-qstat".format(host, name).replace(".", "-")
entry["cm_host"] = name
entry["cm_kind"] = "qstat"
entry["cm_id"] = id
entry["cm_qstat"] = host
entry["cm_refresh"] = time_now
self.db_qstat.insert(data[name][job])
def refresh_qstat(self, host):
'''
obtains a refreshed qstat data set from the specified host. The result is written into the mongo db.
:param host: The host on which to execute qstat
'''
time_now = datetime.now()
data = dict(self.hosts[host].qstat(refresh=True))
self.db_qstat.remove({"cm_host": host, "cm_kind": "qstat"}, safe=True)
for name in data:
banner(name)
for job in data[name]:
entry = data[name][job]
id = "{0}-{1}-qstat".format(host, name).replace(".", "-")
entry["cm_host"] = name
entry["cm_kind"] = "qstat"
entry["cm_id"] = id
entry["cm_qstat"] = host
entry["cm_refresh"] = time_now
self.db_qstat.insert(data[name][job])
def Parallel(credential, f, delta=4, **kwargs):
task = {}
result = {}
update = []
for host in credential:
entry = Cache.get(host, f)
time_stamp = entry["date_modified"]
#time_valid = time_stamp + timedelta(seconds=delta)
#exired = datetime.now() > time_valid
if (entry is None) or (time_expired(time_stamp, delta)):
user = credential[host]
id = Cache.get_id(host, f)
print "submitting -> {0}@{1}".format(user, host)
update.append(host)
task[host] = f.apply_async(args=(host, user),
kwargs=kwargs,
expires=10,
task_id=id)
else:
result[host] = entry["value"]
banner("tasks", c=".")
pprint(task)
print update
for host in update:
print "getting -> {0}".format(host), str(task[host])
result[host] = task[host].get(propagate=False)
banner("info")
print "INFO", task[host].info
banner("result")
print "RESULT", task[host].result
banner("backend")
print "BACKEND", task[host].backend
Cache.update(host, f, result[host])
return result
def Parallel(credential, f, delta=4, **kwargs):
task = {}
result = {}
update = []
for host in credential:
entry = Cache.get(host, f)
time_stamp = entry["date_modified"]
#time_valid = time_stamp + timedelta(seconds=delta)
#exired = datetime.now() > time_valid
if (entry is None) or (time_expired(time_stamp, delta)):
user = credential[host]
id = Cache.get_id(host, f)
print "submitting -> {0}@{1}".format(user, host)
update.append(host)
task[host] = f.apply_async(args=(host, user),
kwargs=kwargs,
expires=10, task_id=id)
else:
result[host] = entry["value"]
banner("tasks", c=".")
pprint(task)
print update
for host in update:
print "getting -> {0}".format(host), str(task[host])
result[host] = task[host].get(propagate=False)
banner("info")
print "INFO", task[host].info
banner("result")
print "RESULT", task[host].result
banner("backend")
print "BACKEND", task[host].backend
Cache.update(host, f, result[host])
return result
def cm_image_command(arguments):
"""
cm-image admin on HOSTS
cm-image admin off HOSTS
"""
path = path_expand(definitions[0])
if arguments["version"]:
print(cloudmesh.__version__)
elif arguments["info"]:
banner("info")
banner("System", c='-')
print("Kind: ", arguments['--kind'])
print("Path: ", path)
print("Version:", cloudmesh.__version__)
banner("List of templates", c='-')
system_name = None
for definition in definitions:
try:
path = path_expand(definition)
if os.path.exists(path):
os.system("cd '%s' ; veewee vbox list" % path)
else:
print("WARNING: path", path, "does not exist")
except KeyError, key:
print('WARNING: no environment variable called', key, 'found')
print()
print("To build one, please use one of the")
print()
print(" cm-image build OS")
print()
print("Next you need to register the image")
print()
print(" cm-image register OS")
print()
print("where OS is one of the labels listed above.")
print()
print()
print("To build one, please use one of the")
print()
print(" cm-image build OS")
print()
print("Next you need to register the image")
print()
print(" cm-image register OS")
print()
print("where OS is one of the labels listed above.")
print()
elif arguments["build"]:
banner("build")
system_name = arguments["OS"]
if arguments['--gui']:
gui = ""
else:
gui = '--nogui'
if arguments['--kind'] == "vbox":
os.system("cd '%s' ; veewee vbox build '%s' --force %s" %
(path, system_name, gui))
# due to some bug the following does not work
# os.system("veewee vbox build %s --workdir='%s' --force" % (path,
# system_name)
else:
def test_qinfo(self):
for host in self.hosts:
banner(host)
self.get_qinfo(host)
def cm_manage():
"""Usage:
cm-manage config projects list
cm-manage config projects
cm-manage config [-f FILE] [-o OUT] [-p PROJECT] cloud NAME [-]
cm-manage config dump [--format=(yaml|dict)]
cm-manage config init [-o OUT] [-u USER]
cm-manage config list
cm-manage config password NAME
cm-manage config show passwords
cm-manage config fetch [-u USER] [-r HOST]
cm-manage --version
cm-manage --help
Arguments:
NAME name of the cloud
Options:
-h --help show this help message and exit
-v --version show version and exit
-f NAME --file=NAME the Name of the cloud to be specified,
if ? a selection is presented
-o OUT --out=OUT writes the result in the specifide file
-p PROJECT --project=PROJECT selects a project (e.g. for eucalyptus
which has project-specific environments)
-u USER --user=USER the user (login) name
-r HOST --remote=HOST the host machine on which the yaml file is
located in the CONFIG directory
[default: india.futuregrid.org]
-d --debug debug
- this option is a - at the end of the command.
If data is written to a file it is also put out to stdout
Description:
Command to generate rc files from our cloudmesh configuration files.
This program generates form a YAML file containing the login
information for a cloud an rc file that can be used to later source
it.
Example:
we assume the yaml file has an entry india-openstack::
cm-manage config -o novarc india-openstack
source novarc
This will create a novarc file and than you can source it::
cm-manage config ? -
Presents a selction of cloud choices and writes the choice into a
file called CONFIG/novarc
"""
default_path = config_file_prefix + '/novarc'
arguments = docopt(cm_manage.__doc__)
DEBUG("arguments", arguments)
home = os.environ['HOME']
# DEBUG("home", home)
#
# This secion deals with handeling "cm config" related commands
######################################################################
is_config = arguments['config'] != None
if is_config:
# DEBUG('Arguments', arguments)
file = arguments['--file']
try:
config = cm_config(file)
# DEBUG("config", config)
except IOError:
print("{0}: Configuration file '{1}' not found".format(
"CM ERROR", file))
sys.exit(1)
except (yaml.scanner.ScannerError,
yaml.parser.ParserError) as yamlerror:
print("{0}: YAML error: {1}, in configuration file '{2}'".format(
"CM ERROR", yamlerror, file))
sys.exit(1)
except:
print("Unexpected error:", sys.exc_info()[0])
sys.exit(1)
name = arguments['NAME']
#
# NOT TESTED
#
if arguments['fetch'] or name == 'fetch':
DEBUG('Arguments', arguments)
# get user
var = {}
var['user'] = arguments['--user']
var['host'] = arguments['--remote']
#
# BUG should be
#
var['file'] = config_file_prefix() + "/cloudmesh.yaml"
if var['user'] is None:
var['user'] = getpass.getuser()
from_location = "%(user)s@%(host)s:%(file)s" % var
to_location = config_file("/%(file)s" % var)
if os.path.isfile(to_location):
print("WARNING: The file %s exists" % to_location)
if not yn_choice("Would you like to replace the file",
default='y'):
sys.exit(1)
print(from_location)
print(to_location)
print("Copy cloudmesh file from %s to %s" %
(from_location, to_location))
result = scp(from_location, to_location)
print(result)
sys.exit(0)
#
# ok
#
# if (arguments['projects'] and arguments['list']) :
if arguments['projects'] and arguments['list']:
projects = config.get('cloudmesh.projects')
print(yaml.dump(projects, default_flow_style=False, indent=4))
sys.exit(0)
#
# OK, needs setting
#
if arguments['projects']:
projects = config.projects('active')
print("Please select from the following:")
print("0 - Cancel")
selected = False
choices = []
while not selected:
counter = 1
for name in projects:
print(counter, "-" "%20s" % name)
choices.append(name)
counter += 1
print("Please select:")
input = int(sys.stdin.readline())
if input == 0:
print("Selection terminated")
sys.exit(0)
selected = (input > 0) and (input < counter)
print("Selected: ", input)
name = choices[input - 1]
print(name)
print("ERROR: THIS JUST SELECTS A PROJECT ID BUT DOES NOT SET IT")
sys.exit(0)
if arguments['init'] or name == 'init':
output = arguments['--out']
username = arguments['--user'] or os.getenv('USER')
location = path_expand(output)
new_yaml_file = open(location, 'w+')
user_yaml = cm_user().generate_yaml(username, 'cloudmesh')
print(yaml.dump(user_yaml, default_flow_style=False),
file=new_yaml_file)
new_yaml_file.close()
print("Written new yaml file in " + location)
sys.exit(0)
#
# OK
#
if arguments['list'] or name == 'list':
for name in config.cloudnames():
if 'cm_type' in config.cloud(name):
print(name, "(%s)" % config.cloud(name)['cm_type'])
sys.exit(0)
#
# NOT TESTED
#
if arguments['password']:
oldpass = getpass("Current password: "******"New password: "******"New password (again): ")
# TODO: some kind of password strength checking?
if newpass1 == newpass2:
config.change_own_password(name, oldpass, newpass1)
else:
print("New passwords did not match; password not changed.")
sys.exit(0)
#
# OK, but does not display the username
#
if arguments['show'] or name == 'show' and arguments['passwords']:
warning = "Your passwords will appear on the screen. Continue?"
if yn_choice(warning, 'n'):
me = ConfigDict(filename=config_file("/me.yaml"))
banner("PASSWORDS")
for name in me['password']:
print("{0}: {1}".format(name, me['password'][name]))
sys.exit(0)
#
# OK
#
if arguments['dump'] or name == 'dump':
format = arguments['--format']
if format == 'yaml':
d = dict(config)
print(yaml.dump(d, default_flow_style=False))
elif format == 'dict' or format is None:
print(config)
sys.exit(0)
#
# NOT TESTED
#
if name in ['?', 'x']:
if file is None:
arguments['--out'] = "%s/%s" % (home, default_path)
print("Please select from the following:")
print("0 - Cancel")
selected = False
choices = []
while not selected:
counter = 1
for name in config.cloudnames():
if 'cm_type' in config.cloud(name):
print("{0} - {1:<30} ({2})".format(
counter, name,
config.cloud(name)['cm_type']))
choices.append(name)
counter += 1
print("Please select:")
input = int(sys.stdin.readline())
if input == 0:
print("Selection terminated")
sys.exit(0)
selected = (input > 0) and (input < counter)
print("Selected: ", input)
name = choices[input - 1]
output = arguments['--out']
#
# OK
#
if name is not None:
cloud = config.cloud(name)
if not cloud:
print("%s: The cloud '%s' is not defined." %
("CM ERROR", name))
print("Try instead:")
for keyname in config.cloudnames():
print(" ", keyname)
sys.exit(1)
if cloud['cm_type'] == 'eucalyptus':
if arguments['--project']:
project = arguments['--project']
if not project in cloud:
print("No such project %s defined in cloud %s." %
(project, name))
sys.exit(1)
else:
project = config.cloud_default(
name, 'project') or config.projects('default')
if not project in cloud:
print("Default project %s not defined in cloud %s." %
(project, name))
sys.exit(1)
rc_func = lambda name: config.rc_euca(name, project)
else:
rc_func = config.rc
result = rc_func(name)
#
# OK
#
if arguments["-"]:
print(result)
else:
if output is None:
arguments['--out'] = "%s/%s" % (home, default_path)
output = arguments['--out']
out = False
if yn_choice("Would you like to review the information",
default="y"):
banner("WARNING: FIle will be written to " + output)
print(result)
print(banner(""))
try:
# First we try to open the file assuming it doesn't exist
out = os.open(output, os.O_CREAT | os.O_EXCL | os.O_WRONLY,
stat.S_IRUSR | stat.S_IWUSR)
except OSError as oserr:
# If file exists, offer to replace it
if oserr.errno == 17:
delete = raw_input(
"'%s' exists; Overwrite it [N|y]? " % output)
if delete.strip().lower() == 'y':
out = os.open(output, os.O_TRUNC | os.O_WRONLY)
if out:
os.write(out, result)
os.close(out)
def test_all(self):
HEADING()
for host in self.hosts:
banner(host)
self.get_qstat(host)
import cloudmesh
from pprint import pprint
cloudmesh.logger(False)
username = cloudmesh.load().username()
cloudmesh.banner("INIT MONGO")
mesh = cloudmesh.mesh("mongo")
#
# authentication as a user - username is requried
# On webgui side, this is achieved by the framework,
# and the username is obtined from g.user.id
#
# On CLI side, a global user object or username variable
# should be maintained upon the start of the shell
# The username could be obtained from yaml file.
cloudmesh.banner("ACTIVATE")
mesh.activate(username)
cloudmesh.banner("GET FLAVOR")
mesh.refresh(username, types=['flavors'], names=["india"])
data = mesh.flavors(cm_user_id=username, clouds=["india"])
pprint(data)
cloudmesh.banner("GET IMAGE")
mesh.refresh(username, types=['images'], names=["india"])
data = mesh.images(cm_user_id=username, clouds=["india"])
import cloudmesh as cm
cm.banner('HW3: ex1')
print cm.version()
'''
class CredentialFromMongo(UserBaseClass):
def __init__(self, user, cloud, datasource=None):
"""data source is a collectionname in cloudmesh_server.yaml"""
"""if day=tasource is none than use the default on which is ?"""
raise NotImplementedError()
'''
if __name__ == "__main__":
# -------------------------------------------------------------------------
banner("YAML read test")
# -------------------------------------------------------------------------
user = UserFromYaml("gvonlasz")
pprint(user)
store = UserDictStore(UserFromYaml, password="******")
store.set("gvonlasz", "~/.cloudmesh/cloudmesh.yaml", password="******")
banner("credentialstore")
pprint(store)
# banner("gvonlasz - hp")
# -------------------------------------------------------------------------
# cred = store.credential("gvonlasz", "hp", password="******")
# pprint (cred)
def shell_commands_dict_output(username,
d,
# choose format manually
print_format=None,
# choose table format if needed
table_format=None,
# more specific table arguments
# --------------------------
# for table format: "key_list"
indexed=False,
# --------------------------
firstheader=None,
header=None,
oneitem=False,
vertical_table=False,
title=None,
count=False):
'''
some shell commands output a dict or table, this function should be
called when commands are printing in such way.
if none of parameters json and table is given, the funtion will try
to find the default printing form from db_defaults
if db_defaults has no item 'shell_print_format', the function will
set 'shell_print_format' = table
param username:: user id
param d:: data to print
param print_format:: print format: table, json, csv
param table_format:: choose table format if needed, DESCRIPTIONS:
type1: key_list:
accept a dict in the form:
{key1: [list1],
key2: [list2],
.......
=>
| key1 | key2 |
| l
| i
| s
| t
acceptable args: indexed: provide index
param firstheader:: designed for table, provide a attribute name for the
first item of each row, since the dict doesn't provide
it
param header:: designed for table(and used to filter input dict), a list of lists,
provides column order, e.g.
[[a,b], [c, d], ...
where 'a' is the printing column name and 'b' is the attribute name
in the dict
if you don't want to change the header name but want to keep the
header order or filter the items to list, for each item in the list
you may provide a string or a list with one item instead of a list of
two items
param oneitem:: designed for table, normally the input dict should be in such
form:
{a: {...}, b:{...}}, where each subitem is a row
if there is only one item, input dict is the {...} of the subitem
above
param title: provide a title for the table
param count: provide count info at the end of the table
'''
format_type = None
if print_format:
format_type = print_format
else:
format_type = get_default_print_format(username)
if format_type not in ALLOWED_PRINT_FORMAT:
Console.error("wrong print format: {0}. (allowed print format: {1})".format(format_type,
", ".join(ALLOWED_PRINT_FORMAT)))
return False
headers = None
order = None
if header:
headers = []
order = []
for i in header:
if isinstance(i, basestring):
headers.append(i)
order.append(i)
elif isinstance(i, list):
if len(i) == 1:
headers.append(i[0])
order.append(i[0])
else:
headers.append(i[0])
order.append(i[1])
else:
print("ERROR: header info is not correct")
return False
# --------------------------------------------------------------------------
# filter the input dict
# --------------------------------------------------------------------------
if order:
new_d = {}
if oneitem:
for item in order:
if item in d:
new_d[item] = d[item]
else:
for i in d:
new_d[i] = {}
for item in order:
if item in d[i]:
new_d[i][item] = d[i][item]
d = new_d
# --------------------------------------------------------------------------
if format_type == "json":
if title:
banner(title)
print(json.dumps(d, indent=4))
elif format_type == "csv":
with open(".temp.csv", "wb") as f:
w = csv.DictWriter(f, d.keys())
w.writeheader()
w.writerow(d)
elif format_type == "table":
if table_format == "key_list":
print (dict_key_list_table_printer(d, indexed=indexed))
return
if title:
print("+" + "-" * (len(title) - 2) + "+")
print(title)
print_data = []
if oneitem:
print_data = [d]
else:
for k in sorted(d):
d[k][' '] = k
print_data.append(d[k])
if header:
if firstheader:
headers = [firstheader] + headers
order = [' '] + order
if vertical_table:
print(array_dict_table_printer(print_data,
order=order,
header=headers,
vertical=True))
else:
print(array_dict_table_printer(print_data, order=order, header=headers))
if count:
c = len(print_data)
sentence = "count: {0}".format(c)
print(sentence)
print("+" + "-" * (len(sentence) - 2) + "+")
def shell_commands_dict_output(
username,
d,
# choose format manually
print_format=None,
# choose table format if needed
table_format=None,
# more specific table arguments
# --------------------------
# for table format: "key_list"
indexed=False,
# --------------------------
firstheader=None,
header=None,
oneitem=False,
vertical_table=False,
title=None,
count=False):
'''
some shell commands output a dict or table, this function should be
called when commands are printing in such way.
if none of parameters json and table is given, the funtion will try
to find the default printing form from db_defaults
if db_defaults has no item 'shell_print_format', the function will
set 'shell_print_format' = table
param username:: user id
param d:: data to print
param print_format:: print format: table, json, csv
param table_format:: choose table format if needed, DESCRIPTIONS:
type1: key_list:
accept a dict in the form:
{key1: [list1],
key2: [list2],
.......
=>
| key1 | key2 |
| l
| i
| s
| t
acceptable args: indexed: provide index
param firstheader:: designed for table, provide a attribute name for the
first item of each row, since the dict doesn't provide
it
param header:: designed for table(and used to filter input dict), a list of lists,
provides column order, e.g.
[[a,b], [c, d], ...
where 'a' is the printing column name and 'b' is the attribute name
in the dict
if you don't want to change the header name but want to keep the
header order or filter the items to list, for each item in the list
you may provide a string or a list with one item instead of a list of
two items
param oneitem:: designed for table, normally the input dict should be in such
form:
{a: {...}, b:{...}}, where each subitem is a row
if there is only one item, input dict is the {...} of the subitem
above
param title: provide a title for the table
param count: provide count info at the end of the table
'''
format_type = None
if print_format:
format_type = print_format
else:
format_type = get_default_print_format(username)
if format_type not in ALLOWED_PRINT_FORMAT:
Console.error(
"wrong print format: {0}. (allowed print format: {1})".format(
format_type, ", ".join(ALLOWED_PRINT_FORMAT)))
return False
headers = None
order = None
if header:
headers = []
order = []
for i in header:
if isinstance(i, basestring):
headers.append(i)
order.append(i)
elif isinstance(i, list):
if len(i) == 1:
headers.append(i[0])
order.append(i[0])
else:
headers.append(i[0])
order.append(i[1])
else:
print("ERROR: header info is not correct")
return False
# --------------------------------------------------------------------------
# filter the input dict
# --------------------------------------------------------------------------
if order:
new_d = {}
if oneitem:
for item in order:
if item in d:
new_d[item] = d[item]
else:
for i in d:
new_d[i] = {}
for item in order:
if item in d[i]:
new_d[i][item] = d[i][item]
d = new_d
# --------------------------------------------------------------------------
if format_type == "json":
if title:
banner(title)
print(json.dumps(d, indent=4))
elif format_type == "csv":
with open(".temp.csv", "wb") as f:
w = csv.DictWriter(f, d.keys())
w.writeheader()
w.writerow(d)
elif format_type == "table":
if table_format == "key_list":
print(dict_key_list_table_printer(d, indexed=indexed))
return
if title:
print("+" + "-" * (len(title) - 2) + "+")
print(title)
print_data = []
if oneitem:
print_data = [d]
else:
for k in sorted(d):
d[k][' '] = k
print_data.append(d[k])
if header:
if firstheader:
headers = [firstheader] + headers
order = [' '] + order
if vertical_table:
print(
array_dict_table_printer(print_data,
order=order,
header=headers,
vertical=True))
else:
print(
array_dict_table_printer(print_data,
order=order,
header=headers))
if count:
c = len(print_data)
sentence = "count: {0}".format(c)
print(sentence)
print("+" + "-" * (len(sentence) - 2) + "+")
def cm_manage():
"""Usage:
cm-manage config projects list
cm-manage config projects
cm-manage config [-f FILE] [-o OUT] [-p PROJECT] cloud NAME [-]
cm-manage config dump [--format=(yaml|dict)]
cm-manage config init [-o OUT] [-u USER]
cm-manage config list
cm-manage config password NAME
cm-manage config show passwords
cm-manage config fetch [-u USER] [-r HOST]
cm-manage --version
cm-manage --help
Arguments:
NAME name of the cloud
Options:
-h --help show this help message and exit
-v --version show version and exit
-f NAME --file=NAME the Name of the cloud to be specified,
if ? a selection is presented
-o OUT --out=OUT writes the result in the specifide file
-p PROJECT --project=PROJECT selects a project (e.g. for eucalyptus
which has project-specific environments)
-u USER --user=USER the user (login) name
-r HOST --remote=HOST the host machine on which the yaml file is
located in the CONFIG directory
[default: india.futuregrid.org]
-d --debug debug
- this option is a - at the end of the command.
If data is written to a file it is also put out to stdout
Description:
Command to generate rc files from our cloudmesh configuration files.
This program generates form a YAML file containing the login
information for a cloud an rc file that can be used to later source
it.
Example:
we assume the yaml file has an entry india-openstack::
cm-manage config -o novarc india-openstack
source novarc
This will create a novarc file and than you can source it::
cm-manage config ? -
Presents a selction of cloud choices and writes the choice into a
file called CONFIG/novarc
"""
default_path = config_file_prefix + '/novarc'
arguments = docopt(cm_manage.__doc__)
DEBUG("arguments", arguments)
home = os.environ['HOME']
# DEBUG("home", home)
#
# This secion deals with handeling "cm config" related commands
######################################################################
is_config = arguments['config'] != None
if is_config:
# DEBUG('Arguments', arguments)
file = arguments['--file']
try:
config = cm_config(file)
# DEBUG("config", config)
except IOError:
print("{0}: Configuration file '{1}' not found".format("CM ERROR", file))
sys.exit(1)
except (yaml.scanner.ScannerError, yaml.parser.ParserError) as yamlerror:
print("{0}: YAML error: {1}, in configuration file '{2}'".format("CM ERROR", yamlerror, file))
sys.exit(1)
except:
print("Unexpected error:", sys.exc_info()[0])
sys.exit(1)
name = arguments['NAME']
#
# NOT TESTED
#
if arguments['fetch'] or name == 'fetch':
DEBUG('Arguments', arguments)
# get user
var = {}
var['user'] = arguments['--user']
var['host'] = arguments['--remote']
#
# BUG should be
#
var['file'] = config_file_prefix() + "/cloudmesh.yaml"
if var['user'] is None:
var['user'] = getpass.getuser()
from_location = "%(user)s@%(host)s:%(file)s" % var
to_location = config_file("/%(file)s" % var)
if os.path.isfile(to_location):
print("WARNING: The file %s exists" % to_location)
if not yn_choice("Would you like to replace the file", default='y'):
sys.exit(1)
print(from_location)
print(to_location)
print("Copy cloudmesh file from %s to %s" % (from_location, to_location))
result = scp(from_location, to_location)
print(result)
sys.exit(0)
#
# ok
#
# if (arguments['projects'] and arguments['list']) :
if arguments['projects'] and arguments['list']:
projects = config.get('cloudmesh.projects')
print(yaml.dump(projects, default_flow_style=False, indent=4))
sys.exit(0)
#
# OK, needs setting
#
if arguments['projects']:
projects = config.projects('active')
print("Please select from the following:")
print("0 - Cancel")
selected = False
choices = []
while not selected:
counter = 1
for name in projects:
print(counter, "-" "%20s" % name)
choices.append(name)
counter += 1
print("Please select:")
input = int(sys.stdin.readline())
if input == 0:
print("Selection terminated")
sys.exit(0)
selected = (input > 0) and (input < counter)
print("Selected: ", input)
name = choices[input - 1]
print(name)
print("ERROR: THIS JUST SELECTS A PROJECT ID BUT DOES NOT SET IT")
sys.exit(0)
if arguments['init'] or name == 'init':
output = arguments['--out']
username = arguments['--user'] or os.getenv('USER')
location = path_expand(output)
new_yaml_file = open(location, 'w+')
user_yaml = cm_user().generate_yaml(username, 'cloudmesh')
print(yaml.dump(
user_yaml, default_flow_style=False), file=new_yaml_file)
new_yaml_file.close()
print("Written new yaml file in " + location)
sys.exit(0)
#
# OK
#
if arguments['list'] or name == 'list':
for name in config.cloudnames():
if 'cm_type' in config.cloud(name):
print(name, "(%s)" % config.cloud(name)['cm_type'])
sys.exit(0)
#
# NOT TESTED
#
if arguments['password']:
oldpass = getpass("Current password: "******"New password: "******"New password (again): ")
# TODO: some kind of password strength checking?
if newpass1 == newpass2:
config.change_own_password(name, oldpass, newpass1)
else:
print("New passwords did not match; password not changed.")
sys.exit(0)
#
# OK, but does not display the username
#
if arguments['show'] or name == 'show' and arguments['passwords']:
warning = "Your passwords will appear on the screen. Continue?"
if yn_choice(warning, 'n'):
me = ConfigDict(filename=config_file("/me.yaml"))
banner("PASSWORDS")
for name in me['password']:
print("{0}: {1}".format(name, me['password'][name]))
sys.exit(0)
#
# OK
#
if arguments['dump'] or name == 'dump':
format = arguments['--format']
if format == 'yaml':
d = dict(config)
print(yaml.dump(d, default_flow_style=False))
elif format == 'dict' or format is None:
print(config)
sys.exit(0)
#
# NOT TESTED
#
if name in ['?', 'x']:
if file is None:
arguments['--out'] = "%s/%s" % (home, default_path)
print("Please select from the following:")
print("0 - Cancel")
selected = False
choices = []
while not selected:
counter = 1
for name in config.cloudnames():
if 'cm_type' in config.cloud(name):
print("{0} - {1:<30} ({2})".format(counter, name, config.cloud(name)['cm_type']))
choices.append(name)
counter += 1
print("Please select:")
input = int(sys.stdin.readline())
if input == 0:
print("Selection terminated")
sys.exit(0)
selected = (input > 0) and (input < counter)
print("Selected: ", input)
name = choices[input - 1]
output = arguments['--out']
#
# OK
#
if name is not None:
cloud = config.cloud(name)
if not cloud:
print("%s: The cloud '%s' is not defined." % ("CM ERROR", name))
print("Try instead:")
for keyname in config.cloudnames():
print(" ", keyname)
sys.exit(1)
if cloud['cm_type'] == 'eucalyptus':
if arguments['--project']:
project = arguments['--project']
if not project in cloud:
print("No such project %s defined in cloud %s." % (project, name))
sys.exit(1)
else:
project = config.cloud_default(
name, 'project') or config.projects('default')
if not project in cloud:
print("Default project %s not defined in cloud %s." % (project, name))
sys.exit(1)
rc_func = lambda name: config.rc_euca(name, project)
else:
rc_func = config.rc
result = rc_func(name)
#
# OK
#
if arguments["-"]:
print(result)
else:
if output is None:
arguments['--out'] = "%s/%s" % (home, default_path)
output = arguments['--out']
out = False
if yn_choice("Would you like to review the information", default="y"):
banner("WARNING: FIle will be written to " + output)
print(result)
print(banner(""))
try:
# First we try to open the file assuming it doesn't exist
out = os.open(
output, os.O_CREAT | os.O_EXCL | os.O_WRONLY, stat.S_IRUSR | stat.S_IWUSR)
except OSError as oserr:
# If file exists, offer to replace it
if oserr.errno == 17:
delete = raw_input(
"'%s' exists; Overwrite it [N|y]? " % output)
if delete.strip().lower() == 'y':
out = os.open(output, os.O_TRUNC | os.O_WRONLY)
if out:
os.write(out, result)
os.close(out)
import cloudmesh
from pprint import pprint
cloudmesh.logger(False)
username = cloudmesh.load().username()
cloudmesh.banner("INIT MONGO")
mesh = cloudmesh.mesh("mongo")
#
# authentication as a user - username is requried
# On webgui side, this is achieved by the framework,
# and the username is obtined from g.user.id
#
# On CLI side, a global user object or username variable
# should be maintained upon the start of the shell
# The username could be obtained from yaml file.
cloudmesh.banner("ACTIVATE")
mesh.activate(username)
cloudmesh.banner("GET FLAVOR")
mesh.refresh(username, types=['flavors'], names=["india"])
data = mesh.flavors(cm_user_id=username, clouds=["india"])
pprint(data)
cloudmesh.banner("GET IMAGE")
mesh.refresh(username, types=['images'], names=["india"])
data = mesh.images(cm_user_id=username, clouds=["india"])
credential['EC2_ACCESS_KEY'] = access_key
credential['EC2_SECERT_KEY'] = secret_key
return credential
class CredentialFromMongo(CredentialBaseClass):
def __init__(self, user, cloud, datasource=None):
"""data source is a collectionname in cloudmesh_server.yaml"""
"""if day=tasource is none than use the default on which is ?"""
raise NotImplementedError()
if __name__ == "__main__":
# -------------------------------------------------------------------------
banner("YAML read test")
# -------------------------------------------------------------------------
banner("gvonlasz - sierra_openstack_grizzly - old")
# -------------------------------------------------------------------------
credential = CredentialFromYaml("gvonlasz", "sierra_openstack_grizzly")
pprint(credential)
banner("gvonlasz - sierra_openstack_grizzly - new")
# -------------------------------------------------------------------------
credential = CredentialFromYaml(
"gvonlasz", "sierra_openstack_grizzly", style=3.0)
pprint(credential)
print credential['credentials']['OS_USERNAME']
print credential.keys()
hosts = []
# hosts.append("localhost")
hosts.append("india.futuregrid.org")
task = {}
watch = StopWatch()
for execute in [Sequential, Parallel]:
name = execute.__name__
banner(name)
watch.start(name)
result = execute(hosts, cm_ssh, username=username, command="qstat")
watch.stop(name)
pprint(result)
banner("PRINT")
for host in result:
print result[host]["output"]
for timer in watch.keys():
print timer, watch.get(timer), "s"
def start_vm(username,
cloudname,
count=1,
flavorname=None,
flavorid=None,
imagename=None,
imageid=None,
groupname=None,
servername=None):
'''
create a vm of a cloud of a user
will check flavor, image existence if provided
user can specify groupname which will be written in metadata, servername which
will replace prefix+index as the vm name
it's better to check cloud active status before use this function
:param username: string
:param cloudname: string
:param count: number of vms to start
:return False if error
TODO: what if fail, how to acknowledge it; no return now as using celery
input key
missing security group
'''
# Changed scope of this import - hyungro lee 12/01/2014
from cloudmesh.experiment.group_usage import add_vm_to_group_while_creating
mongo = cm_mongo()
userobj = cm_user()
cloudobj = CloudManage()
mongo.activate(cm_user_id=username, names=[cloudname])
userinfo = userobj.info(username)
key = None
vm_image_id = None
vm_flavor_id = None
error = ''
# -------------------------
# refresh server flavor or image
to_refresh = ["servers"]
if flavorname is not None or flavorid is not None:
to_refresh.append("flavors")
if imagename is not None or imageid is not None:
to_refresh.append("images")
if to_refresh != []:
mongo.refresh(username, names=[cloudname], types=to_refresh)
# -------------------------
# get exist VM names list, to prevent names duplicate
serverdata = mongo.servers(
clouds=[cloudname], cm_user_id=username)[cloudname]
servers_names_list = []
for k,v in serverdata.iteritems():
servers_names_list.append(v['name'])
# -------------------------
# flavor handler
if flavorname is not None or flavorid is not None:
flavordata = mongo.flavors(
clouds=[cloudname], cm_user_id=username)[cloudname]
same_name_count = 0
for k, v in flavordata.iteritems():
if flavorname is not None:
if flavorname == v['name']:
vm_flavor_id = k
same_name_count = same_name_count + 1
else:
if flavorid == k:
vm_flavor_id = k
break
if vm_flavor_id is None:
error = error + "The flavor you provide doesn't exist. "
if same_name_count > 1:
error = error + "There are more than one flavor with the name you provide" \
"please use flavorid instead or select one by command cloud" \
"set flavor [CLOUD]. "
else:
try:
vm_flavor_id = userinfo["defaults"]["flavors"][cloudname]
except:
pass
if vm_flavor_id in [None, 'none']:
error = error + \
"Please specify a default flavor(command: cloud set flavor [CLOUD]). "
# -------------------------
# image handler
if imagename is not None or imageid is not None:
imagedata = mongo.images(
clouds=[cloudname], cm_user_id=username)[cloudname]
same_name_count = 0
for k, v in imagedata.iteritems():
if imagename is not None:
if imagename == v['name']:
vm_image_id = k
same_name_count = same_name_count + 1
else:
if imageid == k:
vm_image_id = k
break
if vm_image_id is None:
error = error + "The image you provide doesn't exist. "
if same_name_count > 1:
error = error + "There are more than one image with the name you provide" \
"please use imageid instead or select one by command cloud" \
"set image [CLOUD]. "
else:
try:
vm_image_id = userinfo["defaults"]["images"][cloudname]
except:
pass
if vm_image_id in [None, 'none']:
error = error + \
"Please specify a default image(command: cloud set flavor [CLOUD]). "
# -------------------------
# key handler
if "key" in userinfo["defaults"]:
key = userinfo["defaults"]["key"]
elif len(userinfo["keys"]["keylist"].keys()) > 0:
key = userinfo["keys"]["keylist"].keys()[0]
if key:
keycontent = userinfo["keys"]["keylist"][key]
if keycontent.startswith('key '):
keycontent = keycontent[4:]
cm_keys_mongo(username).check_register_key(username, cloudname, key, keycontent)
keynamenew = _keyname_sanitation(username, key)
else:
error = error + \
"No sshkey found. Please Upload one"
# -------------------------
if error != '':
Console.error(error)
return False
# -------------------------
metadata = {'cm_owner': username}
if groupname:
metadata['cm_group'] = groupname
tmpnamefl = cloudobj.get_flavors(
cloudname=cloudname, getone=True, id=vm_flavor_id)['name']
tmpnameim = cloudobj.get_images(
cloudname=cloudname, getone=True, id=vm_image_id)['name']
while count > 0:
userinfo = userobj.info(username)
if servername:
prefix = ''
index = ''
givenvmname = servername
tmpnameser = servername
else:
prefix = userinfo["defaults"]["prefix"]
index = userinfo["defaults"]["index"]
givenvmname = None
tmpnameser = prefix + '_' + str(index)
# ------------------------
# do not allow server name duplicate
if tmpnameser in servers_names_list:
Console.error("vm name '{0}' exists, please use other names or delete it first".format(
tmpnameser))
if not servername:
userobj.set_default_attribute(username, "index", int(index) + 1)
count = count - 1
continue
# ------------------------
# vm start procedure
banner("Starting vm->{0} on cloud->{1} using image->{2}, flavor->{3}, key->{4}"
.format(tmpnameser, cloudname, tmpnameim, tmpnamefl, keynamenew))
useQueue = False
if useQueue:
result = mongo.vm_create_queue(cloudname,
prefix,
index,
vm_flavor_id,
vm_image_id,
keynamenew,
meta=metadata,
cm_user_id=username,
givenvmname=givenvmname)
print("job status:", result.state)
else:
result = mongo.vm_create(cloudname,
prefix,
index,
vm_flavor_id,
vm_image_id,
keynamenew,
meta=metadata,
cm_user_id=username,
givenvmname=givenvmname)
if "server" in result and "adminPass" in result["server"]:
result["server"]["adminPass"] = "******"
pprint(result)
# ------------------------
# add it to the group in database if groupname provided
if groupname:
try:
add_vm_to_group_while_creating(username, groupname, tmpnameser)
except Exception, err:
Console.error(str(err))
return
# ------------------------
# increase index if it is used
if not servername:
userobj.set_default_attribute(username, "index", int(index) + 1)
# ------------------------
servers_names_list.append(tmpnameser)
count = count - 1
import cloudmesh as cm
cm.banner('HW3 ex2')
print cm.shell('cloud list')
print cm.shell('cloud on india')
print cm.shell('cloud list')
def delete_vm(username,
cloudname,
server_id_list=None,
preview=False,
refresh=False):
'''
delete vms of a cloud of a user, this function provides several ways to find and
delete vms
:param server_id_list:: the list of VMs(id) to delete
:param preview:: True if the user wants to preview and confirm before start to delete
TODO: what if fail, how to acknowledge it
range search: now if prefix not given, all vms whose index are in the range will
be deleted, regardless of its prefix
it looks like even though delete a vm and return {msg: seccess}, sometimes refresh
after 5 sec, it might be still there
'''
# changed the scope of this import
# Benefit: other functions are not affected with this import
# drawback: hard to see which module is being loaded in this file
# Hyungro Lee 12/01/2014
from cloudmesh.experiment.group_usage import remove_vm_from_group_while_deleting
try:
mongo = cm_mongo()
except:
Console.error("There is a problem with the mongo server")
return False
if refresh:
mongo.activate(cm_user_id=username, names=[cloudname])
mongo.refresh(cm_user_id=username,
names=[cloudname],
types=['servers'])
serverdata = mongo.servers(
clouds=[cloudname], cm_user_id=username)[cloudname]
# -------------------------
# preview and confirm
confirm_deletion = True
if preview:
if server_id_list == []:
Console.warning("no vm meets the condition")
return False
else:
resserverdata = {}
for i in server_id_list:
resserverdata[i] = serverdata[i]
cloudobj = CloudManage()
itemkeys = {"openstack":
[
['name', 'name'],
['status', 'status'],
['addresses', 'addresses'],
['id', 'id'],
['flavor', 'flavor', 'id'],
['image', 'image', 'id'],
['user_id', 'cm_user_id'],
['metadata', 'metadata'],
['key_name', 'key_name'],
['created', 'created'],
['cloud', 'cm_cloud']
],
"ec2":
[
["name", "id"],
["status", "extra", "status"],
["addresses", "public_ips"],
["flavor", "extra", "instance_type"],
['id', 'id'],
['image', 'extra', 'imageId'],
["user_id", 'user_id'],
["metadata", "metadata"],
["key_name", "extra", "key_name"],
["created", "extra", "launch_time"]
],
"aws":
[
["name", "name"],
["status", "extra", "status"],
["addresses", "public_ips"],
["flavor", "extra", "instance_type"],
['id', 'id'],
['image', 'extra', 'image_id'],
["user_id", "user_id"],
["metadata", "metadata"],
["key_name", "extra", "key_name"],
["created", "extra", "launch_time"]
],
"azure":
[
['name', 'name'],
['status', 'status'],
['addresses', 'vip'],
['flavor', 'flavor', 'id'],
['id', 'id'],
['image', 'image', 'id'],
['user_id', 'user_id'],
['metadata', 'metadata'],
['key_name', 'key_name'],
['created', 'created'],
]
}
cloudobj.print_cloud_servers(username=username,
cloudname=cloudname,
itemkeys=itemkeys,
refresh=False,
output=False,
serverdata=resserverdata)
if yn_choice("confirm to delete these vms?", default='n', tries=3):
pass
else:
confirm_deletion = False
# -------------------------
# deleting
if confirm_deletion:
if server_id_list == []:
return
watch = time.time()
useQueue = False
if useQueue:
# not functioning
cloudmanager = mongo.clouds[username][cloudname]["manager"]
cm_type = mongo.get_cloud_info(username, cloudname)['cm_type']
package = "cloudmesh.iaas.%s.queue" % cm_type
name = "tasks"
imported = getattr(__import__(package, fromlist=[name]), name)
queue_name = "%s-%s" % (cm_type, "servers")
for i in server_id_list:
tempservername = serverdata[i]['name'].encode("ascii")
banner(
"Deleting vm->{0} on cloud->{1}".format(tempservername, cloudname))
result = imported.vm_delete.apply_async(
(cloudname, i, username), queue=queue_name)
print("job status:", result.state)
try:
remove_vm_from_group_while_deleting(username, tempservername)
except Exception, err:
Console.error(str(err))
return
# print result.traceback #########
imported.wait.apply_async(
args=None, kwargs={'t': 10}, queue=queue_name)
handleip = imported.release_unused_public_ips.apply_async(
(cloudname, username), queue=queue_name)
handlerefresh = imported.refresh.apply_async(args=None,
kwargs={'cm_user_id': username,
'names': [cloudname],
'types': ['servers']},
queue=queue_name)
# print handleip.state
# print handleip.traceback
# print handlerefresh.state
# print handlerefresh.traceback
if preview:
print("to check realtime vm status: list vm --refresh")
else:
for i in server_id_list:
tempservername = serverdata[i]['name'].encode("ascii")
banner(
"Deleting vm->{0} on cloud->{1}".format(tempservername, cloudname))
result = mongo.vm_delete(cloudname, i, username)
pprint(result)
try:
remove_vm_from_group_while_deleting(username, tempservername)
except Exception, err:
Console.error(str(err))
return
time.sleep(5)
mongo.release_unused_public_ips(cloudname, username)
mongo.refresh(username, names=[cloudname], types=['servers'])
