def display_launch_table():
launcher_config = ConfigDict(filename=config_file("/cloudmesh_launcher.yaml"))
launch_recipies = launcher_config.get("cloudmesh.launcher.recipies")
columns = launcher_config.get("cloudmesh.launcher.columns")
return render_template('mesh/mesh_launch.html',
recipies=launch_recipies,
columns=columns,
)
def test_getitem_server(self):
HEADING()
filename = "~/.futuregrid/cloudmesh_server.yaml"
config = ConfigDict(filename=filename)
print config
existing = config.get("cloudmesh.server.mongo.db")
test1 = existing is not None
print "QUERY", existing
print "Port", config.get("cloudmesh.server.mongo.port")
def test_getitem_server(self):
HEADING()
filename = "~/.futuregrid/cloudmesh_server.yaml"
config = ConfigDict(filename=filename)
print config
existing = config.get("cloudmesh.server.mongo.db")
test1 = existing is not None
print "QUERY", existing
print "Port", config.get("cloudmesh.server.mongo.port")
def test_getitem_server(self):
HEADING()
filename = config_file("/cloudmesh_server.yaml")
config = ConfigDict(filename=filename)
print(config)
existing = config.get("cloudmesh.server.mongo.db")
test1 = existing is not None
print("QUERY", existing)
print("Port", config.get("cloudmesh.server.mongo.port"))
def test_launcher(self):
HEADING()
filename = config_file("/cloudmesh_launcher.yaml")
config = ConfigDict(filename=filename)
print(config)
existing = config.get("cloudmesh.launcher.recipies")
test1 = existing is not None
print(existing)
try:
none_existing = config.get("cloudmesh.launcher.recipies.xyz")
test2 = False
except:
print("Error")
test2 = True
assert test1 and test2
def test_server(self):
HEADING()
filename = "~/.futuregrid/cloudmesh_server.yaml"
config = ConfigDict(filename=filename)
# print config
existing = config.get("cloudmesh.server.mongo.db")
test1 = existing is not None
print "mongo.db =", existing
try:
none_existing = config.get("cloudmesh.server.mongo.xyz")
test2 = False
except:
print "Error"
test2 = True
assert test1 and test2
def test_launcher(self):
HEADING()
filename = "~/.futuregrid/cloudmesh_launcher.yaml"
config = ConfigDict(filename=filename)
print config
existing = config.get("cloudmesh.launcher.recipies")
test1 = existing is not None
print existing
try:
none_existing = config.get("cloudmesh.launcher.recipies.xyz")
test2 = False
except:
print "Error"
test2 = True
assert test1 and test2
def test_launcher(self):
HEADING()
filename = "~/.futuregrid/cloudmesh_launcher.yaml"
config = ConfigDict(filename=filename)
print config
existing = config.get("cloudmesh.launcher.recipies")
test1 = existing is not None
print existing
try:
none_existing = config.get("cloudmesh.launcher.recipies.xyz")
test2 = False
except:
print "Error"
test2 = True
assert test1 and test2
def test_server(self):
HEADING()
filename = "~/.futuregrid/cloudmesh_server.yaml"
config = ConfigDict(filename=filename)
# print config
existing = config.get("cloudmesh.server.mongo.db")
test1 = existing is not None
print "mongo.db =", existing
try:
none_existing = config.get("cloudmesh.server.mongo.xyz")
test2 = False
except:
print "Error"
test2 = True
assert test1 and test2
def readRackConfig(self, name, dir_yaml, dir_diag):
rack_config = ConfigDict(filename=dir_yaml + "/" +
self.default_rack_yaml)
self.dict_rack_config = rack_config.get("cloudmesh.rack")
lname = name.lower()
# the rack name in cloudmesh_rack.yaml MUST be valid/predefined in
# clouldmesh_cluster.yaml
self.cluster_name = lname if lname in self.cluster_name_list else self.cluster_name_unknown
# diag filename and temporary diag filename
self.filename_diag = dir_diag + "/" + \
self.dict_rack_config["cluster"][self.cluster_name]["diag"]
self.rack_count = self.dict_rack_config["cluster"][
self.cluster_name]["count"]
self.filename_diag_temp = self.filename_diag + ".temp"
# additional process for 'all'
# its count equals the sum of all other cluster exclude 'all' and
# 'unknown'
if name == "all":
raw_set = set(self.dict_rack_config["cluster"].keys())
exclude_set = set(["all", "unknown"])
all_set = raw_set - exclude_set
self.rack_count = 0
for rack in all_set:
self.rack_count += self.dict_rack_config["cluster"][rack][
"count"]
def readRackConfig(self, name, dir_yaml, dir_diag):
rack_config = ConfigDict(
filename=dir_yaml + "/" + self.default_rack_yaml)
self.dict_rack_config = rack_config.get("cloudmesh.rack")
lname = name.lower()
# the rack name in cloudmesh_rack.yaml MUST be valid/predefined in
# clouldmesh_cluster.yaml
self.cluster_name = lname if lname in self.cluster_name_list else self.cluster_name_unknown
# diag filename and temporary diag filename
self.filename_diag = dir_diag + "/" + \
self.dict_rack_config["cluster"][self.cluster_name]["diag"]
self.rack_count = self.dict_rack_config[
"cluster"][self.cluster_name]["count"]
self.filename_diag_temp = self.filename_diag + ".temp"
# additional process for 'all'
# its count equals the sum of all other cluster exclude 'all' and
# 'unknown'
if name == "all":
raw_set = set(self.dict_rack_config["cluster"].keys())
exclude_set = set(["all", "unknown"])
all_set = raw_set - exclude_set
self.rack_count = 0
for rack in all_set:
self.rack_count += self.dict_rack_config[
"cluster"][rack]["count"]
def __init__(self, username, filename, Credential, style=2.0, password=None):
config = ConfigDict(filename=filename)
self.password = password
self[username] = {}
for cloud in config.get("cloudmesh.clouds").keys():
self[username][cloud] = Credential(username,
cloud,
filename,
style=style,
password=self.password)
self.encrypt(username, cloud, style)
def __init__(self, username, filename, Credential, style=2.0, password=None):
config = ConfigDict(filename=filename)
self.password = password
self[username] = {}
for cloud in config.get("cloudmesh.clouds").keys():
self[username][cloud] = Credential(username,
cloud,
filename,
style=style,
password=self.password)
self.encrypt(username, cloud, style)
def view(link=""):
"""run the browser"""
from cloudmesh.config.ConfigDict import ConfigDict
server_config = ConfigDict(filename=config_file("/cloudmesh_server.yaml"))
host = server_config.get("cloudmesh.server.webui.host")
port = server_config.get("cloudmesh.server.webui.port")
url_link = "http://{0}:{1}/{2}".format(host, port, link)
local("%s %s" % (web_browser, url_link))
def get_ldap_user_from_yaml():
me = ConfigDict(filename=config_file("/me.yaml"))
d = {}
for element in ["firstname",
"lastname",
"email",
"phone",
"address"]:
d[element] = me.get("profile.{0}".format(element))
d["cm_user_id"] = me.get("portalname")
d["gidNumber"] = 0
d["uidNumber"] = 0
if "gidNumber" in me.keys():
d["gidNumber"] = me.get("gidNumber")
if "uidNumber" in me.keys():
d["uidNumber"] = me.get("uidNumber")
d["projects"] = me.get("projects")
#
# copy the keys
#
d['keys'] = me.get("keys.keylist")
return d
def view(link=""):
"""run the browser"""
from cloudmesh.config.ConfigDict import ConfigDict
server_config = ConfigDict(filename="~/.futuregrid/cloudmesh_server.yaml")
host = server_config.get("cloudmesh.server.webui.host")
port = server_config.get("cloudmesh.server.webui.port")
url_link = "http://{0}:{1}/{2}".format(host, port, link)
local("%s %s" % (web_browser, url_link))
def launch_servers():
launcher_config = ConfigDict(filename=config_file("/cloudmesh_launcher.yaml"))
celery_config = ConfigDict(filename=config_file("/cloudmesh_celery.yaml"))
launch_recipies = launcher_config.get("cloudmesh.launcher.recipies")
server = request.form.get("server")
name_index = request.form.get("name_index")
return_dict = dict(launch_recipies[server][int(name_index)])
return_dict["user"] = "******" # default user.... change to read from session
return_dict["server"] = server
parameters = launch_recipies[server][int(name_index)]["parameters"] # load the needed set of parameters
for parameter in parameters:
parameters[parameter] = request.form.get("parameter_{0}".format(parameter))
return_dict["parameters"] = parameters
return_dict["name"] = launch_recipies[server][int(name_index)]["name"]
queue = celery_config = celery_config.get("cloudmesh.workers.launcher.queue")
task_launch.apply_async(queue=queue, args=[return_dict])
return "Task has been submitted to the queue.... <br/><br/>Data sent was:<br/>" + str(return_dict)
def display_launch_table():
""" To satisfy Pylint. Will update with proper comments """
if request.method == 'POST':
print "HHHHHH", request.form.keys()
for key in request.form.keys():
print key, ":", request.form[key]
else:
print "HEY JUDE"
launcher_config = ConfigDict(
filename=config_file("/cloudmesh_launcher.yaml"))
launch_recipies = launcher_config.get("cloudmesh.launcher.recipies")
return render_template('mesh/mesh_launch.html', recipies=launch_recipies)
def display_launch_table():
""" To satisfy Pylint. Will update with proper comments """
if request.method == 'POST':
print "HHHHHH", request.form.keys()
for key in request.form.keys():
print key, ":", request.form[key]
else:
print "HEY JUDE"
launcher_config = ConfigDict(
filename=config_file("/cloudmesh_launcher.yaml"))
launch_recipies = launcher_config.get("cloudmesh.launcher.recipies")
return render_template('mesh/mesh_launch.html',
recipies=launch_recipies)
def shell_command_open_web(arguments):
"""
Usage:
web [--fg|--cm] [LINK]
Arguments:
LINK the link on the localhost cm server is opened.
Options:
-v verbose mode
--fg opens a link on the FG portal
--cm opens a link on the CM portal
Description:
Opens a web page with the specified link
"""
link = arguments["LINK"]
if link is None or link == "/":
link = ""
web_browser = "firefox"
if sys.platform == 'darwin':
web_browser = "open"
if arguments["--fg"]:
location = "https://portal.futuregrid.org"
elif arguments["--cm"]:
location = "https://cloudmesh.futuregrid.org"
else:
try:
filename = config_file("/cloudmesh_server.yaml")
server_config = ConfigDict(filename=filename)
host = server_config.get("cloudmesh.server.webui.host")
port = server_config.get("cloudmesh.server.webui.port")
location = "http://{0}:{1}".format(host, port)
except Exception, e:
print("ERROR: some error reading from the config file")
print(e)
return
def shell_command_open_web(arguments):
"""
Usage:
web [--fg|--cm] [LINK]
Arguments:
LINK the link on the localhost cm server is opened.
Options:
-v verbose mode
--fg opens a link on the FG portal
--cm opens a link on the CM portal
Description:
Opens a web page with the specified link
"""
link = arguments["LINK"]
if link is None or link == "/":
link = ""
web_browser = "firefox"
if sys.platform == 'darwin':
web_browser = "open"
if arguments["--fg"]:
location = "https://portal.futuregrid.org"
elif arguments["--cm"]:
location = "https://cloudmesh.futuregrid.org"
else:
try:
filename = config_file("/cloudmesh_server.yaml")
server_config = ConfigDict(filename=filename)
host = server_config.get("cloudmesh.server.webui.host")
port = server_config.get("cloudmesh.server.webui.port")
location = "http://{0}:{1}".format(host, port)
except Exception, e:
print("ERROR: some error reading from the config file")
print(e)
return
def password():
user_config = cm_config(filename=config_file("/cloudmesh.yaml"))
user = user_config.cloud('india')['credentials']
server_config = ConfigDict(filename=config_file("/cloudmesh_server.yaml"))
server = server_config.get('cloudmesh.server.keystone.india')
print(" ".join([
"keystone", "--os-username", server['OS_USERNAME'], "--os-password",
server['OS_PASSWORD'], "--os-tenant-name", server['OS_TENANT_NAME'],
"--os-auth-url", server['OS_AUTH_URL'], "user-password-update",
"--pass", user['OS_PASSWORD'], user['OS_USERNAME']
]))
keystone("--os-username", server['OS_USERNAME'], "--os-password",
server['OS_PASSWORD'], "--os-tenant-name",
server['OS_TENANT_NAME'], "--os-auth-url", server['OS_AUTH_URL'],
"user-password-update", "--pass", user['OS_PASSWORD'],
user['OS_USERNAME'])
def password():
user_config = cm_config(filename="~/.futuregrid/cloudmesh.yaml")
user = user_config.cloud('sierra_openstack_grizzly')['credentials']
server_config = ConfigDict(filename="~/.futuregrid/cloudmesh_server.yaml")
server = server_config.get('cloudmesh.server.keystone.sierra_openstack_grizzly')
print(" ".join(["keystone", "--os-username", server['OS_USERNAME'],
"--os-password", server['OS_PASSWORD'],
"--os-tenant-name", server['OS_TENANT_NAME'],
"--os-auth-url", server['OS_AUTH_URL'],
"user-password-update",
"--pass", user['OS_PASSWORD'], user['OS_USERNAME']]))
keystone("--os-username", server['OS_USERNAME'],
"--os-password", server['OS_PASSWORD'],
"--os-tenant-name", server['OS_TENANT_NAME'],
"--os-auth-url", server['OS_AUTH_URL'],
"user-password-update",
"--pass", user['OS_PASSWORD'], user['OS_USERNAME'])
def readClustersConfig(self, dir_yaml):
clusters_config = ConfigDict(filename=dir_yaml + "/" +
self.default_clusters_yaml)
self.dict_clusters_config = clusters_config.get("cloudmesh.inventory")
# get all possible cluster names from dict_clusters_config
self.cluster_name_list += self.dict_clusters_config.keys()
class Test_cloudmesh:
d = {"portalname": "gvonlasz"}
filename = "etc/cloudmesh.yaml"
def setup(self):
self.t = cm_template(path_expand(self.filename))
self.user = cm_user()
try:
self.setup_inventory()
except:
print("=" * 40)
print("setup_inventory() failed. ldap test will not be performed")
print("=" * 40)
def tearDown(self):
os.system("fab test.info:user")
pass
def test_variables(self):
HEADING()
print(self.t.variables())
assert "portalname" in self.t.variables()
def test_replace_incomplete(self):
try:
print(self.t.replace(self.d, format="dict"))
except:
pass
assert True
def test_user(self):
d = {
"portalname": "gvonlasz",
"sierra_openstack_password": "******",
"project_default": "fg82",
"india_openstack_password": "******",
"projects": "82, 83", # this is still wrong
}
print(self.t.replace(d=d))
# self.t.complete does not exist in cm_template?
# assert self.t.complete
# if not t.complete():
# print "ERROR: undefined variables"
# print t.variables()
def test_userinfo(self):
HEADING()
print(self.user.info("fuwang"))
print(self.user.info("gvonlasz"))
print(self.user.info("nonexistuser"))
print(self.user.info("nova"))
print(self.user.info("fuwang", ["sierra"]))
print(self.user.info("fuwang", ["cloud-non-exist"]))
print("============================")
pprint(self.user["gvonlasz"])
print(self.user.get_name('gvonlasz'))
def setup_inventory(self):
banner("Read Dicts")
self.sample_user = ConfigDict(filename=config_file("/me.yaml"))
self.portalname = self.sample_user.get("portalname")
print("PORTALNAME", self.portalname)
print("SAMPLE USER", self.sample_user)
banner("create user from template, duplicates cm init generate me")
t = cm_template(config_file("/etc/cloudmesh.yaml"))
pprint(set(t.variables()))
self.config = t.replace(kind="dict", values=self.sample_user)
print(type(self.config))
print(self.config)
#
# BUG?
#
self.idp = cm_userLDAP()
self.idp.connect("fg-ldap", "ldap")
self.idp.refresh()
ldap_info = self.idp.get(self.portalname)
print(ldap_info)
print(type(self.config))
self.config['cloudmesh']['projects'] = ldap_info['projects']
self.config['cloudmesh']['keys'] = ldap_info['keys']
try:
self.config['cloudmesh']['projects']['deafult'] = ldap_info[
'projects']['active'][0]
except:
print("ERROR: you have no projects")
def test_print(self):
pprint(self.config)
def test_projects(self):
projects = dict()
keys = dict()
# read the yaml
# read projects info from ldap
# write out new dict/json file
pass
def test_keys(self):
# read the yaml
# read KEYS from ldap
# write out new dict/json file
pass
def test_gregor(self):
banner("ME")
id = ConfigDict(filename=config_file("/me.yaml")).get("portalname")
user = cm_user()
result = user.info(id)
pprint(result)
pass
def test_list(self):
user = cm_user()
list_of_users = user.list_users()
pprint(list_of_users)
print()
print("=========================")
num = len(list_of_users)
print(str(num) + " users listed")
def test_mongo_credential(self):
banner("USER")
pprint(self.user.info("gvonlasz"))
username = "******"
cloudname = "dummy"
password = "******"
tennant = "fg1"
name = username
self.user.set_credential(
username, cloudname, {
"OS_USERNAME": name,
"OS_PASSWORD": password,
"OS_TENANT_NAME": tennant,
"CM_CLOUD_TYPE": "openstack"
})
cred = self.user.get_credential(username, cloudname)
banner("credential")
print(cred)
banner("credentials")
pprint(self.user.get_credentials(username))
banner("find")
result = self.user.userdb_passwd.find({})
for r in result:
pprint(r)
from pprint import pprint
from cloudmesh.config.ConfigDict import ConfigDict
from cloudmesh.config.cm_config import cm_config_server
from cloudmesh_common.util import banner
from cloudmesh.provisioner.queue.celery import celery as p_queue
from cloudmesh.launcher.queue.celery import celery as l_queue
from cloudmesh.pbs.celery import celery as pbs_queue
from celery import Celery
__all__ = [
'start', 'stop', 'list', 'clean', 'gui', 'monitor', 'kill', 'ls', 'lspbs'
]
celery_config = ConfigDict(filename="~/.futuregrid/cloudmesh_celery.yaml",
kind="worker")
workers = celery_config.get("cloudmesh.workers")
"""
for worker in workers:
workers[worker] = {"app":"cloudmesh.launcher{0}.queue",
"hostlist":hostlist.expand_hostlist("l[1-{0}]".format(workers[worker])),
"queue": worker}
"""
# no_workers_launcher =
for worker in workers:
workers[worker]["hostlist"] = hostlist.expand_hostlist("{0}[1-{1}]".format(
workers[worker]["id"], workers[worker]["count"]))
# launcher_workers = {"app":"cloudmesh.launcher.queue",
# "hostlist":hostlist.expand_hostlist("l[1-{0}]".format(workers["launcher"]["count"])),
# "queue":"launcher"}
#
app = Flask(__name__)
api = restful.Api(app)
queue_stat_parser = reqparse.RequestParser()
queue_stat_parser.add_argument('resource', type=str, required=True)
#queue_stat_parser.add_argument('user', type=str)
queue_stat_parser.add_argument('id', type=str)
queue_info_parser = reqparse.RequestParser()
queue_info_parser.add_argument('resource', type=str, required=True)
#queue_info_parser.add_argument('user', type=str)
queue_info_parser.add_argument('queue', type=str)
config = ConfigDict(
prefix="cloudmesh", filename=config_file("/cloudmesh.yaml"))
user = config.get("cloudmesh.profile.username")
def versioned_url(url):
return "/" + version + url
def simple_error(kind, attribute, help="does not exist"):
msg = {}
msg["error:"] = "{0} {1} {2}".format(kind, attribute, help)
return msg
class rest_queue_stat(restful.Resource):
def get(self):
def do_launcher(self, args, arguments):
"""
Usage:
launcher start MENU
launcher stop STACK_NAME
launcher list
launcher show STACK_NAME
launcher menu [--column=COLUMN] [--format=FORMAT]
launcher import [FILEPATH] [--force]
launcher export FILEPATH
launcher help | -h
An orchestration tool with Chef Cookbooks
Arguments:
MENU Name of a cookbook
STACK_NAME Name of a launcher
FILEPATH Filepath
COLUMN column name to display
FORMAT display format (json, table)
help Prints this message
Options:
-v verbose mode
"""
log.info(arguments)
self.cm_mongo = cm_mongo()
self.cm_config = cm_config()
self.user = cm_user()
if arguments["help"] or arguments["-h"]:
print (self.do_launcher.__doc__)
elif arguments['show'] and arguments['STACK_NAME']:
print ("NOT IMPLEMENTED")
return
elif arguments['menu']:
userid = self.cm_config.username()
launchers = self.cm_mongo.launcher_get(userid)
if launchers.count() == 0:
Console.warning("no launcher in database, please import launcher first"
"(launcher import [FILEPATH] [--force])")
return
else:
d = {}
for launcher in launchers:
d[launcher['cm_launcher']] = launcher
if "_id" in d[launcher['cm_launcher']]:
del d[launcher['cm_launcher']]['_id']
columns = None
if arguments['--column']:
if arguments['--column'] != "all":
columns = [x.strip() for x in arguments['--column'].split(',')]
else:
columns = ['name', 'description']
if arguments['--format']:
if arguments['--format'] not in ['table', 'json', 'csv']:
Console.error("please select printing format ",
"among table, json and csv")
return
else:
p_format = arguments['--format']
else:
p_format = None
shell_commands_dict_output(userid,
d,
print_format=p_format,
firstheader="launcher",
header=columns
# vertical_table=True
)
elif arguments['list']:
userid = self.cm_config.username()
self.cm_mongo.activate(userid)
self.cm_mongo.refresh(cm_user_id=userid, types=[self._id])
stacks = self.cm_mongo.stacks(cm_user_id=userid)
launchers = self.filter_launcher(
stacks,
{"search": "contain",
"key": "stack_name",
"value": "launcher"}
)
log.debug(launchers)
d = {}
for k0, v0 in launchers.iteritems():
for k1, v1 in launchers[k0].iteritems():
d[v1['id']] = v1
columns = ['stack_name', 'description', 'stack_status',
'creation_time', 'cm_cloud']
if arguments['--column'] and arguments['--column'] != "all":
columns = [x.strip() for x in arguments['--column'].split(',')]
if arguments['--format']:
if arguments['--format'] not in ['table', 'json', 'csv']:
Console.error("please select printing format among table, json and csv")
return
else:
p_format = arguments['--format']
else:
p_format = None
shell_commands_dict_output(userid,
d,
print_format=p_format,
firstheader="launcher_id",
header=columns
# vertical_table=True
)
elif arguments['start'] and arguments['MENU']:
userid = self.cm_config.username()
def_cloud = self.get_cloud_name(userid)
self.cm_mongo.activate(userid)
userinfo = self.user.info(userid)
if "key" in userinfo["defaults"]:
key = userinfo["defaults"]["key"]
elif len(userinfo["keys"]["keylist"].keys()) > 0:
key = userinfo["keys"]["keylist"].keys()[0]
if key:
keycontent = userinfo["keys"]["keylist"][key]
if keycontent.startswith('key '):
keycontent = keycontent[4:]
cm_keys_mongo(userid).check_register_key(userid, def_cloud, key, keycontent)
keynamenew = _keyname_sanitation(userid, key)
else:
Console.warning("No sshkey found. Please Upload one")
cookbook = arguments['MENU']
s_name = "launcher-{0}-{1}-{2}".format(userid, cookbook, get_rand_string())
dummy = "123456789" # doing nothing. just for test
t_url = \
"https://raw.githubusercontent.com/cloudmesh/cloudmesh/master/heat-templates/centos6/launcher/launcher.yaml"
param = {'KeyName': keynamenew,
'Cookbook': cookbook,
'dummy': dummy}
log.debug(def_cloud, userid, s_name, t_url, param)
res = self.cm_mongo.stack_create(cloud=def_cloud, cm_user_id=userid,
servername=s_name,
template_url=t_url,
parameters=param)
log.debug(res)
if 'error' in res:
print (res['error']['message'])
return res
elif arguments['stop'] and arguments['STACK_NAME']:
userid = self.cm_config.username()
def_cloud = self.get_cloud_name(userid)
s_id = arguments['STACK_NAME']
self.cm_mongo.activate(userid)
res = self.cm_mongo.stack_delete(cloud=def_cloud,
cm_user_id=userid,
server=s_id)
log.debug(res)
return res
elif arguments['import']:
filepath = config_file("/cloudmesh_launcher.yaml")
if arguments['FILEPATH']:
filepath = arguments['FILEPATH']
try:
filename = path_expand(filepath)
fileconfig = ConfigDict(filename=filename)
except Exception, err:
Console.error(
"error while loading '{0}', please check".format(filepath))
print (traceback.format_exc())
print (sys.exc_info()[0])
return
try:
recipes_dict = fileconfig.get("cloudmesh", "launcher", "recipies")
except:
Console.error("error while loading recipies from the file")
# print recipes_dict
userid = self.cm_config.username()
launcher_names = []
launchers = self.cm_mongo.launcher_get(userid)
for launcher in launchers:
launcher_names.append(launcher['cm_launcher'].encode("ascii"))
for key in recipes_dict:
if key in launcher_names:
if arguments['--force']:
self.cm_mongo.launcher_remove(userid, key)
self.cm_mongo.launcher_import(
recipes_dict[key], key, userid)
print ("launcher '{0}' overwritten.".format(key))
else:
print ("ERROR: launcher '{0}' exists, "
"please remove it first, or use "
"'--force' when adding".format(key))
else:
self.cm_mongo.launcher_import(
recipes_dict[key], key, userid)
print ("launcher '{0}' added.".format(key))
app = Flask(__name__)
api = restful.Api(app)
queue_stat_parser = reqparse.RequestParser()
queue_stat_parser.add_argument('resource', type=str, required=True)
#queue_stat_parser.add_argument('user', type=str)
queue_stat_parser.add_argument('id', type=str)
queue_info_parser = reqparse.RequestParser()
queue_info_parser.add_argument('resource', type=str, required=True)
#queue_info_parser.add_argument('user', type=str)
queue_info_parser.add_argument('queue', type=str)
config = ConfigDict(prefix="cloudmesh",
filename=config_file("/cloudmesh.yaml"))
user = config.get("cloudmesh.profile.username")
def versioned_url(url):
return "/" + version + url
def simple_error(kind, attribute, help="does not exist"):
msg = {}
msg["error:"] = "{0} {1} {2}".format(kind, attribute, help)
return msg
class rest_queue_stat(restful.Resource):
def get(self):
args = queue_stat_parser.parse_args()
class Test_cloudmesh:
d = {
"portalname": "gvonlasz"
}
filename = "etc/cloudmesh.yaml"
def setup(self):
self.t = cm_template(path_expand(self.filename))
self.user = cm_user()
try:
self.setup_inventory()
except:
print "=" * 40
print "setup_inventory() failed. ldap test will not be performed"
print "=" * 40
def tearDown(self):
os.system("fab test.info:user")
pass
def test_variables(self):
HEADING()
print self.t.variables()
assert "portalname" in self.t.variables()
def test_replace_incomplete(self):
try:
print self.t.replace(self.d, format="dict")
except:
pass
assert True
def test_user(self):
d = {
"portalname" : "gvonlasz",
"sierra_openstack_password" : "sierra",
"project_default" : "fg82",
"india_openstack_password" : "india",
"projects" : "82, 83", # this is still wrong
}
print self.t.replace(d=d)
# self.t.complete does not exist in cm_template?
# assert self.t.complete
# if not t.complete():
# print "ERROR: undefined variables"
# print t.variables()
def test_userinfo(self):
HEADING()
print self.user.info("fuwang")
print self.user.info("gvonlasz")
print self.user.info("nonexistuser")
print self.user.info("nova")
print self.user.info("fuwang", ["sierra_openstack_grizzly"])
print self.user.info("fuwang", ["cloud-non-exist"])
print "============================"
pprint (self.user["gvonlasz"])
print self.user.get_name('gvonlasz')
def setup_inventory(self):
banner("Read Dicts")
self.sample_user = ConfigDict(filename="~/.futuregrid/me.yaml")
self.portalname = self.sample_user.get("portalname")
print "PORTALNAME", self.portalname
print "SAMPLE USER", self.sample_user
banner("create user from template, duplicates cm init generate me")
t = cm_template("~/.futuregrid/etc/cloudmesh.yaml")
pprint (set(t.variables()))
self.config = t.replace(kind="dict", values=self.sample_user)
print type(self.config)
print self.config
#
# BUG?
#
self.idp = cm_userLDAP ()
self.idp.connect("fg-ldap", "ldap")
self.idp.refresh()
ldap_info = self.idp.get(self.portalname)
print ldap_info
print type(self.config)
self.config['cloudmesh']['projects'] = ldap_info['projects']
self.config['cloudmesh']['keys'] = ldap_info['keys']
try:
self.config['cloudmesh']['projects']['deafult'] = ldap_info['projects']['active'][0]
except:
print "ERROR: you have no projects"
def test_print(self):
pprint (self.config)
def test_projects(self):
projects = dict()
keys = dict()
# read the yaml
# read projects info from ldap
# write out new dict/json file
pass
def test_keys(self):
# read the yaml
# read KEYS from ldap
# write out new dict/json file
pass
def test_gregor(self):
banner("ME")
id = ConfigDict(filename="~/.futuregrid/me.yaml").get("portalname")
user = cm_user()
result = user.info(id)
pprint (result)
pass
def test_list(self):
user = cm_user()
list_of_users = user.list_users()
pprint (list_of_users)
print
print "========================="
num = len(list_of_users)
print str(num) + " users listed"
def test_mongo_credential(self):
banner("USER")
pprint (self.user.info("gvonlasz"))
username = "******"
cloudname = "dummy"
password = "******"
tennant = "fg1"
name = username
self.user.set_credential(username, cloudname,
{"OS_USERNAME": name,
"OS_PASSWORD": password,
"OS_TENANT_NAME": tennant,
"CM_CLOUD_TYPE": "openstack" }
)
cred = self.user.get_credential(username, cloudname)
banner("credential")
print cred
banner("credentials")
pprint(self.user.get_credentials(username))
banner("find")
result = self.user.userdb_passwd.find({})
for r in result:
pprint (r)
import mq
import time
import hostlist
from pprint import pprint
from cloudmesh.config.ConfigDict import ConfigDict
from cloudmesh.config.cm_config import cm_config_server
from cloudmesh_common.util import banner
from cloudmesh.provisioner.queue.celery import celery as p_queue
from cloudmesh.launcher.queue.celery import celery as l_queue
from cloudmesh.pbs.celery import celery as pbs_queue
from celery import Celery
__all__ = ["start", "stop", "list", "clean", "gui", "monitor", "kill", "ls", "lspbs"]
celery_config = ConfigDict(filename="~/.futuregrid/cloudmesh_celery.yaml", kind="worker")
workers = celery_config.get("cloudmesh.workers")
"""
for worker in workers:
workers[worker] = {"app":"cloudmesh.launcher{0}.queue",
"hostlist":hostlist.expand_hostlist("l[1-{0}]".format(workers[worker])),
"queue": worker}
"""
# no_workers_launcher =
for worker in workers:
workers[worker]["hostlist"] = hostlist.expand_hostlist(
"{0}[1-{1}]".format(workers[worker]["id"], workers[worker]["count"])
)
# launcher_workers = {"app":"cloudmesh.launcher.queue",
# "hostlist":hostlist.expand_hostlist("l[1-{0}]".format(workers["launcher"]["count"])),
def readClustersConfig(self, dir_yaml):
clusters_config = ConfigDict(
filename=dir_yaml + "/" + self.default_clusters_yaml)
self.dict_clusters_config = clusters_config.get("cloudmesh.inventory")
# get all possible cluster names from dict_clusters_config
self.cluster_name_list += self.dict_clusters_config.keys()
class CredentialFromYaml(CredentialBaseClass):
password = None
def __init__(self,
username,
cloud,
datasource=None,
yaml_version=2.0,
style=2.0,
password=None):
"""datasource is afilename"""
CredentialBaseClass.__init__(self, username, cloud, datasource)
self.password = password
if datasource != None:
self.filename = datasource
else:
self.filename = "~/.futuregrid/cloudmesh.yaml"
self.config = ConfigDict(filename=self.filename)
self.read(username, cloud, style=style)
def read(self, username, cloud, style=2.0):
self.style = style
self['cm'] = {}
self['cm']['source'] = 'yaml'
self['cm']['filename'] = self.filename
self['cm']['kind'] = self.config.get("meta.kind")
self['cm']['yaml_version'] = self.config.get("meta.yaml_version")
kind = self['cm']['kind']
if kind == "clouds":
self['cm']['filename'] = "~/.futuregrid/cloudmesh.yaml"
self.update(self.config.get("cloudmesh.clouds.{0}".format(cloud)))
elif kind == "server":
self['cm']['filename'] = "~/.futuregrid/cloudmesh_server.yaml"
self.update(self.config.get("cloudmesh.server.keystone.{0}".format(cloud)))
else:
log.error("kind wrong {0}".format(kind))
self['cm']['username'] = username
self['cm']['cloud'] = cloud
self.clean_cm()
self.transform_cm(self['cm']['yaml_version'], style)
# self.remove_default()
def clean_cm(self):
'''temporary so we do not have to modify yaml files for now'''
for key in self.keys():
if key.startswith('cm_'):
new_key = key.replace('cm_', '')
self['cm'][new_key] = self[key]
del self[key]
"""
def remove_default(self):
if 'default' in self.keys():
del self['default']
"""
def transform_cm(self, yaml_version, style):
if yaml_version <= 2.0 and style == 2.0:
for key in self['cm']:
new_key = 'cm_' + key
self[new_key] = self['cm'][key]
del self['cm']
class CredentialFromYaml(CredentialBaseClass):
password = None
def __init__(self,
username,
cloud,
datasource=None,
yaml_version=2.0,
style=2.0,
password=None):
"""datasource is afilename"""
CredentialBaseClass.__init__(self, username, cloud, datasource)
self.password = password
if datasource != None:
self.filename = datasource
else:
self.filename = "~/.futuregrid/cloudmesh.yaml"
self.config = ConfigDict(filename=self.filename)
self.read(username, cloud, style=style)
def read(self, username, cloud, style=2.0):
self.style = style
self['cm'] = {}
self['cm']['source'] = 'yaml'
self['cm']['filename'] = self.filename
self['cm']['kind'] = self.config.get("meta.kind")
self['cm']['yaml_version'] = self.config.get("meta.yaml_version")
kind = self['cm']['kind']
if kind == "clouds":
self['cm']['filename'] = "~/.futuregrid/cloudmesh.yaml"
self.update(self.config.get("cloudmesh.clouds.{0}".format(cloud)))
elif kind == "server":
self['cm']['filename'] = "~/.futuregrid/cloudmesh_server.yaml"
self.update(
self.config.get("cloudmesh.server.keystone.{0}".format(cloud)))
else:
log.error("kind wrong {0}".format(kind))
self['cm']['username'] = username
self['cm']['cloud'] = cloud
self.clean_cm()
self.transform_cm(self['cm']['yaml_version'], style)
# self.remove_default()
def clean_cm(self):
'''temporary so we do not have to modify yaml files for now'''
for key in self.keys():
if key.startswith('cm_'):
new_key = key.replace('cm_', '')
self['cm'][new_key] = self[key]
del self[key]
"""
def remove_default(self):
if 'default' in self.keys():
del self['default']
"""
def transform_cm(self, yaml_version, style):
if yaml_version <= 2.0 and style == 2.0:
for key in self['cm']:
new_key = 'cm_' + key
self[new_key] = self['cm'][key]
del self['cm']
