def get_ldap_user_from_yaml():
me = ConfigDict(filename=config_file("/me.yaml"))
d = {}
for element in ["firstname",
"lastname",
"email",
"phone",
"address"]:
d[element] = me.get("profile.{0}".format(element))
d["cm_user_id"] = me.get("portalname")
d["gidNumber"] = 0
d["uidNumber"] = 0
if "gidNumber" in me.keys():
d["gidNumber"] = me.get("gidNumber")
if "uidNumber" in me.keys():
d["uidNumber"] = me.get("uidNumber")
d["projects"] = me.get("projects")
#
# copy the keys
#
d['keys'] = me.get("keys.keylist")
return d
def setup_inventory(self):
banner("Read Dicts")
self.sample_user = ConfigDict(filename=config_file("/me.yaml"))
self.portalname = self.sample_user.get("portalname")
print("PORTALNAME", self.portalname)
print("SAMPLE USER", self.sample_user)
banner("create user from template, duplicates cm init generate me")
t = cm_template(config_file("/etc/cloudmesh.yaml"))
pprint(set(t.variables()))
self.config = t.replace(kind="dict", values=self.sample_user)
print(type(self.config))
print(self.config)
#
# BUG?
#
self.idp = cm_userLDAP()
self.idp.connect("fg-ldap", "ldap")
self.idp.refresh()
ldap_info = self.idp.get(self.portalname)
print(ldap_info)
print(type(self.config))
self.config['cloudmesh']['projects'] = ldap_info['projects']
self.config['cloudmesh']['keys'] = ldap_info['keys']
try:
self.config['cloudmesh']['projects']['deafult'] = ldap_info[
'projects']['active'][0]
except:
print("ERROR: you have no projects")
def readRackConfig(self, name, dir_yaml, dir_diag):
rack_config = ConfigDict(filename=dir_yaml + "/" +
self.default_rack_yaml)
self.dict_rack_config = rack_config.get("cloudmesh.rack")
lname = name.lower()
# the rack name in cloudmesh_rack.yaml MUST be valid/predefined in
# clouldmesh_cluster.yaml
self.cluster_name = lname if lname in self.cluster_name_list else self.cluster_name_unknown
# diag filename and temporary diag filename
self.filename_diag = dir_diag + "/" + \
self.dict_rack_config["cluster"][self.cluster_name]["diag"]
self.rack_count = self.dict_rack_config["cluster"][
self.cluster_name]["count"]
self.filename_diag_temp = self.filename_diag + ".temp"
# additional process for 'all'
# its count equals the sum of all other cluster exclude 'all' and
# 'unknown'
if name == "all":
raw_set = set(self.dict_rack_config["cluster"].keys())
exclude_set = set(["all", "unknown"])
all_set = raw_set - exclude_set
self.rack_count = 0
for rack in all_set:
self.rack_count += self.dict_rack_config["cluster"][rack][
"count"]
def readRackConfig(self, name, dir_yaml, dir_diag):
rack_config = ConfigDict(
filename=dir_yaml + "/" + self.default_rack_yaml)
self.dict_rack_config = rack_config.get("cloudmesh.rack")
lname = name.lower()
# the rack name in cloudmesh_rack.yaml MUST be valid/predefined in
# clouldmesh_cluster.yaml
self.cluster_name = lname if lname in self.cluster_name_list else self.cluster_name_unknown
# diag filename and temporary diag filename
self.filename_diag = dir_diag + "/" + \
self.dict_rack_config["cluster"][self.cluster_name]["diag"]
self.rack_count = self.dict_rack_config[
"cluster"][self.cluster_name]["count"]
self.filename_diag_temp = self.filename_diag + ".temp"
# additional process for 'all'
# its count equals the sum of all other cluster exclude 'all' and
# 'unknown'
if name == "all":
raw_set = set(self.dict_rack_config["cluster"].keys())
exclude_set = set(["all", "unknown"])
all_set = raw_set - exclude_set
self.rack_count = 0
for rack in all_set:
self.rack_count += self.dict_rack_config[
"cluster"][rack]["count"]
def display_launch_table():
launcher_config = ConfigDict(filename=config_file("/cloudmesh_launcher.yaml"))
launch_recipies = launcher_config.get("cloudmesh.launcher.recipies")
columns = launcher_config.get("cloudmesh.launcher.columns")
return render_template('mesh/mesh_launch.html',
recipies=launch_recipies,
columns=columns,
)
def test_getitem_server(self):
HEADING()
filename = "~/.futuregrid/cloudmesh_server.yaml"
config = ConfigDict(filename=filename)
print config
existing = config.get("cloudmesh.server.mongo.db")
test1 = existing is not None
print "QUERY", existing
print "Port", config.get("cloudmesh.server.mongo.port")
def test_getitem_server(self):
HEADING()
filename = config_file("/cloudmesh_server.yaml")
config = ConfigDict(filename=filename)
print(config)
existing = config.get("cloudmesh.server.mongo.db")
test1 = existing is not None
print("QUERY", existing)
print("Port", config.get("cloudmesh.server.mongo.port"))
def test_getitem_server(self):
HEADING()
filename = "~/.futuregrid/cloudmesh_server.yaml"
config = ConfigDict(filename=filename)
print config
existing = config.get("cloudmesh.server.mongo.db")
test1 = existing is not None
print "QUERY", existing
print "Port", config.get("cloudmesh.server.mongo.port")
def __init__(self):
# read the host file definition from cloudmesh_cluster.yaml
self.server_config = cm_config_server()
self.config = ConfigDict(filename=self.CONFIG_FILE)
self.bootspec_config = ConfigDict(filename=self.BOOTSPEC_FILE)
collection = "inventory"
self.db_inventory = get_mongo_db(collection)
def __init__(self, username, filename, Credential, style=2.0, password=None):
config = ConfigDict(filename=filename)
self.password = password
self[username] = {}
for cloud in config.get("cloudmesh.clouds").keys():
self[username][cloud] = Credential(username,
cloud,
filename,
style=style,
password=self.password)
self.encrypt(username, cloud, style)
def __init__(self, username, filename, Credential, style=2.0, password=None):
config = ConfigDict(filename=filename)
self.password = password
self[username] = {}
for cloud in config.get("cloudmesh.clouds").keys():
self[username][cloud] = Credential(username,
cloud,
filename,
style=style,
password=self.password)
self.encrypt(username, cloud, style)
def view(link=""):
"""run the browser"""
from cloudmesh.config.ConfigDict import ConfigDict
server_config = ConfigDict(filename=config_file("/cloudmesh_server.yaml"))
host = server_config.get("cloudmesh.server.webui.host")
port = server_config.get("cloudmesh.server.webui.port")
url_link = "http://{0}:{1}/{2}".format(host, port, link)
local("%s %s" % (web_browser, url_link))
def display_launch_table():
""" To satisfy Pylint. Will update with proper comments """
if request.method == 'POST':
print "HHHHHH", request.form.keys()
for key in request.form.keys():
print key, ":", request.form[key]
else:
print "HEY JUDE"
launcher_config = ConfigDict(
filename=config_file("/cloudmesh_launcher.yaml"))
launch_recipies = launcher_config.get("cloudmesh.launcher.recipies")
return render_template('mesh/mesh_launch.html', recipies=launch_recipies)
def view(link=""):
"""run the browser"""
from cloudmesh.config.ConfigDict import ConfigDict
server_config = ConfigDict(filename="~/.futuregrid/cloudmesh_server.yaml")
host = server_config.get("cloudmesh.server.webui.host")
port = server_config.get("cloudmesh.server.webui.port")
url_link = "http://{0}:{1}/{2}".format(host, port, link)
local("%s %s" % (web_browser, url_link))
def display_launch_table():
""" To satisfy Pylint. Will update with proper comments """
if request.method == 'POST':
print "HHHHHH", request.form.keys()
for key in request.form.keys():
print key, ":", request.form[key]
else:
print "HEY JUDE"
launcher_config = ConfigDict(
filename=config_file("/cloudmesh_launcher.yaml"))
launch_recipies = launcher_config.get("cloudmesh.launcher.recipies")
return render_template('mesh/mesh_launch.html',
recipies=launch_recipies)
def test_launcher(self):
HEADING()
filename = "~/.futuregrid/cloudmesh_launcher.yaml"
config = ConfigDict(filename=filename)
print config
existing = config.get("cloudmesh.launcher.recipies")
test1 = existing is not None
print existing
try:
none_existing = config.get("cloudmesh.launcher.recipies.xyz")
test2 = False
except:
print "Error"
test2 = True
assert test1 and test2
def test_server(self):
HEADING()
filename = "~/.futuregrid/cloudmesh_server.yaml"
config = ConfigDict(filename=filename)
# print config
existing = config.get("cloudmesh.server.mongo.db")
test1 = existing is not None
print "mongo.db =", existing
try:
none_existing = config.get("cloudmesh.server.mongo.xyz")
test2 = False
except:
print "Error"
test2 = True
assert test1 and test2
def test_server(self):
HEADING()
filename = "~/.futuregrid/cloudmesh_server.yaml"
config = ConfigDict(filename=filename)
# print config
existing = config.get("cloudmesh.server.mongo.db")
test1 = existing is not None
print "mongo.db =", existing
try:
none_existing = config.get("cloudmesh.server.mongo.xyz")
test2 = False
except:
print "Error"
test2 = True
assert test1 and test2
def test_launcher(self):
HEADING()
filename = config_file("/cloudmesh_launcher.yaml")
config = ConfigDict(filename=filename)
print(config)
existing = config.get("cloudmesh.launcher.recipies")
test1 = existing is not None
print(existing)
try:
none_existing = config.get("cloudmesh.launcher.recipies.xyz")
test2 = False
except:
print("Error")
test2 = True
assert test1 and test2
def test_launcher(self):
HEADING()
filename = "~/.futuregrid/cloudmesh_launcher.yaml"
config = ConfigDict(filename=filename)
print config
existing = config.get("cloudmesh.launcher.recipies")
test1 = existing is not None
print existing
try:
none_existing = config.get("cloudmesh.launcher.recipies.xyz")
test2 = False
except:
print "Error"
test2 = True
assert test1 and test2
class Test_cloudmesh:
username = ConfigDict(
filename=config_file("/cloudmesh.yaml")).get("cloudmesh.hpc.username")
filename = "etc/cloudmesh.yaml"
def setup(self):
self.idp = cm_userLDAP()
self.idp.connect("fg-ldap", "ldap")
self.idp.refresh()
def tearDown(self):
pass
def test_me(self):
print("USERNAME", self.username)
user = self.idp.find_one({'cm_user_id': self.username})
print(user)
def test_list(self):
users = self.idp.list()
pprint(users)
pprint(self.idp.users)
def test_auth(self):
password = getpass.getpass()
if self.idp.authenticate(self.username, password):
print("SUCCESS")
else:
print("FAILED")
def setup_inventory(self):
banner("Read Dicts")
self.sample_user = ConfigDict(filename="~/.futuregrid/me.yaml")
self.portalname = self.sample_user.get("portalname")
print "PORTALNAME", self.portalname
print "SAMPLE USER", self.sample_user
banner("create user from template, duplicates cm init generate me")
t = cm_template("~/.futuregrid/etc/cloudmesh.yaml")
pprint (set(t.variables()))
self.config = t.replace(kind="dict", values=self.sample_user)
print type(self.config)
print self.config
#
# BUG?
#
self.idp = cm_userLDAP ()
self.idp.connect("fg-ldap", "ldap")
self.idp.refresh()
ldap_info = self.idp.get(self.portalname)
print ldap_info
print type(self.config)
self.config['cloudmesh']['projects'] = ldap_info['projects']
self.config['cloudmesh']['keys'] = ldap_info['keys']
try:
self.config['cloudmesh']['projects']['deafult'] = ldap_info['projects']['active'][0]
except:
print "ERROR: you have no projects"
def test_gregor(self):
banner("ME")
id = ConfigDict(filename=config_file("/me.yaml")).get("portalname")
user = cm_user()
result = user.info(id)
pprint(result)
pass
def shell_command_open_web(arguments):
"""
Usage:
web [--fg|--cm] [LINK]
Arguments:
LINK the link on the localhost cm server is opened.
Options:
-v verbose mode
--fg opens a link on the FG portal
--cm opens a link on the CM portal
Description:
Opens a web page with the specified link
"""
link = arguments["LINK"]
if link is None or link == "/":
link = ""
web_browser = "firefox"
if sys.platform == 'darwin':
web_browser = "open"
if arguments["--fg"]:
location = "https://portal.futuregrid.org"
elif arguments["--cm"]:
location = "https://cloudmesh.futuregrid.org"
else:
try:
filename = config_file("/cloudmesh_server.yaml")
server_config = ConfigDict(filename=filename)
host = server_config.get("cloudmesh.server.webui.host")
port = server_config.get("cloudmesh.server.webui.port")
location = "http://{0}:{1}".format(host, port)
except Exception, e:
print("ERROR: some error reading from the config file")
print(e)
return
def shell_command_open_web(arguments):
"""
Usage:
web [--fg|--cm] [LINK]
Arguments:
LINK the link on the localhost cm server is opened.
Options:
-v verbose mode
--fg opens a link on the FG portal
--cm opens a link on the CM portal
Description:
Opens a web page with the specified link
"""
link = arguments["LINK"]
if link is None or link == "/":
link = ""
web_browser = "firefox"
if sys.platform == 'darwin':
web_browser = "open"
if arguments["--fg"]:
location = "https://portal.futuregrid.org"
elif arguments["--cm"]:
location = "https://cloudmesh.futuregrid.org"
else:
try:
filename = config_file("/cloudmesh_server.yaml")
server_config = ConfigDict(filename=filename)
host = server_config.get("cloudmesh.server.webui.host")
port = server_config.get("cloudmesh.server.webui.port")
location = "http://{0}:{1}".format(host, port)
except Exception, e:
print("ERROR: some error reading from the config file")
print(e)
return
def password():
user_config = cm_config(filename=config_file("/cloudmesh.yaml"))
user = user_config.cloud('india')['credentials']
server_config = ConfigDict(filename=config_file("/cloudmesh_server.yaml"))
server = server_config.get('cloudmesh.server.keystone.india')
print(" ".join([
"keystone", "--os-username", server['OS_USERNAME'], "--os-password",
server['OS_PASSWORD'], "--os-tenant-name", server['OS_TENANT_NAME'],
"--os-auth-url", server['OS_AUTH_URL'], "user-password-update",
"--pass", user['OS_PASSWORD'], user['OS_USERNAME']
]))
keystone("--os-username", server['OS_USERNAME'], "--os-password",
server['OS_PASSWORD'], "--os-tenant-name",
server['OS_TENANT_NAME'], "--os-auth-url", server['OS_AUTH_URL'],
"user-password-update", "--pass", user['OS_PASSWORD'],
user['OS_USERNAME'])
def touch(username=None):
"""clean the dirs"""
if username is None:
# bug for some reason the get method does not work
#useranme = ConfigDict(filename="~/.futuregrid/cloudmesh.yaml").get("cloudmesh.hpc.username")
username = ConfigDict(filename="~/.futuregrid/cloudmesh.yaml"
)["cloudmesh"]["hpc"]["username"]
print "Username: "******"india", "sierra", "alamo", "hotel", "foxtrot"]:
local("ssh %s@%s.futuregrid.org hostname -a" % (username, host))
def check(username=None):
"""clean the dirs"""
if username is None:
# bug for some reason the get method does not work
# useranme = ConfigDict(filename=config_file("/cloudmesh.yaml")).get("cloudmesh.hpc.username")
username = ConfigDict(filename=config_file(
"/cloudmesh.yaml"))["cloudmesh"]["hpc"]["username"]
print "Username: "******"india"]:
local("ssh %s@%s.futuregrid.org hostname -a" % (username, host))
def __init__(self):
config = ConfigDict(filename=config_file("/cloudmesh_server.yaml"))
port = config['cloudmesh']['server']['mongo']['port']
get_mongo_db("manage", DBConnFactory.TYPE_MONGOENGINE)
#db = connect('manage', port=port)
self.users = User.objects()
# thism may be wrong
meta = {"db_alias": "default"}
def metric_index():
metric = "metric-summary"
term = "last_3_months"
config = ConfigDict(filename=config_file(
"/cloudmesh_server.yaml"))["cloudmesh"]["server"]["metric"]
address = "{0}:{1}/{2}/{3}".format(config["host"], config["port"], metric,
term)
r = requests.get(address)
return render_template('/metric/index.html', data=r.text)
def __init__(self,
username,
cloud,
datasource=None,
yaml_version=2.0,
style=2.0,
password=None):
"""datasource is afilename"""
CredentialBaseClass.__init__(self, username, cloud, datasource)
self.password = password
if datasource != None:
self.filename = datasource
else:
self.filename = "~/.futuregrid/cloudmesh.yaml"
self.config = ConfigDict(filename=self.filename)
self.read(username, cloud, style=style)
def password():
user_config = cm_config(filename="~/.futuregrid/cloudmesh.yaml")
user = user_config.cloud('sierra_openstack_grizzly')['credentials']
server_config = ConfigDict(filename="~/.futuregrid/cloudmesh_server.yaml")
server = server_config.get('cloudmesh.server.keystone.sierra_openstack_grizzly')
print(" ".join(["keystone", "--os-username", server['OS_USERNAME'],
"--os-password", server['OS_PASSWORD'],
"--os-tenant-name", server['OS_TENANT_NAME'],
"--os-auth-url", server['OS_AUTH_URL'],
"user-password-update",
"--pass", user['OS_PASSWORD'], user['OS_USERNAME']]))
keystone("--os-username", server['OS_USERNAME'],
"--os-password", server['OS_PASSWORD'],
"--os-tenant-name", server['OS_TENANT_NAME'],
"--os-auth-url", server['OS_AUTH_URL'],
"user-password-update",
"--pass", user['OS_PASSWORD'], user['OS_USERNAME'])
def generate(self,
me_file,
out_file):
cloudmesh_yaml = path_expand(self.filename)
user_config = ConfigDict(filename=me_file)
t = cm_template(cloudmesh_yaml)
result = t.replace(kind="dict", values=user_config)
location = path_expand(out_file)
yaml_file = open(location, 'w+')
print >> yaml_file, yaml.dump(result, default_flow_style=False)
yaml_file.close()
log.info("Written new yaml file in " + location)
def metric_project(project_id):
config = ConfigDict(filename=config_file(
"/cloudmesh_server.yaml"))["cloudmesh"]["server"]["metric"]
base_url = "project-summary"
address = "{0}:{1}/{2}/{3}".format(config["host"], config["port"],
base_url, project_id)
try:
r = requests.get(address)
pprint(r.json())
data = dict(yaml.load(r.text))
data = data["message"]
return render_template('/metric/project.html', data=data)
except Exception, e:
print e
error = str(e)
return render_template('error.html', error=error)
def __init__(self, username, datasource=None, password=None):
"""datasource is afilename"""
dict.__init__(self)
self['username'] = username
self['datasource'] = datasource
self.password = password
if datasource != None:
self.filename = datasource
else:
self.filename = "~/.futuregrid/cloudmesh.yaml"
config = ConfigDict(filename=self.filename)
for key in config['cloudmesh']:
self[key] = config['cloudmesh'][key]
self['cm_user_id'] = username
def display_mongo_qstat_refresh(host=None):
celery_config = ConfigDict(filename=config_file("/cloudmesh_celery.yaml"))
log.info("qstat refresh request {0}".format(host))
# timeout = 15;
config = cm_config()
user = config["cloudmesh"]["hpc"]["username"]
if host is None:
hosts = [
"india.futuregrid.org", "lima.futuregrid.org",
"sierra.futuregrid.org", "hotel.futuregrid.org",
"alamo.futuregrid.org"
]
elif host in [
'bravo.futuregrid.org', 'echo.futuregrid.org',
'delta.futuregrid.org'
]:
hosts = ['india.futuregrid.org']
else:
hosts = [host]
error = ""
try:
pbs = pbs_mongo()
# queue = celery_config = celery_config.get("cloudmesh.workers.qstat.queue")
# res = tasks.refresh_qstat.apply_async(queue=queue, priority=0, args=[hosts])
for host in hosts:
pbs.activate(host, user)
pbs.refresh_qstat(host)
# error = res.get(timeout=timeout)
except Exception, e:
print traceback.format_exc()
error = "{0}".format(e)
log.error(error)
return render_template('error.html',
error=error,
type="Some error in qstat",
msg="")
def __init__(self,
username,
cloud,
datasource=None,
yaml_version=2.0,
style=2.0,
password=None):
"""datasource is afilename"""
CredentialBaseClass.__init__(self, username, cloud, datasource)
self.password = password
if datasource != None:
self.filename = datasource
else:
self.filename = "~/.futuregrid/cloudmesh.yaml"
self.config = ConfigDict(filename=self.filename)
self.read(username, cloud, style=style)
def display_mongo_qstat_refresh(host=None):
celery_config = ConfigDict(filename=config_file("/cloudmesh_celery.yaml"))
log.info("qstat refresh request {0}".format(host))
# timeout = 15;
config = cm_config()
user = config["cloudmesh"]["hpc"]["username"]
if host is None:
hosts = ["india.futuregrid.org"]
elif host in [
'bravo.futuregrid.org', 'echo.futuregrid.org',
'delta.futuregrid.org'
]:
hosts = ['india.futuregrid.org']
else:
hosts = [host]
error = ""
try:
pbs = pbs_mongo()
# queue = celery_config = celery_config.get("cloudmesh.workers.qstat.queue")
# res = tasks.refresh_qstat.apply_async(queue=queue, priority=0, args=[hosts])
for host in hosts:
pbs.activate(host, user)
log.info("refresh qstat: {0} {1}".format(host, user))
pbs.refresh_qstat(host)
# error = res.get(timeout=timeout)
except Exception, e:
print traceback.format_exc()
error = "{0}".format(e)
log.error(error)
category = "qstat-{0}".format(host)
flash(error, category=str(category))
def build_JSON(s_time, e_time, label, hosts):
configFile = ConfigDict(filename="~/.futuregrid/me.yaml")
jsonData = JSONEncoder().encode(
{
"summary": label,
"description": {
"hosts": hosts,
"kind": "vm-server",
"project": configFile["projects"]["default"],
"userid": configFile["profile"]["id"],
"displayName": configFile["profile"]["firstname"],
"email": configFile["profile"]["email"]
},
"start": {
"dateTime": s_time,
"timeZone": "America/New_York"
},
"end": {
"dateTime": e_time,
"timeZone": "America/New_York"
}
})
return jsonData
def launch_servers():
launcher_config = ConfigDict(filename=config_file("/cloudmesh_launcher.yaml"))
celery_config = ConfigDict(filename=config_file("/cloudmesh_celery.yaml"))
launch_recipies = launcher_config.get("cloudmesh.launcher.recipies")
server = request.form.get("server")
name_index = request.form.get("name_index")
return_dict = dict(launch_recipies[server][int(name_index)])
return_dict["user"] = "******" # default user.... change to read from session
return_dict["server"] = server
parameters = launch_recipies[server][int(name_index)]["parameters"] # load the needed set of parameters
for parameter in parameters:
parameters[parameter] = request.form.get("parameter_{0}".format(parameter))
return_dict["parameters"] = parameters
return_dict["name"] = launch_recipies[server][int(name_index)]["name"]
queue = celery_config = celery_config.get("cloudmesh.workers.launcher.queue")
task_launch.apply_async(queue=queue, args=[return_dict])
return "Task has been submitted to the queue.... <br/><br/>Data sent was:<br/>" + str(return_dict)
def __init__(self, filename=None):
if filename is None:
filename = self.filename
ConfigDict.__init__(self, filename=filename, kind="launcher")
from cloudmesh_task.tasks import cm_ssh
from cloudmesh_task.parallel import Parallel, Sequential
from cloudmesh.util.stopwatch import StopWatch
from cloudmesh import banner
from pprint import pprint
from cloudmesh.config.cm_config import cm_config
from cloudmesh.config.ConfigDict import ConfigDict
import sys
config = ConfigDict(
filename="~/.cloudmesh/cloudmesh_hpc.yaml")["cloudmesh"]["hpc"]
print config
#hosts = []
# hosts.append("localhost")
# hosts.append("bigred2.uits.iu.edu")
# hosts.append("india.futuregrid.org")
def get_credentials(hosts):
credential = {}
for host in hosts:
credential[host] = config[host]['username']
return credential
hosts = config.keys()
credentials = get_credentials(hosts)
def do_launcher(self, args, arguments):
"""
Usage:
launcher start MENU
launcher stop STACK_NAME
launcher list
launcher show STACK_NAME
launcher menu [--column=COLUMN] [--format=FORMAT]
launcher import [FILEPATH] [--force]
launcher export FILEPATH
launcher help | -h
An orchestration tool with Chef Cookbooks
Arguments:
MENU Name of a cookbook
STACK_NAME Name of a launcher
FILEPATH Filepath
COLUMN column name to display
FORMAT display format (json, table)
help Prints this message
Options:
-v verbose mode
"""
log.info(arguments)
self.cm_mongo = cm_mongo()
self.cm_config = cm_config()
self.user = cm_user()
if arguments["help"] or arguments["-h"]:
print (self.do_launcher.__doc__)
elif arguments['show'] and arguments['STACK_NAME']:
print ("NOT IMPLEMENTED")
return
elif arguments['menu']:
userid = self.cm_config.username()
launchers = self.cm_mongo.launcher_get(userid)
if launchers.count() == 0:
Console.warning("no launcher in database, please import launcher first"
"(launcher import [FILEPATH] [--force])")
return
else:
d = {}
for launcher in launchers:
d[launcher['cm_launcher']] = launcher
if "_id" in d[launcher['cm_launcher']]:
del d[launcher['cm_launcher']]['_id']
columns = None
if arguments['--column']:
if arguments['--column'] != "all":
columns = [x.strip() for x in arguments['--column'].split(',')]
else:
columns = ['name', 'description']
if arguments['--format']:
if arguments['--format'] not in ['table', 'json', 'csv']:
Console.error("please select printing format ",
"among table, json and csv")
return
else:
p_format = arguments['--format']
else:
p_format = None
shell_commands_dict_output(userid,
d,
print_format=p_format,
firstheader="launcher",
header=columns
# vertical_table=True
)
elif arguments['list']:
userid = self.cm_config.username()
self.cm_mongo.activate(userid)
self.cm_mongo.refresh(cm_user_id=userid, types=[self._id])
stacks = self.cm_mongo.stacks(cm_user_id=userid)
launchers = self.filter_launcher(
stacks,
{"search": "contain",
"key": "stack_name",
"value": "launcher"}
)
log.debug(launchers)
d = {}
for k0, v0 in launchers.iteritems():
for k1, v1 in launchers[k0].iteritems():
d[v1['id']] = v1
columns = ['stack_name', 'description', 'stack_status',
'creation_time', 'cm_cloud']
if arguments['--column'] and arguments['--column'] != "all":
columns = [x.strip() for x in arguments['--column'].split(',')]
if arguments['--format']:
if arguments['--format'] not in ['table', 'json', 'csv']:
Console.error("please select printing format among table, json and csv")
return
else:
p_format = arguments['--format']
else:
p_format = None
shell_commands_dict_output(userid,
d,
print_format=p_format,
firstheader="launcher_id",
header=columns
# vertical_table=True
)
elif arguments['start'] and arguments['MENU']:
userid = self.cm_config.username()
def_cloud = self.get_cloud_name(userid)
self.cm_mongo.activate(userid)
userinfo = self.user.info(userid)
if "key" in userinfo["defaults"]:
key = userinfo["defaults"]["key"]
elif len(userinfo["keys"]["keylist"].keys()) > 0:
key = userinfo["keys"]["keylist"].keys()[0]
if key:
keycontent = userinfo["keys"]["keylist"][key]
if keycontent.startswith('key '):
keycontent = keycontent[4:]
cm_keys_mongo(userid).check_register_key(userid, def_cloud, key, keycontent)
keynamenew = _keyname_sanitation(userid, key)
else:
Console.warning("No sshkey found. Please Upload one")
cookbook = arguments['MENU']
s_name = "launcher-{0}-{1}-{2}".format(userid, cookbook, get_rand_string())
dummy = "123456789" # doing nothing. just for test
t_url = \
"https://raw.githubusercontent.com/cloudmesh/cloudmesh/master/heat-templates/centos6/launcher/launcher.yaml"
param = {'KeyName': keynamenew,
'Cookbook': cookbook,
'dummy': dummy}
log.debug(def_cloud, userid, s_name, t_url, param)
res = self.cm_mongo.stack_create(cloud=def_cloud, cm_user_id=userid,
servername=s_name,
template_url=t_url,
parameters=param)
log.debug(res)
if 'error' in res:
print (res['error']['message'])
return res
elif arguments['stop'] and arguments['STACK_NAME']:
userid = self.cm_config.username()
def_cloud = self.get_cloud_name(userid)
s_id = arguments['STACK_NAME']
self.cm_mongo.activate(userid)
res = self.cm_mongo.stack_delete(cloud=def_cloud,
cm_user_id=userid,
server=s_id)
log.debug(res)
return res
elif arguments['import']:
filepath = config_file("/cloudmesh_launcher.yaml")
if arguments['FILEPATH']:
filepath = arguments['FILEPATH']
try:
filename = path_expand(filepath)
fileconfig = ConfigDict(filename=filename)
except Exception, err:
Console.error(
"error while loading '{0}', please check".format(filepath))
print (traceback.format_exc())
print (sys.exc_info()[0])
return
try:
recipes_dict = fileconfig.get("cloudmesh", "launcher", "recipies")
except:
Console.error("error while loading recipies from the file")
# print recipes_dict
userid = self.cm_config.username()
launcher_names = []
launchers = self.cm_mongo.launcher_get(userid)
for launcher in launchers:
launcher_names.append(launcher['cm_launcher'].encode("ascii"))
for key in recipes_dict:
if key in launcher_names:
if arguments['--force']:
self.cm_mongo.launcher_remove(userid, key)
self.cm_mongo.launcher_import(
recipes_dict[key], key, userid)
print ("launcher '{0}' overwritten.".format(key))
else:
print ("ERROR: launcher '{0}' exists, "
"please remove it first, or use "
"'--force' when adding".format(key))
else:
self.cm_mongo.launcher_import(
recipes_dict[key], key, userid)
print ("launcher '{0}' added.".format(key))
def __init__(self, filename=None):
if filename is None:
filename = self.filename
ConfigDict.__init__(self, filename=filename)
# provider = pbs_mongo
app = Flask(__name__)
api = restful.Api(app)
queue_stat_parser = reqparse.RequestParser()
queue_stat_parser.add_argument('resource', type=str, required=True)
#queue_stat_parser.add_argument('user', type=str)
queue_stat_parser.add_argument('id', type=str)
queue_info_parser = reqparse.RequestParser()
queue_info_parser.add_argument('resource', type=str, required=True)
#queue_info_parser.add_argument('user', type=str)
queue_info_parser.add_argument('queue', type=str)
config = ConfigDict(
prefix="cloudmesh", filename=config_file("/cloudmesh.yaml"))
user = config.get("cloudmesh.profile.username")
def versioned_url(url):
return "/" + version + url
def simple_error(kind, attribute, help="does not exist"):
msg = {}
msg["error:"] = "{0} {1} {2}".format(kind, attribute, help)
return msg
class rest_queue_stat(restful.Resource):
import clean
import mq
import time
import hostlist
from pprint import pprint
from cloudmesh.config.ConfigDict import ConfigDict
from cloudmesh.config.cm_config import cm_config_server
from cloudmesh_common.util import banner
from cloudmesh.provisioner.queue.celery import celery as p_queue
from cloudmesh.launcher.queue.celery import celery as l_queue
from cloudmesh.pbs.celery import celery as pbs_queue
from celery import Celery
__all__ = ["start", "stop", "list", "clean", "gui", "monitor", "kill", "ls", "lspbs"]
celery_config = ConfigDict(filename="~/.futuregrid/cloudmesh_celery.yaml", kind="worker")
workers = celery_config.get("cloudmesh.workers")
"""
for worker in workers:
workers[worker] = {"app":"cloudmesh.launcher{0}.queue",
"hostlist":hostlist.expand_hostlist("l[1-{0}]".format(workers[worker])),
"queue": worker}
"""
# no_workers_launcher =
for worker in workers:
workers[worker]["hostlist"] = hostlist.expand_hostlist(
"{0}[1-{1}]".format(workers[worker]["id"], workers[worker]["count"])
)
# launcher_workers = {"app":"cloudmesh.launcher.queue",
class Test_cloudmesh:
d = {
"portalname": "gvonlasz"
}
filename = "etc/cloudmesh.yaml"
def setup(self):
self.t = cm_template(path_expand(self.filename))
self.user = cm_user()
try:
self.setup_inventory()
except:
print "=" * 40
print "setup_inventory() failed. ldap test will not be performed"
print "=" * 40
def tearDown(self):
os.system("fab test.info:user")
pass
def test_variables(self):
HEADING()
print self.t.variables()
assert "portalname" in self.t.variables()
def test_replace_incomplete(self):
try:
print self.t.replace(self.d, format="dict")
except:
pass
assert True
def test_user(self):
d = {
"portalname" : "gvonlasz",
"sierra_openstack_password" : "sierra",
"project_default" : "fg82",
"india_openstack_password" : "india",
"projects" : "82, 83", # this is still wrong
}
print self.t.replace(d=d)
# self.t.complete does not exist in cm_template?
# assert self.t.complete
# if not t.complete():
# print "ERROR: undefined variables"
# print t.variables()
def test_userinfo(self):
HEADING()
print self.user.info("fuwang")
print self.user.info("gvonlasz")
print self.user.info("nonexistuser")
print self.user.info("nova")
print self.user.info("fuwang", ["sierra_openstack_grizzly"])
print self.user.info("fuwang", ["cloud-non-exist"])
print "============================"
pprint (self.user["gvonlasz"])
print self.user.get_name('gvonlasz')
def setup_inventory(self):
banner("Read Dicts")
self.sample_user = ConfigDict(filename="~/.futuregrid/me.yaml")
self.portalname = self.sample_user.get("portalname")
print "PORTALNAME", self.portalname
print "SAMPLE USER", self.sample_user
banner("create user from template, duplicates cm init generate me")
t = cm_template("~/.futuregrid/etc/cloudmesh.yaml")
pprint (set(t.variables()))
self.config = t.replace(kind="dict", values=self.sample_user)
print type(self.config)
print self.config
#
# BUG?
#
self.idp = cm_userLDAP ()
self.idp.connect("fg-ldap", "ldap")
self.idp.refresh()
ldap_info = self.idp.get(self.portalname)
print ldap_info
print type(self.config)
self.config['cloudmesh']['projects'] = ldap_info['projects']
self.config['cloudmesh']['keys'] = ldap_info['keys']
try:
self.config['cloudmesh']['projects']['deafult'] = ldap_info['projects']['active'][0]
except:
print "ERROR: you have no projects"
def test_print(self):
pprint (self.config)
def test_projects(self):
projects = dict()
keys = dict()
# read the yaml
# read projects info from ldap
# write out new dict/json file
pass
def test_keys(self):
# read the yaml
# read KEYS from ldap
# write out new dict/json file
pass
def test_gregor(self):
banner("ME")
id = ConfigDict(filename="~/.futuregrid/me.yaml").get("portalname")
user = cm_user()
result = user.info(id)
pprint (result)
pass
def test_list(self):
user = cm_user()
list_of_users = user.list_users()
pprint (list_of_users)
print
print "========================="
num = len(list_of_users)
print str(num) + " users listed"
def test_mongo_credential(self):
banner("USER")
pprint (self.user.info("gvonlasz"))
username = "******"
cloudname = "dummy"
password = "******"
tennant = "fg1"
name = username
self.user.set_credential(username, cloudname,
{"OS_USERNAME": name,
"OS_PASSWORD": password,
"OS_TENANT_NAME": tennant,
"CM_CLOUD_TYPE": "openstack" }
)
cred = self.user.get_credential(username, cloudname)
banner("credential")
print cred
banner("credentials")
pprint(self.user.get_credentials(username))
banner("find")
result = self.user.userdb_passwd.find({})
for r in result:
pprint (r)
def readClustersConfig(self, dir_yaml):
clusters_config = ConfigDict(
filename=dir_yaml + "/" + self.default_clusters_yaml)
self.dict_clusters_config = clusters_config.get("cloudmesh.inventory")
# get all possible cluster names from dict_clusters_config
self.cluster_name_list += self.dict_clusters_config.keys()
class CredentialFromYaml(CredentialBaseClass):
password = None
def __init__(self,
username,
cloud,
datasource=None,
yaml_version=2.0,
style=2.0,
password=None):
"""datasource is afilename"""
CredentialBaseClass.__init__(self, username, cloud, datasource)
self.password = password
if datasource != None:
self.filename = datasource
else:
self.filename = "~/.futuregrid/cloudmesh.yaml"
self.config = ConfigDict(filename=self.filename)
self.read(username, cloud, style=style)
def read(self, username, cloud, style=2.0):
self.style = style
self['cm'] = {}
self['cm']['source'] = 'yaml'
self['cm']['filename'] = self.filename
self['cm']['kind'] = self.config.get("meta.kind")
self['cm']['yaml_version'] = self.config.get("meta.yaml_version")
kind = self['cm']['kind']
if kind == "clouds":
self['cm']['filename'] = "~/.futuregrid/cloudmesh.yaml"
self.update(self.config.get("cloudmesh.clouds.{0}".format(cloud)))
elif kind == "server":
self['cm']['filename'] = "~/.futuregrid/cloudmesh_server.yaml"
self.update(self.config.get("cloudmesh.server.keystone.{0}".format(cloud)))
else:
log.error("kind wrong {0}".format(kind))
self['cm']['username'] = username
self['cm']['cloud'] = cloud
self.clean_cm()
self.transform_cm(self['cm']['yaml_version'], style)
# self.remove_default()
def clean_cm(self):
'''temporary so we do not have to modify yaml files for now'''
for key in self.keys():
if key.startswith('cm_'):
new_key = key.replace('cm_', '')
self['cm'][new_key] = self[key]
del self[key]
"""
def remove_default(self):
if 'default' in self.keys():
del self['default']
"""
def transform_cm(self, yaml_version, style):
if yaml_version <= 2.0 and style == 2.0:
for key in self['cm']:
new_key = 'cm_' + key
self[new_key] = self['cm'][key]
del self['cm']
