def user_apply():
form = UserRegistrationForm(request.form)
if request.method == 'POST': #and form.validate():
data = dict(request.form)
action = str(data['button'][0])
for key in data:
data[key] = data[key][0]
del data['button']
if action == 'save':
users = Users()
user = MongoUser()
del data['confirm']
for d in data:
user[d] = data[d]
users.add(user)
flash('Thanks for registering')
return redirect('/')
user_config = ConfigDict(filename=config_file("/cloudmesh_user_intf.yaml"))
user_fields = user_config.get("cloudmesh.user")
return render_template('management/user_apply.html',
title="User Application",
states=['save', 'cancel'],
fields=user_fields,
countries_list=[c for c in country_list()],
roles_list=roles)
def setup_inventory(self):
banner("Read Dicts")
self.sample_user = ConfigDict(filename=config_file("/me.yaml"))
self.portalname = self.sample_user.get("portalname")
print("PORTALNAME", self.portalname)
print("SAMPLE USER", self.sample_user)
banner("create user from template, duplicates cm init generate me")
t = cm_template(config_file("/etc/cloudmesh.yaml"))
pprint(set(t.variables()))
self.config = t.replace(kind="dict", values=self.sample_user)
print(type(self.config))
print(self.config)
#
# BUG?
#
self.idp = cm_userLDAP()
self.idp.connect("fg-ldap", "ldap")
self.idp.refresh()
ldap_info = self.idp.get(self.portalname)
print(ldap_info)
print(type(self.config))
self.config['cloudmesh']['projects'] = ldap_info['projects']
self.config['cloudmesh']['keys'] = ldap_info['keys']
try:
self.config['cloudmesh']['projects'][
'deafult'] = ldap_info['projects']['active'][0]
except:
print("ERROR: you have no projects")
def create_user_from_file(cls, file_path):
try:
filename = path_expand(file_path)
file_config = ConfigDict(filename=filename)
except:
Console.error("Could not load file, please check filename and its path")
return
try:
user_config = file_config.get("cloudmesh", "user")
user_name = user_config['username']
user = User()
update_document(user, user_config)
except:
Console.error("Could not get user information from yaml file, "
"please check you yaml file, users information must be "
"under 'cloudmesh' -> 'users' -> user1...")
return
try:
if cls.check_exists(user_name) is False:
cls.add(user)
Console.info("User created in the database.")
else:
Console.error("User with user name " + user_name + " already exists.")
return
except:
Console.error("User creation in database failed, " + str(sys.exc_info()))
return
def create_project_from_file(cls, file_path):
# implement()
# return
try:
filename = path_expand(file_path)
file_config = ConfigDict(filename=filename)
except:
Console.error("Could not load file, please check filename and its path")
return
try:
project_config = file_config.get("cloudmesh", "project")
project = Project()
project_id = uuid.uuid4()
project_config.update({'project_id': project_id})
update_document(project, project_config)
except:
Console.error("Could not get project information from yaml file, "
"please check you yaml file, users information must be "
"under 'cloudmesh' -> 'project' -> project1..." + str(sys.exc_info()[0]))
return
try:
cls.add(project)
Console.info("Project created in the database.")
except:
Console.error("Project creation in database failed, " + str(sys.exc_info()))
return
def user_apply():
form = UserRegistrationForm(request.form)
if request.method == 'POST': #and form.validate():
data = dict(request.form)
action = str(data['button'][0])
for key in data:
data[key] = data[key][0]
del data['button']
if action == 'save':
users = Users()
user = MongoUser()
del data['confirm']
for d in data:
user[d] = data[d]
users.add(user)
flash('Thanks for registering')
return redirect('/')
user_config = ConfigDict(filename=config_file("/cloudmesh_user_intf.yaml"))
user_fields = user_config.get("cloudmesh.user")
return render_template('management/user_apply.html',
title="User Application",
states=['save', 'cancel'],
fields=user_fields,
countries_list=[c for c in country_list()],
roles_list=roles)
def readRackConfig(self, name, dir_yaml, dir_diag):
rack_config = ConfigDict(filename=dir_yaml + "/" +
self.default_rack_yaml)
self.dict_rack_config = rack_config.get("cloudmesh.rack")
lname = name.lower()
# the rack name in cloudmesh_rack.yaml MUST be valid/predefined in
# clouldmesh_cluster.yaml
self.cluster_name = lname if lname in self.cluster_name_list else self.cluster_name_unknown
# diag filename and temporary diag filename
self.filename_diag = dir_diag + "/" + \
self.dict_rack_config["cluster"][self.cluster_name]["diag"]
self.rack_count = self.dict_rack_config["cluster"][
self.cluster_name]["count"]
self.filename_diag_temp = self.filename_diag + ".temp"
# additional process for 'all'
# its count equals the sum of all other cluster exclude 'all' and
# 'unknown'
if name == "all":
raw_set = set(self.dict_rack_config["cluster"].keys())
exclude_set = set(["all", "unknown"])
all_set = raw_set - exclude_set
self.rack_count = 0
for rack in all_set:
self.rack_count += self.dict_rack_config["cluster"][rack][
"count"]
def readRackConfig(self, name, dir_yaml, dir_diag):
rack_config = ConfigDict(
filename=dir_yaml + "/" + self.default_rack_yaml)
self.dict_rack_config = rack_config.get("cloudmesh.rack")
lname = name.lower()
# the rack name in cloudmesh_rack.yaml MUST be valid/predefined in
# clouldmesh_cluster.yaml
self.cluster_name = lname if lname in self.cluster_name_list else self.cluster_name_unknown
# diag filename and temporary diag filename
self.filename_diag = dir_diag + "/" + \
self.dict_rack_config["cluster"][self.cluster_name]["diag"]
self.rack_count = self.dict_rack_config[
"cluster"][self.cluster_name]["count"]
self.filename_diag_temp = self.filename_diag + ".temp"
# additional process for 'all'
# its count equals the sum of all other cluster exclude 'all' and
# 'unknown'
if name == "all":
raw_set = set(self.dict_rack_config["cluster"].keys())
exclude_set = set(["all", "unknown"])
all_set = raw_set - exclude_set
self.rack_count = 0
for rack in all_set:
self.rack_count += self.dict_rack_config[
"cluster"][rack]["count"]
def get_ldap_user_from_yaml():
me = ConfigDict(filename=config_file("/me.yaml"))
d = {}
for element in ["firstname",
"lastname",
"email",
"phone",
"address"]:
d[element] = me.get("profile.{0}".format(element))
d["cm_user_id"] = me.get("portalname")
d["gidNumber"] = 0
d["uidNumber"] = 0
if "gidNumber" in me.keys():
d["gidNumber"] = me.get("gidNumber")
if "uidNumber" in me.keys():
d["uidNumber"] = me.get("uidNumber")
d["projects"] = me.get("projects")
#
# copy the keys
#
d['keys'] = me.get("keys.keylist")
return d
def test_getitem_server(self):
HEADING()
filename = config_file("/cloudmesh_server.yaml")
config = ConfigDict(filename=filename)
print(config)
existing = config.get("cloudmesh.server.mongo.db")
test1 = existing is not None
print("QUERY", existing)
print("Port", config.get("cloudmesh.server.mongo.port"))
def set_debug(self, on):
filename = path_expand("~/.cloudmesh/cmd3.yaml")
config = ConfigDict(filename=filename)
if type(on) == bool:
self.debug = on
else:
self.debug = on.lower() in ['on', 'true']
config["cmd3"]["properties"]["debug"] = self.debug
Console.ok("Debug mode is {:}".format(self.debug))
config.write(filename=filename, output="yaml", attribute_indent=" ")
def __init__(self, username, filename, Credential, style=2.0, password=None):
config = ConfigDict(filename=filename)
self.password = password
self[username] = {}
for cloud in config.get("cloudmesh.clouds").keys():
self[username][cloud] = Credential(username,
cloud,
filename,
style=style,
password=self.password)
self.encrypt(username, cloud, style)
def set_debug(self, on):
filename = path_expand("~/.cloudmesh/cmd3.yaml")
config = ConfigDict(filename=filename)
if type(on) == bool:
self.debug = on
else:
self.debug = on.lower() in ['on', 'true']
config["cmd3"]["properties"]["debug"] = self.debug
Console.ok("Debug mode is {:}".format(self.debug))
config.write(filename=filename, output="yaml", attribute_indent=" ")
def view(link=""):
"""run the browser"""
from cloudmesh_base.ConfigDict import ConfigDict
server_config = ConfigDict(filename=config_file("/cloudmesh_server.yaml"))
host = server_config.get("cloudmesh.server.webui.host")
port = server_config.get("cloudmesh.server.webui.port")
url_link = "http://{0}:{1}/{2}".format(host, port, link)
local("%s %s" % (web_browser, url_link))
def view(link=""):
"""run the browser"""
from cloudmesh_base.ConfigDict import ConfigDict
server_config = ConfigDict(filename=config_file("/cloudmesh_server.yaml"))
host = server_config.get("cloudmesh.server.webui.host")
port = server_config.get("cloudmesh.server.webui.port")
url_link = "http://{0}:{1}/{2}".format(host, port, link)
local("%s %s" % (web_browser, url_link))
def project_apply():
form = ProjectRegistrationForm(request.form)
if request.method == 'POST' and form.validate():
data = dict(request.form)
action = str(data['button'][0])
print "Project Data"
print data
for key in data:
if key in [
'agreement_use', 'agreement_slides', 'agreement_support',
'agreement_software', 'agreement_documentation',
'join_open', 'join_notification'
]:
if str(data[key][0]) == 'y':
data[key] = True
elif str(data[key][0]) == 'n':
data[key] = False
elif key in [
'category', 'managers', 'resources_clusters', 'alumni',
'resources_provision', 'resources_services', 'members',
'keywords'
]:
print data[key][0]
else:
data[key] = data[key][0]
print str(key) + " - " + str(data[key]) + "\n"
del data['button']
if action == 'save':
projects = Projects()
project = MongoProject()
for d in data:
project[d] = data[d]
print project
projects.add(project)
flash('Thanks for registering')
return redirect('/')
project_config = ConfigDict(
filename=config_file("/cloudmesh_project_intf.yaml"))
project_fields = project_config.get("cloudmesh.project")
return render_template('management/project_apply.html',
title="Project Application",
states=['save', 'cancel'],
fields=project_fields,
countries_list=[c for c in country_list()],
states_list=[c for c in states_list()],
disciplines_list=[c for c in disciplines_list()])
def load(self, yaml_filename=None):
"""
Loads the cloudmesh pbs yaml file.
:param yaml_filename: The filename of the yaml file
"""
log.debug("PBS yaml filename: {0}".format(self.yaml_filename))
if yaml_filename is None:
yaml_filename = self.yaml_filename
else:
self.yaml_filename = config_file(yaml_filename)
self.data = ConfigDict(filename=self.yaml_filename)
self.hosts = ssh_config()
def display_launch_table():
""" To satisfy Pylint. Will update with proper comments """
if request.method == 'POST':
print "HHHHHH", request.form.keys()
for key in request.form.keys():
print key, ":", request.form[key]
else:
print "HEY JUDE"
launcher_config = ConfigDict(
filename=config_file("/cloudmesh_launcher.yaml"))
launch_recipies = launcher_config.get("cloudmesh.launcher.recipies")
return render_template('mesh/mesh_launch.html', recipies=launch_recipies)
def __init__(self, quiet=False):
self.quiet = quiet
try:
self.filename = config_file("/cmd3.yaml")
if not self.quiet:
Console.ok("Reading " + self.filename + ". ok.")
except:
Console.error("problem loading cmd3.yaml file")
try:
self.config = ConfigDict(filename=self.filename)
# print(self.config)
except Exception as e:
Console.error("problem with ConfigDict")
print(e)
def display_launch_table():
""" To satisfy Pylint. Will update with proper comments """
if request.method == 'POST':
print "HHHHHH", request.form.keys()
for key in request.form.keys():
print key, ":", request.form[key]
else:
print "HEY JUDE"
launcher_config = ConfigDict(
filename=config_file("/cloudmesh_launcher.yaml"))
launch_recipies = launcher_config.get("cloudmesh.launcher.recipies")
return render_template('mesh/mesh_launch.html',
recipies=launch_recipies)
def test_launcher(self):
HEADING()
filename = config_file("/cloudmesh_launcher.yaml")
config = ConfigDict(filename=filename)
print(config)
existing = config.get("cloudmesh.launcher.recipies")
test1 = existing is not None
print(existing)
try:
none_existing = config.get("cloudmesh.launcher.recipies.xyz")
test2 = False
except:
print("Error")
test2 = True
assert test1 and test2
def test_server(self):
HEADING()
filename = config_file("/cloudmesh_server.yaml")
config = ConfigDict(filename=filename)
# print config
existing = config.get("cloudmesh.server.mongo.db")
test1 = existing is not None
print("mongo.db =", existing)
try:
none_existing = config.get("cloudmesh.server.mongo.xyz")
test2 = False
except:
print("Error")
test2 = True
assert test1 and test2
class PBSDeploy(object):
def load(self):
self.filename = config_file("/cloudmesh_pbs.yaml")
self.data = ConfigDict(filename=self.filename)
def __init__(self, deploy=False):
if deploy:
self.deploy()
self.load()
def __str__(self):
return self.data.json()
@staticmethod
def deploy(force=True):
"""copies the yaml file from etc in the distribution to the .cloudmesh
directory. If the file exits it will not be copied and a warning is
thrown. If the file is the same as in etc no warning is thrown.
"""
xcopy("../etc/", "~/.cloudmesh", "*.yaml", force=force)
def servers(self):
return self.data["cloudmesh"]["pbs"].keys()
def queues(self, server):
server = self.data["cloudmesh"]["pbs"][server]
if "queues" in server:
return server["queues"]
else:
return None
@staticmethod
def qstat(host):
return Shell.ssh(host, "qstat").rstrip()
class PBSDeploy(object):
def load(self):
self.filename = config_file("/cloudmesh_pbs.yaml")
self.data = ConfigDict(filename=self.filename)
def __init__(self, deploy=False):
if deploy:
self.deploy()
self.load()
def __str__(self):
return self.data.json()
@staticmethod
def deploy(force=True):
"""copies the yaml file from etc in the distribution to the .cloudmesh
directory. If the file exits it will not be copied and a warning is
thrown. If the file is the same as in etc no warning is thrown.
"""
xcopy("../etc/", "~/.cloudmesh", "*.yaml", force=force)
def servers(self):
return self.data["cloudmesh"]["pbs"].keys()
def queues(self, server):
server = self.data["cloudmesh"]["pbs"][server]
if "queues" in server:
return server["queues"]
else:
return None
@staticmethod
def qstat(host):
return Shell.ssh(host, "qstat").rstrip()
class Test_cloudmesh:
username = ConfigDict(
filename=config_file("/cloudmesh.yaml")).get("cloudmesh.hpc.username")
filename = "etc/cloudmesh.yaml"
def setup(self):
self.idp = cm_userLDAP()
self.idp.connect("fg-ldap", "ldap")
self.idp.refresh()
def tearDown(self):
pass
def test_me(self):
print("USERNAME", self.username)
user = self.idp.find_one({'cm_user_id': self.username})
print(user)
def test_list(self):
users = self.idp.list()
pprint(users)
pprint(self.idp.users)
def test_auth(self):
password = getpass.getpass()
if self.idp.authenticate(self.username, password):
print("SUCCESS")
else:
print("FAILED")
def __init__(self,
username,
filename,
Credential,
style=2.0,
password=None):
config = ConfigDict(filename=filename)
self.password = password
self[username] = {}
for cloud in config.get("cloudmesh.clouds").keys():
self[username][cloud] = Credential(username,
cloud,
filename,
style=style,
password=self.password)
self.encrypt(username, cloud, style)
def shell_command_open_web(arguments):
"""
::
Usage:
web [--fg|--cm] [LINK]
Arguments:
LINK the link on the localhost cm server is opened.
Options:
-v verbose mode
--fg opens a link on the FG portal
--cm opens a link on the CM portal
Description:
Opens a web page with the specified link
"""
link = arguments["LINK"]
if link is None or link == "/":
link = ""
web_browser = "firefox"
if sys.platform == 'darwin':
web_browser = "open"
if arguments["--fg"]:
location = "https://portal.futuregrid.org"
elif arguments["--cm"]:
location = "https://cloudmesh.futuregrid.org"
else:
try:
filename = config_file("/cloudmesh_server.yaml")
server_config = ConfigDict(filename=filename)
host = server_config.get("cloudmesh.server.webui.host")
port = server_config.get("cloudmesh.server.webui.port")
location = "http://{0}:{1}".format(host, port)
except Exception, e:
print("ERROR: some error reading from the config file")
print(e)
return
def project_apply():
form = ProjectRegistrationForm(request.form)
if request.method == 'POST' and form.validate():
data = dict(request.form)
action = str(data['button'][0])
print "Project Data"
print data
for key in data:
if key in ['agreement_use','agreement_slides','agreement_support',
'agreement_software','agreement_documentation', 'join_open',
'join_notification']:
if str(data[key][0]) == 'y':
data[key] = True
elif str(data[key][0]) == 'n':
data[key] = False
elif key in ['category', 'managers', 'resources_clusters', 'alumni',
'resources_provision', 'resources_services', 'members',
'keywords']:
print data[key][0]
else:
data[key] = data[key][0]
print str(key)+" - "+str(data[key])+"\n"
del data['button']
if action == 'save':
projects = Projects()
project = MongoProject()
for d in data:
project[d] = data[d]
print project
projects.add(project)
flash('Thanks for registering')
return redirect('/')
project_config = ConfigDict(filename=config_file("/cloudmesh_project_intf.yaml"))
project_fields = project_config.get("cloudmesh.project")
return render_template('management/project_apply.html',
title="Project Application",
states=['save', 'cancel'],
fields=project_fields,
countries_list=[c for c in country_list()],
states_list = [c for c in states_list()],
disciplines_list = [c for c in disciplines_list()])
def shell_command_open_web(arguments):
"""
::
Usage:
web [--fg|--cm] [LINK]
Arguments:
LINK the link on the localhost cm server is opened.
Options:
-v verbose mode
--fg opens a link on the FG portal
--cm opens a link on the CM portal
Description:
Opens a web page with the specified link
"""
link = arguments["LINK"]
if link is None or link == "/":
link = ""
web_browser = "firefox"
if sys.platform == 'darwin':
web_browser = "open"
if arguments["--fg"]:
location = "https://portal.futuregrid.org"
elif arguments["--cm"]:
location = "https://cloudmesh.futuregrid.org"
else:
try:
filename = config_file("/cloudmesh_server.yaml")
server_config = ConfigDict(filename=filename)
host = server_config.get("cloudmesh.server.webui.host")
port = server_config.get("cloudmesh.server.webui.port")
location = "http://{0}:{1}".format(host, port)
except Exception, e:
print("ERROR: some error reading from the config file")
print(e)
return
def test_gregor(self):
banner("ME")
id = ConfigDict(filename=config_file("/me.yaml")).get("portalname")
user = cm_user()
result = user.info(id)
pprint(result)
pass
def __init__(self):
# read the host file definition from cloudmesh_cluster.yaml
self.server_config = cm_config_server()
self.config = ConfigDict(filename=self.CONFIG_FILE)
collection = "inventory"
self.db_inventory = get_mongo_db(collection)
def connect(self, **kwargs):
"""
The method is used to get a connection to a specified database. The hostname and port is being read from the
cloudmesh_server.yaml file. If the username and password is not mentioned as part of the command line
arguments, the details are read from the cloudmesh_server.yaml file. If the details are not available in the
yaml file, it tried to connect without the credentials.
:param kwargs:
Can contain credentials for authentication to the database.
:return:
Connection object to the specified server.
"""
self.DB_CONFIG = {}
config = ConfigDict(filename=config_file("/cloudmesh_server.yaml"))
mongo_config = config.get("cloudmesh", "server", "mongo")
#
self.DB_CONFIG["host"] = mongo_config["host"]
self.DB_CONFIG["port"] = int(mongo_config["port"])
self.DB_CONFIG["username"] = mongo_config["username"]
self.DB_CONFIG["password"] = mongo_config["password"]
#
for key, value in kwargs.iteritems():
if key == "user_name":
if value:
self.DB_CONFIG["username"] = value
elif key == "pwd":
if value:
self.DB_CONFIG["password"] = value
if self.DB_CONFIG["username"] and self.DB_CONFIG["password"]:
uri = "mongodb://{0}:{1}@{2}:{3}".format(
self.DB_CONFIG["username"], self.DB_CONFIG["password"],
self.DB_CONFIG["host"], self.DB_CONFIG["port"])
else:
uri = "mongodb://{0}:{1}".format(self.DB_CONFIG["host"],
self.DB_CONFIG["port"])
try:
return MongoClient(uri)
except:
Console.error(
"Failed to connect to Mongoclient DB. May be an authentication issue.\n\t "
)
pass
def connect(self, **kwargs):
"""
The method is used to get a connection to a specified database. The hostname and port is being read from the
cloudmesh_server.yaml file. If the username and password is not mentioned as part of the command line
arguments, the details are read from the cloudmesh_server.yaml file. If the details are not available in the
yaml file, it tried to connect without the credentials.
:param kwargs:
Can contain credentials for authentication to the database.
:return:
Connection object to the specified server.
"""
self.DB_CONFIG = {}
config = ConfigDict(filename=config_file("/cloudmesh_server.yaml"))
mongo_config = config.get("cloudmesh", "server", "mongo")
#
self.DB_CONFIG["host"] = mongo_config["host"]
self.DB_CONFIG["port"] = int(mongo_config["port"])
self.DB_CONFIG["username"] = mongo_config["username"]
self.DB_CONFIG["password"] = mongo_config["password"]
#
for key, value in kwargs.iteritems():
if key == "user_name":
if value:
self.DB_CONFIG["username"] = value
elif key == "pwd":
if value:
self.DB_CONFIG["password"] = value
if self.DB_CONFIG["username"] and self.DB_CONFIG["password"]:
uri = "mongodb://{0}:{1}@{2}:{3}".format(self.DB_CONFIG["username"],
self.DB_CONFIG["password"],
self.DB_CONFIG["host"],
self.DB_CONFIG["port"])
else:
uri = "mongodb://{0}:{1}".format(self.DB_CONFIG["host"],
self.DB_CONFIG["port"])
try:
return MongoClient(uri)
except:
Console.error("Failed to connect to Mongoclient DB. May be an authentication issue.\n\t ")
pass
def load(self, filename):
"""
loads the configuration from the yaml filename
:param filename:
:type filename: string
:return:
"""
self.data = BaseConfigDict(filename=Config.path_expand(filename))
def password():
user_config = cm_config(filename=config_file("/cloudmesh.yaml"))
user = user_config.cloud('india')['credentials']
server_config = ConfigDict(filename=config_file("/cloudmesh_server.yaml"))
server = server_config.get('cloudmesh.server.keystone.india')
print(" ".join([
"keystone", "--os-username", server['OS_USERNAME'], "--os-password",
server['OS_PASSWORD'], "--os-tenant-name", server['OS_TENANT_NAME'],
"--os-auth-url", server['OS_AUTH_URL'], "user-password-update",
"--pass", user['OS_PASSWORD'], user['OS_USERNAME']
]))
Shell.keystone("--os-username", server['OS_USERNAME'], "--os-password",
server['OS_PASSWORD'], "--os-tenant-name",
server['OS_TENANT_NAME'], "--os-auth-url",
server['OS_AUTH_URL'], "user-password-update", "--pass",
user['OS_PASSWORD'], user['OS_USERNAME'])
def __init__(self):
config = ConfigDict(filename=config_file("/cloudmesh_server.yaml"))
port = config['cloudmesh']['server']['mongo']['port']
# db = connect('manage', port=port)
self.users = User.objects()
dbname = get_mongo_dbname_from_collection("manage")
if dbname:
meta = {'db_alias': dbname}
def check(username=None):
"""clean the dirs"""
if username is None:
# bug for some reason the get method does not work
# useranme = ConfigDict(filename=config_file("/cloudmesh.yaml")).get("cloudmesh.hpc.username")
username = ConfigDict(filename=config_file(
"/cloudmesh.yaml"))["cloudmesh"]["hpc"]["username"]
print "Username: "******"india"]:
local("ssh %s@%s.futuregrid.org hostname -a" % (username, host))
def metric_index():
metric = "metric-summary"
term = "last_3_months"
config = ConfigDict(filename=config_file(
"/cloudmesh_server.yaml"))["cloudmesh"]["server"]["metric"]
address = "{0}:{1}/{2}/{3}".format(
config["host"], config["port"], metric, term)
r = requests.get(address)
return render_template('/metric/index.html', data=r.text)
def generate(self, me_file, out_file):
cloudmesh_yaml = path_expand(self.filename)
user_config = ConfigDict(filename=me_file)
t = cm_template(cloudmesh_yaml)
result = t.replace(kind="dict", values=user_config)
location = path_expand(out_file)
yaml_file = open(location, 'w+')
print(yaml.dump(result, default_flow_style=False), file=yaml_file)
yaml_file.close()
log.info("Written new yaml file in " + location)
def __init__(self,
username,
cloud,
datasource=None,
yaml_version=2.0,
style=2.0,
password=None):
"""datasource is afilename"""
CredentialBaseClass.__init__(self, username, cloud, datasource)
self.password = password
if datasource is not None:
self.filename = datasource
else:
self.filename = "~/.cloudmesh/cloudmesh.yaml"
self.config = ConfigDict(filename=self.filename)
self.read(username, cloud, style=style)
def password():
user_config = cm_config(filename=config_file("/cloudmesh.yaml"))
user = user_config.cloud('india')['credentials']
server_config = ConfigDict(filename=config_file("/cloudmesh_server.yaml"))
server = server_config.get('cloudmesh.server.keystone.india')
print(" ".join(["keystone", "--os-username", server['OS_USERNAME'],
"--os-password", server['OS_PASSWORD'],
"--os-tenant-name", server['OS_TENANT_NAME'],
"--os-auth-url", server['OS_AUTH_URL'],
"user-password-update",
"--pass", user['OS_PASSWORD'], user['OS_USERNAME']]))
Shell.keystone("--os-username", server['OS_USERNAME'],
"--os-password", server['OS_PASSWORD'],
"--os-tenant-name", server['OS_TENANT_NAME'],
"--os-auth-url", server['OS_AUTH_URL'],
"user-password-update",
"--pass", user['OS_PASSWORD'], user['OS_USERNAME'])
def do_loglevel(self, args, arguments):
"""
::
Usage:
loglevel
loglevel critical
loglevel error
loglevel warning
loglevel info
loglevel debug
Shows current log level or changes it.
loglevel - shows current log level
critical - shows log message in critical level
error - shows log message in error level including critical
warning - shows log message in warning level including error
info - shows log message in info level including warning
debug - shows log message in debug level including info
"""
if arguments['debug']:
self.loglevel = "DEBUG"
elif arguments['error']:
self.loglevel = "ERROR"
elif arguments['warning']:
self.loglevel = "WARNING"
elif arguments['info']:
self.loglevel = "INFO"
elif arguments['critical']:
self.loglevel = "CRITICAL"
else:
Console.ok("Log level: {0}".format(self.loglevel))
return
Console.ok ("Log level: {0} is set".format(self.loglevel))
filename = path_expand("~/.cloudmesh/cmd3.yaml")
config = ConfigDict(filename=filename)
config["cmd3"]["properties"]["loglevel"] = self.loglevel
config.write(filename=filename, output="yaml", attribute_indent=" ")
def do_loglevel(self, args, arguments):
"""
::
Usage:
loglevel
loglevel critical
loglevel error
loglevel warning
loglevel info
loglevel debug
Shows current log level or changes it.
loglevel - shows current log level
critical - shows log message in critical level
error - shows log message in error level including critical
warning - shows log message in warning level including error
info - shows log message in info level including warning
debug - shows log message in debug level including info
"""
if arguments['debug']:
self.loglevel = "DEBUG"
elif arguments['error']:
self.loglevel = "ERROR"
elif arguments['warning']:
self.loglevel = "WARNING"
elif arguments['info']:
self.loglevel = "INFO"
elif arguments['critical']:
self.loglevel = "CRITICAL"
else:
Console.ok("Log level: {0}".format(self.loglevel))
return
Console.ok("Log level: {0} is set".format(self.loglevel))
filename = path_expand("~/.cloudmesh/cmd3.yaml")
config = ConfigDict(filename=filename)
config["cmd3"]["properties"]["loglevel"] = self.loglevel
config.write(filename=filename, output="yaml", attribute_indent=" ")
def do_admin(self, args, arguments):
"""
::
Usage:
admin password reset
admin password
Options:
Description:
admin password reset
Reset portal password
"""
pprint(arguments)
if arguments["password"] and arguments["reset"]:
Console.ok("password reset ...")
self.password = getpass("Password:"******"~/.cloudmesh/cmd3.yaml")
config = ConfigDict(filename=filename)
config["cmd3"]["properties"]["password"] = self.password
config.write(filename=filename, output="yaml")
Console.ok("Resetting password. ok.")
elif arguments["password"]:
if yn_choice("Would you like to print the password?"):
filename = path_expand("~/.cloudmesh/cmd3.yaml")
config = ConfigDict(filename=filename)
try:
self.password = config["cmd3"]["properties"]["password"]
except Exception:
Console.error("could not find the password. Please set it.")
return
Console.msg("Password: {:}".format(self.password))
def load(self, yaml_filename=None):
"""
Loads the cloudmesh pbs yaml file.
:param yaml_filename: The filename of the yaml file
"""
log.debug("PBS yaml filename: {0}".format(self.yaml_filename))
if yaml_filename is None:
yaml_filename = self.yaml_filename
else:
self.yaml_filename = config_file(yaml_filename)
self.data = ConfigDict(filename=self.yaml_filename)
self.hosts = ssh_config()
class setup_management(object):
def __init__(self, quiet=False):
self.quiet = quiet
try:
self.filename = config_file("/cmd3.yaml")
if not self.quiet:
Console.ok("Reading " + self.filename + ". ok.")
except:
Console.error("problem loading cmd3.yaml file")
try:
self.config = ConfigDict(filename=self.filename)
# print(self.config)
except Exception as e:
Console.error("problem with ConfigDict")
print(e)
def add(self, plugin, dryrun=False):
if not plugin.endswith(".plugins"):
plugin = plugin + ".plugins"
modules = self.config["cmd3"]["modules"]
if plugin in modules:
if not self.quiet:
Console.error("plugin {:} is already activated.".format(plugin))
else:
self.config["cmd3"]["modules"].append(plugin)
if not self.quiet:
Console.ok("Adding plugin " + plugin)
if not dryrun:
self.config.write(self.filename, output="yaml")
else:
print(self.config.yaml())
def delete(self, plugin, dryrun=False):
if not plugin.endswith(".plugins"):
plugin = plugin + ".plugins"
modules = self.config["cmd3"]["modules"]
if plugin not in modules:
Console.error("plugin {:} is already deactivated.".format(plugin))
else:
self.config["cmd3"]["modules"].remove(plugin)
if not self.quiet:
Console.ok("Deleting plugin " + plugin)
if not dryrun:
self.config.write(self.filename, output="yaml")
else:
print(self.config.yaml())
def __str__(self):
with open(self.filename, 'r') as f:
r = f.read()
return r
def load(self, filename="/cloudmesh_pbs.yaml"):
"""
The configuration for the job db is stored in a yaml
file. The defualt location is ~/.cloudmesh/cloudmesh_pbs.yaml
:param filename: the filename of the yaml file
:type filename: str
"""
self.filename = config_file(filename)
self.data = ConfigDict(filename=self.filename)
self.port = self.data["cloudmesh"]["jobdatabase"]["port"]
self.db_path = path_expand(self.data["cloudmesh"]["jobdatabase"]["db_path"])
self.log_file = path_expand(self.db_path + "/dbjobs.log")
self.dbname = self.data["cloudmesh"]["jobdatabase"]["dbname"]
def __init__(self,
username,
cloud,
datasource=None,
yaml_version=2.0,
style=2.0,
password=None):
"""datasource is afilename"""
CredentialBaseClass.__init__(self, username, cloud, datasource)
self.password = password
if datasource is not None:
self.filename = datasource
else:
self.filename = "~/.cloudmesh/cloudmesh.yaml"
self.config = ConfigDict(filename=self.filename)
self.read(username, cloud, style=style)
def __init__(self,
label,
credential=None,
service_url_type='publicURL'):
"""
initializes the openstack cloud from a file
located at cloudmesh.yaml.
However if a credential dict is used it is used instead
"""
self.clear()
self.label = label
self.credential = credential
self.service_url_type = service_url_type
if credential is None:
try:
self.compute_config = ConfigDict(filename="~/.cloudmesh/cloudmesh.yaml")
self.credential = self.compute_config.get("cloudmesh.clouds.{:}.credentials".format(label))
except:
log.error(str(
lineno()) + "No user credentail found! Please check your cloudmesh.yaml file.")
# sys.exit(1)
self.connect()
def __init__(self, filename=None):
if filename is None:
filename = self.filename
ConfigDict.__init__(self, filename=filename, kind="flavor")
# provider = pbs_mongo
app = Flask(__name__)
api = restful.Api(app)
queue_stat_parser = reqparse.RequestParser()
queue_stat_parser.add_argument('resource', type=str, required=True)
# queue_stat_parser.add_argument('user', type=str)
queue_stat_parser.add_argument('id', type=str)
queue_info_parser = reqparse.RequestParser()
queue_info_parser.add_argument('resource', type=str, required=True)
# queue_info_parser.add_argument('user', type=str)
queue_info_parser.add_argument('queue', type=str)
config = ConfigDict(
prefix="cloudmesh", filename=config_file("/cloudmesh.yaml"))
user = config.get("cloudmesh.profile.username")
def versioned_url(url):
return "/" + version + url
def simple_error(kind, attribute, help="does not exist"):
msg = {}
msg["error:"] = "{0} {1} {2}".format(kind, attribute, help)
return msg
class rest_queue_stat(restful.Resource):
def __init__(self, filename=None):
if filename is None:
filename = self.filename
ConfigDict.__init__(self, filename=filename)
class ConfigDict(object):
def __init__(self,
filename,
load_order=None,
verbose=False):
"""
Creates a dictionary from a yaml configuration file
while using the filename to load it in the specified load_order.
The load order is an array of paths in which the file is searched.
By default the load order is set to . and ~/.cloudmesh
:param filename: the filename
:type filename: string
:param load_order: an array with path names in with the filename is looked for.
:type load_order: list of strings
:return: an instance of ConfigDict
:rtype: ConfigDict
"""
self.data = None
if load_order is None:
self.load_order = [".", os.path.join("~", ".cloudmesh")]
else:
self.load_order = load_order
for path in self.load_order:
name = Config.path_expand(os.path.join(path, filename))
if verbose:
print("try Loading ConfigDict", name)
if os.path.isfile(name):
if verbose:
print("Loading ConfigDict", name)
self.load(name)
self.filename = name
return
raise Exception("could not find file {:} in {:}"
.format(filename, self.load_order))
def load(self, filename):
"""
loads the configuration from the yaml filename
:param filename:
:type filename: string
:return:
"""
self.data = BaseConfigDict(filename=Config.path_expand(filename))
def save(self, filename=None):
"""
saves the configuration in the given filename,
if it is none the filename at load time is used.
:param filename: the file name
:type filename: string
:return:
"""
content = self.data.yaml()
with open(Config.path_expand(self.filename), 'w') as f:
f.write(content)
def __setitem__(self, item, value):
"""
sets an item with the given value while using . formatted keys
set('a.b.c", value)
:param item:
:type item:
:param value:
:type value:
:return:
"""
keys = None
if "." in item:
keys = item.split(".")
else:
element = self.data[item]
element = self.data[keys[0]]
for key in keys[1:]:
element = element[key]
element = value
def __getitem__(self, item):
"""
gets an item form the dict. The key is . separated
use it as follows get("a.b.c")
:param item:
:type item:
:return:
"""
if "." in item:
keys = item.split(".")
else:
return self.data[item]
element = self.data[keys[0]]
for key in keys[1:]:
element = element[key]
return element
def __str__(self):
"""
returns the dict in yaml format
:return: returns the yaml output of the dict
:rtype: string
"""
return self.data.yaml()
@property
def yaml(self):
"""
returns the dict in yaml format
:return: returns the yaml output of the dict
:rtype: string:
"""
return self.data.yaml()
def info(self):
"""
prints out the dict type and its content
"""
print(type(self.data))
print(self.data)
@property
def json(self, start=None):
"""
:param start: start key in dot notation
returns the dict in json format
:return: json string version
:rtype: string
"""
if start is not None:
data = self.data[start]
return json.dumps(self.data, indent=4)
@classmethod
def check(cls, filename):
"""
checks the filename if it is syntactically correct and does not include tabs
:param filename:
:type filename: string
:return:
"""
TODO.implement()
class CredentialFromYaml(CredentialBaseClass):
password = None
def __init__(self,
username,
cloud,
datasource=None,
yaml_version=2.0,
style=2.0,
password=None):
"""datasource is afilename"""
CredentialBaseClass.__init__(self, username, cloud, datasource)
self.password = password
if datasource is not None:
self.filename = datasource
else:
self.filename = "~/.cloudmesh/cloudmesh.yaml"
self.config = ConfigDict(filename=self.filename)
self.read(username, cloud, style=style)
def read(self, username, cloud, style=2.0):
self.style = style
self['cm'] = {}
self['cm']['source'] = 'yaml'
self['cm']['filename'] = self.filename
self['cm']['kind'] = self.config.get("meta.kind")
self['cm']['yaml_version'] = self.config.get("meta.yaml_version")
kind = self['cm']['kind']
if kind == "clouds":
self['cm']['filename'] = "~/.cloudmesh/cloudmesh.yaml"
self.update(self.config.get("cloudmesh.clouds.{0}".format(cloud)))
elif kind == "server":
self['cm']['filename'] = "~/.cloudmesh/cloudmesh_server.yaml"
self.update(
self.config.get("cloudmesh.server.keystone.{0}".format(cloud)))
else:
log.error("kind wrong {0}".format(kind))
self['cm']['username'] = username
self['cm']['cloud'] = cloud
self.clean_cm()
self.transform_cm(self['cm']['yaml_version'], style)
# self.remove_default()
def clean_cm(self):
'''temporary so we do not have to modify yaml files for now'''
for key in self.keys():
if key.startswith('cm_'):
new_key = key.replace('cm_', '')
self['cm'][new_key] = self[key]
del self[key]
"""
def remove_default(self):
if 'default' in self.keys():
del self['default']
"""
def transform_cm(self, yaml_version, style):
if yaml_version <= 2.0 and style == 2.0:
for key in self['cm']:
new_key = 'cm_' + key
self[new_key] = self['cm'][key]
del self['cm']
from cloudmesh_task.tasks import cm_ssh
from cloudmesh_task.parallel import Parallel, Sequential
from cloudmesh.util.stopwatch import StopWatch
from cloudmesh_base.util import banner
from pprint import pprint
from cloudmesh.config.cm_config import cm_config
from cloudmesh_base.ConfigDict import ConfigDict
import sys
config = ConfigDict(
filename="~/.cloudmesh/cloudmesh_hpc.yaml")["cloudmesh"]["hpc"]
print config
#hosts = []
# hosts.append("localhost")
# hosts.append("bigred2.uits.iu.edu")
# hosts.append("india.futuregrid.org")
def get_credentials(hosts):
credential = {}
for host in hosts:
credential[host] = config[host]['username']
return credential
hosts = config.keys()
credentials = get_credentials(hosts)