def get_ldap_user_from_yaml():
me = ConfigDict(filename=config_file("/me.yaml"))
d = {}
for element in ["firstname",
"lastname",
"email",
"phone",
"address"]:
d[element] = me.get("profile.{0}".format(element))
d["cm_user_id"] = me.get("portalname")
d["gidNumber"] = 0
d["uidNumber"] = 0
if "gidNumber" in me.keys():
d["gidNumber"] = me.get("gidNumber")
if "uidNumber" in me.keys():
d["uidNumber"] = me.get("uidNumber")
d["projects"] = me.get("projects")
#
# copy the keys
#
d['keys'] = me.get("keys.keylist")
return d
def readRackConfig(self, name, dir_yaml, dir_diag):
rack_config = ConfigDict(filename=dir_yaml + "/" +
self.default_rack_yaml)
self.dict_rack_config = rack_config.get("cloudmesh.rack")
lname = name.lower()
# the rack name in cloudmesh_rack.yaml MUST be valid/predefined in
# clouldmesh_cluster.yaml
self.cluster_name = lname if lname in self.cluster_name_list else self.cluster_name_unknown
# diag filename and temporary diag filename
self.filename_diag = dir_diag + "/" + \
self.dict_rack_config["cluster"][self.cluster_name]["diag"]
self.rack_count = self.dict_rack_config["cluster"][
self.cluster_name]["count"]
self.filename_diag_temp = self.filename_diag + ".temp"
# additional process for 'all'
# its count equals the sum of all other cluster exclude 'all' and
# 'unknown'
if name == "all":
raw_set = set(self.dict_rack_config["cluster"].keys())
exclude_set = set(["all", "unknown"])
all_set = raw_set - exclude_set
self.rack_count = 0
for rack in all_set:
self.rack_count += self.dict_rack_config["cluster"][rack][
"count"]
def setup_inventory(self):
banner("Read Dicts")
self.sample_user = ConfigDict(filename=config_file("/me.yaml"))
self.portalname = self.sample_user.get("portalname")
print("PORTALNAME", self.portalname)
print("SAMPLE USER", self.sample_user)
banner("create user from template, duplicates cm init generate me")
t = cm_template(config_file("/etc/cloudmesh.yaml"))
pprint(set(t.variables()))
self.config = t.replace(kind="dict", values=self.sample_user)
print(type(self.config))
print(self.config)
#
# BUG?
#
self.idp = cm_userLDAP()
self.idp.connect("fg-ldap", "ldap")
self.idp.refresh()
ldap_info = self.idp.get(self.portalname)
print(ldap_info)
print(type(self.config))
self.config['cloudmesh']['projects'] = ldap_info['projects']
self.config['cloudmesh']['keys'] = ldap_info['keys']
try:
self.config['cloudmesh']['projects'][
'deafult'] = ldap_info['projects']['active'][0]
except:
print("ERROR: you have no projects")
def create_user_from_file(cls, file_path):
try:
filename = path_expand(file_path)
file_config = ConfigDict(filename=filename)
except:
Console.error("Could not load file, please check filename and its path")
return
try:
user_config = file_config.get("cloudmesh", "user")
user_name = user_config['username']
user = User()
update_document(user, user_config)
except:
Console.error("Could not get user information from yaml file, "
"please check you yaml file, users information must be "
"under 'cloudmesh' -> 'users' -> user1...")
return
try:
if cls.check_exists(user_name) is False:
cls.add(user)
Console.info("User created in the database.")
else:
Console.error("User with user name " + user_name + " already exists.")
return
except:
Console.error("User creation in database failed, " + str(sys.exc_info()))
return
class Test_cloudmesh:
username = ConfigDict(
filename=config_file("/cloudmesh.yaml")).get("cloudmesh.hpc.username")
filename = "etc/cloudmesh.yaml"
def setup(self):
self.idp = cm_userLDAP()
self.idp.connect("fg-ldap", "ldap")
self.idp.refresh()
def tearDown(self):
pass
def test_me(self):
print("USERNAME", self.username)
user = self.idp.find_one({'cm_user_id': self.username})
print(user)
def test_list(self):
users = self.idp.list()
pprint(users)
pprint(self.idp.users)
def test_auth(self):
password = getpass.getpass()
if self.idp.authenticate(self.username, password):
print("SUCCESS")
else:
print("FAILED")
def user_apply():
form = UserRegistrationForm(request.form)
if request.method == 'POST': #and form.validate():
data = dict(request.form)
action = str(data['button'][0])
for key in data:
data[key] = data[key][0]
del data['button']
if action == 'save':
users = Users()
user = MongoUser()
del data['confirm']
for d in data:
user[d] = data[d]
users.add(user)
flash('Thanks for registering')
return redirect('/')
user_config = ConfigDict(filename=config_file("/cloudmesh_user_intf.yaml"))
user_fields = user_config.get("cloudmesh.user")
return render_template('management/user_apply.html',
title="User Application",
states=['save', 'cancel'],
fields=user_fields,
countries_list=[c for c in country_list()],
roles_list=roles)
def test_gregor(self):
banner("ME")
id = ConfigDict(filename=config_file("/me.yaml")).get("portalname")
user = cm_user()
result = user.info(id)
pprint(result)
pass
def __init__(self):
# read the host file definition from cloudmesh_cluster.yaml
self.server_config = cm_config_server()
self.config = ConfigDict(filename=self.CONFIG_FILE)
collection = "inventory"
self.db_inventory = get_mongo_db(collection)
def test_getitem_server(self):
HEADING()
filename = config_file("/cloudmesh_server.yaml")
config = ConfigDict(filename=filename)
print(config)
existing = config.get("cloudmesh.server.mongo.db")
test1 = existing is not None
print("QUERY", existing)
print("Port", config.get("cloudmesh.server.mongo.port"))
def __init__(self):
config = ConfigDict(filename=config_file("/cloudmesh_server.yaml"))
port = config['cloudmesh']['server']['mongo']['port']
# db = connect('manage', port=port)
self.users = User.objects()
dbname = get_mongo_dbname_from_collection("manage")
if dbname:
meta = {'db_alias': dbname}
def check(username=None):
"""clean the dirs"""
if username is None:
# bug for some reason the get method does not work
# useranme = ConfigDict(filename=config_file("/cloudmesh.yaml")).get("cloudmesh.hpc.username")
username = ConfigDict(filename=config_file(
"/cloudmesh.yaml"))["cloudmesh"]["hpc"]["username"]
print "Username: "******"india"]:
local("ssh %s@%s.futuregrid.org hostname -a" % (username, host))
def generate(self, me_file, out_file):
cloudmesh_yaml = path_expand(self.filename)
user_config = ConfigDict(filename=me_file)
t = cm_template(cloudmesh_yaml)
result = t.replace(kind="dict", values=user_config)
location = path_expand(out_file)
yaml_file = open(location, 'w+')
print(yaml.dump(result, default_flow_style=False), file=yaml_file)
yaml_file.close()
log.info("Written new yaml file in " + location)
def set_debug(self, on):
filename = path_expand("~/.cloudmesh/cmd3.yaml")
config = ConfigDict(filename=filename)
if type(on) == bool:
self.debug = on
else:
self.debug = on.lower() in ['on', 'true']
config["cmd3"]["properties"]["debug"] = self.debug
Console.ok("Debug mode is {:}".format(self.debug))
config.write(filename=filename, output="yaml", attribute_indent=" ")
def metric_index():
metric = "metric-summary"
term = "last_3_months"
config = ConfigDict(filename=config_file(
"/cloudmesh_server.yaml"))["cloudmesh"]["server"]["metric"]
address = "{0}:{1}/{2}/{3}".format(
config["host"], config["port"], metric, term)
r = requests.get(address)
return render_template('/metric/index.html', data=r.text)
def view(link=""):
"""run the browser"""
from cloudmesh_base.ConfigDict import ConfigDict
server_config = ConfigDict(filename=config_file("/cloudmesh_server.yaml"))
host = server_config.get("cloudmesh.server.webui.host")
port = server_config.get("cloudmesh.server.webui.port")
url_link = "http://{0}:{1}/{2}".format(host, port, link)
local("%s %s" % (web_browser, url_link))
def do_admin(self, args, arguments):
"""
::
Usage:
admin password reset
admin password
Options:
Description:
admin password reset
Reset portal password
"""
pprint(arguments)
if arguments['password'] and arguments['reset']:
Console.ok('password reset ...')
self.password = getpass("Password:"******"~/.cloudmesh/cmd3.yaml")
config = ConfigDict(filename=filename)
config["cmd3"]["properties"]["password"] = self.password
config.write(filename=filename, output="yaml")
Console.ok("Resetting password. ok.")
elif arguments['password']:
if yn_choice("Would you like to print the password?"):
filename = path_expand("~/.cloudmesh/cmd3.yaml")
config = ConfigDict(filename=filename)
try:
self.password = config["cmd3"]["properties"]["password"]
except Exception:
Console.error("could not find the password. Please set it.")
return
Console.msg("Password: {:}".format(self.password))
def project_apply():
form = ProjectRegistrationForm(request.form)
if request.method == 'POST' and form.validate():
data = dict(request.form)
action = str(data['button'][0])
print "Project Data"
print data
for key in data:
if key in [
'agreement_use', 'agreement_slides', 'agreement_support',
'agreement_software', 'agreement_documentation',
'join_open', 'join_notification'
]:
if str(data[key][0]) == 'y':
data[key] = True
elif str(data[key][0]) == 'n':
data[key] = False
elif key in [
'category', 'managers', 'resources_clusters', 'alumni',
'resources_provision', 'resources_services', 'members',
'keywords'
]:
print data[key][0]
else:
data[key] = data[key][0]
print str(key) + " - " + str(data[key]) + "\n"
del data['button']
if action == 'save':
projects = Projects()
project = MongoProject()
for d in data:
project[d] = data[d]
print project
projects.add(project)
flash('Thanks for registering')
return redirect('/')
project_config = ConfigDict(
filename=config_file("/cloudmesh_project_intf.yaml"))
project_fields = project_config.get("cloudmesh.project")
return render_template('management/project_apply.html',
title="Project Application",
states=['save', 'cancel'],
fields=project_fields,
countries_list=[c for c in country_list()],
states_list=[c for c in states_list()],
disciplines_list=[c for c in disciplines_list()])
def load(self, yaml_filename=None):
"""
Loads the cloudmesh pbs yaml file.
:param yaml_filename: The filename of the yaml file
"""
log.debug("PBS yaml filename: {0}".format(self.yaml_filename))
if yaml_filename is None:
yaml_filename = self.yaml_filename
else:
self.yaml_filename = config_file(yaml_filename)
self.data = ConfigDict(filename=self.yaml_filename)
self.hosts = ssh_config()
def display_launch_table():
""" To satisfy Pylint. Will update with proper comments """
if request.method == 'POST':
print "HHHHHH", request.form.keys()
for key in request.form.keys():
print key, ":", request.form[key]
else:
print "HEY JUDE"
launcher_config = ConfigDict(
filename=config_file("/cloudmesh_launcher.yaml"))
launch_recipies = launcher_config.get("cloudmesh.launcher.recipies")
return render_template('mesh/mesh_launch.html', recipies=launch_recipies)
def __init__(self, quiet=False):
self.quiet = quiet
try:
self.filename = config_file("/cmd3.yaml")
if not self.quiet:
Console.ok("Reading " + self.filename + ". ok.")
except:
Console.error("problem loading cmd3.yaml file")
try:
self.config = ConfigDict(filename=self.filename)
# print(self.config)
except Exception as e:
Console.error("problem with ConfigDict")
print(e)
def test_server(self):
HEADING()
filename = config_file("/cloudmesh_server.yaml")
config = ConfigDict(filename=filename)
# print config
existing = config.get("cloudmesh.server.mongo.db")
test1 = existing is not None
print("mongo.db =", existing)
try:
none_existing = config.get("cloudmesh.server.mongo.xyz")
test2 = False
except:
print("Error")
test2 = True
assert test1 and test2
def test_launcher(self):
HEADING()
filename = config_file("/cloudmesh_launcher.yaml")
config = ConfigDict(filename=filename)
print(config)
existing = config.get("cloudmesh.launcher.recipies")
test1 = existing is not None
print(existing)
try:
none_existing = config.get("cloudmesh.launcher.recipies.xyz")
test2 = False
except:
print("Error")
test2 = True
assert test1 and test2
def load(self, filename="/cloudmesh_pbs.yaml"):
"""
The configuration for the job db is stored in a yaml
file. The defualt location is ~/.cloudmesh/cloudmesh_pbs.yaml
:param filename: the filename of the yaml file
:type filename: str
"""
self.filename = config_file(filename)
self.data = ConfigDict(filename=self.filename)
self.port = self.data["cloudmesh"]["jobdatabase"]["port"]
self.db_path = path_expand(self.data["cloudmesh"]["jobdatabase"]["db_path"])
self.log_file = path_expand(self.db_path + "/dbjobs.log")
self.dbname = self.data["cloudmesh"]["jobdatabase"]["dbname"]
def __init__(self,
username,
filename,
Credential,
style=2.0,
password=None):
config = ConfigDict(filename=filename)
self.password = password
self[username] = {}
for cloud in config.get("cloudmesh.clouds").keys():
self[username][cloud] = Credential(username,
cloud,
filename,
style=style,
password=self.password)
self.encrypt(username, cloud, style)
def shell_command_open_web(arguments):
"""
::
Usage:
web [--fg|--cm] [LINK]
Arguments:
LINK the link on the localhost cm server is opened.
Options:
-v verbose mode
--fg opens a link on the FG portal
--cm opens a link on the CM portal
Description:
Opens a web page with the specified link
"""
link = arguments["LINK"]
if link is None or link == "/":
link = ""
web_browser = "firefox"
if sys.platform == 'darwin':
web_browser = "open"
if arguments["--fg"]:
location = "https://portal.futuregrid.org"
elif arguments["--cm"]:
location = "https://cloudmesh.futuregrid.org"
else:
try:
filename = config_file("/cloudmesh_server.yaml")
server_config = ConfigDict(filename=filename)
host = server_config.get("cloudmesh.server.webui.host")
port = server_config.get("cloudmesh.server.webui.port")
location = "http://{0}:{1}".format(host, port)
except Exception, e:
print("ERROR: some error reading from the config file")
print(e)
return
def yaml(plugin):
"""
This function reads in the cloudmesh cmd3.yaml file and tests
if the requested plugin module is included in the yaml file.
an example plugin name os "cloudmesh_pbs" to which automatically a
.plugins will be appended if not specified.
:return: True if the plugin is in the yaml file
"""
filename = config_file("/cmd3.yaml")
config = ConfigDict(filename=filename)
if not plugin.endswith(".plugins"):
testname = plugin + ".plugins"
return testname in config["cmd3"]["modules"]
def do_debug(self, args, arguments):
"""
::
Usage:
debug on
debug off
Turns the debug log level on and off.
"""
filename = path_expand("~/.cloudmesh/cmd3.yaml")
config = ConfigDict(filename=filename)
if arguments['on']:
self.set_debug(True)
elif arguments['off']:
self.set_debug(False)
def connect(self, **kwargs):
"""
The method is used to get a connection to a specified database. The hostname and port is being read from the
cloudmesh_server.yaml file. If the username and password is not mentioned as part of the command line
arguments, the details are read from the cloudmesh_server.yaml file. If the details are not available in the
yaml file, it tried to connect without the credentials.
:param kwargs:
Can contain credentials for authentication to the database.
:return:
Connection object to the specified server.
"""
self.DB_CONFIG = {}
config = ConfigDict(filename=config_file("/cloudmesh_server.yaml"))
mongo_config = config.get("cloudmesh", "server", "mongo")
#
self.DB_CONFIG["host"] = mongo_config["host"]
self.DB_CONFIG["port"] = int(mongo_config["port"])
self.DB_CONFIG["username"] = mongo_config["username"]
self.DB_CONFIG["password"] = mongo_config["password"]
#
for key, value in kwargs.iteritems():
if key == "user_name":
if value:
self.DB_CONFIG["username"] = value
elif key == "pwd":
if value:
self.DB_CONFIG["password"] = value
if self.DB_CONFIG["username"] and self.DB_CONFIG["password"]:
uri = "mongodb://{0}:{1}@{2}:{3}".format(
self.DB_CONFIG["username"], self.DB_CONFIG["password"],
self.DB_CONFIG["host"], self.DB_CONFIG["port"])
else:
uri = "mongodb://{0}:{1}".format(self.DB_CONFIG["host"],
self.DB_CONFIG["port"])
try:
return MongoClient(uri)
except:
Console.error(
"Failed to connect to Mongoclient DB. May be an authentication issue.\n\t "
)
pass
def __init__(self, username, datasource=None, password=None):
"""datasource is afilename"""
dict.__init__(self)
self['username'] = username
self['datasource'] = datasource
self.password = password
if datasource is not None:
self.filename = datasource
else:
self.filename = "~/.cloudmesh/cloudmesh.yaml"
config = ConfigDict(filename=self.filename)
for key in config['cloudmesh']:
self[key] = config['cloudmesh'][key]
self['cm_user_id'] = username
def metric_project(project_id):
config = ConfigDict(filename=config_file(
"/cloudmesh_server.yaml"))["cloudmesh"]["server"]["metric"]
base_url = "project-summary"
address = "{0}:{1}/{2}/{3}".format(
config["host"], config["port"], base_url, project_id)
try:
r = requests.get(address)
pprint(r.json())
data = dict(yaml.load(r.text))
data = data["message"]
return render_template('/metric/project.html', data=data)
except Exception, e:
print e
error = str(e)
return render_template('error.html', error=error)
