def __init__(self, deploy=True, yaml_filename="/cloudmesh_pbs.yaml"): """ Creates an object instance of communication with pbs batch queues running on multiple hosts. It also is used to create some configuration parameters if deploy is set to True. it creates in the CLOUDMESH deploy directory the directory pbs and puts the dababse file pbs.db and the file that is used to store the current job number. The job number is shared among all resources and supposed to be unique. :param deploy: If True, creates the configuration files :param yaml_filename: The cloudmesh pbs yaml file. Defaults to cloudmesh_pbs.yaml :return: an object instance of OpenPBS """ self.yaml_filename = config_file(yaml_filename) self.pbs_dir = config_file("/pbs") self.id_file = config_file("/pbs/id.txt") self.db_file = config_file("/pbs/pbs.db") if deploy: self.deploy() self.load() self.id = self.jobid self.pbs_nodes_data = None
def setup_inventory(self): banner("Read Dicts") self.sample_user = ConfigDict(filename=config_file("/me.yaml")) self.portalname = self.sample_user.get("portalname") print("PORTALNAME", self.portalname) print("SAMPLE USER", self.sample_user) banner("create user from template, duplicates cm init generate me") t = cm_template(config_file("/etc/cloudmesh.yaml")) pprint(set(t.variables())) self.config = t.replace(kind="dict", values=self.sample_user) print(type(self.config)) print(self.config) # # BUG? # self.idp = cm_userLDAP() self.idp.connect("fg-ldap", "ldap") self.idp.refresh() ldap_info = self.idp.get(self.portalname) print(ldap_info) print(type(self.config)) self.config['cloudmesh']['projects'] = ldap_info['projects'] self.config['cloudmesh']['keys'] = ldap_info['keys'] try: self.config['cloudmesh']['projects'][ 'deafult'] = ldap_info['projects']['active'][0] except: print("ERROR: you have no projects")
def wsgi(action="start"): pidfile = config_file("/uwsgi/cloudmesh_uwsgi.pid") logfile = config_file("/uwsgi/cloudmesh_uwsgi.log") command = False user_pidfile = os.path.expanduser(pidfile) user_logfile = os.path.expanduser(logfile) if action == "restart": wsgi("stop") action = "start" if action == "start": command = "uwsgi -s /tmp/cloudmesh.sock -M -p 2 -t 10 \ --daemonize={0} \ --pidfile={1} \ --chown-socket=cloudmesh:www-data \ --chdir=cloudmesh_web \ --module=server \ --callable=app".format(user_logfile, user_pidfile) elif action == "stop" or action == "kill": command = "kill -INT `cat {0}`".format(user_pidfile) elif action == "reload": command = "kill -HUP `cat {0}`".format(user_pidfile) if command: local(command)
def configure(): """configure india environment for cloudmesh rapid deployment""" # running on server mode with external port listening filename = config_file('/cloudmesh_server.yaml') print "modify ->", filename yaml_file_replace(filename='/cloudmesh_server.yaml', replacements={ 'browser: True': 'browser: False', 'host: 127.0.0.1': 'host: 0.0.0.0' } ) # port 5000 needs to be open # ec2secgroup_openport('india', 5000) # now managed via nova before vm is started. # new way to replace an attribute in yaml filename = config_file("/cloudmesh.yaml") print "modify ->", filename replacements = { "cloudmesh.clouds.india.cm_service_url_type": "internalURL", } yaml_attribute_replace(filename, replacements, indent_by=4) print "Configuration changes have been made successfully"
def configure(): """configure india environment for cloudmesh rapid deployment""" # running on server mode with external port listening filename = config_file('/cloudmesh_server.yaml') print "modify ->", filename yaml_file_replace(filename='/cloudmesh_server.yaml', replacements={ 'browser: True': 'browser: False', 'host: 127.0.0.1': 'host: 0.0.0.0' }) # port 5000 needs to be open # ec2secgroup_openport('india', 5000) # now managed via nova before vm is started. # new way to replace an attribute in yaml filename = config_file("/cloudmesh.yaml") print "modify ->", filename replacements = { "cloudmesh.clouds.india.cm_service_url_type": "internalURL", } yaml_attribute_replace(filename, replacements, indent_by=4) print "Configuration changes have been made successfully"
def get_ldap_user_from_yaml(): me = ConfigDict(filename=config_file("/me.yaml")) d = {} for element in ["firstname", "lastname", "email", "phone", "address"]: d[element] = me.get("profile.{0}".format(element)) d["cm_user_id"] = me.get("portalname") d["gidNumber"] = 0 d["uidNumber"] = 0 if "gidNumber" in me.keys(): d["gidNumber"] = me.get("gidNumber") if "uidNumber" in me.keys(): d["uidNumber"] = me.get("uidNumber") d["projects"] = me.get("projects") # # copy the keys # d['keys'] = me.get("keys.keylist") return d
class Test_cloudmesh: filename = config_file("/cloudmesh.yaml") def setup(self): pass def tearDown(self): pass def test_policy(self): HEADING() pass def test_simulator(self): HEADING() # hosts = ["1", "2", "3", "a1", "a2", "a3", # "b1", "b2", "b3", "c1", "c2", "c3", ] # image = "a" hosts = ["i066"] image = "ubuntu1304v2btsync" Provisioner = ProvisionerSimulator (success, result) = (None, None) p = Provisioner() (success, result) = p.provision(hosts, image) print(result) print(success) pass
def user_apply(): form = UserRegistrationForm(request.form) if request.method == 'POST': #and form.validate(): data = dict(request.form) action = str(data['button'][0]) for key in data: data[key] = data[key][0] del data['button'] if action == 'save': users = Users() user = MongoUser() del data['confirm'] for d in data: user[d] = data[d] users.add(user) flash('Thanks for registering') return redirect('/') user_config = ConfigDict(filename=config_file("/cloudmesh_user_intf.yaml")) user_fields = user_config.get("cloudmesh.user") return render_template('management/user_apply.html', title="User Application", states=['save', 'cancel'], fields=user_fields, countries_list=[c for c in country_list()], roles_list=roles)
class Test_cloudmesh: username = ConfigDict( filename=config_file("/cloudmesh.yaml")).get("cloudmesh.hpc.username") filename = "etc/cloudmesh.yaml" def setup(self): self.idp = cm_userLDAP() self.idp.connect("fg-ldap", "ldap") self.idp.refresh() def tearDown(self): pass def test_me(self): print("USERNAME", self.username) user = self.idp.find_one({'cm_user_id': self.username}) print(user) def test_list(self): users = self.idp.list() pprint(users) pprint(self.idp.users) def test_auth(self): password = getpass.getpass() if self.idp.authenticate(self.username, password): print("SUCCESS") else: print("FAILED")
def __init__(self): self.filename = config_file("/cloudmesh.yaml") self.config = cm_config(filename=self.filename) self.cm_user_id = self.config.get("cloudmesh.hpc.username") self.clouds = self.config.get("cloudmesh.clouds") self.user_obj = cm_user() self.profile = self.config.profile() self.mongo = cm_mongo()
def _load_projects(self): if not self.cm_shell_project_loaded: filename = config_file("/cloudmesh.yaml") self.projects = cm_projects(filename) if self.echo: log.info( "Reading project information from -> {0}".format(filename)) self.cm_shell_project_loaded = True
def load(self, filename="/cloudmesh_pbs.yaml"): self.filename = config_file(filename) self.data = ConfigDict(filename=self.filename) self.port = self.data["cloudmesh"]["jobdatabase"]["port"] self.db_path = path_expand(self.data["cloudmesh"]["jobdatabase"]["db_path"]) self.log_file = path_expand(self.db_path + "/dbjobs.log") self.dbname = self.data["cloudmesh"]["jobdatabase"]["dbname"]
def test_gregor(self): banner("ME") id = ConfigDict(filename=config_file("/me.yaml")).get("portalname") user = cm_user() result = user.info(id) pprint(result) pass
def test_getitem_server(self): HEADING() filename = config_file("/cloudmesh_server.yaml") config = ConfigDict(filename=filename) print(config) existing = config.get("cloudmesh.server.mongo.db") test1 = existing is not None print("QUERY", existing) print("Port", config.get("cloudmesh.server.mongo.port"))
def _load_keys_from_yaml(self): try: filename = config_file("/cloudmesh.yaml") if self.echo: log.info("Reading keys information from -> {0}" .format(filename)) self.keys = cm_keys_yaml(filename) self.keys_loaded = True except: print("ERROR: could not find the keys in %s" % filename)
def __init__(self): config = ConfigDict(filename=config_file("/cloudmesh_server.yaml")) port = config["cloudmesh"]["server"]["mongo"]["port"] # db = connect('manage', port=port) self.users = User.objects() dbname = get_mongo_dbname_from_collection("manage") if dbname: meta = {"db_alias": dbname}
def check(username=None): """clean the dirs""" if username is None: # bug for some reason the get method does not work # useranme = ConfigDict(filename=config_file("/cloudmesh.yaml")).get("cloudmesh.hpc.username") username = ConfigDict( filename=config_file("/cloudmesh.yaml"))["cloudmesh"]["hpc"]["username"] print "Username: "******"india"]: local("ssh %s@%s.futuregrid.org hostname -a" % (username, host))
class dbconf(ConfigDict): """ reads db config info from cloudmesh_database.yaml """ filename = config_file("/cloudmesh_database.yaml") def __init__(self, filename=None): if filename is None: filename = self.filename ConfigDict.__init__(self, filename=filename)
def __init__(self): config = ConfigDict(filename=config_file("/cloudmesh_server.yaml")) port = config['cloudmesh']['server']['mongo']['port'] # db = connect('manage', port=port) self.users = User.objects() dbname = get_mongo_dbname_from_collection("manage") if dbname: meta = {'db_alias': dbname}
def _get_value_from_yaml(filepath, column_keys): dir = config_file("") cm_file = dir + filepath try: result = open(cm_file, 'r').read() values = yaml.safe_load(Template(result).substitute(os.environ)) except Exception, e: print "ERROR: There is an error in the yaml file", e sys.exit(1)
def check(username=None): """clean the dirs""" if username is None: # bug for some reason the get method does not work # useranme = ConfigDict(filename=config_file("/cloudmesh.yaml")).get("cloudmesh.hpc.username") username = ConfigDict(filename=config_file( "/cloudmesh.yaml"))["cloudmesh"]["hpc"]["username"] print "Username: "******"india"]: local("ssh %s@%s.futuregrid.org hostname -a" % (username, host))
def password(): user_config = cm_config(filename=config_file("/cloudmesh.yaml")) user = user_config.cloud('india')['credentials'] server_config = ConfigDict(filename=config_file("/cloudmesh_server.yaml")) server = server_config.get('cloudmesh.server.keystone.india') print(" ".join([ "keystone", "--os-username", server['OS_USERNAME'], "--os-password", server['OS_PASSWORD'], "--os-tenant-name", server['OS_TENANT_NAME'], "--os-auth-url", server['OS_AUTH_URL'], "user-password-update", "--pass", user['OS_PASSWORD'], user['OS_USERNAME'] ])) Shell.keystone("--os-username", server['OS_USERNAME'], "--os-password", server['OS_PASSWORD'], "--os-tenant-name", server['OS_TENANT_NAME'], "--os-auth-url", server['OS_AUTH_URL'], "user-password-update", "--pass", user['OS_PASSWORD'], user['OS_USERNAME'])
def _load_keys_from_yaml(self): try: filename = config_file("/cloudmesh.yaml") if self.echo: log.info( "Reading keys information from -> {0}".format(filename)) self.keys = cm_keys_yaml(filename) self.keys_loaded = True except: print("ERROR: could not find the keys in %s" % filename)
class cm_config_flavor(ConfigDict): """ reads the information contained in the file cloudmesh_flavor.yaml """ filename = config_file("/cloudmesh_flavor.yaml") def __init__(self, filename=None): if filename is None: filename = self.filename ConfigDict.__init__(self, filename=filename, kind="flavor")
def disciplines_list(): filename = config_file("/cloudmesh_disciplines.yaml") data = yaml.load(open(filename)) disciplines = [] for key, value in data.items(): item = '' item += str(value['name']) disciplines.append(item) disciplines.sort() disciplines.insert(0, 'Other (OTH)') return disciplines
def states_list(): filename = config_file("/cloudmesh_states.yaml") data = yaml.load(open(filename)) states = [] for key, value in data.items(): item = '' item = item + str(value['name']) + "(" + str(key) + ")" states.append(item) states.sort() states.insert(0, 'Other (OTH)') return states
def states_list(): filename = config_file("/cloudmesh_states.yaml") data = yaml.load(open(filename)) states = [] for key, value in data.items(): item = '' item = item + str(value['name']) + "("+str(key)+")" states.append(item) states.sort() states.insert(0,'Other (OTH)') return states
def disciplines_list(): filename = config_file("/cloudmesh_disciplines.yaml") data = yaml.load(open(filename)) disciplines = [] for key, value in data.items(): item = '' item += str(value['name']) disciplines.append(item) disciplines.sort() disciplines.insert(0,'Other (OTH)') return disciplines
def password(): user_config = cm_config(filename=config_file("/cloudmesh.yaml")) user = user_config.cloud('india')['credentials'] server_config = ConfigDict(filename=config_file("/cloudmesh_server.yaml")) server = server_config.get('cloudmesh.server.keystone.india') print(" ".join(["keystone", "--os-username", server['OS_USERNAME'], "--os-password", server['OS_PASSWORD'], "--os-tenant-name", server['OS_TENANT_NAME'], "--os-auth-url", server['OS_AUTH_URL'], "user-password-update", "--pass", user['OS_PASSWORD'], user['OS_USERNAME']])) Shell.keystone("--os-username", server['OS_USERNAME'], "--os-password", server['OS_PASSWORD'], "--os-tenant-name", server['OS_TENANT_NAME'], "--os-auth-url", server['OS_AUTH_URL'], "user-password-update", "--pass", user['OS_PASSWORD'], user['OS_USERNAME'])
def metric_index(): metric = "metric-summary" term = "last_3_months" config = ConfigDict(filename=config_file( "/cloudmesh_server.yaml"))["cloudmesh"]["server"]["metric"] address = "{0}:{1}/{2}/{3}".format( config["host"], config["port"], metric, term) r = requests.get(address) return render_template('/metric/index.html', data=r.text)
def country_list(): filename = config_file("/cloudmesh_country.yaml") data = yaml.load(open(filename)) countries = [] for key, value in data.items(): item = '' item = item + str(value.encode(encoding='UTF-8', errors='strict')) + "(" + str(key) + ")" countries.append(item) countries.sort() countries.insert(0, 'United States(US)') return countries
def disciplines_list(): filename = config_file("/cloudmesh_disciplines.yaml") data = yaml.load(open(filename)) disciplines = [] for key, value in data.items(): item = "" item = item + str(value["name"]) disciplines.append(item) disciplines.sort() disciplines.insert(0, "Other(OTH)") print disciplines pass
def view(link=""): """run the browser""" from cloudmesh_base.ConfigDict import ConfigDict server_config = ConfigDict(filename=config_file("/cloudmesh_server.yaml")) host = server_config.get("cloudmesh.server.webui.host") port = server_config.get("cloudmesh.server.webui.port") url_link = "http://{0}:{1}/{2}".format(host, port, link) local("%s %s" % (web_browser, url_link))
def states_list(): filename = config_file("/cloudmesh_states.yaml") data = yaml.load(open(filename)) states = [] for key, value in data.items(): item = "" item = item + str(value["name"]) + "(" + str(key) + ")" states.append(item) states.sort() states.insert(0, "Other(OTH)") print states pass
def country_list(): filename = config_file("/cloudmesh_country.yaml") data = yaml.load(open(filename)) countries = [] for key, value in data.items(): item = "" item = item + str(value.encode(encoding="UTF-8", errors="strict")) + "(" + str(key) + ")" countries.append(item) countries.sort() countries.insert(0, "United States(US)") print countries pass
def country_list(): filename = config_file("/cloudmesh_country.yaml") data = yaml.load(open(filename)) countries = [] for key, value in data.items(): item = '' item = item + str(value.encode(encoding='UTF-8', errors='strict')) + "(" + str(key) + ")" countries.append(item) countries.sort() countries.insert(0, 'United States(US)') print countries pass
def yaml_file_replace(filename='/cloudmesh_xxx.yaml', replacements={}): filename = config_file(filename) with open(filename, 'r') as f: content = f.read() for _old, _new in replacements.iteritems(): content = content.replace(_old, _new) outfile = open(filename, 'w') outfile.write(content) outfile.close()
class Test_cloudmesh: # filename = None # filename = "credentials-example-keys.yaml" # filename = config_file("/cloudmesh-new.yaml") filename = config_file("/cloudmesh.yaml") def setup(self): self.projects = cm_projects(self.filename) def tearDown(self): pass def test00_wrong_file(self): HEADING() try: self.projects = cm_projects("wrong file") except: pass def test01_print(self): HEADING() print(self.projects) pass def test02_dump(self): HEADING() print(self.projects.dump()) pass def test03_active(self): HEADING() print(self.projects.names("active")) pass def test04_default(self): HEADING() print(self.projects.names("default")) pass def test05_default(self): HEADING() print(self.projects.names("completed")) pass def test06_wrong_status(self): HEADING() try: print(self.projects.names("wrong")) except Exception, e: print(e) pass
def project_apply(): form = ProjectRegistrationForm(request.form) if request.method == 'POST' and form.validate(): data = dict(request.form) action = str(data['button'][0]) print "Project Data" print data for key in data: if key in [ 'agreement_use', 'agreement_slides', 'agreement_support', 'agreement_software', 'agreement_documentation', 'join_open', 'join_notification' ]: if str(data[key][0]) == 'y': data[key] = True elif str(data[key][0]) == 'n': data[key] = False elif key in [ 'category', 'managers', 'resources_clusters', 'alumni', 'resources_provision', 'resources_services', 'members', 'keywords' ]: print data[key][0] else: data[key] = data[key][0] print str(key) + " - " + str(data[key]) + "\n" del data['button'] if action == 'save': projects = Projects() project = MongoProject() for d in data: project[d] = data[d] print project projects.add(project) flash('Thanks for registering') return redirect('/') project_config = ConfigDict( filename=config_file("/cloudmesh_project_intf.yaml")) project_fields = project_config.get("cloudmesh.project") return render_template('management/project_apply.html', title="Project Application", states=['save', 'cancel'], fields=project_fields, countries_list=[c for c in country_list()], states_list=[c for c in states_list()], disciplines_list=[c for c in disciplines_list()])
def load(self, yaml_filename=None): """ Loads the cloudmesh pbs yaml file. :param yaml_filename: The filename of the yaml file """ log.debug("PBS yaml filename: {0}".format(self.yaml_filename)) if yaml_filename is None: yaml_filename = self.yaml_filename else: self.yaml_filename = config_file(yaml_filename) self.data = ConfigDict(filename=self.yaml_filename) self.hosts = ssh_config()
def _set_value_to_yaml(filepath, data): dir = config_file("") cm_file = dir + filepath # make a backup bak = _make_a_backup(cm_file) print "[%s] backup made" % bak # Write yaml with open(cm_file, 'w') as outfile: outfile.write(yaml.dump(data, default_flow_style=False)) print "[%s] updated" % cm_file
def activate_project(self, project): """ this routine is wrong and has been copied from a deprecated code""" self.credentials = credentials_rc("eucalyptus") self.credentials.location = config_file("/india/eucalyptus/") + \ project + "/eucarc" self.credentials.type('eucalyptus') self.access_key = self.credentials._get_rc_variable("accesskey") self.secret_key = self.credentials._get_rc_variable("secretkey") print(self.access_key) print(self.secret_key)
def get_rcfiles(fpath=None): dir = config_file("") read_values = ["OS_TENANT_NAME", "OS_USERNAME", "OS_PASSWORD", "OS_AUTH_URL"] # , "OS_CACERT"] rcfile_path = dir + "/clouds/" new_values = {} for filepath in glob.glob(rcfile_path + "/*/*rc"): filename = os.path.basename(filepath) cloud_name = os.path.basename( os.path.normpath(filepath.replace(filename, ""))) new_values[cloud_name] = get_variables(filepath, read_values) return new_values
def __init__(self, quiet=False): self.quiet = quiet try: self.filename = config_file("/cmd3.yaml") if not self.quiet: Console.ok("Reading " + self.filename + ". ok.") except: Console.error("problem loading cmd3.yaml file") try: self.config = ConfigDict(filename=self.filename) # print(self.config) except Exception, e: Console.error("problem with ConfigDict") print(e)