def setup_inventory(self): banner("Read Dicts") self.sample_user = ConfigDict(filename="~/.futuregrid/me.yaml") self.portalname = self.sample_user.get("portalname") print "PORTALNAME", self.portalname print "SAMPLE USER", self.sample_user banner("create user from template, duplicates cm init generate me") t = cm_template("~/.futuregrid/etc/cloudmesh.yaml") pprint (set(t.variables())) self.config = t.replace(kind="dict", values=self.sample_user) print type(self.config) print self.config # # BUG? # self.idp = cm_userLDAP () self.idp.connect("fg-ldap", "ldap") self.idp.refresh() ldap_info = self.idp.get(self.portalname) print ldap_info print type(self.config) self.config['cloudmesh']['projects'] = ldap_info['projects'] self.config['cloudmesh']['keys'] = ldap_info['keys'] try: self.config['cloudmesh']['projects']['deafult'] = ldap_info['projects']['active'][0] except: print "ERROR: you have no projects"
def start(auth=True): ''' start the mongod service in the location as specified in ~/.futuregrid/cloudmesh_server.yaml ''' banner("Starting mongod") config = cm_config_server().get("cloudmesh.server.mongo") path = path_expand(config["path"]) port = config["port"] if not os.path.exists(path): print "Creating mongodb directory in", path local("mkdir -p {0}".format(path)) with settings(warn_only=True): with hide('output', 'running', 'warnings'): lines = local("ps -ax |grep '[m]ongod.*port {0}'".format(port), capture=True).split("\n") if lines != ['']: pid = lines[0].split(" ")[0] print "NO ACTION: mongo already running in pid {0} for port {1}".format(pid, port) else: print "ACTION: Starting mongod" with_auth = "" if isyes(auth): with_auth = "--auth" local("mongod {2} --bind_ip 127.0.0.1 --fork --dbpath {0} --logpath {0}/mongodb.log --port {1}".format(path, port, with_auth))
def start(auth=True): ''' start the mongod service in the location as specified in ~/.futuregrid/cloudmesh_server.yaml ''' banner("Starting mongod") config = cm_config_server().get("cloudmesh.server.mongo") path = path_expand(config["path"]) port = config["port"] if not os.path.exists(path): print "Creating mongodb directory in", path local("mkdir -p {0}".format(path)) with settings(warn_only=True): with hide('output', 'running', 'warnings'): lines = local("ps -ax |grep '[m]ongod.*port {0}'".format(port), capture=True).split("\n") if lines != ['']: pid = lines[0].split(" ")[0] print "NO ACTION: mongo already running in pid {0} for port {1}".format( pid, port) else: print "ACTION: Starting mongod" with_auth = "" if isyes(auth): with_auth = "--auth" local( "mongod {2} --bind_ip 127.0.0.1 --fork --dbpath {0} --logpath {0}/mongodb.log --port {1}" .format(path, port, with_auth))
def setup_inventory(self): banner("Read Dicts") self.sample_user = ConfigDict(filename="~/.futuregrid/me.yaml") self.portalname = self.sample_user.get("portalname") print "PORTALNAME", self.portalname print "SAMPLE USER", self.sample_user banner("create user from template, duplicates cm init generate me") t = cm_template("~/.futuregrid/etc/cloudmesh.yaml") pprint(set(t.variables())) self.config = t.replace(kind="dict", values=self.sample_user) print type(self.config) print self.config # # BUG? # self.idp = cm_userLDAP() self.idp.connect("fg-ldap", "ldap") self.idp.refresh() ldap_info = self.idp.get(self.portalname) print ldap_info print type(self.config) self.config['cloudmesh']['projects'] = ldap_info['projects'] self.config['cloudmesh']['keys'] = ldap_info['keys'] try: self.config['cloudmesh']['projects']['deafult'] = ldap_info[ 'projects']['active'][0] except: print "ERROR: you have no projects"
def fetchrc(userid=None, outdir=None): banner("download rcfiles (novarc, eucarc, etc) from IaaS platforms") print "" # Task 1. list portal user id ''' try: from cloudmesh.config.ConfigDict import ConfigDict except Exception, e: print "ERROR: your have not yet configured cloudmesh completely. " print " Have you called" print print " ./install cloudmesh" print sys.exit(1) dir = config_file("") config = ConfigDict(dir + "/me.yaml") userid = config["portalname"] ''' if not userid: userid = getpass.getuser() userid = raw_input ("Please enter your portal user id [default: %s]: " % userid) or userid # Task 2. list hostnames to get access. In Futuregrid, india, sierra are # mandatory hosts to be included. host_ids = ["india_openstack_havana", "sierra_openstack_grizzly"] # TEMPORARY # user input is disabled # host_ids = raw_input("Please enter host identifications [default: %s]: " # % ", ".join(host_ids)) or host_ids if isinstance(host_ids, str): host_ids = map(lambda x : x.strip(), host_ids.split(",")) domain_name = ".futuregrid.org" hostnames = map(lambda x : x.split("_")[0] + domain_name, host_ids) key_path = "~/.ssh/id_rsa" # private key path is disabled # key_path = raw_input("Please enter a path of the ssh private key to" + \ # " login the hosts [default: %s]: " % key_path) or \ # key_path try: # cmd = "fab rcfile.download:userid='%s',host_ids='%s',key_path='%s'" \ cmd = "fab -H %s -u %s -i %s rcfile.download:'%s','%s'" \ % (",".join(hostnames), userid, key_path, "\,".join(host_ids), outdir) # print cmd os.system(cmd) except: print sys.exc_info() sys.exit(1)
def fetchrc(userid=None, outdir=None): banner("download rcfiles (novarc, eucarc, etc) from IaaS platforms") print "" # Task 1. list portal user id ''' try: from cloudmesh.config.ConfigDict import ConfigDict except Exception, e: print "ERROR: your have not yet configured cloudmesh completely. " print " Have you called" print print " ./install cloudmesh" print sys.exit(1) dir = config_file("") config = ConfigDict(dir + "/me.yaml") userid = config["portalname"] ''' if not userid: userid = getpass.getuser() userid = raw_input("Please enter your portal user id [default: %s]: " % userid) or userid # Task 2. list hostnames to get access. In Futuregrid, india, sierra are # mandatory hosts to be included. host_ids = ["india_openstack_havana", "sierra_openstack_grizzly"] # TEMPORARY # user input is disabled # host_ids = raw_input("Please enter host identifications [default: %s]: " # % ", ".join(host_ids)) or host_ids if isinstance(host_ids, str): host_ids = map(lambda x: x.strip(), host_ids.split(",")) domain_name = ".futuregrid.org" hostnames = map(lambda x: x.split("_")[0] + domain_name, host_ids) key_path = "~/.ssh/id_rsa" # private key path is disabled # key_path = raw_input("Please enter a path of the ssh private key to" + \ # " login the hosts [default: %s]: " % key_path) or \ # key_path try: # cmd = "fab rcfile.download:userid='%s',host_ids='%s',key_path='%s'" \ cmd = "fab -H %s -u %s -i %s rcfile.download:'%s','%s'" \ % (",".join(hostnames), userid, key_path, "\,".join(host_ids), outdir) # print cmd os.system(cmd) except: print sys.exc_info() sys.exit(1)
def test_gregor(self): banner("ME") id = ConfigDict(filename="~/.futuregrid/me.yaml").get("portalname") user = cm_user() result = user.info(id) pprint (result) pass
def quick(server="server", browser='yes'): """ starts in dir webgui the program server.py and displays a browser on the given port and link""" banner("INSTALL CLOUDMESH") local("python setup.py install") banner("START WEB SERVER") local("cd webui; python {0}.py &".format(server))
def test_gregor(self): banner("ME") id = ConfigDict(filename="~/.futuregrid/me.yaml").get("portalname") user = cm_user() result = user.info(id) pprint(result) pass
def mongo(): register() filename = "~/.futuregrid/cloudmesh.yaml" banner("reding data from {0}".format(filename)) config = cm_config(filename=filename) profile = config.profile() element = { "firstname" : profile["firstname"], "lastname" : profile["lastname"], "uidNumber" : profile["uid"], "phone" : profile["phone"], "gidNumber" : profile["gid"], "address" : profile["address"][0], "cm_user_id" : config.get("cloudmesh.hpc.username"), "email" : profile["email"], "activeclouds" : config.get("cloudmesh.active") } projects = {} active = config.get("cloudmesh.projects.active") if active != ['None']: projects["active"] = active completed = config.get("cloudmesh.projects.completed") if completed != ['None']: projects["completed"] = completed if projects != {}: element["projects"] = projects # get keys and clean the key titles (replace '.' with '_' due to mongo restriction) keys = config.get("cloudmesh.keys.keylist") for keytitle in keys.keys(): if "." in keytitle: keycontent = keys[keytitle] newkeytitle = keytitle.replace(".", "_") del keys[keytitle] keys[newkeytitle] = keycontent element['keys'] = keys pprint (element) # hpc username as key username = element["cm_user_id"] # populate the local userinfo into the same mongo as though it were from LDAP. userstore = cm_userLDAP() userstore.updates(username, element) user_obj = cm_user() user_obj.init_defaults(username)
def delete_defaults(): filename = "~/.futuregrid/cloudmesh.yaml" banner("reding data from {0}".format(filename)) config = cm_config(filename=filename) username = config.get("cloudmesh.hpc.username") print username user = cm_user() user.set_defaults(username, {}) # user.set_default_attribute(username, 'images', {}) info(username)
def test_mongo_credential(self): banner("USER") pprint(self.user.info("gvonlasz")) username = "******" cloudname = "dummy" password = "******" tennant = "fg1" name = username self.user.set_credential( username, cloudname, { "OS_USERNAME": name, "OS_PASSWORD": password, "OS_TENANT_NAME": tennant, "CM_CLOUD_TYPE": "openstack" }) cred = self.user.get_credential(username, cloudname) banner("credential") print cred banner("credentials") pprint(self.user.get_credentials(username)) banner("find") result = self.user.userdb_passwd.find({}) for r in result: pprint(r)
def test_mongo_credential(self): banner("USER") pprint (self.user.info("gvonlasz")) username = "******" cloudname = "dummy" password = "******" tennant = "fg1" name = username self.user.set_credential(username, cloudname, {"OS_USERNAME": name, "OS_PASSWORD": password, "OS_TENANT_NAME": tennant, "CM_CLOUD_TYPE": "openstack" } ) cred = self.user.get_credential(username, cloudname) banner("credential") print cred banner("credentials") pprint(self.user.get_credentials(username)) banner("find") result = self.user.userdb_passwd.find({}) for r in result: pprint (r)
def login(): error = None form = LoginForm() if request.method == 'POST' and form.validate_on_submit(): form.error = None try: # idp = cm_userLDAP () # idp.connect("fg-ldap", "ldap") # user = idp.find_one({'cm_user_id': form.username.data}) # print "MONGO USER" # banner("LDAPUSER") # pprint (user) banner("CM_USER") user_obj = cm_user() user = user_obj.info(form.username.data) except Exception, e: print traceback.format_exc() error = "LDAP server not reachable" error += str(e) return render_template('error.html', form=form, type="Can not reach LDAP", msg="") if user is None: form.error = 'Login Invalid' elif user['cm_user_id'] != form.username.data: form.error = 'Login Invalid' elif user_obj.authenticate(form.username.data, form.password.data): print "LOGIN USER" g.user = load_user(form.username.data) ret = login_user(g.user) identity_changed.send(current_app._get_current_object(), identity=Identity(g.user.id)) return redirect(request.args.get('next') or '/') else: form.error = 'Login Invalid'
def replace(self, kind="text", values=None): try: template = Template(self.content) if kind == "text": self.result = template.render(**values) elif kind == "dict": self.result = yaml.safe_load(template.render(**values)) else: log.error("kind='dict' or 'text' parameter missing in template replace") raise RuntimeError return self.result except UndefinedError, e: banner("ERROR: Undefined variable in template") print e
def replace(self, kind='text', values=None): try: template = Template(self.content) if kind == "text": self.result = template.render(**values) elif kind == "dict": self.result = yaml.safe_load(template.render(**values)) else: log.error("kind='dict' or 'text' parameter missing in template replace") raise RuntimeError return self.result except UndefinedError, e: banner ("ERROR: Undefined variable in template") print e
def start(server="server", browser='yes'): """ starts in dir webgui the program server.py and displays a browser on the given port and link""" banner("KILL THE SERVER") kill() banner("INSTALL CLOUDMESH") local("python setup.py install") banner("START MONGO") local("fab mongo.start") banner("SATRT RABITMQ") local("fab queue.start") banner("START WEB SERVER") local("cd webui; python {0}.py &".format(server))
def shell_command_user(arguments): """ Usage: user list user info [ID] Administrative command to lists the users from LDAP Arguments: list list the users ID list the user with the given ID Options: -v verbose mode """ user = cm_user() if arguments["info"]: id = arguments["ID"] if id is None: config = cm_config() id = config.username() banner("User Information in Mongo for user: {0}".format(id)) user = cm_user() result = user.info(id) pprint(result) elif arguments["list"]: user = cm_user() list_of_users = user.list_users() pprint(list_of_users) print print "=========================" num = len(list_of_users) print str(num) + " users listed" else: print "WRONG PARAMETERS" return
def shell_command_user(arguments): """ Usage: user list user info [ID] Administrative command to lists the users from LDAP Arguments: list list the users ID list the user with the given ID Options: -v verbose mode """ user = cm_user() if (arguments["info"]): id = arguments["ID"] if id is None: config = cm_config() id = config.username() banner("User Information in Mongo for user: {0}".format(id)) user = cm_user() result = user.info(id) pprint(result) elif (arguments["list"]): user = cm_user() list_of_users = user.list_users() pprint(list_of_users) print print "=========================" num = len(list_of_users) print str(num) + " users listed" else: print "WRONG PARAMETERS" return
class cm_template(): def __init__(self, filename): self.filename = path_expand(filename) self.content = open(self.filename, 'r').read() def variables(self): vars = list() lines = self.content.splitlines() for line in lines: if "{{" in line: words = line.split("{{") for word in words: if "}}" in word: name = word.split("}}")[0].strip() vars.append(name) return vars def grep(self, strip=True): result = [] s = set(self.variables()) for attribute in s: grep_result = _grep("-n", attribute, cloudmesh_yaml).split("\n") for r in grep_result: if "{" in r: result.append(str(r).replace(" ", "").replace(":", ": ", 1)) return result def _variables(self): env = Environment() parsed_content = env.parse(self.content) print meta.find_undeclared_variables(parsed_content) def replace(self, kind='text', values=None): try: template = Template(self.content) if kind == "text": self.result = template.render(**values) elif kind == "dict": self.result = yaml.safe_load(template.render(**values)) else: log.error("kind='dict' or 'text' parameter missing in template replace") raise RuntimeError return self.result except UndefinedError, e: banner ("ERROR: Undefined variable in template") print e except Exception, e: banner ("ERROR") print e print sys.exc_info() # return self.content return None
def boot(auth=True): # kill mongo kill() time.sleep(1) # wipe mongo wipe() time.sleep(1) # start mongo without auth start(auth=False) time.sleep(1) if isyes(auth): # create users admin() time.sleep(1) # restart with auth kill() time.sleep(10) start(auth=auth) time.sleep(1) config = cm_config_server().get("cloudmesh.server.mongo") path = path_expand(config["path"]) banner(path) print ls(path) banner("PROCESS") with settings(warn_only=True): local("ps -ax | fgrep mongo")
def shell_command_label(arguments): """ Usage: label [--prefix=PREFIX] [--id=ID] [--width=WIDTH] A command to set the prefix and id for creating an automatic lable for VMs. Without paremeter it prints the currect label. Arguments: PREFIX The prefix for the label ID The start ID which is an integer WIDTH The width of the ID in teh label, padded with 0 Options: -v verbose mode """ print arguments banner("not yet implemented") return
def refresh_qstat(self, host): ''' obtains a refreshed qstat data set from the specified host. The result is written into the mongo db. :param host: The host on which to execute qstat ''' time_now = datetime.now() data = dict(self.hosts[host].qstat(refresh=True)) print "KKKKK", data.keys() self.db_qstat.remove({"cm_host": host, "cm_kind" : "qstat"}, safe=True) for name in data: banner(name) banner(data[name].keys()) for job in data[name]: entry = data[name][job] id = "{0}-{1}-qstat".format(host, name).replace(".", "-") entry["cm_host"] = name entry["cm_kind"] = "qstat" entry["cm_id"] = id entry["cm_qstat"] = host entry["cm_refresh"] = time_now print "Insert", job self.db_qstat.insert(data[name][job])
def refresh_qstat(self, host): ''' obtains a refreshed qstat data set from the specified host. The result is written into the mongo db. :param host: The host on which to execute qstat ''' time_now = datetime.now() data = dict(self.hosts[host].qstat(refresh=True)) print "KKKKK", data.keys() self.db_qstat.remove({"cm_host": host, "cm_kind": "qstat"}, safe=True) for name in data: banner(name) banner(data[name].keys()) for job in data[name]: entry = data[name][job] id = "{0}-{1}-qstat".format(host, name).replace(".", "-") entry["cm_host"] = name entry["cm_kind"] = "qstat" entry["cm_id"] = id entry["cm_qstat"] = host entry["cm_refresh"] = time_now print "Insert", job self.db_qstat.insert(data[name][job])
def cm_image_command(arguments): """ cm-image admin on HOSTS cm-image admin off HOSTS """ path = path_expand(definitions[0]) if arguments["version"]: print cloudmesh.__version__ elif arguments["info"]: banner("info") banner("System", c='-') print "Kind: ", arguments['--kind'] print "Path: ", path print "Version:", cloudmesh.__version__ banner("List of templates", c='-') system_name = None for definition in definitions: try: path = path_expand(definition) if os.path.exists(path): os.system("cd '%s' ; veewee vbox list" % path) else: print "WARNING: path", path, "does not exist" except KeyError, key: print 'WARNING: no environment variable called', key, 'found' print print "To build one, please use one of the" print print " cm-image build OS" print print "Next you need to register the image" print print " cm-image register OS" print print "where OS is one of the labels listed above." print
def admin(): """creates a password protected user for mongo""" banner("create auth user") config = cm_config_server().get("cloudmesh.server.mongo") user = config["username"] password = config["password"] # # setting up the list of dbs # dbs = set() # print config["collections"] for collection in config["collections"]: dbs.add(config['collections'][collection]['db']) # setting the admin user script = [] script.append('db.addUser("{0}", "{1}");'.format(user, password)) script.append('db.auth("{0}", "{1}");'.format(user, password)) # setting a password for each db for db in dbs: script.append('db = db.getSiblingDB("{0}");'.format(db)) script.append('db.addUser("{0}", "{1}");'.format(user, password)) script.append("use admin;") script.append('db.addUser("{0}", "{1}");'.format(user, password)) script.append('db.auth("{0}", "{1}");'.format(user, password)) script.append('db.shutdownServer();') mongo_script = '\n'.join(script) # print mongo_script command = "echo -e '{0}' | mongo".format(mongo_script) print command banner("Executing js") os.system(command) banner ("Debugging reminder, remove in final version") print "USER", user print "PASSWORD", password
def admin(): """creates a password protected user for mongo""" banner("create auth user") config = cm_config_server().get("cloudmesh.server.mongo") user = config["username"] password = config["password"] # # setting up the list of dbs # dbs = set() # print config["collections"] for collection in config["collections"]: dbs.add(config['collections'][collection]['db']) # setting the admin user script = [] script.append('db.addUser("{0}", "{1}");'.format(user, password)) script.append('db.auth("{0}", "{1}");'.format(user, password)) # setting a password for each db for db in dbs: script.append('db = db.getSiblingDB("{0}");'.format(db)) script.append('db.addUser("{0}", "{1}");'.format(user, password)) script.append("use admin;") script.append('db.addUser("{0}", "{1}");'.format(user, password)) script.append('db.auth("{0}", "{1}");'.format(user, password)) script.append('db.shutdownServer();') mongo_script = '\n'.join(script) # print mongo_script command = "echo -e '{0}' | mongo".format(mongo_script) print command banner("Executing js") os.system(command) banner("Debugging reminder, remove in final version") print "USER", user print "PASSWORD", password
def wipe(): """wipes out all traces from mongo""" kill() config = cm_config_server().get("cloudmesh.server.mongo") path = path_expand(config["path"]) banner("{0}".format(path)) local("mkdir -p {0}".format(path)) result = str(ls(path)) banner(path, "-") print result print 70 * "-" if result != "": if yn_choice("deleting the directory", default="n"): local("rm -rf {0}".format(path)) local("mkdir -p {0}".format(path)) banner("{0}".format(path)) local("ls {0}".format(path))
def test_all(self): HEADING() for host in self.hosts: banner(host) self.get_qstat(host)
def cm_manage(): """Usage: cm-manage config projects list cm-manage config projects cm-manage config [-f FILE] [-o OUT] [-p PROJECT] cloud NAME [-] cm-manage config dump [--format=(yaml|dict)] cm-manage config init [-o OUT] [-u USER] cm-manage config list cm-manage config password NAME cm-manage config show passwords cm-manage config fetch [-u USER] [-r HOST] cm-manage --version cm-manage --help Arguments: NAME name of the cloud Options: -h --help show this help message and exit -v --version show version and exit -f NAME --file=NAME the Name of the cloud to be specified, if ? a selection is presented -o OUT --out=OUT writes the result in the specifide file -p PROJECT --project=PROJECT selects a project (e.g. for eucalyptus which has project-specific environments) -u USER --user=USER the user (login) name -r HOST --remote=HOST the host machine on which the yaml file is located in the ~/.futuregrid directory [default: sierra.futuregrid.org] -d --debug debug - this option is a - at the end of the command. If data is written to a file it is also put out to stdout Description: Command to generate rc files from our cloudmesh configuration files. This program generates form a YAML file containing the login information for a cloud an rc file that can be used to later source it. Example: we assume the yaml file has an entry india-openstack:: cm-manage config -o novarc india-openstack source novarc This will create a novarc file and than you can source it:: cm-manage config ? - Presents a selction of cloud choices and writes the choice into a file called ~/.futuregrid/novarc """ default_path = '.futuregrid/novarc' arguments = docopt(cm_manage.__doc__) DEBUG("arguments", arguments) home = os.environ['HOME'] # DEBUG("home", home) # # This secion deals with handeling "cm config" related commands ###################################################################### is_config = arguments['config'] != None if is_config: # DEBUG('Arguments', arguments) file = arguments['--file'] try: config = cm_config(file) # DEBUG("config", config) except IOError: print "{0}: Configuration file '{1}' not found".format("CM ERROR", file) sys.exit(1) except (yaml.scanner.ScannerError, yaml.parser.ParserError) as yamlerror: print "{0}: YAML error: {1}, in configuration file '{2}'".format("CM ERROR", yamlerror, file) sys.exit(1) except: print "Unexpected error:", sys.exc_info()[0] sys.exit(1) name = arguments['NAME'] # # NOT TESTED # if arguments['fetch'] or name == 'fetch': DEBUG('Arguments', arguments) # get user var = {} var['user'] = arguments['--user'] var['host'] = arguments['--remote'] var['file'] = ".futuregrid/cloudmesh.yaml" if var['user'] is None: var['user'] = getpass.getuser() from_location = "%(user)s@%(host)s:%(file)s" % var to_location = config_file("/%(file)s" % var) if os.path.isfile(to_location): print "WARNING: The file %s exists" % to_location if not yn_choice("Would you like to replace the file", default='y'): sys.exit(1) print from_location print to_location print "Copy cloudmesh file from %s to %s" % (from_location, to_location) result = scp(from_location, to_location) print result sys.exit(0) # # ok # # if (arguments['projects'] and arguments['list']) : if arguments['projects'] and arguments['list']: projects = config.get('cloudmesh.projects') print yaml.dump(projects, default_flow_style=False, indent=4) sys.exit(0) # # OK, needs setting # if arguments['projects']: projects = config.projects('active') print "Please select from the following:" print "0 - Cancel" selected = False choices = [] while not selected: counter = 1 for name in projects: print counter, "-" "%20s" % name choices.append(name) counter += 1 print "Please select:" input = int(sys.stdin.readline()) if input == 0: print "Selection terminated" sys.exit(0) selected = (input > 0) and (input < counter) print "Selected: ", input name = choices[input - 1] print name print "ERROR: THIS JUST SELECTS A PROJECT ID BUT DOES NOT SET IT" sys.exit(0) if arguments['init'] or name == 'init': output = arguments['--out'] username = arguments['--user'] or os.getenv('USER') location = path_expand(output) new_yaml_file = open(location, 'w+') user_yaml = cm_user().generate_yaml(username, 'cloudmesh') print >> new_yaml_file, yaml.dump(user_yaml, default_flow_style=False) new_yaml_file.close() print "Written new yaml file in " + location sys.exit(0) # # OK # if arguments['list'] or name == 'list': for name in config.cloudnames(): if 'cm_type' in config.cloud(name): print name, "(%s)" % config.cloud(name)['cm_type'] sys.exit(0) # # NOT TESTED # if arguments['password']: oldpass = getpass("Current password: "******"New password: "******"New password (again): ") # TODO: some kind of password strength checking? if newpass1 == newpass2: config.change_own_password(name, oldpass, newpass1) else: print "New passwords did not match; password not changed." sys.exit(0) # # OK, but does not display the username # if arguments['show'] or name == 'show' and arguments['passwords']: warning = "Your passwords will appear on the screen. Continue?" if yn_choice(warning, 'n'): me = ConfigDict(filename=config_file("/.futuregrid/me.yaml")) banner("PASSWORDS") for name in me['password']: print "{0}: {1}".format(name, me['password'][name]) sys.exit(0) # # OK # if arguments['dump'] or name == 'dump': format = arguments['--format'] if format == 'yaml': d = dict(config) print yaml.dump(d, default_flow_style=False) elif format == 'dict' or format is None: print config sys.exit(0) # # NOT TESTED # if name in ['?', 'x']: if file is None: arguments['--out'] = "%s/%s" % (home, default_path) print "Please select from the following:" print "0 - Cancel" selected = False choices = [] while not selected: counter = 1 for name in config.cloudnames(): if 'cm_type' in config.cloud(name): print "{0} - {1:<30} ({2})".format(counter, name, config.cloud(name)['cm_type']) choices.append(name) counter += 1 print "Please select:" input = int(sys.stdin.readline()) if input == 0: print "Selection terminated" sys.exit(0) selected = (input > 0) and (input < counter) print "Selected: ", input name = choices[input - 1] output = arguments['--out'] # # OK # if name is not None: cloud = config.cloud(name) if not cloud: print "%s: The cloud '%s' is not defined." % ("CM ERROR", name) print "Try instead:" for keyname in config.cloudnames(): print " ", keyname sys.exit(1) if cloud['cm_type'] == 'eucalyptus': if arguments['--project']: project = arguments['--project'] if not project in cloud: print "No such project %s defined in cloud %s." % (project, name) sys.exit(1) else: project = config.cloud_default( name, 'project') or config.projects('default') if not project in cloud: print "Default project %s not defined in cloud %s." % (project, name) sys.exit(1) rc_func = lambda name: config.rc_euca(name, project) else: rc_func = config.rc result = rc_func(name) # # OK # if arguments["-"]: print result else: if output is None: arguments['--out'] = "%s/%s" % (home, default_path) output = arguments['--out'] out = False if yn_choice("Would you like to review the information", default="y"): banner("WARNING: FIle will be written to " + output) print result print banner("") try: # First we try to open the file assuming it doesn't exist out = os.open( output, os.O_CREAT | os.O_EXCL | os.O_WRONLY, stat.S_IRUSR | stat.S_IWUSR) except OSError as oserr: # If file exists, offer to replace it if oserr.errno == 17: delete = raw_input( "'%s' exists; Overwrite it [N|y]? " % output) if delete.strip().lower() == 'y': out = os.open(output, os.O_TRUNC | os.O_WRONLY) if out: os.write(out, result) os.close(out)
print print "To build one, please use one of the" print print " cm-image build OS" print print "Next you need to register the image" print print " cm-image register OS" print print "where OS is one of the labels listed above." print elif arguments["build"]: banner("build") system_name = arguments["OS"] if arguments['--gui']: gui = "" else: gui = '--nogui' if arguments['--kind'] == "vbox": os.system("cd '%s' ; veewee vbox build '%s' --force %s" % (path, system_name, gui)) # due to some bug the following does not work # os.system("veewee vbox build %s --workdir='%s' --force" % (path, system_name) else: print "ERROR: wrong options"
result = t.replace(kind="dict", values=user_config) location = path_expand(out_file) yaml_file = open(location, "w+") print >> yaml_file, yaml.dump(result, default_flow_style=False) yaml_file.close() log.info("Written new yaml file in " + location) if __name__ == "__main__": cloudmesh_yaml = config_file("/etc/cloudmesh.yaml") user_config = ConfigDict(filename=config_file("/me.yaml")) t = cm_template(cloudmesh_yaml) banner("VARIABLES") s = set(t.variables()) print ("\n".join(s)) banner("GREP") s = t.grep() print ("\n".join(s)) # banner("YAML FILE") # result = t.replace(kind="dict", values=user_config) # print yaml.dump(result, default_flow_style=False) # location = config_file('/cloudmesh-new.yaml') # yaml_file = open(location, 'w+') # print >> yaml_file, yaml.dump(result, default_flow_style=False) # yaml_file.close() # print "Written new yaml file in " + location
# values = yaml.safe_load(Template(result).substitute(os.environ)) # print json.dumps(values, indent=4) except Exception, e: print "ERROR: There is an error in the yaml file", e sys.exit(1) for cloud in values['clouds']: values['clouds'][cloud]['default'] = {} values['clouds'][cloud]['default']['image'] = None values['clouds'][cloud]['default']['flavor'] = None file_from_template(cloudmesh_template, cloudmesh_out, values) print "# Created: {0}".format(me_file) banner(c="-") # sys.exit() # # format = "yaml" # if format in ["json"]: # result = json.dumps(values, indent=4) # elif format in ["yaml", "yml"]: # result = yaml.dump(values, default_flow_style=False) # banner("done", c="-") # print "# Template: {0}".format(filename_template) # print "# Values : {0}".format(filename_values) # print "# Backup : {0}".format(filename_bak)
def install_command(args): """ Usage: install -h | --help install --version install cloudmesh install delete_yaml install system install query install new install apply_credentials install vagrant install rc fetch [--username=<username>] [--outdir=<outdir>] install rc fill install rc login [--username=<username>] """ arguments = docopt(install_command.__doc__, args) if arguments["cloudmesh"]: deploy() elif arguments["new"]: new_cloudmesh_yaml() elif arguments["delete_yaml"]: answer = yn_choice("THIS COMMAND IS REAL DANGEROUS AND WILL DELETE ALL YOUR YAML FILE. Proceed", default='y') if answer: print "You fool we just deleted your yaml files" cp("etc/*.yaml", "~/.futuregrid/") else: print "puuh you interrupted" pass elif arguments["system"]: banner("Installing Ubuntu System Requirements") if is_ubuntu(): ubuntu() elif is_osx(): osx() elif is_centos(): centos() elif arguments["query"]: import platform print "System: ", platform.system() # print "Uname: ", platform.uname() print "Machine: ", platform.machine() print "Processor: ", platform.processor() print "Platform: ", platform.platform() print "Python: ", platform.python_version() print "Virtualenv:", hasattr(sys, 'real_prefix') elif arguments["vagrant"]: vagrant() elif arguments["rc"] and arguments["fetch"]: fetchrc(arguments["--username"], arguments["--outdir"]) elif arguments["rc"] and arguments["fill"]: get_fg_username_password_from_rcfiles() elif arguments["rc"] and arguments["login"]: verify_ssh_login(arguments["--username"])
def init_shell_command(arguments): """ Usage: init [--force] generate yaml init [--force] generate me init [--force] generate none init [--force] generate FILENAME init list [KIND] [--json] init list clouds [--file=FILENAME] [--json] init inspect --file=FILENAME init fill --file=FILENAME [VALUES] Initializes cloudmesh from a yaml file Arguments: generate generates a yaml file yaml specifies if a yaml file is used for generation the file is located at ~/.futuregrid/me.yaml me same as yaml none specifies if a yaml file is used for generation the file is located at ~/.futuregrid/etc/none.yaml FILENAME The filename to be generated or from which to read information. VALUES yaml file with the velues to be sed in the FILENAME KIND The kind of the yaml file. Options: --force force mode does not ask. This may be dangerous as it overwrites the ~/.futuregrid/cloudmesh.yaml file --file=FILENAME The file --json make the output format json -v verbose mode Description: init list [KIND] [--json] list the versions and types of the yaml files in the ~/.futuregrid and ~/.futuregrid/etc directories. init list clouds [--file=FILENAME] Lists the available clouds in the configuration yaml file. init inspect --file=FILENAME print the variables in the yaml template """ if arguments["inspect"]: filename = arguments['--file'] if filename is None: filename = config_file('/cloudmesh.yaml') content = open(filename, 'r').read() t = cm_template(filename) sorted_vars = sorted(set(t.variables())) print "\n".join(sorted_vars) # banner("PARSER") # env = Environment() # ast = env.parse(content) # for v in meta.find_undeclared_variables(ast): # print v if arguments["list"] and not arguments["clouds"]: dirs = [path_expand(config_dir + '/*.yaml'), path_expand(config_dir + '/etc/*.yaml')] file_list = [] for dir in dirs: file_list.extend(glob.glob(dir)) vector = {} vector['kind'] = [] vector['yaml_version'] = [] vector['meta'] = [] vector['filename'] = [] for filename in file_list: values = {'kind': "-", 'yaml_version': "-", 'meta': "-"} head_of_file = find_meta (filename) values = {'kind': "-", 'yaml_version': "-", 'meta': "-"} for line in head_of_file: if ":" in line: (attribute, value) = line.strip().split(":") if attribute in ["kind","yaml_version"]: values[attribute] = value.strip() if attribute in ["meta"]: values[attribute] = "+" if arguments["KIND"] is None or values['kind'] == arguments['KIND']: for attribute in values.keys(): vector[attribute].append(values[attribute]) vector['filename'].append(filename) vector['Kind'] = vector.pop('kind') vector['Version'] = vector.pop('yaml_version') vector['Meta'] = vector.pop('meta') vector['Filename'] = vector.pop('filename') banner("Configuration Directory: {0}".format(config_dir), c="-") print column_table(vector) #print filename, values["kind"], values["version"] if arguments["list"] and arguments["clouds"]: filename = arguments['--file'] if filename is None: filename = config_file('/cloudmesh.yaml') config = cm_config(filename) data = {} data['Clouds'] = config.cloudnames() data['Labels'] = [] data['Type'] = [] data['Version'] = [] for cloud_key in data['Clouds']: data['Labels'].append(config.cloud(cloud_key)['cm_label']) data['Type'].append(config.cloud(cloud_key)['cm_type']) data['Version'].append(config.cloud(cloud_key)['cm_type_version']) if arguments["--json"]: print json.dumps(data, sort_keys=True, indent=4) else: print column_table(data, ['Labels','Clouds','Type','Version']) if arguments["fill"]: filename_template = arguments['--file'] if filename_template is None: filename_template = config_file('/etc/cloudmesh.yaml') filename_template = path_expand(filename_template) filename_values = arguments['VALUES'] if filename_values is None: filename_values = config_file('/me.yaml') content = open(filename_template, 'r').read() t = cm_template(filename_template) sorted_vars = sorted(set(t.variables())) try: values = ConfigDict(filename=filename_values) except Exception, e: print "ERROR: There is an error in the yaml file", e for cloud in values['clouds']: values['clouds'][cloud]['default'] = {} values['clouds'][cloud]['default']['image'] = None values['clouds'][cloud]['default']['flavor'] = None banner("%s -> %s" % (filename_values, filename_template)) env = Environment(undefined=IgnoreUndefined) template = env.from_string(content) result = template.render(values) print result
def lsl(): print pbs_queue banner("Dict") pprint(l_queue.__dict__) i = l_queue.control.inspect() c = l_queue.control banner("Active Queues") pprint(i.active_queues()) banner("Registered") pprint(i.registered()) banner("Active") pprint(i.active()) banner("Scheduled") pprint(i.scheduled()) banner("Reserved") pprint(i.reserved()) # banner("Revoked") # pprint (i.resoked()) banner("Stats") pprint(i.stats()) banner("Ping") pprint(c.ping(timeout=0.5))
def test_qinfo(self): for host in self.hosts: banner(host) self.get_qinfo(host)
def cm_manage(): """Usage: cm-manage config projects list cm-manage config projects cm-manage config [-f FILE] [-o OUT] [-p PROJECT] cloud NAME [-] cm-manage config dump [--format=(yaml|dict)] cm-manage config init [-o OUT] [-u USER] cm-manage config list cm-manage config password NAME cm-manage config show passwords cm-manage config fetch [-u USER] [-r HOST] cm-manage --version cm-manage --help Arguments: NAME name of the cloud Options: -h --help show this help message and exit -v --version show version and exit -f NAME --file=NAME the Name of the cloud to be specified, if ? a selection is presented -o OUT --out=OUT writes the result in the specifide file -p PROJECT --project=PROJECT selects a project (e.g. for eucalyptus which has project-specific environments) -u USER --user=USER the user (login) name -r HOST --remote=HOST the host machine on which the yaml file is located in the ~/.futuregrid directory [default: sierra.futuregrid.org] -d --debug debug - this option is a - at the end of the command. If data is written to a file it is also put out to stdout Description: Command to generate rc files from our cloudmesh configuration files. This program generates form a YAML file containing the login information for a cloud an rc file that can be used to later source it. Example: we assume the yaml file has an entry india-openstack:: cm-manage config -o novarc india-openstack source novarc This will create a novarc file and than you can source it:: cm-manage config ? - Presents a selction of cloud choices and writes the choice into a file called ~/.futuregrid/novarc """ default_path = '.futuregrid/novarc' arguments = docopt(cm_manage.__doc__) DEBUG("arguments", arguments) home = os.environ['HOME'] # DEBUG("home", home) # # This secion deals with handeling "cm config" related commands ###################################################################### is_config = arguments['config'] != None if is_config: # DEBUG('Arguments', arguments) file = arguments['--file'] try: config = cm_config(file) # DEBUG("config", config) except IOError: print "{0}: Configuration file '{1}' not found".format( "CM ERROR", file) sys.exit(1) except (yaml.scanner.ScannerError, yaml.parser.ParserError) as yamlerror: print "{0}: YAML error: {1}, in configuration file '{2}'".format( "CM ERROR", yamlerror, file) sys.exit(1) except: print "Unexpected error:", sys.exc_info()[0] sys.exit(1) name = arguments['NAME'] # # NOT TESTED # if arguments['fetch'] or name == 'fetch': DEBUG('Arguments', arguments) # get user var = {} var['user'] = arguments['--user'] var['host'] = arguments['--remote'] var['file'] = ".futuregrid/cloudmesh.yaml" if var['user'] is None: var['user'] = getpass.getuser() from_location = "%(user)s@%(host)s:%(file)s" % var to_location = config_file("/%(file)s" % var) if os.path.isfile(to_location): print "WARNING: The file %s exists" % to_location if not yn_choice("Would you like to replace the file", default='y'): sys.exit(1) print from_location print to_location print "Copy cloudmesh file from %s to %s" % (from_location, to_location) result = scp(from_location, to_location) print result sys.exit(0) # # ok # # if (arguments['projects'] and arguments['list']) : if arguments['projects'] and arguments['list']: projects = config.get('cloudmesh.projects') print yaml.dump(projects, default_flow_style=False, indent=4) sys.exit(0) # # OK, needs setting # if arguments['projects']: projects = config.projects('active') print "Please select from the following:" print "0 - Cancel" selected = False choices = [] while not selected: counter = 1 for name in projects: print counter, "-" "%20s" % name choices.append(name) counter += 1 print "Please select:" input = int(sys.stdin.readline()) if input == 0: print "Selection terminated" sys.exit(0) selected = (input > 0) and (input < counter) print "Selected: ", input name = choices[input - 1] print name print "ERROR: THIS JUST SELECTS A PROJECT ID BUT DOES NOT SET IT" sys.exit(0) if arguments['init'] or name == 'init': output = arguments['--out'] username = arguments['--user'] or os.getenv('USER') location = path_expand(output) new_yaml_file = open(location, 'w+') user_yaml = cm_user().generate_yaml(username, 'cloudmesh') print >> new_yaml_file, yaml.dump(user_yaml, default_flow_style=False) new_yaml_file.close() print "Written new yaml file in " + location sys.exit(0) # # OK # if arguments['list'] or name == 'list': for name in config.cloudnames(): if 'cm_type' in config.cloud(name): print name, "(%s)" % config.cloud(name)['cm_type'] sys.exit(0) # # NOT TESTED # if arguments['password']: oldpass = getpass("Current password: "******"New password: "******"New password (again): ") # TODO: some kind of password strength checking? if newpass1 == newpass2: config.change_own_password(name, oldpass, newpass1) else: print "New passwords did not match; password not changed." sys.exit(0) # # OK, but does not display the username # if arguments['show'] or name == 'show' and arguments['passwords']: warning = "Your passwords will appear on the screen. Continue?" if yn_choice(warning, 'n'): me = ConfigDict(filename=config_file("/.futuregrid/me.yaml")) banner("PASSWORDS") for name in me['password']: print "{0}: {1}".format(name, me['password'][name]) sys.exit(0) # # OK # if arguments['dump'] or name == 'dump': format = arguments['--format'] if format == 'yaml': d = dict(config) print yaml.dump(d, default_flow_style=False) elif format == 'dict' or format is None: print config sys.exit(0) # # NOT TESTED # if name in ['?', 'x']: if file is None: arguments['--out'] = "%s/%s" % (home, default_path) print "Please select from the following:" print "0 - Cancel" selected = False choices = [] while not selected: counter = 1 for name in config.cloudnames(): if 'cm_type' in config.cloud(name): print "{0} - {1:<30} ({2})".format( counter, name, config.cloud(name)['cm_type']) choices.append(name) counter += 1 print "Please select:" input = int(sys.stdin.readline()) if input == 0: print "Selection terminated" sys.exit(0) selected = (input > 0) and (input < counter) print "Selected: ", input name = choices[input - 1] output = arguments['--out'] # # OK # if name is not None: cloud = config.cloud(name) if not cloud: print "%s: The cloud '%s' is not defined." % ("CM ERROR", name) print "Try instead:" for keyname in config.cloudnames(): print " ", keyname sys.exit(1) if cloud['cm_type'] == 'eucalyptus': if arguments['--project']: project = arguments['--project'] if not project in cloud: print "No such project %s defined in cloud %s." % ( project, name) sys.exit(1) else: project = config.cloud_default( name, 'project') or config.projects('default') if not project in cloud: print "Default project %s not defined in cloud %s." % ( project, name) sys.exit(1) rc_func = lambda name: config.rc_euca(name, project) else: rc_func = config.rc result = rc_func(name) # # OK # if arguments["-"]: print result else: if output is None: arguments['--out'] = "%s/%s" % (home, default_path) output = arguments['--out'] out = False if yn_choice("Would you like to review the information", default="y"): banner("WARNING: FIle will be written to " + output) print result print banner("") try: # First we try to open the file assuming it doesn't exist out = os.open(output, os.O_CREAT | os.O_EXCL | os.O_WRONLY, stat.S_IRUSR | stat.S_IWUSR) except OSError as oserr: # If file exists, offer to replace it if oserr.errno == 17: delete = raw_input( "'%s' exists; Overwrite it [N|y]? " % output) if delete.strip().lower() == 'y': out = os.open(output, os.O_TRUNC | os.O_WRONLY) if out: os.write(out, result) os.close(out)
# values = yaml.safe_load(Template(result).substitute(os.environ)) # print json.dumps(values, indent=4) except Exception, e: print "ERROR: There is an error in the yaml file", e sys.exit(1) for cloud in values['clouds']: values['clouds'][cloud]['default'] = {} values['clouds'][cloud]['default']['image'] = None values['clouds'][cloud]['default']['flavor'] = None file_from_template(cloudmesh_template, cloudmesh_out, values) print "# Created: {0}".format(me_file) banner(c="-") # sys.exit() # # format = "yaml" # if format in ["json"]: # result = json.dumps(values, indent=4) # elif format in ["yaml", "yml"]: # result = yaml.dump(values, default_flow_style=False) # banner("done", c="-") # print "# Template: {0}".format(filename_template) # print "# Values : {0}".format(filename_values)
result = t.replace(kind="dict", values=user_config) location = path_expand(out_file) yaml_file = open(location, 'w+') print >> yaml_file, yaml.dump(result, default_flow_style=False) yaml_file.close() log.info("Written new yaml file in " + location) if __name__ == "__main__": cloudmesh_yaml = config_file("/etc/cloudmesh.yaml") user_config = ConfigDict(filename=config_file("/me.yaml")) t = cm_template(cloudmesh_yaml) banner("VARIABLES") s = set(t.variables()) print ("\n".join(s)) banner("GREP") s = t.grep() print ("\n".join(s)) # banner("YAML FILE") # result = t.replace(kind="dict", values=user_config) # print yaml.dump(result, default_flow_style=False) # location = config_file('/cloudmesh-new.yaml') # yaml_file = open(location, 'w+') # print >> yaml_file, yaml.dump(result, default_flow_style=False) # yaml_file.close()
def install_command(args): """ Usage: install -h | --help install --version install cloudmesh install delete_yaml install system install query install new install apply_credentials install vagrant install rc fetch [--username=<username>] [--outdir=<outdir>] install rc fill install rc login [--username=<username>] """ arguments = docopt(install_command.__doc__, args) if arguments["cloudmesh"]: deploy() elif arguments["new"]: new_cloudmesh_yaml() elif arguments["delete_yaml"]: answer = yn_choice( "THIS COMMAND IS REAL DANGEROUS AND WILL DELETE ALL YOUR YAML FILE. Proceed", default='y') if answer: print "You fool we just deleted your yaml files" cp("etc/*.yaml", "~/.futuregrid/") else: print "puuh you interrupted" pass elif arguments["system"]: banner("Installing Ubuntu System Requirements") if is_ubuntu(): ubuntu() elif is_osx(): osx() elif is_centos(): centos() elif arguments["query"]: import platform print "System: ", platform.system() # print "Uname: ", platform.uname() print "Machine: ", platform.machine() print "Processor: ", platform.processor() print "Platform: ", platform.platform() print "Python: ", platform.python_version() print "Virtualenv:", hasattr(sys, 'real_prefix') elif arguments["vagrant"]: vagrant() elif arguments["rc"] and arguments["fetch"]: fetchrc(arguments["--username"], arguments["--outdir"]) elif arguments["rc"] and arguments["fill"]: get_fg_username_password_from_rcfiles() elif arguments["rc"] and arguments["login"]: verify_ssh_login(arguments["--username"])