def get_nova_credentials(kind="yaml", cloud=None): d = {} if kind in ["env"]: d['version'] = '2' d['username'] = os.environ['OS_USERNAME'] d['api_key'] = os.environ['OS_PASSWORD'] d['auth_url'] = os.environ['OS_AUTH_URL'] d['project_id'] = os.environ['OS_TENANT_NAME'] d['cacert'] = path_expand(os.environ['OS_CACERT']) elif kind in ["yaml"]: if cloud is None: raise Exception("cloud not specified") config = dict(ConfigDict(filename="~/.cloudmesh/cloudmesh.yaml")["cloudmesh"]["clouds"][cloud]) cred = dict(config["credentials"]) d['version'] = '2' d['username'] = cred['OS_USERNAME'] d['api_key'] = cred['OS_PASSWORD'] d['auth_url'] = cred['OS_AUTH_URL'] d['project_id'] = cred['OS_TENANT_NAME'] if 'OS_CACERT' in cred: d['cacert'] = path_expand(cred['OS_CACERT']) else: raise Exception ("unsupported kind: " + kind) return d
def run(self): banner("Setup the cloudmesh management yaml files ") yamlfiles = ['cloudmesh_user.yaml', 'cloudmesh_project.yaml'] dir_path = path_expand("~/.cloudmesh/{0}".format("accounts")) if not os.path.exists(dir_path): Shell.mkdir(dir_path) for yamlfile in yamlfiles: filename = path_expand("~/.cloudmesh/{0}/{1}".format( "accounts", yamlfile)) if os.path.isfile(filename): Console.error( "File {0} already exists. If you like to reinstall it, please remove the file" .format(yamlfile)) else: Console.info("Copying file: {0} -> {1} ".format( path_expand("etc/{0}/{1}".format("accounts", yamlfile)), filename)) shutil.copy( "etc/{0}/{1}".format("accounts", yamlfile), path_expand("~/.cloudmesh/{0}/{1}".format( "accounts", yamlfile)))
def test_clean(self): HEADING() self.db.clean() dbpath = self.db.config['dbpath'] logpath = self.db.config['dbpath'] assert not os.path.isfile(path_expand(dbpath)) and os.path.isfile(path_expand(logpath))
def load(self, filename="/cloudmesh_pbs.yaml"): self.filename = config_file(filename) self.data = ConfigDict(filename=self.filename) self.port = self.data["cloudmesh"]["jobdatabase"]["port"] self.db_path = path_expand(self.data["cloudmesh"]["jobdatabase"]["db_path"]) self.log_file = path_expand(self.db_path + "/dbjobs.log") self.dbname = self.data["cloudmesh"]["jobdatabase"]["dbname"]
def test_clean(self): HEADING() self.db.clean() dbpath = self.db.config['dbpath'] logpath = self.db.config['dbpath'] assert not os.path.isfile(path_expand(dbpath)) and os.path.isfile( path_expand(logpath))
def __init__(self): self.data = { 'notebook_dir': path_expand('~/notebook'), 'profile_nbserver_dir': path_expand('~/.ipython/profile_nbserver'), 'cert': '~/.ipython/profile_nbserver/mycert.pem', 'cloudmesh': path_expand('~/.cloudmesh'), 'pid': None }
def dump(self, filename): """ :param filename: name of the db file that will receive the content of cloudmesh.db :return: """ from_file = path_expand("~/.cloudmesh/cloudmesh.db") to_file = path_expand("~/.cloudmesh/{}".format(filename)) shutil.copyfile(from_file, to_file) print("dump")
def load(self, filename): """ :param filename: name of the db file located on ./cloudmesh that will be copied do cloudmesh.db :return: """ print("load") from_file = path_expand("~/.cloudmesh/{}".format(filename)) to_file = path_expand("~/.cloudmesh/cloudmesh.db") shutil.copyfile(from_file, to_file)
def generate_from_dict(self, d, out_file): cloudmesh_yaml = path_expand(self.filename) t = cm_template(cloudmesh_yaml) result = t.replace(kind="dict", values=d) location = path_expand(out_file) yaml_file = open(location, 'w+') print(yaml.dump(result, default_flow_style=False), file=yaml_file) yaml_file.close() log.info("Written new yaml file in " + location)
def create_config(cls, username): dir_path = path_expand("~/.cloudmesh/{0}".format("accounts")) if not os.path.exists(dir_path): Shell.mkdir(dir_path) filename = path_expand("~/.cloudmesh/{0}/{1}".format("accounts", ".config")) data = dict(user=username) with open(filename, 'w') as outfile: outfile.write(yaml.dump(data, default_flow_style=True))
def run(self): banner("Setup the cmd3.yaml file") cmd3_yaml = path_expand("~/.cloudmesh/cmd3.yaml") if os.path.isfile(cmd3_yaml): print ("ERROR: the file {0} already exists".format(cmd3_yaml)) print print ("If you like to reinstall it, please remove the file") else: print ("Copy file: {0} -> {1} ".format(path_expand("etc/cmd3.yaml"), cmd3_yaml)) Shell.mkdir("~/.cloudmesh") shutil.copy("etc/cmd3.yaml", path_expand("~/.cloudmesh/cmd3.yaml"))
def run(self): banner("Setup the cloudmesh_database.yaml file") database_yaml = path_expand("~/.cloudmesh/cloudmesh_database.yaml") if os.path.isfile(database_yaml): print ("WARNING: the file {0} already exists".format(database_yaml)) print print ("If you like to reinstall it, please remove the file") else: print ("Copy file: {0} -> {1} ".format(path_expand("etc/cloudmesh_database.yaml"), database_yaml)) os.makedirs(path_expand("~/.cloudmesh")) shutil.copy("etc/cloudmesh_database.yaml", path_expand("~/.cloudmesh/cloudmesh_database.yaml"))
def run(self): banner("Setup the cmd3.yaml file") cmd3_yaml = path_expand("~/.cloudmesh/cmd3.yaml") if os.path.isfile(cmd3_yaml): print ("ERROR: the file {0} already exists".format(cmd3_yaml)) print() print ("If you like to reinstall it, please remove the file") else: print ("Copy file: {0} -> {1} ".format(path_expand("etc/cmd3.yaml"), cmd3_yaml)) Shell.mkdir("~/.cloudmesh") shutil.copy("etc/cmd3.yaml", path_expand("~/.cloudmesh/cmd3.yaml"))
def load(self, filename="/cloudmesh_pbs.yaml"): """ The configuration for the job db is stored in a yaml file. The defualt location is ~/.cloudmesh/cloudmesh_pbs.yaml :param filename: the filename of the yaml file :type filename: str """ self.filename = config_file(filename) self.data = ConfigDict(filename=self.filename) self.port = self.data["cloudmesh"]["jobdatabase"]["port"] self.db_path = path_expand(self.data["cloudmesh"]["jobdatabase"]["db_path"]) self.log_file = path_expand(self.db_path + "/dbjobs.log") self.dbname = self.data["cloudmesh"]["jobdatabase"]["dbname"]
def add_from_yaml(self, filename): """ adds jobs of the following form to the database :: job2: program: myprg parameters: -a -l input: - in1.txt - in2.txt output: - out1.txt - out2.txt group: experiment1 label: job2 host: None start: +2h :param filename: the yaml filename """ d = None print(filename) stream = file(path_expand(filename), 'r') # if f does not exists error try: jobs = yaml.load(stream) except Exception, e: print("ERROR: loading file", filename) print(e) return
def do_reset(self, args, arguments): """ :: Usage: reset Description: DANGER: This method erases the database. Examples: clean """ filename = path_expand("~/.cloudmesh/cloudmesh.db") if os.path.exists(filename): os.remove(filename) Console.ok("Database reset") r = self.do_quit(None) Console.error( "Quitting the shell does not yet work. please exit the shell now.") return ""
def create_project_from_file(cls, file_path): # implement() # return try: filename = path_expand(file_path) file_config = ConfigDict(filename=filename) except: Console.error("Could not load file, please check filename and its path") return try: project_config = file_config.get("cloudmesh", "project") project = Project() project_id = uuid.uuid4() project_config.update({'project_id': project_id}) update_document(project, project_config) except: Console.error("Could not get project information from yaml file, " "please check you yaml file, users information must be " "under 'cloudmesh' -> 'project' -> project1..." + str(sys.exc_info()[0])) return try: cls.add(project) Console.info("Project created in the database.") except: Console.error("Project creation in database failed, " + str(sys.exc_info())) return
def boot(auth=True): # wipe mongo wipe() time.sleep(1) PROGRESS.next() # start mongo without auth start(auth=False) time.sleep(5) PROGRESS.next() if isyes(auth): # create users admin() time.sleep(2) PROGRESS.next() # restart with auth kill() time.sleep(10) PROGRESS.next() start(auth=auth) time.sleep(1) PROGRESS.next() config = cm_config_server().get("cloudmesh.server.mongo") path = path_expand(config["path"]) banner(path) print Shell.ls(path) banner("PROCESS") with settings(warn_only=True): local("ps -ax | fgrep mongo") PROGRESS.next()
def check_file_for_tabs(cls, filename, verbose=True): """identifies if the file contains tabs and returns True if it does. It also prints the location of the lines and columns. If verbose is set to False, the location is not printed. :param filename: the filename :type filename: str :rtype: True if there are tabs in the file """ filename = path_expand(filename) file_contains_tabs = False with file(filename) as f: lines = f.read().split("\n") line_no = 1 for line in lines: if "\t" in line: file_contains_tabs = True location = [ i for i in range(len(line)) if line.startswith('\t', i)] if verbose: print("Tab found in line", line_no, "and column(s)", location) line_no += 1 return file_contains_tabs
def _set_rabbitmq_env(self): location = path_expand("~/.cloudmesh/rabbitm") if sys.platform == "darwin": sh.mkdir("-p", location) self.rabbit_env["RABBITMQ_MNESIA_BASE"] = location self.rabbit_env["RABBITMQ_LOG_BASE"] = location os.environ["RABBITMQ_MNESIA_BASE"] = location os.environ["RABBITMQ_LOG_BASE"] = location self.rabbit_env["rabbitmq_server"] = \ "/usr/local/opt/rabbitmq/sbin/rabbitmq-server" self.rabbit_env["rabbitmqctl"] = \ "/usr/local/opt/rabbitmq/sbin/rabbitmqctl" elif sys.platform == "linux2": sh.mkdir("-p", location) self.rabbit_env["RABBITMQ_MNESIA_BASE"] = location self.rabbit_env["RABBITMQ_LOG_BASE"] = location os.environ["RABBITMQ_MNESIA_BASE"] = location os.environ["RABBITMQ_LOG_BASE"] = location self.rabbit_env["rabbitmq_server"] = "/usr/sbin/rabbitmq-server" self.rabbit_env["rabbitmqctl"] = "/usr/sbin/rabbitmqctl" else: print("WARNING: cloudmesh rabbitmq user install not supported, " "using system install")
def set_rabbitmq_env(): # BUG undefined global RABBITMQ_SERVER location = path_expand("~/.cloudmesh/rabbitm") if sys.platform == "darwin": Shell.mkdir(location) rabbit_env["RABBITMQ_MNESIA_BASE"] = location rabbit_env["RABBITMQ_LOG_BASE"] = location os.environ["RABBITMQ_MNESIA_BASE"] = location os.environ["RABBITMQ_LOG_BASE"] = location rabbit_env[ "rabbitmq_server"] = "/usr/local/opt/rabbitmq/sbin/rabbitmq-server" rabbit_env["rabbitmqctl"] = "/usr/local/opt/rabbitmq/sbin/rabbitmqctl" elif sys.platform == "linux2": Shell.mkdir(location) rabbit_env["RABBITMQ_MNESIA_BASE"] = location rabbit_env["RABBITMQ_LOG_BASE"] = location os.environ["RABBITMQ_MNESIA_BASE"] = location os.environ["RABBITMQ_LOG_BASE"] = location rabbit_env["rabbitmq_server"] = "/usr/sbin/rabbitmq-server" rabbit_env["rabbitmqctl"] = "/usr/sbin/rabbitmqctl" else: print "WARNING: cloudmesh rabbitmq user install not supported, " \ "using system install"
def _create_dir(self, path=None): if path is not None: self.data['notebook_dir'] = path_expand(path) try: os.makedirs("{notebook_dir}".format(**self.data), 0700) except: pass
def database_filename(self): """ The name of the cloudmesh_job file :return: """ return path_expand( self.data.get("cloudmesh", "pbsdatabase", "filename"))
def create_user_from_file(cls, file_path): try: filename = path_expand(file_path) file_config = ConfigDict(filename=filename) except: Console.error("Could not load file, please check filename and its path") return try: user_config = file_config.get("cloudmesh", "user") user_name = user_config['username'] user = User() update_document(user, user_config) except: Console.error("Could not get user information from yaml file, " "please check you yaml file, users information must be " "under 'cloudmesh' -> 'users' -> user1...") return try: if cls.check_exists(user_name) is False: cls.add(user) Console.info("User created in the database.") else: Console.error("User with user name " + user_name + " already exists.") return except: Console.error("User creation in database failed, " + str(sys.exc_info())) return
def key_add(self, name, filename, cloud=None): keyfile = path_expand(filename) if self.cloud_type(cloud) == "openstack": with open(os.path.expanduser(filename), 'r') as public_key: try: self.client[cloud].keypairs.create(name=name, public_key=public_key.read()) except exceptions.Conflict, e: print ("key already exists: {0}".format(str(e)))
def open(self, filename=None): if filename is not None: self.filename = filename else: self.filename = path_expand(self.pbs.database_filename()) path = os.path.dirname(self.filename) Shell.mkdir(path) self.load()
def get_credential(self, cloud=None, expand=False): if expand: d = self.get("cloudmesh.clouds.{0}.credentials".format(cloud)) for key in d: d[key] = path_expand(str(d[key])) return d else: return self.cloud(key)['credentials']
def _set_filename(self, filename): """ Sets the filename to be used. :param filename: the filename """ self['filename'] = filename self['location'] = path_expand(self["filename"])
def test_expand(self): HEADING() result = self.config.get('cloudmesh.clouds.india') dir = result['credentials']['OS_CACERT'] print(dir) assert dir.startswith("~") dir = path_expand(dir) print(dir) assert not dir.startswith("~")
def run(self): banner("Setup the cloudmesh management yaml files ") yamlfiles = ['cloudmesh_user.yaml', 'cloudmesh_project.yaml'] dir_path = path_expand("~/.cloudmesh/{0}".format("accounts")) if not os.path.exists(dir_path): Shell.mkdir(dir_path) for yamlfile in yamlfiles: filename = path_expand("~/.cloudmesh/{0}/{1}".format("accounts", yamlfile)) if os.path.isfile(filename): Console.error("File {0} already exists. If you like to reinstall it, please remove the file".format(yamlfile)) else: Console.info("Copying file: {0} -> {1} ".format(path_expand("etc/{0}/{1}".format("accounts", yamlfile)), filename)) shutil.copy("etc/{0}/{1}".format("accounts", yamlfile), path_expand("~/.cloudmesh/{0}/{1}".format("accounts", yamlfile)))
def setup(self): self.t = cm_template(path_expand(self.filename)) self.user = cm_user() try: self.setup_inventory() except: print("=" * 40) print("setup_inventory() failed. ldap test will not be performed") print("=" * 40)
def cm_image_command(arguments): """ cm-image admin on HOSTS cm-image admin off HOSTS """ path = path_expand(definitions[0]) if arguments["version"]: print(cloudmesh.__version__) elif arguments["info"]: banner("info") banner("System", c='-') print("Kind: ", arguments['--kind']) print("Path: ", path) print("Version:", cloudmesh.__version__) banner("List of templates", c='-') system_name = None for definition in definitions: try: path = path_expand(definition) if os.path.exists(path): os.system("cd '%s' ; veewee vbox list" % path) else: print("WARNING: path", path, "does not exist") except KeyError, key: print('WARNING: no environment variable called', key, 'found') print() print("To build one, please use one of the") print() print(" cm-image build OS") print() print("Next you need to register the image") print() print(" cm-image register OS") print() print("where OS is one of the labels listed above.") print()
def create_cloudmesh_yaml(filename): if not os.path.exists(filename): path = os.path.dirname(filename) if not os.path.isdir(path): Shell.mkdir(path) etc_path = os.path.dirname(cloudmesh_client.__file__) etc_file = os.path.join(etc_path, "etc", "cloudmesh.yaml") to_dir = path_expand("~/.cloudmesh") shutil.copy(etc_file, to_dir) Console.ok("~/.cloudmesh/cloudmesh.yaml created")
def create_cmd3_yaml_file(force=False, verbose=True): def print_error(kind, path): if verbose: Console.error("the {0} {1} already exists".format(kind, path)) Console.msg("") Console.msg("If you like to reinstall it, " "please remove the file first") Console.msg("") if verbose: banner("create cmd3.yaml") cmd3_yaml = path_expand("~/.cloudmesh/cmd3.yaml") if force or not os.path.isfile(cmd3_yaml): Shell.mkdir(path_expand("~/.cloudmesh")) import cmd3 content = pkg_resources.resource_string(cmd3.__name__, "etc/cmd3.yaml") Console.ok("Generate yaml file") with open(cmd3_yaml, "w") as cmd3_file: cmd3_file.write(content) else: print_error('file', cmd3_yaml) if verbose: banner("create cmd3_template") # # # copy tree filename = '~/.cloudmesh/etc/cmd3_template' if os.path.isdir(path_expand(filename)): print_error('directory', filename) else: import glob import shutil import cmd3.etc.cmd3_template f1 = cmd3.etc.cmd3_template.__file__ cmd3_etc_path = os.path.dirname(f1) pattern = os.path.join(cmd3_etc_path, '*') for src in glob.glob(pattern): if os.path.isfile(src): continue shutil.copytree(src, path_expand(filename))
def cat(filename): """prints the contents of a file with the given name. :param filename: name of the file, which can include ~ and $ environment variables :type: string """ location = path_expand(filename) banner(filename) with open(location, 'r') as f: print(f.read())
def set_debug(self, on): filename = path_expand("~/.cloudmesh/cmd3.yaml") config = ConfigDict(filename=filename) if type(on) == bool: self.debug = on else: self.debug = on.lower() in ['on', 'true'] config["cmd3"]["properties"]["debug"] = self.debug Console.ok("Debug mode is {:}".format(self.debug)) config.write(filename=filename, output="yaml", attribute_indent=" ")
def userkeys(self, attribute=None, expand=True): if attribute is None: return self['cloudmesh']['keys'] else: if attribute == 'default': name = self['cloudmesh']['keys']['default'] value = self['cloudmesh']['keys']['keylist'][name] else: value = self['cloudmesh']['keys']['keylist'][attribute] if expand: value = path_expand(value) return value
def run(self): banner("Reset the cloudmesh management yaml files ") yaml_files = ['cloudmesh_user.yaml', 'cloudmesh_project.yaml'] dir_path = path_expand("~/.cloudmesh/{0}".format("accounts")) if not os.path.exists(dir_path): Shell.mkdir(dir_path) for yaml_file in yaml_files: filename = path_expand("~/.cloudmesh/{0}/{1}".format( "accounts", yaml_file)) if os.path.isfile(filename): Console.info("Removing file: {0}".format(filename)) Shell.rm(filename) Console.info("Copying file: {0} -> {1} ".format( path_expand("etc/{0}/{1}".format("accounts", yaml_file)), filename)) shutil.copy( "etc/{0}/{1}".format("accounts", yaml_file), path_expand("~/.cloudmesh/{0}/{1}".format( "accounts", yaml_file))) else: Console.info("Copying file: {0} -> {1} ".format( path_expand("etc/{0}/{1}".format("accounts", yaml_file)), filename)) shutil.copy( "etc/{0}/{1}".format("accounts", yaml_file), path_expand("~/.cloudmesh/{0}/{1}".format( "accounts", yaml_file)))
def do_admin(self, args, arguments): """ :: Usage: admin password reset admin password Options: Description: admin password reset Reset portal password """ pprint(arguments) if arguments['password'] and arguments['reset']: Console.ok('password reset ...') self.password = getpass("Password:"******"~/.cloudmesh/cmd3.yaml") config = ConfigDict(filename=filename) config["cmd3"]["properties"]["password"] = self.password config.write(filename=filename, output="yaml") Console.ok("Resetting password. ok.") elif arguments['password']: if yn_choice("Would you like to print the password?"): filename = path_expand("~/.cloudmesh/cmd3.yaml") config = ConfigDict(filename=filename) try: self.password = config["cmd3"]["properties"]["password"] except Exception: Console.error("could not find the password. Please set it.") return Console.msg("Password: {:}".format(self.password))
def do_admin(self, args, arguments): """ :: Usage: admin password reset admin password Options: Description: admin password reset Reset portal password """ pprint(arguments) if arguments["password"] and arguments["reset"]: Console.ok("password reset ...") self.password = getpass("Password:"******"~/.cloudmesh/cmd3.yaml") config = ConfigDict(filename=filename) config["cmd3"]["properties"]["password"] = self.password config.write(filename=filename, output="yaml") Console.ok("Resetting password. ok.") elif arguments["password"]: if yn_choice("Would you like to print the password?"): filename = path_expand("~/.cloudmesh/cmd3.yaml") config = ConfigDict(filename=filename) try: self.password = config["cmd3"]["properties"]["password"] except Exception: Console.error("could not find the password. Please set it.") return Console.msg("Password: {:}".format(self.password))
def read_yaml_config(filename, check=True, osreplace=True, exit=True): """ reads in a yaml file from the specified filename. If check is set to true the code will fail if the file does not exist. However if it is set to false and the file does not exist, None is returned. :param filename: the file name :param check: if True fails if the file does not exist, if False and the file does not exist return will be None """ location = filename if location is not None: location = path_expand(location) if not os.path.exists(location) and not check: return None if check and os.path.exists(location): # test for tab in yaml file if check_file_for_tabs(location): log.error("The file {0} contains tabs. yaml " "Files are not allowed to contain tabs".format(location)) sys.exit() result = None try: if osreplace: result = open(location, 'r').read() t = Template(result) result = t.substitute(os.environ) # data = yaml.safe_load(result) data = ordered_load(result, yaml.SafeLoader) else: f = open(location, "r") # data = yaml.safe_load(f) data = ordered_load(result, yaml.SafeLoader) f.close() return data except Exception, e: log.error( "The file {0} fails with a yaml read error".format(filename)) log.error(str(e)) print(traceback.format_exc()) sys.exit()