def execute(self, name, command): """executes the command on the named host""" if name in ["localhost"]: r = '\n'.join(Shell.sh("-c", command).split()[-1:]) else: r = '\n'.join(Shell.ssh(name, command).split()[-1:]) return r
def create_cloudmesh_yaml(filename): if not os.path.exists(filename): path = os.path.dirname(filename) if not os.path.isdir(path): Shell.mkdir(path) etc_path = os.path.dirname(cloudmesh_client.__file__) etc_file = os.path.join(etc_path, "etc", "cloudmesh.yaml") to_dir = path_expand("~/.cloudmesh") shutil.copy(etc_file, to_dir) Console.ok("~/.cloudmesh/cloudmesh.yaml created")
def create_cloudmesh_yaml(filename): if not os.path.exists(filename): path = os.path.dirname(filename) if not os.path.isdir(path): Shell.mkdir(path) etc_path = os.path.dirname(cloudmesh_client.__file__) etc_file = os.path.join(etc_path, "etc", "cloudmesh.yaml") to_dir = path_expand("~/.cloudmesh") shutil.copy(etc_file, to_dir) os.system("chmod -R go-rwx " + path_expand("~/.cloudmesh")) Console.ok("~/.cloudmesh/cloudmesh.yaml created")
def delete(cls, cluster, job, group=None): """ This method is used to terminate a job with the specified or a group of jobs job_id or job_name in a given cluster :param group: :param cluster: the cluster like comet :param job: the job id or name :return: success message or error """ try: if group is not None: # get the job ids from the db arguments = {'cluster': cluster, 'group': group} db_jobs = cls.cm.find('batchjob', **arguments) list1 = [] for i in db_jobs: list1.append(db_jobs[i]['job_id']) # read active jobs active_jobs = json.loads(cls.queue(cluster)) list2 = [] for i in active_jobs: list2.append(active_jobs[i]['jobid']) # find intersection res = set(list1).intersection(set(list2)) if res is not None: for j in res: cmd = 'scancel {}'.format(str(j)) Shell.ssh(cluster, cmd) print("Deleted {}".format(j)) return "All jobs for group {} killed successfully".format(group) else: args = 'scancel ' if job.isdigit(): args += job else: args += "-n {}".format(job) Shell.ssh(cluster, args) return "Job {} killed successfully".format(job) except Exception as ex: print("in exceptio") print(ex) return ex
def _grep(self, search, platform): if not search: search = "'OS_PASSWORD': '******'" cmd = "egrep -ri \"{0}\" * | cut -d\":\" -f1 > a.tmp".format(search) print("[{0}]:{1}".format(platform, cmd)) os.system(cmd) res = Shell.cat("a.tmp") if res: print ('[{0}]: [ERROR] PASSWORD(OR SECRET KEY) DETECTED, SEE FILES ' 'BELOW'.format(platform)) print ("") print (res) else: print ("[{0}]: NO PASSWORD DETECTED".format(platform)) Shell.rm("a.tmp") print ("")
def list(cls, verbose=False): def convert(line): entry = (' '.join(line.split())).split(' ') data = dotdict() data.id = entry[0] data.name = entry[1] data.provider = entry[2] data.state = entry[3] data.directory = entry[4] return data result = Shell.execute("vagrant", "global-status --prune") if verbose: print(result) if "There are no active" in result: return None lines = [] for line in result.split("\n")[2:]: if line == " ": break else: lines.append(convert(line)) return lines
def execute(cls, name, command, cwd=None): vms = cls.to_dict(cls.list()) arg = "ssh {} -c {}".format(name, command) result = Shell.execute("vagrant", ["ssh", name, "-c", command], cwd=vms[name]["directory"]) return result
def _grep(self, search, platform): if not search: search = "'OS_PASSWORD': '******'" cmd = "egrep -ri \"{0}\" * | cut -d\":\" -f1 > a.tmp".format(search) print("[{0}]:{1}".format(platform, cmd)) os.system(cmd) res = Shell.cat("a.tmp") if res: print('[{0}]: [ERROR] PASSWORD(OR SECRET KEY) DETECTED, SEE FILES ' 'BELOW'.format(platform)) print("") print(res) else: print("[{0}]: NO PASSWORD DETECTED".format(platform)) Shell.rm("a.tmp") print("")
def emails(self, format_arg=None): ''' returns the emails of the authors either as a text or as a dict. The format is specified as an argument. :param format_arg: if "dict" is specified a dict will be returned :rtype: dict or arry of e-mails dependent on format_arg ''' if format_arg is None: format_string = "'%aN' <%cE>" elif format_arg == 'dict': format_string = "%aN\t%cE" #result = Shell.sort(Shell.git("log", # "--all", # "--format=" + format_string, # _tty_in=True, # _tty_out=False, # _piped=True), "-u") result = Shell.execute("git", ["log", "--all", "--format=" + format_string]) if format_arg is None: return result elif format_arg == "dict": result = iter(result.replace("\n", "\t").split("\t")[:-1]) emails = dict(zip(result, result)) for name in emails: emails[name] = emails[name] return emails
def run(command): parameter = command.split(" ") shell_command = parameter[0] args = parameter[1:] result = Shell.execute(shell_command, args) print(result) return result
def stat(self, email): ''' returns a statistic of a git author with the given e_mail. :param email: name of the author :rtype: a dict with the statistics ''' sums = [0, 0, 0] for line in Shell.git("log", "--all", "--stat", '--author={0}'.format(email), _tty_in=True, _tty_out=False, _iter=True): line = line[:-1] if " files changed" in line: line = line.replace(" insertions(+)", "") line = line.replace(" insertion(+)", "") line = line.replace(" deletion(-)", "") line = line.replace(" deletions(-)", "") line = line.replace(" files changed", "") line = line.split(",") data = [int(i) for i in line] for index in range(0, len(data)): sums[index] += data[index] return { "email": email, "fileschanged": sums[0], "inserted": sums[1], "deleted": sums[2], "lineschanged": sums[1] + sums[2] }
def run(command): print ("EXECUTING:", command) parameter = command.split(" ") shell_command = parameter[0] args = parameter[1:] result = Shell.execute(shell_command, args) return result
def stop(self): """stops the mongo service.""" r = Shell.execute('mongod stop'.split(' ')) print(r) # subprocess.call(mongod --shutdown) log_print('stopped') self.status()
def list(cls, cloud, start=None, end=None, tenant=None, format="table"): # set the environment variables set_os_environ(cloud) try: # execute the command args = ["usage"] if start is not None: args.extend(["--start", start]) if end is not None: args.extend(["--end", end]) if tenant is not None: args.extend(["--tenant", tenant]) result = Shell.execute("nova", args) result = Nova.remove_subjectAltName_warning(result) lines = result.splitlines() dates = lines[0] # TODO: as stated below, nova returns additional lines, # on my pc, SecurityWarning is returned, so filtering.. for l in lines[1:]: if l.__contains__("SecurityWarning"): lines.remove(l) table = '\n'.join(lines[1:]) dates = dates.replace("Usage from ", "").replace("to", "").replace(" +", " ")[:-1].split() # # TODO: for some reason the nova command has returned not the # first + char, so we could not ignore the line we may set - as # additional comment char, but that did not work # d = TableParser.convert(table, comment_chars="+#") # d["0"]["start"] = "start" # d["0"]["end"] = "end" d["0"]["start"] = dates[0] d["0"]["end"] = dates[1] # del d['0'] return Printer.write(d, order=[ "start", "end", "servers", "cpu hours", "ram mb-hours", "disk gb-hours" ], output=format) except Exception as e: return e
def setup(self, cloud="cloud=chameleon"): self.cloud = cloud print "Ignore Error: \n Please define a key first, e.g.: cm key add --ssh <keyname> \n " \ "-- If key has been successfully added into the database and uploaded into the cloud \n" \ "Ignore Error: \n problem uploading key veera to cloud chameleon: Key pair 'veera' already exists.\n " \ "******************************************************************************************************" result = Shell.cm("reset") print result result = Shell.cm("key add --ssh") print result result = Shell.cm("key", "upload") print result result = Shell.cm("default", self.cloud) print result result = Shell.cm("refresh", "on") print result
def run(command): print(command) parameter = command.split(" ") shell_command = parameter[0] args = parameter[1:] result = Shell.execute(shell_command, args) print(result) return result
def do_delete(self, names): names = str(names).split(' ') for name in names: delete_machine = "delete " + name print delete_machine result = Shell.cm("vm", delete_machine) print result
def run(self, command): command = command.format(**self.data) banner(command) parameter = command.split(" ") shell_command = parameter[0] args = parameter[1:] result = Shell.execute(shell_command, args) return result
def list(cls, cloud, start=None, end=None, tenant=None, format="table"): # set the environment variables set_os_environ(cloud) try: # execute the command args = ["usage"] if start is not None: args.extend(["--start", start]) if end is not None: args.extend(["--end", end]) if tenant is not None: args.extend(["--tenant", tenant]) result = Shell.execute("nova", args) result = Nova.remove_subjectAltName_warning(result) lines = result.splitlines() dates = lines[0] # TODO: as stated below, nova returns additional lines, # on my pc, SecurityWarning is returned, so filtering.. for l in lines[1:]: if l.__contains__("SecurityWarning"): lines.remove(l) table = '\n'.join(lines[1:]) dates = dates.replace("Usage from ", "").replace("to", "").replace(" +", " ")[:-1].split() # # TODO: for some reason the nova command has returned not the # first + char, so we could not ignore the line we may set - as # additional comment char, but that did not work # d = TableParser.convert(table, comment_chars="+#") # d["0"]["start"] = "start" # d["0"]["end"] = "end" d["0"]["start"] = dates[0] d["0"]["end"] = dates[1] # del d['0'] return Printer.write(d, order=["start", "end", "servers", "cpu hours", "ram mb-hours", "disk gb-hours"], output=format) except Exception as e: return e
def ec2(cls, cloud, zipfile): def sanitize(name): return name.replace(".zip", "").replace("@", "_") def find_exports(filename): with open(filename, "r") as f: content = f.read() data = {} for line in content.split("\n"): if line.startswith("export "): line = line.replace("export ", "") attribute, value = line.split("=", 1) value = value.replace("${NOVA_KEY_DIR}/", "") # remove comments data[attribute] = value.split("#")[0].strip() return data base = sanitize(os.path.basename(zipfile)) dest = sanitize(os.path.join( path_expand("~"), ".cloudmesh", "clouds", cloud, os.path.basename(zipfile))) Console.msg("Unzip file {} -> {}".format(zipfile, dest)) r = Shell.unzip(zipfile, dest) rcfile = os.path.join(dest, "ec2rc.sh") data = find_exports(rcfile) data["DEST"] = dest data["CLOUD"] = cloud d = { "cm_heading": "{CLOUD}, EC2".format(**data), "cm_host": None, "cm_label": "{CLOUD}_ec2".format(**data), "cm_type": "ec2", "cm_type_version": "ec2", "credentials": { "EC2_ACCESS_KEY": "{EC2_ACCESS_KEY}".format(**data), "EC2_SECRET_KEY": "{EC2_SECRET_KEY}".format(**data), "keyname": "TBD_not_used", "userid": "TBD_not_used", "EC2_URL": "{EC2_URL}".format(**data), "EC2_USER_ID": "{EC2_USER_ID}", "EC2_PRIVATE_KEY": "{DEST}/pk.pem".format(**data), "EC2_CERT": "{DEST}/cert.pem".format(**data), "NOVA_CERT": "{DEST}/cacert.pem".format(**data), "EUCALYPTUS_CERT": "{DEST}/cacert.pem".format(**data), }, "default": { "flavor": "m1.small", "image": "None", } } config = ConfigDict("cloudmesh.yaml") config["cloudmesh"]["clouds"][cloud] = d config.save()
def rm(cls, cluster, id=None, format=None): data = { "CLUSTER": cluster, "ID": id, } result = None if id is not None: try: result = Shell.ssh(cluster, "rm -rf experiment/{ID}".format(**data)) except Exception as e: pass else: try: result = Shell.ssh(cluster, "rm -rf experiment/*").split("\n") except Exception as e: pass return result
def run(self, command): command = command.format(**self.data) banner(command, c='-') parameter = command.split(" ") shell_command = parameter[0] args = parameter[1:] result = Shell.execute(shell_command, args) print(result) return str(result)
def version(verbose=False): result = Shell.execute("vagrant", ["version"]) if verbose: return result else: lines = result.split("\n") for line in lines: if "Installed Version:" in line: return line.replace("Installed Version:", "").strip() return None
def sync(cls, cloudname, localdir, remotedir, operation=None): """ Syncs a local directory with a remote directory. Either from local to remote OR vice-versa :param cloudname: :param localdir: :param remotedir: :param operation: get/put :return: """ # Get the operating system os_type = cls.operating_system() # fix the local dir path localdirpath = Config.path_expand(localdir) # check if local directory exists if not os.path.exists(localdirpath): if operation == "put": Console.error("The local directory [{}] does not exist." .format(localdirpath)) return None elif operation == "get": # for receiving, create local dir os.mkdir(localdirpath) Console.msg("Created local directory [{}] for sync." .format(localdirpath)) """ rsync now works on windows machines as well. we install rsync (v5.4.1.20150827) on windows via chocolatey $ choco install rsync """ host = cls.get_host(cloudname) if host is None: Console.error("Cloud [{}] not found in cloudmesh.yaml file." .format(cloudname)) return None else: args = None if operation == "put": args = [ "-r", localdir, host + ":" + remotedir ] elif operation == "get": args = [ "-r", host + ":" + remotedir, localdir ] # call rsync return Shell.rsync(*args)
def run(self, command): self.cloud = "aws" command = command.format(**self.data) banner(command, c="-") print(command) parameter = command.split(" ") shell_command = parameter[0] args = parameter[1:] result = Shell.execute(shell_command, args) print(result) return result
def info(cls, name=None): result = Shell.execute("vagrant", ["ssh-config"], cwd=name) lines = result.split("\n") data = {} for line in lines: attribute, value = line.strip().split(" ", 1) if attribute == "IdentityFile": value = value.replace('"', '') data[attribute] = value return data
def run(self, command): self.cloud = "azure" command = command.format(**self.data) banner(command, c="-") print(command) parameter = command.split(" ") shell_command = parameter[0] args = parameter[1:] result = Shell.execute(shell_command, args) print(result) return result
def pid(self): process_id = None output = Shell.ps('-ax') for line in output.split("\n"): if 'mongod' in line and "--port" in line: log_print(line) process_id = line.split(" ")[0] return process_id return process_id
def ec2(cls, cloud, zipfile): def sanitize(name): return name.replace(".zip", "").replace("@", "_") def find_exports(filename): with open(filename, "r") as f: content = f.read() data = {} for line in content.split("\n"): if line.startswith("export "): line = line.replace("export ", "") attribute, value = line.split("=", 1) value = value.replace("${NOVA_KEY_DIR}/", "") # remove comments data[attribute] = value.split("#")[0].strip() return data base = sanitize(os.path.basename(zipfile)) dest = sanitize( os.path.join(path_expand("~"), ".cloudmesh", "clouds", cloud, os.path.basename(zipfile))) Console.msg("Unzip file {} -> {}".format(zipfile, dest)) r = Shell.unzip(zipfile, dest) rcfile = os.path.join(dest, "ec2rc.sh") data = find_exports(rcfile) data["DEST"] = dest data["CLOUD"] = cloud d = { "cm_heading": "{CLOUD}, EC2".format(**data), "cm_host": None, "cm_label": "{CLOUD}_ec2".format(**data), "cm_type": "ec2", "cm_type_version": "ec2", "credentials": { "EC2_ACCESS_KEY": "{EC2_ACCESS_KEY}".format(**data), "EC2_SECRET_KEY": "{EC2_SECRET_KEY}".format(**data), "keyname": "TBD_not_used", "userid": "TBD_not_used", "EC2_URL": "{EC2_URL}".format(**data), "EC2_USER_ID": "{EC2_USER_ID}", "EC2_PRIVATE_KEY": "{DEST}/pk.pem".format(**data), "EC2_CERT": "{DEST}/cert.pem".format(**data), "NOVA_CERT": "{DEST}/cacert.pem".format(**data), "EUCALYPTUS_CERT": "{DEST}/cacert.pem".format(**data), }, "default": { "flavor": "m1.small", "image": "None", } } from pprint import pprint config = ConfigDict("cloudmesh.yaml") config["cloudmesh"]["clouds"][cloud] = d config.save()
def status(cls, host): msg = "Unknown host" try: msg = Shell.ping("-c", "1", host) except: pass if "1 packets transmitted, 1 packets received" in msg: return True elif "Unknown host" in msg: return False else: return False
def find_tunnel(): r = Shell.execute("ps", ["-ax"]).split("\n") pid = None info = None for line in r: if ("localhost" in line and "nucleus" in line) or ("comet" in line and "tunnel" in line) and not 'status' in line: info = line.strip() break if info: pid = int(info.split(" ", 1)[0]) return pid
def do_version(self, args, arguments): """ Usage: version [--format=FORMAT] [--check=CHECK] Options: --format=FORMAT the format to print the versions in [default: table] --check=CHECK boolean tp conduct an additional check [default: True] Description: Prints out the version number """ python_version, pip_version = get_python() try: git_hash_version = Shell.execute('git', 'log -1 --format=%h', traceflag=False, witherror=False) except: git_hash_version = 'N/A' versions = { "cloudmesh_client": { "name": "cloudmesh_client", "version": str(cloudmesh_client.__version__) }, # "cloudmesh_base": { # "name": "cloudmesh_base", # "version": str(cloudmesh_base.__version__) # }, "python": { "name": "python", "version": str(python_version) }, "pip": { "name": "pip", "version": str(pip_version) }, "git": { "name": "git hash", "version": str(git_hash_version) } } print( Printer.write(versions, output=arguments["--format"], order=["name", "version"])) if arguments["--check"] in ["True"]: check_python()
def info(cls, name=None): result = Shell.execute("vagrant", ["ssh-config"], cwd=name) lines = result.split("\n") data = {} for line in lines: attribute, value = line.strip().split(" ", 1) if attribute == "IdentityFile": value = value.replace('"','') data[attribute] = value return data
def find_tunnel(): r = Shell.execute("ps", ["-ax"]).split("\n") pid = None info = None for line in r: if ("localhost" in line and "nucleus" in line) or ( "comet" in line and "tunnel" in line) and 'status' not in line: info = line.strip() break if info: pid = int(info.split(" ", 1)[0]) return pid
def list(cls, cluster, id=None, format=None): data = { "CLUSTER": cluster, "ID": id, } result = None if id is not None: try: result = Shell.ssh(cluster, "ls experiment/{ID}".format(**data)) result = result.split("\n") except Exception as e: result = None else: try: result = Shell.ssh(cluster, "ls experiment").split("\n") ids = sorted([int(i) for i in result]) if format not in [None, 'txt']: result = ids except Exception as e: result = None return result
def info(cls, cluster, format='json', all=False): if all: result = Shell.ssh(cluster, 'sinfo --format=\"%all\"') else: result = Shell.ssh( cluster, 'sinfo --format=\"%P|%a|%l|%D|%t|%N\"') # ignore leading lines till header is found l = result.splitlines() for i, res in enumerate(l): if 'PARTITION|AVAIL|' in res: result = "\n".join(l[i:]) break parser = TableParser(strip=False) d = parser.to_dict(result) # add cluster and updated to each entry for key in list(d.keys()): d[key]['cluster'] = cluster d[key]['updated'] = datetime.now().strftime("%Y-%m-%d %H:%M:%S") if format == 'json': return json.dumps(d, indent=4, separators=(',', ': ')) else: return (Printer.write(d, order=['cluster', 'partition', 'avail', 'timelimit', 'nodes', 'state', 'nodelist', 'updated'], output=format))
def do_version(self, args, arguments): """ Usage: version [--format=FORMAT] [--check=CHECK] Options: --format=FORMAT the format to print the versions in [default: table] --check=CHECK boolean tp conduct an additional check [default: True] Description: Prints out the version number """ python_version, pip_version = get_python() try: git_hash_version = Shell.execute('git', 'log -1 --format=%h') except: git_hash_version = 'N/A' versions = { "cloudmesh_client": { "name": "cloudmesh_client", "version": str(cloudmesh_client.__version__) }, # "cloudmesh_base": { # "name": "cloudmesh_base", # "version": str(cloudmesh_base.__version__) # }, "python": { "name": "python", "version": str(python_version) }, "pip": { "name": "pip", "version": str(pip_version) }, "git": { "name": "git hash", "version": str(git_hash_version) } } print(dict_printer(versions, output=arguments["--format"], order=["name", "version"])) if arguments["--check"] in ["True"]: check_python()
def sync(cls, cloudname, localdir, remotedir, operation=None): """ Syncs a local directory with a remote directory. Either from local to remote OR vice-versa :param cloudname: :param localdir: :param remotedir: :param operation: get/put :return: """ # Get the operating system os_type = cls.operating_system() # fix the local dir path localdirpath = Config.path_expand(localdir) # check if local directory exists if not os.path.exists(localdirpath): if operation == "put": Console.error( "The local directory [{}] does not exist.".format( localdirpath)) return None elif operation == "get": # for receiving, create local dir os.mkdir(localdirpath) Console.msg("Created local directory [{}] for sync.".format( localdirpath)) """ rsync now works on windows machines as well. we install rsync (v5.4.1.20150827) on windows via chocolatey $ choco install rsync """ host = cls.get_host(cloudname) if host is None: Console.error( "Cloud [{}] not found in cloudmesh.yaml file.".format( cloudname)) return None else: args = None if operation == "put": args = ["-r", localdir, host + ":" + remotedir] elif operation == "get": args = ["-r", host + ":" + remotedir, localdir] # call rsync return Shell.rsync(*args)
def queue(cls, cluster, format='json', job=None): try: args = 'squeue ' if job is not None: if job.isdigit(): args += ' -j {} '.format(str(job)) # search by job id else: args += ' -n {} '.format(job) # search by job name f = '--format=%all' args += f result = Shell.ssh(cluster, args) # TODO: process till header is found...(Need a better way) l = result.splitlines() for i, res in enumerate(l): if 'ACCOUNT|GRES|' in res: result = "\n".join(str(x) for x in l[i:]) break parser = TableParser(strip=True) d = parser.to_dict(result) # add cluster and updated to each entry for key in list(d.keys()): d[key]['cluster'] = cluster d[key]['updated'] = datetime.now().strftime("%Y-%m-%d %H:%M:%S") if format == 'json': return json.dumps(d, indent=4, separators=(',', ': ')) else: return (Printer.write(d, order=['cluster', 'jobid', 'partition', 'name', 'user', 'st', 'time', 'nodes', 'nodelist', 'updated'], output=format)) except Exception as e: Error.traceback(e) return e
def list(cls): def convert(line): line = line.replace("(", "") line = line.replace(")", "") line = line.replace(",", "") entry = line.split(" ") data = dotdict() data.name = entry[0] data.provider = entry[1] data.date = entry[2] return data result = Shell.execute("vagrant", ["box", "list"]) lines = [] for line in result.split("\n"): lines.append(convert(line)) return lines
def authors(self, format_arg=None): ''' returns the authors of the authors either as a text or as a dict. The format is specified as an argument. :param format_arg: if "dict" is specified a dict will be returned ''' #result = Shell.git("shortlog", "-s", "-n", _tty_in=True, _tty_out=False) result = Shell.execute("git", ["shortlog", "-s", "-n"]) if format_arg is None: return result elif format_arg == "dict": list_string = result.replace("\n", "\t").split("\t")[:-1] it = iter(list_string[::-1]) authors = dict(zip(it, it)) for name in authors: authors[name] = int(authors[name]) return authors
def output(cls, cluster, id=None, format=None): data = { "CLUSTER": cluster, "ID": id, } result = None if id is None: ids = list(cluster) else: ids = [id] result = [] for i in ids: try: result.append(Shell.ssh(cluster, "cat experiment/{}/*.out".format(i))) except: result.append("") # if result == []: # result = None return result
def output(cls, cluster, id=None, format=None): data = { "CLUSTER": cluster, "ID": id, } result = None if id is None: ids = list(cluster) else: ids = [id] result = [] for i in ids: try: result.append( Shell.ssh(cluster, "cat experiment/{}/*.out".format(i))) except: result.append("") # if result == []: # result = None return result
def boot(cls, **kwargs): arg = dotdict(kwargs) arg.cwd = kwargs.get("cwd", None) vms = cls.to_dict(cls.list()) if arg.name in vms: Console.error("vm {name} already booted".format(**arg), traceflag=False) return None # print result else: cls.create(**kwargs) Console.ok("{name} created".format(**arg)) Console.ok("{name} booting ...".format(**arg)) result = Shell.execute("vagrant", ["up", arg.name], cwd=arg.name) Console.ok("{name} ok.".format(**arg)) return result
def add(cls, name): result = Shell.execute("vagrant", ["box", "add", name]) return result
def run(cls, cluster, group, cmd, **kwargs): # determine the script name.. # # TODO: script count is variable in data base, we test if fil exists and if it # does increase counter till we find one that does not, that will be new counter. # new counter will than be placed in db. # # define get_script_name(directory, prefix, counter) # there maybe s a similar thing already in the old cloudmesh # # if not kwargs['-name']: # # old_count = Shell.ssh(cluster, # "ls {}*.sh | wc -l | sed 's/$/ count/'". # format(username)) # c = [f for f in old_count.splitlines() if 'count' in f] # script_count = c[0].split()[0] # else: # script_count = kwargs['-name'] config = cls.read_config(cluster) if config["credentials"]["username"] == 'TBD': return "Please enter username in cloudmesh.yaml for cluster {}".format(cluster) cls.incr() data = { "cluster": cluster, "count": cls.counter(), "username": config["credentials"]["username"], "remote_experiment_dir": config["default"]["experiment_dir"], "queue": config["default"]["queue"], "id": None, "nodes": 1, "tasks_per_node": 1, } data["script_base_name"] = "{username}-{count}".format(**data) data["script_name"] = "{username}-{count}.sh".format(**data) data["script_output"] = "{username}-{count}.out".format(**data) data["script_error"] = "{username}-{count}.err".format(**data) data["remote_experiment_dir"] = \ "{remote_experiment_dir}/{count}".format(**data).format(**data) data["group"] = group # overwrite defaults option_mapping = {'-t': '{tasks_per_node}'.format(**data), '-N': '{nodes}'.format(**data), '-p': '{queue}'.format(**data), '-o': '{script_output}'.format(**data), '-D': '{remote_experiment_dir}'.format(**data), '-e': '{script_error}'.format(**data)} # map(lambda k, v: # option_mapping.__setitem__(k, kwargs.get(k) or v), # option_mapping.items()) # # rewrite for better readability for (k, v) in iteritems(option_mapping): option_mapping[k] = kwargs.get(k) or v config = cls.read_config(cluster) project = None try: project = config["credentials"]["project"] if project.lower() not in ["tbd", "none"]: option_mapping["-A"] = project except: pass for key in option_mapping: data[key] = option_mapping[key] # create the options for the script options = "" for key, value in option_mapping.items(): options += '#SBATCH {} {}\n'.format(key, value) cls.create_remote_dir(cluster, data["remote_experiment_dir"]) # if the command is a script, copy the script if os.path.isfile(Config.path_expand(cmd)): _from = Config.path_expand(cmd) _to = '{cluster}:{remote_experiment_dir}'.format(**data) local_file_name = cmd.split('/')[-1] Shell.execute("rsync", [_from, _to]) data["command"] = '{remote_experiment_dir}/{local_file_name}'.format(local_file_name=local_file_name, **data) else: data["command"] = cmd data["options"] = options script = textwrap.dedent( """ #! /bin/sh {options} echo '#CLOUDMESH: BATCH ENVIRONMENT' echo 'BASIL_RESERVATION_ID:' $BASIL_RESERVATION_ID echo 'SLURM_CPU_BIND:' $SLURM_CPU_BIND echo 'SLURM_JOB_ID:' $SLURM_JOB_ID echo 'SLURM_JOB_CPUS_PER_NODE:' $SLURM_JOB_CPUS_PER_NODE echo 'SLURM_JOB_DEPENDENCY:' $SLURM_JOB_DEPENDENCY echo 'SLURM_JOB_NAME:' $SLURM_JOB_NAME echo 'SLURM_JOB_NODELIST:' $SLURM_JOB_NODELIST echo 'SLURM_JOB_NUM_NODES:' $SLURM_JOB_NUM_NODES echo 'SLURM_MEM_BIND:' $SLURM_MEM_BIND echo 'SLURM_TASKS_PER_NODE:' $SLURM_TASKS_PER_NODE echo 'MPIRUN_NOALLOCATE:' $MPIRUN_NOALLOCATE echo 'MPIRUN_NOFREE:' $MPIRUN_NOFREE echo 'SLURM_NTASKS_PER_CORE:' $SLURM_NTASKS_PER_CORE echo 'SLURM_NTASKS_PER_NODE:' $SLURM_NTASKS_PER_NODE echo 'SLURM_NTASKS_PER_SOCKET:' $SLURM_NTASKS_PER_SOCKET echo 'SLURM_RESTART_COUNT:' $SLURM_RESTART_COUNT echo 'SLURM_SUBMIT_DIR:' $SLURM_SUBMIT_DIR echo 'MPIRUN_PARTITION:' $MPIRUN_PARTITION d=$(date) echo \"#CLOUDMESH: status, start, $d\" srun -l echo \"#CLOUDMESH: status, start, $d\" srun -l {command} d=$(date) srun -l echo \"#CLOUDMESH: status, finished, $d\" d=$(date) echo \"#CLOUDMESH: status, finished, $d\" """ ).format(**data).replace("\r\n", "\n").strip() _from = Config.path_expand('~/.cloudmesh/{script_name}'.format(**data)) _to = '{cluster}:{remote_experiment_dir}'.format(**data) data["from"] = _from data["to"] = _to data["script"] = script # write the script to local # print(_from) # print(_to) with open(_from, 'w') as local_file: local_file.write(script) # copy to remote host Shell.scp(_from, _to) # delete local file # Shell.execute('rm', _from) # import sys; sys.exit() # run the sbatch command cmd = 'sbatch {remote_experiment_dir}/{script_name}'.format(**data) data["cmd"] = cmd # print ("CMD>", cmd) result = Shell.ssh(cluster, cmd) data["output"] = result # find id for line in result.split("\n"): # print ("LLL>", line) if "Submitted batch job" in line: data["job_id"] = int(line.replace("Submitted batch job ", "").strip()) break # # HACK, should not depend on Model.py # # from cloudmesh_client.db.model import BATCHJOB # name = "" # BATCHJOB(name, # cluster=data["cluster"], # id=data["id"], # script=data["script"]) # has user and username which seems wrong # here what we have in data and want to store the - options are obviously wrong # and need to be full names # noinspection PyPep8,PyPep8 """ {'-D': '/N/u/gvonlasz/experiment/3', '-N': '1', '-o': 'gvonlasz-3.out', '-p': 'delta', '-t': '1', 'cluster': 'india', 'cmd': 'sbatch /N/u/gvonlasz/experiment/3/gvonlasz-3.sh', 'command': 'uname', 'count': 3, 'from': '/Users/big/.cloudmesh/gvonlasz-3.sh', 'id': 1346, 'options': '#SBATCH -t 1\n#SBATCH -o gvonlasz-3.out\n#SBATCH -N 1\n#SBATCH -p delta\n#SBATCH -D /N/u/gvonlasz/experiment/3\n', 'output': 'Submitted batch job 1346', 'queue': 'delta', 'remote_experiment_dir': '/N/u/gvonlasz/experiment/3', 'script': "#! /bin/sh\n#SBATCH -t 1\n#SBATCH -o gvonlasz-3.out\n#SBATCH -N 1\n#SBATCH -p delta\n#SBATCH -D /N/u/gvonlasz/experiment/3\n\nsrun -l echo '#CLOUDMESH: Starting'\nsrun -l uname\nsrun -l echo '#CLOUDMESH: Test ok'", 'script_base_name': 'gvonlasz-3', 'script_name': 'gvonlasz-3.sh', 'script_output': 'gvonlasz-3.out', 'to': 'india:/N/u/gvonlasz/experiment/3', 'username': '******'} """ """ we also want to store what part of the .out file, BASIL_RESERVATION_ID: SLURM_CPU_BIND: SLURM_JOB_ID: 1351 SLURM_JOB_CPUS_PER_NODE: 12 SLURM_JOB_DEPENDENCY: SLURM_JOB_NAME: gvonlasz-8.sh SLURM_JOB_NODELIST: d001 SLURM_JOB_NUM_NODES: 1 SLURM_MEM_BIND: SLURM_TASKS_PER_NODE: 12 MPIRUN_NOALLOCATE: MPIRUN_NOFREE: SLURM_NTASKS_PER_CORE: SLURM_NTASKS_PER_NODE: SLURM_NTASKS_PER_SOCKET: SLURM_RESTART_COUNT: SLURM_SUBMIT_DIR: /N/u/gvonlasz MPIRUN_PARTITION: so maybe we want to use some of the names here as they reflect the env vars """ # # add data to database # # remove the - options for key in ['-t', '-N', '-p', '-o', '-D', '-e']: if key in data: print(key, data[key]) del data[key] data['status'] = 'started' cls.add_db(**data) return data
def test(cls, cluster, time): result = Shell.ssh(cluster, "srun -n1 -t {} echo '#CLOUDMESH: Test ok'".format( time)) return result
def do_nova(self, args, arguments): """ :: Usage: nova set CLOUD nova info [CLOUD] [--password] nova help nova [--group=GROUP] ARGUMENTS... A simple wrapper for the openstack nova command Arguments: GROUP The group to add vms to ARGUMENTS The arguments passed to nova help Prints the nova manual set reads the information from the current cloud and updates the environment variables if the cloud is an openstack cloud info the environment values for OS Options: --group=GROUP Add VM to GROUP group --password Prints the password -v verbose mode """ # pprint(arguments) cloud = arguments['CLOUD'] or Default.cloud if not cloud: Console.error("Default cloud not set!") return "" group = arguments["--group"] or Default.group if not group: Console.error("Default group not set!") return "" if arguments["help"]: os.system("nova help") return "" elif arguments["info"]: set_os_environ(cloud) d = {} # # TODO: this naturally does not work as clouds will have # different parameters. ALos it does not unset previous # parameters from other clouds. See register # for attribute in ['OS_USERNAME', 'OS_TENANT_NAME', 'OS_AUTH_URL', 'OS_CACERT', 'OS_PASSWORD', 'OS_REGION']: try: d[attribute] = os.environ[attribute] except: Console.warning("OS environment variable {:} not found" .format(attribute)) d[attribute] = None if not arguments["--password"]: d['OS_PASSWORD'] = "******" print(Printer.row_table(d, order=None, labels=["Variable", "Value"])) msg = "info. OK." Console.ok(msg) return "" elif arguments["set"]: if cloud: set_os_environ(cloud) msg = "{0} is set".format(cloud) Console.ok(msg) else: Console.error("CLOUD is required") else: # nova ARGUMENTS... print("Cloud = {0}".format(cloud)) try: set_os_environ(cloud) args = arguments["ARGUMENTS"] # arguments may contain multiple optional arguments if len(args) == 1: args = args[0].split() result = Shell.execute("nova", args) print (result) # print(Nova.remove_subjectAltName_warning(result)) """ If request for nova boot, add the vm to group specified, or else add to default group """ if "boot" in args: # Logic to find ID of VM in the result fields = [] for field in result.split("|"): fields.append(field.strip()) index = fields.index('id') + 1 vm_id = fields[index] # Add to group Group.add(name=group, species="vm", member=vm_id, category=cloud) except Exception as ex: Console.error("Error executing Nova command: {}".format(ex)) return ""
def do_vm(self, args, arguments): """ :: Usage: vm default [--cloud=CLOUD][--format=FORMAT] vm refresh [all][--cloud=CLOUD] vm boot [--name=NAME] [--cloud=CLOUD] [--username=USERNAME] [--image=IMAGE] [--flavor=FLAVOR] [--group=GROUP] [--public] [--secgroup=SECGROUP] [--key=KEY] [--dryrun] vm boot [--n=COUNT] [--cloud=CLOUD] [--username=USERNAME] [--image=IMAGE] [--flavor=FLAVOR] [--group=GROUP] [--public] [--secgroup=SECGROUP] [--key=KEY] [--dryrun] vm ping [NAME] [N] vm console [NAME] [--group=GROUP] [--cloud=CLOUD] [--force] vm start [NAMES] [--group=GROUP] [--cloud=CLOUD] [--force] vm stop [NAMES] [--group=GROUP] [--cloud=CLOUD] [--force] vm terminate [NAMES] [--group=GROUP] [--cloud=CLOUD] [--force] vm delete [NAMES] [--group=GROUP] [--cloud=CLOUD] [--keep] [--dryrun] vm ip assign [NAMES] [--cloud=CLOUD] vm ip show [NAMES] [--group=GROUP] [--cloud=CLOUD] [--format=FORMAT] [--refresh] vm ip inventory [NAMES] [--header=HEADER] [--file=FILE] vm ssh [NAME] [--username=USER] [--quiet] [--ip=IP] [--cloud=CLOUD] [--key=KEY] [--command=COMMAND] vm rename [OLDNAMES] [NEWNAMES] [--force] [--dryrun] vm list [NAMES] [--cloud=CLOUDS|--active] [--group=GROUP] [--format=FORMAT] [--refresh] vm status [NAMES] vm wait [--cloud=CLOUD] [--interval=SECONDS] vm info [--cloud=CLOUD] [--format=FORMAT] vm check NAME vm username USERNAME [NAMES] [--cloud=CLOUD] Arguments: COMMAND positional arguments, the commands you want to execute on the server(e.g. ls -a) separated by ';', you will get a return of executing result instead of login to the server, note that type in -- is suggested before you input the commands NAME server name. By default it is set to the name of last vm from database. NAMES server name. By default it is set to the name of last vm from database. KEYPAIR_NAME Name of the openstack keypair to be used to create VM. Note this is not a path to key. NEWNAMES New names of the VM while renaming. OLDNAMES Old names of the VM while renaming. Options: --username=USERNAME the username to login into the vm. If not specified it will be guessed from the image name and the cloud --ip=IP give the public ip of the server --cloud=CLOUD give a cloud to work on, if not given, selected or default cloud will be used --count=COUNT give the number of servers to start --detail for table print format, a brief version is used as default, use this flag to print detailed table --flavor=FLAVOR give the name or id of the flavor --group=GROUP give the group name of server --secgroup=SECGROUP security group name for the server --image=IMAGE give the name or id of the image --key=KEY specify a key to use, input a string which is the full path to the private key file --keypair_name=KEYPAIR_NAME Name of the openstack keypair to be used to create VM. Note this is not a path to key. --user=USER give the user name of the server that you want to use to login --name=NAME give the name of the virtual machine --force rename/ delete vms without user's confirmation --command=COMMAND specify the commands to be executed Description: commands used to boot, start or delete servers of a cloud vm default [options...] Displays default parameters that are set for vm boot either on the default cloud or the specified cloud. vm boot [options...] Boots servers on a cloud, user may specify flavor, image .etc, otherwise default values will be used, see how to set default values of a cloud: cloud help vm start [options...] Starts a suspended or stopped vm instance. vm stop [options...] Stops a vm instance . vm delete [options...] Delete servers of a cloud, user may delete a server by its name or id, delete servers of a group or servers of a cloud, give prefix and/or range to find servers by their names. Or user may specify more options to narrow the search vm floating_ip_assign [options...] assign a public ip to a VM of a cloud vm ip show [options...] show the ips of VMs vm ssh [options...] login to a server or execute commands on it vm list [options...] same as command "list vm", please refer to it vm status [options...] Retrieves status of last VM booted on cloud and displays it. Tip: give the VM name, but in a hostlist style, which is very convenient when you need a range of VMs e.g. sample[1-3] => ['sample1', 'sample2', 'sample3'] sample[1-3,18] => ['sample1', 'sample2', 'sample3', 'sample18'] Quoting commands: cm vm login gvonlasz-004 --command=\"uname -a\" """ """ # terminate # issues a termination to the cloud, keeps vm in database # delete # issues a terminate if not already done # (remember you do not have to go to cloud if state is already terminated) # deletes the vm from database # # bulk rename rename abc[0-1] def[3-4] renames the abc0,abc1 -> def3,def4 if arguments["rename"]: oldnames = Parameter.expand(arguments["OLDNAME"]) newnames = Parameter.expand(arguments["NEWNAME"]) # check if new names ar not already taken # to be implemented if len(oldnames) == len(newnames): for i in range(0, len(oldnames)): oldname = oldnames[i] newname = newnames[i] if newname is None or newname == '': print("New node name cannot be empty") else: print(Cluster.rename_node(clusterid, oldname, newname)) """ cm = CloudmeshDatabase() def _print_dict(d, header=None, output='table'): return Printer.write(d, order=["id", "name", "status"], output=output, sort_keys=True) def _print_dict_ip(d, header=None, output='table'): return Printer.write(d, order=["network", "version", "addr"], output=output, sort_keys=True) def get_vm_name(name=None, offset=0, fill=3): if name is None: count = Default.get_counter(name='name') + offset prefix = Default.user if prefix is None or count is None: Console.error("Prefix and Count could not be retrieved correctly.", traceflag=False) return name = prefix + "-" + str(count).zfill(fill) return name def _refresh_cloud(cloud): try: msg = "Refresh VMs for cloud {:}.".format(cloud) if Vm.refresh(cloud=cloud): Console.ok("{:} OK.".format(msg)) else: Console.error("{:} failed".format(msg), traceflag=False) except Exception as e: Console.error("Problem running VM refresh", traceflag=False) def _get_vm_names(): vm_list = cm.find(kind="vm") vms = [vm["name"] for vm in vm_list] names = pattern = arguments["NAMES"] if pattern is not None: if "*" in pattern: names = search(vms, pattern) else: names = Parameter.expand(names) if names == ['last'] or names is None: names == [Default.vm] return vm_list, names cloud = arguments["--cloud"] or Default.cloud config = ConfigDict("cloudmesh.yaml") active_clouds = config["cloudmesh"]["active"] def _refresh(cloud): all = arguments["all"] or None if all is None: _refresh_cloud(cloud) else: for cloud in active_clouds: _refresh_cloud(cloud) arg = dotdict(arguments) arg.cloud = arguments["--cloud"] or Default.cloud arg.image = arguments["--image"] or Default.get(name="image", category=arg.cloud) arg.flavor = arguments["--flavor"] or Default.get(name="flavor", category=arg.cloud) arg.group = arguments["--group"] or Default.group arg.secgroup = arguments["--secgroup"] or Default.secgroup arg.key = arguments["--key"] or Default.key arg.dryrun = arguments["--dryrun"] arg.name = arguments["--name"] arg.format = arguments["--format"] or 'table' arg.refresh = Default.refresh or arguments["--refresh"] arg.count = int(arguments["--n"] or 1) arg.dryrun = arguments["--dryrun"] arg.verbose = not arguments["--quiet"] # # in many cases use NAMES # if arg.NAMES is not None: # arg.names = Parameter.expand(arg.NAMES) # gvonlasz[001-002] gives ["gvonlasz-001", "gvonlasz-002"] # else: # arg.names = None # if arguments["boot"]: arg.username = arguments["--username"] or Image.guess_username(arg.image) is_name_provided = arg.name is not None arg.user = Default.user for index in range(0, arg.count): vm_details = dotdict({ "cloud": arg.cloud, "name": get_vm_name(arg.name, index), "image": arg.image, "flavor": arg.flavor, "key": arg.key, "secgroup": arg.secgroup, "group": arg.group, "username": arg.username, "user": arg.user }) # correct the username vm_details.username = Image.guess_username_from_category( vm_details.cloud, vm_details.image, username=arg.username) try: if arg.dryrun: print(Printer.attribute(vm_details, output=arg.format)) msg = "dryrun info. OK." Console.ok(msg) else: vm_id = Vm.boot(**vm_details) if vm_id is None: msg = "info. failed." Console.error(msg, traceflag=False) return "" # set name and counter in defaults Default.set_vm(value=vm_details.name) if is_name_provided is False: Default.incr_counter("name") # Add to group if vm_id is not None: Group.add(name=vm_details.group, species="vm", member=vm_details.name, category=vm_details.cloud) msg = "info. OK." Console.ok(msg) except Exception as e: Console.error("Problem booting instance {name}".format(**vm_details), traceflag=False) elif arguments["username"]: arg.username = arguments["--username"] or Image.guess_username(arg.image) cloud = arg.cloud username = arg.USERNAME if arg.NAMES is None: names = [Default.vm] else: names = Parameter.expand(arg.NAMES) if len(names) == 0: return for name in names: arg.name = name Console.ok("Set username for {cloud}:{name} to {USERNAME}".format(**arg)) Vm.set_login_user(name=name, cloud=cloud, username=username) elif arguments["default"]: try: count = Default.get_counter() prefix = Username() if prefix is None or count is None: Console.error("Prefix and Count could not be retrieved correctly.", traceflag=False) return vm_name = prefix + "-" + str(count).zfill(3) arg = { "name": vm_name, "cloud": arguments["--cloud"] or Default.cloud } for attribute in ["image", "flavor"]: arg[attribute] = Default.get(name=attribute, category=cloud) for attribute in ["key", "group", "secgroup"]: arg[attribute] = Default.get(name=attribute, category='general') output = arguments["--format"] or "table" print(Printer.attribute(arg, output=output)) msg = "info. OK." Console.ok(msg) ValueError("default command not implemented properly. Upon " "first install the defaults should be read from yaml.") except Exception as e: # Error.traceback(e) Console.error("Problem listing defaults", traceflag=False) elif arguments["ping"]: try: if arguments["NAME"] is None and arguments["N"] is None: name = arguments["NAME"] or Default.vm n = arguments["N"] or 1 elif arguments["NAME"].isdigit(): n = arguments["NAME"] name = Default.vm else: name = arguments["NAME"] or Default.vm n = arguments["N"] or 1 print("Ping:", name, str(n)) vm = dotdict(Vm.list(name=name, category=cloud, output="dict")["dict"]) ip = vm.floating_ip result = Shell.ping(host=ip, count=n) print(result) except Exception as e: Console.error(e.message, traceflag=False) elif arguments["console"]: try: name = arguments["NAME"] or Default.vm vm = dotdict(Vm.list(name=name, category=cloud, output="dict")["dict"]) cloud_provider = CloudProvider(cloud).provider vm_list = cloud_provider.list_console(vm.uuid) print(vm_list) msg = "info. OK." Console.ok(msg) except Exception as e: # Error.traceback(e) Console.error("Problem retrieving status of the VM", traceflag=False) elif arguments["status"]: try: cloud_provider = CloudProvider(cloud).provider vm_list = cloud_provider.list_vm(cloud) vms = [vm_list[i]["name"] for i in vm_list ] print ("V", vms) pattern = arguments["NAMES"] if pattern is not None: if "*" in pattern: print ("serach") names = search(vms, pattern) else: names = Parameter.expand() for i in vm_list: if vm_list[i]["name"] in names: print("{} {}".format(vm_list[i]["status"], vm_list[i]["name"])) else: print("{} {}".format(vm_list[0]["status"], vm_list[0]["name"])) except Exception as e: # Error.traceback(e) Console.error("Problem retrieving status of the VM", traceflag=True) elif arguments["wait"]: interval = arguments["--interval"] or 5 try: cloud_provider = CloudProvider(cloud).provider for i in range(1,10): vm_list = cloud_provider.list_vm(cloud) time.sleep(float(1)) d = {} for id in vm_list: vm = vm_list[id] d[vm["name"]] = vm["status"] print (d) print("{} {}".format(vm_list[0]["status"], vm_list[0]["name"])) if vm_list[0]["status"] in ['ACTIVE']: return except Exception as e: # Error.traceback(e) Console.error("Problem retrieving status of the VM", traceflag=True) elif arguments["info"]: try: cloud_provider = CloudProvider(cloud).provider vms = cloud_provider.list_vm(cloud) vm = vms[0] output_format = arguments["--format"] or "table" print(Printer.attribute(vm, output=output_format)) msg = "info. OK." Console.ok(msg) except Exception as e: # Error.traceback(e) Console.error("Problem retrieving status of the VM", traceflag=False) elif arguments["check"]: test = {} try: names = Parameter.expand(arguments["NAME"]) id = 0 for name in names: print("Not implemented: {}".format(name)) # TODO: check the status of the vms status = "active" # TODO: check if they have a floating ip # TODO: get ip floating_ip = "127.0.0.1" ip = True # ping # TODO: ping the machine with the shell command ping = True # check if one can login and run a command check = False try: r = Shell.execute("uname", "-a") # do a real check check = True except: check = False test[name] = { "id": id, "name": name, "status": status, "ip": ip, "ping": ping, "login": check } id += 1 pprint(test) print(Printer.write(test, order=["id", "name", "status", "ip", "ping", "login"], output="table", sort_keys=True)) msg = "not yet implemented. failed." Console.error(msg, traceflag=False) except Exception as e: # Error.traceback(e) Console.error("Problem retrieving status of the VM", traceflag=False) elif arguments["start"]: try: servers = Parameter.expand(arguments["NAMES"]) # If names not provided, take the last vm from DB. if len(servers) == 0: last_vm = Default.vm if last_vm is None: Console.error("No VM records in database. Please run vm refresh.", traceflag=False) return "" name = last_vm["name"] # print(name) servers = list() servers.append(name) group = arguments["--group"] force = arguments["--force"] # if default cloud not set, return error if not cloud: Console.error("Default cloud not set.", traceflag=False) return "" Vm.start(cloud=cloud, servers=servers) msg = "info. OK." Console.ok(msg) except Exception as e: # Error.traceback(e) Console.error("Problem starting instances", traceflag=False) elif arguments["stop"]: try: servers = Parameter.expand(arguments["NAMES"]) # If names not provided, take the last vm from DB. if servers is None or len(servers) == 0: last_vm = Default.vm if last_vm is None: Console.error("No VM records in database. Please run vm refresh.", traceflag=False) return "" name = last_vm["name"] # print(name) servers = list() servers.append(name) group = arguments["--group"] force = arguments["--force"] # if default cloud not set, return error if not cloud: Console.error("Default cloud not set.", traceflag=False) return "" Vm.stop(cloud=cloud, servers=servers) msg = "info. OK." Console.ok(msg) except Exception as e: # Error.traceback(e) Console.error("Problem stopping instances", traceflag=False) elif arguments["refresh"]: _refresh(cloud) elif arguments["delete"]: dryrun = arguments["--dryrun"] group = arguments["--group"] force = not arguments["--keep"] cloud = arguments["--cloud"] vms, servers = _get_vm_names() if servers in [None, []]: Console.error("No vms found.", traceflag=False) return "" for server in servers: if dryrun: Console.ok("Dryrun: delete {}".format(server)) else: Vm.delete(servers=[server], force=force) return "" elif arguments["ip"] and arguments["assign"]: if arguments["NAMES"] is None: names = [Default.vm] else: names = Parameter.expand(arguments["NAMES"]) for name in names: # ip = Network.get_floatingip(....) vm = dotdict(Vm.list(name=name, category=cloud, output="dict")["dict"]) if vm.floating_ip is None: Console.ok("Assign IP to {}".format(name)) try: floating_ip = Network.find_assign_floating_ip(cloudname=cloud, instance_id=name) Vm.refresh(cloud=cloud) if floating_ip is not None: print( "Floating IP assigned to {:} is {:}".format( name, floating_ip)) msg = "info. OK." Console.ok(msg) except Exception as e: Console.error("Problem assigning floating ips.", traceflag=False) else: Console.error("VM {} already has a floating ip: {}".format(name, vm.floating_ip), traceflag=False) elif arguments["ip"] and arguments["inventory"]: vms, names = _get_vm_names() if names in [None, []]: if str(Default.vm) in ['None', None]: Console.error("The default vm is not set.", traceflag=False) return "" else: names = [Default.vm] header = arguments["--header"] or "[servers]" filename = arguments["--file"] or "inventory.txt" try: vm_ips = [] for vm in vms: if vm["name"] in names: print (vm["name"]) vm_ips.append(vm["floating_ip"]) result = header + "\n" result += '\n'.join(vm_ips) Console.ok("Creating inventory file: {}".format(filename)) Console.ok(result) with open(filename, 'w') as f: f.write(result) except Exception as e: Console.error("Problem getting ip addresses for instance", traceflag=True) elif arguments["ip"] and arguments["show"]: if arguments["NAMES"] is None: if str(Default.vm) in ['None', None]: Console.error("The default vm is not set.", traceflag=False) return "" else: names = [Default.vm] else: names = Parameter.expand(arguments["NAMES"]) group = arguments["--group"] output_format = arguments["--format"] or "table" refresh = arguments["--refresh"] try: ips = Ip.list(cloud=arg.cloud, output=output_format, names=names) print(ips) except Exception as e: Console.error("Problem getting ip addresses for instance", traceflag=False) elif arguments["ssh"]: def _print(msg): if arg.verbose: Console.msg(msg) chameleon = "chameleon" in ConfigDict(filename="cloudmesh.yaml")["cloudmesh"]["clouds"][arg.cloud][ "cm_host"] if chameleon: arg.username = "******" elif arg.cloud == "azure": arg.username = ConfigDict(filename="cloudmesh.yaml")["cloudmesh"]["clouds"]["azure"]["default"]["username"] else: if arg.username is None: Console.error("Could not guess the username of the vm", traceflag=False) return arg.username = arguments["--username"] or Image.guess_username(arg.image) arg.command = arguments["--command"] data = dotdict({ 'name': arguments["NAME"] or Default.vm, 'username': arg.username, 'cloud': arg.cloud, 'command': arg.command }) _print("login {cloud}:{username}@{name}".format(**data)) vm = Vm.get(data.name, category=data.cloud) Vm.set_login_user(name=data.name, cloud=data.cloud, username=data.username) data.floating_ip = vm.floating_ip data.key = arguments["--key"] or Default.key _print(Printer.attribute(data)) ''' if vm.username is None: user_from_db = Vm.get_login_user(vm.name, vm.cloud) user_suggest = user_from_db or Default.user username = input("Username (Default: {}):".format(user_suggest)) or user_suggest Vm.set_login_user(name=data.name, cloud=cloud, username=data.username) ''' ip = arguments["--ip"] commands = arguments["--command"] ip_addresses = [] cloud_provider = CloudProvider(cloud).provider ip_addr = cloud_provider.get_ips(vm.name) ipaddr_dict = Vm.construct_ip_dict(ip_addr, cloud) for entry in ipaddr_dict: ip_addresses.append(ipaddr_dict[entry]["addr"]) if len(ip_addresses) > 0: if ip is not None: if ip not in ip_addresses: Console.error("IP Address specified does not match with the host.", traceflag=False) return "" else: _print("Determining IP Address to use with a ping test.") # This part assumes that the ping is allowed to the machine. for ipadd in ip_addresses: _print("Checking {:}...".format(ipadd)) try: # Evading ping test, as ping is not enabled for VMs on Azure cloud # socket.gethostbyaddr(ipadd) # ip will be set if above command is successful. ip = ipadd except socket.herror: _print("Cannot reach {:}.".format(ipadd)) if ip is None: _print("Unable to connect to the machine") return "" else: _print("IP to be used is: {:}".format(ip)) # # TODO: is this correctly implemented # if not cloud == 'azure': SecGroup.enable_ssh(cloud=cloud) if arg.verbose: Console.info("Connecting to Instance at IP:" + format(ip)) # Constructing the ssh command to connect to the machine. sshcommand = "ssh" if arg.key is not None: sshcommand += " -i {:}".format(arg.key) sshcommand += " -o StrictHostKeyChecking=no" sshcommand += " {:}@{:}".format(data.username, ip) if commands is not None: sshcommand += " \"{:}\"".format(commands) # print(sshcommand) os.system(sshcommand) else: Console.error("No Public IPs found for the instance", traceflag=False) elif arguments["list"]: # groups = Group.list(output="dict") arg = dotdict(arguments) arg.names = arguments["NAMES"] arg.group = arguments["--group"] if arg.group is None: arg.group = [] else: arg.group = Parameter.expand(arguments["--group"]) arg.refresh = arguments["--refresh"] or Default.refresh if arg.NAMES is not None: arg.names = Parameter.expand(arguments["NAMES"]) else: arg.names = ["all"] _format = arguments["--format"] or "table" if arguments["--active"]: clouds = active_clouds else: if arguments["--cloud"]: clouds = Parameter.expand(arguments["--cloud"]) else: clouds = [Default.cloud] try: d = ConfigDict("cloudmesh.yaml") for cloud in clouds: if arg.refresh: _refresh(cloud) Console.ok("Listing VMs on Cloud: {:}".format(cloud)) vms = Vm.list(category=cloud, output="raw") # print ("XXX", type(vms), vms) if vms is None: break result = [] if "all" in arg.names: if result is None: result = [] else: result = vms elif arg.group is not None and len(arg.group) > 0: for vm in vms: if vm["group"] in arg.group: result.append(vm) elif arg.names is not None and len(arg.names) > 0: for vm in vms: if vm["name"] in arg.names: result.append(vm) if len(result) > 0: # print(result) (order, header) = CloudProvider(cloud).get_attributes("vm") print(Printer.write(result, order=order, output=_format) ) else: Console.error("No data found with requested parameters.", traceflag=False) except Exception as e: # Error.traceback(e) Console.error("Problem listing all instances", traceflag=False) elif arguments["rename"]: try: oldnames = Parameter.expand(arguments["OLDNAMES"]) newnames = Parameter.expand(arguments["NEWNAMES"]) force = arguments["--force"] if oldnames is None or newnames is None: Console.error("Wrong VMs specified for rename", traceflag=False) elif len(oldnames) != len(newnames): Console.error("The number of VMs to be renamed is wrong", traceflat=False) else: for i in range(0, len(oldnames)): oldname = oldnames[i] newname = newnames[i] if arguments["--dryrun"]: Console.ok("Rename {} to {}".format(oldname, newname)) else: Vm.rename(cloud=cloud, oldname=oldname, newname=newname, force=force ) msg = "info. OK." Console.ok(msg) except Exception as e: # Error.traceback(e) Console.error("Problem deleting instances", traceflag=False) return ""