def qsub(self, name, host, script, template=None, kind="dict"): """ Executes the qsub command on a given host. NOTE this method may not yet be fully implemented :param name: name of the script :param host: host on which the script is to be run :param script: The name of the script :param template: The script is wrapped into a template :param kind: The return is passed as dict, yaml, xml :return: """ self.jobid_incr() jobscript = self.create_script(name, script, template) # copy the script to the remote host self._write_to_file(jobscript, name) # copy script to remote host remote_path = self.data.get("cloudmesh", "pbs", host, "scripts") print(remote_path) xmkdir(host, remote_path) manager_host = self.manager(host) # call qsub on the remot host r = Shell.scp(name, manager_host + ":" + remote_path) jobid = Shell.ssh(manager_host, "qsub {0}/{1}".format(remote_path, name)).rstrip() return self.jobstatus(host, jobid, kind=kind)
def run(self): banner("Setup the cloudmesh management yaml files ") yamlfiles = ['cloudmesh_user.yaml', 'cloudmesh_project.yaml'] dir_path = path_expand("~/.cloudmesh/{0}".format("accounts")) if not os.path.exists(dir_path): Shell.mkdir(dir_path) for yamlfile in yamlfiles: filename = path_expand("~/.cloudmesh/{0}/{1}".format( "accounts", yamlfile)) if os.path.isfile(filename): Console.error( "File {0} already exists. If you like to reinstall it, please remove the file" .format(yamlfile)) else: Console.info("Copying file: {0} -> {1} ".format( path_expand("etc/{0}/{1}".format("accounts", yamlfile)), filename)) shutil.copy( "etc/{0}/{1}".format("accounts", yamlfile), path_expand("~/.cloudmesh/{0}/{1}".format( "accounts", yamlfile)))
def run(self): banner("Reset the cloudmesh management yaml files ") yaml_files = ['cloudmesh_user.yaml', 'cloudmesh_project.yaml'] dir_path = path_expand("~/.cloudmesh/{0}".format("accounts")) if not os.path.exists(dir_path): Shell.mkdir(dir_path) for yaml_file in yaml_files: filename = path_expand("~/.cloudmesh/{0}/{1}".format( "accounts", yaml_file)) if os.path.isfile(filename): Console.info("Removing file: {0}".format(filename)) Shell.rm(filename) Console.info("Copying file: {0} -> {1} ".format( path_expand("etc/{0}/{1}".format("accounts", yaml_file)), filename)) shutil.copy( "etc/{0}/{1}".format("accounts", yaml_file), path_expand("~/.cloudmesh/{0}/{1}".format( "accounts", yaml_file))) else: Console.info("Copying file: {0} -> {1} ".format( path_expand("etc/{0}/{1}".format("accounts", yaml_file)), filename)) shutil.copy( "etc/{0}/{1}".format("accounts", yaml_file), path_expand("~/.cloudmesh/{0}/{1}".format( "accounts", yaml_file)))
def execute(self, name, command): """executes the command on the named host""" if name in ["localhost"]: r = '\n'.join(Shell.sh("-c", command).split()[-1:]) else: r = '\n'.join(Shell.ssh(name, command).split()[-1:]) return r
def open(self, filename=None): if filename is not None: self.filename = filename else: self.filename = path_expand(self.pbs.database_filename()) path = os.path.dirname(self.filename) Shell.mkdir(path) self.load()
def create_cloudmesh_yaml(filename): if not os.path.exists(filename): path = os.path.dirname(filename) if not os.path.isdir(path): Shell.mkdir(path) etc_path = os.path.dirname(cloudmesh_client.__file__) etc_file = os.path.join(etc_path, "etc", "cloudmesh.yaml") to_dir = path_expand("~/.cloudmesh") shutil.copy(etc_file, to_dir) Console.ok("~/.cloudmesh/cloudmesh.yaml created")
def create_config(cls, username): dir_path = path_expand("~/.cloudmesh/{0}".format("accounts")) if not os.path.exists(dir_path): Shell.mkdir(dir_path) filename = path_expand("~/.cloudmesh/{0}/{1}".format("accounts", ".config")) data = dict(user=username) with open(filename, 'w') as outfile: outfile.write(yaml.dump(data, default_flow_style=True))
def delete(cls, cluster, job, group=None): """ This method is used to terminate a job with the specified or a group of jobs job_id or job_name in a given cluster :param group: :param cluster: the cluster like comet :param job: the job id or name :return: success message or error """ try: if group is not None: # get the job ids from the db cm = CloudmeshDatabase() arguments = {'cluster': cluster, 'group': group} db_jobs = cm.find('batchjob', **arguments) list1 = [] for i in db_jobs: list1.append(db_jobs[i]['job_id']) # read active jobs active_jobs = json.loads(cls.queue(cluster)) list2 = [] for i in active_jobs: list2.append(active_jobs[i]['jobid']) # find intersection res = set(list1).intersection(set(list2)) if res is not None: for j in res: cmd = 'scancel {}'.format(str(j)) Shell.ssh(cluster, cmd) print("Deleted {}".format(j)) return "All jobs for group {} killed successfully".format(group) else: args = 'scancel ' if job.isdigit(): args += job else: args += "-n {}".format(job) Shell.ssh(cluster, args) return "Job {} killed successfully".format(job) except Exception as ex: print("in exceptio") print(ex) return ex
def run(self): banner("Setup the cmd3.yaml file") cmd3_yaml = path_expand("~/.cloudmesh/cmd3.yaml") if os.path.isfile(cmd3_yaml): print ("ERROR: the file {0} already exists".format(cmd3_yaml)) print print ("If you like to reinstall it, please remove the file") else: print ("Copy file: {0} -> {1} ".format(path_expand("etc/cmd3.yaml"), cmd3_yaml)) Shell.mkdir("~/.cloudmesh") shutil.copy("etc/cmd3.yaml", path_expand("~/.cloudmesh/cmd3.yaml"))
def run(self): banner("Setup the cmd3.yaml file") cmd3_yaml = path_expand("~/.cloudmesh/cmd3.yaml") if os.path.isfile(cmd3_yaml): print ("ERROR: the file {0} already exists".format(cmd3_yaml)) print() print ("If you like to reinstall it, please remove the file") else: print ("Copy file: {0} -> {1} ".format(path_expand("etc/cmd3.yaml"), cmd3_yaml)) Shell.mkdir("~/.cloudmesh") shutil.copy("etc/cmd3.yaml", path_expand("~/.cloudmesh/cmd3.yaml"))
def start(self): """ Starts the cloudmesh_job process in the background. """ for key in ['dbpath', 'logpath']: path = os.dirname(self.config[key]) Shell.mkdir(path) r = Shell.sh("mongod", "--fork", "--logpath", self.config['logpath'], "--prot", self.config['port'], '--dbpath', self.config['dbpath']) print(r) # TODO # get the id from r self.config['id'] = None # put here the real id
def start(self): try: mongod = Shell.which("mongod") command = [ mongod, "--dbpath", str(self.db_path), "--port", str(self.port), "--fork", "--logpath", str(self.log_file), "--bind_ip", "127.0.0.1" ] print(" ".join(command)) #a = subprocess.call(command) os.system(" ".join(command)) os.system ("ps aux | fgrep mongod |fgrep -v fgrep") Console.ok("MongoDB has been deployed") self.info() except Exception, e: Console.error("we had a problem starting the mongo daemon") print(e) Console.error("MongoDB has stopped") # TODO remove the exit in final code, for debugging only sys.exit()
def _grep(self, search, platform): if not search: search = "'OS_PASSWORD': '******'" cmd = "egrep -ri \"{0}\" * | cut -d\":\" -f1 > a.tmp".format(search) print("[{0}]:{1}".format(platform, cmd)) os.system(cmd) res = Shell.cat("a.tmp") if res: print ('[{0}]: [ERROR] PASSWORD(OR SECRET KEY) DETECTED, SEE FILES ' 'BELOW'.format(platform)) print ("") print (res) else: print ("[{0}]: NO PASSWORD DETECTED".format(platform)) Shell.rm("a.tmp") print ("")
def stat(self, email): """ returns a statistic of a git author with the given e_mail. :param email: name of the author :rtype: a dict with the statistics """ result = Shell.git("log", "--all", "--stat", '--author={0}'.format(email)).split("\n") sums = [0, 0, 0] for line in result: if " files changed" in line: line = line.strip() line = line.replace(" insertions(+)", "") line = line.replace(" insertion(+)", "") line = line.replace(" deletion(-)", "") line = line.replace(" deletions(-)", "") line = line.replace(" files changed", "") line = line.split(",") data = [int(i) for i in line] for index in range(0, len(data)): sums[index] += data[index] return { "email": email, "fileschanged": sums[0], "inserted": sums[1], "deleted": sums[2], "lineschanged": sums[1] + sums[2] }
def get_authors_by_date(header=False): """lists the authors of a git repository sorted by date. Example: 0001 (2015-02-25): Gregor von Laszewski ([email protected]) 0002 (2015-04-14): Fugang Wang ([email protected]) :rtype: str """ # modified from https://github.com/jgehrcke/git-authors result = "" # if header: # result = result + "Authors\n" # result = result + "=======\n\n" r = Shell.git("log", "--encoding=utf-8", "--full-history", "--reverse", "--format=format:%at;%an;%ae").split("\n") seen = set() for line in r: timestamp, name, email = line.strip().split(";") if name not in seen: seen.add(name) day = time.strftime("%Y-%m-%d", time.gmtime(float(timestamp))) result = result + "* {:04d} ({:}): {:} ({:})\n".format( len(seen), day, name, email) return result
def emails(self, output=None): """ returns the emails of the authors either as a text or as a dict. The format is specified as an argument. :param output: if "dict" is specified a dict will be returned :rtype: dict or array of e-mails dependent on output """ format_string = "'%aN' <%cE>" if output == 'dict': format_string = "%aN\t%cE" result = sorted( set( Shell.git("log", "--all", "--format=" + format_string).split("\n"))) if output is None: return result elif output == "dict": emails = {} for l in result: (name, email) = l.split("\t") emails[name] = email return emails
def stat(self, email): """ returns a statistic of a git author with the given e_mail. :param email: name of the author :rtype: a dict with the statistics """ result = Shell.git("log", "--all", "--stat", '--author={0}'.format(email)).split("\n") sums = [0, 0, 0] for line in result: if " files changed" in line: line = line.strip() line = line.replace(" insertions(+)", "") line = line.replace(" insertion(+)", "") line = line.replace(" deletion(-)", "") line = line.replace(" deletions(-)", "") line = line.replace(" files changed", "") line = line.split(",") data = [int(i) for i in line] for index in range(0, len(data)): sums[index] += data[index] return {"email": email, "fileschanged": sums[0], "inserted": sums[1], "deleted": sums[2], "lineschanged": sums[1] + sums[2]}
def get_authors_by_date(header=False): """lists the authors of a git repository sorted by date. Example: 0001 (2015-02-25): Gregor von Laszewski ([email protected]) 0002 (2015-04-14): Fugang Wang ([email protected]) :rtype: str """ # modified from https://github.com/jgehrcke/git-authors result = "" # if header: # result = result + "Authors\n" # result = result + "=======\n\n" r = Shell.git("log", "--encoding=utf-8", "--full-history", "--reverse", "--format=format:%at;%an;%ae").split("\n") seen = set() for line in r: timestamp, name, email = line.strip().split(";") if name not in seen: seen.add(name) day = time.strftime("%Y-%m-%d", time.gmtime(float(timestamp))) result = result + "* {:04d} ({:}): {:} ({:})\n".format(len(seen), day, name, email) return result
def list_quotas(cls, cloud, format): Quota.set_os_environment(cloud) result = Shell.execute("nova", "quota-show") d = Quota.convert_to_dict(result) return dict_printer(d, order=['Quota', 'Limit'], output=format)
def start(self): """ Starts the cloudmesh_job process in the background. """ for key in ['dbpath', 'logpath']: path = os.dirname(self.config[key]) Shell.mkdir(path) r = Shell.sh("mongod", "--fork", "--logpath", self.config['logpath'], "--prot", self.config['port'], '--dbpath', self.config['dbpath']) print (r) # TODO # get the id from r self.config['id'] = None # put here the real id
def list(cls, cloud, start=None, end=None, tenant=None, format="table"): # set the environment variables set_os_environ(cloud) try: # execute the command args = ["usage"] if start is not None: args.extend(["--start", start]) if end is not None: args.extend(["--end", end]) if tenant is not None: args.extend(["--tenant", tenant]) result = Shell.execute("nova", args) result = Nova.remove_subjectAltName_warning(result) lines = result.splitlines() dates = lines[0] # TODO: as stated below, nova returns additional lines, # on my pc, SecurityWarning is returned, so filtering.. for l in lines[1:]: if l.__contains__("SecurityWarning"): lines.remove(l) table = '\n'.join(lines[1:]) dates = dates.replace("Usage from ", "").replace("to", "").replace(" +", " ")[:-1].split() # # TODO: for some reason the nova command has returned not the # first + char, so we could not ignore the line we may set - as # additional comment char, but that did not work # d = TableParser.convert(table, comment_chars="+#") # d["0"]["start"] = "start" # d["0"]["end"] = "end" d["0"]["start"] = dates[0] d["0"]["end"] = dates[1] # del d['0'] return dict_printer(d, order=["start", "end", "servers", "cpu hours", "ram mb-hours", "disk gb-hours"], output=format) except Exception, e: return e
def image_list(self): images = {} lines = Shell.vagrant("box", "list") for line in lines: (name, kind) = line.split("(") name = name.strip() kind = kind.split(")")[0].strip() images[name] = {"name": name, "kind": kind} return images
def run(self): banner("Setup the cloudmesh management yaml files ") yamlfiles = ['cloudmesh_user.yaml', 'cloudmesh_project.yaml'] dir_path = path_expand("~/.cloudmesh/{0}".format("accounts")) if not os.path.exists(dir_path): Shell.mkdir(dir_path) for yamlfile in yamlfiles: filename = path_expand("~/.cloudmesh/{0}/{1}".format("accounts", yamlfile)) if os.path.isfile(filename): Console.error("File {0} already exists. If you like to reinstall it, please remove the file".format(yamlfile)) else: Console.info("Copying file: {0} -> {1} ".format(path_expand("etc/{0}/{1}".format("accounts", yamlfile)), filename)) shutil.copy("etc/{0}/{1}".format("accounts", yamlfile), path_expand("~/.cloudmesh/{0}/{1}".format("accounts", yamlfile)))
def deploy(self): """ creates the directories if they do not exist """ try: r = Shell.mkdir(self.db_path) except Exception, e: Console.error("Problem creating the database directory {:}".format(self.db_path)) print(e)
def stop(self): """ Stops the cloudmesh_job process """ id = self.config['id'] # TODO use pythonic way to kill # p = psutil.Process(pid) # p.terminate() #or p.kill() r = Shell.kill('-9', id) pass
def delete(cls, cluster, job): """ This method is used to terminate a job with the specified job_id or job_name in a given cluster :param cluster: the cluster like comet :param job: the job id or name :return: success message or error """ try: args = 'scancel ' if job.isdigit(): args += job else: args += "-n {}".format(job) Shell.ssh(cluster, args) return "Job {} killed successfully".format(job) except Exception as ex: return ex
def run(self): banner("Reset the cloudmesh management yaml files ") yaml_files = ['cloudmesh_user.yaml', 'cloudmesh_project.yaml'] dir_path = path_expand("~/.cloudmesh/{0}".format("accounts")) if not os.path.exists(dir_path): Shell.mkdir(dir_path) for yaml_file in yaml_files: filename = path_expand("~/.cloudmesh/{0}/{1}".format("accounts", yaml_file)) if os.path.isfile(filename): Console.info("Removing file: {0}".format(filename)) Shell.rm(filename) Console.info("Copying file: {0} -> {1} ".format(path_expand("etc/{0}/{1}".format("accounts", yaml_file)), filename)) shutil.copy("etc/{0}/{1}".format("accounts", yaml_file), path_expand("~/.cloudmesh/{0}/{1}".format("accounts", yaml_file))) else: Console.info("Copying file: {0} -> {1} ".format(path_expand("etc/{0}/{1}".format("accounts", yaml_file)), filename)) shutil.copy("etc/{0}/{1}".format("accounts", yaml_file), path_expand("~/.cloudmesh/{0}/{1}".format("accounts", yaml_file)))
def create_cmd3_yaml_file(force=False, verbose=True): def print_error(kind, path): if verbose: Console.error("the {0} {1} already exists".format(kind, path)) Console.msg("") Console.msg("If you like to reinstall it, " "please remove the file first") Console.msg("") if verbose: banner("create cmd3.yaml") cmd3_yaml = path_expand("~/.cloudmesh/cmd3.yaml") if force or not os.path.isfile(cmd3_yaml): Shell.mkdir(path_expand("~/.cloudmesh")) import cmd3 content = pkg_resources.resource_string(cmd3.__name__, "etc/cmd3.yaml") Console.ok("Generate yaml file") with open(cmd3_yaml, "w") as cmd3_file: cmd3_file.write(content) else: print_error('file', cmd3_yaml) if verbose: banner("create cmd3_template") # # # copy tree filename = '~/.cloudmesh/etc/cmd3_template' if os.path.isdir(path_expand(filename)): print_error('directory', filename) else: import glob import shutil import cmd3.etc.cmd3_template f1 = cmd3.etc.cmd3_template.__file__ cmd3_etc_path = os.path.dirname(f1) pattern = os.path.join(cmd3_etc_path, '*') for src in glob.glob(pattern): if os.path.isfile(src): continue shutil.copytree(src, path_expand(filename))
def tag(self): v = self.find() banner("v") print v #v = ".".join(self.version) os.system("python shell_plugins.py install") if self.git_commit_needed(): banner("git commit") command = "git commit -m 'version {:}' {:}".format(v,self.filename) print "CCC", command os.system(command) os.system("git push") else: print "git commit not needed" Shell.git("tag", "-a", v, "-m", "version {:}".format(v)) Shell.git("push", "origin", "--tags")
def rm(cls, cluster, id=None, format=None): data = { "CLUSTER": cluster, "ID": id, } result = None if id is not None: try: result = Shell.ssh(cluster, "rm -rf experiment/{ID}".format(**data)) except Exception, e: pass
def tag(self): v = self.find() banner("v") print v #v = ".".join(self.version) os.system("python shell_plugins.py install") if self.git_commit_needed(): banner("git commit") command = "git commit -m 'version {:}' {:}".format( v, self.filename) print "CCC", command os.system(command) os.system("git push") else: print "git commit not needed" Shell.git("tag", "-a", v, "-m", "version {:}".format(v)) Shell.git("push", "origin", "--tags")
def list(cls, cluster, id=None, format=None): data = { "CLUSTER": cluster, "ID": id, } result = None if id is not None: try: result = Shell.ssh(cluster, "ls experiment/{ID}".format(**data)) result = result.split("\n") except Exception, e: result = None
def status(cls, host): msg = "Unknown host" try: msg = Shell.ping("-c", "1", host) except: pass if "1 packets transmitted, 1 packets received" in msg: return True elif "Unknown host" in msg: return False else: return False
def find_tunnel(): r = Shell.execute("ps", ["-ax"]).split("\n") pid = None info = None for line in r: if ("localhost" in line and "nucleus" in line) or ("comet" in line and "tunnel" in line) and not 'status' in line: info = line.strip() break if info: pid = int(info.split(" ", 1)[0]) return pid
def info(cls, cluster, format='json', all=False): if all: result = Shell.ssh(cluster, 'sinfo --format=\"%all\"') else: result = Shell.ssh( cluster, 'sinfo --format=\"%P|%a|%l|%D|%t|%N\"') # ignor leading lines till header is found l = result.splitlines() for i, res in enumerate(l): if 'PARTITION|AVAIL|' in res: result = "\n".join(l[i:]) break parser = TableParser(strip=False) d = parser.to_dict(result) # add cluster and updated to each entry for key in d.keys(): d[key]['cluster'] = cluster d[key]['updated'] = datetime.now().strftime("%Y-%m-%d %H:%M:%S") if format == 'json': return json.dumps(d, indent=4, separators=(',', ': ')) else: return (dict_printer(d, order=['cluster', 'partition', 'avail', 'timelimit', 'nodes', 'state', 'nodelist', 'updated'], output=format))
def list_limits(cls, cloud, format, tenant=" "): # set the environment variables Quota.set_os_environment(cloud) # execute the command args = ["limits", "--tenant", tenant] result = Shell.execute("nova", args) # print results in a format if "ERROR" in result: return result else: d = Limits.convert_to_dict(result) return dict_printer(d, order=["Name", "Used", "Max"], output=format)
def nodes(self, host, refresh=True): """ returns the information from the command pbsnodes in a dict. :param host: the name of the host as specified in the .ssh/config file :param refresh: if False, reads returns a cached value if True, issues a new command and refreshes the cach :return: information of the pbsnodes command in a dict """ manager_host = self.manager(host) if self.pbs_nodes_data is None or refresh: try: result = Shell.ssh(manager_host, "pbsnodes", "-a") except: raise RuntimeError( "can not execute pbs nodes on host {0}".format( manager_host)) pbsinfo = {} nodes = result.split("\n\n") for node in nodes: pbs_data = node.split("\n") pbs_data = [e.strip() for e in pbs_data] name = pbs_data[0] if name != "": pbsinfo[name] = {u'name': name} for element in pbs_data[1:]: try: (attribute, value) = element.split(" = ") if attribute == 'status': status_elements = value.split(",") pbsinfo[name][attribute] = {} for e in status_elements: (a, v) = e.split("=") pbsinfo[name][attribute][a] = v elif attribute == 'jobs': pbsinfo[name][attribute] = value.split(',') elif attribute == 'note' and ( value.strip().startswith("{") or value.strip().startswith("[")): pbsinfo[name][attribute] = literal_eval(value) else: pbsinfo[name][attribute] = value except: pass self.pbs_nodes_data = pbsinfo return self.pbs_nodes_data
def authors(self, output=None): """ returns the authors of the authors either as a text or as a dict. The format is specified as an argument. :param output: if "dict" is specified a dict will be returned """ result = Shell.git("shortlog", "-s", "-n") if output is None: return result elif output == "dict": authors = {} for line in result.split("\n"): l = " ".join(line.split()).strip() (number, name) = l.split(" ", 1) authors[name] = int(number) return authors
def remote(cls, host, force=False): """ TODO: there is a bug in the instalation of kilo the openrc file on the remote machine is not called openrc.sh but contains username and project number. :param host: the remote host :param force: :return: """ config = ConfigDict("cloudmesh.yaml") host_spec = config["cloudmesh.clouds." + host] host_credentials = host_spec["credentials"] if 'cm_openrc' in host_spec: Console.ok("looking for openrc") else: Console.error("no cm_openrc specified in the host") return hostname = config["cloudmesh.clouds." + host + ".cm_host"] Console.ok("fetching information from {:} ...".format(host)) openrc = host_spec["cm_openrc"] directory = os.path.dirname(openrc) base = os.path.basename(openrc) _from_dir = "{:}:{:}".format(hostname, directory).replace("~/", "") _to_dir = os.path.dirname(Config.path_expand(directory)) openrc_file = Config.path_expand(openrc) print("From: ", _from_dir) print("To: ", _to_dir) print("Openrc:", openrc_file) cls.make_dir(_to_dir) r = "" Console.ok("Reading rc file from {}".format(host)) try: r = Shell.scp('-r', _from_dir, _to_dir) except Exception, e: print(e) return
def nodes(self, host, refresh=True): """ returns the information from the command pbsnodes in a dict. :param host: the name of the host as specified in the .ssh/config file :param refresh: if False, reads returns a cached value if True, issues a new command and refreshes the cach :return: information of the pbsnodes command in a dict """ manager_host = self.manager(host) if self.pbs_nodes_data is None or refresh: try: result = Shell.ssh(manager_host, "pbsnodes", "-a") except: raise RuntimeError( "can not execute pbs nodes on host {0}".format(manager_host)) pbsinfo = {} nodes = result.split("\n\n") for node in nodes: pbs_data = node.split("\n") pbs_data = [e.strip() for e in pbs_data] name = pbs_data[0] if name != "": pbsinfo[name] = {u'name': name} for element in pbs_data[1:]: try: (attribute, value) = element.split(" = ") if attribute == 'status': status_elements = value.split(",") pbsinfo[name][attribute] = {} for e in status_elements: (a, v) = e.split("=") pbsinfo[name][attribute][a] = v elif attribute == 'jobs': pbsinfo[name][attribute] = value.split(',') elif attribute == 'note' and ( value.strip().startswith("{") or value.strip().startswith("[")): pbsinfo[name][attribute] = literal_eval(value) else: pbsinfo[name][attribute] = value except: pass self.pbs_nodes_data = pbsinfo return self.pbs_nodes_data
def test_012_yaml_load(self): """ tests adding jobs from a YAML file :return: """ HEADING() db = self.db db.connect() # Clear all jobs currently in the database to ensure a correct final assertion db.clear() # Add the jobs outlined in the YAML file db.add_from_yaml("etc/jobs.yaml") count_fgrep = len(Shell.fgrep("input:", "etc/jobs.yaml").split("\n")) # Assert that the correct number jobs have been added assert (db.count() == count_fgrep)
def test_15_jobscript(self): HEADING() db = self.db db.connect() name = "test1" contents = "hallo" db.add_script(name, contents) db.add_script("test2", "other") script = db.get_script(name) print("Script:", script) print("Content:", contents) db.write_script(name, "/tmp/script.txt") what = Shell.cat("/tmp/script.txt") assert contents == what assert contents == script
def jobstatus(self, host, jobid, kind='dict'): """ The status of a specific job :param host: The host on which the job is running :param jobid: The jobid as specified by the queing system :param kind: The output can be returned as dict, xml, and yaml :return: """ manager_host = self.manager(host) qstat_xml_data = Shell.ssh(manager_host, "qstat", "-x", jobid).rstrip() if kind == 'xml': r = qstat_xml_data else: r = self.qstat_xml_to_dict(qstat_xml_data) r[unicode(jobid)][u"cm_jobid"] = self.jobid r[unicode(jobid)]["cm_Variable_list"] = self.variable_list(r) if kind == 'yaml': r = yaml.dump(r, default_flow_style=False) return r
def start(self): """ starts the database service :return: the pid """ if self.isup(): _pid = self.pid() Console.error("A mongod process on port {:} is already running with pid {:}".format(self.port, _pid)) return else: Console.ok("STARTING") try: mongod = Shell.which("mongod") command = [ mongod, "--dbpath", str(self.db_path), "--port", str(self.port), "--fork", "--logpath", str(self.log_file), "--bind_ip", "127.0.0.1" ] print(" ".join(command)) # a = subprocess.call(command) os.system(" ".join(command)) self.ps() Console.ok("MongoDB has been deployed") self.info() return None # implement the return of the pid for the process. # store the pid in self.pid except Exception, e: Console.error("we had a problem starting the mongo daemon") print(e) Console.error("MongoDB has stopped") # TODO remove the exit in final code, for debugging only sys.exit()
def test_013_find_files(self): """ tests searching for a file the file being searched for exists in 3 files: twice as input and twice as output :return: """ HEADING() db = self.db db.connect() # Clear all jobs currently in the database to ensure a correct final assertion db.clear() # Add the jobs outlined in the YAML file db.add_from_yaml("etc/jobs.yaml") inputs, outputs = db.find_jobs_with_file("in1.txt") # Assert that the lengths of the inputs and outputs arrays are correct count_fgrep = len( Shell.fgrep("in1.txt", "etc/jobs.yaml").strip().split("\n")) assert (len(inputs) == count_fgrep)
def qstat(self, host, user=True, format='dict'): """ executes the qstat command on a particular host and returns the information as dict. :param host: The host as specified in ~/.ssh/config :param user: If True, only retirns information for the user If False, all jobs for all users are returned :param format: :return: """ data = None username = self.username(host) manager_host = self.manager(host) xml_data = Shell.ssh(manager_host, "qstat", "-x").rstrip() if format == 'dict': data = OpenPBS.qstat_xml_to_dict(xml_data) selected_data = {} for jobid in data: (owner, cm_host) = data[jobid]['Job_Owner'].split('@') if not user: selected_data[unicode(jobid)] = data[unicode(jobid)] elif owner == username: selected_data[unicode(jobid)] = data[unicode(jobid)] data = selected_data for jobid in data: data[unicode(jobid)][u"cm_jobid"] = jobid if "Variable_list" in data[unicode(jobid)]: data[unicode( jobid)][u"cm_Variable_list"] = self.variable_list( data, jobid) elif format == "xml": if user is not None: print("WARNING: " "restrictiong xml data for a user not supported.") data = xml_data return dict(data)
def ping(host): """ping the specified host. :param host: the name or ip of the host """ try: result = Shell.ping("-o", "-c", "1", host).strip().split("\n") except: pass try: (attributes, values) = result[-1].replace("round-trip", "")\ .strip().split("=") attributes = attributes.strip().split("/") values = values.strip().split("/") data = dict(zip(attributes, values)) data['loss'] = result[-2].split(",")[2].split("%")[0].strip() + "%" except: data = {} data['host'] = host return data
def vbox_controlvm(*args): a = ["controlvm"] + args return Shell.execute('VBoxManage', a)
def vbox_vminfo(*args): a = ["showvminfo"] + args return Shell.execute('VBoxManage', a)
def vbox_list(*args): a = ["list", "vms", "-l"] + args return Shell.execute('VBoxManage', a)
def vbox_startcm(*args): a = ["startvm"] + args return Shell.execute('VBoxManage', a)