def submit(self, add=True, dbpath=None, configpath=None, silent=False): """Submit this Job using qsub add: Should this job be added to the JobDB database? dbpath: Specify a non-default JobDB database silent: do not print qsub stdout Raises PBSError if error submitting the job. """ try: self.jobID = misc_pbs.submit(substr=self.sub_string(), silent=silent) except misc.PBSError as e: #pylint: disable=invalid-name raise e if add: db = jobdb.JobDB(dbpath=dbpath, configpath=configpath) #pylint: disable=invalid-name status = jobdb.job_status_dict(jobid=self.jobID, jobname=self.name, rundir=os.getcwd(), jobstatus="?", auto=self.auto, qsubstr=self.sub_string(), walltime=misc.seconds( self.walltime), nodes=self.nodes, procs=self.nodes * self.ppn) db.add(status) db.close()
def select_recent_id(self, recent_time): """ Return a list of all jobids which were modified in the last 'recent_time' """ mintime = int(time.time() - misc.seconds(recent_time)) recent_job = [] self.curs.execute("SELECT jobid FROM jobs WHERE modifytime>=?", (mintime, )) for r in sql_iter(self.curs): #pylint: disable=invalid-name recent_job.append(r["jobid"]) return recent_job
def select_recent_id(self, recent_time): """ Return a list of all jobids which were modified in the last 'recent_time' """ mintime = int(time.time() - misc.seconds(recent_time)) recent_job = [] self.curs.execute("SELECT jobid FROM jobs WHERE modifytime>=?",(mintime,)) for r in sql_iter(self.curs): recent_job.append( r["jobid"] ) return recent_job
def select_recent_series_id(self, recent_time): """ Return a list of lists of jobids (one for each series) which were modified in the last 'recent_time' """ mintime = int(time.time() - misc.seconds(recent_time)) recent_job = [] self.curs.execute("SELECT jobid FROM jobs WHERE modifytime>=?",(mintime,)) for r in sql_iter(self.curs): if r["continuation_jobid"] == "-": recent_job.append(select_series_id(jobid)) return recent_job
def select_recent_series_id(self, recent_time): """ Return a list of lists of jobids (one for each series) which were modified in the last 'recent_time' """ mintime = int(time.time() - misc.seconds(recent_time)) recent_job = [] self.curs.execute("SELECT jobid FROM jobs WHERE modifytime>=?", (mintime, )) for r in sql_iter(self.curs): #pylint: disable=invalid-name if r["continuation_jobid"] == "-": recent_job.append(self.select_series_id(r["jobid"])) return recent_job
def submit(self, add=True, dbpath=None): """Submit this Job using qsub add: Should this job be added to the JobDB database? dbpath: Specify a non-default JobDB database Raises PBSError if error submitting the job. """ try: self.jobID = misc.submit(qsubstr=self.qsub_string()) except PBSError as e: raise e if add: db = jobdb.JobDB(dbpath=dbpath) status = jobdb.job_status_dict(jobid = self.jobID, jobname = self.name, rundir = os.getcwd(), \ jobstatus = "?", auto = self.auto, qsubstr = self.qsub_string(), \ walltime = misc.seconds(self.walltime), nodes = self.nodes, procs = self.nodes*self.ppn) db.add(status) db.close()
def submit(self, add=True, dbpath=None): """Submit this Job using qsub add: Should this job be added to the JobDB database? dbpath: Specify a non-default JobDB database Raises PBSError if error submitting the job. """ #TODO: fix generic exception catching try: self.jobID = misc.submit(qsubstr=self.qsub_string()) except Exception as e: raise e if add: db = jobdb.JobDB(dbpath=dbpath) status = jobdb.job_status_dict(jobid = self.jobID, jobname = self.name, rundir = os.getcwd(), \ jobstatus = "?", auto = self.auto, qsubstr = self.qsub_string(), \ walltime = misc.seconds(self.walltime), nodes = self.nodes, procs = self.nodes*self.ppn) db.add(status) db.close()
def job_status(jobid=None): """Return job status using qstat Returns a dict of dict, with jobid as key in outer dict. Inner dict contains: "name", "nodes", "procs", "walltime", "jobstatus": status ("Q","C","R", etc.) "qstatstr": result of qstat -f jobid, None if not found "elapsedtime": None if not started, else seconds as int "starttime": None if not started, else seconds since epoch as int "completiontime": None if not completed, else seconds since epoch as int *This should be edited to return job_status_dict()'s* """ status = dict() stdout = _qstat(jobid=jobid, full=True) sout = StringIO.StringIO(stdout) ### TODO: figure out why jobstatus is being initialized as a None vs as a dict() and then checked for content ### pylint: disable=fixme jobstatus = None for line in sout: m = re.search(r"Job Id:\s*(.*)\s", line) #pylint: disable=invalid-name if m: if jobstatus is not None: if jobstatus["jobstatus"] == "R": #pylint: disable=unsubscriptable-object jobstatus["elapsedtime"] = int( time.time()) - jobstatus["starttime"] #pylint: disable=unsubscriptable-object status[jobstatus["jobid"]] = jobstatus #pylint: disable=unsubscriptable-object jobstatus = dict() jobstatus["jobid"] = m.group(1).split(".")[0] jobstatus["qstatstr"] = line jobstatus["elapsedtime"] = None jobstatus["starttime"] = None jobstatus["completiontime"] = None continue jobstatus["qstatstr"] += line #results = line.split() #jobid = results[0].split(".")[0] #jobstatus = dict() #jobstatus["jobid"] = jobid #jobstatus["jobname"] = results[3] m = re.match(r"\s*Job_Name\s*=\s*(.*)\s", line) #pylint: disable=invalid-name if m: jobstatus["jobname"] = m.group(1) continue #jobstatus["nodes"] = int(results[5]) #jobstatus["procs"] = int(results[6]) m = re.match(r"\s*Resource_List\.nodes\s*=\s*(.*):ppn=(.*)\s", line) #pylint: disable=invalid-name if m: jobstatus["nodes"] = m.group(1) jobstatus["procs"] = int(m.group(1)) * int(m.group(2)) continue #jobstatus["walltime"] = int(seconds(results[8])) m = re.match(r"\s*Resource_List\.walltime\s*=\s*(.*)\s", line) #pylint: disable=invalid-name if m: jobstatus["walltime"] = int(seconds(m.group(1))) continue #jobstatus["jobstatus"] = results[9] m = re.match(r"\s*job_state\s*=\s*(.*)\s", line) #pylint: disable=invalid-name if m: jobstatus["jobstatus"] = m.group(1) continue #elapsedtime = line.split()[10] #if elapsedtime == "--": # jobstatus["elapsedtime"] = None #else: # jobstatus["elapsedtime"] = int(seconds(elapsedtime)) # #qstatstr = qstat(jobid, full=True) #if not re.match("^qstat: Unknown Job Id Error.*",qstatstr): # jobstatus["qstatstr"] = qstatstr # m = re.search("Job_Name = (.*)\n",qstatstr) # if m: # jobstatus["jobname"] = m.group(1) #m = re.match("\s*resources_used.walltime\s*=\s*(.*)\s",line) #if m: # print line # jobstatus["elapsedtime"] = int(seconds(m.group(1))) m = re.match(r"\s*start_time\s*=\s*(.*)\s", line) #pylint: disable=invalid-name if m: jobstatus["starttime"] = int( time.mktime( datetime.datetime.strptime( m.group(1), "%a %b %d %H:%M:%S %Y").timetuple())) continue m = re.search(r"\s*comp_time\s*=\s*(.*)\s", line) #pylint: disable=invalid-name if m: jobstatus["completiontime"] = int( time.mktime( datetime.datetime.strptime( m.group(1), "%a %b %d %H:%M:%S %Y").timetuple())) continue if jobstatus is not None: if jobstatus["jobstatus"] == "R": jobstatus["elapsedtime"] = int( time.time()) - jobstatus["starttime"] status[jobstatus["jobid"]] = jobstatus return status
def job_status(jobid=None): """Return job status using qstat Returns a dict of dict, with jobid as key in outer dict. Inner dict contains: "name", "nodes", "procs", "walltime", "jobstatus": status ("Q","C","R", etc.) "qstatstr": result of qstat -f jobid, None if not found "elapsedtime": None if not started, else seconds as int "starttime": None if not started, else seconds since epoch as int "completiontime": None if not completed, else seconds since epoch as int *This should be edited to return job_status_dict()'s* """ status = dict() stdout = _qstat(jobid=jobid, full=True) sout = StringIO.StringIO(stdout) jobstatus = None for line in sout: m = re.search(r"Job Id:\s*(.*)", line) #pylint: disable=invalid-name if m: if jobstatus is not None: if jobstatus["jobstatus"] == "R": #pylint: disable=unsubscriptable-object jobstatus["elapsedtime"] = int(time.time()) - jobstatus["starttime"] #pylint: disable=unsubscriptable-object status[jobstatus["jobid"]] = jobstatus #pylint: disable=unsubscriptable-object jobstatus = dict() jobstatus["jobid"] = m.group(1).split(".")[0] jobstatus["qstatstr"] = line jobstatus["elapsedtime"] = None jobstatus["starttime"] = None jobstatus["completiontime"] = None jobstatus["walltime"] = None continue jobstatus["qstatstr"] += line m = re.match(r"\s*Job_Name\s*=\s*(.*)", line) #pylint: disable=invalid-name if m: jobstatus["jobname"] = m.group(1) continue #jobstatus["nodes"] = int(results[5]) #jobstatus["procs"] = int(results[6]) m = re.match(r"\s*Resource_List\.nodect\s*=\s*(.*)", line) #pylint: disable=invalid-name if m: jobstatus["nodes"] = m.group(1) continue m = re.match(r"\s*Resource_List\.ncpus\s*=\s*(.*)", line) if m: jobstatus["procs"] = int(m.group(1)) m = re.match(r"\s*Resource_List\.walltime\s*=\s*(.*)", line) #pylint: disable=invalid-name if m: jobstatus["walltime"] = int(seconds(m.group(1))) continue #jobstatus["jobstatus"] = results[9] m = re.match(r"\s*job_state\s*=\s*(.*)", line) #pylint: disable=invalid-name if m: jobstatus["jobstatus"] = m.group(1) continue m = re.match(r"\s*stime\s*=\s*(.*)", line) #pylint: disable=invalid-name if m: jobstatus["starttime"] = int(time.mktime(datetime.datetime.strptime( m.group(1), "%a %b %d %H:%M:%S %Y").timetuple())) continue m = re.search(r"\s*comp_time\s*=\s*(.*)", line) #pylint: disable=invalid-name if m: jobstatus["completiontime"] = int(time.mktime(datetime.datetime.strptime( m.group(1), "%a %b %d %H:%M:%S %Y").timetuple())) continue if jobstatus is not None: if jobstatus["jobstatus"] == "R": jobstatus["elapsedtime"] = int(time.time()) - jobstatus["starttime"] status[jobstatus["jobid"]] = jobstatus return status