def submit(self, add=True, dbpath=None, configpath=None, silent=False):
"""Submit this Job using qsub
add: Should this job be added to the JobDB database?
dbpath: Specify a non-default JobDB database
silent: do not print qsub stdout
Raises PBSError if error submitting the job.
"""
try:
self.jobID = misc_pbs.submit(substr=self.sub_string(),
silent=silent)
except misc.PBSError as e: #pylint: disable=invalid-name
raise e
if add:
db = jobdb.JobDB(dbpath=dbpath, configpath=configpath) #pylint: disable=invalid-name
status = jobdb.job_status_dict(jobid=self.jobID,
jobname=self.name,
rundir=os.getcwd(),
jobstatus="?",
auto=self.auto,
qsubstr=self.sub_string(),
walltime=misc.seconds(
self.walltime),
nodes=self.nodes,
procs=self.nodes * self.ppn)
db.add(status)
db.close()
def select_recent_id(self, recent_time):
""" Return a list of all jobids which were modified in the last 'recent_time' """
mintime = int(time.time() - misc.seconds(recent_time))
recent_job = []
self.curs.execute("SELECT jobid FROM jobs WHERE modifytime>=?", (mintime, ))
for r in sql_iter(self.curs): #pylint: disable=invalid-name
recent_job.append(r["jobid"])
return recent_job
def select_recent_id(self, recent_time):
""" Return a list of all jobids which were modified in the last 'recent_time' """
mintime = int(time.time() - misc.seconds(recent_time))
recent_job = []
self.curs.execute("SELECT jobid FROM jobs WHERE modifytime>=?",(mintime,))
for r in sql_iter(self.curs):
recent_job.append( r["jobid"] )
return recent_job
def select_recent_series_id(self, recent_time):
""" Return a list of lists of jobids (one for each series) which were modified in the last 'recent_time' """
mintime = int(time.time() - misc.seconds(recent_time))
recent_job = []
self.curs.execute("SELECT jobid FROM jobs WHERE modifytime>=?",(mintime,))
for r in sql_iter(self.curs):
if r["continuation_jobid"] == "-":
recent_job.append(select_series_id(jobid))
return recent_job
def select_recent_series_id(self, recent_time):
""" Return a list of lists of jobids (one for each series) which were
modified in the last 'recent_time' """
mintime = int(time.time() - misc.seconds(recent_time))
recent_job = []
self.curs.execute("SELECT jobid FROM jobs WHERE modifytime>=?", (mintime, ))
for r in sql_iter(self.curs): #pylint: disable=invalid-name
if r["continuation_jobid"] == "-":
recent_job.append(self.select_series_id(r["jobid"]))
return recent_job
def submit(self, add=True, dbpath=None):
"""Submit this Job using qsub
add: Should this job be added to the JobDB database?
dbpath: Specify a non-default JobDB database
Raises PBSError if error submitting the job.
"""
try:
self.jobID = misc.submit(qsubstr=self.qsub_string())
except PBSError as e:
raise e
if add:
db = jobdb.JobDB(dbpath=dbpath)
status = jobdb.job_status_dict(jobid = self.jobID, jobname = self.name, rundir = os.getcwd(), \
jobstatus = "?", auto = self.auto, qsubstr = self.qsub_string(), \
walltime = misc.seconds(self.walltime), nodes = self.nodes, procs = self.nodes*self.ppn)
db.add(status)
db.close()
def submit(self, add=True, dbpath=None):
"""Submit this Job using qsub
add: Should this job be added to the JobDB database?
dbpath: Specify a non-default JobDB database
Raises PBSError if error submitting the job.
"""
#TODO: fix generic exception catching
try:
self.jobID = misc.submit(qsubstr=self.qsub_string())
except Exception as e:
raise e
if add:
db = jobdb.JobDB(dbpath=dbpath)
status = jobdb.job_status_dict(jobid = self.jobID, jobname = self.name, rundir = os.getcwd(), \
jobstatus = "?", auto = self.auto, qsubstr = self.qsub_string(), \
walltime = misc.seconds(self.walltime), nodes = self.nodes, procs = self.nodes*self.ppn)
db.add(status)
db.close()
def job_status(jobid=None):
"""Return job status using qstat
Returns a dict of dict, with jobid as key in outer dict.
Inner dict contains:
"name", "nodes", "procs", "walltime",
"jobstatus": status ("Q","C","R", etc.)
"qstatstr": result of qstat -f jobid, None if not found
"elapsedtime": None if not started, else seconds as int
"starttime": None if not started, else seconds since epoch as int
"completiontime": None if not completed, else seconds since epoch as int
*This should be edited to return job_status_dict()'s*
"""
status = dict()
stdout = _qstat(jobid=jobid, full=True)
sout = StringIO.StringIO(stdout)
### TODO: figure out why jobstatus is being initialized as a None vs as a dict() and then checked for content ### pylint: disable=fixme
jobstatus = None
for line in sout:
m = re.search(r"Job Id:\s*(.*)\s", line) #pylint: disable=invalid-name
if m:
if jobstatus is not None:
if jobstatus["jobstatus"] == "R": #pylint: disable=unsubscriptable-object
jobstatus["elapsedtime"] = int(
time.time()) - jobstatus["starttime"] #pylint: disable=unsubscriptable-object
status[jobstatus["jobid"]] = jobstatus #pylint: disable=unsubscriptable-object
jobstatus = dict()
jobstatus["jobid"] = m.group(1).split(".")[0]
jobstatus["qstatstr"] = line
jobstatus["elapsedtime"] = None
jobstatus["starttime"] = None
jobstatus["completiontime"] = None
continue
jobstatus["qstatstr"] += line
#results = line.split()
#jobid = results[0].split(".")[0]
#jobstatus = dict()
#jobstatus["jobid"] = jobid
#jobstatus["jobname"] = results[3]
m = re.match(r"\s*Job_Name\s*=\s*(.*)\s", line) #pylint: disable=invalid-name
if m:
jobstatus["jobname"] = m.group(1)
continue
#jobstatus["nodes"] = int(results[5])
#jobstatus["procs"] = int(results[6])
m = re.match(r"\s*Resource_List\.nodes\s*=\s*(.*):ppn=(.*)\s", line) #pylint: disable=invalid-name
if m:
jobstatus["nodes"] = m.group(1)
jobstatus["procs"] = int(m.group(1)) * int(m.group(2))
continue
#jobstatus["walltime"] = int(seconds(results[8]))
m = re.match(r"\s*Resource_List\.walltime\s*=\s*(.*)\s", line) #pylint: disable=invalid-name
if m:
jobstatus["walltime"] = int(seconds(m.group(1)))
continue
#jobstatus["jobstatus"] = results[9]
m = re.match(r"\s*job_state\s*=\s*(.*)\s", line) #pylint: disable=invalid-name
if m:
jobstatus["jobstatus"] = m.group(1)
continue
#elapsedtime = line.split()[10]
#if elapsedtime == "--":
# jobstatus["elapsedtime"] = None
#else:
# jobstatus["elapsedtime"] = int(seconds(elapsedtime))
#
#qstatstr = qstat(jobid, full=True)
#if not re.match("^qstat: Unknown Job Id Error.*",qstatstr):
# jobstatus["qstatstr"] = qstatstr
# m = re.search("Job_Name = (.*)\n",qstatstr)
# if m:
# jobstatus["jobname"] = m.group(1)
#m = re.match("\s*resources_used.walltime\s*=\s*(.*)\s",line)
#if m:
# print line
# jobstatus["elapsedtime"] = int(seconds(m.group(1)))
m = re.match(r"\s*start_time\s*=\s*(.*)\s", line) #pylint: disable=invalid-name
if m:
jobstatus["starttime"] = int(
time.mktime(
datetime.datetime.strptime(
m.group(1), "%a %b %d %H:%M:%S %Y").timetuple()))
continue
m = re.search(r"\s*comp_time\s*=\s*(.*)\s", line) #pylint: disable=invalid-name
if m:
jobstatus["completiontime"] = int(
time.mktime(
datetime.datetime.strptime(
m.group(1), "%a %b %d %H:%M:%S %Y").timetuple()))
continue
if jobstatus is not None:
if jobstatus["jobstatus"] == "R":
jobstatus["elapsedtime"] = int(
time.time()) - jobstatus["starttime"]
status[jobstatus["jobid"]] = jobstatus
return status
def job_status(jobid=None):
"""Return job status using qstat
Returns a dict of dict, with jobid as key in outer dict.
Inner dict contains:
"name", "nodes", "procs", "walltime",
"jobstatus": status ("Q","C","R", etc.)
"qstatstr": result of qstat -f jobid, None if not found
"elapsedtime": None if not started, else seconds as int
"starttime": None if not started, else seconds since epoch as int
"completiontime": None if not completed, else seconds since epoch as int
*This should be edited to return job_status_dict()'s*
"""
status = dict()
stdout = _qstat(jobid=jobid, full=True)
sout = StringIO.StringIO(stdout)
jobstatus = None
for line in sout:
m = re.search(r"Job Id:\s*(.*)", line) #pylint: disable=invalid-name
if m:
if jobstatus is not None:
if jobstatus["jobstatus"] == "R": #pylint: disable=unsubscriptable-object
jobstatus["elapsedtime"] = int(time.time()) - jobstatus["starttime"] #pylint: disable=unsubscriptable-object
status[jobstatus["jobid"]] = jobstatus #pylint: disable=unsubscriptable-object
jobstatus = dict()
jobstatus["jobid"] = m.group(1).split(".")[0]
jobstatus["qstatstr"] = line
jobstatus["elapsedtime"] = None
jobstatus["starttime"] = None
jobstatus["completiontime"] = None
jobstatus["walltime"] = None
continue
jobstatus["qstatstr"] += line
m = re.match(r"\s*Job_Name\s*=\s*(.*)", line) #pylint: disable=invalid-name
if m:
jobstatus["jobname"] = m.group(1)
continue
#jobstatus["nodes"] = int(results[5])
#jobstatus["procs"] = int(results[6])
m = re.match(r"\s*Resource_List\.nodect\s*=\s*(.*)", line) #pylint: disable=invalid-name
if m:
jobstatus["nodes"] = m.group(1)
continue
m = re.match(r"\s*Resource_List\.ncpus\s*=\s*(.*)", line)
if m:
jobstatus["procs"] = int(m.group(1))
m = re.match(r"\s*Resource_List\.walltime\s*=\s*(.*)", line) #pylint: disable=invalid-name
if m:
jobstatus["walltime"] = int(seconds(m.group(1)))
continue
#jobstatus["jobstatus"] = results[9]
m = re.match(r"\s*job_state\s*=\s*(.*)", line) #pylint: disable=invalid-name
if m:
jobstatus["jobstatus"] = m.group(1)
continue
m = re.match(r"\s*stime\s*=\s*(.*)", line) #pylint: disable=invalid-name
if m:
jobstatus["starttime"] = int(time.mktime(datetime.datetime.strptime(
m.group(1), "%a %b %d %H:%M:%S %Y").timetuple()))
continue
m = re.search(r"\s*comp_time\s*=\s*(.*)", line) #pylint: disable=invalid-name
if m:
jobstatus["completiontime"] = int(time.mktime(datetime.datetime.strptime(
m.group(1), "%a %b %d %H:%M:%S %Y").timetuple()))
continue
if jobstatus is not None:
if jobstatus["jobstatus"] == "R":
jobstatus["elapsedtime"] = int(time.time()) - jobstatus["starttime"]
status[jobstatus["jobid"]] = jobstatus
return status
