def __init__(self, scheduler, projectId):
self.checkScheduler(scheduler)
self.scheduler = scheduler # SGE
self.projectId = projectId
self.groupId2group = {} # key=identifier value=object
self.path2db = "%s/%s_%s.db" % (os.getcwd(), self.projectId,
Utils.uniq_alphanum(5))
self.db = DbSqlite(self.path2db)
self.setUpJobTable()
class JobManager(object):
"""
All job groups of a given job manager will share the same scheduler.
"""
def __init__(self, scheduler, projectId):
self.checkScheduler(scheduler)
self.scheduler = scheduler # SGE
self.projectId = projectId
self.groupId2group = {} # key=identifier value=object
self.path2db = "%s/%s_%s.db" % (os.getcwd(), self.projectId,
Utils.uniq_alphanum(5))
self.db = DbSqlite(self.path2db)
self.setUpJobTable()
def __getitem__(self, jobGroupId):
return self.groupId2group[jobGroupId]
@staticmethod
def checkScheduler(scheduler):
if scheduler not in ["SGE"]:
msg = "unknown scheduler '%s'" % scheduler
raise ValueError(msg)
@staticmethod
def checkQueue(scheduler, queue):
if scheduler == "SGE":
p = subprocess.check_output(["qconf", "-sql"])
p = p.split("\n")
if queue not in p:
msg = "unknown queue '%s'" % queue
raise ValueError(msg)
def setUpJobTable(self):
"""
The "status" column can take three different values: waiting, success, and error.
Right after submission, it will be "waiting".
Once it is not in the output of "qstat" anymore, the output file will be scanned.
Depending on the result, the status will be updated to "success" or "error".
"""
cmd = "CREATE TABLE jobs"
cmd += " (jobid INT,"
cmd += " jobname TEXT NOT NULL,"
cmd += " jobdir TEXT NOT NULL,"
cmd += " groupid TEXT NOT NULL,"
cmd += " queue TEXT NOT NULL,"
cmd += " resources TEXT,"
cmd += " status TEXT NOT NULL,"
cmd += " datetime TEXT DEFAULT CURRENT_TIMESTAMP NOT NULL)"
self.db.execomm(cmd)
def insert(self, iJobGroup):
JobManager.checkQueue(self.scheduler, iJobGroup.queue)
self.groupId2group[iJobGroup.id] = iJobGroup
self.groupId2group[iJobGroup.id].scheduler = self.scheduler
def submit(self, jobGroupId):
self.groupId2group[jobGroupId].submit(self.db)
def wait(self, jobGroupId, verbose=1):
self.groupId2group[jobGroupId].wait(self.db, verbose)
def close(self):
self.db.conn.close()
os.remove(self.path2db)
