def __init__(self, scheduler, projectId): self.checkScheduler(scheduler) self.scheduler = scheduler # SGE self.projectId = projectId self.groupId2group = {} # key=identifier value=object self.path2db = "%s/%s_%s.db" % (os.getcwd(), self.projectId, Utils.uniq_alphanum(5)) self.db = DbSqlite(self.path2db) self.setUpJobTable()
class JobManager(object): """ All job groups of a given job manager will share the same scheduler. """ def __init__(self, scheduler, projectId): self.checkScheduler(scheduler) self.scheduler = scheduler # SGE self.projectId = projectId self.groupId2group = {} # key=identifier value=object self.path2db = "%s/%s_%s.db" % (os.getcwd(), self.projectId, Utils.uniq_alphanum(5)) self.db = DbSqlite(self.path2db) self.setUpJobTable() def __getitem__(self, jobGroupId): return self.groupId2group[jobGroupId] @staticmethod def checkScheduler(scheduler): if scheduler not in ["SGE"]: msg = "unknown scheduler '%s'" % scheduler raise ValueError(msg) @staticmethod def checkQueue(scheduler, queue): if scheduler == "SGE": p = subprocess.check_output(["qconf", "-sql"]) p = p.split("\n") if queue not in p: msg = "unknown queue '%s'" % queue raise ValueError(msg) def setUpJobTable(self): """ The "status" column can take three different values: waiting, success, and error. Right after submission, it will be "waiting". Once it is not in the output of "qstat" anymore, the output file will be scanned. Depending on the result, the status will be updated to "success" or "error". """ cmd = "CREATE TABLE jobs" cmd += " (jobid INT," cmd += " jobname TEXT NOT NULL," cmd += " jobdir TEXT NOT NULL," cmd += " groupid TEXT NOT NULL," cmd += " queue TEXT NOT NULL," cmd += " resources TEXT," cmd += " status TEXT NOT NULL," cmd += " datetime TEXT DEFAULT CURRENT_TIMESTAMP NOT NULL)" self.db.execomm(cmd) def insert(self, iJobGroup): JobManager.checkQueue(self.scheduler, iJobGroup.queue) self.groupId2group[iJobGroup.id] = iJobGroup self.groupId2group[iJobGroup.id].scheduler = self.scheduler def submit(self, jobGroupId): self.groupId2group[jobGroupId].submit(self.db) def wait(self, jobGroupId, verbose=1): self.groupId2group[jobGroupId].wait(self.db, verbose) def close(self): self.db.conn.close() os.remove(self.path2db)