def JSONDecoderTaskTree(dct): if dct.has_key("tid"): task = EngineDB.Task() task.tid = dct.get("tid") task.id = dct.get("id") task.title = dct.get("title") task.service = dct.get("service") task.minslots = dct.get("atleast", 1) task.maxslots = dct.get("atmost", 1) task.serialsubtasks = True if dct.get("serialsubtasks", False) == 1 else False task.ptids = [0] if dct.has_key("ants"): task.__dict__["instanceTids"] = dct.get("ants") task.cids = dct.get("cids", []) return task elif dct.has_key("data") and dct.has_key("children") \ and dct["data"] != OUTER_TASK_TREE_SENTINAL: # this puts the subtasks as a member of a task object (which is dct["data"]) dct["data"].children = dct["children"] return dct["data"] elif dct.has_key("data") and dct["data"] == OUTER_TASK_TREE_SENTINAL: # this ensures the top-level JSON wrapper just returns a list of tasks return dct["children"] else: raise FileSystemDBError, "invalid task tree"
def sync(self): self.log("synchronizing with filesystem") def jobDirSort(a, b): jidA = a.split("/")[-1] dateA = jidA[:8] jidB = b.split("/")[-1] dateB = jidB[:8] if dateA != dateB: return -cmp(dateA, dateB) else: jidA = jidA[8:] jidA = int(jidA) if jidA.isdigit() else 0 jidB = jidB[8:] jidB = int(jidB) if jidB.isdigit() else 0 return -cmp(jidA, jidB) # establish connection with postgres database db = EngineDB.EngineDB(dbhost=self.config.dbHostname(), db=self.config.dbDatabaseName(), user=MIGRATE_USER, password=MIGRATE_PASSWORD) try: db.open() except rpg.sql.SQLError, err: raise DBMigrateToolError, str(err)
def JSONDecoderJobInfo(dct): job = EngineDB.Job() job.jid = dct.get("jid", 0) job.owner = dct.get("user", "") job.spoolhost = dct.get("spoolhost", "") job.spoolfile = dct.get("sourcefile", "") job.spoolcwd = dct.get("cwd", "") job.spooladdr = dct.get("spooladdr", "") job.spooltime = datetime.datetime.fromtimestamp(dct.get("spooltime", 0)) job.title = dct.get("title", "") job.priority = dct.get("priority", 0) if job.priority < 0: job.priority = abs(job.priority) # currently the pausetime is not stored, so let's just set it to the spooltime job.pausetime = job.spooltime job.crews = dct.get("crews", []) job.tags = dct.get("tags", []) job.service = dct.get("service", "") job.envkey = dct.get("envkey", []) job.editpolicy = dct.get("editpolicy", "") job.minslots = dct.get("atleast", 1) job.maxslots = dct.get("atmost", 1) job.etalevel = dct.get("etalevel", 1) job.afterjids = dct.get("afterJids", []) if dct.has_key("afterTime"): job.aftertime = datetime.datetime.fromtimestamp(dct.get("afterTime")) job.metadata = dct.get("metadata", "") job.comment = dct.get("comment", "") return job
def execSQLWithResult(self, sql, user=SUPERUSER, dbname=None, *args): """Execute the supplied sql statement using the EngineDB's client.""" dbname = dbname or self.config.dbDatabaseName() db = EngineDB.EngineDB(user=user, db=dbname, dbhost=self.config.dbHostname(), port=self.config.dbPort()) try: db.open() db._execute(sql) result = db.cursor.fetchall() db.close() except rpg.sql.SQLError, err: raise ExecSQLError( "execSQLWithResult(): problem running '%s' as '%s' with args '%s': %s" % (sql, user, str(args), err))
def newTask (self, jid, tid, sst): task = EngineDB.Task() task.jid = jid task.tid = tid task.state = EngineDB.STATE_BLOCKED task.cids = [] task.serialsubtasks = True if sst else False # sst may be numeric task.statetime = self.spooltime setattr(task, "kids", []) # for construction use, not recorded to db setattr(task, "idx", 0) # for construction use, not recorded to db if tid > 0: # tid 0 is special and "reserved" for the job itself # commands attached to the job object are placed into # the db with their tid=0, but the task itself is not. task.idx = len(self.allTasks) self.allTasks.append( task ) return task
def upgradeDB(self, upgrades): """This method executes the upgrade actions.""" self.log("Upgrading database schema.") db = EngineDB.EngineDB(user=SUPERUSER, db=self.config.dbDatabaseName(), dbhost=self.config.dbHostname(), port=self.config.dbPort()) db.open() db._execute("begin") i = 0 for u in upgrades: i += 1 self.log("Applying upgrade %d of %d: %s" % (i, len(upgrades), str(u))) db._execute(u.getSQL()) db._execute("UPDATE param SET value='%s' WHERE name='schema-version'" % upgrade.SCHEMA_VERSION) db._execute("end")
def addJob (self, dct, spooladdr, task0): job = EngineDB.Job() job.jid = self.jid job.spooladdr = spooladdr job.spooltime = self.spooltime job.owner = dct.get("owner", "") job.spoolhost = dct.get("spoolhost", "") job.spoolfile = dct.get("spoolfile", "") job.spoolcwd = dct.get("spoolcwd", "") job.title = dct.get("title", "") job.priority = dct.get("priority", 0) if job.priority < 0: job.priority = abs(job.priority + 1) # also acct for T1-style shift # currently the pausetime is not stored, so just set it # to the spooltime for now job.pausetime = job.spooltime if 0 != dct.get("paused", 0): job.pausetime = job.spooltime job.crews = dct.get("crews", []) job.maxactive = dct.get("maxactive", 0) job.tags = dct.get("tags", "").split() job.service = dct.get("service", "") job.envkey = dct.get("envkey", []) job.serialsubtasks = task0.serialsubtasks job.editpolicy = dct.get("editpolicy", "") job.projects = dct.get("projects", []) job.tier = dct.get("tier", "") job.minslots = dct.get("minslots") job.maxslots = dct.get("maxslots") job.metadata = dct.get("metadata", "") job.comment = dct.get("comment", "") job.etalevel = dct.get("etalevel", 1) job.dirmap = dct.get("dirmaps", []) job.afterjids = dct.get("afterjids", []) if dct.has_key("aftertime"): job.aftertime = datetime.datetime.fromtimestamp(dct.get("aftertime")) job.numtasks = len( self.allTasks ) job.numready = self.nready job.numblocked = job.numtasks - job.numready job.maxcid = self.lastCID job.maxtid = self.lastTID return job
def __init__ (self, credentialsChannel, dbgmode=None, spoolTime=None, plpyModule=None): self.engineDB = EngineDB.EngineDB() self.dbgmode = dbgmode self.jobObj = [] self.allCmds = [] self.allTasks = [] self.instanceMap = {} self.nready = 0 self.jid = 0 self.lastTID = 0 self.lastCID = 0 self.tidOffset = 0 if spoolTime == None: spoolTime = time.time() self.spooltime = datetime.datetime.fromtimestamp( float(spoolTime) ) if plpyModule: self.dbconn = None self.dbcursor = PlpyCursor(plpyModule) else: if credentialsChannel: try: # open connection to psql import tractor.base.rpg.osutil as osutil if osutil.getlocalos() == "Linux": # preload libpq.so file so that later imports of psycopg2 will get the proper library # since rmanpy as compiled only looks in the install dir's lib/, but libpq is in lib/psql/lib ctypes.cdll.LoadLibrary(os.path.join(installDir(), "lib", "psql", "lib", "libpq.so.5")) # this doesn't appear to be necessary on OSX since the stock install comes with a valid libpq.so import psycopg2 dbConnInfo = credentialsChannel.read() self.dbconn = psycopg2.connect( dbConnInfo ) self.dbcursor = self.dbconn.cursor() except: raise else: print >>sys.stderr, "note: no database credentials provided." self.dbcursor = False
def ddl(dbname=DEFAULT_DATABASE_NAME): import upgrade db = EngineDB.EngineDB(db=dbname) # the first parts are executed before the database has been switched to "tractor" parts = [PREAMBLE, db.getCreate()] # create base (non-login) and login roles which inherit permissions from archtype base roles for baseRole, loginRoles in ROLES_BY_BASE_ROLE.iteritems(): parts.append("CREATE ROLE %s;" % baseRole) for loginRole in loginRoles: parts.append("CREATE ROLE %s WITH LOGIN IN ROLE %s;" % (loginRole, baseRole)) # grant permissions to base roles tableStr = ",".join([table.tablename.lower() for table in db.Tables]) parts.extend([ "GRANT SELECT,DELETE,INSERT,UPDATE ON %s TO writeroles;" % tableStr, "GRANT SELECT ON %s TO readroles;" % tableStr ]) # grant permission to views viewStr = ",".join([view.name for view in db.Views]) parts.extend([ "GRANT SELECT ON %s TO writeroles;" % viewStr, "GRANT SELECT ON %s TO readroles;" % viewStr ]) # limiting to one boostrap connection prevents multiple engines from running at once parts.append("ALTER ROLE bootstrap CONNECTION LIMIT 1;") # change the table owner to a role that can create inherited tables for table in db.Tables: parts.append("ALTER TABLE %s OWNER TO %s;" % (table.tablename.lower(), TABLE_OWNER)) parts.extend([ # start numbering jobs at 1 "INSERT INTO param VALUES ('jidcounter', 0);", # default install limits result set size "INSERT INTO param VALUES ('maxrecords', 10000);", # default to archiving deleted jobs "INSERT INTO param VALUES ('archiving', 1);", # default to archiving deleted jobs "INSERT INTO param VALUES ('schema-version', '%s');" % upgrade.SCHEMA_VERSION, # the high-level plpython functions defined above ]) return "\n".join(parts)
def addCmd (self, dct, ptask): self.lastCID += 1 cmd = EngineDB.Command() cmd.jid = self.jid cmd.tid = ptask.tid cmd.cid = self.lastCID ptask.cids.append( cmd.cid ) cmd.argv = dct.get("argv", []) cmd.msg = dct.get("msg") cmd.service = dct.get("service") cmd.tags = dct.get("tags", "").split() cmd.id = dct.get("id") cmd.refersto = dct.get("refersto") cmd.minslots = dct.get("minslots") cmd.maxslots = dct.get("maxslots") cmd.envkey = dct.get("envkey", []) cmd.retryrcodes = dct.get("retryrc", []) cmd.resumewhile = dct.get("resumewhile", []) cmd.resumepin = dct.get("resumepin", 0) t = dct.get("type", "RC") n = len(t) cmd.local = (n > 0 and t[0] == "L") cmd.expand = (n > 2 and t[2] == "X") if n > 1: if t[1] == "D": cmd.runtype = "cleanup" elif t[1] == "P": if n > 2 and t[2]=="D": cmd.runtype = "post_done" elif n > 2 and t[2]=="E": cmd.runtype = "post_error" else: cmd.runtype = "post_always" self.allCmds.append( cmd ) return cmd
def JSONDecoderCmdList(dct): if dct.has_key("cid"): cmd = EngineDB.Command() cmd.cid = dct.get("cid") cmd.argv = dct.get("argv", []) cmdtype = dct.get("type", "RC") if len(cmdtype) < 2: cmdtype = "RC" cmd.local = True if cmdtype[0] == "L" else False cmd.cleanup = True if cmdtype[1] == "D" else False cmd.expand = True if len(cmdtype) > 2 and cmdtype[2] == "X" else False cmd.msg = dct.get("msg") cmd.service = dct.get("service") tagstr = dct.get("tags") cmd.tags = tagstr.split() if tagstr else [] cmd.id = dct.get("id") cmd.refersto = dct.get("refersto") cmd.minslots = dct.get("atleast", 1) cmd.maxslots = dct.get("maxThreads", 1) cmd.envkey = dct.get("envkey", []) cmd.retryrcodes = dct.get("retryrc", []) return cmd else: return dct.values()
raise FileSystemDBError(msg) f.close() # traverse task tree and express tasks as updates taskStack = taskTree while taskStack: task = taskStack.pop() task.jid = self.job.jid self.tasks.append(task) if hasattr(task, "children"): taskStack.extend(task.children) # put parent tid in all children for child in task.children: child.ptids = [task.tid] # create a task with tid 0 for containing job-level commands zeroTask = EngineDB.Task() zeroTask.jid = self.job.jid zeroTask.tid = 0 zeroTask.title = "LAST" # NOTE: we're not yet maintaining the zero task's list of children zeroTask.children = [] self.tasks.append(zeroTask) # populate task LUT for task in self.tasks: self.taskByTid[task.tid] = task # extend the ptids list for tasks that are referred to by other tasks as instances for task in self.tasks: for tid in task.__dict__.get("instanceTids", []): childTask = self.taskByTid.get(tid)
parts.extend([ # start numbering jobs at 1 "INSERT INTO param VALUES ('jidcounter', 0);", # default install limits result set size "INSERT INTO param VALUES ('maxrecords', 10000);", # default to archiving deleted jobs "INSERT INTO param VALUES ('archiving', 1);", # default to archiving deleted jobs "INSERT INTO param VALUES ('schema-version', '%s');" % upgrade.SCHEMA_VERSION, # the high-level plpython functions defined above ]) return "\n".join(parts) if __name__=='__main__': import argparse parser = argparse.ArgumentParser(prog="ddl.py") parser.add_argument("--functions", action="store_true", help="dump only function definitions") parser.add_argument("--views", action="store_true", help="dump only view definitions") args = parser.parse_args() if args.functions: db = EngineDB.EngineDB(db=DEFAULT_DATABASE_NAME) for f in db.Functions: print f.getCreate() elif args.views: import upgrade for view in EngineDB.EngineDB.Views: print view.getCreate(), ";" else: print ddl()