def JSONDecoderTaskTree(dct):
if dct.has_key("tid"):
task = EngineDB.Task()
task.tid = dct.get("tid")
task.id = dct.get("id")
task.title = dct.get("title")
task.service = dct.get("service")
task.minslots = dct.get("atleast", 1)
task.maxslots = dct.get("atmost", 1)
task.serialsubtasks = True if dct.get("serialsubtasks",
False) == 1 else False
task.ptids = [0]
if dct.has_key("ants"):
task.__dict__["instanceTids"] = dct.get("ants")
task.cids = dct.get("cids", [])
return task
elif dct.has_key("data") and dct.has_key("children") \
and dct["data"] != OUTER_TASK_TREE_SENTINAL:
# this puts the subtasks as a member of a task object (which is dct["data"])
dct["data"].children = dct["children"]
return dct["data"]
elif dct.has_key("data") and dct["data"] == OUTER_TASK_TREE_SENTINAL:
# this ensures the top-level JSON wrapper just returns a list of tasks
return dct["children"]
else:
raise FileSystemDBError, "invalid task tree"
def sync(self):
self.log("synchronizing with filesystem")
def jobDirSort(a, b):
jidA = a.split("/")[-1]
dateA = jidA[:8]
jidB = b.split("/")[-1]
dateB = jidB[:8]
if dateA != dateB:
return -cmp(dateA, dateB)
else:
jidA = jidA[8:]
jidA = int(jidA) if jidA.isdigit() else 0
jidB = jidB[8:]
jidB = int(jidB) if jidB.isdigit() else 0
return -cmp(jidA, jidB)
# establish connection with postgres database
db = EngineDB.EngineDB(dbhost=self.config.dbHostname(),
db=self.config.dbDatabaseName(),
user=MIGRATE_USER,
password=MIGRATE_PASSWORD)
try:
db.open()
except rpg.sql.SQLError, err:
raise DBMigrateToolError, str(err)
def JSONDecoderJobInfo(dct):
job = EngineDB.Job()
job.jid = dct.get("jid", 0)
job.owner = dct.get("user", "")
job.spoolhost = dct.get("spoolhost", "")
job.spoolfile = dct.get("sourcefile", "")
job.spoolcwd = dct.get("cwd", "")
job.spooladdr = dct.get("spooladdr", "")
job.spooltime = datetime.datetime.fromtimestamp(dct.get("spooltime", 0))
job.title = dct.get("title", "")
job.priority = dct.get("priority", 0)
if job.priority < 0:
job.priority = abs(job.priority)
# currently the pausetime is not stored, so let's just set it to the spooltime
job.pausetime = job.spooltime
job.crews = dct.get("crews", [])
job.tags = dct.get("tags", [])
job.service = dct.get("service", "")
job.envkey = dct.get("envkey", [])
job.editpolicy = dct.get("editpolicy", "")
job.minslots = dct.get("atleast", 1)
job.maxslots = dct.get("atmost", 1)
job.etalevel = dct.get("etalevel", 1)
job.afterjids = dct.get("afterJids", [])
if dct.has_key("afterTime"):
job.aftertime = datetime.datetime.fromtimestamp(dct.get("afterTime"))
job.metadata = dct.get("metadata", "")
job.comment = dct.get("comment", "")
return job
def execSQLWithResult(self, sql, user=SUPERUSER, dbname=None, *args):
"""Execute the supplied sql statement using the EngineDB's client."""
dbname = dbname or self.config.dbDatabaseName()
db = EngineDB.EngineDB(user=user,
db=dbname,
dbhost=self.config.dbHostname(),
port=self.config.dbPort())
try:
db.open()
db._execute(sql)
result = db.cursor.fetchall()
db.close()
except rpg.sql.SQLError, err:
raise ExecSQLError(
"execSQLWithResult(): problem running '%s' as '%s' with args '%s': %s"
% (sql, user, str(args), err))
def newTask (self, jid, tid, sst):
task = EngineDB.Task()
task.jid = jid
task.tid = tid
task.state = EngineDB.STATE_BLOCKED
task.cids = []
task.serialsubtasks = True if sst else False # sst may be numeric
task.statetime = self.spooltime
setattr(task, "kids", []) # for construction use, not recorded to db
setattr(task, "idx", 0) # for construction use, not recorded to db
if tid > 0:
# tid 0 is special and "reserved" for the job itself
# commands attached to the job object are placed into
# the db with their tid=0, but the task itself is not.
task.idx = len(self.allTasks)
self.allTasks.append( task )
return task
def upgradeDB(self, upgrades):
"""This method executes the upgrade actions."""
self.log("Upgrading database schema.")
db = EngineDB.EngineDB(user=SUPERUSER,
db=self.config.dbDatabaseName(),
dbhost=self.config.dbHostname(),
port=self.config.dbPort())
db.open()
db._execute("begin")
i = 0
for u in upgrades:
i += 1
self.log("Applying upgrade %d of %d: %s" %
(i, len(upgrades), str(u)))
db._execute(u.getSQL())
db._execute("UPDATE param SET value='%s' WHERE name='schema-version'" %
upgrade.SCHEMA_VERSION)
db._execute("end")
def addJob (self, dct, spooladdr, task0):
job = EngineDB.Job()
job.jid = self.jid
job.spooladdr = spooladdr
job.spooltime = self.spooltime
job.owner = dct.get("owner", "")
job.spoolhost = dct.get("spoolhost", "")
job.spoolfile = dct.get("spoolfile", "")
job.spoolcwd = dct.get("spoolcwd", "")
job.title = dct.get("title", "")
job.priority = dct.get("priority", 0)
if job.priority < 0:
job.priority = abs(job.priority + 1) # also acct for T1-style shift
# currently the pausetime is not stored, so just set it
# to the spooltime for now
job.pausetime = job.spooltime
if 0 != dct.get("paused", 0):
job.pausetime = job.spooltime
job.crews = dct.get("crews", [])
job.maxactive = dct.get("maxactive", 0)
job.tags = dct.get("tags", "").split()
job.service = dct.get("service", "")
job.envkey = dct.get("envkey", [])
job.serialsubtasks = task0.serialsubtasks
job.editpolicy = dct.get("editpolicy", "")
job.projects = dct.get("projects", [])
job.tier = dct.get("tier", "")
job.minslots = dct.get("minslots")
job.maxslots = dct.get("maxslots")
job.metadata = dct.get("metadata", "")
job.comment = dct.get("comment", "")
job.etalevel = dct.get("etalevel", 1)
job.dirmap = dct.get("dirmaps", [])
job.afterjids = dct.get("afterjids", [])
if dct.has_key("aftertime"):
job.aftertime = datetime.datetime.fromtimestamp(dct.get("aftertime"))
job.numtasks = len( self.allTasks )
job.numready = self.nready
job.numblocked = job.numtasks - job.numready
job.maxcid = self.lastCID
job.maxtid = self.lastTID
return job
def __init__ (self, credentialsChannel, dbgmode=None,
spoolTime=None, plpyModule=None):
self.engineDB = EngineDB.EngineDB()
self.dbgmode = dbgmode
self.jobObj = []
self.allCmds = []
self.allTasks = []
self.instanceMap = {}
self.nready = 0
self.jid = 0
self.lastTID = 0
self.lastCID = 0
self.tidOffset = 0
if spoolTime == None:
spoolTime = time.time()
self.spooltime = datetime.datetime.fromtimestamp( float(spoolTime) )
if plpyModule:
self.dbconn = None
self.dbcursor = PlpyCursor(plpyModule)
else:
if credentialsChannel:
try:
# open connection to psql
import tractor.base.rpg.osutil as osutil
if osutil.getlocalos() == "Linux":
# preload libpq.so file so that later imports of psycopg2 will get the proper library
# since rmanpy as compiled only looks in the install dir's lib/, but libpq is in lib/psql/lib
ctypes.cdll.LoadLibrary(os.path.join(installDir(), "lib", "psql", "lib", "libpq.so.5"))
# this doesn't appear to be necessary on OSX since the stock install comes with a valid libpq.so
import psycopg2
dbConnInfo = credentialsChannel.read()
self.dbconn = psycopg2.connect( dbConnInfo )
self.dbcursor = self.dbconn.cursor()
except:
raise
else:
print >>sys.stderr, "note: no database credentials provided."
self.dbcursor = False
def ddl(dbname=DEFAULT_DATABASE_NAME):
import upgrade
db = EngineDB.EngineDB(db=dbname)
# the first parts are executed before the database has been switched to "tractor"
parts = [PREAMBLE, db.getCreate()]
# create base (non-login) and login roles which inherit permissions from archtype base roles
for baseRole, loginRoles in ROLES_BY_BASE_ROLE.iteritems():
parts.append("CREATE ROLE %s;" % baseRole)
for loginRole in loginRoles:
parts.append("CREATE ROLE %s WITH LOGIN IN ROLE %s;" % (loginRole, baseRole))
# grant permissions to base roles
tableStr = ",".join([table.tablename.lower() for table in db.Tables])
parts.extend([
"GRANT SELECT,DELETE,INSERT,UPDATE ON %s TO writeroles;" % tableStr,
"GRANT SELECT ON %s TO readroles;" % tableStr
])
# grant permission to views
viewStr = ",".join([view.name for view in db.Views])
parts.extend([
"GRANT SELECT ON %s TO writeroles;" % viewStr,
"GRANT SELECT ON %s TO readroles;" % viewStr
])
# limiting to one boostrap connection prevents multiple engines from running at once
parts.append("ALTER ROLE bootstrap CONNECTION LIMIT 1;")
# change the table owner to a role that can create inherited tables
for table in db.Tables:
parts.append("ALTER TABLE %s OWNER TO %s;" % (table.tablename.lower(), TABLE_OWNER))
parts.extend([
# start numbering jobs at 1
"INSERT INTO param VALUES ('jidcounter', 0);",
# default install limits result set size
"INSERT INTO param VALUES ('maxrecords', 10000);",
# default to archiving deleted jobs
"INSERT INTO param VALUES ('archiving', 1);",
# default to archiving deleted jobs
"INSERT INTO param VALUES ('schema-version', '%s');" % upgrade.SCHEMA_VERSION,
# the high-level plpython functions defined above
])
return "\n".join(parts)
def addCmd (self, dct, ptask):
self.lastCID += 1
cmd = EngineDB.Command()
cmd.jid = self.jid
cmd.tid = ptask.tid
cmd.cid = self.lastCID
ptask.cids.append( cmd.cid )
cmd.argv = dct.get("argv", [])
cmd.msg = dct.get("msg")
cmd.service = dct.get("service")
cmd.tags = dct.get("tags", "").split()
cmd.id = dct.get("id")
cmd.refersto = dct.get("refersto")
cmd.minslots = dct.get("minslots")
cmd.maxslots = dct.get("maxslots")
cmd.envkey = dct.get("envkey", [])
cmd.retryrcodes = dct.get("retryrc", [])
cmd.resumewhile = dct.get("resumewhile", [])
cmd.resumepin = dct.get("resumepin", 0)
t = dct.get("type", "RC")
n = len(t)
cmd.local = (n > 0 and t[0] == "L")
cmd.expand = (n > 2 and t[2] == "X")
if n > 1:
if t[1] == "D":
cmd.runtype = "cleanup"
elif t[1] == "P":
if n > 2 and t[2]=="D":
cmd.runtype = "post_done"
elif n > 2 and t[2]=="E":
cmd.runtype = "post_error"
else:
cmd.runtype = "post_always"
self.allCmds.append( cmd )
return cmd
def JSONDecoderCmdList(dct):
if dct.has_key("cid"):
cmd = EngineDB.Command()
cmd.cid = dct.get("cid")
cmd.argv = dct.get("argv", [])
cmdtype = dct.get("type", "RC")
if len(cmdtype) < 2:
cmdtype = "RC"
cmd.local = True if cmdtype[0] == "L" else False
cmd.cleanup = True if cmdtype[1] == "D" else False
cmd.expand = True if len(cmdtype) > 2 and cmdtype[2] == "X" else False
cmd.msg = dct.get("msg")
cmd.service = dct.get("service")
tagstr = dct.get("tags")
cmd.tags = tagstr.split() if tagstr else []
cmd.id = dct.get("id")
cmd.refersto = dct.get("refersto")
cmd.minslots = dct.get("atleast", 1)
cmd.maxslots = dct.get("maxThreads", 1)
cmd.envkey = dct.get("envkey", [])
cmd.retryrcodes = dct.get("retryrc", [])
return cmd
else:
return dct.values()
raise FileSystemDBError(msg)
f.close()
# traverse task tree and express tasks as updates
taskStack = taskTree
while taskStack:
task = taskStack.pop()
task.jid = self.job.jid
self.tasks.append(task)
if hasattr(task, "children"):
taskStack.extend(task.children)
# put parent tid in all children
for child in task.children:
child.ptids = [task.tid]
# create a task with tid 0 for containing job-level commands
zeroTask = EngineDB.Task()
zeroTask.jid = self.job.jid
zeroTask.tid = 0
zeroTask.title = "LAST"
# NOTE: we're not yet maintaining the zero task's list of children
zeroTask.children = []
self.tasks.append(zeroTask)
# populate task LUT
for task in self.tasks:
self.taskByTid[task.tid] = task
# extend the ptids list for tasks that are referred to by other tasks as instances
for task in self.tasks:
for tid in task.__dict__.get("instanceTids", []):
childTask = self.taskByTid.get(tid)
parts.extend([
# start numbering jobs at 1
"INSERT INTO param VALUES ('jidcounter', 0);",
# default install limits result set size
"INSERT INTO param VALUES ('maxrecords', 10000);",
# default to archiving deleted jobs
"INSERT INTO param VALUES ('archiving', 1);",
# default to archiving deleted jobs
"INSERT INTO param VALUES ('schema-version', '%s');" % upgrade.SCHEMA_VERSION,
# the high-level plpython functions defined above
])
return "\n".join(parts)
if __name__=='__main__':
import argparse
parser = argparse.ArgumentParser(prog="ddl.py")
parser.add_argument("--functions", action="store_true", help="dump only function definitions")
parser.add_argument("--views", action="store_true", help="dump only view definitions")
args = parser.parse_args()
if args.functions:
db = EngineDB.EngineDB(db=DEFAULT_DATABASE_NAME)
for f in db.Functions:
print f.getCreate()
elif args.views:
import upgrade
for view in EngineDB.EngineDB.Views:
print view.getCreate(), ";"
else:
print ddl()