def sync(self):
self.log("synchronizing with filesystem")
def jobDirSort(a, b):
jidA = a.split("/")[-1]
dateA = jidA[:8]
jidB = b.split("/")[-1]
dateB = jidB[:8]
if dateA != dateB:
return -cmp(dateA, dateB)
else:
jidA = jidA[8:]
jidA = int(jidA) if jidA.isdigit() else 0
jidB = jidB[8:]
jidB = int(jidB) if jidB.isdigit() else 0
return -cmp(jidA, jidB)
# establish connection with postgres database
db = EngineDB.EngineDB(dbhost=self.config.dbHostname(),
db=self.config.dbDatabaseName(),
user=MIGRATE_USER,
password=MIGRATE_PASSWORD)
try:
db.open()
except rpg.sql.SQLError, err:
raise DBMigrateToolError, str(err)
def execSQLWithResult(self, sql, user=SUPERUSER, dbname=None, *args):
"""Execute the supplied sql statement using the EngineDB's client."""
dbname = dbname or self.config.dbDatabaseName()
db = EngineDB.EngineDB(user=user,
db=dbname,
dbhost=self.config.dbHostname(),
port=self.config.dbPort())
try:
db.open()
db._execute(sql)
result = db.cursor.fetchall()
db.close()
except rpg.sql.SQLError, err:
raise ExecSQLError(
"execSQLWithResult(): problem running '%s' as '%s' with args '%s': %s"
% (sql, user, str(args), err))
def upgradeDB(self, upgrades):
"""This method executes the upgrade actions."""
self.log("Upgrading database schema.")
db = EngineDB.EngineDB(user=SUPERUSER,
db=self.config.dbDatabaseName(),
dbhost=self.config.dbHostname(),
port=self.config.dbPort())
db.open()
db._execute("begin")
i = 0
for u in upgrades:
i += 1
self.log("Applying upgrade %d of %d: %s" %
(i, len(upgrades), str(u)))
db._execute(u.getSQL())
db._execute("UPDATE param SET value='%s' WHERE name='schema-version'" %
upgrade.SCHEMA_VERSION)
db._execute("end")
def __init__ (self, credentialsChannel, dbgmode=None,
spoolTime=None, plpyModule=None):
self.engineDB = EngineDB.EngineDB()
self.dbgmode = dbgmode
self.jobObj = []
self.allCmds = []
self.allTasks = []
self.instanceMap = {}
self.nready = 0
self.jid = 0
self.lastTID = 0
self.lastCID = 0
self.tidOffset = 0
if spoolTime == None:
spoolTime = time.time()
self.spooltime = datetime.datetime.fromtimestamp( float(spoolTime) )
if plpyModule:
self.dbconn = None
self.dbcursor = PlpyCursor(plpyModule)
else:
if credentialsChannel:
try:
# open connection to psql
import tractor.base.rpg.osutil as osutil
if osutil.getlocalos() == "Linux":
# preload libpq.so file so that later imports of psycopg2 will get the proper library
# since rmanpy as compiled only looks in the install dir's lib/, but libpq is in lib/psql/lib
ctypes.cdll.LoadLibrary(os.path.join(installDir(), "lib", "psql", "lib", "libpq.so.5"))
# this doesn't appear to be necessary on OSX since the stock install comes with a valid libpq.so
import psycopg2
dbConnInfo = credentialsChannel.read()
self.dbconn = psycopg2.connect( dbConnInfo )
self.dbcursor = self.dbconn.cursor()
except:
raise
else:
print >>sys.stderr, "note: no database credentials provided."
self.dbcursor = False
def ddl(dbname=DEFAULT_DATABASE_NAME):
import upgrade
db = EngineDB.EngineDB(db=dbname)
# the first parts are executed before the database has been switched to "tractor"
parts = [PREAMBLE, db.getCreate()]
# create base (non-login) and login roles which inherit permissions from archtype base roles
for baseRole, loginRoles in ROLES_BY_BASE_ROLE.iteritems():
parts.append("CREATE ROLE %s;" % baseRole)
for loginRole in loginRoles:
parts.append("CREATE ROLE %s WITH LOGIN IN ROLE %s;" % (loginRole, baseRole))
# grant permissions to base roles
tableStr = ",".join([table.tablename.lower() for table in db.Tables])
parts.extend([
"GRANT SELECT,DELETE,INSERT,UPDATE ON %s TO writeroles;" % tableStr,
"GRANT SELECT ON %s TO readroles;" % tableStr
])
# grant permission to views
viewStr = ",".join([view.name for view in db.Views])
parts.extend([
"GRANT SELECT ON %s TO writeroles;" % viewStr,
"GRANT SELECT ON %s TO readroles;" % viewStr
])
# limiting to one boostrap connection prevents multiple engines from running at once
parts.append("ALTER ROLE bootstrap CONNECTION LIMIT 1;")
# change the table owner to a role that can create inherited tables
for table in db.Tables:
parts.append("ALTER TABLE %s OWNER TO %s;" % (table.tablename.lower(), TABLE_OWNER))
parts.extend([
# start numbering jobs at 1
"INSERT INTO param VALUES ('jidcounter', 0);",
# default install limits result set size
"INSERT INTO param VALUES ('maxrecords', 10000);",
# default to archiving deleted jobs
"INSERT INTO param VALUES ('archiving', 1);",
# default to archiving deleted jobs
"INSERT INTO param VALUES ('schema-version', '%s');" % upgrade.SCHEMA_VERSION,
# the high-level plpython functions defined above
])
return "\n".join(parts)
parts.extend([
# start numbering jobs at 1
"INSERT INTO param VALUES ('jidcounter', 0);",
# default install limits result set size
"INSERT INTO param VALUES ('maxrecords', 10000);",
# default to archiving deleted jobs
"INSERT INTO param VALUES ('archiving', 1);",
# default to archiving deleted jobs
"INSERT INTO param VALUES ('schema-version', '%s');" % upgrade.SCHEMA_VERSION,
# the high-level plpython functions defined above
])
return "\n".join(parts)
if __name__=='__main__':
import argparse
parser = argparse.ArgumentParser(prog="ddl.py")
parser.add_argument("--functions", action="store_true", help="dump only function definitions")
parser.add_argument("--views", action="store_true", help="dump only view definitions")
args = parser.parse_args()
if args.functions:
db = EngineDB.EngineDB(db=DEFAULT_DATABASE_NAME)
for f in db.Functions:
print f.getCreate()
elif args.views:
import upgrade
for view in EngineDB.EngineDB.Views:
print view.getCreate(), ";"
else:
print ddl()