class ListCmds(CmdLineTool.MultiCmdLineTool):
"""Helper object for testing."""
options = [
CmdLineTool.BooleanOption("--force",
help="force all operations. This will "
"allow any operation to be "
"performed on a task or job, even "
"if you are not."),
CmdLineTool.BooleanOption(
"-d", "--debug", help="print debug info."),
] + CmdLineTool.MultiCmdLineTool.options
commands = {
"jobs": JobsCmd,
"tasks": TasksCmd,
}
class OperateCmd(QueryCmd):
"""An operation command will query the database and run an arbitrary
operation on each object returned."""
# the default set of options for all operation commands
options = [
CmdLineTool.StrListOption("-c", "--cols",
help="list of columns that will be "
"printed for each each row."),
CmdLineTool.StrListOption("-s", "--sortby",
help="sort the results in a specified "
"order before performing any "
"operations."),
CmdLineTool.BooleanOption("--ns", "--nosort", dest="nosort",
help="do not sort the results"),
CmdLineTool.IntOption ("--limit",
help="only operate on the first l rows"),
] + CmdLineTool.CmdLineTool.options
def __init__(self, table, *args, **kwargs):
# call the super
super(OperateCmd, self).__init__(table, *args, **kwargs)
# if a list of objects is provided at the command line, then
# subclasses can fill in this value. Setting this will prevent
# the database from being contacted
self.objects = []
def parseArgs(self, *args, **kwargs):
result = super(OperateCmd, self).parseArgs(*args, **kwargs)
# make sure a where exists
if not self.where:
raise OptionParser.HelpRequest(self.getHelpStr())
# make sure a value is set for options we are not supporting, but
# that the super expects to exist.
for name,default in (("distinct", None),
("delimiter", ' '),
("raw", False),
("timefmt", None),
("noformat", False),
("noheader", False)):
self.opts.__dict__.setdefault(name, default)
return result
def opendb(self):
"""We don't need to open the database if the objects member is set."""
if not self.objects:
super(OperateCmd, self).opendb()
def getNewObjs(self, num, curr, objs, distinct):
"""Return at most 'num' new objects for the operationLoop. The
objects returned are tq Operation Job or Task objects. By
default the mrd cursor is checked for waiting job objects, but
a list can be provided which will be modified."""
# a list of all the operation objects we are going to return
newobjs = []
# loop until we have enough objects
while len(newobjs) < num and curr < len(objs):
obj = objs[curr]
curr += 1
# check if this object is distinct
if not self.isDistinct(obj, distinct):
if self.parent.opts.debug: print 'already worked on'
continue
# skip over this object if we don't have permission
if not (self.parent.force or obj.hasPermission()):
msg = "permission denied: not job owner"
if self.parent.color:
msg = terminal.TerminalColor('red').colorStr(msg)
print self.formatter.format(obj), msg
continue
# we passed all the tests, so add it to the list
newobjs.append(obj)
return newobjs,curr
def processResult(self, qresult):
"""Process and print all the objects returned from the query."""
# if we had no results, then do nothing
if not qresult or \
(self.opts.limit is not None and self.opts.limit < 0):
return
# make sure the query result doesn't cache objects after they are
# created, as we will only need them once.
if isinstance(qresult, Database.QueryResult):
qresult.cacheObjects = False
file = sys.stdout
# set the widths of our formatter so everything is neatly spaced
if not self.opts.noformat:
self.formatter.setWidths(qresult)
# print a header
if not self.opts.noheader:
print >>file, self.formatter.header()
# keep track of which objects are distinct
distinct = {}
# our current list of objects that have had the operation started,
# but the operation has not finished. Typically because the dispatcher
# is busy reading from it and would block the process.
queue = []
# keep track of how many objects we've worked on
workedon = 0
# time we last created an object (only useful when pausing between
# operations).
lastcreate = 0
# make sure threads is set to at least one
if self.parent.threads < 1:
self.parent.threads = 1
# setup a blocking flag
if not self.parent.yes or self.parent.threads <= 1:
self.blocking = True
else:
self.blocking = False
# keep track of our current index into the query result.
curr = 0
try:
# start operating on the objects
while True:
# if we are running in parallel mode (i.e. non-blocking) we
# want to have as many concurrent connections as possible.
# Thus make sure the queue size is always the size of
# self.threads, or at least as big as it can be.
newobjs = []
# check if we need to pause inbetween operations
if self.parent.pause > 0:
# how much time has passed since our last object create
elapsed = self.parent.pause - (time.time() - lastcreate)
# if we don't have any objects on the queue, and 'pause'
# seconds hasn't elapsed
if not queue and elapsed > 0:
time.sleep(elapsed)
elapsed = 0
# add another item to the queue
if elapsed <= 0 and len(queue) < self.parent.threads:
newobjs,curr = self.getNewObjs(1, curr,
qresult, distinct)
lastcreate = time.time()
else:
# get however many more we can take on now
newobjs,curr = self.getNewObjs(
self.parent.threads - len(queue),
curr, qresult, distinct)
# if we have nothing left, then we're all done!
if not (queue or newobjs) or \
(not queue and \
self.opts.limit not in [None, 0] and \
self.opts.limit <= workedon):
break
# start the new operations now. If we are in blocking
# mode, (self.yesToAll == False or self.threads <= 1),
# then the entire operation will be done here. If we are
# in parallel mode (i.e. non-blocking), then the connect
# will initialize here, raise a WouldBlock exception, and
# be put onto the queue.
for obj in newobjs:
try:
# work on this object
self.processObject(obj)
except Sockets.WouldBlock:
# if this would block, then add it to the queue
queue.append(obj)
except Sockets.SocketError, err:
# catch any other socket related error
obj.post(self, str(err), error=True)
obj.close()
except EngineClient.EngineClientError, err:
obj.post(self, str(err), error=True)
obj.close()
except:
class AliasHelpOld(TqHelp):
"""Help command for listing aliases that can be used in a query."""
usage = "alias"
description = """
Set/unset and view search clause aliases.
This feature is useful if you have a particular search clause or
desired format. Several aliases are set by default in the cli and
can be accessed by running the alias command with no arguments.
"""
examples = """
Syntax for setting and unsetting aliases is:
> tq alias -t TYPE -u USEDBY --add alias_name value_of_alias ...
> tq alias -t TYPE -u USEDBY --delete alias_name
The 'type' argument is used to identify whether the alias is for a command,
search clause, or a name (db field) alias. The 'usedby' argument is used
to identify the scope of the alias (i.e. it is only used by the jobs command,
or only in the slots, or it is used globally by all commands).
Examples:
add a search alias for the jobs command that will always print your
errored jobs
> tq alias -t where -u jobs --add myerrors "mine and error"
delete the where alias we just made
> tq alias -t where -u jobs --delete myerrors
add a command alias to list the nemo jobs in reverse priority order
> tq alias -t cmd --add nemoqueue "jobs -s pri,-spooled not errwait and title like nemo"
delete the command alias we just made
> tq alias -t cmd --delete nemoqueue
"""
options = [
CmdLineTool.BooleanOption("-a", "--add",
help="add/overwrite an alias"),
CmdLineTool.StringOption(
"-d", "--delete", help="delete/remove an alias"),
CmdLineTool.StringOption("-t",
"--type",
default="where",
help="the type of alias (cmd, args, "
"or where)"),
CmdLineTool.StringOption("-u",
"--usedby",
default="global",
help="the command or class this alias "
"will be used by, i.e. jobs, "
"machines, slots, etc. if the "
"alias type is a where. If not "
"provided then the alias will be "
"global to everything under a "
"given type."),
] + CmdLineTool.CmdLineTool.options
def getExamples(self):
return self.examples.lstrip('\n').rstrip()
def parseArgs(self, *args, **kwargs):
# do the actual parsing
result = super(AliasHelp, self).parseArgs(*args, **kwargs)
# make sure everything is okay
if self.opts.add:
if not self.args:
raise tq.TqError, "no alias provided"
elif len(self.args) == 1:
raise tq.TqError, "no value for alias provided"
# save the alias as a key, value pair
self.opts.add = (self.args[0], ' '.join(self.args[1:]))
if self.opts.add or self.opts.delete:
if self.opts.type not in ("cmd", "command", "args", "where"):
raise tq.TqError, "unknown alias type"
#if self.opts.type == "name" and \
# (self.opts.usedby not in ["global"] + \
# [t.baseClass.__name__ for t in FieldsHelp.tables]):
# raise tq.TqError, "unknown class type"
if self.opts.type == "where":
if self.opts.usedby != "global":
cmd2table = {}
for tbl, cmd in FieldsHelp.table2cmd.items():
cmd2table[cmd] = tbl.baseClass.__name__
cmd2table["tasks"] = "Task"
try:
self.opts.usedby = cmd2table[self.opts.usedby]
except KeyError:
raise tq.TqError, "cannot add/delete " \
"aliases for the '%s' command" % self.opts.usedby
elif self.opts.type in ("args", "where") and \
self.opts.usedby != "global" and \
self.opts.usedby not in self.parent.commands:
raise tq.TqError, "unknown command name"
# this is a hack so we don't have to change the code in
# do_alias, but we can call command types with 'cmd'
if self.opts.type == "cmd":
self.opts.type = "command"
def execute(self):
"""Display the help message for this topic."""
# should we add an alias?
if self.opts.add:
# make sure the alias name is valid
are = re.compile(r'^[a-zA-Z_0-9]+$')
name = self.opts.add[0]
if not are.match(name):
raise tq.TqError, \
"invalid alias name '%s' provided." % name
def makedicts(root, dictnames):
top = dictnames.pop(0)
if not root.has_key(top):
root[top] = {}
if dictnames:
makedicts(root[top], dictnames)
#mprefs = MisterD.UserPrefs()
mprefs = {}
oprefs = self.parent.userPrefs
prefs = None
atype = self.opts.type
used = self.opts.usedby
val = self.opts.add[1]
# command aliases don't have a used by because they are
# global by definition
if atype == 'command':
if val.split()[0] not in self.parent.commands:
raise tq.TqError, "'%s' cannot begin a " \
"command alias, it is not a tq command." % \
val.split()[0]
makedicts(oprefs.aliases, ["command", name])
oprefs.aliases.command[name] = val
prefs = oprefs
elif atype == "args":
if used == "global":
makedicts(oprefs.aliases, ["args", "global", name])
oprefs.aliases.args["global"][name] = val
else:
makedicts(oprefs.aliases, ["args", "cmds", used, name])
oprefs.aliases.args.cmds[used][name] = val
prefs = oprefs
elif used == "global":
makedicts(mprefs.aliases, ["where", "global", name])
mprefs.aliases.where["global"][name] = val
prefs = mprefs
else:
makedicts(mprefs.aliases, ["where", "table", used, name])
mprefs.aliases.where.table[used][name] = val
prefs = mprefs
try:
prefs.save()
except IOError, errObj:
sys.stderr.write('unable to save new alias.\n')
sys.stderr.write('%s\n' % str(errObj))
return
# should we delete an alias
if self.opts.delete:
#mprefs = MisterD.UserPrefs()
mprefs = {}
oprefs = self.parent.userPrefs
atype = self.opts.type
used = self.opts.usedby
name = self.opts.delete
prefs = None
try:
# command aliases do not have a usedby category
if atype == 'command':
del oprefs.aliases.command[name]
prefs = oprefs
elif atype == "args":
if used == "global":
del oprefs.aliases.args[used][name]
else:
del oprefs.aliases.args.cmds[used][name]
prefs = oprefs
elif used == "global":
del mprefs.aliases.where["global"][name]
prefs = mprefs
else:
del mprefs.aliases.where.table[used][name]
prefs = mprefs
except KeyError:
if EngineDB.EngineClientDB.getWhereAlias(name):
raise tq.TqError, "default aliases cannot be " \
"deleted, only overridden."
err = "alias '%s' not found under type='%s'" % \
(name, atype)
if atype == 'command':
err += " and used by='%s'" % used
raise tq.TqError, err
try:
prefs.save()
except IOError, errObj:
sys.stderr.write('unable to save new alias.\n')
sys.stderr.write('%s\n' % str(errObj))
return
class DBMigrateTool(CmdLineTool.BasicCmdLineTool):
description = """
This program is used to migrate jobs in a Tractor 1.x data directory
to the Tractor 2.x relational database.
"""
options = [
CmdLineTool.BooleanOption(
"--complete", dest="complete", help="migrate completed jobs"),
CmdLineTool.BooleanOption("--incomplete",
dest="incomplete",
help="migrate incomplete (not done) jobs"),
CmdLineTool.BooleanOption(
"--deleted", dest="deleted", help="migrate deleted jobs"),
CmdLineTool.BooleanOption(
"--all",
dest="all",
help="migrate all jobs (complete, incomplete, and deleted)"),
CmdLineTool.StringOption("--src-data-dir",
dest="dataDir",
help="path to tractor 1.0 data directory"),
CmdLineTool.StringOption("--dest-config-dir",
dest="configDir",
help="path to tractor 2.0 config directory"),
] + CmdLineTool.BasicCmdLineTool.options
def __init__(self, *args, **kwargs):
super(DBMigrateTool, self).__init__(*args, **kwargs)
self.config = None
def parseArgs(self, *args, **kwargs):
"""This method gets called before execute() to validate command line arguments."""
result = super(DBMigrateTool, self).parseArgs(*args, **kwargs)
# no additional args should be supplied on the command line once flags have been removed
if self.args:
raise CmdLineTool.HelpRequest, self.getHelpStr()
if self.opts.configDir:
if not os.path.exists(self.opts.configDir):
raise OptionParser.OptionParserError(
"Config dir %s does not exist" % self.opts.configDir)
if not self.opts.complete and not self.opts.incomplete and not self.opts.deleted and not self.opts.all:
raise OptionParser.OptionParserError(
"One of --complete, --incomplete, --deleted, or --all must be specified."
)
if not self.opts.dataDir:
raise OptionParser.OptionParserError(
"--src-data-dir must be specified.")
if not self.opts.configDir:
raise OptionParser.OptionParserError(
"--dest-config-dir must be specified.")
if self.opts.all:
self.opts.complete = True
self.opts.incomplete = True
self.opts.deleted = True
return result
def execute(self):
"""This method gets called automatically by CmdLineTool, and is the core logic of the program."""
# this enables the postgresql server to find the proper python
self.config = EngineConfig.EngineConfig(self.opts.configDir,
self.installDir())
self.sync()
def installDir(self):
"""Return the full path to this tractor installation."""
thisScriptPath = os.path.dirname(sys.argv[0])
installDir = os.path.join(thisScriptPath,
RELATIVE_PATH_TO_INSTALL_ROOT)
installDir = os.path.realpath(installDir)
return installDir
def isDebugMode(self):
"""Returns True if debug mode is turned on by way of config file or command line option."""
return self.opts.debug or (self.config and self.config.isDbDebug())
def dprint(self, msg, inverse=False):
if self.isDebugMode(
) and not inverse or not self.isDebugMode() and inverse:
sys.stderr.write(msg)
sys.stderr.write("\n")
def log(self, msg):
sys.stderr.write(msg)
sys.stderr.write("\n")
def runCommand(self, argv, input=None, errIsOut=False, **kw):
"""Run the specified command, returning the return code, stdout, and stderr."""
self.dprint("running %s" % argv)
stdin = subprocess.PIPE if input is not None else None
stderr = subprocess.STDOUT if errIsOut else subprocess.PIPE
proc = subprocess.Popen(argv,
stdin=stdin,
stdout=subprocess.PIPE,
stderr=stderr,
**kw)
# NOTE: http://docs.python.org/2/library/subprocess.html warns that input to Popen.communicate() shouldn't be "large"
out, err = proc.communicate(input=input)
rcode = proc.wait()
self.dprint(out)
if not errIsOut:
self.dprint(err)
return rcode, out, err
def sync(self):
self.log("synchronizing with filesystem")
def jobDirSort(a, b):
jidA = a.split("/")[-1]
dateA = jidA[:8]
jidB = b.split("/")[-1]
dateB = jidB[:8]
if dateA != dateB:
return -cmp(dateA, dateB)
else:
jidA = jidA[8:]
jidA = int(jidA) if jidA.isdigit() else 0
jidB = jidB[8:]
jidB = int(jidB) if jidB.isdigit() else 0
return -cmp(jidA, jidB)
# establish connection with postgres database
db = EngineDB.EngineDB(dbhost=self.config.dbHostname(),
db=self.config.dbDatabaseName(),
user=MIGRATE_USER,
password=MIGRATE_PASSWORD)
try:
db.open()
except rpg.sql.SQLError, err:
raise DBMigrateToolError, str(err)
# first walk the job data directory so that newer jobs can be processed first
jobDirs = []
jobsDir = os.path.join(self.opts.dataDir, "jobs")
self.dprint("jobsDir = %s" % jobsDir)
joroots = []
for root, dirs, files in os.walk(jobsDir):
if FileSystemDB.JobDB.JOB_INFO_FILENAME in files:
# it's a job database
jobDirs.append(root)
# process each job directory in reverse date + jobid order (newest jobs first)
jobDirs.sort(jobDirSort)
for jobDir in jobDirs:
self.dprint("process job in %s" % jobDir)
# extract job id from job directory
parts = jobDir.split("/")
jid = parts[-1]
if not jid.isdigit():
self.log("Job directory does not end in a job id: %s" % jobDir)
continue # for jobDir
jid = int(jid)
# set up a JobDB to analyze job
jobDB = FileSystemDB.JobDB(jobDir, readFiles=False)
# don't process deleted jobs if deemed so
isDeleted = jobDB.isDeleted()
if isDeleted:
if not self.opts.deleted:
self.dprint("Skipping deleted job %d." % jid)
continue # for jobDir
else:
# don't process non-deleted jobs if complete/incomplete flags haven't been set
if not self.opts.complete and not self.opts.incomplete:
self.dprint("Skipping non-deleted job %d." % jid)
continue # for jobDir
# do not process job if it's already in database
jobInDB = db._execute("SELECT jid FROM oldjob WHERE oldjid=%d" %
jid)
rows = db.cursor.fetchall()
if len(rows) > 0:
self.log("Skipping migrated job %d." % jid)
continue # for jobDir
# read job info
try:
jobDB.readJobInfo()
except FileSystemDB.JobDBError, err:
self.log(str(err))
continue # for jobDir
# filter out complete/incomplete jobs
isComplete = jobDB.isComplete()
if not isDeleted:
if isComplete:
if not self.opts.complete:
self.dprint("Skipping complete job %d." % jid)
continue # for jobDir
else:
if not self.opts.incomplete:
self.dprint("Skipping incomplete job %d." % jid)
continue # for jobDir
# read remaining info files
jobDB.readTaskInfo()
jobDB.readCmdList()
jobDB.readActivityLog()
jobDB.countStates()
# deleted jobs need to go in an archive partition
if isDeleted:
db._execute("SELECT TractorPartitionCreate('%s') as suffix" %
jobDB.job.spooltime.strftime("%Y-%m-%d"))
result = db.cursor.fetchall()
if not len(result):
self.log(
"Skipping deleted job %d with spooltime %s due to problems creating archive partition."
% (jid, str(jobDB.job.spooltime)))
continue # for jobDir
suffix = result[0]["suffix"]
else:
suffix = ""
# bulk insert records on a per-table basis using postgresql's COPY FROM
db._execute("begin")
db._execute("SELECT TractorNewJid() as nextjid")
result = db.cursor.fetchall()
nextjid = result[0]["nextjid"]
# set job id of job as well as tasks, commands, and invocations
jobDB.setJid(nextjid)
s = StringIO.StringIO(PGFormat.formatObjsForCOPY([jobDB.job]))
db.cursor.copy_from(s, "job" + suffix)
s = StringIO.StringIO(PGFormat.formatObjsForCOPY(jobDB.tasks))
db.cursor.copy_from(s, "task" + suffix)
s = StringIO.StringIO(PGFormat.formatObjsForCOPY(jobDB.commands))
db.cursor.copy_from(s, "command" + suffix)
s = StringIO.StringIO(PGFormat.formatObjsForCOPY(
jobDB.invocations))
db.cursor.copy_from(s, "invocation" + suffix)
# add entry to oldjob table to show it has been migrated
db._execute("INSERT INTO oldjob VALUES (%d, %d)" % (nextjid, jid))
db._execute("end")
class DBCmdLineTool(CmdLineTool.MultiCmdLineTool):
"""Generalized object for querying any table from a database from
the command line.
@cvar tableByCmd: mapping of command name to the Table object that
should be used when as an input paramater to L{TableQueryCmd}.
This mapping is used as a backup if an entry is not found in
the 'commands' mapping.
@cvar sortByCmd: mapping of command name to a list of fields the
results should be sorted by.
"""
options = [
CmdLineTool.BooleanOption("--nocolor",
help="do not display any output in color."),
CmdLineTool.BooleanOption("--zeros",
help="some fields do not display a value "
"if it is equal to zero, this will "
"force zeros to be printed no matter "
"what."),
RawDataOption(),
TimeFormatOption(),
CmdLineTool.BooleanOption("-d", "--debug", help="print debug info."),
] + CmdLineTool.MultiCmdLineTool.options
commands = {
"fields": FieldsHelp,
}
tableByCmd = {}
sortByCmd = {}
def __init__(self,
database,
formatterClass,
tableQueryCmd=TableQueryCmd,
**kwargs):
"""
@param database: the L{Database.Database} object that will be queried.
@type database: L{Database.Database} object
@param formatterClass: the L{DBFormatter.DBFormatter} subclass that
will be used when displaying the result.
@type formatterClass: L{DBFormatter.DBFormatter} class
"""
self.database = database
self.formatterClass = formatterClass
self.tableQueryCmd = tableQueryCmd
super(DBCmdLineTool, self).__init__(**kwargs)
def getNewCommand(self, cmd, *args, **kwargs):
"""Overloaded from the super so we can ensure that our TableQueryCmd
objects are properly initialized."""
cmdkwargs = kwargs.copy()
if cmd == "fields":
cmdkwargs['tables'] = self.database.Tables
try:
return super(DBCmdLineTool,
self).getNewCommand(cmd, *args, **cmdkwargs)
except CmdLineTool.UnknownCommand:
# check for a default list to sortby
sortby = self.sortByCmd.get(cmd)
kwargs = kwargs.copy()
kwargs['defaultSort'] = sortby
try:
tableobj = self.tableByCmd[cmd]
except KeyError:
raise CmdLineTool.UnknownCommand(cmd)
# assume we need to make a TableQueryCmd object
cmdobj = self.tableQueryCmd(self.database, tableobj,
self.formatterClass, *args, **kwargs)
if not cmdobj.usage:
cmdobj.usage = cmd
return cmdobj
class DBControlTool(CmdLineTool.BasicCmdLineTool):
progname = "tractor-dbctl"
description = """
This program is used to manage starting and stopping of the postgresql
database server that is used with the tractor engine.
"""
options = [
CmdLineTool.BooleanOption(
"--start-for-engine",
dest="startForEngine",
help=
"start the postgresql server and automatically build/upgrade schema as configured in db.config (used by engine)"
),
CmdLineTool.BooleanOption(
"--stop-for-engine",
dest="stopForEngine",
help=
"stop the postgresql server as configured in db.config (used by engine)"
),
CmdLineTool.BooleanOption(
"--start", dest="start", help="start the postgresql server"),
CmdLineTool.BooleanOption(
"--stop", dest="stop", help="stop the postgresql server"),
CmdLineTool.BooleanOption(
"--status",
dest="status",
help="check the status of the postgresql server"),
CmdLineTool.BooleanOption(
"--init",
dest="init",
help=
"initialize the postgresql data directory; server must not be running"
),
CmdLineTool.BooleanOption(
"--build",
dest="build",
help="build the tractor database; server must be running"),
CmdLineTool.BooleanOption(
"--destroy",
dest="destroy",
help=
"remove the postgres database directory; server must not be running"
),
CmdLineTool.BooleanOption(
"--upgrade", dest="upgrade", help="update the database schema"),
CmdLineTool.BooleanOption("--check-upgrade",
dest="checkUpgrade",
help="check whether upgrades are required"),
CmdLineTool.BooleanOption(
"--no-auto-upgrade",
dest="noAutoUpgrade",
help="prevent automatic upgrading of the database schema"),
CmdLineTool.BooleanOption(
"--purge-jobs", dest="purgeJobs", help="remove all jobs"),
CmdLineTool.StringOption(
"--purge-archive-to-year-month",
dest="purgeArchiveToYearMonth",
help=
"remove archived jobs up to and including specified year-month in YY-MM format"
),
CmdLineTool.BooleanOption(
"--vacuum",
dest="vacuum",
help="rebuild tables of non-deleted jobs to save space"),
CmdLineTool.BooleanOption(
"--reset-job-counter",
dest="resetJobCounter",
help=
"reset the job id counter so that job ids (jids) start at 1; can only be used with --purge-jobs"
),
CmdLineTool.BooleanOption("--show-params",
dest="showParams",
help="show stored database paramaters"),
CmdLineTool.BooleanOption(
"--update-params",
dest="updateParams",
help="update database with paramaters stored in config files"),
CmdLineTool.BooleanOption(
"--tail-log",
dest="tailLog",
help="tail and follow the last lines of postgres message log"),
CmdLineTool.BooleanOption(
"--logs-usage",
dest="logsUsage",
help="report disk space used by postgresql message log files"),
CmdLineTool.BooleanOption("--purge-logs",
dest="purgeLogs",
help="remove postgresql message log files"),
CmdLineTool.StringOption(
"--backup", dest="backup", help="write backup to specified file"),
CmdLineTool.StringOption(
"--restore",
dest="restore",
help="restore database from specified backup file"),
CmdLineTool.StringOption("--config-dir",
dest="configDir",
help="path to tractor config directory"),
] + CmdLineTool.BasicCmdLineTool.options
def __init__(self, *args, **kwargs):
super(DBControlTool, self).__init__(*args, **kwargs)
self.config = None
if osutil.getlocalos() == "Linux":
# preload libpq.so file so that later imports of psycopg2 will get the proper library
# since rmanpy as compiled only looks in the install dir's lib/, but libpq is in lib/psql/lib
ctypes.cdll.LoadLibrary(
os.path.join(self.installDir(), "lib", "psql", "lib",
"libpq.so.5"))
# this doesn't appear to be necessary on OSX since the stock install comes with a valid libpq.so
def parseArgs(self, *args, **kwargs):
"""This method gets called before execute() to validate command line arguments."""
result = super(DBControlTool, self).parseArgs(*args, **kwargs)
# no additional args should be supplied on the command line once flags have been removed
if self.args:
raise CmdLineTool.HelpRequest, self.getHelpStr()
if self.opts.configDir:
if not os.path.exists(self.opts.configDir):
raise OptionParser.OptionParserError(
"Config dir %s does not exist" % self.opts.configDir)
if self.opts.resetJobCounter and not self.opts.purgeJobs:
raise OptionParser.OptionParserError(
"--reset-job-counter can only be used with --purge-jobs.")
if not self.opts.configDir:
raise OptionParser.OptionParserError(
"--config-dir must be specified.")
return result
def execute(self):
"""This method gets called automatically by CmdLineTool, and is the core logic of the program."""
self.config = EngineConfig.EngineConfig(self.configDir(),
self.installDir())
# check that process is owned by proper user; files will be owned by that user
processOwner = osutil.getusername()
dataDirOwner = osutil.ownerForPath(self.config.pgDataDir())
configuredOwner = self.config.tractorEngineOwner()
if configuredOwner and processOwner != configuredOwner:
raise DBControlToolError, "tractor-dbctl is configured to be run by %s; the owner of this process is %s." \
% (configuredOwner, processOwner)
# check that engine owner owns the data directory
if dataDirOwner != processOwner:
raise DBControlToolError, "The database data dir %s is owned by %s; the owner of this process is %s." \
% (self.config.pgDataDir(), dataDirOwner, processOwner)
if self.opts.status:
self.status()
elif self.opts.start:
self.start()
elif self.opts.stop:
self.stop()
elif self.opts.startForEngine:
self.startForEngine()
elif self.opts.stopForEngine:
self.stopForEngine()
elif self.opts.init:
self.init()
elif self.opts.build:
self.build()
elif self.opts.destroy:
self.destroy()
elif self.opts.checkUpgrade:
self.checkUpgrade()
elif self.opts.upgrade:
self.upgrade()
elif self.opts.purgeJobs:
self.purgeJobs()
elif self.opts.purgeArchiveToYearMonth:
self.purgeArchiveToYearMonth()
elif self.opts.vacuum:
self.vacuum()
elif self.opts.backup:
self.backup()
elif self.opts.restore:
self.restore()
elif self.opts.purgeLogs:
self.purgeLogs()
elif self.opts.logsUsage:
self.tailLog()
elif self.opts.tailLog:
self.tailLog()
elif self.opts.showParams:
self.showParams()
elif self.opts.updateParams:
self.updateParams()
else:
raise OptionParser.OptionParserError(
"No operations were specified. Use --help for options.")
def isDebugMode(self):
"""Returns True if debug mode is turned on by way of config file or command line option."""
return self.opts.debug or (self.config and self.config.isDbDebug())
def installDir(self):
"""Return the full path to this tractor installation."""
thisScriptPath = os.path.dirname(sys.argv[0])
installDir = os.path.join(thisScriptPath,
RELATIVE_PATH_TO_INSTALL_ROOT)
installDir = os.path.realpath(installDir)
return installDir
def configDir(self):
"""Return the full path to the config dir."""
return os.path.realpath(self.opts.configDir)
def dprint(self, msg, inverse=False):
if self.isDebugMode(
) and not inverse or not self.isDebugMode() and inverse:
sys.stderr.write(msg)
sys.stderr.write("\n")
def log(self, msg):
sys.stderr.write(msg)
sys.stderr.write("\n")
def runCommand(self, argv, input=None, errIsOut=False, **kw):
"""Run the specified command, returning the return code, stdout, and stderr."""
self.dprint("Running %s" % argv)
stdin = subprocess.PIPE if input is not None else None
stderr = subprocess.STDOUT if errIsOut else subprocess.PIPE
proc = subprocess.Popen(argv,
stdin=stdin,
stdout=subprocess.PIPE,
stderr=stderr,
**kw)
# NOTE: http://docs.python.org/2/library/subprocess.html warns that input to Popen.communicate() shouldn't be "large"
out, err = proc.communicate(input=input)
rcode = proc.wait()
self.dprint(out)
if not errIsOut:
self.dprint(err)
return rcode, out, err
def start(self):
"""Start the postgresql server."""
if self.isPostgreSQLRunning():
raise DBControlToolError, "A postgresql server is already running on the data directory %s." % self.config.pgDataDir(
)
self.startPostgreSQL()
def stop(self):
"""Stop the postgresql server."""
if not self.isPostgreSQLRunning():
raise DBControlToolError, "The postgresql server is NOT running on the data directory %s." % self.config.pgDataDir(
)
self.stopPostgreSQL()
def status(self):
"""Check the status of the postgresql server."""
if self.isPostgreSQLRunning():
self.log(
"A postgresql server is running on the data directory %s." %
self.config.pgDataDir())
if self.isPostgreSQLReachable():
self.log(
"The postgresql server is reachable through port %d." %
self.config.dbPort())
if self.databaseExists():
self.log("The database %s exists." %
self.config.dbDatabaseName())
else:
self.log("The database %s does NOT exist." %
self.config.dbDatabaseName())
else:
self.log(
"The postgresql server is NOT reachable through port %d." %
self.config.dbPort())
else:
self.log(
"A postgresql server is NOT running on the data directory %s."
% self.config.pgDataDir())
def init(self):
"""Initialize the postgresql data directory."""
# make sure postgresql server isn't already running
if self.isPostgreSQLRunning():
raise DBControlToolError, "The postgresql server is already running. It must first be stopped with --stop."
# make sure there isn't an existing postgres database
if os.path.exists(self.config.pgDataDir()) and len(
os.listdir(self.config.pgDataDir())) > 0:
raise DBControlToolError, "%s is not an empty directory." % self.config.pgDataDir(
)
self.initPostgreSQL()
def build(self):
if self.databaseExists():
raise DBControlToolError, "Database already exists."
self.buildDB()
def startForEngine(self):
"""Conditionally start the postgresql server, performing any database initialization as required."""
# initialize db if required and configured to do so
if not os.path.exists(self.config.pgDataDir()) or not os.path.exists(
os.path.join(self.config.pgDataDir(), "postgresql.conf")):
self.dprint(
"Considering db initialization because one of the following do not exist:\n%s\n%s"
% (self.config.pgDataDir(),
os.path.join(self.config.pgDataDir(), "postgresql.conf")))
if self.config.doDbInit():
self.initPostgreSQL()
else:
raise DBControlToolError, "The postgresql data dir %s was not found and %s is not configured to create a new one." % (
self.config.pgDataDir(), self.progname)
# start postgresql server if required and configured to do so
if self.isPostgreSQLRunning():
if self.config.doDbUseExisting():
self.dprint(
"A postgresql server is already running. The system has been configured to use it."
)
else:
raise DBControlToolError, "A postgresql server is already running. Set %s to True in %s to use an existing postgresql server." \
% (EngineConfig.DB_USE_EXISTING, self.config.dbSiteConfigFilename())
else:
if self.config.doDbStartup():
self.startPostgreSQL()
time.sleep(POST_START_SLEEP)
if not self.isPostgreSQLRunning(
maxAttempts=MAX_DB_CONNECT_ATTEMPTS):
raise DBControlToolError, "Failed to start a postgresql server on data directory %s. A different server may be already running; to check, try 'ps -elf | grep postgres'. Or another service is using port %d; to check, try 'sudo fuser %d/tcp'.\%s may have more info." \
% (self.config.pgDataDir(), self.config.dbPort(), self.config.dbPort(), self.config.pgLogFilename())
else:
raise DBControlToolError, "A postgresql server was not started because %s is set to False in %s to prevent the automatic starting of a postgresql server. Change this setting to True, or manually ensure your custom postgresql server is running and set %s to True." \
% (EngineConfig.DB_STARTUP, self.config.dbSiteConfigFilename(), EngineConfig.DB_USE_EXISTING)
# test if reachable
if not self.isPostgreSQLReachable(maxAttempts=MAX_DB_CONNECT_ATTEMPTS):
raise DBControlToolError, "Unable to connect to postgresql server on port %d. Check %s for more info." % (
self.config.dbPort(), self.config.pgLogFilename())
# build database
if not self.databaseExists():
self.buildDB()
else:
self.dprint(
"Database %s exists. Building database is not required." %
self.config.dbDatabaseName())
# ensure that languages are loaded
if not self.isLanguageReady("plpython2u"):
raise DBControlToolError, "Unable to verify that plpython2u has been loaded."
if not self.isLanguageReady("plpgsql"):
raise DBControlToolError, "Unable to verify that plpgsql has been loaded."
# see if an upgrade is required
if self.config.doDbAutoUpgrade() and not self.opts.noAutoUpgrade:
upgrades = self.getUpgrades()
if upgrades:
self.upgradeDB(upgrades)
def stopForEngine(self):
"""Stop the postgresql server."""
if self.config.doDbShutdown() and self.isPostgreSQLRunning():
self.stopPostgreSQL()
def destroy(self):
"""This method removes the existing a new postgresql database directory.
There must be no existing postgresql server running."""
# make sure postgresql server is not running
if self.isPostgreSQLRunning():
raise DBControlToolError, "The postgresql server is already running. It must be shutdown with --stop before using --destroy."
# make sure there isn't an existing postgres database
if not os.path.exists(self.config.pgDataDir()):
raise DBControlToolError, "%s does not exist." % self.config.pgDataDir(
)
# remove the contents of the data directory
self.log("Removing contents of data directory %s." %
self.config.pgDataDir())
for filename in os.listdir(self.config.pgDataDir()):
fullFilename = os.path.join(self.config.pgDataDir(), filename)
try:
if os.path.isfile(fullFilename):
os.remove(fullFilename)
else:
shutil.rmtree(
os.path.join(self.config.pgDataDir(), fullFilename))
except (IOError, OSError), err:
self.log("Unable to remove %s: %s" % (filename, str(err)))
class TractorCLT(CmdLineTool.MultiCmdLineTool):
"""The main command object for the cli just includes all the sub-command objects."""
version = "2.0"
DEFAULT_THREADS = 10
DEFAULT_TIMEOUT = 30
DEFAULT_TIMEFMT = os.environ.get("TRACTOR_TIMEFMT")
# setup all the commands in this app
commands = {
"jobs" : ListCmds.JobsCmd,
"tasks" : ListCmds.TasksCmd,
"commands" : ListCmds.CommandsCmd,
"cmds" : ListCmds.CommandsCmd,
"invocations" : ListCmds.InvocationsCmd,
"invos" : ListCmds.InvocationsCmd,
"blades" : ListCmds.BladesCmd,
"params" : ListCmds.ParamsCmd,
"chcrews" : JobCmds.ChangeCrewsCmd,
"chpri" : JobCmds.ChangePriorityCmd,
"delay" : JobCmds.DelayJobCmd,
"delete" : JobCmds.DeleteJobCmd,
"jattr" : JobCmds.ChangeJobAttributeCmd,
"pause" : JobCmds.PauseJobCmd,
"interrupt" : JobCmds.InterruptJobCmd,
"restart" : JobCmds.RestartJobCmd,
"retryallerrs" : JobCmds.RetryAllErrorsCmd,
"skipallerrs" : JobCmds.SkipAllErrorsCmd,
"undelay" : JobCmds.UndelayJobCmd,
"undelete" : JobCmds.UndeleteJobCmd,
"unpause" : JobCmds.UnpauseJobCmd,
"retry" : TaskCmds.RetryTaskCmd,
"resume" : TaskCmds.ResumeTaskCmd,
"skip" : TaskCmds.SkipTaskCmd,
"log" : TaskCmds.PrintLogCmd,
"chkeys" : CommandCmds.ChangeKeysCmd,
"cattr" : CommandCmds.ChangeCommandAttributeCmd,
"nimby" : BladeCmds.NimbyBladeCmd,
"unnimby" : BladeCmds.UnnimbyBladeCmd,
"trace" : BladeCmds.TraceBladeCmd,
"help" : HelpCmds.MainHelp,
None : HelpCmds.MainHelp,
"alias" : HelpCmds.AliasHelp,
}
# command only viewable by rpg folk
adminCommands = {
}
# command aliases
aliases = {
"aliases" : "alias",
"machs" : "machines",
"queue" : "jobs -s pri,-spooled not errwait and",
}
# set the options that the main program will have
options = [
CmdLineTool.BooleanOption ("-y", "--yes",
help="answer 'yes' to all questions"),
CmdLineTool.BooleanOption ("-a", "--archives",
help="search archive tables"),
CmdLineTool.BooleanOption ("--force",
help="force all operations. This will "
"allow any operation to be performed "
"on a task or job, even if you are "
"not the owner."),
CmdLineTool.BooleanOption ("--nocolor", dest="color", default=True,
help="do not display any output in color."),
CmdLineTool.BooleanOption ("--zeros",
help="some fields do not display a value "
"if it is equal to zero, this will "
"force zeros to be printed no matter "
"what."),
CmdLineTool.BooleanOption ("--nothreads", dest="threads", const=1,
help="do not use threads, when performing "
"task/job operations, threads are "
"only used when --yes is applied."),
CmdLineTool.IntOption ("--threads", default=DEFAULT_THREADS,
help="maximum number of concurrent "
"threads, default=%d, setting this "
"to 1 or lower is the same as "
"--nothreads." % DEFAULT_THREADS),
CmdLineTool.SecondsOption ("-p", "--pause",
help="pause p seconds inbetween each "
"action (e.g. retry, skip, etc.) "
"This can be a floating point value."),
CmdLineTool.SecondsOption ("-t", "--timeout", default=DEFAULT_TIMEOUT,
help="timeout in seconds before canceling "
"an operation, default=%d seconds." %
DEFAULT_TIMEOUT),
CmdLineTool.BooleanOption ("--login", default=False,
help="explicitly log in to engine rather than use existing session id "
"query"),
CmdLineTool.BooleanOption ("--logout", default=False,
help="automatically log out from engine after performing "
"query"),
DBCmdLineTool.RawDataOption(),
DBCmdLineTool.TimeFormatOption(default=DEFAULT_TIMEFMT),
DBCmdLineTool.FullTimeOption(),
CmdLineTool.StringOption ("--engine",
help="<hostname>:<port> of engine"),
CmdLineTool.StringOption ("-u", "--user", help="engine username"),
CmdLineTool.StringOption ("--password", "--pw", help="engine user password"),
CmdLineTool.StringOption ("--password-file", "--pwfile", help="path to json file containing engine username and password"),
CmdLineTool.BooleanOption ("--traceback", help="show full traceback on exceptions (developer option)"),
CmdLineTool.BooleanOption ("-d", "--debug", help="display debug info"),
CmdLineTool.VersionOption ("-v", "--version", help="display version info"),
] + CmdLineTool.MultiCmdLineTool.options
def __init__(self, *args, **kwds):
self.__userPrefs = None
super(TractorCLT, self).__init__(*args, **kwds)
def getUserPrefs(self):
"""Return a pointer to the user prefs object."""
if self.__userPrefs is None:
self.__userPrefs = {} # cli.UserPrefs()
return self.__userPrefs
userPrefs = property(fget=getUserPrefs)
def parseArgs(self, *args, **kwds):
super(TractorCLT, self).parseArgs(*args, **kwds)
self.opts.nocolor = not self.opts.color
# set all the options as members of ourself
for var in ("yes", "force", "color", "zeros", "threads", "pause",
"timeout"):
setattr(self, var, getattr(self.opts, var))
global ShowFullTraceback
ShowFullTraceback = self.opts.traceback
def getHelpStr(self):
"""Print a custom help message."""
# get the help command
helpcmd = self.getCommand("help")
# run the main help
helpcmd.run()
def getNewCommand(self, cmd, *args, **kwds):
"""Get a L{CmdLineTool} instance that should be used for the
provided command.
@param cmd: name of the command being referenced.
@type cmd: string
@return: L{CmdLineTool} that will be used
@rtype: L{CmdLineTool} instance
"""
kwds["raiseArgErrors"] = True
try:
return self.commands[cmd](*args, **kwds)
except KeyError:
raise CmdLineTool.UnknownCommand(cmd)
def getCommandToRun(self):
try:
return super(TractorCLT, self).getCommandToRun()
except CmdLineTool.UnknownCommand, err:
# check for the fields command which we will redirect to the
# main help command
if self.args and self.args[0] == "fields":
command = "help"
cmdobj = self.getCommand(command)
args = ["fields"] + self.args[1:]
return (command, cmdobj, args)
# check the user prefs for command aliases
try:
alias = self.userPrefs.aliases.command[self.args[0]]
except (AttributeError, KeyError):
# check the global command alias list
alias = self.aliases.get(self.args[0])
if alias:
# check for additional arguments in the alias
import rpg.stringutil as stringutil
aliasArgs = stringutil.quotedSplit(alias, removeQuotes=True)
command = aliasArgs[0]
cmdobj = self.getCommand(command)
args = aliasArgs[1:] + self.args[1:]
return (command, cmdobj, args)
raise CmdLineTool.UnknownCommand, err
class TraceBladeCmd(BladeOperationCmd):
usage = "trace"
description = """
Run a tracer on one or more blades. Running a tracer on a blade will
display output regarding its decision making process to run a task.
"""
examples = """
The typical syntax to trace blade is:
> tq trace WHERE_STRING
Examples:
trace a blade named boxy
> tq trace name=boxy
"""
options = [
CmdLineTool.BooleanOption("-u",
"--unique",
help="print the name of the blade "
"on each line to "
"distinguish output from blades. "
"Useful when grepping though more "
"than one trace."),
] + BladeOperationCmd.options
def runQuery(self):
"""Overloaded so the header can be suppressed if only one trace
is being printed."""
# check if we need to contact the db
if self.objects:
result = self.objects
else:
result = self.db.getBlades(members=self.members,
where=self.where,
limit=self.opts.limit,
orderby=self.opts.sortby,
objtype=tq.OperationRow)
if not result:
raise tq.TqError, "no blades found"
# do not display the header
if len(result) == 1:
self.opts.noheader = True
return result
def processObject(self, obj):
"""Operate on the provided blade object."""
# print a header if that is desired
if not self.opts.noheader:
hdr = self.formatter.format(obj)
if self.parent.color:
hdr = terminal.TerminalColor('yellow').colorStr(hdr)
print hdr
trace = query.trace(obj).get((obj.name, obj.ipaddr), "")
# make each line unique if that is desired
if self.opts.unique:
pre = '[%s] ' % obj.name
trace = pre + trace.replace('\n', '\n' + pre)
print trace
obj.post(self, "End of trace.")