class AliasHelpOld(TqHelp):
"""Help command for listing aliases that can be used in a query."""
usage = "alias"
description = """
Set/unset and view search clause aliases.
This feature is useful if you have a particular search clause or
desired format. Several aliases are set by default in the cli and
can be accessed by running the alias command with no arguments.
"""
examples = """
Syntax for setting and unsetting aliases is:
> tq alias -t TYPE -u USEDBY --add alias_name value_of_alias ...
> tq alias -t TYPE -u USEDBY --delete alias_name
The 'type' argument is used to identify whether the alias is for a command,
search clause, or a name (db field) alias. The 'usedby' argument is used
to identify the scope of the alias (i.e. it is only used by the jobs command,
or only in the slots, or it is used globally by all commands).
Examples:
add a search alias for the jobs command that will always print your
errored jobs
> tq alias -t where -u jobs --add myerrors "mine and error"
delete the where alias we just made
> tq alias -t where -u jobs --delete myerrors
add a command alias to list the nemo jobs in reverse priority order
> tq alias -t cmd --add nemoqueue "jobs -s pri,-spooled not errwait and title like nemo"
delete the command alias we just made
> tq alias -t cmd --delete nemoqueue
"""
options = [
CmdLineTool.BooleanOption("-a", "--add",
help="add/overwrite an alias"),
CmdLineTool.StringOption(
"-d", "--delete", help="delete/remove an alias"),
CmdLineTool.StringOption("-t",
"--type",
default="where",
help="the type of alias (cmd, args, "
"or where)"),
CmdLineTool.StringOption("-u",
"--usedby",
default="global",
help="the command or class this alias "
"will be used by, i.e. jobs, "
"machines, slots, etc. if the "
"alias type is a where. If not "
"provided then the alias will be "
"global to everything under a "
"given type."),
] + CmdLineTool.CmdLineTool.options
def getExamples(self):
return self.examples.lstrip('\n').rstrip()
def parseArgs(self, *args, **kwargs):
# do the actual parsing
result = super(AliasHelp, self).parseArgs(*args, **kwargs)
# make sure everything is okay
if self.opts.add:
if not self.args:
raise tq.TqError, "no alias provided"
elif len(self.args) == 1:
raise tq.TqError, "no value for alias provided"
# save the alias as a key, value pair
self.opts.add = (self.args[0], ' '.join(self.args[1:]))
if self.opts.add or self.opts.delete:
if self.opts.type not in ("cmd", "command", "args", "where"):
raise tq.TqError, "unknown alias type"
#if self.opts.type == "name" and \
# (self.opts.usedby not in ["global"] + \
# [t.baseClass.__name__ for t in FieldsHelp.tables]):
# raise tq.TqError, "unknown class type"
if self.opts.type == "where":
if self.opts.usedby != "global":
cmd2table = {}
for tbl, cmd in FieldsHelp.table2cmd.items():
cmd2table[cmd] = tbl.baseClass.__name__
cmd2table["tasks"] = "Task"
try:
self.opts.usedby = cmd2table[self.opts.usedby]
except KeyError:
raise tq.TqError, "cannot add/delete " \
"aliases for the '%s' command" % self.opts.usedby
elif self.opts.type in ("args", "where") and \
self.opts.usedby != "global" and \
self.opts.usedby not in self.parent.commands:
raise tq.TqError, "unknown command name"
# this is a hack so we don't have to change the code in
# do_alias, but we can call command types with 'cmd'
if self.opts.type == "cmd":
self.opts.type = "command"
def execute(self):
"""Display the help message for this topic."""
# should we add an alias?
if self.opts.add:
# make sure the alias name is valid
are = re.compile(r'^[a-zA-Z_0-9]+$')
name = self.opts.add[0]
if not are.match(name):
raise tq.TqError, \
"invalid alias name '%s' provided." % name
def makedicts(root, dictnames):
top = dictnames.pop(0)
if not root.has_key(top):
root[top] = {}
if dictnames:
makedicts(root[top], dictnames)
#mprefs = MisterD.UserPrefs()
mprefs = {}
oprefs = self.parent.userPrefs
prefs = None
atype = self.opts.type
used = self.opts.usedby
val = self.opts.add[1]
# command aliases don't have a used by because they are
# global by definition
if atype == 'command':
if val.split()[0] not in self.parent.commands:
raise tq.TqError, "'%s' cannot begin a " \
"command alias, it is not a tq command." % \
val.split()[0]
makedicts(oprefs.aliases, ["command", name])
oprefs.aliases.command[name] = val
prefs = oprefs
elif atype == "args":
if used == "global":
makedicts(oprefs.aliases, ["args", "global", name])
oprefs.aliases.args["global"][name] = val
else:
makedicts(oprefs.aliases, ["args", "cmds", used, name])
oprefs.aliases.args.cmds[used][name] = val
prefs = oprefs
elif used == "global":
makedicts(mprefs.aliases, ["where", "global", name])
mprefs.aliases.where["global"][name] = val
prefs = mprefs
else:
makedicts(mprefs.aliases, ["where", "table", used, name])
mprefs.aliases.where.table[used][name] = val
prefs = mprefs
try:
prefs.save()
except IOError, errObj:
sys.stderr.write('unable to save new alias.\n')
sys.stderr.write('%s\n' % str(errObj))
return
# should we delete an alias
if self.opts.delete:
#mprefs = MisterD.UserPrefs()
mprefs = {}
oprefs = self.parent.userPrefs
atype = self.opts.type
used = self.opts.usedby
name = self.opts.delete
prefs = None
try:
# command aliases do not have a usedby category
if atype == 'command':
del oprefs.aliases.command[name]
prefs = oprefs
elif atype == "args":
if used == "global":
del oprefs.aliases.args[used][name]
else:
del oprefs.aliases.args.cmds[used][name]
prefs = oprefs
elif used == "global":
del mprefs.aliases.where["global"][name]
prefs = mprefs
else:
del mprefs.aliases.where.table[used][name]
prefs = mprefs
except KeyError:
if EngineDB.EngineClientDB.getWhereAlias(name):
raise tq.TqError, "default aliases cannot be " \
"deleted, only overridden."
err = "alias '%s' not found under type='%s'" % \
(name, atype)
if atype == 'command':
err += " and used by='%s'" % used
raise tq.TqError, err
try:
prefs.save()
except IOError, errObj:
sys.stderr.write('unable to save new alias.\n')
sys.stderr.write('%s\n' % str(errObj))
return
class DBMigrateTool(CmdLineTool.BasicCmdLineTool):
description = """
This program is used to migrate jobs in a Tractor 1.x data directory
to the Tractor 2.x relational database.
"""
options = [
CmdLineTool.BooleanOption(
"--complete", dest="complete", help="migrate completed jobs"),
CmdLineTool.BooleanOption("--incomplete",
dest="incomplete",
help="migrate incomplete (not done) jobs"),
CmdLineTool.BooleanOption(
"--deleted", dest="deleted", help="migrate deleted jobs"),
CmdLineTool.BooleanOption(
"--all",
dest="all",
help="migrate all jobs (complete, incomplete, and deleted)"),
CmdLineTool.StringOption("--src-data-dir",
dest="dataDir",
help="path to tractor 1.0 data directory"),
CmdLineTool.StringOption("--dest-config-dir",
dest="configDir",
help="path to tractor 2.0 config directory"),
] + CmdLineTool.BasicCmdLineTool.options
def __init__(self, *args, **kwargs):
super(DBMigrateTool, self).__init__(*args, **kwargs)
self.config = None
def parseArgs(self, *args, **kwargs):
"""This method gets called before execute() to validate command line arguments."""
result = super(DBMigrateTool, self).parseArgs(*args, **kwargs)
# no additional args should be supplied on the command line once flags have been removed
if self.args:
raise CmdLineTool.HelpRequest, self.getHelpStr()
if self.opts.configDir:
if not os.path.exists(self.opts.configDir):
raise OptionParser.OptionParserError(
"Config dir %s does not exist" % self.opts.configDir)
if not self.opts.complete and not self.opts.incomplete and not self.opts.deleted and not self.opts.all:
raise OptionParser.OptionParserError(
"One of --complete, --incomplete, --deleted, or --all must be specified."
)
if not self.opts.dataDir:
raise OptionParser.OptionParserError(
"--src-data-dir must be specified.")
if not self.opts.configDir:
raise OptionParser.OptionParserError(
"--dest-config-dir must be specified.")
if self.opts.all:
self.opts.complete = True
self.opts.incomplete = True
self.opts.deleted = True
return result
def execute(self):
"""This method gets called automatically by CmdLineTool, and is the core logic of the program."""
# this enables the postgresql server to find the proper python
self.config = EngineConfig.EngineConfig(self.opts.configDir,
self.installDir())
self.sync()
def installDir(self):
"""Return the full path to this tractor installation."""
thisScriptPath = os.path.dirname(sys.argv[0])
installDir = os.path.join(thisScriptPath,
RELATIVE_PATH_TO_INSTALL_ROOT)
installDir = os.path.realpath(installDir)
return installDir
def isDebugMode(self):
"""Returns True if debug mode is turned on by way of config file or command line option."""
return self.opts.debug or (self.config and self.config.isDbDebug())
def dprint(self, msg, inverse=False):
if self.isDebugMode(
) and not inverse or not self.isDebugMode() and inverse:
sys.stderr.write(msg)
sys.stderr.write("\n")
def log(self, msg):
sys.stderr.write(msg)
sys.stderr.write("\n")
def runCommand(self, argv, input=None, errIsOut=False, **kw):
"""Run the specified command, returning the return code, stdout, and stderr."""
self.dprint("running %s" % argv)
stdin = subprocess.PIPE if input is not None else None
stderr = subprocess.STDOUT if errIsOut else subprocess.PIPE
proc = subprocess.Popen(argv,
stdin=stdin,
stdout=subprocess.PIPE,
stderr=stderr,
**kw)
# NOTE: http://docs.python.org/2/library/subprocess.html warns that input to Popen.communicate() shouldn't be "large"
out, err = proc.communicate(input=input)
rcode = proc.wait()
self.dprint(out)
if not errIsOut:
self.dprint(err)
return rcode, out, err
def sync(self):
self.log("synchronizing with filesystem")
def jobDirSort(a, b):
jidA = a.split("/")[-1]
dateA = jidA[:8]
jidB = b.split("/")[-1]
dateB = jidB[:8]
if dateA != dateB:
return -cmp(dateA, dateB)
else:
jidA = jidA[8:]
jidA = int(jidA) if jidA.isdigit() else 0
jidB = jidB[8:]
jidB = int(jidB) if jidB.isdigit() else 0
return -cmp(jidA, jidB)
# establish connection with postgres database
db = EngineDB.EngineDB(dbhost=self.config.dbHostname(),
db=self.config.dbDatabaseName(),
user=MIGRATE_USER,
password=MIGRATE_PASSWORD)
try:
db.open()
except rpg.sql.SQLError, err:
raise DBMigrateToolError, str(err)
# first walk the job data directory so that newer jobs can be processed first
jobDirs = []
jobsDir = os.path.join(self.opts.dataDir, "jobs")
self.dprint("jobsDir = %s" % jobsDir)
joroots = []
for root, dirs, files in os.walk(jobsDir):
if FileSystemDB.JobDB.JOB_INFO_FILENAME in files:
# it's a job database
jobDirs.append(root)
# process each job directory in reverse date + jobid order (newest jobs first)
jobDirs.sort(jobDirSort)
for jobDir in jobDirs:
self.dprint("process job in %s" % jobDir)
# extract job id from job directory
parts = jobDir.split("/")
jid = parts[-1]
if not jid.isdigit():
self.log("Job directory does not end in a job id: %s" % jobDir)
continue # for jobDir
jid = int(jid)
# set up a JobDB to analyze job
jobDB = FileSystemDB.JobDB(jobDir, readFiles=False)
# don't process deleted jobs if deemed so
isDeleted = jobDB.isDeleted()
if isDeleted:
if not self.opts.deleted:
self.dprint("Skipping deleted job %d." % jid)
continue # for jobDir
else:
# don't process non-deleted jobs if complete/incomplete flags haven't been set
if not self.opts.complete and not self.opts.incomplete:
self.dprint("Skipping non-deleted job %d." % jid)
continue # for jobDir
# do not process job if it's already in database
jobInDB = db._execute("SELECT jid FROM oldjob WHERE oldjid=%d" %
jid)
rows = db.cursor.fetchall()
if len(rows) > 0:
self.log("Skipping migrated job %d." % jid)
continue # for jobDir
# read job info
try:
jobDB.readJobInfo()
except FileSystemDB.JobDBError, err:
self.log(str(err))
continue # for jobDir
# filter out complete/incomplete jobs
isComplete = jobDB.isComplete()
if not isDeleted:
if isComplete:
if not self.opts.complete:
self.dprint("Skipping complete job %d." % jid)
continue # for jobDir
else:
if not self.opts.incomplete:
self.dprint("Skipping incomplete job %d." % jid)
continue # for jobDir
# read remaining info files
jobDB.readTaskInfo()
jobDB.readCmdList()
jobDB.readActivityLog()
jobDB.countStates()
# deleted jobs need to go in an archive partition
if isDeleted:
db._execute("SELECT TractorPartitionCreate('%s') as suffix" %
jobDB.job.spooltime.strftime("%Y-%m-%d"))
result = db.cursor.fetchall()
if not len(result):
self.log(
"Skipping deleted job %d with spooltime %s due to problems creating archive partition."
% (jid, str(jobDB.job.spooltime)))
continue # for jobDir
suffix = result[0]["suffix"]
else:
suffix = ""
# bulk insert records on a per-table basis using postgresql's COPY FROM
db._execute("begin")
db._execute("SELECT TractorNewJid() as nextjid")
result = db.cursor.fetchall()
nextjid = result[0]["nextjid"]
# set job id of job as well as tasks, commands, and invocations
jobDB.setJid(nextjid)
s = StringIO.StringIO(PGFormat.formatObjsForCOPY([jobDB.job]))
db.cursor.copy_from(s, "job" + suffix)
s = StringIO.StringIO(PGFormat.formatObjsForCOPY(jobDB.tasks))
db.cursor.copy_from(s, "task" + suffix)
s = StringIO.StringIO(PGFormat.formatObjsForCOPY(jobDB.commands))
db.cursor.copy_from(s, "command" + suffix)
s = StringIO.StringIO(PGFormat.formatObjsForCOPY(
jobDB.invocations))
db.cursor.copy_from(s, "invocation" + suffix)
# add entry to oldjob table to show it has been migrated
db._execute("INSERT INTO oldjob VALUES (%d, %d)" % (nextjid, jid))
db._execute("end")
class TractorCLT(CmdLineTool.MultiCmdLineTool):
"""The main command object for the cli just includes all the sub-command objects."""
version = "2.0"
DEFAULT_THREADS = 10
DEFAULT_TIMEOUT = 30
DEFAULT_TIMEFMT = os.environ.get("TRACTOR_TIMEFMT")
# setup all the commands in this app
commands = {
"jobs" : ListCmds.JobsCmd,
"tasks" : ListCmds.TasksCmd,
"commands" : ListCmds.CommandsCmd,
"cmds" : ListCmds.CommandsCmd,
"invocations" : ListCmds.InvocationsCmd,
"invos" : ListCmds.InvocationsCmd,
"blades" : ListCmds.BladesCmd,
"params" : ListCmds.ParamsCmd,
"chcrews" : JobCmds.ChangeCrewsCmd,
"chpri" : JobCmds.ChangePriorityCmd,
"delay" : JobCmds.DelayJobCmd,
"delete" : JobCmds.DeleteJobCmd,
"jattr" : JobCmds.ChangeJobAttributeCmd,
"pause" : JobCmds.PauseJobCmd,
"interrupt" : JobCmds.InterruptJobCmd,
"restart" : JobCmds.RestartJobCmd,
"retryallerrs" : JobCmds.RetryAllErrorsCmd,
"skipallerrs" : JobCmds.SkipAllErrorsCmd,
"undelay" : JobCmds.UndelayJobCmd,
"undelete" : JobCmds.UndeleteJobCmd,
"unpause" : JobCmds.UnpauseJobCmd,
"retry" : TaskCmds.RetryTaskCmd,
"resume" : TaskCmds.ResumeTaskCmd,
"skip" : TaskCmds.SkipTaskCmd,
"log" : TaskCmds.PrintLogCmd,
"chkeys" : CommandCmds.ChangeKeysCmd,
"cattr" : CommandCmds.ChangeCommandAttributeCmd,
"nimby" : BladeCmds.NimbyBladeCmd,
"unnimby" : BladeCmds.UnnimbyBladeCmd,
"trace" : BladeCmds.TraceBladeCmd,
"help" : HelpCmds.MainHelp,
None : HelpCmds.MainHelp,
"alias" : HelpCmds.AliasHelp,
}
# command only viewable by rpg folk
adminCommands = {
}
# command aliases
aliases = {
"aliases" : "alias",
"machs" : "machines",
"queue" : "jobs -s pri,-spooled not errwait and",
}
# set the options that the main program will have
options = [
CmdLineTool.BooleanOption ("-y", "--yes",
help="answer 'yes' to all questions"),
CmdLineTool.BooleanOption ("-a", "--archives",
help="search archive tables"),
CmdLineTool.BooleanOption ("--force",
help="force all operations. This will "
"allow any operation to be performed "
"on a task or job, even if you are "
"not the owner."),
CmdLineTool.BooleanOption ("--nocolor", dest="color", default=True,
help="do not display any output in color."),
CmdLineTool.BooleanOption ("--zeros",
help="some fields do not display a value "
"if it is equal to zero, this will "
"force zeros to be printed no matter "
"what."),
CmdLineTool.BooleanOption ("--nothreads", dest="threads", const=1,
help="do not use threads, when performing "
"task/job operations, threads are "
"only used when --yes is applied."),
CmdLineTool.IntOption ("--threads", default=DEFAULT_THREADS,
help="maximum number of concurrent "
"threads, default=%d, setting this "
"to 1 or lower is the same as "
"--nothreads." % DEFAULT_THREADS),
CmdLineTool.SecondsOption ("-p", "--pause",
help="pause p seconds inbetween each "
"action (e.g. retry, skip, etc.) "
"This can be a floating point value."),
CmdLineTool.SecondsOption ("-t", "--timeout", default=DEFAULT_TIMEOUT,
help="timeout in seconds before canceling "
"an operation, default=%d seconds." %
DEFAULT_TIMEOUT),
CmdLineTool.BooleanOption ("--login", default=False,
help="explicitly log in to engine rather than use existing session id "
"query"),
CmdLineTool.BooleanOption ("--logout", default=False,
help="automatically log out from engine after performing "
"query"),
DBCmdLineTool.RawDataOption(),
DBCmdLineTool.TimeFormatOption(default=DEFAULT_TIMEFMT),
DBCmdLineTool.FullTimeOption(),
CmdLineTool.StringOption ("--engine",
help="<hostname>:<port> of engine"),
CmdLineTool.StringOption ("-u", "--user", help="engine username"),
CmdLineTool.StringOption ("--password", "--pw", help="engine user password"),
CmdLineTool.StringOption ("--password-file", "--pwfile", help="path to json file containing engine username and password"),
CmdLineTool.BooleanOption ("--traceback", help="show full traceback on exceptions (developer option)"),
CmdLineTool.BooleanOption ("-d", "--debug", help="display debug info"),
CmdLineTool.VersionOption ("-v", "--version", help="display version info"),
] + CmdLineTool.MultiCmdLineTool.options
def __init__(self, *args, **kwds):
self.__userPrefs = None
super(TractorCLT, self).__init__(*args, **kwds)
def getUserPrefs(self):
"""Return a pointer to the user prefs object."""
if self.__userPrefs is None:
self.__userPrefs = {} # cli.UserPrefs()
return self.__userPrefs
userPrefs = property(fget=getUserPrefs)
def parseArgs(self, *args, **kwds):
super(TractorCLT, self).parseArgs(*args, **kwds)
self.opts.nocolor = not self.opts.color
# set all the options as members of ourself
for var in ("yes", "force", "color", "zeros", "threads", "pause",
"timeout"):
setattr(self, var, getattr(self.opts, var))
global ShowFullTraceback
ShowFullTraceback = self.opts.traceback
def getHelpStr(self):
"""Print a custom help message."""
# get the help command
helpcmd = self.getCommand("help")
# run the main help
helpcmd.run()
def getNewCommand(self, cmd, *args, **kwds):
"""Get a L{CmdLineTool} instance that should be used for the
provided command.
@param cmd: name of the command being referenced.
@type cmd: string
@return: L{CmdLineTool} that will be used
@rtype: L{CmdLineTool} instance
"""
kwds["raiseArgErrors"] = True
try:
return self.commands[cmd](*args, **kwds)
except KeyError:
raise CmdLineTool.UnknownCommand(cmd)
def getCommandToRun(self):
try:
return super(TractorCLT, self).getCommandToRun()
except CmdLineTool.UnknownCommand, err:
# check for the fields command which we will redirect to the
# main help command
if self.args and self.args[0] == "fields":
command = "help"
cmdobj = self.getCommand(command)
args = ["fields"] + self.args[1:]
return (command, cmdobj, args)
# check the user prefs for command aliases
try:
alias = self.userPrefs.aliases.command[self.args[0]]
except (AttributeError, KeyError):
# check the global command alias list
alias = self.aliases.get(self.args[0])
if alias:
# check for additional arguments in the alias
import rpg.stringutil as stringutil
aliasArgs = stringutil.quotedSplit(alias, removeQuotes=True)
command = aliasArgs[0]
cmdobj = self.getCommand(command)
args = aliasArgs[1:] + self.args[1:]
return (command, cmdobj, args)
raise CmdLineTool.UnknownCommand, err
class DBControlTool(CmdLineTool.BasicCmdLineTool):
progname = "tractor-dbctl"
description = """
This program is used to manage starting and stopping of the postgresql
database server that is used with the tractor engine.
"""
options = [
CmdLineTool.BooleanOption(
"--start-for-engine",
dest="startForEngine",
help=
"start the postgresql server and automatically build/upgrade schema as configured in db.config (used by engine)"
),
CmdLineTool.BooleanOption(
"--stop-for-engine",
dest="stopForEngine",
help=
"stop the postgresql server as configured in db.config (used by engine)"
),
CmdLineTool.BooleanOption(
"--start", dest="start", help="start the postgresql server"),
CmdLineTool.BooleanOption(
"--stop", dest="stop", help="stop the postgresql server"),
CmdLineTool.BooleanOption(
"--status",
dest="status",
help="check the status of the postgresql server"),
CmdLineTool.BooleanOption(
"--init",
dest="init",
help=
"initialize the postgresql data directory; server must not be running"
),
CmdLineTool.BooleanOption(
"--build",
dest="build",
help="build the tractor database; server must be running"),
CmdLineTool.BooleanOption(
"--destroy",
dest="destroy",
help=
"remove the postgres database directory; server must not be running"
),
CmdLineTool.BooleanOption(
"--upgrade", dest="upgrade", help="update the database schema"),
CmdLineTool.BooleanOption("--check-upgrade",
dest="checkUpgrade",
help="check whether upgrades are required"),
CmdLineTool.BooleanOption(
"--no-auto-upgrade",
dest="noAutoUpgrade",
help="prevent automatic upgrading of the database schema"),
CmdLineTool.BooleanOption(
"--purge-jobs", dest="purgeJobs", help="remove all jobs"),
CmdLineTool.StringOption(
"--purge-archive-to-year-month",
dest="purgeArchiveToYearMonth",
help=
"remove archived jobs up to and including specified year-month in YY-MM format"
),
CmdLineTool.BooleanOption(
"--vacuum",
dest="vacuum",
help="rebuild tables of non-deleted jobs to save space"),
CmdLineTool.BooleanOption(
"--reset-job-counter",
dest="resetJobCounter",
help=
"reset the job id counter so that job ids (jids) start at 1; can only be used with --purge-jobs"
),
CmdLineTool.BooleanOption("--show-params",
dest="showParams",
help="show stored database paramaters"),
CmdLineTool.BooleanOption(
"--update-params",
dest="updateParams",
help="update database with paramaters stored in config files"),
CmdLineTool.BooleanOption(
"--tail-log",
dest="tailLog",
help="tail and follow the last lines of postgres message log"),
CmdLineTool.BooleanOption(
"--logs-usage",
dest="logsUsage",
help="report disk space used by postgresql message log files"),
CmdLineTool.BooleanOption("--purge-logs",
dest="purgeLogs",
help="remove postgresql message log files"),
CmdLineTool.StringOption(
"--backup", dest="backup", help="write backup to specified file"),
CmdLineTool.StringOption(
"--restore",
dest="restore",
help="restore database from specified backup file"),
CmdLineTool.StringOption("--config-dir",
dest="configDir",
help="path to tractor config directory"),
] + CmdLineTool.BasicCmdLineTool.options
def __init__(self, *args, **kwargs):
super(DBControlTool, self).__init__(*args, **kwargs)
self.config = None
if osutil.getlocalos() == "Linux":
# preload libpq.so file so that later imports of psycopg2 will get the proper library
# since rmanpy as compiled only looks in the install dir's lib/, but libpq is in lib/psql/lib
ctypes.cdll.LoadLibrary(
os.path.join(self.installDir(), "lib", "psql", "lib",
"libpq.so.5"))
# this doesn't appear to be necessary on OSX since the stock install comes with a valid libpq.so
def parseArgs(self, *args, **kwargs):
"""This method gets called before execute() to validate command line arguments."""
result = super(DBControlTool, self).parseArgs(*args, **kwargs)
# no additional args should be supplied on the command line once flags have been removed
if self.args:
raise CmdLineTool.HelpRequest, self.getHelpStr()
if self.opts.configDir:
if not os.path.exists(self.opts.configDir):
raise OptionParser.OptionParserError(
"Config dir %s does not exist" % self.opts.configDir)
if self.opts.resetJobCounter and not self.opts.purgeJobs:
raise OptionParser.OptionParserError(
"--reset-job-counter can only be used with --purge-jobs.")
if not self.opts.configDir:
raise OptionParser.OptionParserError(
"--config-dir must be specified.")
return result
def execute(self):
"""This method gets called automatically by CmdLineTool, and is the core logic of the program."""
self.config = EngineConfig.EngineConfig(self.configDir(),
self.installDir())
# check that process is owned by proper user; files will be owned by that user
processOwner = osutil.getusername()
dataDirOwner = osutil.ownerForPath(self.config.pgDataDir())
configuredOwner = self.config.tractorEngineOwner()
if configuredOwner and processOwner != configuredOwner:
raise DBControlToolError, "tractor-dbctl is configured to be run by %s; the owner of this process is %s." \
% (configuredOwner, processOwner)
# check that engine owner owns the data directory
if dataDirOwner != processOwner:
raise DBControlToolError, "The database data dir %s is owned by %s; the owner of this process is %s." \
% (self.config.pgDataDir(), dataDirOwner, processOwner)
if self.opts.status:
self.status()
elif self.opts.start:
self.start()
elif self.opts.stop:
self.stop()
elif self.opts.startForEngine:
self.startForEngine()
elif self.opts.stopForEngine:
self.stopForEngine()
elif self.opts.init:
self.init()
elif self.opts.build:
self.build()
elif self.opts.destroy:
self.destroy()
elif self.opts.checkUpgrade:
self.checkUpgrade()
elif self.opts.upgrade:
self.upgrade()
elif self.opts.purgeJobs:
self.purgeJobs()
elif self.opts.purgeArchiveToYearMonth:
self.purgeArchiveToYearMonth()
elif self.opts.vacuum:
self.vacuum()
elif self.opts.backup:
self.backup()
elif self.opts.restore:
self.restore()
elif self.opts.purgeLogs:
self.purgeLogs()
elif self.opts.logsUsage:
self.tailLog()
elif self.opts.tailLog:
self.tailLog()
elif self.opts.showParams:
self.showParams()
elif self.opts.updateParams:
self.updateParams()
else:
raise OptionParser.OptionParserError(
"No operations were specified. Use --help for options.")
def isDebugMode(self):
"""Returns True if debug mode is turned on by way of config file or command line option."""
return self.opts.debug or (self.config and self.config.isDbDebug())
def installDir(self):
"""Return the full path to this tractor installation."""
thisScriptPath = os.path.dirname(sys.argv[0])
installDir = os.path.join(thisScriptPath,
RELATIVE_PATH_TO_INSTALL_ROOT)
installDir = os.path.realpath(installDir)
return installDir
def configDir(self):
"""Return the full path to the config dir."""
return os.path.realpath(self.opts.configDir)
def dprint(self, msg, inverse=False):
if self.isDebugMode(
) and not inverse or not self.isDebugMode() and inverse:
sys.stderr.write(msg)
sys.stderr.write("\n")
def log(self, msg):
sys.stderr.write(msg)
sys.stderr.write("\n")
def runCommand(self, argv, input=None, errIsOut=False, **kw):
"""Run the specified command, returning the return code, stdout, and stderr."""
self.dprint("Running %s" % argv)
stdin = subprocess.PIPE if input is not None else None
stderr = subprocess.STDOUT if errIsOut else subprocess.PIPE
proc = subprocess.Popen(argv,
stdin=stdin,
stdout=subprocess.PIPE,
stderr=stderr,
**kw)
# NOTE: http://docs.python.org/2/library/subprocess.html warns that input to Popen.communicate() shouldn't be "large"
out, err = proc.communicate(input=input)
rcode = proc.wait()
self.dprint(out)
if not errIsOut:
self.dprint(err)
return rcode, out, err
def start(self):
"""Start the postgresql server."""
if self.isPostgreSQLRunning():
raise DBControlToolError, "A postgresql server is already running on the data directory %s." % self.config.pgDataDir(
)
self.startPostgreSQL()
def stop(self):
"""Stop the postgresql server."""
if not self.isPostgreSQLRunning():
raise DBControlToolError, "The postgresql server is NOT running on the data directory %s." % self.config.pgDataDir(
)
self.stopPostgreSQL()
def status(self):
"""Check the status of the postgresql server."""
if self.isPostgreSQLRunning():
self.log(
"A postgresql server is running on the data directory %s." %
self.config.pgDataDir())
if self.isPostgreSQLReachable():
self.log(
"The postgresql server is reachable through port %d." %
self.config.dbPort())
if self.databaseExists():
self.log("The database %s exists." %
self.config.dbDatabaseName())
else:
self.log("The database %s does NOT exist." %
self.config.dbDatabaseName())
else:
self.log(
"The postgresql server is NOT reachable through port %d." %
self.config.dbPort())
else:
self.log(
"A postgresql server is NOT running on the data directory %s."
% self.config.pgDataDir())
def init(self):
"""Initialize the postgresql data directory."""
# make sure postgresql server isn't already running
if self.isPostgreSQLRunning():
raise DBControlToolError, "The postgresql server is already running. It must first be stopped with --stop."
# make sure there isn't an existing postgres database
if os.path.exists(self.config.pgDataDir()) and len(
os.listdir(self.config.pgDataDir())) > 0:
raise DBControlToolError, "%s is not an empty directory." % self.config.pgDataDir(
)
self.initPostgreSQL()
def build(self):
if self.databaseExists():
raise DBControlToolError, "Database already exists."
self.buildDB()
def startForEngine(self):
"""Conditionally start the postgresql server, performing any database initialization as required."""
# initialize db if required and configured to do so
if not os.path.exists(self.config.pgDataDir()) or not os.path.exists(
os.path.join(self.config.pgDataDir(), "postgresql.conf")):
self.dprint(
"Considering db initialization because one of the following do not exist:\n%s\n%s"
% (self.config.pgDataDir(),
os.path.join(self.config.pgDataDir(), "postgresql.conf")))
if self.config.doDbInit():
self.initPostgreSQL()
else:
raise DBControlToolError, "The postgresql data dir %s was not found and %s is not configured to create a new one." % (
self.config.pgDataDir(), self.progname)
# start postgresql server if required and configured to do so
if self.isPostgreSQLRunning():
if self.config.doDbUseExisting():
self.dprint(
"A postgresql server is already running. The system has been configured to use it."
)
else:
raise DBControlToolError, "A postgresql server is already running. Set %s to True in %s to use an existing postgresql server." \
% (EngineConfig.DB_USE_EXISTING, self.config.dbSiteConfigFilename())
else:
if self.config.doDbStartup():
self.startPostgreSQL()
time.sleep(POST_START_SLEEP)
if not self.isPostgreSQLRunning(
maxAttempts=MAX_DB_CONNECT_ATTEMPTS):
raise DBControlToolError, "Failed to start a postgresql server on data directory %s. A different server may be already running; to check, try 'ps -elf | grep postgres'. Or another service is using port %d; to check, try 'sudo fuser %d/tcp'.\%s may have more info." \
% (self.config.pgDataDir(), self.config.dbPort(), self.config.dbPort(), self.config.pgLogFilename())
else:
raise DBControlToolError, "A postgresql server was not started because %s is set to False in %s to prevent the automatic starting of a postgresql server. Change this setting to True, or manually ensure your custom postgresql server is running and set %s to True." \
% (EngineConfig.DB_STARTUP, self.config.dbSiteConfigFilename(), EngineConfig.DB_USE_EXISTING)
# test if reachable
if not self.isPostgreSQLReachable(maxAttempts=MAX_DB_CONNECT_ATTEMPTS):
raise DBControlToolError, "Unable to connect to postgresql server on port %d. Check %s for more info." % (
self.config.dbPort(), self.config.pgLogFilename())
# build database
if not self.databaseExists():
self.buildDB()
else:
self.dprint(
"Database %s exists. Building database is not required." %
self.config.dbDatabaseName())
# ensure that languages are loaded
if not self.isLanguageReady("plpython2u"):
raise DBControlToolError, "Unable to verify that plpython2u has been loaded."
if not self.isLanguageReady("plpgsql"):
raise DBControlToolError, "Unable to verify that plpgsql has been loaded."
# see if an upgrade is required
if self.config.doDbAutoUpgrade() and not self.opts.noAutoUpgrade:
upgrades = self.getUpgrades()
if upgrades:
self.upgradeDB(upgrades)
def stopForEngine(self):
"""Stop the postgresql server."""
if self.config.doDbShutdown() and self.isPostgreSQLRunning():
self.stopPostgreSQL()
def destroy(self):
"""This method removes the existing a new postgresql database directory.
There must be no existing postgresql server running."""
# make sure postgresql server is not running
if self.isPostgreSQLRunning():
raise DBControlToolError, "The postgresql server is already running. It must be shutdown with --stop before using --destroy."
# make sure there isn't an existing postgres database
if not os.path.exists(self.config.pgDataDir()):
raise DBControlToolError, "%s does not exist." % self.config.pgDataDir(
)
# remove the contents of the data directory
self.log("Removing contents of data directory %s." %
self.config.pgDataDir())
for filename in os.listdir(self.config.pgDataDir()):
fullFilename = os.path.join(self.config.pgDataDir(), filename)
try:
if os.path.isfile(fullFilename):
os.remove(fullFilename)
else:
shutil.rmtree(
os.path.join(self.config.pgDataDir(), fullFilename))
except (IOError, OSError), err:
self.log("Unable to remove %s: %s" % (filename, str(err)))
class ChangeJobAttributeCmd(JobOperationCmd):
usage = "jattr"
description = """
Change an attribute of one or more jobs. The attribute and value of a job
must be specified, and may require a different format, depending on the
attribute. For example, certain attributes may expect lists, where others
a string. By default the user is prompted before each job has its
attribute changed, unless the --yes flag is set. Also, you can only change
the attributes of your own job, unless the --force flag is set.
"""
examples = """
The typical syntax to change the attributes of jobs is:
> tq jattr SEARCH_CLAUSE -k attribute [-a|--add|-r|--remove|-v|--value] value
> tq jattr jid [jid2 jid3 ...] -k attribute [-a|--add|-r|--remove|-v|--value] value
Examples:
change the metadata of all potsi's jobs to "watch this job"
> tq jattr user=potsi -k metadata -v "watch this job"
change the afterjids of all potsi's jobs submitted after 5pm today
> tq jattr user=potsi and "spooled >= 5pm" -k afterjid -v 10020,11021
add "apple" to the tags of all potsi's jobs that are at priority 100
> tq chpri user=potsi and priority=100 -k tags --add apple
"""
# add option to specify job attribute and value
options = [
CmdLineTool.StringOption("-k",
"--key",
help="the name of the job attribute "
"that will be changed"),
CmdLineTool.StringOption("-v",
"--value",
help="the value the job attribute "
"that will be set to; can be a"
"commad-delimited list for lists"),
CmdLineTool.StringOption(
"-a", "--add", help="value will be appended to current value"),
CmdLineTool.StringOption(
"-r", "--remove", help="value will be removed from current value"),
] + JobOperationCmd.options
IntListAttrs = ["afterjids"]
StrListAttrs = ["tags", "envkey"]
def parseArgs(self, *args, **kwargs):
result = super(ChangeJobAttributeCmd, self).parseArgs(*args, **kwargs)
# make sure key and value were specified
if self.opts.key is None:
raise tq.TqError, "no key provided"
if self.opts.value is None:
raise tq.TqError, "no value provided :)"
if self.opts.key in self.IntListAttrs:
values = stringutil.str2list(self.opts.value)
for value in values:
if not value.isdigit():
raise tq.TqError, "value must be a comma separated list of integers"
return result
def processObject(self, obj):
"""Operate on the provided job object."""
# ask the user if we should continue
if not obj.pre(self, "Change %s of the job?" % self.opts.key):
return
# represent value in type appropriate to member; it will be packed for transmission
if self.opts.key in self.IntListAttrs:
value = [int(v) for v in stringutil.str2list(self.opts.value)]
elif self.opts.key in self.StrListAttrs:
value = stringutil.str2list(self.opts.value)
else:
value = self.opts.value
# try to run the operation
query.jattr(obj, key=self.opts.key, value=value)
obj.post(self, "%s changed" % self.opts.key)