class OperateCmd(QueryCmd):
"""An operation command will query the database and run an arbitrary
operation on each object returned."""
# the default set of options for all operation commands
options = [
CmdLineTool.StrListOption("-c", "--cols",
help="list of columns that will be "
"printed for each each row."),
CmdLineTool.StrListOption("-s", "--sortby",
help="sort the results in a specified "
"order before performing any "
"operations."),
CmdLineTool.BooleanOption("--ns", "--nosort", dest="nosort",
help="do not sort the results"),
CmdLineTool.IntOption ("--limit",
help="only operate on the first l rows"),
] + CmdLineTool.CmdLineTool.options
def __init__(self, table, *args, **kwargs):
# call the super
super(OperateCmd, self).__init__(table, *args, **kwargs)
# if a list of objects is provided at the command line, then
# subclasses can fill in this value. Setting this will prevent
# the database from being contacted
self.objects = []
def parseArgs(self, *args, **kwargs):
result = super(OperateCmd, self).parseArgs(*args, **kwargs)
# make sure a where exists
if not self.where:
raise OptionParser.HelpRequest(self.getHelpStr())
# make sure a value is set for options we are not supporting, but
# that the super expects to exist.
for name,default in (("distinct", None),
("delimiter", ' '),
("raw", False),
("timefmt", None),
("noformat", False),
("noheader", False)):
self.opts.__dict__.setdefault(name, default)
return result
def opendb(self):
"""We don't need to open the database if the objects member is set."""
if not self.objects:
super(OperateCmd, self).opendb()
def getNewObjs(self, num, curr, objs, distinct):
"""Return at most 'num' new objects for the operationLoop. The
objects returned are tq Operation Job or Task objects. By
default the mrd cursor is checked for waiting job objects, but
a list can be provided which will be modified."""
# a list of all the operation objects we are going to return
newobjs = []
# loop until we have enough objects
while len(newobjs) < num and curr < len(objs):
obj = objs[curr]
curr += 1
# check if this object is distinct
if not self.isDistinct(obj, distinct):
if self.parent.opts.debug: print 'already worked on'
continue
# skip over this object if we don't have permission
if not (self.parent.force or obj.hasPermission()):
msg = "permission denied: not job owner"
if self.parent.color:
msg = terminal.TerminalColor('red').colorStr(msg)
print self.formatter.format(obj), msg
continue
# we passed all the tests, so add it to the list
newobjs.append(obj)
return newobjs,curr
def processResult(self, qresult):
"""Process and print all the objects returned from the query."""
# if we had no results, then do nothing
if not qresult or \
(self.opts.limit is not None and self.opts.limit < 0):
return
# make sure the query result doesn't cache objects after they are
# created, as we will only need them once.
if isinstance(qresult, Database.QueryResult):
qresult.cacheObjects = False
file = sys.stdout
# set the widths of our formatter so everything is neatly spaced
if not self.opts.noformat:
self.formatter.setWidths(qresult)
# print a header
if not self.opts.noheader:
print >>file, self.formatter.header()
# keep track of which objects are distinct
distinct = {}
# our current list of objects that have had the operation started,
# but the operation has not finished. Typically because the dispatcher
# is busy reading from it and would block the process.
queue = []
# keep track of how many objects we've worked on
workedon = 0
# time we last created an object (only useful when pausing between
# operations).
lastcreate = 0
# make sure threads is set to at least one
if self.parent.threads < 1:
self.parent.threads = 1
# setup a blocking flag
if not self.parent.yes or self.parent.threads <= 1:
self.blocking = True
else:
self.blocking = False
# keep track of our current index into the query result.
curr = 0
try:
# start operating on the objects
while True:
# if we are running in parallel mode (i.e. non-blocking) we
# want to have as many concurrent connections as possible.
# Thus make sure the queue size is always the size of
# self.threads, or at least as big as it can be.
newobjs = []
# check if we need to pause inbetween operations
if self.parent.pause > 0:
# how much time has passed since our last object create
elapsed = self.parent.pause - (time.time() - lastcreate)
# if we don't have any objects on the queue, and 'pause'
# seconds hasn't elapsed
if not queue and elapsed > 0:
time.sleep(elapsed)
elapsed = 0
# add another item to the queue
if elapsed <= 0 and len(queue) < self.parent.threads:
newobjs,curr = self.getNewObjs(1, curr,
qresult, distinct)
lastcreate = time.time()
else:
# get however many more we can take on now
newobjs,curr = self.getNewObjs(
self.parent.threads - len(queue),
curr, qresult, distinct)
# if we have nothing left, then we're all done!
if not (queue or newobjs) or \
(not queue and \
self.opts.limit not in [None, 0] and \
self.opts.limit <= workedon):
break
# start the new operations now. If we are in blocking
# mode, (self.yesToAll == False or self.threads <= 1),
# then the entire operation will be done here. If we are
# in parallel mode (i.e. non-blocking), then the connect
# will initialize here, raise a WouldBlock exception, and
# be put onto the queue.
for obj in newobjs:
try:
# work on this object
self.processObject(obj)
except Sockets.WouldBlock:
# if this would block, then add it to the queue
queue.append(obj)
except Sockets.SocketError, err:
# catch any other socket related error
obj.post(self, str(err), error=True)
obj.close()
except EngineClient.EngineClientError, err:
obj.post(self, str(err), error=True)
obj.close()
except:
class TractorCLT(CmdLineTool.MultiCmdLineTool):
"""The main command object for the cli just includes all the sub-command objects."""
version = "2.0"
DEFAULT_THREADS = 10
DEFAULT_TIMEOUT = 30
DEFAULT_TIMEFMT = os.environ.get("TRACTOR_TIMEFMT")
# setup all the commands in this app
commands = {
"jobs" : ListCmds.JobsCmd,
"tasks" : ListCmds.TasksCmd,
"commands" : ListCmds.CommandsCmd,
"cmds" : ListCmds.CommandsCmd,
"invocations" : ListCmds.InvocationsCmd,
"invos" : ListCmds.InvocationsCmd,
"blades" : ListCmds.BladesCmd,
"params" : ListCmds.ParamsCmd,
"chcrews" : JobCmds.ChangeCrewsCmd,
"chpri" : JobCmds.ChangePriorityCmd,
"delay" : JobCmds.DelayJobCmd,
"delete" : JobCmds.DeleteJobCmd,
"jattr" : JobCmds.ChangeJobAttributeCmd,
"pause" : JobCmds.PauseJobCmd,
"interrupt" : JobCmds.InterruptJobCmd,
"restart" : JobCmds.RestartJobCmd,
"retryallerrs" : JobCmds.RetryAllErrorsCmd,
"skipallerrs" : JobCmds.SkipAllErrorsCmd,
"undelay" : JobCmds.UndelayJobCmd,
"undelete" : JobCmds.UndeleteJobCmd,
"unpause" : JobCmds.UnpauseJobCmd,
"retry" : TaskCmds.RetryTaskCmd,
"resume" : TaskCmds.ResumeTaskCmd,
"skip" : TaskCmds.SkipTaskCmd,
"log" : TaskCmds.PrintLogCmd,
"chkeys" : CommandCmds.ChangeKeysCmd,
"cattr" : CommandCmds.ChangeCommandAttributeCmd,
"nimby" : BladeCmds.NimbyBladeCmd,
"unnimby" : BladeCmds.UnnimbyBladeCmd,
"trace" : BladeCmds.TraceBladeCmd,
"help" : HelpCmds.MainHelp,
None : HelpCmds.MainHelp,
"alias" : HelpCmds.AliasHelp,
}
# command only viewable by rpg folk
adminCommands = {
}
# command aliases
aliases = {
"aliases" : "alias",
"machs" : "machines",
"queue" : "jobs -s pri,-spooled not errwait and",
}
# set the options that the main program will have
options = [
CmdLineTool.BooleanOption ("-y", "--yes",
help="answer 'yes' to all questions"),
CmdLineTool.BooleanOption ("-a", "--archives",
help="search archive tables"),
CmdLineTool.BooleanOption ("--force",
help="force all operations. This will "
"allow any operation to be performed "
"on a task or job, even if you are "
"not the owner."),
CmdLineTool.BooleanOption ("--nocolor", dest="color", default=True,
help="do not display any output in color."),
CmdLineTool.BooleanOption ("--zeros",
help="some fields do not display a value "
"if it is equal to zero, this will "
"force zeros to be printed no matter "
"what."),
CmdLineTool.BooleanOption ("--nothreads", dest="threads", const=1,
help="do not use threads, when performing "
"task/job operations, threads are "
"only used when --yes is applied."),
CmdLineTool.IntOption ("--threads", default=DEFAULT_THREADS,
help="maximum number of concurrent "
"threads, default=%d, setting this "
"to 1 or lower is the same as "
"--nothreads." % DEFAULT_THREADS),
CmdLineTool.SecondsOption ("-p", "--pause",
help="pause p seconds inbetween each "
"action (e.g. retry, skip, etc.) "
"This can be a floating point value."),
CmdLineTool.SecondsOption ("-t", "--timeout", default=DEFAULT_TIMEOUT,
help="timeout in seconds before canceling "
"an operation, default=%d seconds." %
DEFAULT_TIMEOUT),
CmdLineTool.BooleanOption ("--login", default=False,
help="explicitly log in to engine rather than use existing session id "
"query"),
CmdLineTool.BooleanOption ("--logout", default=False,
help="automatically log out from engine after performing "
"query"),
DBCmdLineTool.RawDataOption(),
DBCmdLineTool.TimeFormatOption(default=DEFAULT_TIMEFMT),
DBCmdLineTool.FullTimeOption(),
CmdLineTool.StringOption ("--engine",
help="<hostname>:<port> of engine"),
CmdLineTool.StringOption ("-u", "--user", help="engine username"),
CmdLineTool.StringOption ("--password", "--pw", help="engine user password"),
CmdLineTool.StringOption ("--password-file", "--pwfile", help="path to json file containing engine username and password"),
CmdLineTool.BooleanOption ("--traceback", help="show full traceback on exceptions (developer option)"),
CmdLineTool.BooleanOption ("-d", "--debug", help="display debug info"),
CmdLineTool.VersionOption ("-v", "--version", help="display version info"),
] + CmdLineTool.MultiCmdLineTool.options
def __init__(self, *args, **kwds):
self.__userPrefs = None
super(TractorCLT, self).__init__(*args, **kwds)
def getUserPrefs(self):
"""Return a pointer to the user prefs object."""
if self.__userPrefs is None:
self.__userPrefs = {} # cli.UserPrefs()
return self.__userPrefs
userPrefs = property(fget=getUserPrefs)
def parseArgs(self, *args, **kwds):
super(TractorCLT, self).parseArgs(*args, **kwds)
self.opts.nocolor = not self.opts.color
# set all the options as members of ourself
for var in ("yes", "force", "color", "zeros", "threads", "pause",
"timeout"):
setattr(self, var, getattr(self.opts, var))
global ShowFullTraceback
ShowFullTraceback = self.opts.traceback
def getHelpStr(self):
"""Print a custom help message."""
# get the help command
helpcmd = self.getCommand("help")
# run the main help
helpcmd.run()
def getNewCommand(self, cmd, *args, **kwds):
"""Get a L{CmdLineTool} instance that should be used for the
provided command.
@param cmd: name of the command being referenced.
@type cmd: string
@return: L{CmdLineTool} that will be used
@rtype: L{CmdLineTool} instance
"""
kwds["raiseArgErrors"] = True
try:
return self.commands[cmd](*args, **kwds)
except KeyError:
raise CmdLineTool.UnknownCommand(cmd)
def getCommandToRun(self):
try:
return super(TractorCLT, self).getCommandToRun()
except CmdLineTool.UnknownCommand, err:
# check for the fields command which we will redirect to the
# main help command
if self.args and self.args[0] == "fields":
command = "help"
cmdobj = self.getCommand(command)
args = ["fields"] + self.args[1:]
return (command, cmdobj, args)
# check the user prefs for command aliases
try:
alias = self.userPrefs.aliases.command[self.args[0]]
except (AttributeError, KeyError):
# check the global command alias list
alias = self.aliases.get(self.args[0])
if alias:
# check for additional arguments in the alias
import rpg.stringutil as stringutil
aliasArgs = stringutil.quotedSplit(alias, removeQuotes=True)
command = aliasArgs[0]
cmdobj = self.getCommand(command)
args = aliasArgs[1:] + self.args[1:]
return (command, cmdobj, args)
raise CmdLineTool.UnknownCommand, err