def scriptOptions(parser, args=None, arg_keys=None):
parser.addBool(None,
' ',
'parseable',
default=False,
help='Output tabular data in parseable format')
parser.addBool(None,
' ',
'pivot',
default=False,
help='Output pivoted tabular data')
parser.addText(None,
' ',
'textwidth',
default=100,
help='Output tabular data with selected width')
parser.addAccu(None, 'v', 'verbose', default=0, help='Increase verbosity')
(opts, args, config_dict) = parser.parse(args, arg_keys)
logging.getLogger().setLevel(logging.DEFAULT - opts.verbose)
utils.verbosity(opts.verbose)
if opts.parseable:
utils.printTabular.mode = 'parseable'
elif opts.pivot:
utils.printTabular.mode = 'longlist'
utils.printTabular.wraplen = int(opts.textwidth)
return utils.Result(opts=opts,
args=args,
config_dict=config_dict,
parser=parser)
def __init__(self, config):
InfoScanner.__init__(self, config)
newVerbosity = utils.verbosity(utils.verbosity() - 3)
extConfigFN = config.getPath('source config')
extConfig = createConfig(extConfigFN).changeView(setSections = ['global'])
self._extWorkDir = extConfig.getWorkPath()
self._extTask = extConfig.getPlugin(['task', 'module'], cls = 'TaskModule')
selector = config.get('source job selector', '')
extJobDB = JobDB(extConfig, jobSelector = lambda jobNum, jobObj: jobObj.state == Job.SUCCESS)
self._selected = sorted(extJobDB.getJobs(JobSelector.create(selector, task = self._extTask)))
utils.verbosity(newVerbosity + 3)
def __init__(self, config):
from grid_control.tasks import TaskModule
newVerbosity = utils.verbosity(utils.verbosity() - 3)
extConfigFN = config.getPath('source config')
extConfig = createConfigFactory(extConfigFN).getConfig(setSections = ['global'])
self.extWorkDir = extConfig.getWorkPath()
self.extTask = extConfig.getPlugin(['task', 'module'], cls = TaskModule).getInstance()
selector = config.get('source job selector', '')
extJobDB = JobDB(extConfig, jobSelector = lambda jobNum, jobObj: jobObj.state == Job.SUCCESS)
self.selected = sorted(extJobDB.getJobs(JobSelector.create(selector, task = self.extTask)))
utils.verbosity(newVerbosity + 3)
def getEntries(self, path, metadata, events, seList, objStore): newVerbosity = utils.verbosity(utils.verbosity() - 3) if 'GC_TASK' in objStore: tmp = dict(objStore['GC_TASK'].getTaskConfig()) if 'GC_JOBNUM' in metadata: tmp.update(objStore['GC_TASK'].getJobConfig(metadata['GC_JOBNUM'])) for (newKey, oldKey) in objStore['GC_TASK'].getVarMapping().items(): tmp[newKey] = tmp.get(oldKey) metadata.update(utils.filterDict(tmp, kF = lambda k: k not in self.ignoreVars)) utils.verbosity(newVerbosity + 3) yield (path, metadata, events, seList, objStore)
def gc_cmd_line_parser(cmd_line_args):
parser = Options(usage = '%s [OPTIONS] <config file>', add_help_option = False)
parser.addBool(None, ' ', 'debug', default = False)
parser.addBool(None, ' ', 'help-conf', default = False)
parser.addBool(None, ' ', 'help-confmin', default = False)
parser.addBool(None, 'c', 'continuous', default = False)
parser.addBool(None, 'h', 'help', default = False)
parser.addBool(None, 'i', 'init', default = False)
parser.addBool(None, 'q', 'resync', default = False)
parser.addBool(None, 's', 'no-submission', default = True, dest = 'submission')
parser.addBool(None, 'G', 'gui', default = False, dest = 'gui_ansi')
parser.addBool(None, 'W', 'webserver', default = False, dest = 'gui_cp')
parser.addAccu(None, 'v', 'verbose')
parser.addList(None, 'l', 'logging')
parser.addList(None, 'o', 'override')
parser.addText(None, ' ', 'action')
parser.addText(None, 'd', 'delete')
parser.addText(None, 'J', 'job-selector')
parser.addText(None, 'm', 'max-retry')
parser.addText(None, ' ', 'reset')
# Deprecated options - refer to new report script instead
for (sopt, lopt) in [('-r', 'report'), ('-R', 'site-report'), ('-T', 'time-report'),
('-M', 'task-report'), ('-D', 'detail-report'), ('', 'help-vars')]:
parser.addBool(None, sopt, lopt, default = False, dest = 'old_report')
(opts, args, _) = parser.parse(args = cmd_line_args)
opts.gui = None
if opts.gui_ansi:
opts.gui = 'ANSIGUI'
elif opts.gui_cp:
opts.gui = 'CPWebserver'
opts.continuous = opts.continuous or None # either True or None
# Display help
if opts.help:
sys.stderr.write('%s\n\n%s\n' % (parser.usage(), open(utils.pathShare('help.txt'), 'r').read()))
sys.exit(os.EX_USAGE)
# Require single config file argument
if len(args) == 0:
utils.exitWithUsage(parser.usage(), 'Config file not specified!')
elif len(args) > 1:
utils.exitWithUsage(parser.usage(), 'Invalid command line arguments: %r' % cmd_line_args)
# Warn about deprecated report options
if opts.old_report:
utils.deprecated('Please use the more versatile report tool in the scripts directory!')
# Configure preliminary logging
utils.verbosity(opts.verbose)
logging.getLogger().setLevel(max(1, logging.DEFAULT - opts.verbose))
if opts.debug: # Setup initial debug handler before it is reconfigured by logging_setup
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(ExceptionFormatter(showCodeContext = 2, showVariables = 1, showFileStack = 1))
logging.getLogger('exception').addHandler(handler)
return (opts, args)
def __init__(self, config):
from grid_control import TaskModule
newVerbosity = utils.verbosity(utils.verbosity() - 3)
extConfigFN = config.getPath('source config')
extConfig = Config([DefaultFilesConfigFiller(), FileConfigFiller([extConfigFN])], extConfigFN)
extConfig.opts = type('DummyType', (), {'init': False, 'resync': False})
self.extWorkDir = extConfig.getWorkPath()
self.extTask = extConfig.getClass('global', ['task', 'module'], cls = TaskModule).getInstance()
selector = config.get('source job selector', '')
extJobDB = JobDB(extConfig, jobSelector = lambda jobNum, jobObj: jobObj.state == Job.SUCCESS)
self.selected = sorted(extJobDB.getJobs(JobSelector.create(selector, task = self.extTask)))
utils.verbosity(newVerbosity + 3)
def gc_cmd_line_parser(cmd_line_args):
parser = Options(usage = '%s [OPTIONS] <config file>', add_help_option = False)
parser.addBool(None, ' ', 'debug', default = False)
parser.addBool(None, ' ', 'help-conf', default = False)
parser.addBool(None, ' ', 'help-confmin', default = False)
parser.addBool(None, 'c', 'continuous', default = False)
parser.addBool(None, 'h', 'help', default = False)
parser.addBool(None, 'i', 'init', default = False)
parser.addBool(None, 'q', 'resync', default = False)
parser.addBool(None, 's', 'no-submission', default = True, dest = 'submission')
parser.addBool(None, 'G', 'gui', default = False, dest = 'gui_ansi')
parser.addBool(None, 'W', 'webserver', default = False, dest = 'gui_cp')
parser.addAccu(None, 'v', 'verbose')
parser.addList(None, 'l', 'logging')
parser.addList(None, 'o', 'override')
parser.addText(None, ' ', 'action')
parser.addText(None, 'd', 'delete')
parser.addText(None, 'J', 'job-selector')
parser.addText(None, 'm', 'max-retry')
parser.addText(None, ' ', 'reset')
# Deprecated options - refer to new report script instead
for (sopt, lopt) in [('-r', 'report'), ('-R', 'site-report'), ('-T', 'time-report'),
('-M', 'task-report'), ('-D', 'detail-report'), ('', 'help-vars')]:
parser.addBool(None, sopt, lopt, default = False, dest = 'old_report')
(opts, args, _) = parser.parse(args = cmd_line_args)
opts.gui = None
if opts.gui_ansi:
opts.gui = 'ANSIGUI'
elif opts.gui_cp:
opts.gui = 'CPWebserver'
opts.continuous = opts.continuous or None # either True or None
# Display help
if opts.help:
sys.stderr.write('%s\n\n%s\n' % (parser.usage(), open(utils.pathShare('help.txt'), 'r').read()))
sys.exit(os.EX_USAGE)
# Require single config file argument
if len(args) == 0:
utils.exitWithUsage(parser.usage(), 'Config file not specified!')
elif len(args) > 1:
utils.exitWithUsage(parser.usage(), 'Invalid command line arguments: %r' % cmd_line_args)
# Warn about deprecated report options
if opts.old_report:
utils.deprecated('Please use the more versatile report tool in the scripts directory!')
# Configure preliminary logging
utils.verbosity(opts.verbose)
logging.getLogger().setLevel(max(1, logging.DEFAULT - opts.verbose))
if opts.debug: # Setup initial debug handler before it is reconfigured by logging_setup
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(ExceptionFormatter(showCodeContext = 2, showVariables = 1, showFileStack = 1))
logging.getLogger('exception').addHandler(handler)
return (opts, args)
def getEntries(self, path, metadata, events, seList, objStore):
newVerbosity = utils.verbosity(utils.verbosity() - 3)
if 'GC_TASK' in objStore:
tmp = dict(objStore['GC_TASK'].getTaskConfig())
if 'GC_JOBNUM' in metadata:
tmp.update(objStore['GC_TASK'].getJobConfig(
metadata['GC_JOBNUM']))
for (newKey,
oldKey) in objStore['GC_TASK'].getVarMapping().items():
tmp[newKey] = tmp.get(oldKey)
metadata.update(
utils.filterDict(tmp, kF=lambda k: k not in self._ignoreVars))
utils.verbosity(newVerbosity + 3)
yield (path, metadata, events, seList, objStore)
def scriptOptions(parser, args = None, arg_keys = None): parser.addBool(None, ' ', 'parseable', default = False, help = 'Output tabular data in parseable format') parser.addBool(None, ' ', 'pivot', default = False, help = 'Output pivoted tabular data') parser.addText(None, ' ', 'textwidth', default = 100, help = 'Output tabular data with selected width') parser.addAccu(None, 'v', 'verbose', default = 0, help = 'Increase verbosity') (opts, args, config_dict) = parser.parse(args, arg_keys) logging.getLogger().setLevel(logging.DEFAULT - opts.verbose) utils.verbosity(opts.verbose) if opts.parseable: utils.printTabular.mode = 'parseable' elif opts.pivot: utils.printTabular.mode = 'longlist' utils.printTabular.wraplen = int(opts.textwidth) return utils.Result(opts = opts, args = args, config_dict = config_dict, parser = parser)
def parseOptions(parser):
parser.add_option('', '--parseable', dest='displaymode', const='parseable', action='store_const',
help='Output tabular data in parseable format')
parser.add_option('-P', '--pivot', dest='displaymode', const='longlist', action='store_const',
help='Output pivoted tabular data')
parser.add_option('', '--textwidth', dest='textwidth', default=100,
help='Output tabular data with selected width')
parser.add_option('-v', '--verbose', dest='verbosity', default=0, action='count',
help='Increase verbosity')
(opts, args) = parser.parse_args()
logging.getLogger().setLevel(logging.DEFAULT - opts.verbosity)
utils.verbosity(opts.verbosity)
utils.printTabular.mode = opts.displaymode
utils.printTabular.wraplen = int(opts.textwidth)
return (opts, args)
def __init__(self, config):
InfoScanner.__init__(self, config)
newVerbosity = utils.verbosity(utils.verbosity() - 3)
extConfigFN = config.getPath('source config')
extConfig = createConfig(extConfigFN).changeView(
setSections=['global'])
self._extWorkDir = extConfig.getWorkPath()
self._extTask = extConfig.getPlugin(['task', 'module'],
cls='TaskModule')
selector = config.get('source job selector', '')
extJobDB = JobDB(
extConfig,
jobSelector=lambda jobNum, jobObj: jobObj.state == Job.SUCCESS)
self._selected = sorted(
extJobDB.getJobs(JobSelector.create(selector, task=self._extTask)))
utils.verbosity(newVerbosity + 3)
def _update(self, jobObj, jobNum, state, showWMS = False):
if jobObj.state == state:
return
oldState = jobObj.state
jobObj.update(state)
self.jobDB.commit(jobNum, jobObj)
jobNumLen = int(math.log10(max(1, len(self.jobDB))) + 1)
jobStatus = ['Job %s state changed from %s to %s ' % (str(jobNum).ljust(jobNumLen), Job.enum2str(oldState), Job.enum2str(state))]
if showWMS and jobObj.wmsId:
jobStatus.append('(WMS:%s)' % jobObj.wmsId.split('.')[1])
if (state == Job.SUBMITTED) and (jobObj.attempt > 1):
jobStatus.append('(retry #%s)' % (jobObj.attempt - 1))
elif (state == Job.QUEUED) and (jobObj.get('dest') != 'N/A'):
jobStatus.append('(%s)' % jobObj.get('dest'))
elif (state in [Job.WAITING, Job.ABORTED, Job.DISABLED]) and jobObj.get('reason'):
jobStatus.append('(%s)' % jobObj.get('reason'))
elif (state == Job.SUCCESS) and (jobObj.get('runtime', None) is not None):
jobStatus.append('(runtime %s)' % strTime(jobObj.get('runtime') or 0))
elif state == Job.FAILED:
msg = []
retCode = jobObj.get('retcode')
if retCode:
msg.append('error code: %d' % retCode)
if (utils.verbosity() > 0) and (retCode in self._task.errorDict):
msg.append(self._task.errorDict[retCode])
if jobObj.get('dest'):
msg.append(jobObj.get('dest'))
if len(msg):
jobStatus.append('(%s)' % str.join(' - ', msg))
self._log_user_time.info(str.join(' ', jobStatus))
def getJobConfig(self, jobNum):
data = CMSSW.getJobConfig(self, jobNum)
nickdata = self.getVarsForNick(data.get("DATASETNICK"))
data.update(nickdata)
data["LUMI_RANGE"] = self.getActiveLumiFilter(data["LUMI_RANGE"], jobNum)
if utils.verbosity() > 0:
utils.vprint("Nickname: %s" % data.get("DATASETNICK"), 1)
utils.vprint(" * Config files: %s" % data["CMSSW_CONFIG"], 1)
utils.vprint(" * Lumi range: %s" % data["LUMI_RANGE"], 1)
utils.vprint(
" * Variables: %s" % utils.filterDict(nickdata, lambda k: k not in ["CMSSW_CONFIG", "LUMI_RANGE"]), 1
)
return data
def display(self):
reports = []
for jobNum in self._jobs:
jobObj = self._jobDB.get(jobNum)
if not jobObj or (jobObj.state == Job.INIT):
continue
reports.append({0: jobNum, 1: Job.enum2str(jobObj.state), 2: jobObj.wmsId})
if utils.verbosity() > 0:
history = jobObj.history.items()
history.reverse()
for at, dest in history:
if dest != 'N/A':
reports.append({1: at, 2: ' -> ' + dest})
elif jobObj.get('dest', 'N/A') != 'N/A':
reports.append({2: ' -> ' + jobObj.get('dest')})
utils.printTabular(lzip(irange(3), ['Job', 'Status / Attempt', 'Id / Destination']), reports, 'rcl')
def display(self):
reports = []
for jobNum in self._jobs:
jobObj = self._jobDB.get(jobNum)
if not jobObj or (jobObj.state == Job.INIT):
continue
reports.append({0: jobNum, 1: Job.enum2str(jobObj.state), 2: jobObj.wmsId})
if utils.verbosity() > 0:
history = jobObj.history.items()
history.reverse()
for at, dest in history:
if dest != "N/A":
reports.append({1: at, 2: " -> " + dest})
elif jobObj.get("dest", "N/A") != "N/A":
reports.append({2: " -> " + jobObj.get("dest")})
utils.printTabular(zip(range(3), ["Job", "Status / Attempt", "Id / Destination"]), reports, "rcl")
utils.vprint()
def _update(self, jobObj, jobNum, state, showWMS=False):
if jobObj.state == state:
return
oldState = jobObj.state
jobObj.update(state)
self.jobDB.commit(jobNum, jobObj)
jobNumLen = int(math.log10(max(1, len(self.jobDB))) + 1)
jobStatus = [
'Job %s state changed from %s to %s ' %
(str(jobNum).ljust(jobNumLen), Job.enum2str(oldState),
Job.enum2str(state))
]
if showWMS and jobObj.wmsId:
jobStatus.append('(WMS:%s)' % jobObj.wmsId.split('.')[1])
if (state == Job.SUBMITTED) and (jobObj.attempt > 1):
jobStatus.append('(retry #%s)' % (jobObj.attempt - 1))
elif (state == Job.QUEUED) and (jobObj.get('dest') != 'N/A'):
jobStatus.append('(%s)' % jobObj.get('dest'))
elif (state in [Job.WAITING, Job.ABORTED, Job.DISABLED
]) and jobObj.get('reason'):
jobStatus.append('(%s)' % jobObj.get('reason'))
elif (state == Job.SUCCESS) and (jobObj.get('runtime', None)
is not None):
jobStatus.append('(runtime %s)' %
strTime(jobObj.get('runtime') or 0))
elif state == Job.FAILED:
msg = []
retCode = jobObj.get('retcode')
if retCode:
msg.append('error code: %d' % retCode)
if (utils.verbosity() > 0) and (retCode
in self._task.errorDict):
msg.append(self._task.errorDict[retCode])
if jobObj.get('dest'):
msg.append(jobObj.get('dest'))
if len(msg):
jobStatus.append('(%s)' % str.join(' - ', msg))
self._log_user_time.info(str.join(' ', jobStatus))
def printInfoForJob(splitInfo):
if splitInfo.get(DataSplitter.Invalid, False):
utils.vprint(' Status: Invalidated splitting', -1)
utils.vprint(('Dataset: %s' % splitInfo[DataSplitter.Dataset]).ljust(50), -1, newline = False)
utils.vprint(('Events: %d' % splitInfo[DataSplitter.NEntries]).ljust(20), -1, newline = False)
utils.vprint(' ID: %s' % splitInfo.get(DataSplitter.DatasetID, 0), -1)
utils.vprint((' Block: %s' % splitInfo.get(DataSplitter.BlockName, 0)).ljust(50), -1, newline = False)
utils.vprint((' Skip: %d' % splitInfo.get(DataSplitter.Skipped, 0)).ljust(20), -1, newline = False)
if splitInfo.get(DataSplitter.Nickname):
utils.vprint('Nick: %s' % splitInfo[DataSplitter.Nickname], -1)
else:
utils.vprint('', -1)
if splitInfo.get(DataSplitter.Locations) != None:
utils.vprint('Locations: %s' % utils.wrapList(splitInfo[DataSplitter.Locations], 70, ',\n '), -1)
for idx, head in enumerate(splitInfo.get(DataSplitter.MetadataHeader, [])):
oneFileMetadata = map(lambda x: repr(x[idx]), splitInfo[DataSplitter.Metadata])
utils.vprint('%7s: %s' % (head, utils.wrapList(oneFileMetadata, 70, ',\n ')), -1)
utils.vprint(' Files: ', -1, newline = False)
if utils.verbosity() > 2:
utils.vprint(str.join('\n ', splitInfo[DataSplitter.FileList]), -1)
else:
utils.vprint('%d files selected' % len(splitInfo[DataSplitter.FileList]), -1)
def display(self):
reports = []
for jobNum in self._jobs:
jobObj = self._jobDB.get(jobNum)
if not jobObj or (jobObj.state == Job.INIT):
continue
reports.append({
0: jobNum,
1: Job.enum2str(jobObj.state),
2: jobObj.wmsId
})
if utils.verbosity() > 0:
history = jobObj.history.items()
history.reverse()
for at, dest in history:
if dest != 'N/A':
reports.append({1: at, 2: ' -> ' + dest})
elif jobObj.get('dest', 'N/A') != 'N/A':
reports.append({2: ' -> ' + jobObj.get('dest')})
utils.printTabular(
lzip(irange(3), ['Job', 'Status / Attempt', 'Id / Destination']),
reports, 'rcl')
def _update(self, jobObj, jobNum, state, showWMS = False):
if jobObj.state == state:
return
oldState = jobObj.state
jobObj.update(state)
self.jobDB.commit(jobNum, jobObj)
jobNumLen = int(math.log10(max(1, len(self.jobDB))) + 1)
utils.vprint('Job %s state changed from %s to %s ' % (str(jobNum).ljust(jobNumLen), Job.enum2str(oldState), Job.enum2str(state)), -1, True, False)
if showWMS and jobObj.wmsId:
print '(WMS:%s)' % jobObj.wmsId.split('.')[1],
if (state == Job.SUBMITTED) and (jobObj.attempt > 1):
print '(retry #%s)' % (jobObj.attempt - 1)
elif (state == Job.QUEUED) and jobObj.get('dest') != 'N/A':
print '(%s)' % jobObj.get('dest')
elif (state in [Job.WAITING, Job.ABORTED, Job.DISABLED]) and jobObj.get('reason'):
print '(%s)' % jobObj.get('reason')
elif (state == Job.SUCCESS) and jobObj.get('runtime', None) != None:
print '(runtime %s)' % utils.strTime(utils.QM(jobObj.get('runtime') != '', jobObj.get('runtime'), 0))
elif (state == Job.FAILED):
msg = []
if jobObj.get('retcode'):
msg.append('error code: %d' % jobObj.get('retcode'))
try:
if utils.verbosity() > 0:
msg.append(self._task.errorDict[jobObj.get('retcode')])
except Exception:
pass
if jobObj.get('dest'):
msg.append(jobObj.get('dest'))
if len(msg):
print '(%s)' % str.join(' - ', msg),
print
else:
print
def fillParameterInfo(self, pNum, result):
splitInfo = self.dataSplitter.getSplitInfo(pNum)
if utils.verbosity() > 2:
utils.vprint('Dataset task number: %d' % pNum)
DataSplitter.printInfoForJob(splitInfo)
self.dataProc.process(pNum, splitInfo, result)
parser.add_option('-v', '--verbose', dest='verbosity', default=0, action='count')
parser.add_option('-G', '--gui', dest='gui', action='store_const', const = 'ANSIGUI')
parser.add_option('-W', '--webserver', dest='gui', action='store_const', const = 'CPWebserver')
# Deprecated options - refer to new report script instead
parser.add_option('-r', '--report', dest='old_report', default=False, action='store_true')
parser.add_option('-R', '--site-report', dest='old_report', default=False, action='store_true')
parser.add_option('-T', '--time-report', dest='old_report', default=False, action='store_true')
parser.add_option('-M', '--task-report', dest='old_report', default=False, action='store_true')
parser.add_option('-D', '--detail-report', dest='old_report', default=False, action='store_true')
parser.add_option('', '--help-vars', dest='old_report', default=False, action='store_true')
(opts, args) = parser.parse_args()
if opts.help:
utils.eprint('%s\n%s' % (usage, open(utils.pathShare('help.txt'), 'r').read()))
sys.exit(os.EX_USAGE)
utils.verbosity(opts.verbosity)
logging.getLogger().setLevel(logging.DEFAULT - opts.verbosity)
if opts.debug:
logging.getLogger('exception').addHandler(logging.StreamHandler(sys.stdout))
# we need exactly one positional argument (config file)
if len(args) != 1:
utils.exitWithUsage(usage, 'Config file not specified!')
if opts.old_report:
utils.deprecated('Please use the more versatile report tool in the scripts directory!')
# Config filler which collects data from command line arguments
class OptsConfigFiller(ConfigFiller):
def __init__(self, optParser):
self._optParser = optParser