def main(opts, args):
# try to open config file
config = getConfig(args[0], section='global')
# Initialise task module
task = None
if opts.use_task:
task = config.getPlugin(['task', 'module'], cls='TaskModule')
# Initialise job database
jobDB = config.getPlugin('job database', 'JobDB', cls='JobDB')
activity = utils.ActivityLog('Filtering job entries')
selected = jobDB.getJobs(JobSelector.create(opts.job_selector, task=task))
activity.finish()
report = Report.createInstance(opts.report, jobDB, task, selected,
opts.string)
report.display()
def lumi_calc(opts, workDir, jobList, splitter):
(lumiDict, readDict, writeDict) = process_jobs(opts, workDir, jobList,
splitter)
activity = utils.ActivityLog('Simplifying lumi sections')
lumis = {}
for sample in lumiDict:
for run in lumiDict[sample]:
for lumi in lumiDict[sample][run]:
lumis.setdefault(sample, []).append(([run, lumi], [run, lumi]))
for sample in lumiDict:
lumis[sample] = mergeLumi(lumis[sample])
activity.finish()
for sample, lumi_list in lumis.items():
print('Sample: %s' % sample)
if opts.job_events:
print('=========================================')
print('Number of events processed: %12s' % readDict.get(sample))
print(' Number of events written: %12d' %
sum(writeDict.get(sample, {}).values()))
if writeDict.get(sample, None):
sys.stdout.write('\n')
head = [(0, ' Output filename'), (1, 'Events')]
utils.printTabular(
head,
lmap(lambda pfn: {
0: pfn,
1: writeDict[sample][pfn]
}, writeDict[sample]))
if opts.job_json:
json_fn = os.path.join(opts.output_dir or workDir,
'processed_%s.json' % sample)
outputJSON(lumi_list, open(json_fn, 'w'))
print('Saved processed lumi sections in ' + json_fn)
if opts.job_gc:
sys.stdout.write('\n')
print('List of processed lumisections:')
print('-----------------------------------------')
outputGC(lumi_list)
sys.stdout.write('\n')
def iter_jobs(opts, workDir, jobList, splitter):
(splitInfo, fip) = ({}, FileInfoProcessor())
for jobNum in jobList:
activity = utils.ActivityLog('Reading job logs - [%d / %d]' %
(jobNum, jobList[-1]))
if opts.parameterized:
fi = fip.process(os.path.join(workDir, 'output',
'job_%d' % jobNum))
outputName = fi[0][FileInfoProcessor.NameDest].split('.')[0]
outputName = outputName.replace(opts.replace % jobNum,
'_').replace('/', '_').replace(
'__', '_').strip('_')
else:
if splitter:
splitInfo = splitter.getSplitInfo(jobNum)
outputName = splitInfo.get(
DataSplitter.Nickname, splitInfo.get(DataSplitter.DatasetID,
0))
yield (jobNum, outputName)
activity.finish()