def test_wrongMPoutputDir(self):
dataDict = {
'ESD':
argFile(
"data15_13TeV.00267167.physics_Main.recon.ESD.f594._lb0176._SFO-1._0002"
),
'HIST_ESD_INT':
argFile("tmp.HIST_ESD_INT"),
'DRAW_EMU':
argFile(
"data15_13TeV.00267167.physics_Main.recon.DRAW_EMU.f594._lb0176._SFO-1._0002"
),
'DRAW_EGZ':
argFile(
"data15_13TeV.00267167.physics_Main.recon.DRAW_EGZ.f594._lb0176._SFO-1._0002"
),
'DRAW_TAUMUH':
argFile(
"data15_13TeV.00267167.physics_Main.recon.DRAW_TAUMUH.f594._lb0176._SFO-1._0002"
),
'DRAW_NOTHERE':
argFile(
"data15_13TeV.00267167.physics_Main.recon.DRAW_NOTHERE.f594._lb0176._SFO-1._0002"
),
'DRAW_ZMUMU':
argFile(
"data15_13TeV.00267167.physics_Main.recon.DRAW_ZMUMU.f594._lb0176._SFO-1._0002"
),
}
self.assertRaises(trfExceptions.TransformExecutionException,
athenaMPOutputHandler,
"athenaMP-outputs-RAWtoESD-r2e-missing",
"athenaMP-workers-RAWtoESD-r2e", dataDict, 20)
def test_basicMPoutputs(self):
dataDict = {
'BS':
argFile(
"data15_13TeV.00267167.physics_Main.merge.RAW._lb0176._SFO-1._0001.1",
io="input"),
'ESD':
argFile(
"data15_13TeV.00267167.physics_Main.recon.ESD.f594._lb0176._SFO-1._0002"
),
'HIST_ESD_INT':
argFile("tmp.HIST_ESD_INT"),
'DRAW_EMU':
argFile(
"data15_13TeV.00267167.physics_Main.recon.DRAW_EMU.f594._lb0176._SFO-1._0002"
),
'DRAW_EGZ':
argFile(
"data15_13TeV.00267167.physics_Main.recon.DRAW_EGZ.f594._lb0176._SFO-1._0002"
),
'DRAW_TAUMUH':
argFile(
"data15_13TeV.00267167.physics_Main.recon.DRAW_TAUMUH.f594._lb0176._SFO-1._0002"
),
'DRAW_ZMUMU':
argFile(
"data15_13TeV.00267167.physics_Main.recon.DRAW_ZMUMU.f594._lb0176._SFO-1._0002"
),
}
self.assertEqual(
athenaMPOutputHandler("athenaMP-outputs-RAWtoESD-r2e",
"athenaMP-workers-RAWtoESD-r2e", dataDict,
8), None)
def execute(self):
msg.debug('Entering transform execution phase')
try:
# Intercept a few special options here
if 'dumpargs' in self._argdict:
self.parser.dumpArgs()
sys.exit(0)
# Graph stuff!
msg.info('Resolving execution graph')
self._setupGraph()
if 'showSteps' in self._argdict:
for exe in self._executors:
print "Executor Step: {0} (alias {1})".format(
exe.name, exe.substep)
if msg.level <= logging.DEBUG:
print " {0} -> {1}".format(exe.inData, exe.outData)
sys.exit(0)
if 'showGraph' in self._argdict:
print self._executorGraph
sys.exit(0)
# Graph stuff!
msg.info('Starting to trace execution path')
self._tracePath()
msg.info('Execution path found with {0} step(s): {1}'.format(
len(self._executorPath),
' '.join([exe['name'] for exe in self._executorPath])))
if 'showPath' in self._argdict:
msg.debug('Execution path list is: {0}'.format(
self._executorPath))
# Now print it nice
print 'Executor path is:'
for node in self._executorPath:
print ' {0}: {1} -> {2}'.format(node['name'],
list(node['input']),
list(node['output']))
sys.exit(0)
msg.debug('Execution path is {0}'.format(self._executorPath))
# Prepare files for execution (separate method?)
for dataType in [
data for data in self._executorGraph.data
if 'NULL' not in data
]:
if dataType in self._dataDictionary:
msg.debug(
'Data type {0} maps to existing argument {1}'.format(
dataType, self._dataDictionary[dataType]))
else:
fileName = 'tmp.' + dataType
# How to pick the correct argFile class?
for (prefix, suffix) in (('tmp', ''), ('output', 'File'),
('input', 'File')):
stdArgName = prefix + dataType + suffix
if stdArgName in self.parser._argClass:
msg.debug(
'Matched data type {0} to argument {1}'.format(
dataType, stdArgName))
self._dataDictionary[
dataType] = self.parser._argClass[stdArgName](
fileName)
self._dataDictionary[dataType].io = 'temporary'
break
if dataType not in self._dataDictionary:
if 'HIST' in fileName:
self._dataDictionary[dataType] = argHISTFile(
fileName,
io='temporary',
type=dataType.lower())
else:
self._dataDictionary[dataType] = argFile(
fileName,
io='temporary',
type=dataType.lower())
msg.debug(
'Did not find any argument matching data type {0} - setting to plain argFile: {1}'
.format(dataType,
self._dataDictionary[dataType]))
self._dataDictionary[dataType].name = fileName
# Now we can set the final executor configuration properly, with the final dataDictionary
for executor in self._executors:
executor.conf.setFromTransform(self)
self.validateInFiles()
for executionStep in self._executorPath:
msg.debug('Now preparing to execute {0}'.format(executionStep))
executor = self._executorDictionary[executionStep['name']]
executor.preExecute(input=executionStep['input'],
output=executionStep['output'])
try:
executor.execute()
executor.postExecute()
finally:
executor.validate()
self._processedEvents = self.getProcessedEvents()
self.validateOutFiles()
msg.debug('Transform executor succeeded')
self._exitCode = 0
self._exitMsg = trfExit.codeToName(self._exitCode)
except trfExceptions.TransformNeedCheckException as e:
msg.warning(
'Transform executor signaled NEEDCHECK condition: {0}'.format(
e.errMsg))
self._exitCode = e.errCode
self._exitMsg = e.errMsg
self.generateReport(fast=False)
except trfExceptions.TransformException as e:
msg.critical('Transform executor raised %s: %s' %
(e.__class__.__name__, e.errMsg))
self._exitCode = e.errCode
self._exitMsg = e.errMsg
# Try and write a job report...
self.generateReport(fast=True)
finally:
# Clean up any orphaned processes and exit here if things went bad
infanticide(message=True)
if self._exitCode:
msg.warning(
'Transform now exiting early with exit code {0} ({1})'.
format(e.errCode, e.errMsg))
sys.exit(self._exitCode)
def addOutputArgs(parser,dict):
if not 'part' in dict:
checkPart = ['doNoisyStrip']
else:
checkPart = dict['part']._value
if not 'splitNoisyStrip' in dict:
checkSplit= 0
else:
checkSplit = dict['splitNoisyStrip']._value
if not 'prefix' in dict:
checkPrefix=''
else:
checkPrefix=dict['prefix']._value
# # # add outputfiles
if checkPrefix != '':
checkPrefix += '.'
if not 'doHV' in checkPart and checkSplit != 1:
parser.add_argument('--outputCOOL', type=trfArgClasses.argFactory(trfArgClasses.argFile, runarg=True,io='output'),
help = 'COOL DB',group='Calibration',default=trfArgClasses.argFile([checkPrefix+'mycool.db'],runarg=True))
#NOISY STRIPS OUTPUT FILES
if 'doNoisyStrip' in checkPart and checkSplit == 1:
parser.add_argument('--outputHITMapFile', type=trfArgClasses.argFactory(trfArgClasses.argNTUPFile, runarg=True,io='output'),
help = 'HitMap output file',group='Calibration',default=trfArgClasses.argNTUPFile([checkPrefix+'SCTHitMaps.root'],runarg=True))
parser.add_argument('--outputLBFile', type=trfArgClasses.argFactory(trfArgClasses.argNTUPFile, runarg=True,io='output'),
help = 'LB output file',group='Calibration',default=trfArgClasses.argNTUPFile([checkPrefix+'SCTLB.root'],runarg=True))
if 'doNoisyStrip' in checkPart and checkSplit != 1:
parser.add_argument('--outputHITMapFile', type=trfArgClasses.argFactory(trfArgClasses.argNTUPFile, runarg=True,io='output'),
help = 'HitMap output file',group='Calibration',default=trfArgClasses.argNTUPFile([checkPrefix+'SCTHitMaps.root'],runarg=True))
parser.add_argument('--outputLBFile', type=trfArgClasses.argFactory(trfArgClasses.argNTUPFile, runarg=True,io='output'),
help = 'LB output file',group='Calibration',default=trfArgClasses.argNTUPFile([checkPrefix+'SCTLB.root'],runarg=True))
parser.add_argument('--outputBSSummaryFile', type=trfArgClasses.argFactory(trfArgClasses.argFile, runarg=True,io='output'),
help = 'Bad Strips summary file',group='Calibration',default=trfArgClasses.argFile([checkPrefix+'BadStripsSummaryFile.xml'],runarg=True))
parser.add_argument('--outputBSAllFile', type=trfArgClasses.argFactory(trfArgClasses.argFile, runarg=True,io='output'),
help = 'Bad Strips All file',group='Calibration',default=trfArgClasses.argFile([checkPrefix+'BadStripsAllFile.xml'],runarg=True))
parser.add_argument('--outputBSNewFile', type=trfArgClasses.argFactory(trfArgClasses.argFile, runarg=True,io='output'),
help = 'Bad Strips New file',group='Calibration',default=trfArgClasses.argFile([checkPrefix+'BadStripsNewFile.xml'],runarg=True))
if 'doHV' in checkPart:
parser.add_argument('--outputBadModulesFile', type=trfArgClasses.argFactory(trfArgClasses.argFile, runarg=True,io='output'),
help = 'Bad Modules file',group='Calibration',default=trfArgClasses.argFile([checkPrefix+'BadModulesFile.xml'],runarg=True))
#DEAD CHIP OUTPUT FILES
if 'doDeadChip' in checkPart:
parser.add_argument('--outputHITMapFile', type=trfArgClasses.argFactory(trfArgClasses.argNTUPFile, runarg=True,io='output'),
help = 'HitMap output file',group='Calibration',default=trfArgClasses.argNTUPFile([checkPrefix+'SCTHitMaps.root'],runarg=True))
parser.add_argument('--outputDeadChipFile', type=trfArgClasses.argFactory(trfArgClasses.argFile, runarg=True,io='output'),
help = 'Dead Chip file',group='Calibration',default=trfArgClasses.argFile([checkPrefix+'DeadChipsFile.xml'],runarg=True))
parser.add_argument('--outputDeadSummaryFile', type=trfArgClasses.argFactory(trfArgClasses.argFile, runarg=True,io='output'),
help = 'Dead Chip Summary file',group='Calibration',default=trfArgClasses.argFile([checkPrefix+'DeadSummaryFile.xml'],runarg=True))
parser.add_argument('--outputBSErrorsFile', type=trfArgClasses.argFactory(trfArgClasses.argNTUPFile, runarg=True,io='output'),
help = 'BS Errors file',group='Calibration',default=trfArgClasses.argNTUPFile([checkPrefix+'SCTBSErrors.root'],runarg=True))
#DEAD STRIP OUTPUT FILES
if 'doDeadStrip' in checkPart:
parser.add_argument('--outputHITMapFile', type=trfArgClasses.argFactory(trfArgClasses.argNTUPFile, runarg=True,io='output'),
help = 'HitMap output file',group='Calibration',default=trfArgClasses.argNTUPFile([checkPrefix+'SCTHitMaps.root'],runarg=True))
parser.add_argument('--outputDeadStripFile', type=trfArgClasses.argFactory(trfArgClasses.argFile, runarg=True,io='output'),
help = 'Dead Strip file',group='Calibration',default=trfArgClasses.argFile([checkPrefix+'DeadStripsFile.xml'],runarg=True))
parser.add_argument('--outputDeadSummaryFile', type=trfArgClasses.argFactory(trfArgClasses.argFile, runarg=True,io='output'),
help = 'Dead Strip Summary file',group='Calibration',default=trfArgClasses.argFile([checkPrefix+'DeadSummaryFile.xml'],runarg=True))
parser.add_argument('--outputBSErrorsFile', type=trfArgClasses.argFactory(trfArgClasses.argNTUPFile, runarg=True,io='output'),
help = 'BS Errors file',group='Calibration',default=trfArgClasses.argNTUPFile([checkPrefix+'SCTBSErrors.root'],runarg=True))
#NOISE OCCUPANCY OUTPUT FILES
if 'doNoiseOccupancy' in checkPart:
parser.add_argument('--outputNOFile', type=trfArgClasses.argFactory(trfArgClasses.argFile, runarg=True,io='output'),
help = 'Noise Occupancy file',group='Calibration',default=trfArgClasses.argFile([checkPrefix+'NoiseOccupancyFile.xml'],runarg=True))
parser.add_argument('--outputNOSummaryFile', type=trfArgClasses.argFactory(trfArgClasses.argFile, runarg=True,io='output'),
help = 'Noise Occupancy Summary file',group='Calibration',default=trfArgClasses.argFile([checkPrefix+'NoiseOccupancySummaryFile.xml'],runarg=True))
#LORENTZ ANGLE OUTPUT FILES
if 'doLorentzAngle' in checkPart:
parser.add_argument('--outputLAFile', type=trfArgClasses.argFactory(trfArgClasses.argFile, runarg=True,io='output'),
help = 'Lorentz Angle file',group='Calibration',default=trfArgClasses.argFile([checkPrefix+'LorentzAngleFile.xml'],runarg=True))
parser.add_argument('--outputLASummaryFile', type=trfArgClasses.argFactory(trfArgClasses.argFile, runarg=True,io='output'),
help = 'Lorentz Angle Summary file',group='Calibration',default=trfArgClasses.argFile([checkPrefix+'LorentzAngleSummaryFile.xml'],runarg=True))
#RAW OCCUPANCY OUTPUT FILES
if 'doRawOccupancy' in checkPart:
parser.add_argument('--outputROSummaryFile', type=trfArgClasses.argFactory(trfArgClasses.argFile, runarg=True,io='output'),
help = 'Raw Occupancy Summary file',group='Calibration',default=trfArgClasses.argFile([checkPrefix+'RawOccupancySummaryFile.xml'],runarg=True))
#EFFICIENCY OUTPUT FILES
if 'doEfficiency' in checkPart:
parser.add_argument('--outputEffModuleFile', type=trfArgClasses.argFactory(trfArgClasses.argFile, runarg=True,io='output'),
help = 'Efficiency file',group='Calibration',default=trfArgClasses.argFile([checkPrefix+'EfficiencyModuleSummary.xml'],runarg=True))
parser.add_argument('--outputEffSummaryFile', type=trfArgClasses.argFactory(trfArgClasses.argFile, runarg=True,io='output'),
help = 'Efficiency Summary file',group='Calibration',default=trfArgClasses.argFile([checkPrefix+'EfficiencySummaryFile.xml'],runarg=True))
#BS ERRORS OUTPUT FILES
if 'doBSErrorDB' in checkPart:
parser.add_argument('--outputBSModuleFile', type=trfArgClasses.argFactory(trfArgClasses.argFile, runarg=True,io='output'),
help = 'BS Errors Summary file',group='Calibration',default=trfArgClasses.argFile([checkPrefix+'BSErrorModuleSummary.xml'],runarg=True))
parser.add_argument('--outputBSSummaryFile', type=trfArgClasses.argFactory(trfArgClasses.argFile, runarg=True,io='output'),
help = 'BS Errors Summary file',group='Calibration',default=trfArgClasses.argFile([checkPrefix+'BSErrorSummaryFile.xml'],runarg=True))
def addArgs(parser):
# N.B. Although the trf does not generate these files, badge them as 'output' to run full validation
parser.add_argument('--outputBSFile',
type=trfArgClasses.argFactory(trfArgClasses.argBSFile,
io='output',
type='bs',
multipleOK=True),
nargs='+')
parser.add_argument('--outputESDFile',
type=trfArgClasses.argFactory(
trfArgClasses.argPOOLFile,
io='output',
type='esd',
multipleOK=True),
nargs='+')
parser.add_argument('--outputAODFile',
type=trfArgClasses.argFactory(
trfArgClasses.argPOOLFile,
io='output',
type='aod',
multipleOK=True),
nargs='+')
parser.add_argument('--outputRDOFile',
type=trfArgClasses.argFactory(
trfArgClasses.argPOOLFile,
io='output',
type='rdo',
multipleOK=True),
nargs='+')
parser.add_argument('--outputTAGFile',
type=trfArgClasses.argFactory(trfArgClasses.argTAGFile,
io='output',
type='tag',
multipleOK=True),
nargs='+')
parser.add_argument('--outputEVNTFile',
type=trfArgClasses.argFactory(
trfArgClasses.argEVNTFile,
io='output',
type='evnt',
multipleOK=True),
nargs='+')
parser.add_argument('--outputHISTFile',
type=trfArgClasses.argFactory(
trfArgClasses.argHISTFile,
io='output',
type='hist',
multipleOK=True),
nargs='+')
parser.add_argument('--outputTXT_FTKIPFile',
type=trfArgClasses.argFactory(
trfArgClasses.argFTKIPFile,
io='output',
multipleOK=True),
nargs='+')
parser.add_argument('--outputNTUP_FTKSIMFile',
type=trfArgClasses.argFactory(
trfArgClasses.argNTUPFile,
io='output',
type='ntup_ftksim',
treeNames=['ftkdata'],
multipleOK=True),
nargs='+')
parser.add_argument('--outputEvtFile',
type=trfArgClasses.argFactory(
trfArgClasses.argHepEvtAsciiFile,
io='output',
type='evt',
multipleOK=True),
nargs='+')
parser.add_argument('--logfile',
type=trfArgClasses.argFactory(trfArgClasses.argString))
# This is a dummy argument which makes the graph tracer code happy
parser.add_argument('--inputFile',
type=trfArgClasses.argFactory(trfArgClasses.argFile,
io='input',
type='dummy'),
default=trfArgClasses.argFile(type='dummy'))
