def testPerfMon(jobOptName="PerfMonTests/test_perfMonSvc_noopalg.py", perfMonFileName="perfmon.noopalg.root", evtMax=50000): print("") print("#" * 80) print("## testing PerfMonSvc [%s]" % jobOptName) print(" ==> [%s]... (%i)" % (perfMonFileName, evtMax)) perfMonFileName = workDir(perfMonFileName) refPerfMonFileName = "ref." + os.path.basename(perfMonFileName) chkPerfMonFileName = "chk." + os.path.basename(perfMonFileName) outPerfMonFileName = "ana." + os.path.basename(perfMonFileName) refPerfMonFileName = refPerfMonFileName.replace(".root", ".pmon.gz") chkPerfMonFileName = chkPerfMonFileName.replace(".root", ".pmon.gz") ## create the reference file athena = ChapPy.Athena(jobOptions=[ ChapPy.JobOptions(jobOptName), ChapPy.JobOptionsCmd("jobproperties.PerfMonFlags.OutputFile = '%s'" % refPerfMonFileName) ], checkLeak=False, logFile=refPerfMonFileName + ".log") athena.EvtMax = evtMax sc = athena.run() if sc != 0: print("ERROR: could not create the 'ref' perfmon file !!") return ScOutput(sc, "ERROR") ## create the to-be-checked file athena = ChapPy.Athena(jobOptions=[ ChapPy.JobOptions(jobOptName), ChapPy.JobOptionsCmd("jobproperties.PerfMonFlags.OutputFile = '%s'" % chkPerfMonFileName) ], checkLeak=False, logFile=chkPerfMonFileName + ".log") athena.EvtMax = evtMax sc = athena.run() if sc != 0: print("ERROR: could not create the 'chk' perfmon file !!") return ScOutput(sc, "ERROR") #outPerfMonFileName = "ana." + os.path.basename( perfMonFileName ) print(" :::running [perfmon.py]...") cmd = "perfmon.py %s %s -o %s" % \ ( chkPerfMonFileName, refPerfMonFileName, outPerfMonFileName ) sc, out = subprocess.getstatusoutput(cmd) if sc != 0: print("## Problem while doing [perfmon] !!") print(out) out = "ERROR" return ScOutput(sc, out) out = "OK" print("## [DONE]") return ScOutput(sc, out)
def installRefFiles(fileNames): for refFile in fileNames: for fileName in [refFile, workDir(refFile)]: if os.path.exists(fileName): os.remove(fileName) sc, out = subprocess.getstatusoutput("get_files %s" % refFile) if sc != 0: print("## ERROR: could not retrieve [%s]" % refFile) print("## reason:\n", out) continue if os.path.exists(refFile) and \ os.path.exists(workDir(refFile)) and \ os.path.samefile( refFile, workDir(refFile) ): print(" -%s" % workDir(refFile)) continue sc, out = subprocess.getstatusoutput("mv %s %s" % (refFile, workDir(refFile))) if sc != 0: print ("## ERROR: could not install [%s] into [%s]" %\ ( refFile, workDir(refFile) )) print("## reason:\n", out) continue else: print(" -%s" % workDir(refFile)) return
def installRefFiles( fileNames, quiet=True ): for refFile in fileNames: for fileName in [ refFile, workDir(refFile) ]: if os.path.exists(fileName): os.remove(fileName) sc,out = commands.getstatusoutput( "get_files %s" % refFile ) if sc != 0: print "## ERROR: could not retrieve [%s]" % refFile print "## reason:\n",out continue if os.path.exists(refFile) and \ os.path.exists(workDir(refFile)) and \ os.path.samefile( refFile, workDir(refFile) ): if not quiet: print " -%s" % workDir(refFile) continue try: shutil.move(refFile, workDir(refFile)) if not quiet: print " -%s" % workDir(refFile) except OSError,err: print "## ERROR: could not install [%s] into [%s]" %\ ( refFile, workDir(refFile) ) print "## reason:\n",err continue
print("#" * 80) print("## testing symlinks for TruthParticleContainer...") bench = BenchSequence("Symlinks for TruthParticleContainer") print("## installing reference files...") installRefFiles([ "mc.aod.symlinks.ref", "mc.aod.pysymlinks.ref", ]) evtMax = 5 ###----------------------------------------------------- print("## Testing [writing]...") jobOptions = [ ChapPy.JobOptionsCmd("OUTPUT=\"%s\"" % workDir("mc.aod.symlinks.pool")), ChapPy.JobOptions("McParticleTests/symlinkTest_mcAod_jobOptions.py"), ] athena = ChapPy.Athena(jobOptions=jobOptions, logFile=workDir("mc.aod.symlinks.pool.log"), checkLeak=False) athena.EvtMax = evtMax athena.run() ###----------------------------------------------------- print("\n") print("#" * 80) bench += doPostCheck("McAod symlinks", workDir("mc.aod.symlinks.ref"), workDir("mc.aod.symlinks.pool.log"), "grep \"^McAodSymLinkTests\"")
def doReadWriteTest(genName="TruthParticles", evtMax=100): ###----------------------------------------------------- print("\n") print("#" * 80) print("## Preparing input data... [%s]" % genName) templateJobO = """ OUTPUT='%(OutputFile)s'; DUMP=True; include( 'McParticleTests/iotest_WriteGenEvent_jobOptions.py' ); """ jobOptions = [ ChapPy.JobOptionsCmd(templateJobO % { 'OutputFile': workDir("mc.event.%s.pool" % uuid), }) ] athena = ChapPy.Athena(jobOptions=jobOptions, logFile=workDir("mc.event.%s.pool.log" % uuid), checkLeak=False) athena.EvtMax = evtMax sc = athena.run() if sc != 0: return ScOutput(sc, "ERROR") ###----------------------------------------------------- print("\n") print("#" * 80) print("## Testing [writing-%s]..." % genName) templateJobO = """ INPUT=['%(InputFile)s']; OUTPUT= '%(OutputFile)s'; DUMPTUPLE=True; TUPLEFILENAME='%(TupleFile)s'; include( 'McParticleTests/iotest_WriteTruthParticles_jobOptions.py' ); jobproperties.PerfMonFlags.OutputFile = '%(PmonFile)s'; """ jobOptions = [ ChapPy.JobOptionsCmd( templateJobO % { 'InputFile': workDir("mc.event.%s.pool" % uuid), 'OutputFile': outFiles['ref'], 'TupleFile': outFiles['ref'].replace(".pool", ".tuple.root"), 'PmonFile': workDir("write.mcaod.%s.%s.perfmon.pmon.gz" % (genName, uuid)), }) ] athena = ChapPy.Athena(jobOptions=jobOptions, logFile=outFiles['ref'] + ".log", checkLeak=False) athena.EvtMax = evtMax sc = athena.run() if sc != 0: return ScOutput(sc, "ERROR") else: inFile = workDir("write.mcaod.%s.%s.perfmon.pmon.gz" % (genName, uuid)) outFile = workDir("out.write.mcaod.%s.%s.perfmon.root" % (genName, uuid)) subprocess.getoutput("perfmon.py %s -o %s" % (inFile, outFile)) ###----------------------------------------------------- print("\n") print("#" * 80) print("## Testing [reading-%s]..." % genName) templateJobO = """ INPUT=['%(InputFile)s']; OUTPUT= '%(OutputFile)s'; DUMPTUPLE=True; TUPLEFILENAME='%(TupleFile)s'; include( 'McParticleTests/iotest_ReadTruthParticles_jobOptions.py' ); jobproperties.PerfMonFlags.OutputFile = '%(PmonFile)s'; """ jobOptions = [ ChapPy.JobOptionsCmd( templateJobO % { 'InputFile': outFiles['ref'], 'OutputFile': outFiles['chk'], 'TupleFile': outFiles['chk'].replace(".pool", ".tuple.root"), 'PmonFile': workDir("read.mcaod.%s.%s.perfmon.pmon.gz" % (genName, uuid)), }) ] athena = ChapPy.Athena(jobOptions=jobOptions, logFile=outFiles['chk'] + ".log", checkLeak=False) athena.EvtMax = evtMax sc = athena.run() if sc != 0: return ScOutput(sc, "ERROR") else: inFile = workDir("read.mcaod.%s.%s.perfmon.pmong.gz" % (genName, uuid)) outFile = workDir("out.read.mcaod.%s.%s.perfmon.root" % (genName, uuid)) subprocess.getoutput("perfmon.py %s -o %s" % (inFile, outFile)) ###----------------------------------------------------- print("\n") print("#" * 80) return doMcAodTupleValidation( outFiles['ref'].replace(".pool", ".tuple.root"), outFiles['chk'].replace(".pool", ".tuple.root"))
uuid = "%s_%s" % (os.getpid(), timeofday()) print("#" * 80) print("## testing TruthParticleContainer (read/write) persistency...") print("## Job uuid:", uuid) bench = BenchSequence("TruthParticleContainer (read/write) persistency") ###----------------------------------------------------- dataFilesDir = "/afs/cern.ch/atlas/offline/data/testfile" dataRefDir = "/afs/cern.ch/atlas/offline/test" asciiFiles = { 'w': { 'ref': os.path.join(dataRefDir, "mc.aod.etisols.rel_13.ref"), 'chk': workDir("mc.aod.txt"), }, 'r': { 'ref': os.path.join(dataRefDir, "reaccessed.mc.aod.etisols.rel_13.ref"), 'chk': workDir("reaccessed.mc.aod.txt"), }, } outFiles = { 'ref': workDir("mc.aod.%s.pool" % uuid), 'chk': workDir("reaccessed.mc.aod.%s.pool" % uuid) } evtMax = 100
###----------------------------------------------------- ## Little helper to validate output of jobs from TestTools.iobench import doValidation from TestTools.iobench import ScOutput from TestTools.iobench import BenchSequence EVTMAX = 100 NTIMES = 1 def timeofday(): from time import time return int(time()) uuid = "%s_%s" % (os.getpid(),timeofday()) genEventFile = workDir("mc.event.%s.pool"%uuid) mcAodFile = workDir("mc.aod.%s.pool"%uuid) print "#"*80 print "## testing TruthParticleContainer I/O..." print "## Job uuid:",uuid benchSequence = BenchSequence( "TruthParticleContainer I/O" ) ###----------------------------------------------------- if os.path.exists(genEventFile): print "## Re-using input data..." else: print "## Preparing input data..." jobOptions = [ ChapPy.JobOptionsCmd( "OUTPUT=\"%s\"" % genEventFile ), ChapPy.JobOptionsCmd( "DUMP=False" ),
from TestTools.iobench import BenchSequence print "#" * 80 print "## testing McEventCollection (read/back) persistency..." bench = BenchSequence("McEventCollection (read/back) persistency") ###----------------------------------------------------- dataFilesDir = "/afs/cern.ch/atlas/offline/data/testfile" dataRefDir = "/afs/cern.ch/atlas/offline/test" asciiFiles = { 'rel_1004': { 'ref': os.path.join(dataRefDir, 'mc.event.rel_1004.ref'), 'chk': workDir('mc.event.rel_1004.txt'), 'in': os.path.join( dataFilesDir, "q02initialprod.0001.H_2e2mu.q02dig_1004.etacut.0001_extract.pool.root" ) }, 'rel_11041': { 'ref': os.path.join(dataRefDir, 'mc.event.rel_11041.ref'), 'chk': workDir('mc.event.rel_11041.txt'), 'in': os.path.join( dataFilesDir, "mc11.004100.T1_McAtNLO_top.digit_test.RDO.v11000401._00001.pool.root"
uuid = "%s_%s" % (os.getpid(), timeofday()) print("#" * 80) print("## testing McEventCollection I/O...") print("## Job uuid:", uuid) benchSequence = BenchSequence("McEventCollection I/O") EVTMAX = 100 NTIMES = 1 ###----------------------------------------------------- print("## Testing [writing]...") jobOptions = [ ChapPy.JobOptionsCmd("OUTPUT=\"%s\"" % workDir("mc.io.event.%s.pool" % uuid)), ChapPy.JobOptionsCmd("DUMP=False"), ChapPy.JobOptions("McParticleTests/iotest_WriteGenEvent_jobOptions.py"), ] athena = ChapPy.Athena( jobOptions=jobOptions, #logFile = "/dev/stdout", checkLeak=False) athena.EvtMax = EVTMAX bench = iobench.AthBench( athena, nTimes=NTIMES, logFile=workDir("write.%s.log" % os.path.basename(workDir("mc.io.event.%s.pool" % uuid))))
def doReadWriteTest(genName="pythia", evtMax=100): """A simple wrapper around the read/write tests...""" genName = genName.lower() ###----------------------------------------------------- print "" print "#" * 80 print "## Testing [writing-%s]..." % genName templateJobO = """ OUTPUT='%(OutputFile)s'; DUMPTUPLE=True; GENERATOR='%(Generator)s'; TUPLEFILENAME='%(TupleFile)s'; include( 'McParticleTests/iotest_WriteGenEvent_jobOptions.py' ); jobproperties.PerfMonFlags.OutputFile = '%(PmonFile)s'; """ jobOptions = [ ChapPy.JobOptionsCmd( templateJobO % { 'OutputFile' : workDir("mc.event.%s.pool" % genName), 'Generator' : genName.upper(), 'TupleFile' : outFiles['gen_%s' % genName]['ref']\ .replace(".pool",".tuple.root"), 'PmonFile' : workDir("write.genevent.%s.pmon.gz" % genName), } ) ] athena = ChapPy.Athena(jobOptions=jobOptions, logFile=workDir("write.genevent.%s.log" % genName), checkLeak=False) athena.EvtMax = evtMax sc = athena.run() if sc != 0: return ScOutput(sc, "ERROR") else: inFile = workDir("write.genevent.%s.pmon.gz" % genName) outFile = workDir("out.write.genevent.%s.pmon.root" % genName) print commands.getoutput("perfmon.py %s -o %s" % (inFile, outFile)) ###----------------------------------------------------- print "" print "#" * 80 print "## Testing [ASCII-writing-%s (1)]..." % genName templateJobO = """ INPUT=%(InputFile)s; OUTPUT='%(OutputFile)s'; DUMPTUPLE=True; GENERATOR='%(Generator)s'; include( 'McParticleAlgs/GenEventAsciiWriter_jobOptions.py' ); """ jobOptions = [ ChapPy.JobOptionsCmd( templateJobO % { 'InputFile': [ workDir("mc.event.%s.pool" % genName), ], 'OutputFile': workDir("mc.event.%s.1.ascii" % genName), 'Generator': genName.upper(), }) ] athena = ChapPy.Athena(jobOptions=jobOptions, logFile=workDir("write.genevent.ascii.1.%s.log" % genName), checkLeak=False) athena.EvtMax = evtMax sc = athena.run() if sc != 0: return ScOutput(sc, "ERROR") ###----------------------------------------------------- print "" print "#" * 80 print "## Testing [ASCII-writing-%s (2)]..." % genName templateJobO = """ INPUT=%(InputFile)s; OUTPUT='%(OutputFile)s'; DUMPTUPLE=True; GENERATOR='%(Generator)s'; include( 'McParticleAlgs/GenEventAsciiWriter_jobOptions.py' ); """ jobOptions = [ ChapPy.JobOptionsCmd( templateJobO % { 'InputFile': [ workDir("mc.event.%s.pool" % genName), ], 'OutputFile': workDir("mc.event.%s.2.ascii" % genName), 'Generator': genName.upper(), }) ] athena = ChapPy.Athena(jobOptions=jobOptions, logFile=workDir("write.genevent.ascii.2.%s.log" % genName), checkLeak=False) athena.EvtMax = evtMax sc = athena.run() if sc != 0: return ScOutput(sc, "ERROR") ###----------------------------------------------------- print "\n" print "#" * 80 print "## Testing [ASCII-reading-%s]..." % genName templateJobO = """ INPUT=%(InputFiles)s; include( 'McAsciiEventSelector/Example_McAsciiReader_jobOptions.py' ); """ jobOptions = [ ChapPy.JobOptionsCmd( templateJobO % { 'InputFiles' : [ workDir("mc.event.%s.1.ascii" % genName), workDir("mc.event.%s.2.ascii" % genName),], 'TupleFile' : outFiles['gen_%s' % genName]['chk']\ .replace(".pool",".tuple.root"), 'PmonFile' : workDir("read.genevent.%s.pmon.gz" % genName), } ) ] athena = ChapPy.Athena(jobOptions=jobOptions, logFile=workDir("read.genevent.%s.log" % genName), checkLeak=False) athena.EvtMax = evtMax sc = athena.run() if sc != 0: return ScOutput(sc, "ERROR") ###----------------------------------------------------- print "\n" print "#" * 80 return ScOutput(0, "OK")
###----------------------------------------------------- ## Little helper to validate output of jobs from TestTools.iobench import ScOutput from TestTools.iobench import BenchSequence from McParticleTests.tests import doTupleValidation print "#" * 80 print "## testing McEventCollection (read/write) persistency..." print "#" * 80 bench = BenchSequence("McEventCollection (read/write) ASCII persistency") ###----------------------------------------------------- outFiles = { 'gen_pythia': { 'ref': workDir("mc.event.pythia.pool"), 'chk': workDir("reaccessed.mc.event.pythia.pool"), }, 'gen_herwig': { 'ref': workDir("mc.event.herwig.pool"), 'chk': workDir("reaccessed.mc.event.herwig.pool"), }, } evtMax = 50 def doReadWriteTest(genName="pythia", evtMax=100): """A simple wrapper around the read/write tests...""" genName = genName.lower() ###-----------------------------------------------------
uuid = "%s_%s" % (os.getpid(), timeofday()) print "#" * 80 print "## testing McEventCollection (read/write) persistency..." print "## Job uuid:", uuid print "#" * 80 bench = BenchSequence("McEventCollection (read/write) persistency") ###----------------------------------------------------- dataFilesDir = "/afs/cern.ch/atlas/offline/data/testfile" dataRefDir = "/afs/cern.ch/atlas/offline/test" outFiles = { 'gen_pythia': { 'ref': workDir("mc.event.pythia.%s.pool" % uuid), 'chk': workDir("reaccessed.mc.event.pythia.%s.pool" % uuid), }, 'gen_herwig': { 'ref': workDir("mc.event.herwig.%s.pool" % uuid), 'chk': workDir("reaccessed.mc.event.herwig.%s.pool" % uuid), }, } evtMax = 100 def doReadWriteTest(genName="pythia", evtMax=100): """A simple wrapper around the read/write tests...""" genName = genName.lower() ###-----------------------------------------------------
def doReadWriteTest(genName="pythia", evtMax=100): """A simple wrapper around the read/write tests...""" genName = genName.lower() ###----------------------------------------------------- print("") print("#" * 80) print("## Testing [writing-%s]..." % genName) templateJobO = """ OUTPUT='%(OutputFile)s'; DUMPTUPLE=True; GENERATOR='%(Generator)s'; TUPLEFILENAME='%(TupleFile)s'; include( 'McParticleTests/iotest_WriteGenEvent_jobOptions.py' ); jobproperties.PerfMonFlags.OutputFile = '%(PmonFile)s'; """ jobOptions = [ ChapPy.JobOptionsCmd( templateJobO % { 'OutputFile' : workDir("mc.event.%s.%s.pool" % (genName,uuid)), 'Generator' : genName.upper(), 'TupleFile' : outFiles['gen_%s' % genName]['ref']\ .replace(".pool",".tuple.root"), 'PmonFile' : workDir("write.genevent.%s.%s.pmon.gz"%(genName,uuid)), } ) ] athena = ChapPy.Athena(jobOptions=jobOptions, logFile=workDir("write.genevent.%s.%s.log" % (genName, uuid)), checkLeak=False) athena.EvtMax = evtMax sc = athena.run() if sc != 0: return ScOutput(sc, "ERROR") else: inFile = workDir("write.genevent.%s.%s.pmon.gz" % (genName, uuid)) outFile = workDir("out.write.genevent.%s.%s.pmon.root" % (genName, uuid)) print(subprocess.getoutput("perfmon.py %s -o %s" % (inFile, outFile))) ###----------------------------------------------------- print("\n") print("#" * 80) print("## Testing [reading-%s]..." % genName) templateJobO = """ INPUT=['%(InputFile)s']; DUMPTUPLE=True; TUPLEFILENAME='%(TupleFile)s'; include( 'McParticleTests/iotest_ReadGenEvent_jobOptions.py' ); jobproperties.PerfMonFlags.OutputFile = '%(PmonFile)s'; """ jobOptions = [ ChapPy.JobOptionsCmd( templateJobO % { 'InputFile' : workDir("mc.event.%s.%s.pool" % (genName,uuid)), 'TupleFile' : outFiles['gen_%s' % genName]['chk']\ .replace(".pool",".tuple.root"), 'PmonFile' : workDir("read.genevent.%s.%s.pmon.gz" % (genName,uuid)), } ) ] athena = ChapPy.Athena(jobOptions=jobOptions, logFile=workDir("read.genevent.%s.%s.log" % (genName, uuid)), checkLeak=False) athena.EvtMax = evtMax sc = athena.run() if sc != 0: return ScOutput(sc, "ERROR") else: inFile = workDir("read.genevent.%s.%s.pmon.gz" % (genName, uuid)) outFile = workDir("out.read.genevent.%s.%s.pmon.root" % (genName, uuid)) print(subprocess.getoutput("perfmon.py %s -o %s" % (inFile, outFile))) ###----------------------------------------------------- print("\n") print("#" * 80) return doTupleValidation( outFiles['gen_%s' % genName]['ref'].replace(".pool", ".tuple.root"), outFiles['gen_%s' % genName]['chk'].replace(".pool", ".tuple.root"))