def testPerfMon(jobOptName="PerfMonTests/test_perfMonSvc_noopalg.py",
perfMonFileName="perfmon.noopalg.root",
evtMax=50000):
print("")
print("#" * 80)
print("## testing PerfMonSvc [%s]" % jobOptName)
print(" ==> [%s]... (%i)" % (perfMonFileName, evtMax))
perfMonFileName = workDir(perfMonFileName)
refPerfMonFileName = "ref." + os.path.basename(perfMonFileName)
chkPerfMonFileName = "chk." + os.path.basename(perfMonFileName)
outPerfMonFileName = "ana." + os.path.basename(perfMonFileName)
refPerfMonFileName = refPerfMonFileName.replace(".root", ".pmon.gz")
chkPerfMonFileName = chkPerfMonFileName.replace(".root", ".pmon.gz")
## create the reference file
athena = ChapPy.Athena(jobOptions=[
ChapPy.JobOptions(jobOptName),
ChapPy.JobOptionsCmd("jobproperties.PerfMonFlags.OutputFile = '%s'" %
refPerfMonFileName)
],
checkLeak=False,
logFile=refPerfMonFileName + ".log")
athena.EvtMax = evtMax
sc = athena.run()
if sc != 0:
print("ERROR: could not create the 'ref' perfmon file !!")
return ScOutput(sc, "ERROR")
## create the to-be-checked file
athena = ChapPy.Athena(jobOptions=[
ChapPy.JobOptions(jobOptName),
ChapPy.JobOptionsCmd("jobproperties.PerfMonFlags.OutputFile = '%s'" %
chkPerfMonFileName)
],
checkLeak=False,
logFile=chkPerfMonFileName + ".log")
athena.EvtMax = evtMax
sc = athena.run()
if sc != 0:
print("ERROR: could not create the 'chk' perfmon file !!")
return ScOutput(sc, "ERROR")
#outPerfMonFileName = "ana." + os.path.basename( perfMonFileName )
print(" :::running [perfmon.py]...")
cmd = "perfmon.py %s %s -o %s" % \
( chkPerfMonFileName, refPerfMonFileName, outPerfMonFileName )
sc, out = subprocess.getstatusoutput(cmd)
if sc != 0:
print("## Problem while doing [perfmon] !!")
print(out)
out = "ERROR"
return ScOutput(sc, out)
out = "OK"
print("## [DONE]")
return ScOutput(sc, out)
def installRefFiles(fileNames):
for refFile in fileNames:
for fileName in [refFile, workDir(refFile)]:
if os.path.exists(fileName):
os.remove(fileName)
sc, out = subprocess.getstatusoutput("get_files %s" % refFile)
if sc != 0:
print("## ERROR: could not retrieve [%s]" % refFile)
print("## reason:\n", out)
continue
if os.path.exists(refFile) and \
os.path.exists(workDir(refFile)) and \
os.path.samefile( refFile, workDir(refFile) ):
print(" -%s" % workDir(refFile))
continue
sc, out = subprocess.getstatusoutput("mv %s %s" %
(refFile, workDir(refFile)))
if sc != 0:
print ("## ERROR: could not install [%s] into [%s]" %\
( refFile, workDir(refFile) ))
print("## reason:\n", out)
continue
else:
print(" -%s" % workDir(refFile))
return
def installRefFiles( fileNames, quiet=True ):
for refFile in fileNames:
for fileName in [ refFile, workDir(refFile) ]:
if os.path.exists(fileName):
os.remove(fileName)
sc,out = commands.getstatusoutput( "get_files %s" % refFile )
if sc != 0:
print "## ERROR: could not retrieve [%s]" % refFile
print "## reason:\n",out
continue
if os.path.exists(refFile) and \
os.path.exists(workDir(refFile)) and \
os.path.samefile( refFile, workDir(refFile) ):
if not quiet:
print " -%s" % workDir(refFile)
continue
try:
shutil.move(refFile, workDir(refFile))
if not quiet:
print " -%s" % workDir(refFile)
except OSError,err:
print "## ERROR: could not install [%s] into [%s]" %\
( refFile, workDir(refFile) )
print "## reason:\n",err
continue
print("#" * 80)
print("## testing symlinks for TruthParticleContainer...")
bench = BenchSequence("Symlinks for TruthParticleContainer")
print("## installing reference files...")
installRefFiles([
"mc.aod.symlinks.ref",
"mc.aod.pysymlinks.ref",
])
evtMax = 5
###-----------------------------------------------------
print("## Testing [writing]...")
jobOptions = [
ChapPy.JobOptionsCmd("OUTPUT=\"%s\"" % workDir("mc.aod.symlinks.pool")),
ChapPy.JobOptions("McParticleTests/symlinkTest_mcAod_jobOptions.py"),
]
athena = ChapPy.Athena(jobOptions=jobOptions,
logFile=workDir("mc.aod.symlinks.pool.log"),
checkLeak=False)
athena.EvtMax = evtMax
athena.run()
###-----------------------------------------------------
print("\n")
print("#" * 80)
bench += doPostCheck("McAod symlinks", workDir("mc.aod.symlinks.ref"),
workDir("mc.aod.symlinks.pool.log"),
"grep \"^McAodSymLinkTests\"")
def doReadWriteTest(genName="TruthParticles", evtMax=100):
###-----------------------------------------------------
print("\n")
print("#" * 80)
print("## Preparing input data... [%s]" % genName)
templateJobO = """
OUTPUT='%(OutputFile)s';
DUMP=True;
include( 'McParticleTests/iotest_WriteGenEvent_jobOptions.py' );
"""
jobOptions = [
ChapPy.JobOptionsCmd(templateJobO % {
'OutputFile': workDir("mc.event.%s.pool" % uuid),
})
]
athena = ChapPy.Athena(jobOptions=jobOptions,
logFile=workDir("mc.event.%s.pool.log" % uuid),
checkLeak=False)
athena.EvtMax = evtMax
sc = athena.run()
if sc != 0:
return ScOutput(sc, "ERROR")
###-----------------------------------------------------
print("\n")
print("#" * 80)
print("## Testing [writing-%s]..." % genName)
templateJobO = """
INPUT=['%(InputFile)s'];
OUTPUT= '%(OutputFile)s';
DUMPTUPLE=True;
TUPLEFILENAME='%(TupleFile)s';
include( 'McParticleTests/iotest_WriteTruthParticles_jobOptions.py' );
jobproperties.PerfMonFlags.OutputFile = '%(PmonFile)s';
"""
jobOptions = [
ChapPy.JobOptionsCmd(
templateJobO % {
'InputFile':
workDir("mc.event.%s.pool" % uuid),
'OutputFile':
outFiles['ref'],
'TupleFile':
outFiles['ref'].replace(".pool", ".tuple.root"),
'PmonFile':
workDir("write.mcaod.%s.%s.perfmon.pmon.gz" % (genName, uuid)),
})
]
athena = ChapPy.Athena(jobOptions=jobOptions,
logFile=outFiles['ref'] + ".log",
checkLeak=False)
athena.EvtMax = evtMax
sc = athena.run()
if sc != 0:
return ScOutput(sc, "ERROR")
else:
inFile = workDir("write.mcaod.%s.%s.perfmon.pmon.gz" % (genName, uuid))
outFile = workDir("out.write.mcaod.%s.%s.perfmon.root" %
(genName, uuid))
subprocess.getoutput("perfmon.py %s -o %s" % (inFile, outFile))
###-----------------------------------------------------
print("\n")
print("#" * 80)
print("## Testing [reading-%s]..." % genName)
templateJobO = """
INPUT=['%(InputFile)s'];
OUTPUT= '%(OutputFile)s';
DUMPTUPLE=True;
TUPLEFILENAME='%(TupleFile)s';
include( 'McParticleTests/iotest_ReadTruthParticles_jobOptions.py' );
jobproperties.PerfMonFlags.OutputFile = '%(PmonFile)s';
"""
jobOptions = [
ChapPy.JobOptionsCmd(
templateJobO % {
'InputFile':
outFiles['ref'],
'OutputFile':
outFiles['chk'],
'TupleFile':
outFiles['chk'].replace(".pool", ".tuple.root"),
'PmonFile':
workDir("read.mcaod.%s.%s.perfmon.pmon.gz" % (genName, uuid)),
})
]
athena = ChapPy.Athena(jobOptions=jobOptions,
logFile=outFiles['chk'] + ".log",
checkLeak=False)
athena.EvtMax = evtMax
sc = athena.run()
if sc != 0:
return ScOutput(sc, "ERROR")
else:
inFile = workDir("read.mcaod.%s.%s.perfmon.pmong.gz" % (genName, uuid))
outFile = workDir("out.read.mcaod.%s.%s.perfmon.root" %
(genName, uuid))
subprocess.getoutput("perfmon.py %s -o %s" % (inFile, outFile))
###-----------------------------------------------------
print("\n")
print("#" * 80)
return doMcAodTupleValidation(
outFiles['ref'].replace(".pool", ".tuple.root"),
outFiles['chk'].replace(".pool", ".tuple.root"))
uuid = "%s_%s" % (os.getpid(), timeofday())
print("#" * 80)
print("## testing TruthParticleContainer (read/write) persistency...")
print("## Job uuid:", uuid)
bench = BenchSequence("TruthParticleContainer (read/write) persistency")
###-----------------------------------------------------
dataFilesDir = "/afs/cern.ch/atlas/offline/data/testfile"
dataRefDir = "/afs/cern.ch/atlas/offline/test"
asciiFiles = {
'w': {
'ref': os.path.join(dataRefDir, "mc.aod.etisols.rel_13.ref"),
'chk': workDir("mc.aod.txt"),
},
'r': {
'ref': os.path.join(dataRefDir,
"reaccessed.mc.aod.etisols.rel_13.ref"),
'chk': workDir("reaccessed.mc.aod.txt"),
},
}
outFiles = {
'ref': workDir("mc.aod.%s.pool" % uuid),
'chk': workDir("reaccessed.mc.aod.%s.pool" % uuid)
}
evtMax = 100
###-----------------------------------------------------
## Little helper to validate output of jobs
from TestTools.iobench import doValidation
from TestTools.iobench import ScOutput
from TestTools.iobench import BenchSequence
EVTMAX = 100
NTIMES = 1
def timeofday():
from time import time
return int(time())
uuid = "%s_%s" % (os.getpid(),timeofday())
genEventFile = workDir("mc.event.%s.pool"%uuid)
mcAodFile = workDir("mc.aod.%s.pool"%uuid)
print "#"*80
print "## testing TruthParticleContainer I/O..."
print "## Job uuid:",uuid
benchSequence = BenchSequence( "TruthParticleContainer I/O" )
###-----------------------------------------------------
if os.path.exists(genEventFile):
print "## Re-using input data..."
else:
print "## Preparing input data..."
jobOptions = [
ChapPy.JobOptionsCmd( "OUTPUT=\"%s\"" % genEventFile ),
ChapPy.JobOptionsCmd( "DUMP=False" ),
from TestTools.iobench import BenchSequence
print "#" * 80
print "## testing McEventCollection (read/back) persistency..."
bench = BenchSequence("McEventCollection (read/back) persistency")
###-----------------------------------------------------
dataFilesDir = "/afs/cern.ch/atlas/offline/data/testfile"
dataRefDir = "/afs/cern.ch/atlas/offline/test"
asciiFiles = {
'rel_1004': {
'ref':
os.path.join(dataRefDir, 'mc.event.rel_1004.ref'),
'chk':
workDir('mc.event.rel_1004.txt'),
'in':
os.path.join(
dataFilesDir,
"q02initialprod.0001.H_2e2mu.q02dig_1004.etacut.0001_extract.pool.root"
)
},
'rel_11041': {
'ref':
os.path.join(dataRefDir, 'mc.event.rel_11041.ref'),
'chk':
workDir('mc.event.rel_11041.txt'),
'in':
os.path.join(
dataFilesDir,
"mc11.004100.T1_McAtNLO_top.digit_test.RDO.v11000401._00001.pool.root"
uuid = "%s_%s" % (os.getpid(), timeofday())
print("#" * 80)
print("## testing McEventCollection I/O...")
print("## Job uuid:", uuid)
benchSequence = BenchSequence("McEventCollection I/O")
EVTMAX = 100
NTIMES = 1
###-----------------------------------------------------
print("## Testing [writing]...")
jobOptions = [
ChapPy.JobOptionsCmd("OUTPUT=\"%s\"" %
workDir("mc.io.event.%s.pool" % uuid)),
ChapPy.JobOptionsCmd("DUMP=False"),
ChapPy.JobOptions("McParticleTests/iotest_WriteGenEvent_jobOptions.py"),
]
athena = ChapPy.Athena(
jobOptions=jobOptions,
#logFile = "/dev/stdout",
checkLeak=False)
athena.EvtMax = EVTMAX
bench = iobench.AthBench(
athena,
nTimes=NTIMES,
logFile=workDir("write.%s.log" %
os.path.basename(workDir("mc.io.event.%s.pool" % uuid))))
def doReadWriteTest(genName="pythia", evtMax=100):
"""A simple wrapper around the read/write tests..."""
genName = genName.lower()
###-----------------------------------------------------
print ""
print "#" * 80
print "## Testing [writing-%s]..." % genName
templateJobO = """
OUTPUT='%(OutputFile)s';
DUMPTUPLE=True;
GENERATOR='%(Generator)s';
TUPLEFILENAME='%(TupleFile)s';
include( 'McParticleTests/iotest_WriteGenEvent_jobOptions.py' );
jobproperties.PerfMonFlags.OutputFile = '%(PmonFile)s';
"""
jobOptions = [
ChapPy.JobOptionsCmd( templateJobO % {
'OutputFile' : workDir("mc.event.%s.pool" % genName),
'Generator' : genName.upper(),
'TupleFile' : outFiles['gen_%s' % genName]['ref']\
.replace(".pool",".tuple.root"),
'PmonFile' : workDir("write.genevent.%s.pmon.gz" % genName),
} )
]
athena = ChapPy.Athena(jobOptions=jobOptions,
logFile=workDir("write.genevent.%s.log" % genName),
checkLeak=False)
athena.EvtMax = evtMax
sc = athena.run()
if sc != 0:
return ScOutput(sc, "ERROR")
else:
inFile = workDir("write.genevent.%s.pmon.gz" % genName)
outFile = workDir("out.write.genevent.%s.pmon.root" % genName)
print commands.getoutput("perfmon.py %s -o %s" % (inFile, outFile))
###-----------------------------------------------------
print ""
print "#" * 80
print "## Testing [ASCII-writing-%s (1)]..." % genName
templateJobO = """
INPUT=%(InputFile)s;
OUTPUT='%(OutputFile)s';
DUMPTUPLE=True;
GENERATOR='%(Generator)s';
include( 'McParticleAlgs/GenEventAsciiWriter_jobOptions.py' );
"""
jobOptions = [
ChapPy.JobOptionsCmd(
templateJobO % {
'InputFile': [
workDir("mc.event.%s.pool" % genName),
],
'OutputFile': workDir("mc.event.%s.1.ascii" % genName),
'Generator': genName.upper(),
})
]
athena = ChapPy.Athena(jobOptions=jobOptions,
logFile=workDir("write.genevent.ascii.1.%s.log" %
genName),
checkLeak=False)
athena.EvtMax = evtMax
sc = athena.run()
if sc != 0:
return ScOutput(sc, "ERROR")
###-----------------------------------------------------
print ""
print "#" * 80
print "## Testing [ASCII-writing-%s (2)]..." % genName
templateJobO = """
INPUT=%(InputFile)s;
OUTPUT='%(OutputFile)s';
DUMPTUPLE=True;
GENERATOR='%(Generator)s';
include( 'McParticleAlgs/GenEventAsciiWriter_jobOptions.py' );
"""
jobOptions = [
ChapPy.JobOptionsCmd(
templateJobO % {
'InputFile': [
workDir("mc.event.%s.pool" % genName),
],
'OutputFile': workDir("mc.event.%s.2.ascii" % genName),
'Generator': genName.upper(),
})
]
athena = ChapPy.Athena(jobOptions=jobOptions,
logFile=workDir("write.genevent.ascii.2.%s.log" %
genName),
checkLeak=False)
athena.EvtMax = evtMax
sc = athena.run()
if sc != 0:
return ScOutput(sc, "ERROR")
###-----------------------------------------------------
print "\n"
print "#" * 80
print "## Testing [ASCII-reading-%s]..." % genName
templateJobO = """
INPUT=%(InputFiles)s;
include( 'McAsciiEventSelector/Example_McAsciiReader_jobOptions.py' );
"""
jobOptions = [
ChapPy.JobOptionsCmd( templateJobO % {
'InputFiles' : [ workDir("mc.event.%s.1.ascii" % genName),
workDir("mc.event.%s.2.ascii" % genName),],
'TupleFile' : outFiles['gen_%s' % genName]['chk']\
.replace(".pool",".tuple.root"),
'PmonFile' : workDir("read.genevent.%s.pmon.gz" % genName),
} )
]
athena = ChapPy.Athena(jobOptions=jobOptions,
logFile=workDir("read.genevent.%s.log" % genName),
checkLeak=False)
athena.EvtMax = evtMax
sc = athena.run()
if sc != 0:
return ScOutput(sc, "ERROR")
###-----------------------------------------------------
print "\n"
print "#" * 80
return ScOutput(0, "OK")
###-----------------------------------------------------
## Little helper to validate output of jobs
from TestTools.iobench import ScOutput
from TestTools.iobench import BenchSequence
from McParticleTests.tests import doTupleValidation
print "#" * 80
print "## testing McEventCollection (read/write) persistency..."
print "#" * 80
bench = BenchSequence("McEventCollection (read/write) ASCII persistency")
###-----------------------------------------------------
outFiles = {
'gen_pythia': {
'ref': workDir("mc.event.pythia.pool"),
'chk': workDir("reaccessed.mc.event.pythia.pool"),
},
'gen_herwig': {
'ref': workDir("mc.event.herwig.pool"),
'chk': workDir("reaccessed.mc.event.herwig.pool"),
},
}
evtMax = 50
def doReadWriteTest(genName="pythia", evtMax=100):
"""A simple wrapper around the read/write tests..."""
genName = genName.lower()
###-----------------------------------------------------
uuid = "%s_%s" % (os.getpid(), timeofday())
print "#" * 80
print "## testing McEventCollection (read/write) persistency..."
print "## Job uuid:", uuid
print "#" * 80
bench = BenchSequence("McEventCollection (read/write) persistency")
###-----------------------------------------------------
dataFilesDir = "/afs/cern.ch/atlas/offline/data/testfile"
dataRefDir = "/afs/cern.ch/atlas/offline/test"
outFiles = {
'gen_pythia': {
'ref': workDir("mc.event.pythia.%s.pool" % uuid),
'chk': workDir("reaccessed.mc.event.pythia.%s.pool" % uuid),
},
'gen_herwig': {
'ref': workDir("mc.event.herwig.%s.pool" % uuid),
'chk': workDir("reaccessed.mc.event.herwig.%s.pool" % uuid),
},
}
evtMax = 100
def doReadWriteTest(genName="pythia", evtMax=100):
"""A simple wrapper around the read/write tests..."""
genName = genName.lower()
###-----------------------------------------------------
def doReadWriteTest(genName="pythia", evtMax=100):
"""A simple wrapper around the read/write tests..."""
genName = genName.lower()
###-----------------------------------------------------
print("")
print("#" * 80)
print("## Testing [writing-%s]..." % genName)
templateJobO = """
OUTPUT='%(OutputFile)s';
DUMPTUPLE=True;
GENERATOR='%(Generator)s';
TUPLEFILENAME='%(TupleFile)s';
include( 'McParticleTests/iotest_WriteGenEvent_jobOptions.py' );
jobproperties.PerfMonFlags.OutputFile = '%(PmonFile)s';
"""
jobOptions = [
ChapPy.JobOptionsCmd( templateJobO % {
'OutputFile' : workDir("mc.event.%s.%s.pool" % (genName,uuid)),
'Generator' : genName.upper(),
'TupleFile' : outFiles['gen_%s' % genName]['ref']\
.replace(".pool",".tuple.root"),
'PmonFile' : workDir("write.genevent.%s.%s.pmon.gz"%(genName,uuid)),
} )
]
athena = ChapPy.Athena(jobOptions=jobOptions,
logFile=workDir("write.genevent.%s.%s.log" %
(genName, uuid)),
checkLeak=False)
athena.EvtMax = evtMax
sc = athena.run()
if sc != 0:
return ScOutput(sc, "ERROR")
else:
inFile = workDir("write.genevent.%s.%s.pmon.gz" % (genName, uuid))
outFile = workDir("out.write.genevent.%s.%s.pmon.root" %
(genName, uuid))
print(subprocess.getoutput("perfmon.py %s -o %s" % (inFile, outFile)))
###-----------------------------------------------------
print("\n")
print("#" * 80)
print("## Testing [reading-%s]..." % genName)
templateJobO = """
INPUT=['%(InputFile)s'];
DUMPTUPLE=True;
TUPLEFILENAME='%(TupleFile)s';
include( 'McParticleTests/iotest_ReadGenEvent_jobOptions.py' );
jobproperties.PerfMonFlags.OutputFile = '%(PmonFile)s';
"""
jobOptions = [
ChapPy.JobOptionsCmd( templateJobO % {
'InputFile' : workDir("mc.event.%s.%s.pool" % (genName,uuid)),
'TupleFile' : outFiles['gen_%s' % genName]['chk']\
.replace(".pool",".tuple.root"),
'PmonFile' : workDir("read.genevent.%s.%s.pmon.gz" % (genName,uuid)),
} )
]
athena = ChapPy.Athena(jobOptions=jobOptions,
logFile=workDir("read.genevent.%s.%s.log" %
(genName, uuid)),
checkLeak=False)
athena.EvtMax = evtMax
sc = athena.run()
if sc != 0:
return ScOutput(sc, "ERROR")
else:
inFile = workDir("read.genevent.%s.%s.pmon.gz" % (genName, uuid))
outFile = workDir("out.read.genevent.%s.%s.pmon.root" %
(genName, uuid))
print(subprocess.getoutput("perfmon.py %s -o %s" % (inFile, outFile)))
###-----------------------------------------------------
print("\n")
print("#" * 80)
return doTupleValidation(
outFiles['gen_%s' % genName]['ref'].replace(".pool", ".tuple.root"),
outFiles['gen_%s' % genName]['chk'].replace(".pool", ".tuple.root"))