def processAppKerOutput(appstdout=None, stdout=None, stderr=None, geninfo=None, appKerNResVars=None):
# set App Kernel Description
parser = AppKerOutputParser(
name="test",
version=1,
description="test the resource deployment",
url="http://xdmod.buffalo.edu",
measurement_name="test",
)
# set obligatory parameters and statistics
# set common parameters and statistics
parser.setCommonMustHaveParsAndStats()
# set app kernel custom sets
parser.setMustHaveStatistic("Wall Clock Time")
parser.setMustHaveStatistic("Shell is BASH")
# parse common parameters and statistics
parser.parseCommonParsAndStats(appstdout, stdout, stderr, geninfo)
# set statistics
if hasattr(parser, "wallClockTime"):
parser.setStatistic("Wall Clock Time", total_seconds(parser.wallClockTime), "Second")
# read output
lines = []
if os.path.isfile(stdout):
fin = open(stdout, "rt")
lines = fin.readlines()
fin.close()
# process the output
parser.setStatistic("Shell is BASH", 0)
j = 0
while j < len(lines):
if lines[j].count("Checking that the shell is BASH") > 0 and lines[j + 1].count("bash") > 0:
parser.setStatistic("Shell is BASH", 1)
j += 1
if stdout != None:
if hasattr(parser, "filesExistance"):
for k, v in parser.filesExistance.iteritems():
parser.setStatistic(k + " exists", int(v))
if hasattr(parser, "dirAccess"):
for k, v in parser.dirAccess.iteritems():
parser.setStatistic(k + " accessible", int(v))
if __name__ == "__main__":
# output for testing purpose
print "parsing complete:", parser.parsingComplete()
parser.printParsNStatsAsMustHave()
print parser.getXML()
# return complete XML overwize return None
return parser.getXML()
def processAppKerOutput(appstdout=None,stdout=None,stderr=None,geninfo=None,appKerNResVars=None):
#set App Kernel Description
parser=AppKerOutputParser(
name = 'xdmod.benchmark.graph.graph500',
version = 1,
description = "Graph500 Benchmark",
url = 'http://www.Graph500.org',
measurement_name = 'Graph500'
)
#set obligatory parameters and statistics
#set common parameters and statistics
parser.setCommonMustHaveParsAndStats()
#set app kernel custom sets
parser.setMustHaveParameter('App:Version')
parser.setMustHaveParameter('Edge Factor')
parser.setMustHaveParameter('Input File')
parser.setMustHaveParameter('Number of Roots to Check')
parser.setMustHaveParameter('Number of Edges')
parser.setMustHaveParameter('Number of Vertices')
parser.setMustHaveParameter('Scale')
parser.setMustHaveStatistic('Harmonic Mean TEPS')
parser.setMustHaveStatistic('Harmonic Standard Deviation TEPS')
parser.setMustHaveStatistic('Median TEPS')
parser.setMustHaveStatistic('Wall Clock Time')
#parse common parameters and statistics
parser.parseCommonParsAndStats(appstdout,stdout,stderr,geninfo)
if hasattr(parser,'appKerWallClockTime'):
parser.setStatistic("Wall Clock Time", total_seconds(parser.appKerWallClockTime), "Second")
elif hasattr(parser,'wallClockTime'):
parser.setStatistic("Wall Clock Time", total_seconds(parser.wallClockTime), "Second")
#read output
lines=[]
if os.path.isfile(appstdout):
fin=open(appstdout,"rt")
lines=fin.readlines()
fin.close()
#process the output
parser.successfulRun=True
Nerrors=0
j=0
while j<len(lines):
m=re.match(r'^Graph500 version:\s+(.+)',lines[j])
if m:parser.setParameter("App:Version",m.group(1).strip())
m=re.match(r'ERROR:\s+(.+)',lines[j])
if m:Nerrors+=1
m=re.match(r'^Reading input from\s+(.+)',lines[j])
if m:parser.setParameter("Input File",m.group(1))
m=re.match(r'^SCALE:\s+(\d+)',lines[j])
if m:parser.setParameter("Scale",m.group(1))
m=re.match(r'^edgefactor:\s+(\d+)',lines[j])
if m:parser.setParameter("Edge Factor",m.group(1))
m=re.match(r'^NBFS:\s+(\d+)',lines[j])
if m:parser.setParameter("Number of Roots to Check",m.group(1))
m=re.match(r'^median_TEPS:\s+(\d[0-9.e\+]+)',lines[j])
if m:parser.setStatistic("Median TEPS", m.group(1), "Traversed Edges Per Second" )
m=re.match(r'^harmonic_mean_TEPS:\s+(\d[0-9.e\+]+)',lines[j])
if m:
parser.successfulRun=True
parser.setStatistic("Harmonic Mean TEPS", m.group(1), "Traversed Edges Per Second" )
m=re.match(r'^harmonic_stddev_TEPS:\s+(\d[0-9.e\+]+)',lines[j])
if m:parser.setStatistic("Harmonic Standard Deviation TEPS", m.group(1), "Traversed Edges Per Second" )
m=re.match(r'^median_validate:\s+([\d.]+)\s+s',lines[j])
if m:parser.setStatistic("Median Validation Time", m.group(1), "Second" )
m=re.match(r'^mean_validate:\s+([\d.]+)\s+s',lines[j])
if m:parser.setStatistic("Mean Validation Time", m.group(1), "Second" )
m=re.match(r'^stddev_validate:\s+([\d.]+)\s+s',lines[j])
if m:parser.setStatistic("Standard Deviation Validation Time", m.group(1), "Second" )
j+=1
if Nerrors>0:
parser.successfulRun=False
if parser.getParameter('Scale')!=None and parser.getParameter('Edge Factor')!=None :
SCALE=int(parser.getParameter('Scale'))
edgefactor=int(parser.getParameter('Edge Factor'))
parser.setParameter("Number of Vertices",2**SCALE)
parser.setParameter("Number of Edges",edgefactor*2**SCALE)
if __name__ == "__main__":
#output for testing purpose
parser.parsingComplete(True)
print "parsing complete:",parser.parsingComplete()
parser.printParsNStatsAsMustHave()
print parser.getXML()
#return complete XML overwize return None
return parser.getXML()
def processAppKerOutput(appstdout=None,stdout=None,stderr=None,geninfo=None,appKerNResVars=None):
#set App Kernel Description
parser=AppKerOutputParser(
name = 'xdmod.benchmark.mpi.imb',
version = 1,
description = "Intel MPI Benchmarks",
url = 'http://www.intel.com/software/imb',
measurement_name = 'Intel MPI Benchmarks'
)
#set obligatory parameters and statistics
#set common parameters and statistics
parser.setCommonMustHaveParsAndStats()
#set app kernel custom sets
parser.setMustHaveParameter('App:MPI Thread Environment')
parser.setMustHaveParameter('App:MPI Version')
parser.setMustHaveParameter('App:Max Message Size')
parser.setMustHaveStatistic('Max Exchange Bandwidth')
parser.setMustHaveStatistic("Max MPI-2 Bidirectional 'Get' Bandwidth (aggregate)")
parser.setMustHaveStatistic("Max MPI-2 Bidirectional 'Get' Bandwidth (non-aggregate)")
parser.setMustHaveStatistic("Max MPI-2 Bidirectional 'Put' Bandwidth (aggregate)")
parser.setMustHaveStatistic("Max MPI-2 Bidirectional 'Put' Bandwidth (non-aggregate)")
parser.setMustHaveStatistic("Max MPI-2 Unidirectional 'Get' Bandwidth (aggregate)")
parser.setMustHaveStatistic("Max MPI-2 Unidirectional 'Get' Bandwidth (non-aggregate)")
parser.setMustHaveStatistic("Max MPI-2 Unidirectional 'Put' Bandwidth (aggregate)")
parser.setMustHaveStatistic("Max MPI-2 Unidirectional 'Put' Bandwidth (non-aggregate)")
parser.setMustHaveStatistic('Max PingPing Bandwidth')
parser.setMustHaveStatistic('Max PingPong Bandwidth')
parser.setMustHaveStatistic('Max SendRecv Bandwidth')
parser.setMustHaveStatistic('Min AllGather Latency')
parser.setMustHaveStatistic('Min AllGatherV Latency')
parser.setMustHaveStatistic('Min AllReduce Latency')
parser.setMustHaveStatistic('Min AllToAll Latency')
parser.setMustHaveStatistic('Min AllToAllV Latency')
parser.setMustHaveStatistic('Min Barrier Latency')
parser.setMustHaveStatistic('Min Broadcast Latency')
parser.setMustHaveStatistic('Min Gather Latency')
parser.setMustHaveStatistic('Min GatherV Latency')
#parser.setMustHaveStatistic("Min MPI-2 'Accumulate' Latency (aggregate)")
#parser.setMustHaveStatistic("Min MPI-2 'Accumulate' Latency (non-aggregate)")
parser.setMustHaveStatistic('Min MPI-2 Window Creation Latency')
parser.setMustHaveStatistic('Min Reduce Latency')
parser.setMustHaveStatistic('Min ReduceScatter Latency')
parser.setMustHaveStatistic('Min Scatter Latency')
parser.setMustHaveStatistic('Min ScatterV Latency')
parser.setMustHaveStatistic('Wall Clock Time')
#parse common parameters and statistics
parser.parseCommonParsAndStats(appstdout,stdout,stderr,geninfo)
if hasattr(parser,'appKerWallClockTime'):
parser.setStatistic("Wall Clock Time", total_seconds(parser.appKerWallClockTime), "Second")
# Intel MPI benchmark suite contains three classes of benchmarks:
#
# Single-transfer, which needs only 2 processes
# Parallel-transfer, which can use as many processes that are available
# Collective, which can use as many processes that are available
# The parameters mapping table
Params = {
"MPI Thread Environment" : [ "MPI Thread Environment", "", "" ],
"MPI Version" : [ "MPI Version", "", "" ],
"Maximum message length in bytes" : [ "Max Message Size", "MByte", "<val>/1024/1024" ]
}
# The result mapping table
Metrics = {
"PingPing" : [ "PingPing Bandwidth", "MByte per Second", "max" ],
"PingPong" : [ "PingPong Bandwidth", "MByte per Second", "max" ],
"Multi-PingPing" : [ "PingPing Bandwidth", "MByte per Second", "max" ],
"Multi-PingPong" : [ "PingPong Bandwidth", "MByte per Second", "max" ],
"Sendrecv" : [ "SendRecv Bandwidth", "MByte per Second", "max" ],
"Exchange" : [ "Exchange Bandwidth", "MByte per Second", "max" ],
"Allreduce" : [ "AllReduce Latency", "us", "min" ],
"Reduce" : [ "Reduce Latency", "us", "min" ],
"Reduce_scatter" : [ "ReduceScatter Latency", "us", "min" ],
"Allgather" : [ "AllGather Latency", "us", "min" ],
"Allgatherv" : [ "AllGatherV Latency", "us", "min" ],
"Gather" : [ "Gather Latency", "us", "min" ],
"Gatherv" : [ "GatherV Latency", "us", "min" ],
"Scatter" : [ "Scatter Latency", "us", "min" ],
"Scatterv" : [ "ScatterV Latency", "us", "min" ],
"Alltoall" : [ "AllToAll Latency", "us", "min" ],
"Alltoallv" : [ "AllToAllV Latency", "us", "min" ],
"Bcast" : [ "Broadcast Latency", "us", "min" ],
"Barrier" : [ "Barrier Latency", "us", "min" ],
"Window" : [ "MPI-2 Window Creation Latency", "us", "min" ],
"Multi-Unidir_Get" : [ "MPI-2 Unidirectional 'Get' Bandwidth", "MByte per Second", "max" ],
"Multi-Unidir_Put" : [ "MPI-2 Unidirectional 'Put' Bandwidth", "MByte per Second", "max" ],
"Multi-Bidir_Get" : [ "MPI-2 Bidirectional 'Get' Bandwidth", "MByte per Second", "max" ],
"Multi-Bidir_Put" : [ "MPI-2 Bidirectional 'Put' Bandwidth", "MByte per Second", "max" ],
"Unidir_Get" : [ "MPI-2 Unidirectional 'Get' Bandwidth", "MByte per Second", "max" ],
"Unidir_Put" : [ "MPI-2 Unidirectional 'Put' Bandwidth", "MByte per Second", "max" ],
"Bidir_Get" : [ "MPI-2 Bidirectional 'Get' Bandwidth", "MByte per Second", "max" ],
"Bidir_Put" : [ "MPI-2 Bidirectional 'Put' Bandwidth", "MByte per Second", "max" ],
"Accumulate" : [ "MPI-2 'Accumulate' Latency", "us", "min" ]
}
#read output
lines=[]
if os.path.isfile(appstdout):
fin=open(appstdout,"rt")
lines=fin.readlines()
fin.close()
#process the output
parser.successfulRun=False
aggregateMode=None
metric=None
j=-1
while j<len(lines)-1:
j+=1
m=re.search(r'All processes entering MPI_Finalize',lines[j])
if m:parser.successfulRun=True
m=re.match(r'^# Benchmarking\s+(\S+)',lines[j])
if m:
if m.group(1) in Metrics:
metric=m.group(1)
continue
m=re.match(r'^#\s+MODE:\s+(\S+)',lines[j])
if m and metric and aggregateMode==None:
aggregateMode=m.group(1)
continue
m=re.match(r'^# (.+): (.+)',lines[j])
if m:# benchmark parameters
param=m.group(1).strip()
if param in Params:
val=m.group(2).strip()
v=Params[param][2]
if v.find('<val>')>=0:
val=float(val)
val=eval(v.replace('<val>','val'))
parser.setParameter( "App:" + Params[param][0], str(val) + " ", Params[param][1] )
continue
m=re.match(r'^\s+([1-9]\d*)\s+\d+',lines[j])
if m and metric:# this effectively skips the first line of result, which has #bytes = 0
results=[]
while m:
numbers=lines[j].split()
results.append(float(numbers[-1]))# tokenize the line, and extract the last column
j+=1
if j<len(lines):
m=re.match(r'^\s+([1-9]\d*)\s+\d+',lines[j])
else:
break
metricName=Metrics[metric][0]
if aggregateMode:
metricName+=" ("+aggregateMode.lower()+ ")"
if len(results)>0:
if Metrics[metric][1]=='us':
statname=Metrics[metric][2][0].upper()+Metrics[metric][2][1:]+" "+metricName
statval=eval(Metrics[metric][2]+"(results)")
parser.setStatistic(statname,statval*1e-6, "Second" )
else:
statname=Metrics[metric][2][0].upper()+Metrics[metric][2][1:]+" "+metricName
statval=eval(Metrics[metric][2]+"(results)")
parser.setStatistic(statname,statval, Metrics[metric][1])
aggregateMode=None
metric=None
if parser.getParameter("App:MPI Thread Environment")==None:
parser.setParameter("App:MPI Thread Environment","")
if __name__ == "__main__":
#output for testing purpose
print "parsing complete:",parser.parsingComplete(Verbose=True)
parser.printParsNStatsAsMustHave()
print parser.getXML()
#Print out missing parameters for debug purpose
parser.parsingComplete(Verbose=True)
#return complete XML overwize return None
return parser.getXML()
def processAppKerOutput(appstdout=None,stdout=None,stderr=None,geninfo=None,appKerNResVars=None):
#set App Kernel Description
parser=AppKerOutputParser(
name = 'xdmod.app.phys.quantum_espresso',
version = 1,
description = "Quantum ESPRESSO (PWSCF)",
url = 'http://www.quantum-espresso.org',
measurement_name = 'Quantum_ESPRESSO'
)
#set obligatory parameters and statistics
#set common parameters and statistics
parser.setCommonMustHaveParsAndStats()
#set app kernel custom sets
parser.setMustHaveParameter('App:Version')
parser.setMustHaveParameter('Input:Number of Atoms per Cell')
parser.setMustHaveParameter('Input:Number of Atomic Types')
parser.setMustHaveParameter('Input:Number of Electrons')
parser.setMustHaveStatistic('Wall Clock Time')
parser.setMustHaveStatistic('User Time')
parser.setMustHaveStatistic("Per-Process Dynamical Memory")
parser.setMustHaveStatistic("Time Spent in Program Initialization")
parser.setMustHaveStatistic("Time Spent in Electron Energy Calculation")
parser.setMustHaveStatistic("Time Spent in Force Calculation")
#This statistic probably was working for a different set of inputs, optional now
#parser.setMustHaveStatistic("Time Spent in Stress Calculation")
#This statistic probably was working for a different set of inputs, optional now
#parser.setMustHaveStatistic("Time Spent in Potential Updates (Charge Density and Wavefunctions Extrapolations)")
#parse common parameters and statistics
parser.parseCommonParsAndStats(appstdout,stdout,stderr,geninfo)
#read output
lines=[]
if os.path.isfile(appstdout):
fin=open(appstdout,"rt")
lines=fin.readlines()
fin.close()
#process the output
parser.successfulRun=False
j=0
while j<len(lines):
m=re.match(r'^\s+Program PWSCF\s+([\w\.]+)\s+starts',lines[j])
if m:parser.setParameter("App:Version",m.group(1).strip())
m=re.match(r'^\s+number of atoms\/cell\s*=\s*([\d\.]+)',lines[j])
if m:parser.setParameter("Input:Number of Atoms per Cell",m.group(1).strip())
m=re.match(r'^\s+number of atomic types\s*=\s*([\d\.]+)',lines[j])
if m:parser.setParameter("Input:Number of Atomic Types",m.group(1).strip())
m=re.match(r'^\s+number of electrons\s*=\s*([\d\.]+)',lines[j])
if m:parser.setParameter("Input:Number of Electrons",m.group(1).strip())
m=re.match(r'^\s+per-process dynamical memory:\s*([\d\.]+)\s*Mb',lines[j])
if m:parser.setStatistic("Per-Process Dynamical Memory", (m.group(1).strip()), "MByte" );
m=re.match(r'^\s+init_run\s+:\s*([\d\.]+)s CPU',lines[j])
if m:parser.setStatistic("Time Spent in Program Initialization", (m.group(1).strip()), "Second" );
m=re.match(r'^\s+electrons\s+:\s*([\d\.]+)s CPU',lines[j])
if m:parser.setStatistic("Time Spent in Electron Energy Calculation", (m.group(1).strip()), "Second" );
m=re.match(r'^\s+forces\s+:\s*([\d\.]+)s CPU',lines[j])
if m:parser.setStatistic("Time Spent in Force Calculation", (m.group(1).strip()), "Second" );
m=re.match(r'^\s+stress\s+:\s*([\d\.]+)s CPU',lines[j])
if m:parser.setStatistic("Time Spent in Stress Calculation", (m.group(1).strip()), "Second" );
m=re.match(r'^\s+update_pot\s+:\s*([\d\.]+)s CPU',lines[j])
if m:parser.setStatistic("Time Spent in Potential Updates (Charge Density and Wavefunctions Extrapolations)", float(m.group(1).strip()), "Second" );
m=re.match(r'^\s+PWSCF\s+:(.+CPU.+)',lines[j])
if m:
runTimes=m.group(1).strip().split(',')
for runTime in runTimes:
v=runTime.split()
if len(v)>1:
sec=0.0
if v[0].lower().find("m")>=0:
m=re.match(r'^([0-9]+)m([0-9.]+)s',v[0])
sec=float(m.group(1))*60.0+float(m.group(2))
else:
m=re.match(r'^([0-9.]+)s',v[0])
sec=float(m.group(1))
if v[1].upper().find("CPU")>=0:
parser.setStatistic("User Time", sec, "Second" );
if v[1].upper().find("WALL")>=0:
parser.setStatistic("Wall Clock Time", sec, "Second" );
if re.match(r'^\s+JOB DONE',lines[j]):
parser.successfulRun=True
j+=1
if __name__ == "__main__":
#output for testing purpose
print "parsing complete:",parser.parsingComplete(True)
if hasattr(parser, 'successfulRun'):print "successfulRun",parser.successfulRun
parser.printParsNStatsAsMustHave()
print parser.getXML()
#return complete XML overwize return None
return parser.getXML()
def processAppKerOutput(appstdout=None,stdout=None,stderr=None,geninfo=None,appKerNResVars=None):
#set App Kernel Description
parser=AppKerOutputParser(
name = 'xdmod.app.md.lammps',
version = 1,
description = "LAMMPS: Large-scale Atomic/Molecular Massively Parallel Simulator",
url = 'http://lammps.sandia.gov',
measurement_name = 'LAMMPS'
)
#set obligatory parameters and statistics
#set common parameters and statistics
parser.setCommonMustHaveParsAndStats()
#set app kernel custom sets
parser.setMustHaveParameter('App:Version')
parser.setMustHaveParameter('Input:Number of Atoms')
parser.setMustHaveParameter('Input:Number of Steps')
parser.setMustHaveParameter('Input:Timestep')
parser.setMustHaveStatistic('Molecular Dynamics Simulation Performance')
parser.setMustHaveStatistic('Per-Process Memory')
parser.setMustHaveStatistic('Time Spent in Bond Potential Calculation')
parser.setMustHaveStatistic('Time Spent in Communication')
parser.setMustHaveStatistic('Time Spent in Long-Range Coulomb Potential (K-Space) Calculation')
parser.setMustHaveStatistic('Time Spent in Neighbor List Regeneration')
parser.setMustHaveStatistic('Time Spent in Pairwise Potential Calculation')
parser.setMustHaveStatistic('Wall Clock Time')
#parse common parameters and statistics
parser.parseCommonParsAndStats(appstdout,stdout,stderr,geninfo)
#read output
lines=[]
if os.path.isfile(appstdout):
fin=open(appstdout,"rt")
lines=fin.readlines()
fin.close()
#process the output
parser.successfulRun=False
wallClockTime=None
simulationUnits=None
numSteps=None
stepSize=None
j=0
while j<len(lines):
m=re.match(r'^LAMMPS\s+\(([\w ]+)\)',lines[j])
if m:parser.setParameter("App:Version",m.group(1).strip())
m=re.match(r'^Memory usage per processor = ([\d\.]+) Mbyte',lines[j])
if m:parser.setStatistic( "Per-Process Memory",m.group(1).strip(), "MByte" )
m=re.match(r'^Loop time of ([\d\.]+) on',lines[j])
if m:
parser.successfulRun=True
wallClockTime=float(m.group(1).strip())
parser.setStatistic("Wall Clock Time", wallClockTime, "Second" )
m1=re.search(r'(\d+) atoms',lines[j])
if m1:parser.setParameter( "Input:Number of Atoms", m1.group(1).strip())
m=re.match(r'^units\s+(\w+)',lines[j])
if m:simulationUnits=m.group(1).strip().lower()
m=re.match(r'^run\s+(\d+)',lines[j])
if m:
numSteps=int(m.group(1).strip())
parser.setParameter("Input:Number of Steps", numSteps)
m=re.match(r'^timestep\s+([\d\.]+)',lines[j])
if m:stepSize=float(m.group(1).strip())
m=re.match(r'^Pair\s+time.+= ([\d\.]+)',lines[j])
if parser.successfulRun and m:
parser.setStatistic("Time Spent in Pairwise Potential Calculation", m.group(1).strip(), "Second" )
m=re.match(r'^Bond\s+time.+= ([\d\.]+)',lines[j])
if parser.successfulRun and m:
parser.setStatistic("Time Spent in Bond Potential Calculation", m.group(1).strip(), "Second" )
m=re.match(r'^Kspce\s+time.+= ([\d\.]+)',lines[j])
if parser.successfulRun and m:
parser.setStatistic("Time Spent in Long-Range Coulomb Potential (K-Space) Calculation", m.group(1).strip(), "Second" )
m=re.match(r'^Neigh\s+time.+= ([\d\.]+)',lines[j])
if parser.successfulRun and m:
parser.setStatistic("Time Spent in Neighbor List Regeneration", m.group(1).strip(), "Second" )
m=re.match(r'^Comm\s+time.+= ([\d\.]+)',lines[j])
if parser.successfulRun and m:
parser.setStatistic("Time Spent in Communication", m.group(1).strip(), "Second" )
j+=1
if parser.successfulRun and numSteps and simulationUnits != "lj":
# The default value for $stepSize is (see http://lammps.sandia.gov/doc/units.html):
#
# 0.005 tau for $simulationUnits eq "lj"
# 1e-15 second for $simulationUnits eq "real" or "metal"
# 1e-18 second for $simulationUnits eq "electron"
# 1e-8 second for $simulationUnits eq "si" or "cgs"
# If $simulationUnits is (see http://lammps.sandia.gov/doc/units.html)
#
# "lj", the unit for $stepSize is tau
# "real" or "electron", the unit for $stepSize is 1e-15 second
# "metal", the unit for $stepSize is 1e-12 second
# "si" or "cgs", the unit for $stepSize is second
# The default $simulationUnits is "lj"
#
# We ignore "lj" since "lj" is unitless.
if stepSize==None:
if simulationUnits == "real":stepSize = 1.0
if simulationUnits.find("electron")>=0 or simulationUnits.find("metal")>=0: stepSize = 0.001
if simulationUnits.find("si")>=0 or simulationUnits.find("cgs")>=0: stepSize = 1.0e-8
stepSizeInSec=stepSize
if stepSize:
if simulationUnits.find("electron")>=0 or simulationUnits.find("real")>=0: stepSizeInSec=stepSize*1.0e-15
if simulationUnits=="metal": stepSizeInSec=stepSize*1.0e-12
if stepSizeInSec:
parser.setParameter("Input:Timestep", stepSizeInSec, "Second per Step" );
parser.setStatistic("Molecular Dynamics Simulation Performance", 1.0e-9*( 1.0e9 * stepSizeInSec * numSteps ) / ( wallClockTime / 86400.0 ), "Second per Day" )
if __name__ == "__main__":
#output for testing purpose
print "parsing complete:",parser.parsingComplete()
parser.printParsNStatsAsMustHave()
print parser.getXML()
#return complete XML overwize return None
return parser.getXML()
def processAppKerOutput(appstdout=None,stdout=None,stderr=None,geninfo=None,appKerNResVars=None):
#set App Kernel Description
parser=AppKerOutputParser(
name = 'xdmod.app.md.charmm',
version = 1,
description = "CHARMM: Chemistry at Harvard Macromolecular Mechanics",
url = 'http://www.charmm.org',
measurement_name = 'CHARMM'
)
#set obligatory parameters and statistics
#set common parameters and statistics
parser.setCommonMustHaveParsAndStats()
#set app kernel custom sets
parser.setMustHaveParameter('App:Version')
parser.setMustHaveParameter('Input:Number of Angles')
parser.setMustHaveParameter('Input:Number of Atoms')
parser.setMustHaveParameter('Input:Number of Bonds')
parser.setMustHaveParameter('Input:Number of Dihedrals')
parser.setMustHaveParameter('Input:Number of Steps')
parser.setMustHaveParameter('Input:Timestep')
parser.setMustHaveStatistic('Molecular Dynamics Simulation Performance')
parser.setMustHaveStatistic('Time Spent in External Energy Calculation')
parser.setMustHaveStatistic('Time Spent in Integration')
parser.setMustHaveStatistic('Time Spent in Internal Energy Calculation')
parser.setMustHaveStatistic('Time Spent in Non-Bond List Generation')
parser.setMustHaveStatistic('Time Spent in Waiting (Load Unbalance-ness)')
parser.setMustHaveStatistic('User Time')
parser.setMustHaveStatistic('Wall Clock Time')
#parse common parameters and statistics
parser.parseCommonParsAndStats(appstdout,stdout,stderr,geninfo)
#read output
lines=[]
if os.path.isfile(appstdout):
fin=open(appstdout,"rt")
lines=fin.readlines()
fin.close()
#process the output
parser.successfulRun=False
wallClockTime = 0.0
numSteps = 0
stepSize = 0.0
timeBreakdownColumns=None
numAtoms = 0
numBonds = 0
numAngles = 0
numDihedrals = 0
j=0
while j<len(lines):
m0=re.search(r'\s+Chemistry at HARvard Macromolecular Mechanics',lines[j])
m1=re.search(r'\sVersion\s+([\da-zA-Z]+)',lines[j+1])
if m0 and m1:parser.setParameter("App:Version",m1.group(1).strip())
if re.search(r'Summary of the structure file counters',lines[j]):
j+=1
for k in range(256):
if re.search(r'CHARMM>',lines[j]):break
m=re.search(r'Number of atoms\s+=\s+(\d+)',lines[j])
if m:numAtoms+=int(m.group(1).strip())
m=re.search(r'Number of bonds\s+=\s+(\d+)',lines[j])
if m:numBonds+=int(m.group(1).strip())
m=re.search(r'Number of angles\s+=\s+(\d+)',lines[j])
if m:numAngles+=int(m.group(1).strip())
m=re.search(r'Number of dihedrals\s+=\s+(\d+)',lines[j])
if m:numDihedrals+=int(m.group(1).strip())
j+=1
if re.search(r'<MAKGRP> found',lines[j]):
j+=1
for k in range(256):
if re.search(r'NUMBER OF DEGREES OF FREEDOM',lines[j]):break
m=re.search(r'NSTEP\s+=\s+(\d+)',lines[j])
if m:
numSteps=int(m.group(1).strip())
parser.setParameter( "Input:Number of Steps", numSteps )
if re.search(r'TIME STEP\s+=',lines[j]):
m=re.search(r'([\d\-Ee\.]+)\s+PS',lines[j])
if m:
stepSize = 1000.0*float(m.group(1).strip())
parser.setParameter( "Input:Timestep", stepSize *1e-15, "Second per Step" )
j+=1
if re.search(r'NORMAL TERMINATION BY NORMAL STOP',lines[j]):
parser.successfulRun=True
if re.search(r'JOB ACCOUNTING INFORMATION',lines[j]):
parser.successfulRun=True
j+=1
for k in range(256):
if j>len(lines)-1: break
m=re.search(r'ELAPSED TIME:\s*([\d\.]+)\s*MINUTES',lines[j])
if m:
wallClockTime=60.0 * float(m.group(1).strip())
parser.setStatistic( "Wall Clock Time", wallClockTime, "Second" )
m=re.search(r'CPU TIME:\s*([\d\.]+)\s*MINUTES',lines[j])
if m:parser.setStatistic( "User Time", 60.0 * float(m.group(1).strip()), "Second" )
m=re.search(r'ELAPSED TIME:\s*([\d\.]+)\s*SECONDS',lines[j])
if m:
wallClockTime=float(m.group(1).strip())
parser.setStatistic( "Wall Clock Time", wallClockTime, "Second" )
m=re.search(r'CPU TIME:\s*([\d\.]+)\s*SECONDS',lines[j])
if m:parser.setStatistic( "User Time", m.group(1).strip(), "Second" )
j+=1
if j>len(lines)-1: break
if re.search(r'Parallel load balance \(sec',lines[j]):
j+=1
# grab the column headers from the output, e.g.
#
# Parallel load balance (sec.):
# Node Eext Eint Wait Comm List Integ Total
# 0 205.5 6.4 1.2 31.2 23.2 2.8 270.4
# 1 205.2 7.3 1.1 31.2 23.3 3.2 271.2
# 2 205.2 7.7 0.6 32.3 23.3 3.2 272.3
# 3 205.2 7.8 0.6 32.1 23.3 3.3 272.3
#PARALLEL> Average timing for all nodes:
# 4 205.3 7.3 0.9 31.7 23.3 3.1 271.6
timeBreakdownColumns=lines[j].strip().split()
if re.search(r'PARALLEL>\s*Average timing for all nodes',lines[j]) and timeBreakdownColumns:
j+=1
timeBreakdown = lines[j].strip().split()
if len(timeBreakdownColumns)==len(timeBreakdown):
for k in range(len(timeBreakdown)):
if timeBreakdownColumns[k] == "Eext":
parser.setStatistic( "Time Spent in External Energy Calculation", timeBreakdown[k], "Second" )
if timeBreakdownColumns[k] == "Eint":
parser.setStatistic( "Time Spent in Internal Energy Calculation", timeBreakdown[k], "Second" )
if timeBreakdownColumns[k] == "Wait":
parser.setStatistic( "Time Spent in Waiting (Load Unbalance-ness)", timeBreakdown[k], "Second" )
if timeBreakdownColumns[k] == "List":
parser.setStatistic( "Time Spent in Non-Bond List Generation", timeBreakdown[k], "Second" )
if timeBreakdownColumns[k] == "Integ":
parser.setStatistic( "Time Spent in Integration", timeBreakdown[k], "Second" )
j+=1
if numAtoms>0:parser.setParameter( "Input:Number of Atoms", numAtoms )
if numBonds>0:parser.setParameter( "Input:Number of Bonds", numBonds )
if numAngles>0:parser.setParameter( "Input:Number of Angles", numAngles )
if numDihedrals>0:parser.setParameter( "Input:Number of Dihedrals", numDihedrals )
if wallClockTime>0.0 and numSteps>0 and stepSize>0.0:
# $stepSize is in femtoseconds
# $wallClockTime is in seconds
parser.setStatistic( "Molecular Dynamics Simulation Performance", ( 1e-6 * stepSize * numSteps ) / ( wallClockTime / 86400.0 )*1e-9, "Second per Day" )
if __name__ == "__main__":
#output for testing purpose
print "parsing complete:",parser.parsingComplete()
parser.printParsNStatsAsMustHave()
print parser.getXML()
#return complete XML overwize return None
return parser.getXML()
def processAppKerOutput(appstdout=None,stdout=None,stderr=None,geninfo=None,appKerNResVars=None):
#set App Kernel Description
parser=AppKerOutputParser(
name = 'xdmod.benchmark.io.ior',
version = 1,
description = "IOR (Interleaved-Or-Random) Benchmark",
url = 'http://freshmeat.net/projects/ior',
measurement_name = 'IOR'
)
#set obligatory parameters and statistics
#set common parameters and statistics
parser.setCommonMustHaveParsAndStats()
#set app kernel custom sets
parser.setMustHaveParameter('App:Version')
if appKerNResVars==None or (appKerNResVars!=None and 'testHDF5' in appKerNResVars and appKerNResVars['testHDF5']==True):
parser.setMustHaveParameter('HDF Version')
parser.setMustHaveParameter('HDF5 Collective N-to-1 Test File System')
parser.setMustHaveParameter('HDF5 Independent N-to-1 Test File System')
parser.setMustHaveParameter('HDF5 N-to-N Test File System')
if appKerNResVars==None or (appKerNResVars!=None and 'testMPIIO' in appKerNResVars and appKerNResVars['testMPIIO']==True):
parser.setMustHaveParameter('MPIIO Collective N-to-1 Test File System')
parser.setMustHaveParameter('MPIIO Independent N-to-1 Test File System')
parser.setMustHaveParameter('MPIIO N-to-N Test File System')
if appKerNResVars==None or (appKerNResVars!=None and 'testPOSIX' in appKerNResVars and appKerNResVars['testPOSIX']==True):
parser.setMustHaveParameter('POSIX N-to-1 Test File System')
parser.setMustHaveParameter('POSIX N-to-N Test File System')
if appKerNResVars==None or (appKerNResVars!=None and 'testNetCDF' in appKerNResVars and appKerNResVars['testNetCDF']==True):
parser.setMustHaveParameter('Parallel NetCDF Collective N-to-1 Test File System')
parser.setMustHaveParameter('Parallel NetCDF Independent N-to-1 Test File System')
parser.setMustHaveParameter('Parallel NetCDF Version')
parser.setMustHaveParameter('Per-Process Data Size')
parser.setMustHaveParameter('Per-Process I/O Block Size')
parser.setMustHaveParameter('RunEnv:Nodes')
parser.setMustHaveParameter('Transfer Size Per I/O')
if appKerNResVars==None or (appKerNResVars!=None and 'testHDF5' in appKerNResVars and appKerNResVars['testHDF5']==True):
parser.setMustHaveStatistic('HDF5 Collective N-to-1 Read Aggregate Throughput')
parser.setMustHaveStatistic('HDF5 Collective N-to-1 Write Aggregate Throughput')
parser.setMustHaveStatistic('HDF5 Independent N-to-1 Read Aggregate Throughput')
parser.setMustHaveStatistic('HDF5 Independent N-to-1 Write Aggregate Throughput')
parser.setMustHaveStatistic('HDF5 N-to-N Read Aggregate Throughput')
parser.setMustHaveStatistic('HDF5 N-to-N Write Aggregate Throughput')
if appKerNResVars==None or (appKerNResVars!=None and 'testMPIIO' in appKerNResVars and appKerNResVars['testMPIIO']==True):
parser.setMustHaveStatistic('MPIIO Collective N-to-1 Read Aggregate Throughput')
parser.setMustHaveStatistic('MPIIO Collective N-to-1 Write Aggregate Throughput')
parser.setMustHaveStatistic('MPIIO Independent N-to-1 Read Aggregate Throughput')
parser.setMustHaveStatistic('MPIIO Independent N-to-1 Write Aggregate Throughput')
parser.setMustHaveStatistic('MPIIO N-to-N Read Aggregate Throughput')
parser.setMustHaveStatistic('MPIIO N-to-N Write Aggregate Throughput')
if appKerNResVars==None or (appKerNResVars!=None and 'testPOSIX' in appKerNResVars and appKerNResVars['testPOSIX']==True):
parser.setMustHaveStatistic('POSIX N-to-1 Read Aggregate Throughput')
parser.setMustHaveStatistic('POSIX N-to-1 Write Aggregate Throughput')
parser.setMustHaveStatistic('POSIX N-to-N Read Aggregate Throughput')
parser.setMustHaveStatistic('POSIX N-to-N Write Aggregate Throughput')
if appKerNResVars==None or (appKerNResVars!=None and 'testNetCDF' in appKerNResVars and appKerNResVars['testNetCDF']==True):
parser.setMustHaveStatistic('Parallel NetCDF Collective N-to-1 Read Aggregate Throughput')
parser.setMustHaveStatistic('Parallel NetCDF Collective N-to-1 Write Aggregate Throughput')
parser.setMustHaveStatistic('Parallel NetCDF Independent N-to-1 Read Aggregate Throughput')
parser.setMustHaveStatistic('Parallel NetCDF Independent N-to-1 Write Aggregate Throughput')
parser.setMustHaveStatistic('Number of Tests Passed')
parser.setMustHaveStatistic('Number of Tests Started')
parser.setMustHaveStatistic('Wall Clock Time')
parser.completeOnPartialMustHaveStatistics=True
#parse common parameters and statistics
parser.parseCommonParsAndStats(appstdout,stdout,stderr,geninfo)
if hasattr(parser,'appKerWallClockTime'):
parser.setStatistic("Wall Clock Time", total_seconds(parser.appKerWallClockTime), "Second")
#read output
lines=[]
if os.path.isfile(appstdout):
fin=open(appstdout,"rt")
lines=fin.readlines()
fin.close()
#process the output
#find which version of IOR was used
ior_output_version=None
j=0
while j<len(lines)-1:
#IOR RELEASE: IOR-2.10.3
m=re.match(r'^#\s+IOR RELEASE:\s(.+)',lines[j])
if m:ior_output_version=2
m=re.match(r'^IOR-[3-9]\.[0-9]+\.[0-9]: MPI Coordinated Test of Parallel I/O',lines[j])
if m:ior_output_version=3
j+=1
if ior_output_version==None:
print "ERROR: unknown version of IOR output!!!"
testsPassed=0
totalNumberOfTests=0
parser.successfulRun=False
if ior_output_version==2:
METRICS={}
j=-1
while j<len(lines)-1:
m=re.match(r'^# (.+?):(.+)',lines[j])
if m:
METRICS[m.group(1).strip()]=m.group(2).strip()
if m.group(1).strip()=="segmentCount":
METRICS[m.group(1).strip()]=m.group(2).strip().split()[0]
m=re.match(r'^# IOR command line used:',lines[j])
if m:totalNumberOfTests+=1
if "IOR RELEASE" in METRICS:
parser.setParameter( "App:Version", METRICS["IOR RELEASE"])
if "Compile-time HDF Version" in METRICS:
parser.setParameter( "HDF Version", METRICS["Compile-time HDF Version"])
if "Compile-time PNETCDF Version" in METRICS:
parser.setParameter( "Parallel NetCDF Version", METRICS["Compile-time PNETCDF Version"])
if "blockSize" in METRICS and "segmentCount" in METRICS:
#print METRICS["blockSize"],METRICS["segmentCount"]
parser.setParameter( "Per-Process Data Size", ( float(METRICS["blockSize"]) / 1024.0 / 1024.0 ) * int(METRICS["segmentCount"]), "MByte" )
parser.setParameter( "Per-Process I/O Block Size", ( float(METRICS["blockSize"]) / 1024.0 / 1024.0 ), "MByte" )
if "reorderTasks" in METRICS:
if int(METRICS["reorderTasks"])!=0:
parser.setParameter( "Reorder Tasks for Read-back Tests", "Yes" )
if "repetitions" in METRICS:
if 1 < int(METRICS["repetitions"]):
parser.setParameter( "Test Repetitions", METRICS["repetitions"])
if "transferSize" in METRICS:
parser.setParameter( "Transfer Size Per I/O", ( float(METRICS["transferSize"]) / 1024.0 / 1024.0 ), "MByte" )
if "mpiio hints passed to MPI_File_open" in METRICS:
parser.setParameter( "MPI-IO Hints", METRICS["mpiio hints passed to MPI_File_open"])
if "Write bandwidth" in METRICS and "Read bandwidth" in METRICS and \
"api" in METRICS and "filePerProc" in METRICS and "collective" in METRICS and \
"randomOffset" in METRICS and "Run finished" in METRICS:
testsPassed+=1
label = METRICS["api"];
m=re.search(r'NCMPI',label,re.I)
if m:label = "Parallel NetCDF"
m=re.search(r'POSIX',label,re.I)
if m and "useO_DIRECT" in METRICS:
if int(METRICS["useO_DIRECT"]) != 0:
label += ' (O_DIRECT)'
if m==None:
# POSIX doesn't have collective I/O
if int(METRICS["collective"]) == 0: label += ' Independent'
else: label += ' Collective'
if int(METRICS["randomOffset"]) == 0: label += ''
else: label += ' Random'
if int(METRICS["filePerProc"]) == 0: label += ' N-to-1'
else: label += ' N-to-N'
# for N-to-N (each process writes to its own file), it must be
# Independent, so we can remove the redundant words such as
# "Independent" and "Collective"
m=re.search(r' (Independent|Collective).+N-to-N',label,re.I)
if m:
label=label.replace(" Independent","").replace(" Collective","")
# now we have the label, get test-specific parameters
m=re.search(r'MPIIO',label,re.I)
if m:
if "useFileView" in METRICS:
if 0 != int(METRICS["useFileView"]):
parser.setParameter( label + " Uses MPI_File_set_view", "Yes" )
if "useSharedFilePointer" in METRICS:
if 0 != int(METRICS["useSharedFilePointer"]):
parser.setParameter( label + " Uses Shared File Pointer", "Yes" )
m=re.search(r'POSIX',label,re.I)
if m:
if "fsyncPerWrite" in METRICS:
if 0 != int(METRICS["fsyncPerWrite"]):
parser.setParameter( label + " Uses fsync per Write", "Yes" )
m=re.search(r'mean=(\S+)',METRICS["Write bandwidth"])
if m:
metric = m.group(1).strip()
writeLabel = label
# writes are always sequential
writeLabel = writeLabel.replace(" Random","")
parser.setStatistic( writeLabel+" Write Aggregate Throughput", "%.2f"%(float(metric) / 1024.0 / 1024.0, ), "MByte per Second" );
#parser.setParameter( "${writeLabel} Test File", METRICS["testFileName"} ) if ( exists METRICS["testFileName"} );
if "fileSystem" in METRICS:
parser.setParameter( writeLabel+" Test File System", METRICS["fileSystem"])
parser.successfulRun=True
if "File Open Time (Write)" in METRICS:
m2=re.search(r'mean=(\S+)',METRICS["File Open Time (Write)"])
if m2:
parser.setStatistic( writeLabel+" File Open Time (Write)", m2.group(1).strip(), "Second" );
if "File Close Time (Write)" in METRICS:
m2=re.search(r'mean=(\S+)',METRICS["File Close Time (Write)"])
if m2:
parser.setStatistic( writeLabel+" File Close Time (Write)", m2.group(1).strip(), "Second" );
m=re.search(r'mean=(\S+)',METRICS["Read bandwidth"])
if m:
parser.setStatistic( label+" Read Aggregate Throughput", "%.2f"%(float(m.group(1).strip()) / 1024.0 / 1024.0, ), "MByte per Second" );
parser.successfulRun=True
if "File Open Time (Read)" in METRICS:
m2=re.search(r'mean=(\S+)',METRICS["File Open Time (Read)"])
if m2:
parser.setStatistic( writeLabel+" File Open Time (Read)", m2.group(1).strip(), "Second" );
if "File Close Time (Read)" in METRICS:
m2=re.search(r'mean=(\S+)',METRICS["File Close Time (Read)"])
if m2:
parser.setStatistic( writeLabel+" File Close Time (Read)", m2.group(1).strip(), "Second" );
METRICS={}
j+=1
if ior_output_version==3:
i=0
input_summary={}
rsl_w={}
rsl_r={}
filesystem=None
while i<len(lines)-1:
m=re.match(r'^IOR-([3-9]\.[0-9]+\.[0-9]+): MPI Coordinated Test of Parallel I/O',lines[i])
if m:parser.setParameter( "App:Version", m.group(1).strip())
m=re.match(r'^File System To Test:(.+)',lines[i])
if m:filesystem=m.group(1).strip()
m=re.match(r'^# Starting Test:',lines[i])
if m:totalNumberOfTests+=1
m0=re.match(r'^Summary:$',lines[i])
if m0:
#input summary section
input_summary={}
i+=1
while i<len(lines):
m1=re.match(r'^\t([^=\n\r\f\v]+)=(.+)',lines[i])
if m1:
input_summary[m1.group(1).strip()]=m1.group(2).strip()
else:
break
i+=1
#process input_summary
input_summary['filesystem']=filesystem
input_summary['API']=input_summary['api']
if input_summary['api'].count("MPIIO")>0:
input_summary['API']="MPIIO"
input_summary['API_Version']=input_summary['api'].replace("MPIIO","").strip()
parser.setParameter( "MPIIO Version", input_summary['API_Version'])
if input_summary['api'].count("HDF5")>0:
input_summary['API']="HDF5"
input_summary['API_Version']=input_summary['api'].replace("HDF5-","").replace("HDF5","").strip()
parser.setParameter( "HDF Version", input_summary['API_Version'])
if input_summary['api'].count("NCMPI")>0:
input_summary['API']="Parallel NetCDF"
input_summary['API_Version']=input_summary['api'].replace("NCMPI","").strip()
parser.setParameter( "Parallel NetCDF Version", input_summary['API_Version'])
input_summary['fileAccessPattern']=""
input_summary['collectiveOrIndependent']=""
if input_summary['access'].count('single-shared-file')>0:
input_summary['fileAccessPattern']="N-to-1"
if input_summary['access'].count('file-per-process')>0:
input_summary['fileAccessPattern']="N-to-N"
if input_summary['access'].count('independent')>0:
input_summary['collectiveOrIndependent']="Independent"
if input_summary['access'].count('collective')>0:
input_summary['collectiveOrIndependent']="Collective"
if input_summary['fileAccessPattern']=="N-to-N" and input_summary['collectiveOrIndependent']=="Independent":
input_summary['collectiveOrIndependent']=""
if input_summary['collectiveOrIndependent']!="":
input_summary['method']=" ".join((input_summary['API'],
input_summary['collectiveOrIndependent'],
input_summary['fileAccessPattern']))
else:
input_summary['method']=" ".join((input_summary['API'],
input_summary['fileAccessPattern']))
if input_summary['filesystem']!=None:
parser.setParameter(input_summary['method']+' Test File System',input_summary['filesystem'])
if "pattern" in input_summary:
m1=re.match(r'^segmented \(([0-9]+) segment',input_summary["pattern"])
if m1:input_summary["segmentCount"]=int(m1.group(1).strip())
if "blocksize" in input_summary and "segmentCount" in input_summary:
val,unit=input_summary["blocksize"].split()
blockSize=getMiB(float(val),unit)
segmentCount=input_summary["segmentCount"]
parser.setParameter( "Per-Process Data Size", blockSize*segmentCount, "MByte" )
parser.setParameter( "Per-Process I/O Block Size", blockSize, "MByte" )
if "xfersize" in input_summary:
val,unit=input_summary["xfersize"].split()
transferSize=getMiB(float(val),unit)
parser.setParameter( "Transfer Size Per I/O", transferSize, "MByte" )
m0=re.match(r'^access\s+bw\(MiB/s\)\s+block\(KiB\)\s+xfer\(KiB\)\s+open\(s\)\s+wr/rd\(s\)\s+close\(s\)\s+total\(s\)\s+iter',lines[i])
if m0:
i+=1
while i<len(lines):
m1=re.match(r'^write\s+([0-9\.]+)+\s+([0-9\.]+)+\s+([0-9\.]+)+\s+([0-9\.]+)+\s+([0-9\.]+)+\s+([0-9\.]+)+\s+([0-9\.]+)+\s+([0-9\.]+)+',lines[i])
m2=re.match(r'^read\s+([0-9\.]+)+\s+([0-9\.]+)+\s+([0-9\.]+)+\s+([0-9\.]+)+\s+([0-9\.]+)+\s+([0-9\.]+)+\s+([0-9\.]+)+\s+([0-9\.]+)+',lines[i])
if m1 or m2:
if m1:
access="Write"
bw,block,xfer,open_s,wrrd_s,close_s,total_s,iter=m1.groups()
else:
access="Read"
bw,block,xfer,open_s,wrrd_s,close_s,total_s,iter=m2.groups()
testsPassed+=1
parser.successfulRun=True
parser.setStatistic( input_summary['method']+" %s Aggregate Throughput"%access, bw, "MByte per Second" );
parser.setStatistic( input_summary['method']+" File Open Time (%s)"%access, open_s, "Second" );
parser.setStatistic( input_summary['method']+" File Close Time (%s)"%access, close_s, "Second" );
m1=re.match(r'^Summary of all tests:',lines[i])
if m1:break
i+=1
#reset variables
input_summary={}
rsl_w={}
rsl_r={}
#filesystem=None
i+=1
parser.setStatistic('Number of Tests Passed',testsPassed )
parser.setStatistic('Number of Tests Started',totalNumberOfTests )
if __name__ == "__main__":
#output for testing purpose
print "parsing complete:",parser.parsingComplete(Verbose=True)
parser.printParsNStatsAsMustHave()
print parser.getXML()
#return complete XML overwize return None
return parser.getXML()
def processAppKerOutput(appstdout=None,stdout=None,stderr=None,geninfo=None,appKerNResVars=None):
#set App Kernel Description
parser=AppKerOutputParser(
name = 'xdmod.app.astro.enzo',
version = 1,
description = "Enzo: an Adaptive Mesh Refinement Code for Astrophysics",
url = 'http://enzo-project.org',
measurement_name = 'Enzo'
)
#set obligatory parameters and statistics
#set common parameters and statistics
parser.setCommonMustHaveParsAndStats()
#set app kernel custom sets
parser.setMustHaveParameter('App:Version')
parser.setMustHaveStatistic('Wall Clock Time')
#parse common parameters and statistics
parser.parseCommonParsAndStats(appstdout,stdout,stderr,geninfo)
#read output
lines=[]
if os.path.isfile(appstdout):
fin=open(appstdout,"rt")
lines=fin.readlines()
fin.close()
parser.setParameter("App:Version","unknown")
#process the output
successfulRun=False
j=0
while j<len(lines):
m=re.match(r'^Mercurial Branch\s+(\S+)',lines[j])
if m:
branch=m.group(1)
revision=""
if j+1<len(lines):
m=re.match(r'^Mercurial Revision\s+(\S+)',lines[j+1])
if m:revision=m.group(1)
parser.setParameter("App:Version","Branch:"+branch+" Revision:"+revision)
m=re.match(r'^Time\s*=\s*([0-9\.]+)\s+CycleNumber\s*=\s*([0-9]+)\s+Wallclock\s*=\s*([0-9\.]+)',lines[j])
if m:
parser.setStatistic("Final Simulation Time", m.group(1), "Enzo Time Unit")
parser.setStatistic("Total Cycles", m.group(2))
parser.setStatistic("Wall Clock Time", m.group(3), "Second")
successfulRun=True
m=re.match(r'^Successful run, exiting.',lines[j])
if m:
successfulRun=True
#performance
m=re.match(r'^Cycle_Number\s+([0-9]+)',lines[j])
if m:
j+=1
performanceMetrics={}
while j<len(lines):
if lines[j].strip()!="":
v=lines[j].strip().split()
if v[0] not in performanceMetrics:
performanceMetrics[v[0]]=float(v[1])
else:
performanceMetrics[v[0]]+=float(v[1])
else:
if j+1<len(lines):
m=re.match(r'^Cycle_Number\s+([0-9]+)',lines[j+1])
if m: pass
else: break
else:
break
j+=1
metric="CommunicationTranspose"
if metric in performanceMetrics:parser.setStatistic("Communication Transpose Time", performanceMetrics[metric], "Second")
metric="ComputePotentialFieldLevelZero"
if metric in performanceMetrics:parser.setStatistic("Gravitational Potential Field Computing Time", performanceMetrics[metric], "Second")
metric="EvolvePhotons"
if metric in performanceMetrics:parser.setStatistic("Radiative Transfer Calculation Time", performanceMetrics[metric], "Second")
metric="Group_WriteAllData"
if metric in performanceMetrics:parser.setStatistic("All Data Group Write Time", performanceMetrics[metric], "Second")
metric="Level_00"
if metric in performanceMetrics:parser.setStatistic("All Grid Level 00 Calculation Time", performanceMetrics[metric], "Second")
metric="Level_01"
if metric in performanceMetrics:parser.setStatistic("All Grid Level 01 Calculation Time", performanceMetrics[metric], "Second")
metric="Level_02"
if metric in performanceMetrics:parser.setStatistic("All Grid Level 02 Calculation Time", performanceMetrics[metric], "Second")
metric="RebuildHierarchy"
if metric in performanceMetrics:parser.setStatistic("Grid Hierarchy Rebuilding Time", performanceMetrics[metric], "Second")
metric="SetBoundaryConditions"
if metric in performanceMetrics:parser.setStatistic("Boundary Conditions Setting Time", performanceMetrics[metric], "Second")
metric="SolveForPotential"
if metric in performanceMetrics:parser.setStatistic("Poisson Equation Solving Time", performanceMetrics[metric], "Second")
metric="SolveHydroEquations"
if metric in performanceMetrics:parser.setStatistic("Hydro Equations Solving Time", performanceMetrics[metric], "Second")
metric="Total"
if metric in performanceMetrics:parser.setStatistic("Total Time Spent in Cycles", performanceMetrics[metric], "Second")
j+=1
if __name__ == "__main__":
#output for testing purpose
print "parsing complete:",parser.parsingComplete()
parser.printParsNStatsAsMustHave()
print parser.getXML()
#return complete XML overwize return None
return parser.getXML()
def PushToDBRaw(self,cur,task_id,time_finished,Verbose=True):
print "Pushing to DB"
resultFile=os.path.join(self.taskDir,"result.xml")
jobfilesDir=os.path.join(self.taskDir,"jobfiles")
(batchJobDir,stdoutFile,stderrFile,appstdoutFile,taskexeclogFile)=self.GetResultFiles()
#sanity check
fin=open(resultFile,"r")
content=fin.read()
fin.close()
if content[0]!='<':
#need to reformat
i0=content.find("<rep:report")
i1=content.find("</rep:report>")
fout=open(resultFile,"w")
content=fout.write("<?xml version='1.0'?>\n"+content[i0:i1+len("</rep:report>")]+"\n")
fout.close()
import xml.etree.ElementTree as ET
try:
tree = ET.parse(resultFile)
root = tree.getroot()
except:
self.statusinfo = "(1)==Resulting XML file content==\n"+content+\
"\n==Previous status=="+self.status+\
"\n==Previous status info=="+self.statusinfo+"\n"+\
traceback.format_exc()
self.status="Cannot process final XML file"
self.WriteErrorXML(resultFile,bCDATA=True)
instance_id=task_id
collected=None
committed=None
resource=None
executionhost=None
reporter=None
reporternickname=None
status=None
message=None
stderr=None
body=None
memory=None
cputime=None
walltime=None
memory=0.0
cputime=0.0
walltime=0.0
job_id=None
cur.execute("""SELECT instance_id,collected,committed,resource,executionhost,reporter,reporternickname,status,message,stderr,body,memory,cputime,walltime,job_id
FROM akrr_xdmod_instanceinfo WHERE instance_id=%s""",(task_id,))
raw=cur.fetchone()
if raw!=None: #.i.e. not new entry
(instance_id,collected,committed,resource,executionhost,reporter,reporternickname,status,message,stderr,body,memory,cputime,walltime,job_id)=raw
if hasattr(self,"RemoteJobID"):job_id=self.RemoteJobID
else:
collected=time_finished
committed=datetime.datetime.today().strftime("%Y-%m-%d %H:%M:%S")
resource=self.resourceName
executionhost=self.resource.get('__regexp__',self.resourceName)
reporter=self.appName
#reporternickname="%s.%d"%(self.appName,self.resourceParam['ncpus'])
reporternickname=akrr.replaceATvarAT(self.app['nickname'],[self.resource,self.app,self.resourceParam, self.appParam])
if hasattr(self,"RemoteJobID"):job_id=self.RemoteJobID
#Process XML file
import xml.etree.ElementTree as ET
root=None
try:
tree = ET.parse(resultFile)
root = tree.getroot()
except:
self.statusinfo = "(2)==Resulting XML file content==\n"+content+\
"\n==Previous status=="+self.status+\
"\n==Previous status info=="+self.statusinfo
self.status="Cannot process final XML file"
root=None
memory=0.0
cputime=0.0
walltime=0.0
print "TTTTTTTT ",root,status
if root!=None:
completed=None
try:
t=root.find('exitStatus').find('completed').text
print 'exitStatus:completed',t
if t.strip().upper()=="TRUE":completed=True
else: completed=False
if completed:
root.find('body').find('performance').find('benchmark').find('statistics')
self.status="Task was completed successfully."
self.statusinfo="Done"
except:
completed=None
print "TTTTTTTT completedstatus",completed,status
error=None
try:
t=root.find('xdtas').find('batchJob').find('status').text
if t.upper()=="ERROR":error=True
else: error=False
except:
error=None
status=0
if completed!=None:
if completed:
status=1
print "TTTTTTTT completedstatus",completed,status
if completed!=None:
if completed==True:
bodyET=root.find('body').find('performance')
body=ET.tostring(bodyET)
import xml.dom.minidom
xml = xml.dom.minidom.parseString(body)
body = xml.toprettyxml()
body=body.replace("""<?xml version="1.0" ?>\n""","")
statisticsET=root.find('body').find('performance').find('benchmark').find('statistics')
for e in statisticsET:
if e.find('ID').text=='Memory':
memory=float(e.find('value').text)
if e.find('ID').text=='Wall Clock Time':
walltime=float(e.find('value').text)
else:
#i.e. error caught by parser on resource
try:
message="ERROR: error caught by parser"
stderr=root.find('exitStatus').find('errorMessage').text
body="""<xdtas>
<batchJob>
<status>Error</status>
<errorCause>ERROR: error caught by parser</errorCause>
<reporter>%s</reporter>
<errorMsg>%s</errorMsg>
</batchJob>
</xdtas>
"""%(reporter,stderr)
except:
message="ERROR: error caught by parser"
stderr=None
body="""<xdtas>
<batchJob>
<status>Error</status>
<errorCause>ERROR: error caught by parser</errorCause>
<reporter>%s</reporter>
<errorMsg></errorMsg>
</batchJob>
</xdtas>
"""%(reporter)
elif completed==None:
if error!=None: # i.e. xml with error generated afterwards
if error==True:
#fin=open(resultFile,"r")
#body=fin.read()
#fin.close
message=root.find('xdtas').find('batchJob').find('errorCause').text
stderr=root.find('xdtas').find('batchJob').find('errorMsg').text
bodyET=root.find('xdtas')
body=ET.tostring(bodyET)
else:
raise IOError("This condition should not happens")
else:
raise IOError("This condition should not happens")
else:
raise IOError("This condition should not happens")
#memory=None
#cputime=None
#walltime=None
else:#i.e. xml was not parsed
status=0
message=self.status
stderr=self.statusinfo
body="""<xdtas>
<batchJob>
<status>Error</status>
<errorCause>Unknown error</errorCause>
<reporter>AKRR Server</reporter>
<errorMsg>"Cannot process final XML file</errorMsg>
</batchJob>
</xdtas>
"""
message=akrr.CleanUnicode(message)
stderr=akrr.CleanUnicode(stderr)
body=akrr.CleanUnicode(body)
#Get Nodes
nodes=None
nodesFileName=os.path.join(jobfilesDir,"gen.info")
if(os.path.isfile(nodesFileName)):
parser=AppKerOutputParser()
parser.parseCommonParsAndStats(geninfo=nodesFileName)
if hasattr(parser,'geninfo') and 'nodeList' in parser.geninfo:
nodesList=parser.geninfo['nodeList'].split()
nodes=";"
for line in nodesList:
line=line.strip()
nodes+="%s;"%(line)
if len(nodes.strip().strip(';'))==0:
nodes=None
import xml.dom.minidom
#print "#"*120
#print resultFile
#print instance_id,collected,committed,resource,executionhost,reporter,reporternickname,status,message,stderr,memory,cputime,walltime
#print body
#cur.execute("""SELECT * FROM akrr_xdmod_instanceinfo WHERE instance_id=%s""",(task_id,))
internal_failure_code=0
if 'masterTaskID' in self.taskParam and appstdoutFile==None:
internal_failure_code=10004
print "TTTTTTTT completedstatus",completed,status
if raw!=None: #.i.e. new entry
print "Updating"
cur.execute("""UPDATE akrr_xdmod_instanceinfo
SET instance_id=%s,collected=%s,committed=%s,resource=%s,executionhost=%s,reporter=%s,
reporternickname=%s,status=%s,message=%s,stderr=%s,body=%s,memory=%s,cputime=%s,walltime=%s,job_id=%s,nodes=%s,internal_failure=%s
WHERE instance_id=%s""",
(instance_id,collected,committed,resource,executionhost,reporter,reporternickname,status,message,stderr,body,memory,cputime,walltime,job_id,nodes,internal_failure_code,
instance_id))
#(instance_id,collected,committed,resource,executionhost,reporter,reporternickname,status,message,stderr,body,memory,cputime,walltime)=raw
else:
cur.execute("""INSERT INTO akrr_xdmod_instanceinfo
(instance_id,collected,committed,resource,executionhost,reporter,reporternickname,status,message,stderr,body,memory,cputime,walltime,job_id,nodes,internal_failure)
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)""",
(instance_id,collected,committed,resource,executionhost,reporter,reporternickname,status,message,stderr,body,memory,cputime,walltime,job_id,nodes,internal_failure_code))
#print appstdoutFile,stdoutFile,stderrFile
appstdoutFileContent=None
stdoutFileContent=None
stderrFileContent=None
taskexeclogFileContent=None
if appstdoutFile!=None:
fin=open(appstdoutFile,"r")
appstdoutFileContent=fin.read()
fin.close()
else:
appstdoutFileContent="Does Not Present"
if stdoutFile!=None:
fin=open(stdoutFile,"r")
stdoutFileContent=fin.read()
fin.close()
else:
stdoutFileContent="Does Not Present"
if stderrFile!=None:
fin=open(stderrFile,"r")
stderrFileContent=fin.read()
fin.close()
else:
stderrFileContent="Does Not Present"
if taskexeclogFile!=None:
fin=open(taskexeclogFile,"r")
taskexeclogFileContent=fin.read()
fin.close()
else:
taskexeclogFileContent="Does Not Present"
cur.execute("""SELECT task_id
FROM akrr_errmsg WHERE task_id=%s""",(task_id,))
raw=cur.fetchall()
cur.execute("""SELECT @@max_allowed_packet""")
max_allowed_packet=cur.fetchall()[0][0]
if (len(appstdoutFileContent)+len(stderrFileContent)+len(stdoutFileContent)+len(taskexeclogFileContent))>0.9*max_allowed_packet:
print "WARNING: length of files exceed max_allowed_packet will trancate files"
if len(appstdoutFileContent)>0.2*max_allowed_packet:
appstdoutFileContent=appstdoutFileContent[:int(0.2*max_allowed_packet)]
appstdoutFileContent+="\nWARNING: File was trancated because it length of files exceed max_allowed_packet\n"
if len(stderrFileContent)>0.2*max_allowed_packet:
stderrFileContent=stderrFileContent[:int(0.2*max_allowed_packet)]
stderrFileContent+="\nWARNING: File was trancated because it length of files exceed max_allowed_packet\n"
if len(stdoutFileContent)>0.2*max_allowed_packet:
stdoutFileContent=stdoutFileContent[:int(0.2*max_allowed_packet)]
stdoutFileContent+="\nWARNING: File was trancated because it length of files exceed max_allowed_packet\n"
if len(taskexeclogFileContent)>0.2*max_allowed_packet:
taskexeclogFileContent=taskexeclogFileContent[:int(0.2*max_allowed_packet)]
taskexeclogFileContent+="\nWARNING: File was trancated because it length of files exceed max_allowed_packet\n"
if len(raw)>0: #.i.e. updating existing entry
print "Updating",raw
cur.execute("""UPDATE akrr_errmsg
SET appstdout=%s,stderr=%s,stdout=%s,taskexeclog=%s
WHERE task_id=%s""",
((appstdoutFileContent),(stderrFileContent),(stdoutFileContent),taskexeclogFileContent,instance_id))
else:
cur.execute("""INSERT INTO akrr_errmsg
(task_id,appstdout,stderr,stdout,taskexeclog)
VALUES (%s,%s,%s,%s,%s)""",
(instance_id,(appstdoutFileContent),(stderrFileContent),(stdoutFileContent),taskexeclogFileContent))
def processAppKerOutput(appstdout=None,stdout=None,stderr=None,geninfo=None,appKerNResVars=None):
#set App Kernel Description
parser=AppKerOutputParser(
name = 'xdmod.benchmark.hpcc',
version = 1,
description = "HPC Challenge Benchmarks",
url = 'http://icl.cs.utk.edu/hpcc/',
measurement_name = 'xdmod.benchmark.hpcc'
)
#set obligatory parameters and statistics
#set common parameters and statistics
parser.setCommonMustHaveParsAndStats()
#set app kernel custom sets
parser.setMustHaveParameter('App:Version')
parser.setMustHaveParameter('Input:DGEMM Problem Size')
parser.setMustHaveParameter('Input:High Performance LINPACK Grid Cols')
parser.setMustHaveParameter('Input:High Performance LINPACK Grid Rows')
parser.setMustHaveParameter('Input:High Performance LINPACK Problem Size')
parser.setMustHaveParameter('Input:MPI Ranks')
parser.setMustHaveParameter('Input:MPIRandom Problem Size')
parser.setMustHaveParameter('Input:OpenMP Threads')
parser.setMustHaveParameter('Input:PTRANS Problem Size')
parser.setMustHaveParameter('Input:STREAM Array Size')
parser.setMustHaveParameter('RunEnv:CPU Speed')
parser.setMustHaveParameter('RunEnv:Nodes')
parser.setMustHaveStatistic('Average Double-Precision General Matrix Multiplication (DGEMM) Floating-Point Performance')
parser.setMustHaveStatistic("Average STREAM 'Add' Memory Bandwidth")
parser.setMustHaveStatistic("Average STREAM 'Copy' Memory Bandwidth")
parser.setMustHaveStatistic("Average STREAM 'Scale' Memory Bandwidth")
parser.setMustHaveStatistic("Average STREAM 'Triad' Memory Bandwidth")
parser.setMustHaveStatistic('Fast Fourier Transform (FFTW) Floating-Point Performance')
parser.setMustHaveStatistic('High Performance LINPACK Efficiency')
parser.setMustHaveStatistic('High Performance LINPACK Floating-Point Performance')
parser.setMustHaveStatistic('High Performance LINPACK Run Time')
parser.setMustHaveStatistic('MPI Random Access')
parser.setMustHaveStatistic('Parallel Matrix Transpose (PTRANS)')
parser.setMustHaveStatistic('Wall Clock Time')
#parse common parameters and statistics
parser.parseCommonParsAndStats(appstdout,stdout,stderr,geninfo)
if hasattr(parser,'appKerWallClockTime'):
parser.setStatistic("Wall Clock Time", total_seconds(parser.appKerWallClockTime), "Second")
# Intel MPI benchmark suite contains three classes of benchmarks:
#
# Single-transfer, which needs only 2 processes
# Parallel-transfer, which can use as many processes that are available
# Collective, which can use as many processes that are available
# The parameters mapping table
Params = {
"CommWorldProcs" : [ "MPI Ranks", "", "" ],
"HPL_N" : [ "High Performance LINPACK Problem Size", "", "" ],
"HPL_nprow" : [ "High Performance LINPACK Grid Rows", "", "" ],
"HPL_npcol" : [ "High Performance LINPACK Grid Cols", "", "" ],
"PTRANS_n" : [ "PTRANS Problem Size", "", "" ],
"MPIRandomAccess_N" : [ "MPIRandom Problem Size", "MWord", "val/1024/1024" ],
"STREAM_VectorSize" : [ "STREAM Array Size", "MWord", "" ],
"DGEMM_N" : [ "DGEMM Problem Size", "", "" ],
"omp_get_num_threads" : [ "OpenMP Threads", "", "" ],
}
# The result mapping table
Metrics = {
"HPL_Tflops" : [ "High Performance LINPACK Floating-Point Performance",
"MFLOP per Second", "val*1e6" ],
"HPL_time" : [ "High Performance LINPACK Run Time",
"Second", "" ],
"PTRANS_GBs" : [ "Parallel Matrix Transpose (PTRANS)",
"MByte per Second", "val*1024" ],
"MPIRandomAccess_GUPs" : [ "MPI Random Access",
"MUpdate per Second", "val*1000" ],
"MPIFFT_Gflops" : [ "Fast Fourier Transform (FFTW) Floating-Point Performance",
"MFLOP per Second", "val*1000" ],
"StarDGEMM_Gflops" : [ "Average Double-Precision General Matrix Multiplication (DGEMM) Floating-Point Performance",
"MFLOP per Second", "val*1000" ],
"StarSTREAM_Copy" : [ "Average STREAM 'Copy' Memory Bandwidth",
"MByte per Second", "val*1024" ],
"StarSTREAM_Scale" : [ "Average STREAM 'Scale' Memory Bandwidth",
"MByte per Second", "val*1024" ],
"StarSTREAM_Add" : [ "Average STREAM 'Add' Memory Bandwidth",
"MByte per Second", "val*1024" ],
"StarSTREAM_Triad" : [ "Average STREAM 'Triad' Memory Bandwidth",
"MByte per Second", "val*1024" ]
}
#read output
lines=[]
if os.path.isfile(appstdout):
fin=open(appstdout,"rt")
lines=fin.readlines()
fin.close()
#process the output
parser.successfulRun=False
resultBegin=None
hpl_tflops=None
numCores=None
values={}
j=-1
while j<len(lines)-1:
j+=1
m=re.search(r'End of HPC Challenge tests',lines[j])
if m:parser.successfulRun=True
m=re.match(r'^Begin of Summary section',lines[j])
if m:
resultBegin=1
continue
m=re.match(r'^(\w+)=([\w\.]+)',lines[j])
if m and resultBegin:
metricName=m.group(1).strip()
values[metricName] = m.group(2).strip()
if metricName=="HPL_Tflops":hpl_tflops = float(values[metricName])
if metricName=="CommWorldProcs":numCores = int(values[metricName])
m=re.match(r'^Running on ([0-9\.]+) processors',lines[j])
if m:
numCores = int(m.group(1).strip())
if hpl_tflops==None or numCores==None:
parser.successfulRun=False
hpccVersion=None
MHz=None
theoreticalGFlops=None
if "VersionMajor" in values and "VersionMinor" in values and "VersionMicro" in values:
hpccVersion=values["VersionMajor"]+"."+values["VersionMinor"]+"."+values["VersionMicro"]
if "VersionRelease" in values:
hpccVersion+=values["VersionRelease"]
if hpccVersion:
parser.setParameter("App:Version", hpccVersion)
for k,v in Params.iteritems():
if not k in values:
continue
val=values[k]
if v[2].find('val')>=0:
val=float(val)
val=eval(v[2])
if v[1]=="":
v[1]=None
parser.setParameter( "Input:" + v[0],val, v[1])
for k,v in Metrics.iteritems():
if not k in values:
continue
val=values[k]
if v[2].find('val')>=0:
val=float(val)
val=eval(v[2])
if v[1]=="":
v[1]=None
parser.setStatistic(v[0],val, v[1])
if "cpuSpeed" in parser.geninfo:
ll=parser.geninfo["cpuSpeed"].splitlines()
cpuSpeedMax=0.0
for l in ll:
m=re.search(r'([\d\.]+)$',l)
if m:
v=float(m.group(1).strip())
if v>cpuSpeedMax:cpuSpeedMax=v
if cpuSpeedMax>0.0:
parser.setParameter("RunEnv:CPU Speed",cpuSpeedMax, "MHz" )
MHz=cpuSpeedMax
#print appKerNResVars
#print MHz
#print numCores
if appKerNResVars !=None:
if 'resource' in appKerNResVars and 'app' in appKerNResVars:
if 'theoreticalGFlopsPerCore' in appKerNResVars['app']:
resname=appKerNResVars['resource']['name']
if resname in appKerNResVars['app']['theoreticalGFlopsPerCore']:
theoreticalGFlops=appKerNResVars['app']['theoreticalGFlopsPerCore'][resname] * numCores
print "theoreticalGFlops",resname,theoreticalGFlops
if theoreticalGFlops==None and MHz!=None:
# Most modern x86 & POWER processors are superscale and can issue 4 instructions per cycle
theoreticalGFlops = MHz * numCores * 4 / 1000.0
if theoreticalGFlops and hpl_tflops:
# Convert both to GFlops and derive the Efficiency
percent = ( 1000.0 * hpl_tflops / theoreticalGFlops ) * 100.0;
parser.setStatistic("High Performance LINPACK Efficiency", "%.3f"%percent, "Percent")
if __name__ == "__main__":
#output for testing purpose
print "parsing complete:",parser.parsingComplete(Verbose=True)
parser.printParsNStatsAsMustHave()
print parser.getXML()
#return complete XML overwize return None
return parser.getXML()
def processAppKerOutput(appstdout=None,stdout=None,stderr=None,geninfo=None,appKerNResVars=None):
#set App Kernel Description
parser=AppKerOutputParser(
name = 'xdmod.app.chem.gamess',
version = 1,
description = "Gamess: General Atomic and Molecular Electronic Structure System",
url = 'http://www.msg.ameslab.gov',
measurement_name = 'Gamess'
)
#set obligatory parameters and statistics
#set common parameters and statistics
parser.setCommonMustHaveParsAndStats()
#set app kernel custom sets
parser.setMustHaveParameter('App:Version')
parser.setMustHaveStatistic('Wall Clock Time')
parser.setMustHaveStatistic('User Time')
parser.setMustHaveStatistic('Time Spent in MP2 Energy Calculation')
parser.setMustHaveStatistic('Time Spent in Restricted Hartree-Fock Calculation')
#parse common parameters and statistics
parser.parseCommonParsAndStats(appstdout,stdout,stderr,geninfo)
#read output
lines=[]
if os.path.isfile(appstdout):
fin=open(appstdout,"rt")
lines=fin.readlines()
fin.close()
#process the output
startTime=None
endTime=None
MP2EnergyCalculationTime=0.0
RHFCalculationTime=0.0
efficiency=None
j=0
while j<len(lines):
m=re.search(r'GAMESS VERSION = ([^*]+)',lines[j])
if m:parser.setParameter("App:Version",m.group(1).strip())
m=re.search(r'PARALLEL VERSION RUNNING ON\s*([\d\.]+) PROCESSORS IN\s*([\d\.]+) NODE',lines[j])
if m:
parser.setParameter("App:NCores",m.group(1).strip())
parser.setParameter("App:NNodes",m.group(2).strip())
m=re.search(r'EXECUTION OF GAMESS BEGUN (.+)',lines[j])
if m:startTime=parser.getDateTimeLocal(m.group(1).strip())
m=re.search(r'EXECUTION OF GAMESS TERMINATED NORMALLY (.+)',lines[j])
if m:endTime=parser.getDateTimeLocal(m.group(1).strip())
if re.search(r'DONE WITH MP2 ENERGY',lines[j]):
j+=1
m=re.search(r'STEP CPU TIME=\s*([\d\.]+)',lines[j])
if m:MP2EnergyCalculationTime+=float(m.group(1).strip())
if re.search(r'END OF RHF CALCULATION',lines[j]):
j+=1
m=re.search(r'STEP CPU TIME=\s*([\d\.]+)',lines[j])
if m:RHFCalculationTime+=float(m.group(1).strip())
m=re.search(r'TOTAL WALL CLOCK TIME.+CPU UTILIZATION IS\s+([\d\.]+)',lines[j])
if m:efficiency=float(m.group(1).strip())
j+=1
if startTime and endTime:
wallTime=total_seconds(endTime-startTime)
if wallTime >= 0.0:
parser.setStatistic('Wall Clock Time', str(wallTime), "Second" )
if efficiency:
parser.setStatistic( "User Time", str((0.01 * efficiency * wallTime)), "Second" )
parser.setStatistic("Time Spent in MP2 Energy Calculation", str(MP2EnergyCalculationTime), "Second" )
parser.setStatistic("Time Spent in Restricted Hartree-Fock Calculation", str(RHFCalculationTime),"Second" )
if "attemptsToLaunch" in parser.geninfo:
parser.setStatistic("Attempts to Launch", parser.geninfo['attemptsToLaunch'] )
else:
parser.setStatistic("Attempts to Launch", 1 )
if __name__ == "__main__":
#output for testing purpose
print "parsing complete:",parser.parsingComplete()
parser.printParsNStatsAsMustHave()
print parser.getXML()
#return complete XML overwize return None
return parser.getXML()
def processAppKerOutput(appstdout=None, stdout=None, stderr=None, geninfo=None, appKerNResVars=None):
# set App Kernel Description
parser = AppKerOutputParser(
name="xdmod.bundle",
version=1,
description="bundled tasks",
url="https://xdmod.ccr.buffalo.edu",
measurement_name="BUNDLE",
)
parser.setMustHaveParameter("RunEnv:Nodes")
parser.setMustHaveStatistic("Wall Clock Time")
parser.setMustHaveStatistic("Success Rate")
parser.setMustHaveStatistic("Successful Subtasks")
parser.setMustHaveStatistic("Total Number of Subtasks")
# set obligatory parameters and statistics
# set common parameters and statistics
parser.parseCommonParsAndStats(appstdout, stdout, stderr, geninfo)
if hasattr(parser, "wallClockTime"):
parser.setStatistic("Wall Clock Time", total_seconds(parser.wallClockTime), "Second")
# check the status of subtasks
# appKerNResVars['taskId']=self.task_id
# appKerNResVars['subTasksId']=self.subTasksId
successRate = 0.0
totalSubtasks = 0
successfulSubtasks = 0
try:
db = MySQLdb.connect(
host=akrr.export_db_host, user=akrr.export_db_user, passwd=akrr.export_db_passwd, db=akrr.export_db_name
)
cur = db.cursor()
for subTaskId in appKerNResVars["subTasksId"]:
cur.execute(
"""SELECT instance_id,status FROM akrr_xdmod_instanceinfo
WHERE instance_id=%s ;""",
(subTaskId,),
)
raw = cur.fetchall()
instance_id, status = raw[0]
successRate += status
successfulSubtasks += status
successRate /= len(appKerNResVars["subTasksId"])
totalSubtasks = len(appKerNResVars["subTasksId"])
cur.close()
del db
except:
print traceback.format_exc()
parser.setStatistic("Success Rate", successRate)
parser.setStatistic("Successful Subtasks", successfulSubtasks)
parser.setStatistic("Total Number of Subtasks", totalSubtasks)
# if successfulSubtasks==totalSubtasks:
if __name__ == "__main__":
# output for testing purpose
print "parsing complete:", parser.parsingComplete(Verbose=True)
parser.printParsNStatsAsMustHave()
print parser.getXML()
# return complete XML overwize return None
return parser.getXML()
def processAppKerOutput(appstdout=None, stdout=None, stderr=None, geninfo=None, appKerNResVars=None):
# set App Kernel Description
parser = AppKerOutputParser(
name="xdmod.app.climate.wrf",
version=1,
description="Weather Research and Forecasting Model",
url="http://www.wrf-model.org",
measurement_name="WRF",
)
# set obligatory parameters and statistics
# set common parameters and statistics
parser.setCommonMustHaveParsAndStats()
# set app kernel custom sets
parser.setMustHaveParameter("App:Version")
parser.setMustHaveParameter("Input:Grid Resolution")
parser.setMustHaveParameter("Input:Simulation Length")
parser.setMustHaveParameter("Input:Simulation Start Date")
parser.setMustHaveParameter("Input:Timestep")
parser.setMustHaveParameter("RunEnv:Nodes")
parser.setMustHaveParameter("WRF Dynamical Solver")
# parser.setMustHaveStatistic('Average Floating-Point Performance')
parser.setMustHaveStatistic("Average Simulation Speed")
parser.setMustHaveStatistic("Mean Time To Simulate One Timestep")
parser.setMustHaveStatistic("Output Data Size")
# parser.setMustHaveStatistic('Peak Floating-Point Performance')
parser.setMustHaveStatistic("Peak Simulation Speed")
parser.setMustHaveStatistic("Time Spent on I/O")
parser.setMustHaveStatistic("Wall Clock Time")
# parse common parameters and statistics
parser.parseCommonParsAndStats(appstdout, stdout, stderr, geninfo)
# read output
lines = []
if os.path.isfile(appstdout):
fin = open(appstdout, "rt")
lines = fin.readlines()
fin.close()
# process the output
IOsize = None
wallClockTime = None
iterationWallClockTime = []
simTimePerIteration = None
dx = None
dy = None
flopsConversion = None
j = 0
while j < len(lines):
m = re.search(r"XDMOD\*\*\*SIZE OF CURRENT DIR BEFORE WRF RUN\s*(\d+)", lines[j])
if m:
IOsize = int(m.group(1).strip())
m = re.search(r"XDMOD\*\*\*SIZE OF CURRENT DIR AFTER WRF RUN\s*(\d+)", lines[j])
if m and IOsize:
parser.setStatistic("Output Data Size", (int(m.group(1).strip()) - IOsize) / 1024.0 / 1024.0, "MByte")
m = re.search(r"XDMOD\*\*\*WRF RUN BEGINS HERE --(.+)", lines[j])
if m:
wallClockTime = parser.getDateTimeLocal(m.group(1).strip())
m = re.search(r"XDMOD\*\*\*WRF RUN HAS FINISHED --(.+)", lines[j])
if m and wallClockTime:
wallClockTime = parser.getDateTimeLocal(m.group(1).strip()) - wallClockTime
parser.setStatistic("Wall Clock Time", wallClockTime.total_seconds(), "Second")
if lines[j].find("XDMOD***RESULT OF rsl.out.0000 BEGINS") >= 0:
# the output from MPI rank #0
IOtime = 0.0
while j < len(lines):
if lines[j].find("XDMOD***RESULT OF rsl.out.0000 ENDS") >= 0:
break
m = re.search(r"Timing for processing restart file.+?:\s+(\d\S+)", lines[j], re.I)
if m:
IOtime += float(m.group(1).strip())
m = re.search(r"Timing for Writing.+?:\s+(\d\S+)", lines[j], re.I)
if m:
IOtime += float(m.group(1).strip())
m = re.search(r"Timing for main: time.+?on domain.+?:\s+(\d\S+)", lines[j], re.I)
if m:
iterationWallClockTime.append(float(m.group(1).strip()))
m = re.search(r"WRF NUMBER OF TILES.+?(\d+)", lines[j])
if m:
ompThreads = int(m.group(1).strip())
if ompThreads > 1:
parser.setParameter("Number of OpenMP Threads", ompThreads)
m = re.match(r"^\s+WRF V(\S+) MODEL", lines[j])
if m:
parser.setParameter("App:Version", m.group(1).strip())
j += 1
parser.setStatistic("Time Spent on I/O", IOtime, "Second")
if re.search("XDMOD\*\*\*RESULT OF wrfout.+?BEGINS", lines[j]) >= 0:
# the output file's header (netCDF dump)
while j < len(lines):
if re.search("XDMOD\*\*\*RESULT OF wrfout.+?ENDS", lines[j]) >= 0:
break
m = re.search(r":DX = (\d+)", lines[j], re.I)
if m:
dx = float(m.group(1).strip()) * 0.001 # in meters
m = re.search(r":DY = (\d+)", lines[j], re.I)
if m:
dy = float(m.group(1).strip()) * 0.001 # in meters
m = re.search(r":DT = (\d+)", lines[j], re.I)
if m:
simTimePerIteration = float(m.group(1).strip()) # in seconds
parser.setParameter("Input:Timestep", simTimePerIteration, "Second per Step")
m = re.search(r':SIMULATION_START_DATE = "(.+?)"', lines[j], re.I)
if m:
parser.setParameter("Input:Simulation Start Date", (m.group(1).strip()))
m = re.search(r':GRIDTYPE = "(.+?)"', lines[j], re.I)
if m:
solver = m.group(1).strip()
if solver == "C":
solver = "Advanced Research WRF (ARW)"
if solver == "E":
solver = "Nonhydrostatic Mesoscale Model (NMM)"
parser.setParameter("WRF Dynamical Solver", solver)
m = re.search(r"Timing for Writing.+?:\s+(\d\S+)", lines[j], re.I)
if m:
IOtime += float(m.group(1).strip())
m = re.search(r"Timing for main: time.+?on domain.+?:\s+(\d\S+)", lines[j], re.I)
if m:
iterationWallClockTime.append(float(m.group(1).strip()))
m = re.search(r"WRF NUMBER OF TILES.+?(\d+)", lines[j])
if m:
ompThreads = int(m.group(1).strip())
if ompThreads > 1:
parser.setParameter("Number of OpenMP Threads", ompThreads)
m = re.match(r"^\s+WRF V(\S+) MODEL", lines[j])
if m:
parser.setParameter("App:Version", m.group(1).strip())
j += 1
if dx and dy:
if (dx - int(dx)) * 1000 < 0.1 and (dy - int(dy)) * 1000 < 0.1: # back compatibility with output format
parser.setParameter("Input:Grid Resolution", "%.0f x %.0f" % (dx, dy), "km^2")
else:
parser.setParameter("Input:Grid Resolution", str(dx) + " x " + str(dy), "km^2")
m = re.search(r"XDMOD\*\*\*FLOATING-POINT PERFORMANCE CONVERSION", lines[j])
if m:
flopsConversion = lines[j + 1].strip()
j += 1
if wallClockTime:
parser.successfulRun = True
else:
parser.successfulRun = False
if len(iterationWallClockTime) > 0 and simTimePerIteration:
parser.setParameter(
"Input:Simulation Length", (len(iterationWallClockTime)) * simTimePerIteration / 3600.0, "Hour"
)
iterationWallClockTime = sorted(iterationWallClockTime)
iterationWallClockTime.pop()
t = 0.0
minT = iterationWallClockTime[0]
for tt in iterationWallClockTime:
t += tt
t = t / len(iterationWallClockTime)
parser.setStatistic("Mean Time To Simulate One Timestep", t, "Second")
parser.setStatistic("Average Simulation Speed", simTimePerIteration / t, "Simulated Second per Second")
parser.setStatistic("Peak Simulation Speed", simTimePerIteration / minT, "Simulated Second per Second")
if flopsConversion:
flopsConversion = flopsConversion.replace("$", "").replace("gflops=", "")
gflops = eval(flopsConversion, {"T": t})
parser.setStatistic("Average Floating-Point Performance", 1000.0 * gflops, "MFLOP per Second")
gflops = eval(flopsConversion, {"T": minT})
parser.setStatistic("Peak Floating-Point Performance", 1000.0 * gflops, "MFLOP per Second")
if __name__ == "__main__":
# output for testing purpose
parsingComplete = parser.parsingComplete(True)
print "parsing complete:", parsingComplete
if hasattr(parser, "successfulRun"):
print "successfulRun", parser.successfulRun
parser.printParsNStatsAsMustHave()
print parser.getXML()
# return complete XML overwize return None
return parser.getXML()
def processAppKerOutput(appstdout=None,stdout=None,stderr=None,geninfo=None,appKerNResVars=None):
#initiate parser
parser=AppKerOutputParser(
name = 'xdmod.benchmark.io.mdtest'
)
#set obligatory parameters and statistics
#set common parameters and statistics (App:ExeBinSignature and RunEnv:Nodes)
parser.setCommonMustHaveParsAndStats()
#set app kernel custom sets
parser.setMustHaveParameter('RunEnv:Nodes')
parser.setMustHaveParameter('Arguments (single directory per process)')
parser.setMustHaveParameter('Arguments (single directory)')
parser.setMustHaveParameter('Arguments (single tree directory per process)')
parser.setMustHaveParameter('Arguments (single tree directory)')
parser.setMustHaveParameter('files/directories (single directory per process)')
parser.setMustHaveParameter('files/directories (single directory)')
parser.setMustHaveParameter('files/directories (single tree directory per process)')
parser.setMustHaveParameter('files/directories (single tree directory)')
parser.setMustHaveParameter('tasks (single directory per process)')
parser.setMustHaveParameter('tasks (single directory)')
parser.setMustHaveParameter('tasks (single tree directory per process)')
parser.setMustHaveParameter('tasks (single tree directory)')
parser.setMustHaveStatistic('Directory creation (single directory per process)')
parser.setMustHaveStatistic('Directory creation (single directory)')
parser.setMustHaveStatistic('Directory creation (single tree directory per process)')
parser.setMustHaveStatistic('Directory creation (single tree directory)')
parser.setMustHaveStatistic('Directory removal (single directory per process)')
parser.setMustHaveStatistic('Directory removal (single directory)')
parser.setMustHaveStatistic('Directory removal (single tree directory per process)')
parser.setMustHaveStatistic('Directory removal (single tree directory)')
parser.setMustHaveStatistic('Directory stat (single directory per process)')
parser.setMustHaveStatistic('Directory stat (single directory)')
parser.setMustHaveStatistic('Directory stat (single tree directory per process)')
parser.setMustHaveStatistic('Directory stat (single tree directory)')
parser.setMustHaveStatistic('File creation (single directory per process)')
parser.setMustHaveStatistic('File creation (single directory)')
parser.setMustHaveStatistic('File creation (single tree directory per process)')
parser.setMustHaveStatistic('File creation (single tree directory)')
parser.setMustHaveStatistic('File read (single directory per process)')
parser.setMustHaveStatistic('File read (single directory)')
parser.setMustHaveStatistic('File read (single tree directory per process)')
parser.setMustHaveStatistic('File read (single tree directory)')
parser.setMustHaveStatistic('File removal (single directory per process)')
parser.setMustHaveStatistic('File removal (single directory)')
parser.setMustHaveStatistic('File removal (single tree directory per process)')
parser.setMustHaveStatistic('File removal (single tree directory)')
parser.setMustHaveStatistic('File stat (single directory per process)')
parser.setMustHaveStatistic('File stat (single directory)')
parser.setMustHaveStatistic('File stat (single tree directory per process)')
parser.setMustHaveStatistic('File stat (single tree directory)')
parser.setMustHaveStatistic('Tree creation (single directory per process)')
parser.setMustHaveStatistic('Tree creation (single directory)')
parser.setMustHaveStatistic('Tree creation (single tree directory per process)')
parser.setMustHaveStatistic('Tree creation (single tree directory)')
parser.setMustHaveStatistic('Tree removal (single directory per process)')
parser.setMustHaveStatistic('Tree removal (single directory)')
parser.setMustHaveStatistic('Tree removal (single tree directory per process)')
parser.setMustHaveStatistic('Tree removal (single tree directory)')
parser.setMustHaveStatistic('Wall Clock Time')
#parse common parameters and statistics
parser.parseCommonParsAndStats(appstdout,stdout,stderr,geninfo)
if hasattr(parser,'appKerWallClockTime'):
parser.setStatistic("Wall Clock Time", total_seconds(parser.appKerWallClockTime), "Second")
#Here can be custom output parsing
#read output
lines=[]
if os.path.isfile(appstdout):
fin=open(appstdout,"rt")
lines=fin.readlines()
fin.close()
#process the output
testname=""
parser.successfulRun=False
j=0
while j<len(lines):
m=re.match(r'^#Testing (.+)',lines[j])
if m:
testname=" ("+m.group(1).strip()+")"
m=re.match(r'^SUMMARY:',lines[j])
if m:
j=j+3
while j<len(lines):
m=re.match(r'([A-Za-z0-9 ]+):\s+[0-9.]+\s+[0-9.]+\s+([0-9.]+)\s+([0-9.]+)',lines[j])
if m:
parser.setStatistic(m.group(1).strip()+testname,m.group(2),"Operations/Second")
else:
break
j=j+1
m=re.search(r'finished at',lines[j])
if m:parser.successfulRun=True
m=re.match(r'^Command line used:.+mdtest\s+(.+)',lines[j])
if m:
parser.setParameter("Arguments"+testname,m.group(1).strip())
m=re.search(r'([0-9]+) tasks, ([0-9]+) files/directories',lines[j])
if m:
parser.setParameter("tasks"+testname,m.group(1).strip())
parser.setParameter("files/directories"+testname,m.group(2).strip())
j=j+1
#parser.setParameter("mega parameter",m.group(1))
#
# m=re.search(r'My mega parameter\s+(\d+)',lines[j])
# if m:parser.setStatistic("mega statistics",m.group(1),"Seconds")
#
# m=re.search(r'Done',lines[j])
# if m:parser.successfulRun=True
#
# j+=1
if __name__ == "__main__":
#output for testing purpose
print "Parsing complete:",parser.parsingComplete(Verbose=True)
print "Following statistics and parameter can be set as obligatory:"
parser.printParsNStatsAsMustHave()
print "\nResulting XML:"
print parser.getXML()
#return complete XML otherwise return None
return parser.getXML()
def processAppKerOutput(appstdout=None,stdout=None,stderr=None,geninfo=None,appKerNResVars=None):
#set App Kernel Description
if(appKerNResVars!=None and 'app' in appKerNResVars and 'name' in appKerNResVars['app']):
akname=appKerNResVars['app']['name']
else:
akname='unknown'
#initiate parser
parser=AppKerOutputParser(
name = akname
)
#set obligatory parameters and statistics
#set common parameters and statistics (App:ExeBinSignature and RunEnv:Nodes)
parser.setCommonMustHaveParsAndStats()
#set app kernel custom sets
parser.setMustHaveParameter('App:ExeBinSignature')
parser.setMustHaveParameter('App:Version')
parser.setMustHaveParameter('Number of Darts Throws')
parser.setMustHaveParameter('Number of Rounds')
parser.setMustHaveParameter('RunEnv:Nodes')
parser.setMustHaveStatistic('Darts Throws per Second')
parser.setMustHaveStatistic('Time for PI Calculation')
parser.setMustHaveStatistic('Wall Clock Time')
#parse common parameters and statistics
parser.parseCommonParsAndStats(appstdout,stdout,stderr,geninfo)
if hasattr(parser,'appKerWallClockTime'):
parser.setStatistic("Wall Clock Time", parser.appKerWallClockTime.total_seconds(), "Second")
#Here can be custom output parsing
#read output
lines=[]
if os.path.isfile(appstdout):
fin=open(appstdout,"rt")
lines=fin.readlines()
fin.close()
#process the output
parser.successfulRun=False
j=0
while j<len(lines):
m=re.search(r'version:\s+(.+)',lines[j])
if m:parser.setParameter('App:Version',m.group(1))
m=re.search(r'number of throws at dartboard:\s+(\d+)',lines[j])
if m:parser.setParameter('Number of Darts Throws',m.group(1))
m=re.search(r'number of rounds for dartz throwing\s+(\d+)',lines[j])
if m:parser.setParameter('Number of Rounds',m.group(1))
m=re.search(r'Time for PI calculation:\s+([0-9\.]+)',lines[j])
if m:parser.setStatistic("Time for PI Calculation",m.group(1),"Seconds")
m=re.search(r'Giga Darts Throws per Second \(GDaPS\):\s+([0-9\.]+)',lines[j])
if m:parser.setStatistic("Darts Throws per Second",m.group(1),"GDaPS")
m=re.search(r'Giga Darts Throws per Second',lines[j])
if m:parser.successfulRun=True
j+=1
if __name__ == "__main__":
#output for testing purpose
print "Parsing complete:",parser.parsingComplete(Verbose=True)
print "Following statistics and parameter can be set as obligatory:"
parser.printParsNStatsAsMustHave()
print "\nResulting XML:"
print parser.getXML()
#return complete XML otherwise return None
return parser.getXML()
def processAppKerOutput(appstdout=None,stdout=None,stderr=None,geninfo=None,appKerNResVars=None):
#set App Kernel Description
parser=AppKerOutputParser(
name = 'xdmod.app.md.namd',
version = 1,
description = "NAMD: Scalable Molecular Dynamics Package",
url = 'http://www.ks.uiuc.edu/Research/namd/',
measurement_name = 'NAMD'
)
#set obligatory parameters and statistics
#set common parameters and statistics
parser.setCommonMustHaveParsAndStats()
#set app kernel custom sets
parser.setMustHaveParameter('App:Version')
parser.setMustHaveParameter('Input:Coordinate File')
parser.setMustHaveParameter('Input:Number of Angles')
parser.setMustHaveParameter('Input:Number of Atoms')
parser.setMustHaveParameter('Input:Number of Bonds')
parser.setMustHaveParameter('Input:Number of Dihedrals')
parser.setMustHaveParameter('Input:Number of Steps')
parser.setMustHaveParameter('Input:Structure File')
parser.setMustHaveParameter('Input:Timestep')
parser.setMustHaveStatistic('Memory')
parser.setMustHaveStatistic('Molecular Dynamics Simulation Performance')
parser.setMustHaveStatistic('Wall Clock Time')
#parse common parameters and statistics
parser.parseCommonParsAndStats(appstdout,stdout,stderr,geninfo)
#read output
lines=[]
if os.path.isfile(appstdout):
fin=open(appstdout,"rt")
lines=fin.readlines()
fin.close()
#process the output
successfulRun=False
j=0
while j<len(lines):
m=re.match(r'^Info: NAMD ([0-9a-zA-Z\.]+)',lines[j])
if m:parser.setParameter("App:Version",m.group(1))
m=re.match(r'^Info: TIMESTEP\s+([0-9\.]+)',lines[j])
if m:parser.setParameter("Input:Timestep",m.group(1)+"e-15", "Second per Step" )
m=re.match(r'^Info: NUMBER OF STEPS\s+([0-9\.]+)',lines[j])
if m:parser.setParameter("Input:Number of Steps",m.group(1))
m=re.match(r'^Info: COORDINATE PDB\s+(.+)',lines[j])
if m:parser.setParameter("Input:Coordinate File",m.group(1))
m=re.match(r'^Info: STRUCTURE FILE\s+(.+)',lines[j])
if m:parser.setParameter("Input:Structure File",m.group(1))
m=re.match(r'^Info: Running on ([0-9\.]+) processors, ([0-9\.]+) nodes, ([0-9\.]+) physical nodes.',lines[j])
if m:
parser.setParameter("App:NCores",m.group(1).strip())
parser.setParameter("App:NNodes",m.group(3).strip())
if re.match(r'^Info: STRUCTURE SUMMARY',lines[j]):
j+=1
for k in range(25):
if re.match(r'^Info: \*\*\*\*\*',lines[j]):
break
m=re.match(r'^Info:\s+([0-9]+)\s+ATOMS\n',lines[j])
if m:parser.setParameter("Input:Number of Atoms",m.group(1))
m=re.match(r'^Info:\s+([0-9]+)\s+BONDS\n',lines[j])
if m:parser.setParameter("Input:Number of Bonds",m.group(1))
m=re.match(r'^Info:\s+([0-9]+)\s+ANGLES\n',lines[j])
if m:parser.setParameter("Input:Number of Angles",m.group(1))
m=re.match(r'^Info:\s+([0-9]+)\s+DIHEDRALS\n',lines[j])
if m:parser.setParameter("Input:Number of Dihedrals",m.group(1))
j+=1
if re.search(r'Info: Benchmark time:',lines[j]):
m=re.search(r' ([0-9.]+) days/ns',lines[j])
if m:parser.setStatistic( "Molecular Dynamics Simulation Performance", str(1.0e-9/float(m.group(1))), "Second per Day" )
m=re.match(r'^WallClock:\s+([0-9\.]+)\s+CPUTime:\s+([0-9\.]+)\s+Memory:\s+([0-9\.]+)',lines[j])
if m:
parser.setStatistic("Wall Clock Time", m.group(1), "Second")
parser.setStatistic("Memory", m.group(3), "MByte")
successfulRun=True
m=re.match(r'^End of program',lines[j])
if m:successfulRun=True
j+=1
if __name__ == "__main__":
#output for testing purpose
print "parsing complete:",parser.parsingComplete()
parser.printParsNStatsAsMustHave()
print parser.getXML()
#return complete XML overwize return None
return parser.getXML()
def processAppKerOutput(appstdout=None,stdout=None,stderr=None,geninfo=None,appKerNResVars=None):
#set App Kernel Description
if(appKerNResVars!=None and 'app' in appKerNResVars and 'name' in appKerNResVars['app']):
akname=appKerNResVars['app']['name']
else:
akname='unknown'
#initiate parser
parser=AppKerOutputParser(
name = akname
)
#set obligatory parameters and statistics
#set common parameters and statistics (App:ExeBinSignature and RunEnv:Nodes)
parser.setCommonMustHaveParsAndStats()
#set app kernel custom sets
#parser.setMustHaveParameter('App:Version')
parser.setMustHaveStatistic('Wall Clock Time')
#parse common parameters and statistics
parser.parseCommonParsAndStats(appstdout,stdout,stderr,geninfo)
if hasattr(parser,'appKerWallClockTime'):
parser.setStatistic("Wall Clock Time", total_seconds(parser.appKerWallClockTime), "Second")
#Here can be custom output parsing
# #read output
# lines=[]
# if os.path.isfile(appstdout):
# fin=open(appstdout,"rt")
# lines=fin.readlines()
# fin.close()
#
# #process the output
# parser.successfulRun=False
# j=0
# while j<len(lines):
# m=re.search(r'My mega parameter\s+(\d+)',lines[j])
# if m:parser.setParameter("mega parameter",m.group(1))
#
# m=re.search(r'My mega parameter\s+(\d+)',lines[j])
# if m:parser.setStatistic("mega statistics",m.group(1),"Seconds")
#
# m=re.search(r'Done',lines[j])
# if m:parser.successfulRun=True
#
# j+=1
if __name__ == "__main__":
#output for testing purpose
print "Parsing complete:",parser.parsingComplete(Verbose=True)
print "Following statistics and parameter can be set as obligatory:"
parser.printParsNStatsAsMustHave()
print "\nResulting XML:"
print parser.getXML()
#return complete XML otherwise return None
return parser.getXML()
def processAppKerOutput(appstdout=None,stdout=None,stderr=None,geninfo=None,appKerNResVars=None):
#set App Kernel Description
parser=AppKerOutputParser(
name = 'xdmod.app.chem.nwchem',
version = 1,
description = "NWChem: Northwest Computational Chemistry Package",
url = 'http://www.emsl.pnl.gov/docs/nwchem',
measurement_name = 'NWChem'
)
#set obligatory parameters and statistics
#set common parameters and statistics
parser.setCommonMustHaveParsAndStats()
#set app kernel custom sets
parser.setMustHaveParameter('App:Version')
parser.setMustHaveParameter('App:Branch')
parser.setMustHaveParameter('Input:File')
parser.setMustHaveStatistic('Wall Clock Time')
parser.setMustHaveStatistic('User Time')
parser.setMustHaveStatistic("Global Arrays 'Create' Calls")
parser.setMustHaveStatistic("Global Arrays 'Destroy' Calls")
parser.setMustHaveStatistic("Global Arrays 'Get' Calls")
parser.setMustHaveStatistic("Global Arrays 'Put' Calls")
parser.setMustHaveStatistic("Global Arrays 'Accumulate' Calls")
parser.setMustHaveStatistic("Global Arrays 'Get' Amount")
parser.setMustHaveStatistic("Global Arrays 'Put' Amount")
parser.setMustHaveStatistic("Global Arrays 'Accumulate' Amount")
#parse common parameters and statistics
parser.parseCommonParsAndStats(appstdout,stdout,stderr,geninfo)
#read output
lines=[]
if os.path.isfile(appstdout):
fin=open(appstdout,"rt")
lines=fin.readlines()
fin.close()
#process the output
startTime=None
endTime=None
MP2EnergyCalculationTime=0.0
RHFCalculationTime=0.0
efficiency=None
j=0
while j<len(lines):
m=re.search(r'Northwest Computational Chemistry Package \(NWChem\) (.+)',lines[j])
if m:parser.setParameter("App:Version",m.group(1).strip())
m=re.search(r'nwchem branch *=(.+)',lines[j])
if m:parser.setParameter("App:Branch",m.group(1).strip())
m=re.search(r'input\s+= (.+)',lines[j])
if m:parser.setParameter("Input:File",m.group(1).strip())
m=re.search(r'Total times\s+cpu:\s+([0-9.]+)s\s+wall:\s+([0-9.]+)s',lines[j])
if m:
parser.setStatistic("Wall Clock Time", m.group(2).strip(), "Second" )
parser.setStatistic("User Time", m.group(1).strip(), "Second" )
# GA Statistics for process 0
# ------------------------------
#
# create destroy get put acc scatter gather read&inc
# calls: 521 521 6.28e+05 6.45e+04 6.78e+05 0 0 0
# number of processes/call 1.05e+00 1.36e+00 1.03e+00 0.00e+00 0.00e+00
# bytes total: 7.33e+09 4.35e+08 1.53e+09 0.00e+00 0.00e+00 0.00e+00
# bytes remote: 5.74e+09 1.31e+08 1.09e+09 0.00e+00 0.00e+00 0.00e+00
# Max memory consumed for GA by this process: 47428032 bytes
if re.search(r'GA Statistics for process',lines[j]):
if re.match(r'^calls',lines[j+4]):
v=lines[j+4].strip().split()
parser.setStatistic("Global Arrays 'Create' Calls", "%.0f"%float(v[1]), "Number of Calls")
parser.setStatistic("Global Arrays 'Destroy' Calls", "%.0f"%float(v[2]), "Number of Calls")
parser.setStatistic("Global Arrays 'Get' Calls", "%.0f"%float(v[3]), "Number of Calls")
parser.setStatistic("Global Arrays 'Put' Calls", "%.0f"%float(v[4]), "Number of Calls")
parser.setStatistic("Global Arrays 'Accumulate' Calls", "%.0f"%float(v[5]), "Number of Calls")
v=lines[j+6].strip().split()
parser.setStatistic("Global Arrays 'Get' Amount", ( float(v[2]) ) / 1048576.0, "MByte")
parser.setStatistic("Global Arrays 'Put' Amount", ( float(v[3]) ) / 1048576.0, "MByte")
parser.setStatistic("Global Arrays 'Accumulate' Amount", ( float(v[4]) ) / 1048576.0, "MByte")
# NWChem can be optionally compiled with PAPI, and it will
# report some GLOPS at the end
# thus here it is optional
m=re.search(r'Aggregate GFLOPS \(Real_time\):\s+([0-9.]+)',lines[j])
if m:parser.setStatistic("Floating-Point Performance (Wall Clock Time)", 1000.0*float(m.group(1).strip()), "MFLOP per Second" );
m=re.search(r'Aggregate GFLOPS \(Proc_time\):\s+([0-9.]+)',lines[j])
if m:parser.setStatistic("Floating-Point Performance (User Time)", 1000.0*float(m.group(1).strip()), "MFLOP per Second" );
j+=1
if __name__ == "__main__":
#output for testing purpose
print "parsing complete:",parser.parsingComplete()
parser.printParsNStatsAsMustHave()
print parser.getXML()
#return complete XML overwize return None
return parser.getXML()
def processAppKerOutput(appstdout=None,stdout=None,stderr=None,geninfo=None,appKerNResVars=None):
#set App Kernel Description
parser=AppKerOutputParser(
name = 'xdmod.app.md.amber',
version = 1,
description = "Amber: Assisted Model Building with Energy Refinement",
url = 'http://ambermd.org',
measurement_name = 'Amber'
)
#set obligatory parameters and statistics
#set common parameters and statistics
parser.setCommonMustHaveParsAndStats()
#set app kernel custom sets
parser.setMustHaveParameter('App:Version')
parser.setMustHaveParameter('Input:Coordinate File')
parser.setMustHaveParameter('Input:Number of Angles')
parser.setMustHaveParameter('Input:Number of Atoms')
parser.setMustHaveParameter('Input:Number of Bonds')
parser.setMustHaveParameter('Input:Number of Dihedrals')
parser.setMustHaveParameter('Input:Number of Steps')
parser.setMustHaveParameter('Input:Structure File')
parser.setMustHaveParameter('Input:Timestep')
parser.setMustHaveStatistic('Molecular Dynamics Simulation Performance')
parser.setMustHaveStatistic('Time Spent in Direct Force Calculation')
parser.setMustHaveStatistic('Time Spent in Non-Bond List Regeneration')
parser.setMustHaveStatistic('Time Spent in Reciprocal Force Calculation')
parser.setMustHaveStatistic('Wall Clock Time')
#parse common parameters and statistics
parser.parseCommonParsAndStats(appstdout,stdout,stderr,geninfo)
#read output
lines=[]
if os.path.isfile(appstdout):
fin=open(appstdout,"rt")
lines=fin.readlines()
fin.close()
#process the output
parser.successfulRun=False
numSteps = 0
stepSize = 0
j=0
while j<len(lines):
m=re.search(r'Amber\s+([0-9a-zA-Z]+)\s+SANDER\s+20[0-9]+',lines[j])
if m:parser.setParameter("App:Version","SANDER "+m.group(1))
m=re.match(r'^\|\s+PMEMD implementation of SANDER, Release\s+([0-9\.]+)',lines[j])
if m:parser.setParameter("App:Version","PMEMD "+m.group(1))
m=re.match(r'^\|\s+INPCRD:\s+(\S+)',lines[j])
if m:parser.setParameter("Input:Coordinate File",m.group(1))
m=re.match(r'^\|\s+PARM:\s+(\S+)',lines[j])
if m:parser.setParameter("Input:Structure File",m.group(1))
if re.search(r'CONTROL\s+DATA\s+FOR\s+THE\s+RUN',lines[j]):
j+=2
for k in range(256):
if re.match(r'^-----------------------------',lines[j]):break
m=re.search(r'nstlim\s+=\s+([0-9]+)',lines[j])
if m:
numSteps = int(m.group(1).strip())
parser.setParameter( "Input:Number of Steps", numSteps )
m=re.search(r'dt\s+=\s+([0-9.]+)',lines[j])
if m:
stepSize = 1000.0 * float(m.group(1).strip())
parser.setParameter( "Input:Timestep", stepSize*1e-15, "Second per Step" )
j+=1
if re.search(r'RESOURCE\s+USE',lines[j]):
j+=2
numBonds = 0
numAngles = 0
numDihedrals = 0
for k in range(256):
if re.match(r'^-----------------------------',lines[j]):break
m=re.search(r'NATOM\s+=\s+([0-9]+)',lines[j])
if m:parser.setParameter("Input:Number of Atoms",m.group(1).strip())
m=re.search(r'NBONH\s+=\s+([0-9]+)',lines[j])
if m:numBonds+=int(m.group(1).strip())
m=re.search(r'NBONA\s+=\s+([0-9]+)',lines[j])
if m:numBonds+=int(m.group(1).strip())
m=re.search(r'NTHETH\s+=\s+([0-9]+)',lines[j])
if m:numAngles+=int(m.group(1).strip())
m=re.search(r'NTHETA\s+=\s+([0-9]+)',lines[j])
if m:numAngles+=int(m.group(1).strip())
m=re.search(r'NPHIH\s+=\s+([0-9]+)',lines[j])
if m:numDihedrals+=int(m.group(1).strip())
m=re.search(r'NPHIA\s+=\s+([0-9]+)',lines[j])
if m:numDihedrals+=int(m.group(1).strip())
j+=1
if numBonds>0:parser.setParameter("Input:Number of Bonds",numBonds)
if numAngles>0:parser.setParameter("Input:Number of Angles",numAngles)
if numDihedrals>0:parser.setParameter("Input:Number of Dihedrals",numDihedrals)
if re.search(r'PME Nonbond Pairlist CPU Time',lines[j]):
j+=2
for k in range(20):
m=re.search(r'Total\s+([\d\.]+)',lines[j])
if m:
parser.setStatistic("Time Spent in Non-Bond List Regeneration", m.group(1), "Second")
break
j+=1
if re.search(r'PME Direct Force CPU Time',lines[j]):
j+=2
for k in range(20):
m=re.search(r'Total\s+([\d\.]+)',lines[j])
if m:
parser.setStatistic("Time Spent in Direct Force Calculation", m.group(1), "Second")
break
j+=1
if re.search(r'PME Reciprocal Force CPU Time',lines[j]):
j+=2
for k in range(20):
m=re.search(r'Total\s+([\d\.]+)',lines[j])
if m:
parser.setStatistic("Time Spent in Reciprocal Force Calculation", m.group(1), "Second")
break
j+=1
m=re.match(r'^\|\s+Master Total wall time:\s+([0-9.]+)\s+seconds',lines[j])
if m:
parser.setStatistic("Wall Clock Time", m.group(1), "Second")
parser.successfulRun=True
# calculate the performance
simulationTime = stepSize * numSteps * 0.000001 # measured in nanoseconds
if simulationTime>0.0:
parser.setStatistic( "Molecular Dynamics Simulation Performance", 1.e-9*simulationTime / ( float(m.group(1)) / 86400.0 ), "Second per Day" )
j+=1
if __name__ == "__main__":
#output for testing purpose
print "parsing complete:",parser.parsingComplete()
parser.printParsNStatsAsMustHave()
print parser.getXML()
#return complete XML overwize return None
return parser.getXML()
