def cnv(self):
data = {}
import tempfile, os, glob
origdir = os.getcwd()
tmpdir = self._tmpdir
try:
os.chdir(tmpdir)
import tarfile
tar = tarfile.open(os.path.abspath(self._fileName), 'r')
tar.extractall(path=tmpdir)
tar.close()
fname = glob.glob("*.dat")[0]
if open(fname, 'r').read(1024).startswith('SQLite format'):
import PyUtils.dbsqlite as dbs
db = dbs.open(fname, 'r')
else:
db = shelve.open(fname)
data['meta'] = {}
for k in db.iterkeys():
data['meta'][k] = db[k]
db.close()
## print "version:",data['meta']['version_id']
from PyRootLib import importRoot
ROOT = importRoot()
root = ROOT.fopen(glob.glob("*.root")[0], "read")
for k in ('ini', 'evt', 'fin'):
data[k] = root.Get("perfmon/%s" % k)
data['meta']['rootFile'] = root
finally:
os.chdir(origdir)
return data
def bookHistos(self, monComp):
ROOT = importRoot()
#Analyzer.__bookHistos(self)
from PyRootLib import setupRootStyle
setupRootStyle()
from App import DataSetMgr
for dataSetName in monComp.data.keys():
dataSet = DataSetMgr.instances[dataSetName]
nEntries = len(dataSet.bins)
minEvt = dataSet.bins[self.minEvt]
maxEvt = dataSet.bins[-1]
## print ":::",dataSetName,minEvt,maxEvt,nEntries
histos = monComp.data[dataSetName]['histos']
monCompName = monComp.name.replace("/", "#")
hId = 'cpu_%s.%s' % (monCompName, dataSetName)
hName = 'cpu_%s' % dataSetName
histos[hName] = ROOT.TH1F( hId,
"[%s] CPU usage [%s];%s;%s" % \
( monComp.name, dataSetName,
"event nbr", "Timing [ms]" ),
nEntries, minEvt, maxEvt )
return
def __writeRootFile(self):
self.msg.debug("create ROOT file...")
from PyRootLib import importRoot
ROOT = importRoot(batch=True)
outName = self.outputFile + ".root"
outFile = ROOT.fopen(outName, 'RECREATE')
for dsName in DataSetMgr.names():
outFile.cd("/")
outFile.mkdir(dsName)
outFile.cd(dsName)
for m in MonitoredComponent.instances.values():
if (not m.name.startswith('PerfMonSlice')
and not self.__filter(m)):
continue
if not dsName in m.data:
continue
for h in m.data[dsName]['histos'].values():
self.msg.debug("writing %s/%s", dsName, h.GetName())
h.Write()
outFile.cd()
if os.path.exists(outName):
self.msg.info(" --> (%10.3f kB) [%s]",
os.stat(outName).st_size / 1024., outName)
self.msg.debug("create ROOT file... [DONE]")
return
def fitHistos(self, monComp):
import PerfMonAna.PyRootLib as prl
ROOT = importRoot(batch=True)
RootFct = ROOT.TF1
dummyCanvas = ROOT.TCanvas('dummyFitCanvas')
histos = [ h for h in self.histos.values()
if hasattr(h, 'tag') and h.tag == 'summary' and \
not h.GetName().startswith("cfg.") ]
for h in histos:
x = h.GetXaxis()
xMin = x.GetXmin()
xMax = x.GetXmax()
name = h.GetName()
modelFct = prl.Polynom(degree=1)
fct = RootFct('fitFct_%s' % name, "pol1", xMin, xMax)
## Q: quiet
## R: Use the range specified in the function range
## O: Do not plot the result of the fit.
h.Fit(fct, "QOR")
nPars = fct.GetNpar()
fitRes= "\t".join( "p[%i] = %12.3f ms " % \
( i, fct.GetParameter(i)) for i in range(nPars)
)
msg = "[%s] fit results: %s \tchi2/NDF = %12.3f/%i" % \
( name, fitRes, fct.GetChisquare(), fct.GetNDF() )
self.msg.info(msg)
pass
# FIXME: not yet for prod!
## histos = [ h for h in self.histos.values()
## if h.GetName().count("avg_") > 0 ]
## for h in histos:
## x = h.GetXaxis()
## xMin, xMax = x.GetXmin(), x.GetXmax()
## name = h.GetName()
## fct = RootFct( "fitFct_%s" % name, "gaus", xMin, xMax )
## fct.SetParameter( 1, h.GetMaximum() )
## h.Fit(fct, "QOR")
## nPars = fct.GetNpar()
## self.msg.info( "[%-50s] %s", name, "\t".join(
## "p[%i] = %12.3f ms " % (i, fct.GetParameter(i)) \
## for i in range(nPars) )
## )
## pass
del dummyCanvas
return
def process(self, dataSetMgr, monCompMgr):
from PyRootLib import importRoot
ROOT = importRoot(batch=True)
from PyRootLib import setupRootStyle
setupRootStyle()
c = ROOT.TCanvas('c_default')
for m in [
self.processIni,
self.processFirstEvt,
self.processEvt,
self.processFin,
#self.processIo, ## FIXME
]:
m(dataSetMgr, monCompMgr)
del c
return
def bookHistos(self, monComp):
ROOT = importRoot()
#Analyzer.__bookHistos(self)
#meta = self.meta.cfg.io
from App import DataSetMgr
for dataSetName in DataSetMgr.names():
dataSet = DataSetMgr.instances[dataSetName]
nEntries = len(dataSet.bins)
minEvt = dataSet.bins[self.minEvt]
maxEvt = dataSet.bins[-1]
histos = monComp.data[dataSetName]['histos']
hId = 'io_w_%s.%s' % (monComp.name, dataSetName)
hName = 'io_w_%s' % dataSetName
histos[hName] = ROOT.TH1F( hId,
"[%s] I/O usage [%s] (W);%s;%s" % \
( monComp.name,
monComp.name.replace("#","/"),
"event nbr", "Timing [ms]" ),
nEntries, minEvt, maxEvt )
hId = 'io_r_%s.%s' % (monComp.name, dataSetName)
hName = 'io_r_%s' % dataSetName
histos[hName] = ROOT.TH1F( hId,
"[%s] I/O usage [%s] (R);%s;%s" % \
( monComp.name,
monComp.name.replace("#","/"),
"event nbr", "Timing [ms]" ),
nEntries, minEvt, maxEvt )
hId = 'io_rr_%s.%s' % (monComp.name, dataSetName)
hName = 'io_rr_%s' % dataSetName
histos[hName] = ROOT.TH1F( hId,
"[%s] I/O usage [%s] (RR);%s;%s" % \
( monComp.name,
monComp.name.replace("#","/"),
"event nbr", "Timing [ms]" ),
nEntries, minEvt, maxEvt )
return
def fitHistos(self, monComp):
# convert page-size into MB
Mb = Units.Mb
# convert page-size into kB
kb = Units.kb
import PerfMonAna.PyRootLib as prl
ROOT = importRoot()
RootFct = ROOT.TF1
dummyCanvas = ROOT.TCanvas( 'dummyFitCanvas' )
histos = [ h for h in self.histos.values()
if hasattr(h, 'tag') and h.tag == 'summary' and \
not h.GetName().startswith("cfg.")
]
for h in histos:
x = h.GetXaxis()
xMin = x.GetXmin()
xMax = x.GetXmax()
name = h.GetName()
# FIXME: do we want to fit only on half the interval ?
# could make sense for mem-leak...
xMin = xMin + ( xMax-xMin) / 2.
modelFct = prl.Polynom( degree = 1 )
fct = RootFct( 'fitFct_%s' % name, "pol1", xMin, xMax )
## Q: quiet
## R: Use the range specified in the function range
## O: Do not plot the result of the fit.
h.Fit(fct, "QOR")
nPars = fct.GetNpar()
fitRes= "\t".join( "p[%i] = %12.3f kB " % \
( i, fct.GetParameter(i)*kb) for i in range(nPars)
)
msg = "[%s] fit results: %s \tchi2/NDF = %12.3f/%i" % \
( name, fitRes, fct.GetChisquare(), fct.GetNDF() )
self.msg.info( msg )
pass
del dummyCanvas
return
def bookHistos(self, monComp):
ROOT = importRoot()
from PyRootLib import setupRootStyle; setupRootStyle();
#Analyzer.__bookHistos(self)
from App import DataSetMgr
for dataSetName in monComp.data.keys():
dataSet = DataSetMgr.instances[dataSetName]
nEntries = len(dataSet.bins)
minEvt = dataSet.bins[self.minEvt]
maxEvt = dataSet.bins[-1]
histos = monComp.data[dataSetName]['histos']
monCompName = monComp.name.replace( "/", "#" )
hId = "mem_%s.%s" % (monCompName, dataSetName)
hName = "mem_%s" % dataSetName
histos[hName] = ROOT.TH1F(
hId,
"[%s] Mem usage [%s];%s;%s" % (
monComp.name, dataSetName,
"event nbr", "V-mem [kB]"
),
nEntries, minEvt, maxEvt
)
hId = "malloc_%s.%s" % (monCompName, dataSetName)
hName = "malloc_%s" % dataSetName
histos[hName] = ROOT.TH1F(
hId,
"[%s] Malloc usage [%s];%s;%s" % (
monComp.name, dataSetName,
"event nbr", "Malloc'ed mem [kB]"
),
nEntries, minEvt, maxEvt
)
return
def processEvt(self, dataSetMgr, monCompMgr):
from App import DataSetMgr
from PyRootLib import importRoot
ROOT = importRoot(batch=True)
## RootFct = ROOT.TF1
## dummyCanvas = ROOT.TCanvas("dummyFitCanvas")
## import PyRootLib as prl
## self.sum['evt']['histos'] = []
## self.sum['evt']['fig'] = []
## short-hand
ms = Units.ms
kb = Units.kb
Mb = Units.Mb
Mb2Kb = 1000.
msg = self.msg
## get top-20 consumers
dsNames = DataSetMgr.names()
color = DataSetMgr.colorIter()
_txt = self.txt['evt']
monComp = monCompMgr['PerfMonSlice']
yMinCpu = []
yMaxCpu = []
yMinIo = []
yMaxIo = []
for dsName in dsNames:
data = monComp.data[dsName]
histos = data['histos'] = {}
if not 'evt' in data:
continue
data = data['evt']
if data is None:
continue
cpu = data['cpu']
cpu_u = cpu['user']
cpu_s = cpu['sys']
cpu_r = cpu['real']
cpu_c = cpu['cpu']
dcpu_u = cpu_u[:, 2]
dcpu_s = cpu_s[:, 2]
dcpu_r = cpu_r[:, 2]
dcpu_c = cpu_c[:, 2]
mem = data['mem']
vmem = mem['vmem']
dvmem = vmem[:, 2]
rss = mem['rss']
drss = rss[:, 2]
mall = mem['mall']
dmall = mall[:, 2]
nallocs = mem['nmall'][:, 2]
nfrees = mem['nfree'][:, 2]
yMinCpu.append(dcpu_c[self.minEvt:].min() * ms)
yMaxCpu.append(dcpu_c[self.minEvt:].max() * ms)
io = monComp.data[dsName]['io']
io_c = io['w']['cpu']
yMinIo.append(io_c[self.minEvt:].min() * ms)
yMaxIo.append(io_c[self.minEvt:].max() * ms)
## data['mem/vmem/d'] = data['mem/vmem/1'] - data['mem/vmem/0']
## data['mem/rss/d' ] = data['mem/rss/1' ] - data['mem/rss/0' ]
## fill-in some data for ASCII summary
if dsName == '000':
_txt['cpu']['slice'] += [
"%-20s %10.3f %10.3f %10.3f ms" % (
"[u/s/r]",
dcpu_u[self.minEvt:].mean() * ms,
dcpu_s[self.minEvt:].mean() * ms,
dcpu_r[self.minEvt:].mean() * ms,
),
]
_txt['mem']['slice'] += [
"%-20s %10.3f %10.3f %10.3f kB" % (
"[dVmem/dRss/dMalloc]",
dvmem[self.minEvt:].mean() * kb,
drss[self.minEvt:].mean() * kb,
dmall[self.minEvt:].mean() * kb,
),
]
_txt['allocs']['slice'] += [
"%-20s %10i %10i" % (
"[nallocs/nfrees]",
nallocs[self.minEvt:].mean(),
nfrees[self.minEvt:].mean(),
),
]
## book ROOT histos
nEntries = len(dataSetMgr[dsName].bins) - 1
minEvt = dataSetMgr[dsName].bins[self.minEvt]
maxEvt = dataSetMgr[dsName].bins[-1]
hId = 'cpu_%s.%s' % (monComp.name, dsName)
hName = 'cpu_%s' % dsName
histos[hName] = ROOT.TH1F(
hId, "[%s] CPU usage [%s];%s;%s" %
(monComp.name, dsName, "event nbr", "Timing [ms]"), nEntries,
minEvt, maxEvt)
hId = 'vmem_%s.%s' % (monComp.name, dsName)
hName = 'vmem_%s' % dsName
histos[hName] = ROOT.TH1F(
hId, "[%s] VMem usage [%s];%s;%s" %
(monComp.name, dsName, "event nbr", "V-Mem [MB]"), nEntries,
minEvt, maxEvt)
hId = 'rss_%s.%s' % (monComp.name, dsName)
hName = 'rss_%s' % dsName
histos[hName] = ROOT.TH1F(
hId, "[%s] RSS usage [%s];%s;%s" %
(monComp.name, dsName, "event nbr", "RSS [MB]"), nEntries,
minEvt, maxEvt)
hId = 'io_%s.%s' % (monComp.name, dsName)
hName = 'io_%s' % dsName
histos[hName] = ROOT.TH1F(
hId, "[%s] I/O time [%s];%s;%s" %
(monComp.name, dsName, "event nbr", "I/O time [ms]"), nEntries,
minEvt, maxEvt)
pass
yMinIo = min(yMinIo)
yMaxIo = max(yMaxIo)
yMinCpu = min(yMinCpu)
yMaxCpu = max(yMaxCpu)
def markForLegend(p):
setattr(p, '_markedForLegend', True)
def isMarked(p):
return hasattr(p, '_markedForLegend')
memLeak = []
for dsName in dsNames:
if not 'evt' in monComp.data[dsName]:
continue
data = monComp.data[dsName]
cpu = data['evt']['cpu']
cpu_c = cpu['cpu']
dcpu_c = cpu_c[:, 2]
## CPU
bins = dataSetMgr[dsName].bins
xbins = bins[self.minEvt:]
if not 'evt/cpu' in monComp.figs:
monComp.figs['evt/cpu'] = plt.figure()
monComp.figs['evt/cpu'].add_subplot(211).hold(True)
monComp.figs['evt/cpu'].add_subplot(212).hold(True)
fig = monComp.figs['evt/cpu']
ax = fig.axes[0]
pl = ax.plot(xbins,
dcpu_c[self.minEvt:] * ms,
linestyle='steps',
label=dsName)
ax.grid(True)
ax.set_title("CPU time [Begin/End-Event]")
ax.set_ylabel('CPU time [ms]')
ax.set_xlabel('Event number')
ax.set_ylim((ax.get_ylim()[0] * 0.9, ax.get_ylim()[1] * 1.1))
markForLegend(pl[0])
h, b = numpy.histogram(dcpu_c[self.minEvt:] * ms,
bins=20,
range=(yMinCpu * 0.90, yMaxCpu * 1.10))
ax = fig.axes[1]
pl = ax.plot(b[:-1], h, label=dsName, ls='steps')
ax.grid(True)
ax.set_xlabel('CPU time [ms]')
ax.set_ylim((ax.get_ylim()[0], ax.get_ylim()[1] * 1.1))
markForLegend(pl[0])
h = data['histos']['cpu_%s' % dsName]
hAvg = bookAvgHist(h, dcpu_c * ms)
data['histos'][h.GetName()] = hAvg
for i in range(len(bins)):
cpuTime = dcpu_c[i] * ms
h.Fill(float(bins[i]), cpuTime)
hAvg.Fill(cpuTime)
## Mem
mem = data['evt']['mem']
vmem = mem['vmem']
dvmem = vmem[:, 2]
dmall = mem['mall'][:, 2]
if not 'evt/mem' in monComp.figs:
monComp.figs['evt/mem'] = plt.figure()
monComp.figs['evt/mem'].add_subplot(311).hold(True)
monComp.figs['evt/mem'].add_subplot(312).hold(True)
monComp.figs['evt/mem'].add_subplot(313).hold(True)
fig = monComp.figs['evt/mem']
ax = fig.axes[0]
pl = ax.plot(
xbins,
vmem[self.minEvt:, 1] * Mb,
linestyle='steps',
#'o',
label=dsName)
ax.set_title("Memory usage [Begin/End-Event]")
ax.set_ylabel('VMem [MB]')
ax.set_xlabel('Event number')
ax.set_ylim((ax.get_ylim()[0] * 0.9, ax.get_ylim()[1] * 1.1))
ax.grid(True)
markForLegend(pl[0])
## fit
#nFit = int(len(bins)/2.)
#x = bins[-nFit:]
#y = data['evt']['mem/vmem/1'][-nFit:] * Mb
begFit = self._fitSlice[0]
endFit = self._fitSlice[1]
x = bins[begFit:endFit]
y = vmem[begFit:endFit, 1] * Mb
coeffs = numpy.lib.polyfit(x, y, deg=1)
memLeak += [coeffs[0] * kb]
ax.plot(x, numpy.lib.polyval(coeffs, x), '-', color='y', lw=2.)
if dsName == '000':
_txt['mem']['slice'] += [
"%-20s p[0] = %8.3f MB \tp[1] = %8.3f kB" % (
"vmem fit:",
coeffs[1] * Mb,
coeffs[0] * kb, #
)
]
ax = fig.axes[1]
pl = ax.plot(xbins,
mem['rss'][self.minEvt:, 1] * Mb,
linestyle='steps',
label=dsName)
ax.set_ylabel('RSS [MB]')
ax.set_xlabel('Event number')
ax.set_ylim((ax.get_ylim()[0] * 0.9, ax.get_ylim()[1] * 1.1))
ax.grid(True)
markForLegend(pl[0])
ax = fig.axes[2]
pl = ax.plot(xbins,
dmall[self.minEvt:] * Mb,
linestyle='steps',
label=dsName)
ax.set_ylabel('Delta Malloc [MB]')
ax.set_xlabel('Event number')
ax.set_ylim((ax.get_ylim()[0] * 0.9, ax.get_ylim()[1] * 1.1))
ax.grid(True)
markForLegend(pl[0])
h = data['histos']['vmem_%s' % dsName]
hAvg = bookAvgHist(h, mem['vmem'][:, 1] * Mb)
data['histos'][h.GetName()] = hAvg
for i in range(len(bins)):
vmem = mem['vmem'][i, 1] * Mb
h.Fill(float(bins[i]), vmem)
hAvg.Fill(vmem)
h = data['histos']['rss_%s' % dsName]
hAvg = bookAvgHist(h, mem['rss'][:, 1] * Mb)
data['histos'][h.GetName()] = hAvg
for i in range(len(bins)):
rss = mem['rss'][i, 1] * Mb
h.Fill(float(bins[i]), rss)
hAvg.Fill(rss)
## I/O
if not 'evt/io' in monComp.figs:
monComp.figs['evt/io'] = plt.figure()
monComp.figs['evt/io'].add_subplot(211).hold(True)
monComp.figs['evt/io'].add_subplot(212).hold(True)
fig = monComp.figs['evt/io']
ax = fig.axes[0]
io_c = data['io']['w']['cpu']
pl = ax.plot(xbins,
io_c[self.minEvt:] * ms,
linestyle='steps',
label=dsName)
ax.set_title("I/O time [CommitOutput]")
ax.set_ylabel('I/O time [ms]')
ax.set_xlabel('Event number')
ax.set_ylim((ax.get_ylim()[0], ax.get_ylim()[1] * 1.1))
ax.grid(True)
markForLegend(pl[0])
yMinIo *= 0.9
yMaxIo *= 1.1
yMinIo, yMaxIo = min(yMinIo, yMaxIo), max(yMinIo, yMaxIo)
h, b = numpy.histogram(io_c[self.minEvt:] * ms,
bins=20,
range=(yMinIo, yMaxIo))
ax = fig.axes[1]
pl = ax.plot(b[:-1], h, label=dsName, ls='steps')
ax.set_xlabel('I/O time [ms]')
ax.set_ylim((ax.get_ylim()[0], ax.get_ylim()[1] * 1.1))
ax.grid(True)
markForLegend(pl[0])
h = data['histos']['io_%s' % dsName]
hAvg = bookAvgHist(h, io_c * ms)
data['histos'][h.GetName()] = hAvg
for i in range(len(bins)):
cpuTime = io_c[i] * ms
h.Fill(float(bins[i]), cpuTime)
hAvg.Fill(cpuTime)
pass # loop over data sets
## handle mem-leak text
ax = monComp.figs['evt/mem'].axes[0]
ax.text(
0.025,
0.97,
"\n".join("[%s] Leak: %8.3f kB" % (n, leak)
for n, leak in zip(DataSetMgr.labels(), memLeak)),
bbox=dict(facecolor='w'),
#color = '',
fontweight='bold',
#fontsize = 'x-larger',
verticalalignment='top',
transform=ax.transAxes)
for figName, fig in monComp.figs.items():
loc = 'best'
if figName == 'evt/mem': loc = 'lower right'
for ax in fig.axes:
objs = [l for l in ax.lines if isMarked(l)]
ax.legend(
objs,
DataSetMgr.labels(),
#loc='lower right'
#loc='best'
loc=loc)
## get top-20 consumers
self._top_consumers(dataSetMgr,
monCompMgr,
compTypes=('alg', ),
title='Event loop',
storeName='evt',
sliceName='evt',
begSlice=self.minEvt,
endSlice=None)
return
def cnv(self):
data = {}
import tempfile, os, glob
import gzip
import shutil
wkdir = os.getcwd()
tmpdir = tempfile.mkdtemp()
global _holder
_holder.toRemove += [tmpdir]
try:
os.chdir(tmpdir)
f = open(os.path.abspath(self._fileName), 'r')
if self._fileName.endswith('.gz'):
f = gzip.GzipFile(fileobj=f)
f.seek(0)
tmpFile = open(tmpFileName, 'w')
shutil.copyfileobj(f, tmpFile)
f.close()
tmpFile.close()
import imp, inspect
tmpFile = open(tmpFileName, 'r')
mod = imp.load_source('_data_loader', '_data_loader.py', tmpFile)
tmpFile.close()
os.remove(tmpFileName)
def _fctLoader(obj):
if inspect.isfunction(obj):
return obj.__name__ == 'getData'
_data = inspect.getmembers(mod, _fctLoader)[0][1]
_data = _data()
data['meta'] = {
'components': _data['meta://components'],
'iocontainers': _data['meta://iocontainers'],
}
compNames = set(_data['components'].keys() + _data['iocontainers'])
from PyRootLib import importRoot
ROOT = importRoot()
root = ROOT.fopen(os.path.join([tmpdir, self._fileName + ".root"]),
"recreate")
## dataSetName = self.name
## ## print ">>>",len(compNames),len(data.keys())
## _data_keys = data.keys()
## for compName in compNames:
## ## print ":::::::::",compName
## monComp = MonitoredComponent(compName, dataSetName)
## monData = monComp.data[dataSetName]
## for storeName in storeNames:
## ## print compName,storeName
## if not monData.has_key(storeName):
## monData[storeName] = {}
## compNameHdr1 = compName + '://' + storeName
## compNameHdr2 = compName + ':///' + storeName
## for k in _data_keys:
## if k.startswith( compNameHdr1 ) or \
## k.startswith( compNameHdr2 ):
## monKey = k[k.find(storeName+'/')+len(storeName)+1:]
## monData[storeName][monKey] = numpy.array(data[k])
## if storeName == 'evt' and monData[storeName].has_key('evtNbr'):
## self.bins = monData[storeName]['evtNbr']
## pass
## pass
## _monitor('5')
## _compsDb = data['meta://components' ]
## _comps = data['meta://components' ].keys()
## _ioconts = data['meta://iocontainers']
## for monComp in MonitoredComponent.instances.values():
## if monComp.type != None:
## continue
## monName = monComp.name
## if monName in _comps : monComp.type = _compsDb[monName]
## elif monName in _ioconts : monComp.type = 'io'
## else : monComp.type = 'usr'
## pass
finally:
os.chdir(wkdir)
return data
