def __init__(self, partIn, maxThreadsIn=8):
super(ValgrindAnalyzer, self).__init__()
self.release = None
self.plat = None
self.part = partIn
self.ignoreLibs = ['libm-2.5.so', 'libpthread-2.5.so', 'cmsRun']
self.libList = []
prodFileName = os.environ[
'CMSSW_RELEASE_BASE'] + '/src/ReleaseProducts.list'
self.rpr = RelProdReader()
self.rpr.readProducts(prodFileName)
vgCmdFileName = os.environ['CMSSW_RELEASE_BASE'] + '/qaLogs/vgCmds.pkl'
from pickle import Unpickler
vgCmdFile = open(vgCmdFileName, 'r')
vgCmdPklr = Unpickler(vgCmdFile)
self.vgCmds = vgCmdPklr.load()
vgCmdFile.close()
self.toDo = []
self.threadList = []
self.maxThreads = maxThreadsIn
self.threadStatus = {}
self.threadTiming = {}
def __init__(self, partIn, maxThreadsIn=8):
super(ValgrindAnalyzer, self).__init__()
self.release = None
self.plat = None
self.part = partIn
self.ignoreLibs = ['libm-2.5.so','libpthread-2.5.so', 'cmsRun']
self.libList = []
prodFileName = os.environ['CMSSW_RELEASE_BASE']+'/src/ReleaseProducts.list'
self.rpr = RelProdReader()
self.rpr.readProducts(prodFileName)
vgCmdFileName = os.environ['CMSSW_RELEASE_BASE']+'/qaLogs/vgCmds.pkl'
from pickle import Unpickler
vgCmdFile = open(vgCmdFileName, 'r')
vgCmdPklr = Unpickler(vgCmdFile)
self.vgCmds = vgCmdPklr.load()
vgCmdFile.close()
self.toDo = []
self.threadList = []
self.maxThreads = maxThreadsIn
self.threadStatus = {}
self.threadTiming = {}
class ValgrindAnalyzer(object):
"""docstring for ValgrindAnalyzer"""
def __init__(self, partIn, maxThreadsIn=8):
super(ValgrindAnalyzer, self).__init__()
self.release = None
self.plat = None
self.part = partIn
self.ignoreLibs = ['libm-2.5.so','libpthread-2.5.so', 'cmsRun']
self.libList = []
prodFileName = os.environ['CMSSW_RELEASE_BASE']+'/src/ReleaseProducts.list'
self.rpr = RelProdReader()
self.rpr.readProducts(prodFileName)
vgCmdFileName = os.environ['CMSSW_RELEASE_BASE']+'/qaLogs/vgCmds.pkl'
from pickle import Unpickler
vgCmdFile = open(vgCmdFileName, 'r')
vgCmdPklr = Unpickler(vgCmdFile)
self.vgCmds = vgCmdPklr.load()
vgCmdFile.close()
self.toDo = []
self.threadList = []
self.maxThreads = maxThreadsIn
self.threadStatus = {}
self.threadTiming = {}
def activeThreads(self):
nActive = 0
for t in self.threadList:
if t.isAlive() :
nActive += 1
return nActive
def findPkg(self, lib):
"""docstring for findPkg"""
return self.rpr.findProduct(lib)
def analyzeAll(self):
import glob
xmlFiles = glob.glob('*/cpu?/*/*_vlgd.xml')
if not xmlFiles:
xmlFiles = glob.glob('*/*/cpu?/*/*_vlgd.xml')
allErrs = 0
if not xmlFiles: print 'revg> ALERT: no xml files found !?!?!'
for item in xmlFiles:
print 'revg> processing ', item
err = self.analyze(item)
if err > 0:
self.toDo.append(item)
allErrs += err
return allErrs
def analyze(self, inFileName):
"""docstring for analyze"""
doc = minidom.parse(open(inFileName))
errorListDoc = doc.getElementsByTagName('error')
for error in errorListDoc:
obj = error.getElementsByTagName("obj")[0].childNodes[0].nodeValue
if '/afs/cern.ch/cms/sw/ReleaseCandidates' in obj:
self.release = obj.split('/')[9]
self.plat = obj.split('/')[6]
stackFrame = error.getElementsByTagName("stack")[0].getElementsByTagName("frame")
for frame in stackFrame:
libName = frame.getElementsByTagName("obj")[0].childNodes[0].nodeValue.split('/')[-1]
# fncName = frame.getElementsByTagName("fn")[0].childNodes[0].nodeValue
if libName in self.ignoreLibs: continue
if libName not in self.libList: self.libList.append(libName)
return len(self.libList)
def showErrLibs(self):
"""docstring for showErrLibs"""
pass
print "revg> found ", len(self.libList), 'faulty libs:'
for lib in self.libList:
print lib
def rebuildPackages(self, dryRun=False):
"""docstring for rebuildPackages"""
print "revg> rebuilding packages ... "
self.startDir = os.getcwd()
cmd = '' # 'cd ..;'
cmd += 'eval `scram run -sh`;'
cmd += 'export CVSROOT=:gserver:cmssw.cvs.cern.ch:/cvs/CMSSW;'
for lib in self.libList:
package = self.findPkg(lib)
if not package: continue
cmd += 'addpkg -z ' + package + ';'
cmd += 'checkdeps -a ;'
for lib in self.libList:
package = self.findPkg(lib)
if not package: continue
cmd += '(cd ' + package + '&& scram b -v USER_CXXFLAGS="-g" -j 10 >vg-rebuild.log 2>&1 );'
print 'revg> in: ', os.getcwd(), "going to execute:", cmd.replace(';','\n')
ret = doCmd(cmd, dryRun)
if ret != 0:
print 'revg> ERROR executing cmd:"'+cmd+'"'
print ' return code :', ret
else:
print 'revg> Packages successfully rebuilt.'
def rerunValgrind(self, dryRun=False):
startDir = os.getcwd()
os.chdir('newPerf-'+self.part)
print 'revg> in ', os.getcwd(), ' going to re-run valgrind commands :'
print 'revg> toDo'
for item in self.toDo:
print ' ', item
# print 'revg> cmds'
# for item in self.vgCmds:
# print ' ', item
# print self.vgCmds
for item in self.toDo:
baseIn, xmlFile = os.path.split(item)
base = baseIn.replace('newPerf-1of2/', '').replace('newPerf-2of2/', '')
cmd = self.vgCmds[base]
step, cpu, what = base.split('/')
cmd = cmd.replace('valgrind --tool', 'valgrind --track-origins=yes --tool')
# make sure we don't run more than the allowed number of threads:
nAct = self.activeThreads()
# print '\n===========> ', nAct, self.maxThreads
while ( nAct >= self.maxThreads) :
# print '\n waiting ... ', nAct, self.maxThreads
# sys.stdout.flush()
time.sleep(1)
nAct = self.activeThreads()
continue
runner = ValgrindReRunner(step, cpu, cmd, dryRun)
self.threadList.append(runner)
runner.start()
print time.asctime()+'revg> waiting for threads to finish'
# wait until all threads are finished
while self.activeThreads() > 0:
time.sleep(5)
print time.asctime()+'revg> all threads finished, harvesting next'
resultInfo = {}
for t in self.threadList:
self.threadStatus[t.id] = t.status
self.threadTiming[t.id] = [t.startTime, t.stopTime]
resultInfo[t.id] = [t.status, t.startTime, t.stopTime, t.cpu, t.steps, t.cmd]
for id, status in self.threadStatus.items():
dTime = time.gmtime(self.threadTiming[id][1]-self.threadTiming[id][0])
dTimeAsc = time.strftime('%H:%M:%S', dTime)
print 'revg: result for %20s : %s, time to process: %s' % (id, str(status), dTimeAsc)
psrPklFileName = 'ValgrindReRunner-results-'+self.part+'.pkl'
try:
from pickle import Pickler
summFile = open(psrPklFileName,'w')
pklr = Pickler(summFile)
pklr.dump(socket.gethostname())
pklr.dump(self.part)
pklr.dump(self.toDo)
pklr.dump(resultInfo)
summFile.close()
print "Successfully wrote results file", psrPklFileName
except Exception, e:
print "ERROR when writing results file:", psrPklFileName,str(e)
os.chdir(startDir)
class ValgrindAnalyzer(object):
"""docstring for ValgrindAnalyzer"""
def __init__(self, partIn, maxThreadsIn=8):
super(ValgrindAnalyzer, self).__init__()
self.release = None
self.plat = None
self.part = partIn
self.ignoreLibs = ['libm-2.5.so', 'libpthread-2.5.so', 'cmsRun']
self.libList = []
prodFileName = os.environ[
'CMSSW_RELEASE_BASE'] + '/src/ReleaseProducts.list'
self.rpr = RelProdReader()
self.rpr.readProducts(prodFileName)
vgCmdFileName = os.environ['CMSSW_RELEASE_BASE'] + '/qaLogs/vgCmds.pkl'
from pickle import Unpickler
vgCmdFile = open(vgCmdFileName, 'r')
vgCmdPklr = Unpickler(vgCmdFile)
self.vgCmds = vgCmdPklr.load()
vgCmdFile.close()
self.toDo = []
self.threadList = []
self.maxThreads = maxThreadsIn
self.threadStatus = {}
self.threadTiming = {}
def activeThreads(self):
nActive = 0
for t in self.threadList:
if t.isAlive():
nActive += 1
return nActive
def findPkg(self, lib):
"""docstring for findPkg"""
return self.rpr.findProduct(lib)
def analyzeAll(self):
import glob
xmlFiles = glob.glob('*/cpu?/*/*_vlgd.xml')
if not xmlFiles:
xmlFiles = glob.glob('*/*/cpu?/*/*_vlgd.xml')
allErrs = 0
if not xmlFiles: print 'revg> ALERT: no xml files found !?!?!'
for item in xmlFiles:
print 'revg> processing ', item
err = self.analyze(item)
if err > 0:
self.toDo.append(item)
allErrs += err
return allErrs
def analyze(self, inFileName):
"""docstring for analyze"""
doc = minidom.parse(open(inFileName))
errorListDoc = doc.getElementsByTagName('error')
for error in errorListDoc:
obj = error.getElementsByTagName("obj")[0].childNodes[0].nodeValue
if '/afs/cern.ch/cms/sw/ReleaseCandidates' in obj:
self.release = obj.split('/')[9]
self.plat = obj.split('/')[6]
stackFrame = error.getElementsByTagName(
"stack")[0].getElementsByTagName("frame")
for frame in stackFrame:
libName = frame.getElementsByTagName(
"obj")[0].childNodes[0].nodeValue.split('/')[-1]
# fncName = frame.getElementsByTagName("fn")[0].childNodes[0].nodeValue
if libName in self.ignoreLibs: continue
if libName not in self.libList: self.libList.append(libName)
return len(self.libList)
def showErrLibs(self):
"""docstring for showErrLibs"""
pass
print "revg> found ", len(self.libList), 'faulty libs:'
for lib in self.libList:
print lib
def rebuildPackages(self, dryRun=False):
"""docstring for rebuildPackages"""
print "revg> rebuilding packages ... "
self.startDir = os.getcwd()
cmd = '' # 'cd ..;'
cmd += 'eval `scram run -sh`;'
cmd += 'export CVSROOT=:gserver:cmssw.cvs.cern.ch:/cvs/CMSSW;'
for lib in self.libList:
package = self.findPkg(lib)
if not package: continue
cmd += 'addpkg -z ' + package + ';'
cmd += 'checkdeps -a ;'
for lib in self.libList:
package = self.findPkg(lib)
if not package: continue
cmd += '(cd ' + package + '&& scram b -v USER_CXXFLAGS="-g" -j 10 >vg-rebuild.log 2>&1 );'
print 'revg> in: ', os.getcwd(), "going to execute:", cmd.replace(
';', '\n')
ret = doCmd(cmd, dryRun)
if ret != 0:
print 'revg> ERROR executing cmd:"' + cmd + '"'
print ' return code :', ret
else:
print 'revg> Packages successfully rebuilt.'
def rerunValgrind(self, dryRun=False):
startDir = os.getcwd()
os.chdir('newPerf-' + self.part)
print 'revg> in ', os.getcwd(), ' going to re-run valgrind commands :'
print 'revg> toDo'
for item in self.toDo:
print ' ', item
# print 'revg> cmds'
# for item in self.vgCmds:
# print ' ', item
# print self.vgCmds
for item in self.toDo:
baseIn, xmlFile = os.path.split(item)
base = baseIn.replace('newPerf-1of2/',
'').replace('newPerf-2of2/', '')
cmd = self.vgCmds[base]
step, cpu, what = base.split('/')
cmd = cmd.replace('valgrind --tool',
'valgrind --track-origins=yes --tool')
# make sure we don't run more than the allowed number of threads:
nAct = self.activeThreads()
# print '\n===========> ', nAct, self.maxThreads
while (nAct >= self.maxThreads):
# print '\n waiting ... ', nAct, self.maxThreads
# sys.stdout.flush()
time.sleep(1)
nAct = self.activeThreads()
continue
runner = ValgrindReRunner(step, cpu, cmd, dryRun)
self.threadList.append(runner)
runner.start()
print time.asctime() + 'revg> waiting for threads to finish'
# wait until all threads are finished
while self.activeThreads() > 0:
time.sleep(5)
print time.asctime() + 'revg> all threads finished, harvesting next'
resultInfo = {}
for t in self.threadList:
self.threadStatus[t.id] = t.status
self.threadTiming[t.id] = [t.startTime, t.stopTime]
resultInfo[t.id] = [
t.status, t.startTime, t.stopTime, t.cpu, t.steps, t.cmd
]
for id, status in self.threadStatus.items():
dTime = time.gmtime(self.threadTiming[id][1] -
self.threadTiming[id][0])
dTimeAsc = time.strftime('%H:%M:%S', dTime)
print 'revg: result for %20s : %s, time to process: %s' % (
id, str(status), dTimeAsc)
psrPklFileName = 'ValgrindReRunner-results-' + self.part + '.pkl'
try:
from pickle import Pickler
summFile = open(psrPklFileName, 'w')
pklr = Pickler(summFile)
pklr.dump(socket.gethostname())
pklr.dump(self.part)
pklr.dump(self.toDo)
pklr.dump(resultInfo)
summFile.close()
print "Successfully wrote results file", psrPklFileName
except Exception, e:
print "ERROR when writing results file:", psrPklFileName, str(e)
os.chdir(startDir)
