def save(self, compress=True):
rawData = self.getData()
from RingerCore import save
try:
save(rawData, self._filePath, protocol='mat')
except ImportError:
self._warning(
("Cannot save matlab file, it seems that scipy is not "
"available."))
return save(rawData, self._filePath, compress=compress)
def save(self, compress=True):
rawData = self.getData()
try:
import scipy.io
scipy.io.savemat(ensureExtension(self._filePath, '.mat'), rawData)
except ImportError:
self._warning(
("Cannot save matlab file, it seems that scipy is not "
"available."))
return save(rawData, self._filePath, compress=compress)
parser.add_argument('-d',
'--data',
action='store',
dest='data',
required=True,
nargs='+',
help="The input tuning files.")
import sys, os
if len(sys.argv) == 1:
parser.print_help()
sys.exit(1)
args = parser.parse_args()
# Take all files
paths = csvStr2List(args.data)
paths = expandFolders(paths)
from RingerCore import load, save, appendToFileName
for f in paths:
ff = load(f)
for k in ff.keys():
if 'SP' in k:
etBin = ff[k]['etBinIdx']
etaBin = ff[k]['etaBinIdx']
print 'etBin = ', etBin, ', etaBin = ', etaBin
outname = f.split('/')[len(f.split('/')) - 2]
cOutputName = appendToFileName(outname, ('et%d_eta%d') % (etBin, etaBin))
save(ff, cOutputName, compress=True)
def save(self, compress = True): return save( self.getData(), self._filePath, compress = compress )
def create_weights(self, discrList, filename):
self._logger.info('Getting weights...')
from TuningTools import TuningToolCores
modelDict = {}
modelDict['__version__'] = self._version
modelDict['__type__'] = 'Fex'
modelDict['__core__'] = TuningToolCores.FastNet
modelDict['metadata'] = {
'UseEtaVar': self._useEtaVar,
'UseLumiVar': self._useLumiVar,
}
modelDict['tuning'] = []
def tolist(a):
if isinstance(a, list): return a
else: return a.tolist()
for model in discrList:
#etBinIdx = model['etBinIdx']
#etaBinIdx = model['etaBinIdx']
## Discriminator configuration
discrData = {}
discrData['etBin'] = model['etBin'].tolist()
if discrData['etBin'][0] == 0 and discrData['etBin'][1] == 20.0:
discrData['etBin'] = [15.0, 20.0]
discrData['etaBin'] = model['etaBin'].tolist()
discrData['nodes'] = tolist(model['discriminator']['nodes'])
discrData['bias'] = tolist(model['discriminator']['bias'])
discrData['weights'] = tolist(model['discriminator']['weights'])
modelDict['tuning'].append({'discriminator': discrData})
if self._toPickle:
self._logger.info('Export weights to pickle format...')
modelDict['__version__'] = self._version
modelDict['__core__'] = TuningToolCores.keras
from RingerCore import save
save(modelDict, filename)
from ROOT import TFile, TTree
from ROOT import std
self._logger.info('Export weights to root format...')
### Create the discriminator root object
fdiscr = TFile(appendToFileName(filename, '.root', separator=''),
'recreate')
self.__createRootParameter('int', '__version__', self._version).Write()
self.__createRootParameter('int', '__core__',
TuningToolCores.FastNet).Write()
fdiscr.mkdir('tuning')
fdiscr.cd('tuning')
tdiscr = TTree('discriminators', '')
for idx, b in enumerate(self._discrBranches):
b[2] = std.vector(b[0])()
tdiscr.Branch(b[1], 'vector<%s>' % b[0], b[2])
for t in modelDict['tuning']:
for idx, b in enumerate(self._discrBranches):
self.__attachToVector(t['discriminator'][b[1]], b[2])
tdiscr.Fill()
tdiscr.Write()
### Create the thresholds root object
fdiscr.mkdir('metadata')
fdiscr.cd('metadata')
for key, value in modelDict['metadata'].iteritems():
self._logger.info('Saving metadata %s as %s', key, value)
self.__createRootParameter('int' if type(value) is int else 'bool',
key, value).Write()
fdiscr.Close()
def create_thresholds(self, discrList, filename):
self._logger.info('Getting thresholds...')
from TuningTools import TuningToolCores
modelDict = {}
modelDict['__version__'] = self._version
modelDict['__type__'] = 'Hypo'
modelDict['__core__'] = TuningToolCores.FastNet
modelDict['metadata'] = {
'UseNoActivationFunctionInTheLastLayer':
self._removeOutputTansigTF,
'DoPileupCorrection': self._doPileupCorrection,
'LumiCut': self._maxPileupLinearCorrectionValue,
}
modelDict['tuning'] = []
for model in discrList:
thresData = {}
#etBinIdx = model['etBinIdx']
#etaBinIdx = model['etaBinIdx']
thresData['etBin'] = model['etBin'].tolist()
thresData['etaBin'] = model['etaBin'].tolist()
thresData['thresholds'] = model['threshold']
modelDict['tuning'].append(thresData)
if self._toPickle:
self._logger.info('Export Thresholds to pickle format...')
modelDict['__version__'] = self._version
modelDict['__core__'] = TuningToolCores.keras
from RingerCore import save
save(modelDict, filename)
from ROOT import TFile, TTree
from ROOT import std
self._logger.info('Export Thresholds to root format...')
### Create the thresholds root object
fthres = TFile(appendToFileName(filename, '.root', separator=''),
'recreate')
self.__createRootParameter('int', '__version__', self._version).Write()
self.__createRootParameter('int', '__core__',
TuningToolCores.FastNet).Write()
fthres.mkdir('tuning')
fthres.cd('tuning')
tthres = TTree('thresholds', '')
for idx, b in enumerate(self._thresBranches):
b[2] = std.vector(b[0])()
tthres.Branch(b[1], 'vector<%s>' % b[0], b[2])
for t in modelDict['tuning']:
for idx, b in enumerate(self._thresBranches):
self.__attachToVector(t[b[1]], b[2])
tthres.Fill()
tthres.Write()
fthres.mkdir('metadata')
fthres.cd('metadata')
for key, value in modelDict['metadata'].iteritems():
self._logger.info('Saving metadata %s as %s', key, value)
self.__createRootParameter('int' if type(value) is int else 'bool',
key, value).Write()
fthres.Close()
os.path.basename(cOutputName))
f = ROOT.TFile(inFile, 'r')
data = {}
for keyName, obj in getall(f):
mainLogger.debug("Reading key: %s", keyName)
shortKey = keyName.split('/')[-1]
if not issubclass(type(obj), ROOT.TH1):
mainLogger.verbose("Ignoring key: %s", shortKey)
continue
hist = obj
result = [exclude in shortKey for exclude in args.exclude_keys]
if result and all(result):
mainLogger.debug("key <%s> does not match any filter",
shortKey)
continue
if result and all(result):
mainLogger.debug("key <%s> matches exclude pattern", shortKey)
continue
if not hist:
mainLogger.warning("Couldn't retrieve histogram with key: %s",
shortKey)
data[shortKey] = rnp.hist2array(hist,
include_overflow=True,
return_edges=True)
savedPath = save(data, cOutputName, protocol='savez_compressed')
mainLogger.info("Successfully created numpy file: %s", cOutputName)
else:
mainLogger.error("Cannot transform file '%s' to numpy format." %
inFile)
# end of (for fileCollection)
def loop(self, **kw):
from scipy.io import loadmat
import gc
output = kw.pop('output', 'Mon')
tuningReport = kw.pop('tuningReport', 'tuningReport')
doBeamer = kw.pop('doBeamer', True)
shortSlides = kw.pop('shortSlides', False)
debug = kw.pop('debug', False)
overwrite = kw.pop('overwrite', False)
choicesfile = kw.pop('choicesfile', None)
basepath = output
basepath += ('_et%d_eta%d') % (self._infoObjs[0].etbinidx(),
self._infoObjs[0].etabinidx())
if choicesfile: choices = loadmat(choicesfile)
if not overwrite and os.path.isdir(basepath):
self._logger.warning("Monitoring output path already exists!")
return
if shortSlides:
self._logger.warning('Short slides enabled! Doing only tables...')
if debug:
self._logger.warning('Debug mode activated!')
wantedPlotNames = {'allBestTstSorts','allBestOpSorts','allWorstTstSorts', 'allWorstOpSorts',\
'allBestTstNeurons','allBestOpNeurons', 'allWorstTstNeurons', 'allWorstOpNeurons'}
perfBenchmarks = dict()
pathBenchmarks = dict()
from PlotHolder import PlotHolder
from PlotHelper import plot_4c, plot_rocs, plot_nnoutput
from TuningMonitoringInfo import MonitoringPerfInfo
#Loop over benchmarks
for infoObj in self._infoObjs:
#Initialize all plos
plotObjects = dict()
perfObjects = dict()
infoObjects = dict()
pathObjects = dict()
#Init PlotsHolder
for plotname in wantedPlotNames:
if 'Sorts' in plotname:
plotObjects[plotname] = PlotHolder(label='Sort')
else:
plotObjects[plotname] = PlotHolder(label='Neuron')
# keyboard()
#Retrieve benchmark name
benchmarkName = infoObj.name()
#Retrieve reference name
reference = infoObj.reference()
#summary
csummary = infoObj.summary()
#benchmark object
cbenchmark = infoObj.rawBenchmark()
#
etBin = infoObj.etbin()
# reference value
refVal = infoObj.rawBenchmark()['refVal']
#Eta bin
etabinidx = infoObj.etabinidx()
#Et bin
etbinidx = infoObj.etbinidx()
#Eta bin
etabin = infoObj.etabin()
#Et bin
etbin = infoObj.etbin()
self._logger.info(
('Start loop over the benchmark: %s and etaBin = %d etBin = %d'
) % (benchmarkName, etabinidx, etbinidx))
import copy
args = dict()
args['reference'] = reference
args['refVal'] = refVal
args['eps'] = cbenchmark['eps']
self._logger.info('Creating plots...')
# Creating plots
for neuron in progressbar(infoObj.neuronBounds(),
len(infoObj.neuronBounds()),
'Loading : ',
60,
False,
logger=self._logger):
if choicesfile:
neuron = choices['choices'][infoObj.name().split(
'_')[-1]][0][0][etbinidx][etabinidx]
# Figure path location
currentPath = ('%s/figures/%s/%s') % (basepath, benchmarkName,
'neuron_' + str(neuron))
neuronName = 'config_' + str(neuron).zfill(3)
# Create folder to store all plot objects
mkdir_p(currentPath)
#Clear all hold plots stored
plotObjects['allBestTstSorts'].clear()
plotObjects['allBestOpSorts'].clear()
infoObjects['allInfoOpBest_' + neuronName] = list()
#plotObjects['allWorstTstSorts'].clear()
#plotObjects['allWorstOpSorts'].clear()
for sort in infoObj.sortBounds(neuron):
sortName = 'sort_' + str(sort).zfill(3)
#Init bounds
initBounds = infoObj.initBounds(neuron, sort)
#Create path list from initBound list
initPaths = [('%s/%s/%s/init_%s') %
(benchmarkName, neuronName, sortName, init)
for init in initBounds]
self._logger.debug('Creating init plots into the path: %s, (neuron_%s,sort_%s)', \
benchmarkName, neuron, sort)
obj = PlotHolder(label='Init')
try: #Create plots holder class (Helper), store all inits
obj.retrieve(self._rootObj, initPaths)
except RuntimeError:
self._logger.fatal('Can not create plot holder object')
#Hold all inits from current sort
obj.set_index_correction(initBounds)
obj.set_best_index(
csummary[neuronName][sortName]['infoTstBest']['init'])
obj.set_worst_index(
csummary[neuronName][sortName]['infoTstWorst']['init'])
plotObjects['allBestTstSorts'].append(
copy.deepcopy(obj.get_best()))
obj.set_best_index(
csummary[neuronName][sortName]['infoOpBest']['init'])
obj.set_worst_index(
csummary[neuronName][sortName]['infoOpWorst']['init'])
plotObjects['allBestOpSorts'].append(
copy.deepcopy(obj.get_best()))
#plotObjects['allWorstTstSorts'].append( copy.deepcopy(tstObj.getBest() )
#plotObjects['allWorstOpSorts'].append( copy.deepcopy(opObj.getBest() )
infoObjects['allInfoOpBest_' + neuronName].append(
copy.deepcopy(
csummary[neuronName][sortName]['infoOpBest']))
#Release memory
del obj
#Loop over sorts
gc.collect()
plotObjects['allBestTstSorts'].set_index_correction(
infoObj.sortBounds(neuron))
plotObjects['allBestOpSorts'].set_index_correction(
infoObj.sortBounds(neuron))
#plotObjects['allWorstTstSorts'].setIdxCorrection( infoObj.sortBounds(neuron) )
#plotObjects['allWorstOpSorts'].setIdxCorrection( infoObj.sortBounds(neuron) )
# Best and worst sorts for this neuron configuration
plotObjects['allBestTstSorts'].set_best_index(
csummary[neuronName]['infoTstBest']['sort'])
plotObjects['allBestTstSorts'].set_worst_index(
csummary[neuronName]['infoTstWorst']['sort'])
plotObjects['allBestOpSorts'].set_best_index(
csummary[neuronName]['infoOpBest']['sort'])
plotObjects['allBestOpSorts'].set_worst_index(
csummary[neuronName]['infoOpWorst']['sort'])
# Hold the information from the best and worst discriminator for this neuron
infoObjects['infoOpBest_' + neuronName] = copy.deepcopy(
csummary[neuronName]['infoOpBest'])
infoObjects['infoOpWorst_' + neuronName] = copy.deepcopy(
csummary[neuronName]['infoOpWorst'])
# Best and worst neuron sort for this configuration
plotObjects['allBestTstNeurons'].append(
copy.deepcopy(plotObjects['allBestTstSorts'].get_best()))
plotObjects['allBestOpNeurons'].append(
copy.deepcopy(plotObjects['allBestOpSorts'].get_best()))
plotObjects['allWorstTstNeurons'].append(
copy.deepcopy(plotObjects['allBestTstSorts'].get_worst()))
plotObjects['allWorstOpNeurons'].append(
copy.deepcopy(plotObjects['allBestOpSorts'].get_worst()))
# Create perf (tables) Objects for test and operation (Table)
perfObjects[neuronName] = MonitoringPerfInfo(
benchmarkName, reference,
csummary[neuronName]['summaryInfoTst'],
csummary[neuronName]['infoOpBest'], cbenchmark)
# Debug information
self._logger.debug(('Crossval indexs: (bestSort = %d, bestInit = %d) (worstSort = %d, bestInit = %d)')%\
(plotObjects['allBestTstSorts'].best, plotObjects['allBestTstSorts'].get_best()['bestInit'],
plotObjects['allBestTstSorts'].worst, plotObjects['allBestTstSorts'].get_worst()['bestInit']))
self._logger.debug(('Operation indexs: (bestSort = %d, bestInit = %d) (worstSort = %d, bestInit = %d)')%\
(plotObjects['allBestOpSorts'].best, plotObjects['allBestOpSorts'].get_best()['bestInit'],
plotObjects['allBestOpSorts'].worst, plotObjects['allBestOpSorts'].get_worst()['bestInit']))
# Figure 1: Plot all validation/test curves for all crossval sorts tested during
# the training. The best sort will be painted with black and the worst sort will
# be on red color. There is a label that will be draw into the figure to show
# the current location (neuron, sort, init) of the best and the worst network.
args['label'] = ('#splitline{#splitline{Total sorts: %d}{etaBin: %d, etBin: %d}}'+\
'{#splitline{sBestIdx: %d iBestIdx: %d}{sWorstIdx: %d iBestIdx: %d}}') % \
(plotObjects['allBestTstSorts'].size(),etabinidx, etbinidx, plotObjects['allBestTstSorts'].best, \
plotObjects['allBestTstSorts'].get_best()['bestInit'], plotObjects['allBestTstSorts'].worst,\
plotObjects['allBestTstSorts'].get_worst()['bestInit'])
args['cname'] = ('%s/plot_%s_neuron_%s_sorts_val') % (
currentPath, benchmarkName, neuron)
args['set'] = 'val'
args['operation'] = False
args['paintListIdx'] = [
plotObjects['allBestTstSorts'].best,
plotObjects['allBestTstSorts'].worst
]
pname1 = plot_4c(plotObjects['allBestTstSorts'], args)
# Figure 2: Plot all validation/test curves for all crossval sorts tested during
# the training. The best sort will be painted with black and the worst sort will
# be on red color. But, here the painted curves represented the best and the worst
# curve from the operation dataset. In other words, we pass all events into the
# network and get the efficiencis than we choose the best operation and the worst
# operation network and paint the validation curve who represent these sorts.
# There is a label that will be draw into the figure to show
# the current location (neuron, sort, init) of the best and the worst network.
args['label'] = ('#splitline{#splitline{Total sorts: %d (operation)}{etaBin: %d, etBin: %d}}'+\
'{#splitline{sBestIdx: %d iBestIdx: %d}{sWorstIdx: %d iBestIdx: %d}}') % \
(plotObjects['allBestOpSorts'].size(),etabinidx, etbinidx, plotObjects['allBestOpSorts'].best, \
plotObjects['allBestOpSorts'].get_best()['bestInit'], plotObjects['allBestOpSorts'].worst,\
plotObjects['allBestOpSorts'].get_worst()['bestInit'])
args['cname'] = ('%s/plot_%s_neuron_%s_sorts_op') % (
currentPath, benchmarkName, neuron)
args['set'] = 'val'
args['operation'] = True
args['paintListIdx'] = [
plotObjects['allBestOpSorts'].best,
plotObjects['allBestOpSorts'].worst
]
pname2 = plot_4c(plotObjects['allBestOpSorts'], args)
# Figure 3: This figure show us in deteails the best operation network for the current hidden
# layer and benchmark analysis. Depend on the benchmark, we draw lines who represents the
# stops for each curve. The current neuron will be the last position of the plotObjects
splotObject = PlotHolder()
args['label'] = ('#splitline{#splitline{Best network neuron: %d}{etaBin: %d, etBin: %d}}'+\
'{#splitline{sBestIdx: %d iBestIdx: %d}{}}') % \
(neuron,etabinidx, etbinidx, plotObjects['allBestOpSorts'].best, plotObjects['allBestOpSorts'].get_best()['bestInit'])
args['cname'] = ('%s/plot_%s_neuron_%s_best_op') % (
currentPath, benchmarkName, neuron)
args['set'] = 'val'
args['operation'] = True
splotObject.append(plotObjects['allBestOpNeurons'][-1])
pname3 = plot_4c(splotObject, args)
# Figure 4: Here, we have a plot of the discriminator output for all dataset. Black histogram
# represents the signal and the red onces represent the background. TODO: Apply this outputs
# using the feedfoward manual method to generate the network outputs and create the histograms.
args['cname'] = ('%s/plot_%s_neuron_%s_best_op_output') % (
currentPath, benchmarkName, neuron)
args['nsignal'] = self._data[0].shape[0]
args['nbackground'] = self._data[1].shape[0]
sbest = plotObjects['allBestOpNeurons'][-1]['bestSort']
args['cut'] = csummary[neuronName][
'sort_' + str(sbest).zfill(3)]['infoOpBest']['cut']
args['rocname'] = 'roc_operation'
pname4 = plot_nnoutput(splotObject, args)
# Figure 5: The receive operation test curve for all sorts using the test dataset as base.
# Here, we will draw the current tunnel and ref value used to set the discriminator threshold
# when the bechmark are Pd or Pf case. When we use the SP case, this tunnel will not be ploted.
# The black curve represents the best sort and the red onces the worst sort. TODO: Put the SP
# point for the best and worst when the benchmark case is SP.
args['cname'] = ('%s/plot_%s_neuron_%s_sorts_roc_tst') % (
currentPath, benchmarkName, neuron)
args['set'] = 'tst'
args['paintListIdx'] = [
plotObjects['allBestTstSorts'].best,
plotObjects['allBestTstSorts'].worst
]
pname5 = plot_rocs(plotObjects['allBestTstSorts'], args)
# Figure 6: The receive operation curve for all sorts using the operation dataset (train+test) as base.
# Here, we will draw the current tunnel and ref value used to set the discriminator threshold
# when the bechmark are Pd or Pf case. When we use the SP case, this tunnel will not be ploted.
# The black curve represents the best sort and the red onces the worst sort. TODO: Put the SP
# point for the best and worst when the benchmark case is SP.
args['cname'] = ('%s/plot_%s_neuron_%s_sorts_roc_op') % (
currentPath, benchmarkName, neuron)
args['set'] = 'operation'
args['paintListIdx'] = [
plotObjects['allBestOpSorts'].best,
plotObjects['allBestOpSorts'].worst
]
pname6 = plot_rocs(plotObjects['allBestOpSorts'], args)
# Map names for beamer, if you add a plot, you must add into
# the path objects holder
pathObjects['neuron_' + str(neuron) + '_sorts_val'] = pname1
pathObjects['neuron_' + str(neuron) + '_sort_op'] = pname2
pathObjects['neuron_' + str(neuron) + '_best_op'] = pname3
pathObjects['neuron_' + str(neuron) +
'_best_op_output'] = pname4
pathObjects['neuron_' + str(neuron) +
'_sorts_roc_tst'] = pname5
pathObjects['neuron_' + str(neuron) + '_sorts_roc_op'] = pname6
if choicesfile: break
#Loop over neurons
#External
pathBenchmarks[benchmarkName] = pathObjects
perfBenchmarks[benchmarkName] = perfObjects
#Release memory
for xname in plotObjects.keys():
del plotObjects[xname]
gc.collect()
#if debug: break
#Loop over benchmark
#Eta bin
# etabinidx = self._infoObjs[0].etabinidx()
#Et bin
binBounds = dict()
if len(etbin) > 0:
binBounds['etbinstr'] = r'$%d < E_{T} \text{[Gev]}<%d$' % etbin
else:
binBounds['etbinstr'] = r'\text{etBin[%d]}' % etbinidx
if len(etabin) > 0:
binBounds['etabinstr'] = r'$%.2f<\eta<%.2f$' % etabin
else:
binBounds['etabinstr'] = r'\text{etaBin[%d]}' % etabinidx
perfBounds = dict()
perfBounds['bounds'] = binBounds
perfBounds['perf'] = perfBenchmarks
fname = basepath + '/' + 'perfBounds'
save(perfBounds, fname)
#Start beamer presentation
if doBeamer:
from BeamerTemplates import BeamerReport, BeamerTables, BeamerFigure, BeamerBlocks
#Eta bin
etabin = self._infoObjs[0].etabin()
etabinidx = self._infoObjs[0].etabinidx()
#Et bin
etbin = self._infoObjs[0].etbin()
etbinidx = self._infoObjs[0].etbinidx()
#Create the beamer manager
reportname = ('%s_et%d_eta%d') % (output, etbinidx, etabinidx)
beamer = BeamerReport(basepath + '/' + reportname,
title=('Tuning Report (et=%d, eta=%d)') %
(etbinidx, etabinidx))
neuronBounds = self._infoObjs[0].neuronBounds()
for neuron in neuronBounds:
#Make the tables for crossvalidation
ptableCross = BeamerTables(
frametitle=[
'Neuron ' + str(neuron) +
': Cross Validation Performance',
'Neuron ' + str(neuron) + ": Operation Best Network"
],
caption=[
'Efficiencies from each benchmark.',
'Efficiencies for the best operation network'
])
block = BeamerBlocks('Neuron ' + str(neuron) + ' Analysis', [
('All sorts (validation)',
'All sorts evolution are ploted, each sort represents the best init;'
),
('All sorts (operation)',
'All sorts evolution only for operation set;'),
('Best operation',
'Detailed analysis from the best sort discriminator.'),
('Tables', 'Cross validation performance')
])
if not shortSlides: block.tolatex(beamer.file())
for info in self._infoObjs:
#If we produce a short presentation, we do not draw all plots
if not shortSlides:
bname = info.name().replace('OperationPoint_', '')
fig1 = BeamerFigure(
pathBenchmarks[info.name()]['neuron_' +
str(neuron) +
'_sorts_val'].replace(
basepath + '/',
''),
0.7,
frametitle=bname + ', Neuron ' + str(neuron) +
': All sorts (validation)')
fig2 = BeamerFigure(pathBenchmarks[info.name()][
'neuron_' + str(neuron) +
'_sorts_roc_tst'].replace(basepath + '/', ''),
0.8,
frametitle=bname + ', Neuron ' +
str(neuron) +
': All ROC sorts (validation)')
fig3 = BeamerFigure(
pathBenchmarks[info.name()]['neuron_' +
str(neuron) +
'_sort_op'].replace(
basepath + '/',
''),
0.7,
frametitle=bname + ', Neuron ' + str(neuron) +
': All sorts (operation)')
fig4 = BeamerFigure(pathBenchmarks[info.name()][
'neuron_' + str(neuron) + '_sorts_roc_op'].replace(
basepath + '/', ''),
0.8,
frametitle=bname + ', Neuron ' +
str(neuron) +
': All ROC sorts (operation)')
fig5 = BeamerFigure(
pathBenchmarks[info.name()]['neuron_' +
str(neuron) +
'_best_op'].replace(
basepath + '/',
''),
0.7,
frametitle=bname + ', Neuron ' + str(neuron) +
': Best Network')
fig6 = BeamerFigure(pathBenchmarks[info.name()][
'neuron_' + str(neuron) +
'_best_op_output'].replace(basepath + '/', ''),
0.8,
frametitle=bname + ', Neuron ' +
str(neuron) +
': Best Network output')
#Draw figures into the tex file
fig1.tolatex(beamer.file())
fig2.tolatex(beamer.file())
fig3.tolatex(beamer.file())
fig4.tolatex(beamer.file())
fig5.tolatex(beamer.file())
fig6.tolatex(beamer.file())
#Concatenate performance table, each line will be a benchmark
#e.g: det, sp and fa
ptableCross.add(
perfBenchmarks[info.name()]['config_' +
str(neuron).zfill(3)])
#if debug: break
ptableCross.tolatex(
beamer.file()) # internal switch is false to true: test
ptableCross.tolatex(beamer.file(
)) # internal swotch is true to false: operation
if debug: break
beamer.close()
self._logger.info('Done! ')
def save(self, compress = True): return save( self.getData(), self._filePath, compress = compress )
changePassed(key, mat)
changeTotal(key, mat)
from RingerCore import load, save
a = load(
'mc15_13TeV.361106.423300.sgn.trigegprobes.bkg.vetotruth.trig.l2calo.eg.std.grid.medium.npz'
)
d = dict(a)
changeOp(d, 'signal_efficiencies', mergeEffTable(veryloose20160701))
changeOp(d, 'background_efficiencies', pfrefs_vloose)
save(
d,
'mc15_13TeV.361106.423300.sgn.trigegprobes.bkg.vetotruth.trig.l2calo.eg.std.grid.veryloose',
protocol='savez_compressed')
d = dict(a)
changeOp(d, 'signal_efficiencies', mergeEffTable(loose20160701))
changeOp(d, 'background_efficiencies', pfrefs_loose)
save(
d,
'mc15_13TeV.361106.423300.sgn.trigegprobes.bkg.vetotruth.trig.l2calo.eg.std.grid.loose',
protocol='savez_compressed')
d = dict(a)
changeOp(d, 'signal_efficiencies', mergeEffTable(tight20160701))
changeOp(d, 'background_efficiencies', pfrefs_tight)
save(
d,
from RingerCore import Logger, LoggingLevel, save, load, expandFolders, traverse
import numpy as np
from TuningTools.coreDef import retrieve_npConstants
npCurrent, _ = retrieve_npConstants()
npCurrent.level = args.output_level
logger = Logger.getModuleLogger( __name__, args.output_level )
files = expandFolders( args.inputs ) # FIXME *.npz
from zipfile import BadZipfile
for f in files:
logger.info("Changing representation of file '%s'...", f)
try:
data = dict(load(f))
except BadZipfile, e:
logger.warning("Couldn't load file '%s'. Reason:\n%s", f, str(e))
continue
logger.debug("Finished loading file '%s'...", f)
for key in data:
if key == 'W':
for obj, idx, parent, _, _ in traverse(data[key],
tree_types = (np.ndarray,),
max_depth = 3):
parent[idx] = obj.T
elif type(data[key]) is np.ndarray:
logger.debug("Checking key '%s'...", key)
data[key] = npCurrent.toRepr(data[key])
path = save(data, f, protocol = 'savez_compressed')
logger.info("Overwritten file '%s'",f)
if args.outputPath is None:
args.outputPath = os.path.dirname(args.inputFile)
if not os.path.isdir( args.outputPath ): mkdir_p( args.outputPath )
f = load(args.inputFile)
# Copy all metada information
baseDict = { k : f[k] for k in f.keys() if not '_etBin_' in k and not '_etaBin_' in k }
nEtBins = f['nEtBins'].item()
nEtaBins = f['nEtaBins'].item()
for etIdx, etaIdx in progressbar( product(xrange(nEtBins), xrange(nEtaBins))
, nEtBins*nEtaBins
, logger = mainLogger
, prefix = 'Juicing file '):
binDict= {k:f[k] for k in f.keys() if 'etBin_%d_etaBin_%d'%(etIdx,etaIdx) in k}
binDict.update(baseDict)
from copy import deepcopy
for layer in caloLayers:
pp=PreProcChain([RingerLayerSegmentation(layer=layer)])
tmpBinDict = deepcopy(binDict)
for key in binDict.keys():
if 'Patterns' in key:
tmpBinDict[key] = pp(binDict[key])
outFile = os.path.join( args.outputPath, os.path.basename( appendToFileName(args.inputFile.replace('calo','calo'+RingerLayer.tostring(layer)),
'et%d_eta%d' % (etIdx, etaIdx) ) ) )
save(tmpBinDict, outFile, protocol = 'savez_compressed' )
if checkExtension( inFile, "root" ):
cOutputName = changeExtension( inFile, '.npz' )
if args.change_output_folder:
import os.path
cOutputName = os.path.join( os.path.abspath(args.change_output_folder) , os.path.basename(cOutputName) )
f = ROOT.TFile( inFile, 'r' )
data = {}
for keyName, obj in getall(f):
mainLogger.debug("Reading key: %s", keyName)
shortKey = keyName.split('/')[-1]
if not issubclass(type(obj), ROOT.TH1):
mainLogger.verbose("Ignoring key: %s", shortKey )
continue
hist = obj
if all( [wanted not in shortKey for wanted in args.filter_keys] ):
mainLogger.debug("key <%s> does not match any filter", shortKey )
continue
if all( [exclude in shortKey for exclude in args.exclude_keys] ):
mainLogger.debug("key <%s> matches exclude pattern", shortKey )
continue
if not hist:
mainLogger.warning("Couldn't retrieve histogram with key: %s", shortKey )
data[ shortKey ] = rnp.hist2array( hist, include_overflow=True, return_edges=True )
savedPath = save(data, cOutputName, protocol = 'savez_compressed')
mainLogger.info("Successfully created numpy file: %s", cOutputName)
else:
mainLogger.error("Cannot transform file '%s' to numpy format." % inFile)
# end of (for fileCollection)
args = parser.parse_args(namespace=LoggerNamespace())
from RingerCore import Logger, LoggingLevel, save, load, expandFolders, traverse
import numpy as np
from TuningTools.coreDef import npCurrent
npCurrent.level = args.output_level
logger = Logger.getModuleLogger(__name__, args.output_level)
files = expandFolders(args.inputs) # FIXME *.npz
from zipfile import BadZipfile
for f in files:
logger.info("Changing representation of file '%s'...", f)
try:
data = dict(load(f))
except BadZipfile, e:
logger.warning("Couldn't load file '%s'. Reason:\n%s", f, str(e))
continue
logger.debug("Finished loading file '%s'...", f)
for key in data:
if key == 'W':
for obj, idx, parent, _, _ in traverse(data[key],
tree_types=(np.ndarray, ),
max_depth=3):
parent[idx] = obj.T
elif type(data[key]) is np.ndarray:
logger.debug("Checking key '%s'...", key)
data[key] = npCurrent.toRepr(data[key])
path = save(data, f, protocol='savez_compressed')
logger.info("Overwritten file '%s'", f)
def create_weights(self, discrList, filename):
self._logger.info('Getting weights...')
from TuningTools import TuningToolCores
modelDict = {}
modelDict['__version__'] = self._version
modelDict['__type__'] = 'Fex'
modelDict['__core__'] = TuningToolCores.FastNet
modelDict['metadata'] = {
'UseEtaVar': self._useEtaVar,
'UseLumiVar': self._useLumiVar,
'UseCaloRings': True,
'UseTrack': False,
'UseShowerShape': False,
'RemoveOutputTansigTF': self._removeOutputTansigTF,
}
modelDict['tuning'] = []
def tolist(a):
if isinstance(a, list): return a
elif isinstance(a, tuple): return a
else: return a.tolist()
from keras.models import model_from_json
from keras.layers import Conv2D, Dense, Activation
import json
from copy import deepcopy
for model in discrList:
etBinIdx = model['etBinIdx']
etaBinIdx = model['etaBinIdx']
## Discriminator configuration
discrData = {}
discrData['etBin'] = tolist(model['etBin'])
discrData['etaBin'] = tolist(model['etaBin'])
discrData['muBin'] = tolist(model['muBin'])
if self._toPickle:
discrData['model'] = deepcopy(model['discriminator']['model'])
discrData['weights'] = deepcopy(
model['discriminator']['weights'])
else:
keras_model = model_from_json(
json.dumps(model['discriminator']['model'],
separators=(',', ':')))
keras_model.set_weights(model['discriminator']['weights'])
### Extract and reshape Keras weights
dense_weights = []
dense_bias = []
dense_nodes = []
dense_tfnames = []
conv_kernel = []
conv_kernel_i = []
conv_kernel_j = []
conv_bias = []
conv_tfnames = []
conv_nodes = []
useConvLayer = False
### Loop over layers
for idx, obj in enumerate(keras_model.layers):
dobj = model['discriminator']['model']['config'][idx][
'config']
if type(obj) is Conv2D:
useConvLayer = True
conv_nodes.append(dobj['filters'])
conv_tfnames.append(str(dobj['activation']))
w, b = obj.get_weights()
for wn in w.T:
for wu in wn:
conv_kernel.extend(
wu.T.reshape(
(dobj['kernel_size'][0] *
dobj['kernel_size'][1], )).tolist())
conv_bias.extend(b.tolist())
conv_kernel_i.append(dobj['kernel_size'][0])
conv_kernel_j.append(dobj['kernel_size'][1])
elif type(obj) is Dense:
dense_nodes.append(dobj['units'])
dense_tfnames.append(str(dobj['activation']))
w, b = obj.get_weights()
dense_weights.extend(w.reshape(-1, order='F'))
dense_bias.extend(b.reshape(-1, order='F'))
# TODO: Need to implement something smart to tread this case
elif type(obj) is Activation:
dense_tfnames.pop()
dense_tfnames.append(str(dobj['activation']))
else:
continue
discrData['dense_nodes'] = tolist(dense_nodes)
discrData['dense_bias'] = tolist(dense_bias)
discrData['dense_weights'] = tolist(dense_weights)
discrData['dense_tfnames'] = tolist(dense_tfnames)
discrData['useConvLayer'] = [useConvLayer]
# Convolutional neural network
if useConvLayer:
discrData['conv_nodes'] = tolist(conv_nodes)
discrData['conv_tfnames'] = tolist(conv_tfnames)
discrData['conv_kernel_i'] = tolist(conv_kernel_i)
discrData['conv_kernel_j'] = tolist(conv_kernel_j)
discrData['conv_kernel'] = tolist(conv_kernel)
discrData['conv_bias'] = tolist(conv_bias)
discrData['conv_input_i'] = [
model['discriminator']['model']['config'][0]['config']
['batch_input_shape'][1]
]
discrData['conv_input_j'] = [
model['discriminator']['model']['config'][0]['config']
['batch_input_shape'][2]
]
i = discrData['conv_input_i'][0] - (sum(conv_kernel_i) -
len(conv_kernel_i))
j = discrData['conv_input_j'][0] - (sum(conv_kernel_j) -
len(conv_kernel_j))
input_layer = i * j * discrData['conv_nodes'][-1]
discrData['dense_nodes'] = [input_layer
] + discrData['dense_nodes']
### Attach
modelDict['tuning'].append({'discriminator': discrData})
if self._toPickle:
self._logger.info('Export weights to pickle format...')
modelDict['__version__'] = self._version
modelDict['__core__'] = TuningToolCores.keras
from RingerCore import save
save(modelDict, filename)
from ROOT import TFile, TTree
from ROOT import std
self._logger.info('Export weights to root format...')
### Create the discriminator root object
fdiscr = TFile(appendToFileName(filename, '.root', separator=''),
'recreate')
self.__createRootParameter('int', '__version__', self._version).Write()
self.__createRootParameter('int', '__core__',
TuningToolCores.FastNet).Write()
fdiscr.mkdir('tuning')
fdiscr.cd('tuning')
tdiscr = TTree('discriminators', '')
for idx, b in enumerate(self._discrBranches):
b[2] = std.vector(b[0])()
tdiscr.Branch(b[1], 'vector<%s>' % b[0], b[2])
for t in modelDict['tuning']:
for idx, b in enumerate(self._discrBranches):
self.__attachToVector(t['discriminator'][b[1]], b[2])
tdiscr.Fill()
tdiscr.Write()
### Create the thresholds root object
fdiscr.mkdir('metadata')
fdiscr.cd('metadata')
for key, value in modelDict['metadata'].iteritems():
self._logger.info('Saving metadata %s as %s', key, value)
self.__createRootParameter('int' if type(value) is int else 'bool',
key, value).Write()
fdiscr.Close()
)
c = monitoring(dirname=args.output, doBeamer=args.doBeamer)
if c['etBinIdx'] > etBinMax: etBinMax = c['etBinIdx']
if c['etaBinIdx'] > etaBinMax: etaBinMax = c['etaBinIdx']
csummaryList.append(c)
del monitoring
### list to binned grid
summary = [[None for _ in range(etaBinMax + 1)] for __ in range(etBinMax + 1)]
for c in csummaryList:
summary[c['etBinIdx']][c['etaBinIdx']] = c
print 'saving summary...'
from RingerCore import save
### Save this for backup
save(summary, args.output + '/summary')
print 'loading...'
### Built the final table
MonitoringTool.ttReport(args.output + '/summary.pic.gz',
args.dataPath,
outname=args.output + '/' + args.output,
title=args.output,
toPDF=False)
MonitoringTool.ttReport(args.output + '/summary.pic.gz',
args.dataPath,
outname=args.output + '/' + args.output,
title=args.output,
toPDF=True)
#MonitoringTool.ttReport( args.output+'_summary.pic.gz', args.dataPath, args.output, toPDF=True )
import os.path
cOutputName = os.path.join(
os.path.abspath(args.change_output_folder),
os.path.basename(cOutputName))
f = ROOT.TFile(inFile, 'r')
mainLogger.debug("Reading key: %s", args.treePath)
tree = f.Get(args.treePath)
if not isinstance(tree, ROOT.TTree):
mainLogger.error("Path %s does not contain a TTree object",
args.treePath)
continue
shortKey = args.treePath.split('/')[-1]
# TODO Save each numpy object key instead of the collection tree:
data = rnp.tree2array(tree,
branches=args.branches,
selection=args.selection)
toSave = {
key:
(data[key] if key != 'elCand2_ringer_rings' else np.concatenate(
data['elCand2_ringer_rings']).reshape(-1, 100))
for key in data.dtype.names
}
#toSave = { shortKey : rnp.tree2array( tree, branches=args.branches, selection=args.selection ) }
savedPath = save(toSave, cOutputName, protocol='savez_compressed')
mainLogger.info("Successfully created numpy file: %s", cOutputName)
else:
mainLogger.error("Cannot transform file '%s' to numpy format." %
inFile)
# end of (for fileCollection)
def changePassed(key, mat):
for lkey in d[key].item():
for rowIdx, array1 in enumerate( d[key].item()[lkey]):
for columnIdx, rawdict in enumerate(array1):
rawdict['_passed'] = mat[rowIdx][columnIdx]
def changeTotal(key, mat):
for lkey in d[key].item():
for rowIdx, array1 in enumerate( d[key].item()[lkey]):
for columnIdx, rawdict in enumerate(array1):
rawdict['_count'] = 100.
changePassed( key, mat )
changeTotal( key, mat )
from RingerCore import load, save
a = load('mc15_13TeV.361106.423300.sgn.trigegprobes.bkg.vetotruth.trig.l2calo.eg.std.grid.medium.npz')
d = dict(a)
changeOp( d, 'signal_efficiencies', mergeEffTable( veryloose20160701 ) )
changeOp( d, 'background_efficiencies', pfrefs_vloose )
save(d, 'mc15_13TeV.361106.423300.sgn.trigegprobes.bkg.vetotruth.trig.l2calo.eg.std.grid.veryloose', protocol = 'savez_compressed')
d = dict(a)
changeOp( d, 'signal_efficiencies', mergeEffTable( loose20160701 ) )
changeOp( d, 'background_efficiencies', pfrefs_loose )
save(d, 'mc15_13TeV.361106.423300.sgn.trigegprobes.bkg.vetotruth.trig.l2calo.eg.std.grid.loose', protocol = 'savez_compressed')
d = dict(a)
changeOp( d, 'signal_efficiencies', mergeEffTable( tight20160701 ) )
changeOp( d, 'background_efficiencies', pfrefs_tight )
save(d, 'mc15_13TeV.361106.423300.sgn.trigegprobes.bkg.vetotruth.trig.l2calo.eg.std.grid.tight', protocol = 'savez_compressed')
