def Run(self, processList):
len_histoList = len(self.t_histoList)
t_tagList = PythonUtils.makeTupleFromFile(processList, ',')
l_files = []
f = open(processList, 'r')
for file in f:
l_files.append(file.strip())
for i in xrange(len(t_tagList)):
pFile, tagText = t_tagList[i]
PythonUtils.doesFileExist(pFile)
t_processList = PythonUtils.makeTupleFromFile(pFile, ',')
for j in xrange(len(self.t_histoList)):
t_plot = []
name = self.t_histoList[j][2]
for k in xrange(len(t_processList)):
histoLocation = t_processList[k][1] + self.t_histoList[j][1]
t_plot.append([
ROOTUtils.retrieveHistogram(self.histFile,
histoLocation, name)
])
saveString = self.saveDir + self.t_histoList[j][
0] + '_' + t_processList[k][7]
PlottingUtils.stackHistograms(
t_plot, t_processList, self.t_histoList[j],
self.t_legendList[j], self.t_textList[j],
self.t_axisList[j], self.l_saveAs, self.t_ratioList[j],
saveString, tagText)
def Run(self):
##
## Check our files exist
##
l_files = [self.processList, self.typeList]
for file in l_files:
PythonUtils.doesFileExist(file)
##
## Make lists from the file contents
##
l_process = PythonUtils.makeListFromFile(self.processList)
l_type = PythonUtils.makeListFromFile(self.typeList)
l_bSub = []
count = 0
for process in l_process:
for type in l_type:
bMainCfg = self.createConfigs(self.processDir, self.typeDir, self.mainDir, self.nEvents,
self.outDir, self.samples, self.selectOn, self.selectBins,
count, process.strip(), type.strip())
bSub = self.createBatchSub(bMainCfg, self.cmdLocation, self.trunkLocation, count)
l_bSub.append(bSub)
count += 1
self.submission(l_bSub, self.queue, self.email)
def saveHistograms(self, d_filledHists, t_type, outFile, sample, baseDir,
selectOn, t_select):
PythonUtils.Info('++++++++++++++++++++++++++++++++++++++++++++++++++')
PythonUtils.Info(' SAVING HISTOGRAMS: ' + baseDir + ' ')
PythonUtils.Info('++++++++++++++++++++++++++++++++++++++++++++++++++')
for bin in t_select:
## Don't have a trainling slash for all - looks messy
PythonUtils.Info('Creating Histograms for process: ' + baseDir +
' in bin: ' + bin[0] + '-' + bin[1])
if bin[0] == 'all':
select = selectOn + '-' + bin[0]
else:
select = selectOn + '-' + bin[0] + '-' + bin[1]
for i in range(len(t_type)):
type, typeFile = t_type[i]
t_typeFile = PythonUtils.makeTupleFromFile(typeFile, ',')
for j in range(len(t_typeFile)):
rBranch, tBranch, sName, sLocation, xMin, xMax, nBuns = t_typeFile[
j]
location = baseDir + '/' + sample + '/' + select + '/' + sLocation
ROOTUtils.saveToFile(
d_filledHists[select][type][location + sName], outFile,
location, sName)
def Run(self):
##
## Check our files exist
##
l_files = [self.processList, self.typeList]
for file in l_files:
PythonUtils.doesFileExist(file)
##
## Make lists from the file contents
##
l_process = PythonUtils.makeListFromFile(self.processList)
l_type = PythonUtils.makeListFromFile(self.typeList)
l_bSub = []
count = 0
for process in l_process:
for type in l_type:
bMainCfg = self.createConfigs(self.processDir, self.typeDir,
self.mainDir, self.nEvents,
self.outDir, self.samples,
self.selectOn, self.selectBins,
count, process.strip(),
type.strip())
bSub = self.createBatchSub(bMainCfg, self.cmdLocation,
self.trunkLocation, count)
l_bSub.append(bSub)
count += 1
self.submission(l_bSub, self.queue, self.email)
def subtractHistograms(self, d_mjHist, d_dHist_f, d_ewHist_f, mjDir, dDir,
ewDir, sample, selectOn, t_selectBins, t_type):
PythonUtils.Info('++++++++++++++++++++++++++++++++++++++++++++++++++')
PythonUtils.Info(' SUBTRACTING HISTOGRAMS ')
PythonUtils.Info('++++++++++++++++++++++++++++++++++++++++++++++++++')
for bin in t_selectBins:
if bin[0] == 'all':
select = selectOn + '-' + bin[0]
else:
select = selectOn + '-' + bin[0] + '-' + bin[1]
for i in range(len(t_type)):
type, typeFile = t_type[i]
t_typeFile = PythonUtils.makeTupleFromFile(typeFile, ',')
for j in range(len(t_typeFile)):
rBranch, tBranch, sName, sLocation, xMin, xMax, nBuns = t_typeFile[
j]
mjLocation = mjDir + '/' + sample + '/' + select + '/' + sLocation
ewLocation = ewDir + '/' + sample + '/' + select + '/' + sLocation
dLocation = dDir + '/' + sample + '/' + select + '/' + sLocation
d_mjHist[select][type][mjLocation +
sName] = d_dHist_f[select][type][
dLocation + sName].Clone()
d_mjHist[select][type][mjLocation + sName].Add(
d_ewHist_f[select][type][ewLocation + sName], -1)
d_mjHist[select][type][
mjLocation + sName] = self.removeNegBins(
d_mjHist[select][type][mjLocation + sName])
return d_mjHist
def Run(self):
## Create lists from configFile info
l_saveAs = PythonUtils.makeListFromString(self.saveAs, ',')
## Check the .list and tree files exist
l_files = [self.treeFile, self.axisList, self.histoList,
self.scatterList, self.textList, self.fileList]
for file in l_files:
PythonUtils.doesFileExist(file)
## Make the tuples from the list files
t_axisList = PythonUtils.makeTupleFromFile(self.axisList, ',')
t_fileList = PythonUtils.makeTupleFromFile(self.fileList, ',')
t_histoList = PythonUtils.makeTupleFromFile(self.histoList, ',')
t_scatterList = PythonUtils.makeTupleFromFile(self.scatterList, ',')
t_textList = PythonUtils.makeTupleFromFile(self.textList, ',')
## Match the first item in the tuples
l_tuples = [t_fileList, t_histoList, t_scatterList, t_textList]
for item in l_tuples:
PythonUtils.firstItemMatching(t_axisList, item)
for i in xrange(len(t_scatterList)):
PlottingUtils.scatterPlot(t_scatterList[i], t_fileList[i], t_histoList[i],
t_axisList[i], t_textList[i], self.treeFile,
self.saveDir, l_saveAs)
def Run(self, processList):
len_histoList = len(self.t_histoList)
t_tagList = PythonUtils.makeTupleFromFile(processList, ',')
l_files = []
f = open(processList, 'r')
for file in f:
l_files.append(file.strip())
for i in xrange(len(t_tagList)):
pFile, tagText = t_tagList[i]
PythonUtils.doesFileExist(pFile)
t_processList = PythonUtils.makeTupleFromFile(pFile, ',')
for j in xrange(len(self.t_histoList)):
t_plot = []
name = self.t_histoList[j][2]
for k in xrange(len(t_processList)):
histoLocation = t_processList[k][1] + self.t_histoList[j][1]
t_plot.append([ROOTUtils.retrieveHistogram(self.histFile, histoLocation, name)])
saveString = self.saveDir + self.t_histoList[j][0] + '_' + t_processList[k][7]
PlottingUtils.stackHistograms(t_plot, t_processList, self.t_histoList[j],
self.t_legendList[j], self.t_textList[j],
self.t_axisList[j], self.l_saveAs,
self.t_ratioList[j], saveString, tagText)
def fillHistograms(self, d_histType, t_var, tFile, tName, sDir, t_muon,
t_select, selectOn, nEvents):
PythonUtils.Info('Filling Histograms: ' + tFile)
f = TFile(tFile, 'READ')
t = f.Get(tName)
for i, entry in enumerate(t):
## only run over nEvents
if nEvents != -1:
if i == nEvents:
break
for sMu in t_muon:
b_mu = 'nMu_' + sMu[0] + sMu[1]
for bin in t_select:
if bin[1]:
select = selectOn + '-' + bin[0] + '-' + bin[1]
else:
select = selectOn + '-' + bin[0]
for j in xrange(len(t_var)):
rBranch, tBranch, sName, sLoc, xMin, xMax, nBins = t_var[
j]
bSelect = ''
if selectOn == 'TRUTH':
bSelect = tBranch
elif selectOn == 'RECO':
bSelect = rBranch
else:
PythonUtils.Error('Option Not available')
if sMu[0] == 'all' or PythonUtils.getSelection(
entry, sMu[0], sMu[1], 'hasMuon'):
if bin[0] == 'all' or PythonUtils.getSelection(
entry, bin[0], bin[1], bSelect):
location = sDir + '/' + b_mu + '/' + select + '/' + sLoc
rValue = getattr(entry, rBranch)
tValue = getattr(entry, tBranch)
if 'cw' in location:
d_histType[b_mu][select][
location + sName].Fill(
tValue / rValue, entry.eventWeight)
elif 'bw' in location:
d_histType[b_mu][select][
location + sName].Fill(
rValue / tValue, entry.eventWeight)
elif 'vw' in location:
d_histType[b_mu][select][
location + sName].Fill(
rValue, entry.eventWeight)
else:
PythonUtils.Error(
'Histogram Type not recognised!')
return d_histType
def bytes_into_table(table, data_bytes):
global read_data_frame, read_types, read_serializers
path = data_bytes.decode('utf-8')
try:
if read_data_frame is None:
deserialize_data_frame(path)
table._data_frame = read_data_frame
read_data_frame = None
read_types = []
read_serializers = {}
finally:
PythonUtils.invoke_safely(None, os.remove, [path])
def Run(self):
## Create the lists from the config file
l_samples = PythonUtils.makeListFromString(self.samples, ',')
t_select = PythonUtils.makeTupleFromString(self.selectBins, ',', '?')
## Check .list Files exist
l_lFiles = [self.inputList, self.typeList]
for file in l_lFiles:
PythonUtils.doesFileExist(file)
## Make the tuples from the list files
t_input = PythonUtils.makeTupleFromFile(self.inputList, ',')
t_type = PythonUtils.makeTupleFromFile(self.typeList, ',')
for i in range(len(t_input)):
iFile, treeName, baseDir = t_input[i]
PythonUtils.doesFileExist(iFile)
for sample in l_samples:
d_histType = self.createHistograms(t_type, sample, baseDir,
self.selectOn, t_select)
d_filledHists = self.fillHistograms(d_histType, t_type, iFile,
treeName, sample, baseDir,
t_select, self.selectOn,
self.nEvents)
self.saveHistograms(d_filledHists, t_type, self.outFile,
sample, baseDir, self.selectOn, t_select)
def subtractHistograms(self, d_mjHist, d_dHist_f, d_ewHist_f, mjDir, dDir,
ewDir, sample, selectOn, t_selectBins, t_type):
PythonUtils.Info('++++++++++++++++++++++++++++++++++++++++++++++++++')
PythonUtils.Info(' SUBTRACTING HISTOGRAMS ')
PythonUtils.Info('++++++++++++++++++++++++++++++++++++++++++++++++++')
for bin in t_selectBins:
if bin[0] == 'all':
select = selectOn + '-' + bin[0]
else:
select = selectOn + '-' + bin[0] + '-' + bin[1]
for i in range(len(t_type)):
type, typeFile = t_type[i]
t_typeFile = PythonUtils.makeTupleFromFile(typeFile, ',')
for j in range(len(t_typeFile)):
rBranch, tBranch, sName, sLocation, xMin, xMax, nBuns = t_typeFile[j]
mjLocation = mjDir + '/' + sample + '/' + select + '/' + sLocation
ewLocation = ewDir + '/' + sample + '/' + select + '/' + sLocation
dLocation = dDir + '/' + sample + '/' + select + '/' + sLocation
d_mjHist[select][type][mjLocation + sName] = d_dHist_f[select][type][dLocation + sName].Clone()
d_mjHist[select][type][mjLocation + sName].Add(d_ewHist_f[select][type][ewLocation + sName], -1)
d_mjHist[select][type][mjLocation + sName] = self.removeNegBins(d_mjHist[select][type][mjLocation + sName])
return d_mjHist
def main():
use = '''
Example:
python PYTHON/MODULES/runProfile.py --MainConfigFile=ConfigFiles/Profile/MainConfigFile/MainConfigFile.cfg --doProfile=0 --createFile=0
'''
parser = OptionParser(usage=use)
parser.add_option('--MainConfigFile',
dest='MainConfigFile',
default='',
type='string',
help='Main config file for creating profile plots')
parser.add_option('--doProfile',
dest='doProfile',
default=0,
type='int',
help='Create the 2D profile plots ')
parser.add_option('--createFile',
dest='createFile',
default=0,
type='int',
help='Create the histogram for the pT bins')
options, args = parser.parse_args()
if options.MainConfigFile:
createProfile = CreateProfile(options.MainConfigFile,
options.doProfile)
createProfile.Run(options.doProfile, options.createFile)
else:
PythonUtils.Error('Configuration File NOT Found!!')
print use
sys.exit(1)
def createHistograms(self, t_var, sDir, tFile, t_muon, selectOn, t_select):
PythonUtils.Info('Creating Histograms: ' + sDir + ' For: ' + tFile)
d_hMuBinType = {}
for sMu in t_muon:
b_mu = 'nMu_' + sMu[0] + sMu[1]
d_hBinType = {}
for bin in t_select:
if bin[1]:
select = selectOn + '-' + bin[0] + '-' + bin[1]
else:
select = selectOn + '-' + bin[0]
d_hType = {}
for i in xrange(len(t_var)):
rBranch, tBranch, sName, sLoc, xMin, xMax, nBins = t_var[i]
hName = 'h_' + sDir.replace(
'/',
'_') + '_' + selectOn + '_' + bin[0] + '_' + rBranch
location = sDir + '/' + b_mu + '/' + select + '/' + sLoc
d_hType[location + sName] = TH1F(hName, hName, int(nBins),
float(xMin), float(xMax))
d_hType[location + sName].SetDirectory(0)
d_hBinType[select] = d_hType
d_hMuBinType[b_mu] = d_hBinType
return d_hMuBinType
def test():
X = pu.readDataFromTxt('testSet.txt')
num = np.size(X)
X = np.reshape(X, (num / 2, 2))
ppx = gmm(X, 4)
index = np.argmax(ppx, axis=1)
plt.figure()
plt.scatter(X[index == 0][:, 0],
X[index == 0][:, 1],
s=60,
c=u'r',
marker=u'o')
plt.scatter(X[index == 1][:, 0],
X[index == 1][:, 1],
s=60,
c=u'b',
marker=u'o')
plt.scatter(X[index == 2][:, 0],
X[index == 2][:, 1],
s=60,
c=u'y',
marker=u'o')
plt.scatter(X[index == 3][:, 0],
X[index == 3][:, 1],
s=60,
c=u'g',
marker=u'o')
def gmm(X, K):
threshold = 1e-15
N, D = np.shape(X)
randV = pu.randIntList(1, N, K)
centroids = X[randV]
pMiu, pPi, pSigma = inti_params(centroids, K, X, N, D)
Lprev = -np.inf
while True:
'Estiamtion Step'
Px = calc_prop(X, N, K, pMiu, pSigma, threshold, D)
pGamma = Px * np.tile(pPi, (N, 1))
pGamma = pGamma / np.tile((np.sum(pGamma, axis=1)), (K, 1)).T
'Maximization Step'
Nk = np.sum(pGamma, axis=0)
pMiu = np.dot(np.dot(np.diag(1 / Nk), pGamma.T), X)
pPi = Nk / N
for kk in range(K):
Xshift = X - np.tile(pMiu[kk], (N, 1))
pSigma[:, :, kk] = (np.dot(
np.dot(Xshift.T, np.diag(pGamma[:, kk])), Xshift)) / Nk[kk]
'check for convergence'
L = np.sum(np.log(np.dot(Px, pPi.T)))
if L - Lprev < threshold:
break
Lprev = L
return Px
def gmm(X, K):
threshold = 1e-15
N, D = np.shape(X)
randV = pu.randIntList(1, N, K)
centroids = X[randV]
pMiu, pPi, pSigma = inti_params(centroids, K, X, N, D)
Lprev = -np.inf
while True:
"Estiamtion Step"
Px = calc_prop(X, N, K, pMiu, pSigma, threshold, D)
pGamma = Px * np.tile(pPi, (N, 1))
pGamma = pGamma / np.tile((np.sum(pGamma, axis=1)), (K, 1)).T
"Maximization Step"
Nk = np.sum(pGamma, axis=0)
pMiu = np.dot(np.dot(np.diag(1 / Nk), pGamma.T), X)
pPi = Nk / N
for kk in range(K):
Xshift = X - np.tile(pMiu[kk], (N, 1))
pSigma[:, :, kk] = (np.dot(np.dot(Xshift.T, np.diag(pGamma[:, kk])), Xshift)) / Nk[kk]
"check for convergence"
L = np.sum(np.log(np.dot(Px, pPi.T)))
if L - Lprev < threshold:
break
Lprev = L
return Px
def overlayProfile(l_prof, l_axis, l_histo, l_legend, l_text, l_legInfo, saveAs, saveString):
aString, xLabel, yLabel, xOffset, yOffset, aLabSize, aTextSize = l_axis
hString, c_x, c_y, yMin, yMax, hBins, hFile = l_histo
lString, l_x1, l_y1, l_x2, l_y2, l_textSize = l_legend
t_str, plotText, t_textSize, t_alignment, t_start, t_gap, t_x = l_text
l_bins = PythonUtils.makeListFromString(hBins, ';')
gROOT.SetBatch(1)
c = TCanvas('c', 'c', int(c_x), int(c_y))
gStyle.SetOptStat(0)
c.SetTicks(1, 1)
for i in xrange(len(l_prof)):
if i == 0:
l_prof[i].Draw('eps')
setupAxes(l_prof[i], xLabel, yLabel, xOffset, yOffset,
aTextSize, l_bins[0], l_bins[len(l_bins) -1], 1)
l_prof[i].SetMinimum(float(yMin))
l_prof[i].SetMaximum(float(yMax))
l_prof[i].SetTitle('')
hLine = TLine(l_prof[i].GetXaxis().GetXmin(), 1, l_prof[i].GetXaxis().GetXmax(), 1)
hLine.SetLineWidth(2)
hLine.SetLineColor(1)
hLine.SetLineStyle(2)
hLine.Draw('same')
else:
l_prof[i].Draw('eps same')
setupTextOnPlot(plotText, t_textSize, t_alignment, t_x, t_start, t_gap)
setupLegend(l_legInfo, l_x1, l_y1, l_x2, l_y2, l_textSize)
savePlots(c, saveString, saveAs)
def Run(self):
## Create the lists from configFiles
l_tags = PythonUtils.makeListFromString(self.tagList, ',')
if not self.splitTag:
haddCmd = 'hadd -f ' + self.outputDir + self.fileIdent + '_' + self.tagList + '0_EW_MJ_0.root '
for root, dir, files in os.walk(self.inputDir):
for file in files:
if 'MJ' in file and 'Data' not in file and '.root' in file:
if 'llbb' not in file and 'lv' not in file and 'vv' not in file:
print file
if self.emptyTree(self.inputDir + file, self.tree):
print self.inputDir + file
haddCmd += self.inputDir + file + ' '
print haddCmd
os.system(haddCmd)
else:
## loop over the tags and hadd the files
for tag in l_tags:
haddCmd = 'hadd -f ' + self.outputDir + self.fileIdent + '_' + tag + '0_EW_MJ_0.root '
for root, dir, files in os.walk(self.inputDir):
for file in files:
if tag in file and self.fileIdent in file and 'MJ' in file and 'Data' not in file:
if self.emptyTree(self.outputDir + file, self.tree):
print self.outputDir + file
haddCmd += self.outputDir + file + ' '
print ''
os.system(haddCmd)
def getResult(initialNumber, randomNumberVariations, neuronActivationFunctions,
learningMethods, inputLayers, outputLayers, hiddenLayers,
epochDict, useDefaults):
'''
Get the result to parse back
'''
l_NN = []
counter = initialNumber - 1
for function in neuronActivationFunctions:
for method in learningMethods:
for iLayer in inputLayers:
for oLayer in outputLayers:
for i in xrange(randomNumberVariations):
counter += 1
d_treeNN = {}
for tree in epochDict:
NN = [
'NN', 'n' + str(counter), tree, function,
method, epochDict[tree],
PythonUtils.updateListVars('r', iLayer, ''),
hiddenLayers, oLayer, useDefaults
]
d_treeNN[tree] = NN
l_NN.append(d_treeNN)
return l_NN
def overlayProfile(l_plot, l_legend, l_histo, l_legendInfo, l_text, l_axis, sName, l_saveAs, tAdd):
## Destruct the lists
a_str, xLabel, yLabel, xOffset, yOffset, a_labelSize, a_textSize = l_axis
l_str, l_x1, l_y1, l_x2, l_y2, l_textSize = l_legendInfo
t_str, plotText, t_textSize, t_alignment, t_start, t_gap, t_x = l_text
h_str, tFile, c_x, c_y, xMin, xMax, yMin, yMax, nBins = l_histo
## Add the plot specific text to the other text
plotText += tAdd
PythonUtils.Info(' OVERLAYING PROFILES: ' + h_str + ' ')
## Setup the Canvas
gROOT.SetBatch(1)
c = TCanvas('c', 'c', int(c_x), int(c_y))
c.SetTicks(1, 1)
c.SetGrid(1, 1)
c.SetLogx(1)
gStyle.SetOptStat(0)
for i in xrange(len(l_plot)):
if i == 0:
l_plot[i].Draw()
setupAxes(l_plot[i], xLabel, yLabel, xOffset, yOffset,
a_textSize, xMin, xMax, 1)
l_plot[i].GetXaxis().SetRange(20, 1200)
else:
l_plot[i].Draw('SAME eps')
setupLegend(l_legend, l_x1, l_y1, l_x2, l_y2, l_textSize)
setupTextOnPlot(plotText, t_textSize, t_alignment, t_x, t_start, t_gap)
savePlots(c, sName, l_saveAs)
def fillProfile(d_profile, t_corrList, tName, l_histo, t_select):
t_profile = []
t_legend = []
h_str, tFile, c_x, x_y, xMin, xMax, yMin, yMax, nBins = l_histo
PythonUtils.Info(' FILLING HISTOGRAMS: ' + h_str + ' ')
f = TFile(tFile, 'READ')
t = f.Get(tName)
for i, entry in enumerate(t):
if i%10000 == 0:
PythonUtils.Info('Filling Event Number: ' + str(i))
l_bin = []
for bin in t_select:
for i in range(len(t_corrList)):
cName, xBranch, yBranch, sBranch, cColour, cMarker, mSize = t_corrList[i]
if bin[0] == 'all' or PythonUtils.getSelection(entry, bin[0], bin[1], sBranch):
dName = h_str + '_' + bin[0] + '_' + bin[1] + '_' + t_corrList[i][0]
xValue = getattr(entry, xBranch)
yValue = getattr(entry, yBranch)
d_profile[bin[0] + bin[1]][dName].Fill(xValue, yValue)
for bin in t_select:
l_bin = []
l_legend = []
for i in range(len(t_corrList)):
cName, xBranch, yBranch, sBranch, cColour, cMarker, mSize = t_corrList[i]
dName = h_str + '_' + bin[0] + '_' + bin[1] + '_' + t_corrList[i][0]
d_profile[bin[0] + bin[1]][dName].GetYaxis().SetRangeUser(float(yMin), float(yMax))
d_profile[bin[0] + bin[1]][dName].GetXaxis().SetRangeUser(float(xMin), float(xMax))
d_profile[bin[0] + bin[1]][dName].SetMarkerStyle(int(cMarker))
d_profile[bin[0] + bin[1]][dName].SetMarkerColor(int(cColour))
d_profile[bin[0] + bin[1]][dName].SetLineColor(int(cColour))
d_profile[bin[0] + bin[1]][dName].SetTitle('')
if int(mSize) != -1:
d_profile[bin[0] + bin[1]][dName].SetMarkerSize(int(mSize))
l_legend.append([d_profile[bin[0] + bin[1]][dName], cName, 'p'])
l_bin.append(d_profile[bin[0] + bin[1]][dName])
t_legend.append(l_legend)
t_profile.append(l_bin)
return t_profile, t_legend
def createProfile(t_corrList, t_select, l_histoList, nBins, dBins):
d_profileBin = {}
h_str, tFile, c_x, c_y, xMin, xMax, yMin, yMax, varBins = l_histoList
PythonUtils.Info(' CREATING PROFILES: ' + h_str + ' ')
for bin in t_select:
PythonUtils.Info('Creating Profiles: ' + h_str + ' in bin: ' + bin[0] +
' -> ' + bin[1])
d_profile = {}
for i in range(len(t_corrList)):
dName = h_str + '_' + bin[0] + '_' + bin[1] + '_' + t_corrList[i][0]
d_profile[dName] = TProfile(dName, dName, int(nBins), dBins,
float(yMin), float(yMax))
d_profileBin[bin[0] + bin[1]] = d_profile
return d_profileBin
def Run(self):
## Make lists from the config file options
l_samples = PythonUtils.makeListFromString(self.samples, ',')
t_selectBins = PythonUtils.makeTupleFromString(self.selectBins, ',',
'?')
## Check the existence of the .list files
l_listFiles = [self.inputList, self.typeList]
for file in l_listFiles:
PythonUtils.doesFileExist(file)
## Make the tuples from these files
t_input = PythonUtils.makeTupleFromFile(self.inputList, ',')
t_type = PythonUtils.makeTupleFromFile(self.typeList, ',')
## Loop over the input files to create a histogram file
for i in xrange(len(t_input)):
dFile, ewFile, treeName, mjDir, dDir, ewDir = t_input[i]
l_inFiles = [dFile, ewFile]
for file in l_inFiles:
PythonUtils.doesFileExist(file)
for sample in l_samples:
## Create the histograms
d_ewHist = self.createHistograms(t_type, sample, ewDir,
self.selectOn, t_selectBins)
d_dHist = self.createHistograms(t_type, sample, dDir,
self.selectOn, t_selectBins)
d_mjHist = self.createHistograms(t_type, sample, mjDir,
self.selectOn, t_selectBins)
## Fill the histograms
d_ewHist_f = self.fillHistograms(d_ewHist, t_type, ewFile,
treeName, sample, ewDir,
t_selectBins, self.selectOn,
self.nEvents)
d_dHist_f = self.fillHistograms(d_dHist, t_type, dFile,
treeName, sample, dDir,
t_selectBins, self.selectOn,
self.nEvents)
## Subtract the histograms
d_mjHist_f = self.subtractHistograms(d_mjHist, d_dHist_f,
d_ewHist_f, mjDir, dDir,
ewDir, sample,
self.selectOn,
t_selectBins, t_type)
## save the histograms
self.saveHistograms(d_mjHist_f, t_type, self.outputFile,
sample, mjDir, self.selectOn, t_selectBins)
self.saveHistograms(d_ewHist_f, t_type, self.outputFile,
sample, ewDir, self.selectOn, t_selectBins)
self.saveHistograms(d_dHist_f, t_type, self.outputFile, sample,
dDir, self.selectOn, t_selectBins)
def fillHistograms(self, d_histType, t_var, tFile, tName, sDir, t_muon, t_select, selectOn, nEvents):
PythonUtils.Info('Filling Histograms: ' + tFile)
f = TFile(tFile, 'READ')
t = f.Get(tName)
for i, entry in enumerate(t):
## only run over nEvents
if nEvents != -1:
if i == nEvents:
break
for sMu in t_muon:
b_mu = 'nMu_' + sMu[0] + sMu[1]
for bin in t_select:
if bin[1]:
select = selectOn + '-' + bin[0] + '-' + bin[1]
else:
select = selectOn + '-' + bin[0]
for j in xrange(len(t_var)):
rBranch, tBranch, sName, sLoc, xMin, xMax, nBins = t_var[j]
bSelect = ''
if selectOn == 'TRUTH':
bSelect = tBranch
elif selectOn == 'RECO':
bSelect = rBranch
else:
PythonUtils.Error('Option Not available')
if sMu[0] == 'all' or PythonUtils.getSelection(entry, sMu[0], sMu[1], 'hasMuon'):
if bin[0] == 'all' or PythonUtils.getSelection(entry, bin[0], bin[1], bSelect):
location = sDir + '/' + b_mu + '/' + select + '/' + sLoc
rValue = getattr(entry, rBranch)
tValue = getattr(entry, tBranch)
if 'cw' in location:
d_histType[b_mu][select][location + sName].Fill(tValue/rValue, entry.eventWeight)
elif 'bw' in location:
d_histType[b_mu][select][location + sName].Fill(rValue/tValue, entry.eventWeight)
elif 'vw' in location:
d_histType[b_mu][select][location + sName].Fill(rValue, entry.eventWeight)
else:
PythonUtils.Error('Histogram Type not recognised!')
return d_histType
def setupTextOnPlot(plotText, t_textSize, t_alignment, t_x, t_start, t_gap):
l_plotText = PythonUtils.makeListFromString(plotText, '?')
t = TLatex()
t.SetNDC()
t.SetTextSize(float(t_textSize))
t.SetTextAlign(int(t_alignment))
for item in l_plotText:
t.DrawLatex(float(t_x), float(t_start), item)
t_start = float(t_start) - float(t_gap)
def getListNN(file):
'''
Get NN List
'''
fileList = open(file, 'r')
result = []
for f in fileList:
NNOptionParser = ConfigParser.SafeConfigParser()
NNOptionParser.read(f.strip())
# NNOptions
initialNumber = NNOptionParser.getint('NNOptions', 'initialNumber')
randomNumberVariations = NNOptionParser.getint(
'NNOptions', 'randomNumberVariations')
neuronActivationFunctions = NNOptionParser.get(
'NNOptions', 'neuronActivationFunctions')
learningMethods = NNOptionParser.get('NNOptions', 'learningMethods')
inputLayers = NNOptionParser.get('NNOptions', 'inputLayers')
hiddenLayers = NNOptionParser.get('NNOptions', 'hiddenLayers')
outputLayers = NNOptionParser.get('NNOptions', 'outputLayers')
epochInfo = NNOptionParser.get('NNOptions', 'epochInfo')
useDefaults = NNOptionParser.get('NNOptions', 'useDefaults')
# GeneralOptions
doInputCombinations = NNOptionParser.getint('GeneralOptions',
'doInputCombinations')
inputComboLength = NNOptionParser.getint('GeneralOptions',
'inputComboLength')
doOutputCombinations = NNOptionParser.getint('GeneralOptions',
'doOutputCombinations')
outputComboLength = NNOptionParser.getint('GeneralOptions',
'outputComboLength')
# Create the lists and dicts for training
activationFunctionList = PythonUtils.makeListFromString(
neuronActivationFunctions, ',')
learningMethodList = PythonUtils.makeListFromString(
learningMethods, ',')
inputLayerList = PythonUtils.makeListFromString(inputLayers, ',')
outputLayerList = PythonUtils.makeListFromString(outputLayers, ',')
epochDict = PythonUtils.makeDictFromString(epochInfo, ':', ',')
result = result + getResult(initialNumber, randomNumberVariations,
activationFunctionList, learningMethodList,
inputLayersList, outputLayersList,
hiddenLayers, epochDict, useDefaults)
def test():
X = pu.readDataFromTxt("testSet.txt")
num = np.size(X)
X = np.reshape(X, (num / 2, 2))
ppx = gmm(X, 4)
index = np.argmax(ppx, axis=1)
plt.figure()
plt.scatter(X[index == 0][:, 0], X[index == 0][:, 1], s=60, c=u"r", marker=u"o")
plt.scatter(X[index == 1][:, 0], X[index == 1][:, 1], s=60, c=u"b", marker=u"o")
plt.scatter(X[index == 2][:, 0], X[index == 2][:, 1], s=60, c=u"y", marker=u"o")
plt.scatter(X[index == 3][:, 0], X[index == 3][:, 1], s=60, c=u"g", marker=u"o")
def Run(self):
## Make lists from the config file options
l_samples = PythonUtils.makeListFromString(self.samples, ',')
t_selectBins = PythonUtils.makeTupleFromString(self.selectBins, ',', '?')
## Check the existence of the .list files
l_listFiles = [self.inputList, self.typeList]
for file in l_listFiles:
PythonUtils.doesFileExist(file)
## Make the tuples from these files
t_input = PythonUtils.makeTupleFromFile(self.inputList, ',')
t_type = PythonUtils.makeTupleFromFile(self.typeList, ',')
## Loop over the input files to create a histogram file
for i in xrange(len(t_input)):
dFile, ewFile, treeName, mjDir, dDir, ewDir = t_input[i]
l_inFiles = [dFile, ewFile]
for file in l_inFiles:
PythonUtils.doesFileExist(file)
for sample in l_samples:
## Create the histograms
d_ewHist = self.createHistograms(t_type, sample, ewDir,
self.selectOn, t_selectBins)
d_dHist = self.createHistograms(t_type, sample, dDir,
self.selectOn, t_selectBins)
d_mjHist = self.createHistograms(t_type, sample, mjDir,
self.selectOn, t_selectBins)
## Fill the histograms
d_ewHist_f = self.fillHistograms(d_ewHist, t_type, ewFile,
treeName, sample, ewDir,
t_selectBins, self.selectOn, self.nEvents)
d_dHist_f = self.fillHistograms(d_dHist, t_type, dFile,
treeName, sample, dDir,
t_selectBins, self.selectOn, self.nEvents )
## Subtract the histograms
d_mjHist_f = self.subtractHistograms(d_mjHist, d_dHist_f, d_ewHist_f,
mjDir, dDir, ewDir, sample,
self.selectOn, t_selectBins, t_type)
## save the histograms
self.saveHistograms(d_mjHist_f, t_type, self.outputFile, sample,
mjDir, self.selectOn, t_selectBins)
self.saveHistograms(d_ewHist_f, t_type, self.outputFile, sample,
ewDir, self.selectOn, t_selectBins)
self.saveHistograms(d_dHist_f, t_type, self.outputFile, sample,
dDir, self.selectOn, t_selectBins)
def GetCore(self, fastq):
"""
Build the Pbs bash core for this specimen
:param fastq:
:return:
"""
s = "FASTADIR=\"{0}\"\n".format(self.cqfasta_out)
s += "FASTQDIR=\"{0}\"\n".format(self.cqfastq_dir)
s += "SPADESDIR=\"{0}{1}/\"\n".format(
self.cqspades_out, PythonUtils.GetFastqPrefix(fastq))
s += "p1_1=\"{0}\"\n".format(fastq)
s += "p1_2=\"{0}\"\n\n".format(re.sub('_R1_', '_R2_', fastq))
s += "cd {0}\n".format(self.cqspades)
s += "./spades.py -k {0} --pe1 {1} --pe2 {2} --careful -o {3}\n\n".format(
",".join(self.kmer), "$FASTQDIR$p1_1", "$FASTQDIR$p1_2",
"$SPADESDIR")
s += "cp $SPADESDIR\"contigs.fasta\" $FASTADIR\"{0}.fasta\"\n\n".format(
PythonUtils.GetFastqPrefix(fastq))
return s
def _respond(self, request, response_message_id, workspace):
name = PayloadDecoder(request.payload).get_next_string()
data_object = workspace.get_variable(name)
o_bytes = bytearray(pickle.dumps(data_object))
o_type = type(data_object).__name__
o_representation = PythonUtils.object_to_string(data_object)
data_frame = pandas.DataFrame([{'bytes': o_bytes, 'type': o_type, 'representation': o_representation}])
data_bytes = workspace.serializer.data_frame_to_bytes(data_frame)
return AbstractRequestHandler._create_response(request, response_message_id,
response_payload=_create_byte_array_payload(data_bytes))
def fillHistograms( self, d_histType, t_type, iFile, treeName,
sample, baseDir, t_select, selectOn, nEvents ):
PythonUtils.Info('++++++++++++++++++++++++++++++++++++++++++++++++++')
PythonUtils.Info(' FILLING HISTOGRAMS: ' + baseDir + ' ')
PythonUtils.Info('++++++++++++++++++++++++++++++++++++++++++++++++++')
f = TFile(iFile, 'READ')
t = f.Get(treeName)
for i, entry in enumerate(t):
## Test for the test, train or all samples
if not PythonUtils.keepEntry(sample, i):
continue
## Run over the number of events: -1 is all
if nEvents != -1:
if i == nEvents:
break
## Loop over the binning
for bin in t_select:
## Don't have a trailing slash for all - looks messy
if bin[0] == 'all':
select = selectOn + '-' + bin[0]
else:
select = selectOn + '-' + bin[0] + '-' + bin[1]
if bin[0] == 'all' or PythonUtils.getSelection(entry, bin[0], bin[1], selectOn):
for j in range(len(t_type)):
type, typeFile = t_type[j]
t_typeFile = PythonUtils.makeTupleFromFile(typeFile, ',')
for k in range(len(t_typeFile)):
rBranch, tBranch, sName, sLocation, xMin, xMax, nBuns = t_typeFile[k]
location = baseDir + '/' + sample + '/' + select + '/' + sLocation
## Setup how the values are filled for different types
d_histBinTypeF = self.doFill(d_histType, entry, rBranch, tBranch,
select, type, location, sName)
return d_histBinTypeF
def writeLatex(self, t_mRr, l_corrName, pTVBin, texFile, lDir, gMean, gRMS, gRes):
## We want the corrections to appear in order
l_corrName.sort()
texName = lDir + texFile + '.tex'
# Write the table for the mean, rms and res for the
# different jet corrections
f = open(texName, 'w')
f.write('\documentclass[11pt, landscape]{aastex} \n')
f.write('\\begin{document}\n')
f.write('\\begin{center}\n')
f.write('\\begin{tabular}{r')
if gMean:
f.write('|c')
if gRMS:
f.write('|c')
if gRes:
f.write('|c')
f.write('} \n')
f.write('Correction ')
if gMean:
f.write('& Mean ')
if gRMS:
f.write('& RMS ')
if gRes:
f.write('& Resolution ')
f.write('\\\ \hline\hline \n')
for corr in l_corrName:
f.write(corr)
for i in xrange(len(t_mRr)):
for j in xrange(len(t_mRr[i])):
t_sort = tuple(sorted(t_mRr[i][j], key = lambda item: item[0]))
for k in xrange(len(t_sort)):
if t_sort[k][0] == corr:
f.write(' & ' + str(round(t_sort[k][1], 5)) + ' $\pm$ ' + str(round(t_sort[k][2], 5)))
f.write(' \\\ \n')
f.write('\hline\n')
f.write('\end{tabular} \n')
f.write('\end{center}\n')
f.write('\end{document}\n')
f.close()
PythonUtils.Info('FILE: ' + texName + ' Written')
return texFile
def createHistograms(self, t_type, sample, baseDir, selectOn, t_select):
PythonUtils.Info('++++++++++++++++++++++++++++++++++++++++++++++++++')
PythonUtils.Info(' CREATING HISTOGRAMS: ' + baseDir + ' ')
PythonUtils.Info('++++++++++++++++++++++++++++++++++++++++++++++++++')
d_histBinType = {}
for bin in t_select:
## Don't have a trailing slash for all - looks messy
PythonUtils.Info('Creating Histograms for process: ' + baseDir +
' in bin: ' + bin[0] + '-' + bin[1])
if bin[0] == 'all':
select = selectOn + '-' + bin[0]
else:
select = selectOn + '-' + bin[0] + '-' + bin[1]
d_histType = {}
for i in range(len(t_type)):
## Decalare the list variables
type, typeFile = t_type[i]
## Open typeFile and loop through it
PythonUtils.doesFileExist(typeFile)
t_typeFile = PythonUtils.makeTupleFromFile(typeFile, ',')
d_hist = {}
for j in range(len(t_typeFile)):
rBranch, tBranch, sName, sLocation, xMin, xMax, nBins = t_typeFile[j]
hName = 'h_' + baseDir + '_' + sample + '_' + selectOn + bin[0] + '_' + type + '_' + rBranch
location = baseDir + '/' + sample + '/' + select + '/' + sLocation
if 'crea' not in type:
d_hist[location + sName] = TH1F(hName, hName, int(nBins), float(xMin), float(xMax))
d_hist[location + sName].SetDirectory(0)
else:
d_hist[location + sName] = TH1D(hName, hName, int(nBins), float(xMin), float(xMax))
d_hist[location + sName].SetDirectory(0)
d_histType[type] = d_hist
d_histBinType[select] = d_histType
return d_histBinType
def saveHistograms(self, d_filledHists, t_var, outFile, sDir, t_muon,
selectOn, t_select):
PythonUtils.Info('Saving Histograms: ' + outFile)
for sMu in t_muon:
b_mu = 'nMu_' + sMu[0] + sMu[1]
for bin in t_select:
if bin[1]:
select = selectOn + '-' + bin[0] + '-' + bin[1]
else:
select = selectOn + '-' + bin[0]
PythonUtils.Info('Saving Histograms for: ' + selectOn +
' In bin range: ' + bin[0] + ' - ' + bin[1])
for i in xrange(len(t_var)):
rBranch, tBranch, sName, sLocation, xMin, xMax, nBins = t_var[
i]
location = sDir + '/' + b_mu + '/' + select + '/' + sLocation
ROOTUtils.saveToFile(
d_filledHists[b_mu][select][location + sName], outFile,
location, sName)
def MakeSpecList(self):
"""
Build the specimen name list from fasta files in
self.assembly_dir
:return:
"""
self.fasta_list = Bash.GetFastaInDir(self.assembly_dir)
print self.fasta_list
self.SpecList = [
PythonUtils.GetFastaPrefix(x) for x in list(self.fasta_list)
]
def WritePbsFile(self, fastq):
"""
Write a Pbs file for this specimen
:param fastq:
:return:
"""
if (str(fastq).find('_R1_') > -1):
#Extract the specimen name from the R1 fastq file name
specimen = PythonUtils.GetFastqPrefix(fastq)
#The Pbs output file
handler = open(self.out_dir + specimen + '.pbs', 'w')
handler.write(self.GetHeader(specimen))
handler.write(self.GetCore(fastq))
handler.close()
def writeLatex(self, t_sens, l_corrName, l_pTV, texFile, latexDir):
## Want the corrections to appear in order
l_corrName.sort()
texName = latexDir + texFile + '.tex'
## Create the latex table for the number of corrections
## and also the number of bins
f = open(texName, 'w')
f.write('\documentclass[11pt, landscape]{aastex} \n')
f.write('\usepackage{pdflscape} \n')
f.write('\\begin{document}\n')
f.write('\\footnotesize \n')
f.write('\\begin{landscape} \n')
f.write('\\begin{tabular}{r' + len(l_pTV) * '|c' + '}\n')
f.write('Correction')
for bin in l_pTV:
f.write(' & ' + bin)
f.write(' \\\ \hline\hline \n')
for corr in l_corrName:
f.write(corr)
for i in xrange(len(t_sens)):
t_sort = tuple(sorted(t_sens[i], key=lambda item: item[0]))
for j in xrange(len(t_sort)):
if t_sort[j][0] == corr:
f.write(' & ' + str(round(t_sort[j][1], 3)) +
' $\pm$ ' + str(round(t_sort[j][2], 3)))
f.write(' \\\ \n')
f.write('\hline\n')
f.write('\end{tabular} \n')
f.write('\end{landscape} \n')
f.write('\end{document}\n')
f.close()
PythonUtils.Info('FILE: ' + texName + ' Written')
return texFile
def createHistograms(self, t_type, sample, baseDir, selectOn, t_select):
PythonUtils.Info('++++++++++++++++++++++++++++++++++++++++++++++++++')
PythonUtils.Info(' CREATING HISTOGRAMS: ' + baseDir + ' ')
PythonUtils.Info('++++++++++++++++++++++++++++++++++++++++++++++++++')
d_histBinType = {}
for bin in t_select:
## Don't have a trailing slash for all - looks messy
PythonUtils.Info('Creating Histograms for process: ' + baseDir +
' in bin: ' + bin[0] + '-' + bin[1])
if bin[0] == 'all':
select = selectOn + '-' + bin[0]
else:
select = selectOn + '-' + bin[0] + '-' + bin[1]
d_histType = {}
for i in range(len(t_type)):
## Decalare the list variables
type, typeFile = t_type[i]
## Open typeFile and loop through it
PythonUtils.doesFileExist(typeFile)
t_typeFile = PythonUtils.makeTupleFromFile(typeFile, ',')
d_hist = {}
for j in range(len(t_typeFile)):
rBranch, tBranch, sName, sLocation, xMin, xMax, nBins = t_typeFile[
j]
hName = 'h_' + baseDir + '_' + sample + '_' + selectOn + bin[
0] + '_' + type + '_' + rBranch
location = baseDir + '/' + sample + '/' + select + '/' + sLocation
if 'crea' in type:
d_hist[location + sName] = TH1F(
hName, hName, int(nBins), float(xMin), float(xMax))
d_hist[location + sName].SetDirectory(0)
else:
d_hist[location + sName] = TH1D(
hName, hName, int(nBins), float(xMin), float(xMax))
d_hist[location + sName].SetDirectory(0)
d_histType[type] = d_hist
d_histBinType[select] = d_histType
return d_histBinType
def saveHistograms(self, d_filledHists, t_type, outFile, sample, baseDir, selectOn, t_select):
PythonUtils.Info('++++++++++++++++++++++++++++++++++++++++++++++++++')
PythonUtils.Info(' SAVING HISTOGRAMS: ' + baseDir + ' ')
PythonUtils.Info('++++++++++++++++++++++++++++++++++++++++++++++++++')
for bin in t_select:
## Don't have a trainling slash for all - looks messy
PythonUtils.Info('Creating Histograms for process: ' + baseDir +
' in bin: ' + bin[0] + '-' + bin[1])
if bin[0] == 'all':
select = selectOn + '-' + bin[0]
else:
select = selectOn + '-' + bin[0] + '-' + bin[1]
for i in range(len(t_type)):
type, typeFile = t_type[i]
t_typeFile = PythonUtils.makeTupleFromFile(typeFile, ',')
for j in range(len(t_typeFile)):
rBranch, tBranch, sName, sLocation, xMin, xMax, nBuns = t_typeFile[j]
location = baseDir + '/' + sample + '/' + select + '/' + sLocation
ROOTUtils.saveToFile(d_filledHists[select][type][location + sName], outFile, location, sName)
def fillHistograms(self, d_histType, t_type, iFile, treeName, sample,
baseDir, t_select, selectOn, nEvents):
PythonUtils.Info('++++++++++++++++++++++++++++++++++++++++++++++++++')
PythonUtils.Info(' FILLING HISTOGRAMS: ' + baseDir + ' ')
PythonUtils.Info('++++++++++++++++++++++++++++++++++++++++++++++++++')
f = TFile(iFile, 'READ')
t = f.Get(treeName)
for i, entry in enumerate(t):
## Test for the test, train or all samples
if not PythonUtils.keepEntry(sample, i):
continue
## Run over the number of events: -1 is all
if nEvents != -1:
if i == nEvents:
break
## Loop over the binning
for bin in t_select:
## Don't have a trailing slash for all - looks messy
if bin[0] == 'all':
select = selectOn + '-' + bin[0]
else:
select = selectOn + '-' + bin[0] + '-' + bin[1]
if bin[0] == 'all' or PythonUtils.getSelection(
entry, bin[0], bin[1], selectOn):
for j in range(len(t_type)):
type, typeFile = t_type[j]
t_typeFile = PythonUtils.makeTupleFromFile(
typeFile, ',')
for k in range(len(t_typeFile)):
rBranch, tBranch, sName, sLocation, xMin, xMax, nBuns = t_typeFile[
k]
location = baseDir + '/' + sample + '/' + select + '/' + sLocation
## Setup how the values are filled for different types
d_histBinTypeF = self.doFill(
d_histType, entry, rBranch, tBranch, select,
type, location, sName)
return d_histBinTypeF
def Run(self, doProfile, createFile):
## Create the lists from the main configFile
t_select = PythonUtils.makeTupleFromString(self.selectBins, ',', '?')
t_muon = PythonUtils.makeTupleFromString(self.muBins, ',', '?')
if createFile:
PythonUtils.Info('Creating Histogram File for Profile Plots')
## Check the list files exist
l_fList = [self.variableList, self.treeList]
for f in l_fList:
PythonUtils.doesFileExist(f)
## Make the lists from the list files
t_var = PythonUtils.makeTupleFromFile(self.variableList, ',')
t_tree = PythonUtils.makeTupleFromFile(self.treeList, ',')
for i in xrange(len(t_tree)):
tFile, tName, sDir = t_tree[i]
PythonUtils.doesFileExist(tFile)
d_histType = self.createHistograms(t_var, sDir, tFile, t_muon,
self.selectOn, t_select)
d_filledHists = self.fillHistograms(d_histType, t_var, tFile, tName, sDir, t_muon,
t_select, self.selectOn, self.nEvents)
self.saveHistograms(d_filledHists, t_var, self.outHist,
sDir, t_muon, self.selectOn, t_select)
if doProfile:
PythonUtils.Info("Creating Profile Plots!")
## Create the Lists from profile cfg
l_saveAs = PythonUtils.makeListFromString(self.plotType, ',')
## Check the list files exist
l_pList = [self.axisList, self.histoList, self.legendList,
self.textList, self.varList, self.inFile]
for f in l_pList:
PythonUtils.doesFileExist(f)
## Create the list of lists from each list file
t_axisList = PythonUtils.makeTupleFromFile(self.axisList, ',')
t_histoList = PythonUtils.makeTupleFromFile(self.histoList, ',')
t_legendList = PythonUtils.makeTupleFromFile(self.legendList, ',')
t_textList = PythonUtils.makeTupleFromFile(self.textList, ',')
## Check teh first Item matches:
l_tList = [t_histoList, t_legendList, t_textList]
for l in l_tList:
PythonUtils.firstItemMatching(l, t_axisList)
## Add the var list files to a list
l_varList = []
f = open(self.varList, 'r')
for line in f:
if line.startswith('#'):
continue
else:
l_varList.append(line)
for i in xrange(len(l_varList)):
PythonUtils.doesFileExist(l_varList[i].strip())
t_varList = PythonUtils.makeTupleFromFile(l_varList[i].strip(), ',')
l_prof = []
t_legend = []
for j in xrange(len(t_varList)):
d_pTMean = {}
l_err = []
for bin in t_select:
if bin[0] == 'all' or bin[1] == 'Inf':
continue
if bin[1]:
select = self.selectOn + '-' + bin[0] + '-' + bin[1]
else:
select = self.selectOn + '-' + bin[0]
location = t_varList[j][2] + '/' + t_varList[j][6] + '/' + select + '/' + t_varList[j][7]
pTbin = float(bin[0]) + (float(bin[1]) - float(bin[0])) / 2
pTMean = ROOTUtils.retrieveHistogram(self.inFile, location, t_varList[j][1]).GetMean()
pTMeanErr = ROOTUtils.retrieveHistogram(self.inFile, location, t_varList[j][1]).GetMeanError()
l_err.append(pTMeanErr)
d_pTMean[pTbin] = pTMean
h_prof, l_legend= PlottingUtils.twoDprofile(d_pTMean, t_histoList[i], t_varList[j], l_err)
l_prof.append(h_prof)
t_legend.append(l_legend)
saveString = self.plotDir + t_histoList[i][0]
PlottingUtils.overlayProfile(l_prof, t_axisList[i], t_histoList[i], t_legendList[i], t_textList[i], t_legend, l_saveAs, saveString)
def Run(self):
## Make the list from the config file
l_range = PythonUtils.makeListFromString(self.plotRange, ',')
t_select = PythonUtils.makeTupleFromString(self.selectBins, ',','?')
## Check the input and process files exist
l_file = [self.processList, self.inputFile]
for file in l_file:
PythonUtils.doesFileExist(file)
## Loop over the ProcessList files
f = open(self.processList, 'r')
for line in f:
if line.startswith('#'):
continue
l_process = PythonUtils.makeListFromString(line.strip(), ',')
PythonUtils.doesFileExist(l_process[0])
t_process = PythonUtils.makeTupleFromFile(l_process[0], ',')
l_yield = []
##ll
ll_VH = ['VH']
ll_VV = ['VV']
ll_top = ['top']
ll_stop = ['stop']
ll_Wl = ['W+l']
ll_Wcl = ['W+cl']
ll_Whf = ['W+hf']
ll_Zl = ['Z+l']
ll_Zcl = ['Z+cl']
ll_Zhf = ['Z+hf']
ll_MJe = ['MJe']
ll_MJm = ['MJm']
ll_tot = ['Total']
ll_data = ['Data']
for bin in t_select:
if bin[0] == 'all':
select = self.selectOn + '-' + bin[0]
else:
select = self.selectOn + '-' + bin[0] + '-' + bin[1]
## nEvents
nSignal = 0
nTot = 0
nTop = 0
nStop = 0
nWl = 0
nWh = 0
nWcl = 0
nZl = 0
nZh = 0
nZcl = 0
nVV = 0
nMJe = 0
nMJm = 0
nData = 0
##nEventsErr
eSignal = 0
eTot = 0
eTop = 0
eStop = 0
eWl = 0
eWh = 0
eWcl = 0
eZl = 0
eZh = 0
eZcl = 0
eVV = 0
eMJe = 0
eMJm = 0
eData = 0
for i in xrange(len(t_process)):
process, sLocation, pType, selection, SF = t_process[i]
location = sLocation + '/all/' + select + '/' + self.histLocation
hist = ROOTUtils.retrieveHistogram(self.inputFile, location, self.histogram)
yValue, yErr = ROOTUtils.getYield(hist, l_range[0], l_range[1], t_process[i][4])
if pType == "SIGNAL":
nTot += yValue
nSignal += yValue
eSignal += yErr
eTot += yErr
elif pType == "W+H":
nTot += yValue
nWh += yValue
eWh += yErr
eTot += yErr
elif pType == "W+CL":
nTot += yValue
nWcl += yValue
eWcl += yErr
eTot += yErr
elif pType == "W+L":
nTot += yValue
nWl += yValue
eWl += yErr
eTot += yErr
elif pType == "Z+H":
nTot += yValue
nZh += yValue
eZh += yErr
eTot += yErr
elif pType == "Z+CL":
nTot += yValue
nZcl += yValue
eZcl += yErr
eTot += yErr
elif pType == "Z+L":
nTot += yValue
nZl += yValue
eZl += yErr
eTot += yErr
elif pType == "TOP":
nTot += yValue
nTop += yValue
eTop += yErr
eTot += yErr
elif pType == "STOP":
nTot += yValue
nStop += yValue
nStop += yErr
eTot += yErr
elif pType == "VV":
nTot += yValue
nVV += yValue
eVV += yErr
eTot += yErr
elif pType == "MJe":
nTot += yValue
nMJe += yValue
nMJe += yErr
eTot += yErr
elif pType == "MJm":
nTot += yValue
nMJm += yValue
eMJm += yErr
eTot += yErr
elif pType == "DATA":
nData += yValue
eData += yErr
else:
PythonUtils.Error('Not a valid selection: ' + t_process[i][2] + '!!!\n')
l_VH = [nSignal, eSignal]
ll_VH.append(l_VH)
l_VV = [nVV, eVV]
ll_VV.append(l_VV)
l_top = [nTop, eTop]
ll_top.append(l_top)
l_stop = [nStop, eStop]
ll_stop.append(l_stop)
l_Wl = [nWl, eWl]
ll_Wl.append(l_Wl)
l_Wcl = [nWcl, eWcl]
ll_Wcl.append(l_Wcl)
l_Whf = [nWh, eWh]
ll_Whf.append(l_Whf)
l_Zl = [nZl, eZl]
ll_Zl.append(l_Zl)
l_Zcl = [nZcl, eZcl]
ll_Zcl.append(l_Zcl)
l_Zhf = [nZh, eZh]
ll_Zhf.append(l_Zhf)
l_MJe = [nMJe, eMJe]
ll_MJe.append(l_MJe)
l_MJm = [nMJm, eMJm]
ll_MJm.append(l_MJm)
l_tot = [nTot, eTot]
ll_tot.append(l_tot)
l_data = [nData, eData]
ll_data.append(l_data)
l_yield = [ll_VH, ll_VV, ll_top, ll_stop, ll_Wl, ll_Wcl, ll_Whf,
ll_Zl, ll_Zcl, ll_Zhf, ll_MJe, ll_MJm, ll_tot, ll_data]
self.printToScreen(l_yield, select + ' ' + t_process[i][3])
if self.createTex:
texFile = self.saveToTex(l_yield, l_process[1], self.outputFile)
self.runLatex(self.outputFile, texFile)
def Run(self):
## Add the trailing slash to the end of latexDir
if not self.latexDir.endswith('/'):
latexDir = self.latexDir + '/'
PythonUtils.doesDirExist(latexDir)
else:
latexDir = self.latexDir
## Check the .list files exist
l_fList = [self.processList, self.histoList,
self.corrList, self.histFile]
for file in l_fList:
PythonUtils.doesFileExist(file)
## Make the lists from the list files
t_histoList = PythonUtils.makeTupleFromFile(self.histoList, ',')
t_corrList = PythonUtils.makeTupleFromFile(self.corrList, ',')
t_pList = PythonUtils.makeTupleFromFile(self.processList, ',')
## Create the variable name dictionary
compTo = ''
d_nameVar = {}
for i in xrange(len(t_corrList)):
d_nameVar[t_corrList[i][0]] = t_corrList[i][1]
if t_corrList[i][2] == '1':
compTo = t_corrList[i][0]
## Construct the list of processes for each region
l_pFiles = []
f = open(self.processList, 'r')
for line in f:
if not line.startswith('#'):
l_pFiles.append(line.strip())
for i in xrange(len(t_pList)):
pFile, tagType = t_pList[i]
PythonUtils.doesFileExist(pFile)
t_processList = PythonUtils.makeTupleFromFile(pFile, ',')
ld_sens = []
l_pTV = []
option = ''
for k in xrange(len(t_histoList)):
l_pTV.append(t_histoList[k][3])
d_sens = {}
l_corrName = []
# option = t_histoList[k][2]
for name, title in d_nameVar.iteritems():
t_h = []
l_corrName.append(name)
for j in xrange(len(t_processList)):
option = t_processList[j][0]
hLocation = t_processList[j][1] + t_histoList[k][1]
t_h.append([ROOTUtils.retrieveHistogram(self.histFile, hLocation, title), t_processList[j][0], t_processList[j][2], t_processList[j][3]])
sens, sensErr = ROOTUtils.calculateSensitivity(t_h, t_histoList[k], name)
d_sens[name] = [sens, sensErr]
d_sens[name] = ROOTUtils.calculateSensitivity(t_h, t_histoList[k], name)
l_sens = [t_histoList[k][0] + ' ' + tagType, d_sens]
ld_sens.append(l_sens)
t_sens = PythonUtils.rankByValue(ld_sens, compTo, 'SENSITIVITY')
if self.doLatex:
latex = self.writeLatex(t_sens, l_corrName, l_pTV,
'Sensitivity_' + tagType + '_' + option, latexDir)
PythonUtils.runLatex(latex, latexDir)
def Run(self):
## Create lists from config file options
l_saveAs = PythonUtils.makeListFromString(self.saveAs, ',')
## Check the .list files exist
l_listFile = [self.axisList, self.fitList, self.histoList, self.legendList,
self.textList, self.variableList, self.histFile]
for file in l_listFile:
PythonUtils.doesFileExist(file)
## Create the tuples from the .list files
t_axisList = PythonUtils.makeTupleFromFile(self.axisList, ',')
t_fitList = PythonUtils.makeTupleFromFile(self.fitList, ',')
t_histoList = PythonUtils.makeTupleFromFile(self.histoList, ',')
t_legendList = PythonUtils.makeTupleFromFile(self.legendList, ',')
t_textList = PythonUtils.makeTupleFromFile(self.textList, ',')
## Math the first titems
l_tuples = [t_fitList, t_histoList, t_legendList, t_textList]
for item in l_tuples:
PythonUtils.firstItemMatching(t_axisList, item)
## Loop over the variable files to get the relevant info
for j in xrange(len(t_axisList)):
f_str, vRebin, vdoNorm, vXMin, vXMax, vFit, vXValue, vYValue = t_fitList[j]
l_files = []
f = open(self.variableList, 'r')
for file in f:
l_files.append(file.strip())
l_twoD = []
t_legend = []
PythonUtils.doesFileExist(l_files[j])
g = open(l_files[j], 'r')
for vFile in g:
PythonUtils.doesFileExist(vFile.strip())
vFileParser = ConfigParser.SafeConfigParser()
vFileParser.read(vFile.strip())
## Options
vTitle = vFileParser.get('Options','title')
vVariable = vFileParser.get('Options','variable')
vColour = vFileParser.getint('Options','colour')
vMarker = vFileParser.getint('Options','marker')
vMarkerSize = vFileParser.getfloat('Options','markerSize')
vFitMin = vFileParser.get('Options','fitMin')
vFitMax = vFileParser.get('Options','fitMax')
## Create l_statInfo
l_statInfo = [t_histoList[j][1], t_histoList[j][1], float(vRebin), int(vdoNorm),
float(vXMin), float(vXMax), vFitMin, vFitMax]
getMean = 0
getRMS = 0
getRES = 0
if vXValue == 'MEAN':
getMean = 1
elif vXValue == 'RMS':
getRMS = 1
elif vXValue == 'RESOLUTION':
getRES = 1
else:
getMean = getRMS = getRES = 0
l_xOption = [getMean, getRMS, getRES]
getMean = 0
getRMS = 0
getRES = 0
if vYValue == 'MEAN':
getMean = 1
elif vYValue == 'RMS':
getRMS = 1
elif vYValue == 'RESOLUTION':
getRES = 1
else:
getMean = getRMS = getRES = 0
l_yOption = [getMean, getRMS, getRES]
## Get the list of stats
l_plot = [ROOTUtils.retrieveHistogram(self.histFile, t_histoList[j][1], vVariable)]
l_xValue, l_xErr = ROOTUtils.getHistoStat(l_plot, l_statInfo, l_xOption, vFit, vVariable)
l_yValue, l_yErr = ROOTUtils.getHistoStat(l_plot, l_statInfo, l_yOption, vFit, vVariable)
## Get the value we want from the list so we can plot it!
xValue = -1
for stat in l_xValue:
if stat != -1:
xValue = stat
for stat in l_yValue:
if stat != -1:
yValue = stat
## Create a 2D plot and save to a list so can overlay them
twoDPlot = PlottingUtils.createTwoD(t_histoList[j], xValue, yValue,
vMarker, vMarkerSize, vColour)
l_twoD.append(twoDPlot)
if vTitle:
l_legend = [twoDPlot, vTitle, 'p']
t_legend.append(l_legend)
## Overlay our 2D plots onto one canvas
PlottingUtils.overlayTwoD(l_twoD, t_legend, t_histoList[j], t_legendList[j],
t_textList[j], t_axisList[j], self.saveDir, l_saveAs)
def main():
use = '''
Example:
python PYTHON/MODULES/runStacking.py --MainConfigFile=ConfigFiles/Stacking/MainConfig/MainConfigFile.cfg
'''
parser = OptionParser(usage = use)
parser.add_option("--MainConfigFile", dest="MainConfigFile", default="", type="string", help="Main Configuration File used for stacking")
options, args = parser.parse_args()
if options.MainConfigFile:
configFileParser = ConfigParser.SafeConfigParser()
configFileParser.read(options.MainConfigFile)
## General Options
debug = configFileParser.getint('GeneralOptions','debug')
doStacking = configFileParser.getint('GeneralOptions','doStacking')
histFile = configFileParser.get('GeneralOptions','histFile')
saveDir = configFileParser.get('GeneralOptions','saveDir')
saveAs = configFileParser.get('GeneralOptions','saveAs')
## List Files
processList = configFileParser.get('ListFiles','processList')
histoList = configFileParser.get('ListFiles','histoList')
legendList = configFileParser.get('ListFiles','legendList')
textList = configFileParser.get('ListFiles','textList')
axisList = configFileParser.get('ListFiles','axisList')
ratioList = configFileParser.get('ListFiles','ratioList')
else:
print "\nERROR: No ConfigFile"
print use
sys.exit(1)
## Make Lists, dicts and tuples from Main Config File
l_saveAs = PythonUtils.makeListFromString(saveAs, ',')
## Make sure our .list files do existi
PythonUtils.doesDirExist(saveDir)
PythonUtils.doesFileExist(histFile)
PythonUtils.doesFileExist(processList)
PythonUtils.doesFileExist(histoList)
PythonUtils.doesFileExist(legendList)
PythonUtils.doesFileExist(textList)
PythonUtils.doesFileExist(axisList)
PythonUtils.doesFileExist(ratioList)
## Make the Tuples and Lists from the .list files
t_histoList = PythonUtils.makeTupleFromFile(histoList, ',')
t_legendList = PythonUtils.makeTupleFromFile(legendList, ',')
t_textList = PythonUtils.makeTupleFromFile(textList, ',')
t_axisList = PythonUtils.makeTupleFromFile(axisList, ',')
t_ratioList = PythonUtils.makeTupleFromFile(ratioList, ',')
## Match the first item to make sure theyre compatible
PythonUtils.firstItemMatching(t_histoList, t_legendList)
PythonUtils.firstItemMatching(t_histoList, t_textList)
PythonUtils.firstItemMatching(t_histoList, t_axisList)
PythonUtils.firstItemMatching(t_histoList, t_ratioList)
## Loops over the contents of processList
createStackedPlots = CreateStackedPlots(t_histoList, t_legendList, t_textList, t_axisList,
t_ratioList, histFile, l_saveAs, saveDir, doStacking)
createStackedPlots.Run(processList)
def Run(self):
# Create lists from config file info
l_saveAs = PythonUtils.makeListFromString(self.saveAs, ',')
## Check the .list files exist!
l_listFile = [self.axisList, self.histoList, self.legendList,
self.overlayList, self.textList, self.histFile]
for file in l_listFile:
PythonUtils.doesFileExist(file)
## Make the tuples from the input files
t_axisList = PythonUtils.makeTupleFromFile(self.axisList, ',')
t_histoList = PythonUtils.makeTupleFromFile(self.histoList, ',')
t_legendList = PythonUtils.makeTupleFromFile(self.legendList, ',')
t_textList = PythonUtils.makeTupleFromFile(self.textList, ',')
## match the first item to make sure they're compatible
l_tuples = [t_histoList, t_legendList, t_textList]
for item in l_tuples:
PythonUtils.firstItemMatching(t_axisList, item)
l_files = []
f = open(self.overlayList, 'r')
for line in f:
if line.startswith('#'):
continue
else:
l_files.append(line.strip())
for i in xrange(len(l_files)):
PythonUtils.doesFileExist(l_files[i])
t_overlayList = PythonUtils.makeTupleFromFile(l_files[i], ',')
t_plots = []
for j in xrange(len(t_overlayList)):
histoLocation = t_overlayList[j][1] + t_histoList[i][1]
t_plots.append([ROOTUtils.retrieveHistogram(self.histFile, histoLocation, t_overlayList[j][2])])
saveString = self.saveDir + t_histoList[i][0]
PlottingUtils.overlayHistograms(t_plots, t_overlayList, t_histoList[i],
t_legendList[i], t_textList[i], t_axisList[i],
l_saveAs, saveString)
def Run(self):
## Check the files in the config file exist
l_file = [self.histFile, self.correctionList,
self.histoList, self.fittingList]
for file in l_file:
PythonUtils.doesFileExist(file)
## Add the trailing slash to the end of latexDir
if not self.latexDir.endswith('/'):
latexDir = self.latexDir + '/'
PythonUtils.doesDirExist(latexDir)
else:
latexDir = self.latexDir
## Make tuples from the list files
t_histoList = PythonUtils.makeTupleFromFile(self.histoList, ',')
t_fittingList = PythonUtils.makeTupleFromFile(self.fittingList, ',')
## Loop over the files in correctionsList each one is a new run
l_cFiles = []
f = open(self.correctionList, 'r')
for file in f:
if file.startswith('#'):
continue
else:
l_cFiles.append(file.strip())
for i in xrange(len(l_cFiles)):
ld_mean = []
ld_rms = []
ld_res = []
l_getOption = [int(t_fittingList[i][2]), int(t_fittingList[i][3]), int(t_fittingList[i][4])]
## Check the file in the list exists
PythonUtils.doesFileExist(l_cFiles[i])
t_corrList = PythonUtils.makeTupleFromFile(l_cFiles[i], ',')
d_meanResRMS = {}
tagType = ''
for j in xrange(len(t_corrList)):
h_location = t_corrList[j][1] + t_histoList[i][1]
t_histoList[i].append(t_corrList[j][3])
t_histoList[i].append(t_corrList[j][4])
l_hist = [t_histoList[i][0], t_histoList[i][1], t_histoList[i][2],
t_histoList[i][3], t_histoList[i][4], t_histoList[i][5],
t_corrList[j][3], t_corrList[j][4]]
pTVBin = t_histoList[i][6]
tagType = t_histoList[i][7]
l_h = [ROOTUtils.retrieveHistogram(self.histFile,
h_location, t_corrList[j][2])]
d_meanResRMS[t_corrList[j][0]] = ROOTUtils.getHistoStat(l_h, l_hist, l_getOption,
t_fittingList[i][5], t_corrList[j][0])
## We now have all the info to calculate the stats from the plots!
d_mean = {}
d_RMS = {}
d_res = {}
getMean, getRMS, getResolution = l_getOption
compareTo = t_fittingList[i][1]
fit = t_fittingList[i][5]
l_corrName = []
for k, v in d_meanResRMS.iteritems():
l_corrName.append(k)
if getMean:
d_mean[k] = [v[0][0], v[1][0]]
if getRMS:
d_RMS[k] = [v[0][1], v[1][1]]
if getResolution:
d_res[k] = [v[0][2], v[1][2]]
ld_mean.append([t_histoList[i][0], d_mean])
ld_rms.append([t_histoList[i][0], d_RMS])
ld_res.append([t_histoList[i][0], d_res])
t_mRr = []
if getMean:
t_mRr.append(PythonUtils.rankByValue(ld_mean, compareTo,
fit + ' Mean'))
if getRMS:
t_mRr.append(PythonUtils.rankByValue(ld_rms, compareTo,
fit + ' RMS'))
if getResolution:
t_mRr.append(PythonUtils.rankByValue(ld_res, compareTo,
fit + ' Resolution'))
if self.doLatex:
latex = self.writeLatex(t_mRr, l_corrName, pTVBin, t_histoList[i][0], latexDir, getMean, getRMS, getResolution)
PythonUtils.runLatex(latex, latexDir)