def GetOutPreFromRoc(self, Name):
train, test = self.DataSet.GetInput(Name)
if "Even" in Name:
OutPreOther, OutPreSame = self.RocEven.MaxPre[0].ravel(
), self.RocEven.MaxPre[1].ravel()
elif "Odd" in Name:
OutPreOther, OutPreSame = self.RocOdd.MaxPre[0].ravel(
), self.RocOdd.MaxPre[1].ravel()
np.save(self.SavePath + Name + "_OutPreOther.npy", OutPreOther)
np.save(self.SavePath + Name + "_OutPreSame.npy", OutPreSame)
Auctest = roc_auc_score(test.OutTrue,
OutPreOther,
sample_weight=test.Weights)
Auctrain = roc_auc_score(train.OutTrue,
OutPreSame,
sample_weight=train.Weights)
print(Name)
print("AUC train: " + str(Auctrain))
print("AUC test: " + str(Auctest))
return OutPreOther, OutPreSame
def RocCurve(path, OutPre, DataSet, ModelNames):
c1 = ROOT.TCanvas("c1", "Canvas", 800, 600)
leg = ROOT.TLegend(0.1, 0.1, 0.3, 0.3)
mg = ROOT.TMultiGraph()
c1.SetGrid()
if (not isinstance(OutPre, list)):
OutPre = [OutPre]
for i, Name in enumerate(ModelNames):
train, test = DataSet.GetInput(Name)
OutTrue = test.OutTrue
weights = test.Weights
fpr, tpr, threshold = roc_curve(OutTrue,
OutPre[i],
sample_weight=weights)
modelAuc = roc_auc_score(OutTrue, OutPre[i], sample_weight=weights)
gr1 = ROOT.TGraph(len(tpr), 1 - fpr, tpr)
gr1.SetLineColor(1 + i)
gr1.SetLineWidth(2)
mg.Add(gr1, "L")
leg.AddEntry(gr1, ModelNames[i] + " (Auc={:05.3f})".format(modelAuc))
mg.Draw("A")
leg.Draw()
mg.GetXaxis().SetTitle("Signal efficiency")
mg.GetYaxis().SetTitle("Background rejection")
mg.GetHistogram().GetXaxis().SetRangeUser(0.2, 1)
mg.GetHistogram().GetYaxis().SetRangeUser(0.2, 1)
c1.SaveAs(path + "RocCurve.png")
def on_epoch_end(self, epoch, logs={}):
if (epoch == 64 or epoch == 59):
y_pred = self.model.predict(self.model.X_test)
self.TestAucs.append(
roc_auc_score(self.model.Y_test,
y_pred,
sample_weight=self.model.W_test))
y_pred_Train = self.model.predict(self.model.X_train)
self.TrainAucs.append(
roc_auc_score(self.model.Y_train,
y_pred_Train,
sample_weight=self.model.W_train))
#Save the Score if the Auc is the current highest
if (self.TestAucs[-1] == max(self.TestAucs)):
self.MaxPre = [y_pred, y_pred_Train]
return
def GetOutPreFromModel(self, Name):
train, test = self.DataSet.GetInput(Name)
vali = self.DataSet.vali
if "Even" in Name:
OutPreOther, OutPreSame = self.ModelEven.predict(
test.Events).ravel(), self.ModelEven.predict(
train.Events).ravel()
OutPreVali = self.ModelEven.predict(vali.Events).ravel()
elif "Odd" in Name:
OutPreOther, OutPreSame = self.ModelOdd.predict(
test.Events).ravel(), self.ModelOdd.predict(
train.Events).ravel()
OutPreVali = self.ModelOdd.predict(vali.Events).ravel()
np.save(self.SavePath + Name + "_OutPreOther.npy", OutPreOther)
np.save(self.SavePath + Name + "_OutPreSame.npy", OutPreSame)
Auctest = roc_auc_score(test.OutTrue,
OutPreOther,
sample_weight=test.Weights)
Auctrain = roc_auc_score(train.OutTrue,
OutPreSame,
sample_weight=train.Weights)
Aucvali = roc_auc_score(vali.OutTrue,
OutPreVali,
sample_weight=vali.Weights)
print(Name)
print("AUC train: " + str(Auctrain))
print("AUC test: " + str(Auctest))
# print("AUC vali: "+str(Aucvali))
# with open("SearchOut.txt","a+") as Out:
# Out.write(Name+"\n")
# Out.write("AUC train: "+str(Auctrain)+"\n")
# Out.write("AUC test: "+str(Auctest)+"\n")
# Out.close()
return OutPreOther, OutPreSame
def on_epoch_end(self, epoch, logs={}):
# # Evalute Loss and Auc for test
# self.TestLosses.append(logs.get('val_loss'))
y_pred = self.model.predict(self.model.X_test)
if (y_pred.shape[1] == 1):
y_pred = y_pred.flatten()
self.TestAucs.append(
roc_auc_score(self.model.Y_test,
y_pred,
sample_weight=self.model.W_test))
else:
self.TestAucs.append(
roc_auc_score(self.model.Y_test,
y_pred[:, 0],
sample_weight=self.model.W_test))
# # Evalute Loss and Auc for train
# self.TrainLosses.append(logs.get('loss'))
y_pred_Train = self.model.predict(self.model.X_train)
if (y_pred_Train.shape[1] == 1):
y_pred_Train = y_pred_Train.flatten()
self.TrainAucs.append(
roc_auc_score(self.model.Y_train,
y_pred_Train,
sample_weight=self.model.W_train))
else:
self.TrainAucs.append(
roc_auc_score(self.model.Y_train,
y_pred_Train[:, 0],
sample_weight=self.model.W_train))
#Save the Score if the Auc is the current highest
if (self.TestAucs[-1] == max(self.TestAucs)):
self.MaxPre = [y_pred, y_pred_Train]
return
def MultiScore(path, OutPreOther, OutPreSame, DataSet, Name, ClassNum):
train, test = DataSet.GetInput(Name)
AucOther = roc_auc_score(GetTrueLabels(test, ClassNum),
OutPreOther,
sample_weight=test.Weights)
AucSame = roc_auc_score(GetTrueLabels(train, ClassNum),
OutPreSame,
sample_weight=train.Weights)
maxValue, minValue = np.max(OutPreOther), np.min(OutPreOther)
if (np.max(OutPreSame) > maxValue):
maxValue = np.max(OutPreSame)
if (np.min(OutPreSame) < minValue):
minValue = np.min(OutPreSame)
c1 = ROOT.TCanvas("c1", "Canvas", 800, 600)
ROOT.gStyle.SetOptStat(0)
LClass = [0, 10, 11, 13]
# Names = {0:r't\bar{t}t\bar{t}',13:r't\bar{t}W',12:r't\bar{t}WW',11:r't\bar{t}Z',10:r't\bar{t}H',2:'V+jets'
# ,3:'VV',4:r't(\bar{t})X',1:'others',9:r't\bar{t} QmisID',8:r't\bar{t} CO',
# 7:r't\bar{t} HF',6:r't\bar{t} light',5:r't\bar{t} other'}
# Color = {13:3,12:ROOT.kBlack,11:6,10:4, #Colorcoding
# 2:ROOT.kAzure+1,3:ROOT.kYellow-7,4:ROOT.kGreen-5,1:ROOT.kGreen-1,
# 9:ROOT.kGreen-10,8:ROOT.kPink-9,7:ROOT.kViolet-4,
# 6:ROOT.kBlue-7,5:ROOT.kBlue-10,0:ROOT.kRed+1}
Names = {0: r't\bar{t}t\bar{t}', 1: r't\bar{t}V', 2: 'rest'}
Color = {0: ROOT.kRed + 1, 1: ROOT.kBlack, 2: ROOT.kGreen}
LHists = []
for Class in LClass:
hist = GetScoreTH(Class, OutPreOther, test.Weights, test.MultiClass,
minValue, maxValue)
hist.SetLineColor(Color[Class])
hist.SetLineWidth(2)
LHists.append(hist)
Pre = OutPreOther[GetTrueLabels(test, ClassNum) == 0]
Weight = test.Weights[GetTrueLabels(test, ClassNum) == 0]
histAll = ROOT.TH1F("hAll", "", 20, minValue, maxValue)
fill_hist(histAll, Pre, weights=Weight)
histAll.Scale(1 / histAll.GetSumOfWeights() / Getdx(histAll))
histAll.SetLineColor(1)
histAll.SetLineWidth(2)
LHists[0].GetXaxis().SetTitle('Score')
Lmax = [hist.GetMaximum() for hist in LHists]
LHists[0].GetYaxis().SetRangeUser(0, max(Lmax) * 1.4)
LHists[0].Draw("Hist")
histAll.Draw("HistSame")
for i in range(1, len(LHists)):
LHists[i].Draw("HistSame")
T1 = ROOT.TText()
T2 = ROOT.TText()
T1.SetTextSize(0.03)
T2.SetTextSize(0.03)
T1.SetTextAlign(21)
T2.SetTextAlign(21)
T1.DrawTextNDC(.23, .87,
r"AUC = " + str(round(AucOther, 3)) + " (on test)")
T2.DrawTextNDC(.23, .84,
r"AUC = " + str(round(AucSame, 3)) + " (on train)")
leg = ROOT.TLegend(0.6, 0.75, 0.9, 0.9)
for i, hist in enumerate(LHists):
leg.AddEntry(hist, Names[LClass[i]])
leg.AddEntry(histAll, "all backgrounds")
leg.Draw()
c1.SaveAs(path + "Score" + Name + '_' + str(ClassNum) + ".png")
return AucOther, AucSame
def Score(path, OutPreOther, OutPreSame, DataSet, Name):
train, test = DataSet.GetInput(Name)
AucOther = roc_auc_score(test.OutTrue,
OutPreOther,
sample_weight=test.Weights)
AucSame = roc_auc_score(train.OutTrue,
OutPreSame,
sample_weight=train.Weights)
maxValue, minValue = np.max(OutPreOther), np.min(OutPreOther)
if (np.max(OutPreSame) > maxValue):
maxValue = np.max(OutPreSame)
if (np.min(OutPreSame) < minValue):
minValue = np.min(OutPreSame)
PreSigOther = OutPreOther[test.OutTrue == 1]
SigWOther = test.Weights[test.OutTrue == 1]
PreBkgOther = OutPreOther[test.OutTrue == 0]
BkgWOther = test.Weights[test.OutTrue == 0]
PreSigSame = OutPreSame[train.OutTrue == 1]
SigWSame = train.Weights[train.OutTrue == 1]
PreBkgSame = OutPreSame[train.OutTrue == 0]
BkgWSame = train.Weights[train.OutTrue == 0]
c1 = ROOT.TCanvas("c1", "Canvas", 800, 600)
ROOT.gStyle.SetOptStat(0)
hSigOther = ROOT.TH1F("h1", "", 40, minValue, maxValue)
hBkgOther = ROOT.TH1F("h2", "", 40, minValue, maxValue)
hSigSame = ROOT.TH1F("h3", "", 40, minValue, maxValue)
hBkgSame = ROOT.TH1F("h4", "", 40, minValue, maxValue)
fill_hist(hSigOther, PreSigOther, weights=SigWOther)
fill_hist(hBkgOther, PreBkgOther, weights=BkgWOther)
fill_hist(hSigSame, PreSigSame, weights=SigWSame)
fill_hist(hBkgSame, PreBkgSame, weights=BkgWSame)
hSigOther.Scale(1 / hSigOther.GetSumOfWeights() / Getdx(hSigOther))
hBkgOther.Scale(1 / hBkgOther.GetSumOfWeights() / Getdx(hBkgOther))
hSigSame.Scale(1 / hSigSame.GetSumOfWeights() / Getdx(hSigSame))
hBkgSame.Scale(1 / hBkgSame.GetSumOfWeights() / Getdx(hBkgSame))
hBkgSame.SetMarkerColor(2)
hBkgSame.SetLineColor(2)
hBkgSame.SetMarkerSize(0.7)
hBkgSame.SetMarkerStyle(21)
hSigSame.SetMarkerColor(4)
hSigSame.SetMarkerSize(0.7)
hSigSame.SetMarkerStyle(21)
hSigOther.GetXaxis().SetTitle('Score')
hBkgOther.SetLineColor(ROOT.kRed)
if (hSigOther.GetMaximum() > hBkgSame.GetMaximum()):
hSigOther.GetYaxis().SetRangeUser(0, hSigOther.GetMaximum() * 1.4)
else:
hSigOther.GetYaxis().SetRangeUser(0, hBkgOther.GetMaximum() * 1.4)
hSigOther.Draw("Hist")
hBkgOther.Draw("HistSame")
hSigSame.Draw("E1 Same")
hBkgSame.Draw("E1 Same")
T1 = ROOT.TText()
T2 = ROOT.TText()
T1.SetTextSize(0.03)
T2.SetTextSize(0.03)
T1.SetTextAlign(21)
T2.SetTextAlign(21)
T1.DrawTextNDC(.23, .87,
r"AUC = " + str(round(AucOther, 3)) + " (on test)")
T2.DrawTextNDC(.23, .84,
r"AUC = " + str(round(AucSame, 3)) + " (on train)")
leg = ROOT.TLegend(0.6, 0.75, 0.9, 0.9)
leg.AddEntry(hSigSame, "Signal (on train sample)")
leg.AddEntry(hBkgSame, "Background (on train sample)")
leg.AddEntry(hSigOther, "Signal (on test sample)")
leg.AddEntry(hBkgOther, "Background (on test sample)")
leg.Draw()
c1.SaveAs(path + "Score" + Name + ".png")
return AucOther, AucSame