def main() : data_plots = [] mc_plots = [] for i in range(5) : data_plots.append(ROOT.TH1F('a_%d'%(i),'Data',48,-6,6)) mc_plots .append(ROOT.TH1F('b_%d'%(i),'MC' ,48,-6,6)) mc_plots[-1].SetLineColor(ROOT.kRed+1) for plot in data_plots + mc_plots : plot.Sumw2() plot.SetLineWidth(2) rand = ROOT.TRandom3(1) for i in range(100000) : for j in range(len(data_plots)) : data_plots[j].Fill(rand.Gaus(0,1) + j*0.2) mc_plots [j].Fill(rand.Gaus(0,1) + j*0.15) # # Bin labeling # bin_edges = [15,20,25,30,40,50] template = '%s^{ }<^{ }p_{T}^{ }<^{ }%s GeV' labels = list(template%(bin_edges[a],bin_edges[a+1]) for a in range(len(bin_edges)-1)) mycanvas = ROOT.TCanvas('mycanvas','blah',600,500) # # The main call: # ShowerShapeEvolutionPlot(mycanvas,labels,data_plots,mc_plots) mycanvas.SetGridx() plotfunc.DrawText(mycanvas,[plotfunc.GetAtlasInternalText(status='Internal') ,plotfunc.GetSqrtsText(13)+', '+plotfunc.GetLuminosityText() ],0.27,0.88,0.57,0.99,totalentries=2) mycanvas.Print(mycanvas.GetName()+'.pdf') raw_input('Press enter to exit') return
def Predict_Digits(net, lstT, plotDigit=True, onlyDigit=-1): nRow, nCol = 2, 5 digitNum = nRow * nCol digitFigs, pxlW, pltInchW = Initial_PlotParams(len(lstT[0][0]), nRow, nCol) digitTitles = ["" for a in digitFigs] digitId = 0 # 隨機測試某筆數字 ---------------------------------------------- start = time.time() sampleNum = 10000 # 不含繪圖,辨識 10000張,費時 1.3 秒,平均每張 0.00013秒 plotNum = 5 iPlot = 0 plotMod = int(sampleNum / plotNum) + 1 correctNum = 0 failNum = 0 for i in range(0, sampleNum): if (lstT[i][1] == onlyDigit) or (onlyDigit < 0): doPlot = (i % plotMod == 0) aId = np.random.randint(0, len(lstT)) label, result, outputY = net.Predict_Digit(lstT[aId], False) if label == result: correctNum += 1 else: doPlot = iPlot < plotNum #(failNum<10) failNum += 1 if doPlot and plotDigit: # Plot_Digit(lstT[aId], result, label) # print("({}): Label={}, Predict={} ( {:.5f} ) -> {} ". # format(i, label,result, outputY, sResult[int(label==result)])) s1 ="{}={},({:.3f})->{}".\ format(label, result, outputY,sResult[int(label==result)] ) digitId = digitId % digitNum digitFigs[digitId] = np.array( lstT[aId][0]).transpose().reshape(pxlW, pxlW) * 255 digitTitles[digitId] = s1 digitId += 1 if (digitId == digitNum): pltFn.Plot_Images(np.array(digitFigs), nRow, nCol, digitTitles, "", pltInchW) iPlot += 1 digitId = 0 dt = time.time() - start accurRatio = correctNum / sampleNum print("\nAccuracy({:.3f}), {}/{}(Correct/Total)".format( accurRatio, correctNum, sampleNum)) print("Elapsed(seconds)) : {:.3f} sec.\n".format(dt)) return accurRatio, dt
def RatioRangeAfterBurner(can,ymin=None,ymax=None) : import PlotFunctions as plotfunc import TAxisFunctions as taxisfunc import ROOT # Set ratio range; add dotted line at 1 if not plotfunc.GetBotPad(can) : return isPull = False for hist in plotfunc.GetBotPad(can).GetListOfPrimitives() : if hasattr(hist,'GetYaxis') : isPull = isPull or (hist.GetYaxis().GetTitle() == 'pull') if ymin == None : if isPull : ymin,ymax = -4,4 else : ymin,ymax = 0,2 taxisfunc.SetYaxisRanges(plotfunc.GetBotPad(can),ymin,ymax) if ymax <= 1 : return xmin,xmax = None,None for i in plotfunc.GetBotPad(can).GetListOfPrimitives() : if issubclass(type(i),ROOT.TH1) : xmin = i.GetXaxis().GetBinLowEdge(1) xmax = i.GetXaxis().GetBinLowEdge(i.GetNbinsX()+1) if xmin != None : lineh = 0 if isPull else 1 line = ROOT.TLine(xmin,lineh,xmax,lineh) line.SetLineStyle(2) plotfunc.GetBotPad(can).cd() line.Draw() plotfunc.tobject_collector.append(line) return
def Test_Encoder_Decoder(encoder, decoder, lstT, sampleNum=10, saveImgPath="", noiseStrength=0.0): # 隨機測試某筆數字 ---------------------------------------------- #start = time.time() pxlW = int(np.sqrt(len(lstT[0][0]))) # 不含繪圖,辨識 10000張,費時 1.3 秒,平均每張 0.00013秒 for i in range(0, sampleNum): aId = np.random.randint(0, len(lstT)) digitId = 0 if type(lstT[aId][1]) == np.array: if len(lstT[aId][1]) == 10: digitId = np.argmax(lstT[aId][1]) else: digitId = -1 elif type(lstT[aId][1]) == np.int64: digitId = lstT[aId][1] oneInput = lstT[aId][0] #rf.Plot_Digit([oneInput.transpose(), lstT[aId][1] ] ) #(1x784) if (noiseStrength > 0.0): oneInput = rn.RvBaseNeuralNetwork.Add_Noise( oneInput, noiseStrength) for j in range(1): # 多做幾次,效果不會比較好 encode = encoder.Get_OutputValues(oneInput) #(784x1) #output = decoder.Plot_Output(encode) #(784x1) output = decoder.Get_OutputValues(encode) # print("Input({}) -> Output : Accuracy={:.3f}". # format(digitId, decoder.Get_Accuracy_EnDeCoder(oneInput, output))) if rfi.PathExists(saveImgPath): imgFn = "{}vdoImg_{}_{}.png".format(saveImgPath, i, j) else: imgFn = "" pltFn.Plot_Images( np.array([ oneInput.transpose().reshape(pxlW, pxlW) * 255, output.transpose().reshape(pxlW, pxlW) * 255 ]), 1, 2, ["({}) Input".format(digitId), "Output"], imgFn) print("") oneInput = output
def ShowDrifttimeVsTime(self, Channel=-1): YMax = np.max( [np.max(self.Ch[ii].GradTime) for ii in range(self.NumChannels)]) YMax = self.RoundUpToNext(YMax, 10) YTicks = self.RoundDownToNext(YMax / 5, 1) Plt.PltTime(Time=self.Ch[0].TimeStamp, Data=[ self.Ch[0].GradTime, self.Ch[1].GradTime, self.Ch[1].GradTime - self.Ch[0].GradTime ], Legend=['Anode', 'Cathode', 'Drift Time'], Label='Time since Trigger [$\mu$s]', XTicks=self.XTicks, YTicks=YTicks, YRange=[0, YMax], SaveName='drift_time', Save=False)
def ShowAmplitudeVsTime(self, Channel=-1): YMax = np.max( [np.max(self.Ch[ii].Max) for ii in range(self.NumChannels)]) YMax = self.RoundUpToNext(YMax, 100) YTicks = self.RoundDownToNext(YMax / 5, 100) print(YTicks) Plt.PltTime( Time=self.Ch[0].TimeStamp, Data=[self.Ch[0].Max, self.Ch[1].Max, self.ChargeCollection * 100], Legend=['Anode', 'Cathode', 'Charge Collection [\%]'], Label='Amplitude [mV]', XTicks=self.XTicks, YTicks=YTicks, YRange=[0, YMax], SaveName='amp_ratio', Title='Charge Collection in Vacuum at 60 V/cm', Save=False)
def main(options, args): plotfunc.SetupStyle() file_d, tree_d, key_d = anaplot.GetChainFromFiles( options.data, treename=options.treename) dcuts = ' && '.join(options.cuts + options.blindcut) if not os.path.exists(options.outdir): os.makedirs(options.outdir) from array import array categories = '.'.join(CouplingsHelpers.categories) print categories for k in key_d: ROOT.makePicoXaod_Categories(tree_d[k], k, dcuts, options.outdir, categories) print 'done.' return
import numpy import PlotFunctions H_t = numpy.load('H_t10.npy') H_e = numpy.load('H_e10.npy') Y_lstm2 = numpy.load('Y_ST2B.npy') t_train = numpy.arange(0, len(H_t)) t_test = numpy.arange(len(H_t), len(H_t) + len(Y_lstm2)) t_array = numpy.arange(0, len(Y_lstm2)) PlotFunctions.Plot_triple(t_train, H_t, t_test, Y_lstm2, t_test, H_e, 'Training Data', 'Approach I predictions', 'Test Data (actual)', 'k-', 'r-', 'b-', "fig_sample.eps")
for i in range(len(ClusterData[day].idx)): y_NN[ClusterData[day].idx[i] - 1] = y_pred[i] y_sim[ClusterData[day].idx[i] - 1] = Y_test[i] #Outputs print y_NN print y_sim print y_deep e_rms = (MathFunctions.rms_flat(y_NN - y_sim)) / (MathFunctions.rms_flat(y_sim)) e_deep = (MathFunctions.rms_flat(y_deep - y_sim)) / (MathFunctions.rms_flat(y_sim)) print e_rms print e_deep PlotFunctions.PlotEnergy(y_NN, y_sim, y_deep) j1, j2 = DataFunctions.find_day_idx(4) PlotFunctions.PlotEnergyDaily(y_NN[j1:j2], y_sim[j1:j2], y_deep[j1:j2]) j1, j2 = DataFunctions.find_day_idx(180) PlotFunctions.PlotEnergyDaily(y_NN[j1:j2], y_sim[j1:j2], y_deep[j1:j2]) #debug #print Sch #print Sch_test #print Sch[0:47, 1] #print weather_idxMax #print Sch_idxMax #print j #print ClusterData[0].X_train[0:23, :] #print ClusterData[1].X_train[0:23, :] #print ClusterData[2].X_train[0:23, :] #mlab.start()
IVALU_RESULTS = IVALU_DATA[IVALU_DATA['2MASS'].isin( IGRINS_RESULTS['2MASS'].values)] APOGEE_RESULTS = pd.read_csv(homedir + 'APOGEE_DATA.txt', names=APOGEE_Header, delim_whitespace=True) OPTICAL_DIF = IGRINS_RESULTS.drop(DROP_LIST, axis=1) - OPTICAL_RESULTS.drop( DROP_LIST, axis=1) DATA = [ IGRINS_RESULTS, OPTICAL_RESULTS, IVALU_RESULTS, APOGEE_RESULTS, OPTICAL_COMPLETE ] Comparison = 'ALL' pl.Comp(Comparison) OPTICAL_RESULTS.name = 'Optical' OPTICAL_COMPLETE.name = 'OComplete' IVALU_RESULTS.name = 'Ivalu' IGRINS_RESULTS.name = 'Igrins' APOGEE_RESULTS.name = 'Apogee' while True: if diagFlag == False: pl.PickElement() element = pl.element if element == 'quit': print('Plotting stopped') break elif (element in PlotInput) and (diagFlag == False):
t_array = numpy.arange(0, len(Y_lstm2)) e_deep = (MathFunctions.rms_flat(Y_lstm2 - H_e))/(MathFunctions.rms_flat(H_e)) e_deep2 = (MathFunctions.rms_flat(Y_lstm2 - H_e))/(MathFunctions.rms_flat(H_t)) print Y_lstm2.shape print H_e.shape print t_array.shape print e_deep print e_deep2 numpy.save('3_LSTM.npy', Y_lstm2) ####Plotting for min-wise PlotFunctions.Plot_double(t_array, H_e, t_array, Y_lstm2, 'Actual conv power','LSTM conv power', 'k-', 'r-', "fig_5houseI.eps") ##################### ###Build 1-min model: def objective(params): #optimize_model = build_lstm_v1.lstm_multi_101(params, train_data2.shape[2], lstm_h1, 60) #Check code here, relu entering optimize_model = build_lstm_v1.lstm_model_110(params, train_data2.shape[2], 60) loss_out = NNFunctions.model_optimizer_101(optimize_model, train_data2, H_t2, val_data2, H_val2, 6) return {'loss': loss_out, 'status': STATUS_OK} trials = Trials() best2 = fmin(objective, space2, algo=tpe.suggest, trials=trials, max_evals=3)
elif choice == 2: X_t, Y_t, X_e, Y_e, train_list, test_list = InterpolateFunctions.train_test_split( train_data, H_t) best = InterpolateFunctions.imputate_optimize(X_t, Y_t) Y_p = NNFun_PSB.PSB_model_DL(X_t, Y_t, X_e, best) H_t = InterpolateFunctions.organize_pred(H_t, Y_p, test_list) #numpy.save('Ht_file_HVAC1.npy', H_t) else: H1 = H_t.copy() small_list, large_list = InterpolateFunctions.interpolate_main( train_data, H_t, start_day, end_day, cons_points) H_t = InterpolateFunctions.interp_linear( H_t, small_list) #H_t is beign changed as numpy arrays are mutable H1 = InterpolateFunctions.interp_linear(H1, small_list) PlotFunctions.Plot_interp_params() H_t = H_t[:, None] #changing numpy array shape to fit the function Y_t, Y_NN = InterpolateFunctions.interp_LSTM_v2(train_data, H_t, large_list) PlotFunctions.Plot_interpolate(H1[s0 * 24:start_day * 24], Y_t[start_day * 24:end_day * 24], Y_NN[start_day * 24:end_day * 24], H1[start_day * 24:end_day * 24], H1[end_day * 24:s1 * 24]) e_interp = InterpolateFunctions.interpolate_calculate_rms( H1[start_day * 24:end_day * 24], Y_t[start_day * 24:end_day * 24]) e_NN = InterpolateFunctions.interpolate_calculate_rms( H1[start_day * 24:end_day * 24], Y_NN[start_day * 24:end_day * 24]) print e_interp
# prvaite libraries--------------------------------------------- import mnist_loader import RvNeuralNetworks as rn from RvNeuralNetworks import * import RvAskInput as ri import RvMiscFunctions as rf import RvNeuNetworkMethods as nm import PlotFunctions as pltFn #%% #Load MNIST **************************************************************** #mnist.pkl.gz(50000) Accuracy 0.96 #mnist_expanded.pkl.gz(250000) Accuracy 0.97 fn = "..\\data\\mnist.pkl.gz" #".datamnist_expanded.pkl.gz" lstTrain, lstV, lstT = mnist_loader.load_data_wrapper(fn) lstTrain = list(lstTrain) lstV = list(lstV) lstT = list(lstT) fns = [] for file in os.listdir(".\\"): if file.endswith(".txt"): fns.append(file) aId = ri.Ask_SelectItem("Select Network file", fns, 0) fn1 = fns[aId] accur1, t1 = pltFn.Predict_Digits_FromNetworkFile(fn1, lstT, True) print("File : \"{}\"\n nmcu:{}, Time:{:.3} sec\n".format(fn1, accur1, t1))
def main(options, args): plotfunc.SetupStyle() files_b, trees_b, keys_b = anaplot.GetTreesFromFiles( options.bkgs, treename=options.treename) files_s, trees_s, keys_s = anaplot.GetTreesFromFiles( options.signal, treename=options.treename) file_d, tree_d, key_d = anaplot.GetChainFromFiles( options.data, treename=options.treename) scales_b = anaplot.GetScales(files_b, trees_b, keys_b, options) scales_s = anaplot.GetScales(files_s, trees_s, keys_s, options) dweight = '' # weight value (and cuts) applied to data weight = options.weight if ''.join(options.cuts): weight = (weight + '*(%s)' % (' && '.join(options.cuts + options.truthcuts))).lstrip('*') dweight = '(' + ' && '.join(options.cuts + options.blindcut) + ')' cans = [] v1 = 'HGamEventInfoAuxDyn.m_yy/1000' v2 = 'HGamEventInfoAuxDyn.catCoup_Moriond2017BDT' bkg_hists = [] sig_hists = [] data_hist = None if options.data: data_hist = anaplot.Get2dVariableHistsFromTrees( tree_d, key_d, v1, v2, dweight, options)[0] data_hist.SetLineWidth(2) data_hist.SetLineColor(1) data_hist.SetMarkerColor(1) if options.bkgs: bkg_hists = anaplot.Get2dVariableHistsFromTrees(trees_b, keys_b, v1, v2, weight, options, scales=scales_b) bkg_hists, keys_b = anaplot.MergeSamples(bkg_hists, options) anaplot.PrepareBkgHistosForStack(bkg_hists, options) if options.signal: sig_hists = anaplot.Get2dVariableHistsFromTrees(trees_s, keys_s, v1, v2, weight, options, scales=scales_s) sig_hists, keys_s = anaplot.MergeSamples(sig_hists, options) sig_hists[-1].SetLineColor(2) sig_hists[-1].SetMarkerColor(2) # get the histograms from the files for c in range(len(CouplingsHelpers.categories)): #if not categories[c] : continue lo_bin = c + 1 hi_bin = c + 1 if c == 0: # inclusive case. lo_bin = 0 hi_bin = 10000 # just to be safe, 10k categories # MERGE VH DILEP CATEGORIES (23 and 24 --> 23): if CouplingsHelpers.categories[c] == 'M17_VHdilep_LOW': hi_bin = hi_bin + 1 # MERGE VH MET HIGH and BSM CATEGORIES (19 and 20 --> 19): if CouplingsHelpers.categories[c] == 'M17_VHMET_HIGH': hi_bin = hi_bin + 1 name = 'c%d_%s' % (c, CouplingsHelpers.categories[c]) bkg_projs = [] sig_projs = [] data_proj = None if options.data: data_proj = data_hist.ProjectionX('%s_data' % (name), lo_bin, hi_bin) PyHelpers.PrintNumberOfEvents(data_proj) if options.bkgs: for i, b in enumerate(bkg_hists): bkg_projs.append( b.ProjectionX('%s_%s' % (name, keys_b[i]), lo_bin, hi_bin)) PyHelpers.PrintNumberOfEvents(bkg_projs[-1]) if options.signal: for i, s in enumerate(sig_hists): sig_projs.append( s.ProjectionX('%s_%s' % (name, keys_s[i]), lo_bin, hi_bin)) PyHelpers.PrintNumberOfEvents(sig_projs[-1]) cans.append( anaplot.DrawHistos(v1, options, bkg_projs, sig_projs, data_proj, name=name)) cans[-1].SetName(anaplot.CleanUpName(name)) anaplot.UpdateCanvases(options, cans) if not options.batch: raw_input('Press enter to exit') anaplot.doSaving(options, cans) # Do this afterwards, to make sure the outdir exists. f = ROOT.TFile('%s/couplings.root' % (options.outdir), 'RECREATE') for can in cans: for i in can.GetListOfPrimitives(): if i.GetName()[-6:] == '_error': continue if 'stack' in i.GetName(): if not issubclass(type(i), ROOT.THStack): continue for j in range(i.GetNhists()): print 'writing from stack:', i.GetHists()[j].GetName() i.GetHists()[j].Write() if issubclass(type(i), ROOT.TH1): print 'writing:', i.GetName() i.Write() f.Close() print 'done.' return
digitFigs = [[] for i in range(10)] nCol = 5 #int(np.sqrt(len(digitFigs))) nRow = 2 #int(len(digitFigs)/nCol)+1 pxls = outputNum pxlW = int(np.sqrt(pxls)) pxls = pxlW * pxlW dpi = 72 zoom = 8 # 28 pxl * zoom pltInchW = pxlW / dpi * nCol * zoom if (None != decoder): #and (None!=encoder): for i in range(sampleLoop): digitId = 0 for j in range(10): encode = randomState.randn(inputNum, 1) encode = np.maximum(0.2, np.minimum(0.8, encode)) output = decoder.Get_OutputValues(encode) digitFigs[j] = output.transpose().reshape(pxlW, pxlW) * 255 print("({}): code -> Output".format(i)) if pathExists: imgFn = "{}vdoImg_{}.png".format(imgPath, i) else: imgFn = "" pltFn.Plot_Images(np.array(digitFigs), nRow, nCol, ["Test DeCoder Fake Image"], imgFn, pltInchW) aviFn = "{}{}".format(imgPath, "Decoder.avi") durationSec = 0.5 #min(2.0, 10/sampleLoop) if ru.ImageFilesToAvi(imgPath, aviFn, durationSec): rfi.OpenFile(aviFn)
#%% #使用 mnist_expanded.pkl.gz(250000筆) 準確率提高到 0.97 fn = ".datamnist.pkl.gz" #".datamnist_expanded.pkl.gz" lstTrain, lstV, lstT = mnist_loader.load_data_wrapper(fn) lstTrain = list(lstTrain) lstV = list(lstV) lstT = list(lstT) fnNetworkData1 = ".\\{}_NetData_DontDelete.txt".format( rn.RvNeuralNetwork.__name__) fnNetworkData2 = ".\\{}_NetData_DropOut.txt".format( rn.RvNeuralNetwork.__name__) fnNetworkData3 = ".\\{}_NetData_CnvLyr.txt".format(rn.RvNeuralNetwork.__name__) # accur1, t1 = pltFn.Predict_Digits_FromNetworkFile(fnNetworkData1, lstT, False) accur2, t2 = pltFn.Predict_Digits_FromNetworkFile(fnNetworkData2, lstT, False) accur3, t3 = pltFn.Predict_Digits_FromNetworkFile(fnNetworkData3, lstT) print("FullConnected Layer:\n Accu:{}, Time:{:.3} sec\n".format(accur1, t1)) print("DropOut Layer:\n Accu:{}, Time:{:.3} sec\n".format(accur2, t2)) print("Convolution Layer:\n Accu:{}, Time:{:.3} sec\n".format(accur3, t3)) """ if (os.path.isfile(fnNetworkData1)): print("\n讀取網路參數檔案以新建網路 - Create_NetworkData():\n") net = rn.RvNeuralNetwork.Create_Network(fnNetworkData1) # 預測所有測試集------------------------------------------------ correctNum,n_test = net.Evaluate_Accuracy(lstT) print("\n預測結果: 正確率({:.3f}), {}/{}(正確/總數)". format(correctNum/n_test, correctNum,n_test))
def main(options, args): style = plotfunc.SetupStyle() style.SetPadRightMargin(0.16) files_b, trees_b, keys_b = anaplot.GetTreesFromFiles( options.bkgs, treename=options.treename, xAODInit=options.xAODInit) files_s, trees_s, keys_s = anaplot.GetTreesFromFiles( options.signal, treename=options.treename, xAODInit=options.xAODInit) files_d, trees_d, keys_d = anaplot.GetTreesFromFiles( options.data, treename=options.treename, xAODInit=options.xAODInit) scales_b = anaplot.GetScales(files_b, trees_b, keys_b, options) scales_s = anaplot.GetScales(files_s, trees_s, keys_s, options) weight = options.weight if ''.join(options.cuts + options.truthcuts): weight = (weight + '*(%s)' % (' && '.join(options.cuts + options.truthcuts).lstrip('& ').rstrip('& ')) ).lstrip('*') dweight = '' # weight value (and cuts) applied to data if ''.join(options.cuts + options.blindcut): dweight = '(' + ' && '.join(options.cuts + options.blindcut).lstrip( '& ').rstrip('& ') + ')' cans = [] for v in options.variables: if v not in options.histformat.keys() or len( options.histformat[v]) < 4: print 'Warning: need to set the label of %s using the histformat option.' % ( v) # get the histograms from the files for vi, v1 in enumerate(options.variables): labelv1 = v1 if v1 in options.histformat.keys() and len( options.histformat[v1]) >= 4: labelv1 = options.histformat[v1][3] for vj, v2 in enumerate(options.variables): if v2 == v1: continue if vj < vi: continue labelv2 = v2 if v2 in options.histformat.keys() and len( options.histformat[v2]) >= 4: labelv2 = options.histformat[v2][3] bkg_hists = [] sig_hists = [] data_hist = None if options.data: data_hists = anaplot.Get2dVariableHistsFromTrees(trees_d, keys_d, v1, v2, dweight, options, files=files_d) data_hist = anaplot.MergeSamples(data_hists, options, requireFullyMerged=True)[0] canname = anaplot.CleanUpName('%s_%s_%s' % (v1, v2, 'data')) cans.append(ROOT.TCanvas(canname, canname, 600, 500)) data_hist.SetMinimum(-0.00001) plotfunc.AddHistogram(cans[-1], data_hist, drawopt='colz') plotfunc.SetAxisLabels(cans[-1], labelv1, labelv2) if options.bkgs: bkg_hists = anaplot.Get2dVariableHistsFromTrees( trees_b, keys_b, v1, v2, weight, options, scales=scales_b, files=files_b) bkg_hists = anaplot.MergeSamples(bkg_hists, options) for i, h in enumerate(bkg_hists): canname = anaplot.CleanUpName('%s_%s_%s' % (v1, v2, keys_b[i])) cans.append(ROOT.TCanvas(canname, canname, 600, 500)) h.SetMinimum(-0.00001) plotfunc.AddHistogram(cans[-1], h, drawopt='colz') plotfunc.SetAxisLabels(cans[-1], labelv1, labelv2) if options.signal: sig_hists = anaplot.Get2dVariableHistsFromTrees( trees_s, keys_s, v1, v2, weight, options, scales=scales_s, files=files_s) sig_hists = anaplot.MergeSamples(sig_hists, options) for i, h in enumerate(sig_hists): canname = anaplot.CleanUpName('%s_%s_%s' % (v1, v2, keys_s[i])) cans.append(ROOT.TCanvas(canname, canname, 600, 500)) h.SetMinimum(-0.00001) plotfunc.AddHistogram(cans[-1], h, drawopt='colz') plotfunc.SetAxisLabels(cans[-1], labelv1, labelv2) if options.afterburner: for can in cans: options.afterburner(can) anaplot.UpdateCanvases(cans, options) if options.xAODInit: ROOT.xAOD.ClearTransientTrees() if not options.batch: import code code.interact(banner='Pausing... Press Contol-D to exit.', local=locals()) anaplot.doSaving(options, cans) print 'done.' return
#%% label = "Max Risk Adjusted Return Portfolio" mu = port.mu cov = port.cov returns = port.returns ax = plf.plot_frontier( w_frontier=w_5, mu=mu, cov=cov, returns=returns, rm=rm, rf=0, alpha=0.01, cmap="viridis", w=w1, label="Portfolio", marker="*", s=16, c="r", height=6, width=10, ax=None, ) # ax = plf.plot_pie(w=w1, title='Portafolio', height=6, width=10, cmap="tab20", # ax=None) # # # w0 = pd.DataFrame(np.ones((100,1))/100) # # # ax = plf.plot_pie(w=w0, title='Portafolio', height=6, width=10, cmap = "tab20", ax=None)
#######Training data: 2015 #EnergyData date_start = '6/1/15 12:00 AM' date_end = '6/30/15 11:59 PM' std_inv = 60 #in minutes #Read data data = DataFunctions.read_PSB_csv(date_start, date_end, 5, 20) conv_critical, crac_critical, crac_normal, conv_normal, HVAC_critical, Elev, HVAC_normal, elec_total = DataFunctions.PSB_elec_split( data) H_t = DataFunctions.fix_data(conv_normal) H_t = DataFunctions.fix_energy_intervals(H_t, 5, std_inv) PlotFunctions.Plot_single(H_t) #schedules X_sch_t = DataFunctions.compile_features(H_t, date_start, std_inv) #Weather Data weather_file = r'/home/sseslab/Documents/SLC PSB data/WBB Weather Data/WBB_2015_June.csv' weather_train = DataFunctions.read_weather_csv(weather_file, date_start, date_end) weather_train = DataFunctions.fix_weather_intervals(weather_train, 5, std_inv) weather_min, weather_max = DataFunctions.get_normalize_params(weather_train) #weather_train = DataFunctions.normalize_vector(weather_train, weather_min, weather_max) weather_train = DataFunctions.interpolate_nans(weather_train) X_t, Y_t, X_e, Y_e, train_list, test_list = InterpolateFunctions.train_test_split( X_sch_t, H_t) best = InterpolateFunctions.imputate_optimize(X_t, Y_t)
H_e = numpy.reshape(H_e, (H_e.shape[0] * 24, 1)) Y_lstm = numpy.reshape(Y_lstm, (Y_lstm.shape[0] * 24, 1)) Y_lstm2 = numpy.reshape(Y_lstm2, (Y_lstm2.shape[0] * 24, 1)) t_train = numpy.arange(0, len(H_t)) t_test = numpy.arange(len(H_t), len(H_t) + len(Y_lstm2)) t_array = numpy.arange(0, len(Y_lstm2)) e_deep = (MathFunctions.rms_flat(Y_lstm2 - H_e)) / (MathFunctions.rms_flat(H_e)) e_deep2 = (MathFunctions.rms_flat(Y_lstm2 - H_e)) / (MathFunctions.rms_flat(H_t)) print e_deep print e_deep2 PlotFunctions.Plot_double(t_array, H_e, t_array, Y_hour, 'Actual conv power', 'LSTM conv power', 'k-', 'r-', "fig_77i1a.eps") ##################### ###Build 1-min model: def objective(params): optimize_model = build_lstm_v1.lstm_model_110(params, train_data2.shape[2], 60) loss_out = NNFunctions.model_optimizer_101(optimize_model, train_data2, H_t2, val_data2, H_val2, 5) return {'loss': loss_out, 'status': STATUS_OK} trials = Trials() best2 = fmin(objective, space2, algo=tpe.suggest, trials=trials, max_evals=1)
def main(options, args): plotfunc.SetupStyle() # The nominal case, where you use an algorithm inside the predefined file macroFileName = 'PicoXaodSkimAlgos.h' algName = options.alg # If a colon is used, it is to specify a different file path if ':' in options.alg: macroFileName = options.alg.split(':')[0] algName = options.alg.split(':')[-1] isLoaded = ROOT.gROOT.LoadMacro(macroFileName) if (isLoaded < 0): print 'Error! Macro compilation of file %s failed. See error msgs.' % ( macroFileName) sys.exit() else: print 'Loaded macro file %s' % (macroFileName) print 'Skimming using the \"%s\" algorithm...' % (algName) # Here "b" stands for simulation files_b, trees_b, keys_b = anaplot.GetTreesFromFiles( options.bkgs, treename=options.treename) files_d, trees_d, keys_d = anaplot.GetTreesFromFiles( options.data, treename=options.treename) simcuts = '' if ''.join(options.cuts + options.truthcuts): simcuts = '(' + ' && '.join(options.cuts + options.truthcuts) + ')' dcuts = '' # cuts applied to data if ''.join(options.cuts + options.blindcut): dcuts = '(' + ' && '.join(options.cuts + options.blindcut) + ')' if not os.path.exists(options.outdir): os.makedirs(options.outdir) for k in keys_d: outfilename = os.path.basename(files_d[k].GetName()).replace( '.root', '') outfilename = '%s_pico' % (outfilename) outfilename = anaplot.CleanUpName(outfilename, originalIsDirectoryName=True, forFileName=True) histname = anaplot.CleanUpName(k, originalIsDirectoryName=True) print 'Making picoXaod for %s' % (k) getattr(ROOT, algName)(files_d[k], trees_d[k], histname, dcuts, ','.join(options.variables), options.outdir, outfilename) for k in keys_b: outfilename = os.path.basename(files_b[k].GetName()).replace( '.root', '') outfilename = '%s_pico' % (outfilename) outfilename = anaplot.CleanUpName(outfilename, originalIsDirectoryName=True, forFileName=True) histname = anaplot.CleanUpName(k, originalIsDirectoryName=True) print 'Making picoXaod for %s' % (k) getattr(ROOT, algName)(files_b[k], trees_b[k], histname, simcuts, ','.join(options.variables), options.outdir, outfilename) print 'done.' return
def main_singleCategory(options, args): if options.category in [30, 31]: print 'Error! Too few AF2 stats. Not going to do it.' return cans = [] options.outdir = '' if options.functions: flist = options.functions.split(',') options.outdir += '_'.join(flist) elif options.family == 'official': #flist = ['Exponential','ExpPoly2','ExpPoly3','Bern4','Bern5','Pow','Pow2','Laurent0','Laurent1','Laurent2'] flist = [ 'Exponential', 'ExpPoly2', 'Bern4', 'Bern5', 'Pow', 'Pow2', 'Laurent1', 'Laurent2' ] options.outdir += 'official_functions' elif options.family == 'selected': flist = [Tools.selected[Tools.categories[options.category]]] options.outdir += 'selected_functions' print flist elif options.family == 'ExpPoly': flist = ['Exponential'] + list('ExpPoly%d' % (d) for d in range(2, 4)) options.outdir += 'exppoly_family' elif options.family == 'Laurent': flist = list('Laurent%d' % (d) for d in range(0, 3)) options.outdir += 'Laurent_family' elif options.family == 'PolyOverX4': flist = ['1/x^4'] + list('poly%d/x^4' % (d) for d in range(1, 6)) options.outdir += 'PolyOverX4_family' elif options.family == 'Bernstein': flist = list('Bern%d' % (d) for d in range(4, 6)) options.outdir += 'Bern_family' elif options.family == 'PowerSum': flist = ['Pow', 'Pow2'] options.outdir += 'PowerSum_family' else: flist = [ 'Exponential', # bad 'ExpPoly2', 'ExpPoly3', ] options.outdir += '_'.join(flist) options.outdir += '_c%02d_%s' % (options.category, Tools.categories[options.category]) functions = [] Tools.PopulateFunctionList(functions, flist) Tools.LinkFunctionsForFtest(functions) if len(functions) == 0: print 'Error! no functions loaded!' import sys sys.exit() for f in functions: f.SetCategory(options.category) f.SetFileName(options.file) f.SetSignalWS(options.signalws) f.Initialize() ## ## Plot the data and af2 (fit bkg-only just before this.) SpuriousSignal_05_M17_ggH_1J_BSM ## for f in functions: f.function.fitTo(f.data, *(Tools.args_bkgonly)) cans.append( plotfunc.RatioCanvas( "TemplateFit_%02d_%s" % (options.category, Tools.categories[options.category]), "main plot", 600, 500)) functions[0].af2hist.SetMarkerSize(0) #rebin = Tools.RebinUntilSmallErrors(functions[0].af2hist,0,Tools.lower_range,Tools.upper_range,errormax=0.3) rebin = 5 functions[0].af2hist.Rebin(rebin) binwidth = functions[0].af2hist.GetBinWidth(1) bins = int((Tools.upper_range - Tools.lower_range) / float(binwidth)) functions[0].af2hist.SetTitle('#gamma#gamma MC') functions[0].af2hist.SetLineWidth(2) if options.family == 'selected': functions[0].af2hist.SetLineColor(ROOT.kGray + 2) functions[0].af2hist.SetFillColor(0) plotfunc.AddHistogram(cans[-1], functions[0].af2hist, drawopt='') for i, f in enumerate(functions): f.obsVar.setBins(int(bins)) f.bins = bins f.frame = f.obsVar.frame() if i: f.af2hist.Rebin(rebin) f.af2_rebinned = ROOT.RooDataHist('af2_rebinned', '', ROOT.RooArgList(f.obsVar), f.af2hist, 1.) #f.af2_rebinned.plotOn(f.frame,ROOT.RooFit.Range("lower,upper")) #f.function.plotOn(f.frame,ROOT.RooFit.NormRange("lower,upper"),ROOT.RooFit.Range("all")) f.chisquare = Tools.GetChiSquare_ForSpuriousSignal( f.frame, f.af2_rebinned, f.function, f.ndof) f.pvalue_chi2 = ROOT.TMath.Prob(f.chisquare * (f.bins - 1 - f.ndof), f.bins - 1 - f.ndof) print 'Original chi2: %2.6f p-value: %2.6f' % (f.chisquare, f.pvalue_chi2) f.minNll = 0 curve = f.frame.getCurve() curve.SetMarkerSize(0) curve.SetLineWidth(1) curve.SetTitle(f.name) curve.SetLineWidth(2) # resid = f.frame.residHist(); resid.SetMarkerSize(0) # plotfunc.AddRatioManual(cans[-1],curve,resid,drawopt1='l',drawopt2='l') pull = f.frame.pullHist() pull.SetMarkerSize(0.7) # for special if options.family == 'selected': color = { 'Pow': ROOT.kBlack + 0, 'Exponential': ROOT.kRed + 1, 'ExpPoly2': ROOT.kBlue + 1, 'Bern3': ROOT.kGreen + 1, 'Bern4': ROOT.kMagenta + 1, 'Bern5': ROOT.kOrange + 1, }.get(f.name) pull.SetMarkerColor(color) pull.SetLineColor(color) curve.SetLineColor(color) curve.SetFillColor(0) plotfunc.AddRatioManual(cans[-1], curve, pull, drawopt1='l', drawopt2='p') f.chisquare_toy = Tools.ChiSquareToys_ForSpuriousSignal( f, options.outdir) if not options.family == 'selected': plotfunc.SetColors(cans[-1]) plotfunc.FormatCanvasAxes(cans[-1]) plotfunc.SetXaxisRanges(cans[-1], Tools.lower_range, Tools.upper_range) plotfunc.SetAxisLabels(cans[-1], 'm_{#gamma#gamma} [GeV]', 'entries', 'pull') the_text = [ plotfunc.GetAtlasInternalText(), plotfunc.GetSqrtsText(13) + ', ' + plotfunc.GetLuminosityText(36.1), Tools.CategoryNames[Tools.categories[options.category]] ] plotfunc.DrawText(cans[-1], the_text, 0.19, 0.70, 0.59, 0.91, totalentries=3) plotfunc.MakeLegend(cans[-1], 0.60, 0.70, 0.90, 0.91, totalentries=3) #plotfunc.GetTopPad(cans[-1]).GetPrimitive('legend').AddEntry(0,'^{ }background-only fit','') #list(plotfunc.GetTopPad(cans[-1]).GetPrimitive('legend').GetListOfPrimitives())[-1].SetLabel('^{ }background-only fit') plotfunc.SetYaxisRanges(plotfunc.GetBotPad(cans[-1]), -4, 4) taxisfunc.AutoFixYaxis(plotfunc.GetTopPad(cans[-1]), ignorelegend=False, minzero=True) for can in cans[:-1]: plotfunc.FormatCanvasAxes(can) anaplot.UpdateCanvases(cans, options) if not options.batch: raw_input('Press enter to exit') anaplot.doSaving(options, cans) return
else: H1 = H_t.copy() small_list, large_list = InterpolateFunctions.interpolate_main( train_data, H_t, start_day, end_day, cons_points) H_t = InterpolateFunctions.interp_linear( H_t, small_list) #H_t is beign changed as numpy arrays are mutable H1 = InterpolateFunctions.interp_linear(H1, small_list) H_t = H_t[:, None] #changing numpy array shape to fit the function train_interp, dummy1, dummy2 = DataFunctions.normalize_103( train_data, train_data, train_data) Y_t, Y_NN = InterpolateFunctions.interp_LSTM(train_interp, H_t, large_list) PlotFunctions.Plot_interpolate(H1[s0 * 24:start_day * 24], Y_t[start_day * 24:end_day * 24], Y_NN[start_day * 24:end_day * 24], H1[start_day * 24:end_day * 24], H1[end_day * 24:s1 * 24]) e_interp = InterpolateFunctions.interpolate_calculate_rms( H1[start_day * 24:end_day * 24], Y_t[start_day * 24:end_day * 24]) e_NN = InterpolateFunctions.interpolate_calculate_rms( H1[start_day * 24:end_day * 24], Y_NN[start_day * 24:end_day * 24]) print e_interp print e_NN H_t = Y_t.copy() numpy.save('H1_file_MD1.npy', H_t) #Aggregating data on a daily basis conv_hour_to_day = 24 H_mean_t, H_sum_t, H_min_t, H_max_t = DataFunctions.aggregate_data(
Y_NN = NN_savemodel.predict(X3) #Y_NN = numpy.reshape(Y_NN, (Y_NN.shape[0]*24, 1)) e_NN = (MathFunctions.rms_flat(Y_NN - H_e)) / (MathFunctions.rms_flat(H_e)) e_NN2 = (MathFunctions.rms_flat(Y_NN - H_e)) / (MathFunctions.rms_flat(H_t)) print e_NN print e_NN2 print "R2: " print r2_score(Y_lstm2, H_e) #Calculate p-value a1, a2 = pearsonr(Y_lstm2, H_e) print "LSTM rho-value: " print a1 b1, b2 = pearsonr(Y_NN, H_e) print "NN rho-value" print b1 ####Plotting PlotFunctions.Plot_double(t_array, H_e, t_array, Y_lstm2, 'Actual conv power', 'LSTM conv power', 'k-', 'r-', "fig_77a.eps") PlotFunctions.Plot_triple(t_train, H_t, t_test, Y_lstm2, t_test, H_e, 'Training Data', 'LSTM predictions', 'Test Data (actual)', 'k-', 'r-', 'b-', "fig_77b.eps") PlotFunctions.Plot_quadruple(t_train, H_t, t_test, Y_lstm2, t_test, Y_NN, t_test, H_e, 'Training Data', 'LSTM predictions', 'MLP Predictions', 'Test Data (actual)', 'k-', 'r-', 'y-', 'b-', "fig_77d.eps")
plt.ylabel('Loss') plt.xlabel('Epoch') plt.legend(['Train', 'Validation'], loc='upper right') plt.savefig(path_save_pic + "performance.png") plt.show() #end params_post = model.get_weights() print("Model evaluation on test data: loss and accuracy\n", model.evaluate(Xtest,Ytest, verbose = 2)) if (plot['distributions']): numLayers = len(model.layers) pfn.jointPlotter(numLayers, path_save_pic, params_pre, params_post, plot_hist = False) #end if (plot['network']): plotNet = npl.plotNet(params_post, path_save_pic, trained = True, asGraph = False) plotNet.plotNetFunction() #end streams.check_create_directory(path_save_model + r'\{}'.format(dataset_id)) model.save(path_save_model + r'\\' + dataset_id + \ r'\\model_' + dataset_id + ".h5") model.save(path_save_model + r'\{}\model_{}.h5'.format(dataset_id, dataset_id)) return model, history.history['acc'] #end
def Main(): #Load MNIST **************************************************************** #mnist.pkl.gz(50000) Accuracy 0.96 #mnist_expanded.pkl.gz(250000) Accuracy 0.97 fn = "..\\data\\mnist.pkl.gz" #".datamnist_expanded.pkl.gz" lstTrain, lstV, lstT = mnist_loader.load_data_wrapper(fn) lstTrain = list(lstTrain) lstV = list(lstV) lstT = list(lstT) # ********************************************************************* path = "..\\TmpLogs\\" if not os.path.isdir(path): os.mkdir(path) fnNetworkData = "{}{}_NetData".format(path, rn.RvNeuralNetwork.__name__) fnSaved = "" #Hyper pameters ------------------------------------------- loop = 10 # loop effect,10, 30 all above 0.95 stepNum = 10 # stepNum effect, 10->0.9, 100->0.5 learnRate = 0.1 # learnRate and lmbda will affect each other lmbda = 5.0 #add lmbda(Regularization) to solve overfitting # Training *********************************************************** # Ask DoTraining- DoTraining = ri.Ask_YesNo("Execute Training?", "y") if DoTraining: """ [784,50,10], loop=10, 0.9695 loop=100, 0.9725 """ # 建立 RvNeuralNetWork---------------------------------------------- inputNeusNum = len(lstTrain[0][0]) lyr1NeuNum = 50 lyr2NeuNum = len(lstTrain[0][1]) lyrsNeus = [inputNeusNum, lyr1NeuNum] lyrsNeus = ri.Ask_Add_Array_Int("Input new layer Neurons num.", lyrsNeus, lyr1NeuNum) lyrsNeus.append(lyr2NeuNum) #net = rn.RvNeuralNetwork( \ # rn.RvNeuralNetwork.LayersNeurons_To_RvNeuralLayers(lyrsNeus)) #net = rn.RvNeuralNetwork.Class_Create_LayersNeurons(lyrsNeus) #net = rn.RvNeuralNetwork(lyrsNeus) # ([784,50,10]) cnvLyrSId = 0 lyrObjs = [] # Add RvConvolutionLayer-------------------------------- EnableCovolutionLayer = ri.Ask_YesNo("Add ConvolutionLayer?", "y") if EnableCovolutionLayer: lyrsNeus, cnvLyrSId, cnvLyr = Add_ConvLayer( lyrObjs, lyrsNeus, inputNeusNum, cnvLyrSId) # Create Layer Object array ------------------------------- for iLyr in range(cnvLyrSId, len(lyrsNeus) - 1): lyrObjs.append( rn.RvNeuralLayer([lyrsNeus[iLyr], lyrsNeus[iLyr + 1]])) net = rn.RvNeuralNetwork(lyrObjs) # Ask nmtivation ------------------_---------- enumActivation = ri.Ask_Enum("Select Activation method.", nm.EnumActivation, nm.EnumActivation.afReLU) for lyr in net.NeuralLayers: lyr.ClassActivation, lyr.ClassCost = \ nm.Get_ClassActivation(enumActivation) net.Motoring_TrainningProcess = rn.Debug net.NetEnableDropOut = ri.Ask_YesNo("Execute DropOut?", "n") if net.NetEnableDropOut: enumDropOut = ri.Ask_Enum("Select DropOut Method.", nm.EnumDropOutMethod, drpOut.eoSmallActivation) rn.gDropOutRatio = ri.Ask_Input_Float("Input DropOut ratio.", rn.gDropOutRatio) net.Set_DropOutMethod(enumDropOut, rn.gDropOutRatio) monitoring = ri.Ask_YesNo("Watch training process?", "y") net.Motoring_TrainningProcess = monitoring # Caculate proper hyper pameters --- DoEvaluate_ProperParams = ri.Ask_YesNo( "Auto-caculating proper hyper pameters?", "n") if DoEvaluate_ProperParams: loop, stepNum, learnRate, lmbda = rf.Evaluate_BestParam_lmbda( net, net.Train, lstTrain[:1000], lstV[:500], loop, stepNum, learnRate, lmbda) loop, stepNum, learnRate, lmbda = rf.Evaluate_BestParam_learnRate( net, net.Train, lstTrain[:1000], lstV[:500], loop, stepNum, learnRate, lmbda) else: loop, stepNum, learnRate, lmbda = rf.Ask_Input_SGD( loop, stepNum, learnRate, lmbda) print( "Hyper pameters: Loop({}), stepNum({}), learnRatio({}), lmbda({})\n" .format(loop, stepNum, learnRate, lmbda)) start = time.time() # 開始網路訓練- net.Train(lstTrain, loop, stepNum, learnRate, lstV, lmbda) dT = time.time() - start fnSaved = rf.Save_NetworkDataFile(net, fnNetworkData, loop, stepNum, learnRate, lmbda, dT) # Ask DoPredict---------------------------------------------------- DoPredict = True if DoPredict: if (os.path.isfile(fnSaved)): fn1 = fnSaved else: fn1 = ".\\{}_NetData_DontDelete.txt".format( rn.RvNeuralNetwork.__name__) rn.Debug_Plot = True #ri.Ask_YesNo("Plot Digits?", "n") # Predict_Digits(net, lstT) pltFn.Predict_Digits_FromNetworkFile(fn1, lstT, rn.Debug_Plot)
elif choice == 2: X_t, Y_t, X_e, Y_e, train_list, test_list = InterpolateFunctions.train_test_split( train_data, H_t) best = InterpolateFunctions.imputate_optimize(X_t, Y_t) Y_p = NNFun_PSB.PSB_model_DL(X_t, Y_t, X_e, best) H_t = InterpolateFunctions.organize_pred(H_t, Y_p, test_list) #numpy.save('Ht_file_HVAC1.npy', H_t) else: H1 = H_t.copy() small_list, large_list = InterpolateFunctions.interpolate_main( train_data, H_t, start_day, end_day, cons_points) H_t = InterpolateFunctions.interp_linear( H_t, small_list) #H_t is beign changed as numpy arrays are mutable H1 = InterpolateFunctions.interp_linear(H1, small_list) PlotFunctions.Plot_interp_params() H_t = H_t[:, None] #changing numpy array shape to fit the function train_interp, dummy1, dummy2 = DataFunctions.normalize_103( train_data, train_data, train_data) Y_t, Y_NN = InterpolateFunctions.interp_LSTM(train_interp, H_t, large_list) PlotFunctions.Plot_interpolate(H1[s0 * 24:start_day * 24], Y_t[start_day * 24:end_day * 24], Y_NN[start_day * 24:end_day * 24], H1[start_day * 24:end_day * 24], H1[end_day * 24:s1 * 24]) e_interp = InterpolateFunctions.interpolate_calculate_rms( H1[start_day * 24:end_day * 24], Y_t[start_day * 24:end_day * 24]) e_NN = InterpolateFunctions.interpolate_calculate_rms( H1[start_day * 24:end_day * 24], Y_NN[start_day * 24:end_day * 24])
#!/usr/bin/env python import ROOT import math import PlotFunctions as plotfunc import PyAnalysisPlotting as anaplot import TAxisFunctions as taxisfunc plotfunc.SetupStyle() import Tools import ChiSquareTools import FunctionsModule import os ROOT.gROOT.LoadMacro('RooFitFunctions.h') ROOT.RooMsgService.instance().setGlobalKillBelow(ROOT.RooFit.FATAL) ROOT.RooMsgService.instance().setSilentMode(True) fcns = { 'official': [ 'Exponential', 'ExpPoly2', 'ExpPoly3', 'Bernstein_4', 'Bernstein_5', 'Pow' ], 'selected': None, # to be filled in main_singleCategory 'ftest': None, # to be filled in main_singleCategory 'ExpPoly': ['Exponential'] + list('ExpPoly%d' % (d) for d in range(2, 4)), 'Laurent': list('Laurent%d' % (d) for d in range(0, 3)), 'PolyOverX4': ['1/x^4'] + list('poly%d/x^4' % (d) for d in range(1, 6)),
def main(options,args) : mystyle = plotfunc.SetupStyle() files_b,trees_b,keys_b = anaplot.GetTreesFromFiles(options.bkgs ,treename=options.treename,xAODInit=options.xAODInit) files_s,trees_s,keys_s = anaplot.GetTreesFromFiles(options.signal,treename=options.treename,xAODInit=options.xAODInit) files_d,trees_d,keys_d = anaplot.GetTreesFromFiles(options.data ,treename=options.treename,xAODInit=options.xAODInit) scales_b = anaplot.GetScales(files_b,trees_b,keys_b,options) scales_s = anaplot.GetScales(files_s,trees_s,keys_s,options) weight = options.weight if ''.join(options.cuts+options.truthcuts) : weight = (weight+'*(%s)'%(' && '.join(options.cuts+options.truthcuts).lstrip('& ').rstrip('& '))).lstrip('*') dweight = '' # weight value (and cuts) applied to data if ''.join(options.cuts+options.blindcut) : dweight = '('+' && '.join(options.cuts+options.blindcut).lstrip('& ').rstrip('& ')+')' cans = [] # get the histograms from the files for v in options.variables : bkg_hists = [] sig_hists = [] data_hist = None if options.data : data_hists = anaplot.GetVariableHistsFromTrees(trees_d,keys_d,v,dweight,options,files=files_d) data_hist = anaplot.MergeSamples(data_hists,options,requireFullyMerged=True)[0] anaplot.PrepareDataHistos(data_hists,options) if options.bkgs : bkg_hists = anaplot.GetVariableHistsFromTrees(trees_b,keys_b,v,weight,options,scales=scales_b,files=files_b) bkg_hists = anaplot.MergeSamples(bkg_hists,options) anaplot.PrepareBkgHistosForStack(bkg_hists,options) if options.signal : sig_hists = anaplot.GetVariableHistsFromTrees(trees_s,keys_s,v,weight,options,scales=scales_s,files=files_s) sig_hists = anaplot.MergeSamples(sig_hists,options) anaplot.PrepareSignalHistos(sig_hists,options) if options.normalize : for hist in [data_hist] + bkg_hists + sig_hists : if not hist : continue hist.Scale(1/float(hist.Integral())) if options.customnormalize : options.customnormalize(v,sig_hists=sig_hists,bkg_hists=bkg_hists,data_hist=data_hist) cans.append(anaplot.DrawHistos(v,options,bkg_hists,sig_hists,data_hist)) if options.afterburner : for can in cans : options.afterburner(can) anaplot.UpdateCanvases(cans,options) if options.xAODInit : ROOT.xAOD.ClearTransientTrees() if not options.batch : import code code.interact(banner='Pausing... Press Contol-D to exit.',local=locals()) anaplot.doSaving(options,cans) print 'done.' return
def main_singleCategory(options, args): if options.category in [19, 23]: # ggH_0J_Cen is 0 print 'Error! This category has been merged away.' return cans = [] options.outdir = '' if options.analysis == 'couplings2017': category_name = Tools.categories_couplings2017[options.category] category_title = Tools.CategoryNames_couplings2017[category_name] fcns['selected'] = [Tools.selected_couplings2017[category_name]] fcns['ftest'] = ChiSquareTools.ftest[category_name] background_label = '#gamma#gamma' lumi = 36.1 elif options.analysis == 'ysy': category_name = Tools.categories_ysy[options.category] category_title = Tools.CategoryNames_ysy[category_name] fcns['selected'] = [Tools.selected_ysy[category_name]] fcns['ftest'] = ChiSquareTools.ftest[category_name] background_label = 'll#gamma' lumi = 139 else: print('Error - do not understand analysis name %s' % (options.analysis)) import sys sys.exit() family_name = { 'official': 'official_functions', 'selected': 'selected_function', }.get(options.family, options.family + '_family') # A list of options is provided if options.functions: flist = options.functions.split(',') options.outdir += '_'.join(flist) # Otherwise a family must be provided: else: flist = fcns.get(options.family) options.outdir += family_name options.outdir += '_c%02d_%s' % (options.category, category_name) functions = [] FunctionsModule.PopulateFunctionList(functions, flist, options.lower, options.upper) ChiSquareTools.LinkFunctionsForFtest(functions) if len(functions) == 0: print 'Error! no functions loaded!' import sys sys.exit() for f in functions: f.SetAnalysis(options.analysis) f.SetCategory(options.category) f.SetFileName(options.file) f.Initialize() # rebin = 5 # if options.category > 16 : # rebin = 1 # if options.category > 23 : # ttH categories # rebin = 10 # Rebin to get 1 bin per GeV rebin = int(functions[0].datahist.GetNbinsX() / 55) print 'Proceeding with Data histogram:', functions[0].datahist functions[0].datahist.Rebin(rebin) for i, f in enumerate(functions): if i: f.datahist.Rebin(rebin) f.datasb_rebinned = ROOT.RooDataHist('data_real_rebinned', '', ROOT.RooArgList(f.obsVar), f.datahist, 1.) if not os.path.exists(options.outdir): os.makedirs(options.outdir) ftest_text = '' cans.append( plotfunc.RatioCanvas( "Ftests_%02d_%s" % (options.category, category_name), "main plot", 600, 500)) functions[0].datahist.SetBinErrorOption(ROOT.TH1.kPoisson) plotfunc.AddHistogram(cans[-1], functions[0].datahist) ## ## Ftests to sideband ## for f in functions: if not hasattr(f, 'ftest_function'): continue nbins_blind = int(f.datasb_rebinned.numEntries() - 10) print 'Sideband numEntries:', f.datasb_rebinned.numEntries( ), 'nbins:', nbins_blind # ndof_bins = nbins_blind-f.ndof-1 # ndof_bins_2 = nbins_blind-f.ftest_function.ndof-1 ndof_bins = nbins_blind - f.ndof ndof_bins_2 = nbins_blind - f.ftest_function.ndof # # Fitting data sidebands - first function # print 'Fitting data sidebands' print f.PrintParameters() ChiSquareTools.FitForChi2_DataSidebands(f) print f.PrintParameters() print 'Fitting data sidebands done' ftest_text += f.PrintParameters() + '\n' chi2 = ROOT.GetChiSquare(f.obsVar, f.function_ext, f.datasb_rebinned, ndof_bins) pvalue_chi2 = ROOT.TMath.Prob(chi2 * (ndof_bins), ndof_bins) # # Plotting stuff - first function # Tools.ClearRooPlot(f.frame) ChiSquareTools.NormalizeToSideband(f) f.datasb_rebinned.plotOn( f.frame, ROOT.RooFit.DataError(ROOT.RooAbsData.Poisson)) print 'about to plot' #f.workspace.var('a1').setVal(-1.6) print f.PrintParameters() f.function_ext.plotOn(f.frame, *(Tools.plotOptions_sb_all)) print f.PrintParameters() print 'about to plot done' curve = f.frame.getCurve() curve.SetMarkerSize(0) curve.SetLineWidth(2) curve.SetTitle('%s, p(#chi^{2}) = %2.1f%%' % (f.name, pvalue_chi2 * 100.)) curve.SetLineColor(ROOT.kRed + 1) curve.SetFillColor(0) pull = f.frame.pullHist() pull.SetMarkerSize(0.7) pull.SetLineColor(ROOT.kRed + 1) pull.SetMarkerColor(ROOT.kRed + 1) plotfunc.AddRatioManual(cans[-1], curve, pull, drawopt1='l', drawopt2='p') # # Fitting data sidebands - first function # print 'Fitting data sidebands (other function)' print f.ftest_function.PrintParameters() ChiSquareTools.FitForChi2_DataSidebands(f.ftest_function) print f.ftest_function.PrintParameters() print 'Fitting data sidebands (other function) done' ftest_text += f.ftest_function.PrintParameters() + '\n' chi2_2 = ROOT.GetChiSquare(f.obsVar, f.ftest_function.function_ext, f.datasb_rebinned, ndof_bins_2) pvalue_chi2_2 = ROOT.TMath.Prob(chi2_2 * (ndof_bins_2), ndof_bins_2) # # Plotting stuff - other function # Tools.ClearRooPlot(f.frame) ChiSquareTools.NormalizeToSideband(f.ftest_function) f.datasb_rebinned.plotOn(f.frame) f.ftest_function.function_ext.plotOn(f.frame, *(Tools.plotOptions_sb_all)) curve = f.frame.getCurve() curve.SetMarkerSize(0) curve.SetLineWidth(2) curve.SetTitle('%s, p(#chi^{2}) = %2.1f%%' % (f.ftest_function.name, pvalue_chi2_2 * 100.)) curve.SetLineColor(ROOT.kAzure - 2) curve.SetFillColor(0) pull = f.frame.pullHist() pull.SetMarkerSize(0.7) pull.SetLineColor(ROOT.kAzure - 2) pull.SetMarkerColor(ROOT.kAzure - 2) plotfunc.AddRatioManual(cans[-1], curve, pull, drawopt1='l', drawopt2='p') ftest = ChiSquareTools.GetF(chi2, chi2_2, ndof_bins, ndof_bins_2) ftest_text += 'chi2/ndf: %2.5f\n' % (chi2) ftest_text += 'chi2_2/ndf: %2.5f\n' % (chi2_2) ftest_text += 'chi2: %2.5f\n' % (chi2 * ndof_bins) ftest_text += 'chi2_2: %2.5f\n' % (chi2_2 * ndof_bins_2) ftest_text += 'ftest: %2.5f\n' % (ftest) #p_ftest = 1.0 - ROOT.TMath.FDistI(ftest,1,ndof_bins_2) p_ftest = 1.0 - ROOT.TMath.FDistI(ftest, ndof_bins - ndof_bins_2, ndof_bins_2) ftest_text += 'p_ftest: %2.5f\n' % (p_ftest) # # Throw Toys # print 'Running toy Ftests' fisher_dist = ChiSquareTools.ToyFtest(f, f.ftest_function, ftest, options.outdir, options.ntoys) print 'Running toy Ftests done' if fisher_dist.Integral(0, 100000): ftest_text += 'p_ftest_toys: %2.5f\n' % ( fisher_dist.Integral(fisher_dist.FindBin(ftest), 100000) / float(fisher_dist.Integral(0, 100000))) # chi2_lowerSideBand = ROOT.RooChi2Var("chi2_low","chi2_low",f.function_ext,f.data_realdata,ROOT.RooFit.DataError(ROOT.RooAbsData.Poisson),ROOT.RooFit.Range("lower")); # chi2_upperSideBand = ROOT.RooChi2Var("chi2_upp","chi2_upp",f.function_ext,f.data_realdata,ROOT.RooFit.DataError(ROOT.RooAbsData.Poisson),ROOT.RooFit.Range("upper")); # chi2_all = ROOT.RooChi2Var("chi2_all","chi2_all",f.function_ext,f.data_realdata,ROOT.RooFit.Range("lower,upper")); # chi2_lowerSideBand = ROOT.RooChi2Var("chi2_low","chi2_low",f.function_ext,f.data_realdata,ROOT.RooFit.Range("lower")); # chi2_upperSideBand = ROOT.RooChi2Var("chi2_upp","chi2_upp",f.function_ext,f.data_realdata,ROOT.RooFit.Range("upper")); # print 'chi2_all .getValV()',chi2_all.getValV() # print 'chi2_lowerSideBand.getValV()',chi2_lowerSideBand.getValV() # print 'chi2_upperSideBand.getValV()',chi2_upperSideBand.getValV() # print 'total:',(chi2_lowerSideBand.getValV()+chi2_upperSideBand.getValV()) / float(f.bins-1-f.ndof) # Get the chi2 from the background-only fit # print 'f.frame.chiSquare(1+f.ndof-10)',f.frame.chiSquare(1+f.ndof)*(f.bins-1-f.ndof) # f.chisquare_sb = f.frame.chiSquare(1+f.ndof-10) # n-1 bins MINUS 10 BINS # f.pvalue_chi2_sb = ROOT.TMath.Prob(f.chisquare_sb*(f.bins-1-f.ndof-10),f.bins-1-f.ndof-10) # print 'chisquare:',f.chisquare_sb,f.pvalue_chi2_sb continue curve = f.frame.getCurve() curve.SetMarkerSize(0) curve.SetLineWidth(1) curve.SetTitle(f.name) curve.SetLineWidth(2) # resid = f.frame.residHist(); resid.SetMarkerSize(0) # plotfunc.AddRatioManual(cans[-1],curve,resid,drawopt1='l',drawopt2='l') pull = f.frame.pullHist() pull.SetMarkerSize(0.7) # for special if options.family == 'selected': color = { 'Pow': ROOT.kBlack + 0, 'Exponential': ROOT.kRed + 1, 'ExpPoly2': ROOT.kBlue + 1, 'Bern3': ROOT.kGreen + 1, 'Bern4': ROOT.kMagenta + 1, 'Bern5': ROOT.kOrange + 1, }.get(f.name) pull.SetMarkerColor(color) pull.SetLineColor(color) curve.SetLineColor(color) curve.SetFillColor(0) plotfunc.AddRatioManual(cans[-1], curve, pull, drawopt1='l', drawopt2='p') plotfunc.SetAxisLabels(cans[-1], 'm_{#gamma#gamma} [GeV]', 'entries', 'pull') the_text = [ plotfunc.GetAtlasInternalText(), plotfunc.GetSqrtsText(13) + ', ' + plotfunc.GetLuminosityText(36.1), category_title, '1-p(F_{%d%d}) = %2.1f%%' % (functions[0].ndof, functions[0].ftest_function.ndof, p_ftest * 100) ] plotfunc.DrawText(cans[-1], the_text, 0.19, 0.63, 0.59, 0.91, totalentries=4) plotfunc.MakeLegend(cans[-1], 0.57, 0.63, 0.90, 0.91, totalentries=4) plotfunc.SetYaxisRanges(plotfunc.GetBotPad(cans[-1]), -4, 4) plotfunc.SetXaxisRanges(cans[-1], functions[0].lower_range, functions[0].upper_range) taxisfunc.AutoFixYaxis(plotfunc.GetTopPad(cans[-1]), forcemin=0.001) print ftest_text a = open('%s/ftests.txt' % (options.outdir), 'w') a.write(options.file + '\n') a.write(ftest_text + '\n') a.close() for can in cans: plotfunc.FormatCanvasAxes(can) anaplot.UpdateCanvases(cans, options) if not options.batch: raw_input('Press enter to exit') anaplot.doSaving(options, cans) return