def main() :
data_plots = []
mc_plots = []
for i in range(5) :
data_plots.append(ROOT.TH1F('a_%d'%(i),'Data',48,-6,6))
mc_plots .append(ROOT.TH1F('b_%d'%(i),'MC' ,48,-6,6))
mc_plots[-1].SetLineColor(ROOT.kRed+1)
for plot in data_plots + mc_plots :
plot.Sumw2()
plot.SetLineWidth(2)
rand = ROOT.TRandom3(1)
for i in range(100000) :
for j in range(len(data_plots)) :
data_plots[j].Fill(rand.Gaus(0,1) + j*0.2)
mc_plots [j].Fill(rand.Gaus(0,1) + j*0.15)
#
# Bin labeling
#
bin_edges = [15,20,25,30,40,50]
template = '%s^{ }<^{ }p_{T}^{ }<^{ }%s GeV'
labels = list(template%(bin_edges[a],bin_edges[a+1]) for a in range(len(bin_edges)-1))
mycanvas = ROOT.TCanvas('mycanvas','blah',600,500)
#
# The main call:
#
ShowerShapeEvolutionPlot(mycanvas,labels,data_plots,mc_plots)
mycanvas.SetGridx()
plotfunc.DrawText(mycanvas,[plotfunc.GetAtlasInternalText(status='Internal')
,plotfunc.GetSqrtsText(13)+', '+plotfunc.GetLuminosityText()
],0.27,0.88,0.57,0.99,totalentries=2)
mycanvas.Print(mycanvas.GetName()+'.pdf')
raw_input('Press enter to exit')
return
def Predict_Digits(net, lstT, plotDigit=True, onlyDigit=-1):
nRow, nCol = 2, 5
digitNum = nRow * nCol
digitFigs, pxlW, pltInchW = Initial_PlotParams(len(lstT[0][0]), nRow, nCol)
digitTitles = ["" for a in digitFigs]
digitId = 0
# 隨機測試某筆數字 ----------------------------------------------
start = time.time()
sampleNum = 10000 # 不含繪圖，辨識 10000張，費時 1.3 秒，平均每張 0.00013秒
plotNum = 5
iPlot = 0
plotMod = int(sampleNum / plotNum) + 1
correctNum = 0
failNum = 0
for i in range(0, sampleNum):
if (lstT[i][1] == onlyDigit) or (onlyDigit < 0):
doPlot = (i % plotMod == 0)
aId = np.random.randint(0, len(lstT))
label, result, outputY = net.Predict_Digit(lstT[aId], False)
if label == result: correctNum += 1
else:
doPlot = iPlot < plotNum #(failNum<10)
failNum += 1
if doPlot and plotDigit:
# Plot_Digit(lstT[aId], result, label)
# print("({}): Label={}, Predict={} ( {:.5f} ) -> {} ".
# format(i, label,result, outputY, sResult[int(label==result)]))
s1 ="{}={},({:.3f})->{}".\
format(label, result,
outputY,sResult[int(label==result)] )
digitId = digitId % digitNum
digitFigs[digitId] = np.array(
lstT[aId][0]).transpose().reshape(pxlW, pxlW) * 255
digitTitles[digitId] = s1
digitId += 1
if (digitId == digitNum):
pltFn.Plot_Images(np.array(digitFigs), nRow, nCol,
digitTitles, "", pltInchW)
iPlot += 1
digitId = 0
dt = time.time() - start
accurRatio = correctNum / sampleNum
print("\nAccuracy({:.3f}), {}/{}(Correct/Total)".format(
accurRatio, correctNum, sampleNum))
print("Elapsed(seconds)) : {:.3f} sec.\n".format(dt))
return accurRatio, dt
def RatioRangeAfterBurner(can,ymin=None,ymax=None) :
import PlotFunctions as plotfunc
import TAxisFunctions as taxisfunc
import ROOT
# Set ratio range; add dotted line at 1
if not plotfunc.GetBotPad(can) :
return
isPull = False
for hist in plotfunc.GetBotPad(can).GetListOfPrimitives() :
if hasattr(hist,'GetYaxis') :
isPull = isPull or (hist.GetYaxis().GetTitle() == 'pull')
if ymin == None :
if isPull :
ymin,ymax = -4,4
else :
ymin,ymax = 0,2
taxisfunc.SetYaxisRanges(plotfunc.GetBotPad(can),ymin,ymax)
if ymax <= 1 :
return
xmin,xmax = None,None
for i in plotfunc.GetBotPad(can).GetListOfPrimitives() :
if issubclass(type(i),ROOT.TH1) :
xmin = i.GetXaxis().GetBinLowEdge(1)
xmax = i.GetXaxis().GetBinLowEdge(i.GetNbinsX()+1)
if xmin != None :
lineh = 0 if isPull else 1
line = ROOT.TLine(xmin,lineh,xmax,lineh)
line.SetLineStyle(2)
plotfunc.GetBotPad(can).cd()
line.Draw()
plotfunc.tobject_collector.append(line)
return
def Test_Encoder_Decoder(encoder,
decoder,
lstT,
sampleNum=10,
saveImgPath="",
noiseStrength=0.0):
# 隨機測試某筆數字 ----------------------------------------------
#start = time.time()
pxlW = int(np.sqrt(len(lstT[0][0])))
# 不含繪圖，辨識 10000張，費時 1.3 秒，平均每張 0.00013秒
for i in range(0, sampleNum):
aId = np.random.randint(0, len(lstT))
digitId = 0
if type(lstT[aId][1]) == np.array:
if len(lstT[aId][1]) == 10:
digitId = np.argmax(lstT[aId][1])
else:
digitId = -1
elif type(lstT[aId][1]) == np.int64:
digitId = lstT[aId][1]
oneInput = lstT[aId][0]
#rf.Plot_Digit([oneInput.transpose(), lstT[aId][1] ] ) #(1x784)
if (noiseStrength > 0.0):
oneInput = rn.RvBaseNeuralNetwork.Add_Noise(
oneInput, noiseStrength)
for j in range(1): # 多做幾次，效果不會比較好
encode = encoder.Get_OutputValues(oneInput) #(784x1)
#output = decoder.Plot_Output(encode) #(784x1)
output = decoder.Get_OutputValues(encode)
# print("Input({}) -> Output : Accuracy={:.3f}".
# format(digitId, decoder.Get_Accuracy_EnDeCoder(oneInput, output)))
if rfi.PathExists(saveImgPath):
imgFn = "{}vdoImg_{}_{}.png".format(saveImgPath, i, j)
else:
imgFn = ""
pltFn.Plot_Images(
np.array([
oneInput.transpose().reshape(pxlW, pxlW) * 255,
output.transpose().reshape(pxlW, pxlW) * 255
]), 1, 2, ["({}) Input".format(digitId), "Output"], imgFn)
print("")
oneInput = output
def ShowDrifttimeVsTime(self, Channel=-1):
YMax = np.max(
[np.max(self.Ch[ii].GradTime) for ii in range(self.NumChannels)])
YMax = self.RoundUpToNext(YMax, 10)
YTicks = self.RoundDownToNext(YMax / 5, 1)
Plt.PltTime(Time=self.Ch[0].TimeStamp,
Data=[
self.Ch[0].GradTime, self.Ch[1].GradTime,
self.Ch[1].GradTime - self.Ch[0].GradTime
],
Legend=['Anode', 'Cathode', 'Drift Time'],
Label='Time since Trigger [$\mu$s]',
XTicks=self.XTicks,
YTicks=YTicks,
YRange=[0, YMax],
SaveName='drift_time',
Save=False)
def ShowAmplitudeVsTime(self, Channel=-1):
YMax = np.max(
[np.max(self.Ch[ii].Max) for ii in range(self.NumChannels)])
YMax = self.RoundUpToNext(YMax, 100)
YTicks = self.RoundDownToNext(YMax / 5, 100)
print(YTicks)
Plt.PltTime(
Time=self.Ch[0].TimeStamp,
Data=[self.Ch[0].Max, self.Ch[1].Max, self.ChargeCollection * 100],
Legend=['Anode', 'Cathode', 'Charge Collection [\%]'],
Label='Amplitude [mV]',
XTicks=self.XTicks,
YTicks=YTicks,
YRange=[0, YMax],
SaveName='amp_ratio',
Title='Charge Collection in Vacuum at 60 V/cm',
Save=False)
def main(options, args):
plotfunc.SetupStyle()
file_d, tree_d, key_d = anaplot.GetChainFromFiles(
options.data, treename=options.treename)
dcuts = ' && '.join(options.cuts + options.blindcut)
if not os.path.exists(options.outdir):
os.makedirs(options.outdir)
from array import array
categories = '.'.join(CouplingsHelpers.categories)
print categories
for k in key_d:
ROOT.makePicoXaod_Categories(tree_d[k], k, dcuts, options.outdir,
categories)
print 'done.'
return
import numpy
import PlotFunctions
H_t = numpy.load('H_t10.npy')
H_e = numpy.load('H_e10.npy')
Y_lstm2 = numpy.load('Y_ST2B.npy')
t_train = numpy.arange(0, len(H_t))
t_test = numpy.arange(len(H_t), len(H_t) + len(Y_lstm2))
t_array = numpy.arange(0, len(Y_lstm2))
PlotFunctions.Plot_triple(t_train, H_t, t_test, Y_lstm2, t_test, H_e,
'Training Data', 'Approach I predictions',
'Test Data (actual)', 'k-', 'r-', 'b-',
"fig_sample.eps")
for i in range(len(ClusterData[day].idx)):
y_NN[ClusterData[day].idx[i] - 1] = y_pred[i]
y_sim[ClusterData[day].idx[i] - 1] = Y_test[i]
#Outputs
print y_NN
print y_sim
print y_deep
e_rms = (MathFunctions.rms_flat(y_NN -
y_sim)) / (MathFunctions.rms_flat(y_sim))
e_deep = (MathFunctions.rms_flat(y_deep -
y_sim)) / (MathFunctions.rms_flat(y_sim))
print e_rms
print e_deep
PlotFunctions.PlotEnergy(y_NN, y_sim, y_deep)
j1, j2 = DataFunctions.find_day_idx(4)
PlotFunctions.PlotEnergyDaily(y_NN[j1:j2], y_sim[j1:j2], y_deep[j1:j2])
j1, j2 = DataFunctions.find_day_idx(180)
PlotFunctions.PlotEnergyDaily(y_NN[j1:j2], y_sim[j1:j2], y_deep[j1:j2])
#debug
#print Sch
#print Sch_test
#print Sch[0:47, 1]
#print weather_idxMax
#print Sch_idxMax
#print j
#print ClusterData[0].X_train[0:23, :]
#print ClusterData[1].X_train[0:23, :]
#print ClusterData[2].X_train[0:23, :]
#mlab.start()
IVALU_RESULTS = IVALU_DATA[IVALU_DATA['2MASS'].isin(
IGRINS_RESULTS['2MASS'].values)]
APOGEE_RESULTS = pd.read_csv(homedir + 'APOGEE_DATA.txt',
names=APOGEE_Header,
delim_whitespace=True)
OPTICAL_DIF = IGRINS_RESULTS.drop(DROP_LIST, axis=1) - OPTICAL_RESULTS.drop(
DROP_LIST, axis=1)
DATA = [
IGRINS_RESULTS, OPTICAL_RESULTS, IVALU_RESULTS, APOGEE_RESULTS,
OPTICAL_COMPLETE
]
Comparison = 'ALL'
pl.Comp(Comparison)
OPTICAL_RESULTS.name = 'Optical'
OPTICAL_COMPLETE.name = 'OComplete'
IVALU_RESULTS.name = 'Ivalu'
IGRINS_RESULTS.name = 'Igrins'
APOGEE_RESULTS.name = 'Apogee'
while True:
if diagFlag == False:
pl.PickElement()
element = pl.element
if element == 'quit':
print('Plotting stopped')
break
elif (element in PlotInput) and (diagFlag == False):
t_array = numpy.arange(0, len(Y_lstm2))
e_deep = (MathFunctions.rms_flat(Y_lstm2 - H_e))/(MathFunctions.rms_flat(H_e))
e_deep2 = (MathFunctions.rms_flat(Y_lstm2 - H_e))/(MathFunctions.rms_flat(H_t))
print Y_lstm2.shape
print H_e.shape
print t_array.shape
print e_deep
print e_deep2
numpy.save('3_LSTM.npy', Y_lstm2)
####Plotting for min-wise
PlotFunctions.Plot_double(t_array, H_e, t_array, Y_lstm2, 'Actual conv power','LSTM conv power', 'k-', 'r-', "fig_5houseI.eps")
#####################
###Build 1-min model:
def objective(params):
#optimize_model = build_lstm_v1.lstm_multi_101(params, train_data2.shape[2], lstm_h1, 60) #Check code here, relu entering
optimize_model = build_lstm_v1.lstm_model_110(params, train_data2.shape[2], 60)
loss_out = NNFunctions.model_optimizer_101(optimize_model, train_data2, H_t2, val_data2, H_val2, 6)
return {'loss': loss_out, 'status': STATUS_OK}
trials = Trials()
best2 = fmin(objective, space2, algo=tpe.suggest, trials=trials, max_evals=3)
elif choice == 2:
X_t, Y_t, X_e, Y_e, train_list, test_list = InterpolateFunctions.train_test_split(
train_data, H_t)
best = InterpolateFunctions.imputate_optimize(X_t, Y_t)
Y_p = NNFun_PSB.PSB_model_DL(X_t, Y_t, X_e, best)
H_t = InterpolateFunctions.organize_pred(H_t, Y_p, test_list)
#numpy.save('Ht_file_HVAC1.npy', H_t)
else:
H1 = H_t.copy()
small_list, large_list = InterpolateFunctions.interpolate_main(
train_data, H_t, start_day, end_day, cons_points)
H_t = InterpolateFunctions.interp_linear(
H_t, small_list) #H_t is beign changed as numpy arrays are mutable
H1 = InterpolateFunctions.interp_linear(H1, small_list)
PlotFunctions.Plot_interp_params()
H_t = H_t[:, None] #changing numpy array shape to fit the function
Y_t, Y_NN = InterpolateFunctions.interp_LSTM_v2(train_data, H_t,
large_list)
PlotFunctions.Plot_interpolate(H1[s0 * 24:start_day * 24],
Y_t[start_day * 24:end_day * 24],
Y_NN[start_day * 24:end_day * 24],
H1[start_day * 24:end_day * 24],
H1[end_day * 24:s1 * 24])
e_interp = InterpolateFunctions.interpolate_calculate_rms(
H1[start_day * 24:end_day * 24], Y_t[start_day * 24:end_day * 24])
e_NN = InterpolateFunctions.interpolate_calculate_rms(
H1[start_day * 24:end_day * 24], Y_NN[start_day * 24:end_day * 24])
print e_interp
# prvaite libraries---------------------------------------------
import mnist_loader
import RvNeuralNetworks as rn
from RvNeuralNetworks import *
import RvAskInput as ri
import RvMiscFunctions as rf
import RvNeuNetworkMethods as nm
import PlotFunctions as pltFn
#%%
#Load MNIST ****************************************************************
#mnist.pkl.gz(50000） Accuracy 0.96
#mnist_expanded.pkl.gz(250000） Accuracy 0.97
fn = "..\\data\\mnist.pkl.gz" #".datamnist_expanded.pkl.gz"
lstTrain, lstV, lstT = mnist_loader.load_data_wrapper(fn)
lstTrain = list(lstTrain)
lstV = list(lstV)
lstT = list(lstT)
fns = []
for file in os.listdir(".\\"):
if file.endswith(".txt"):
fns.append(file)
aId = ri.Ask_SelectItem("Select Network file", fns, 0)
fn1 = fns[aId]
accur1, t1 = pltFn.Predict_Digits_FromNetworkFile(fn1, lstT, True)
print("File : \"{}\"\n nmcu:{}, Time:{:.3} sec\n".format(fn1, accur1, t1))
def main(options, args):
plotfunc.SetupStyle()
files_b, trees_b, keys_b = anaplot.GetTreesFromFiles(
options.bkgs, treename=options.treename)
files_s, trees_s, keys_s = anaplot.GetTreesFromFiles(
options.signal, treename=options.treename)
file_d, tree_d, key_d = anaplot.GetChainFromFiles(
options.data, treename=options.treename)
scales_b = anaplot.GetScales(files_b, trees_b, keys_b, options)
scales_s = anaplot.GetScales(files_s, trees_s, keys_s, options)
dweight = '' # weight value (and cuts) applied to data
weight = options.weight
if ''.join(options.cuts):
weight = (weight + '*(%s)' %
(' && '.join(options.cuts + options.truthcuts))).lstrip('*')
dweight = '(' + ' && '.join(options.cuts + options.blindcut) + ')'
cans = []
v1 = 'HGamEventInfoAuxDyn.m_yy/1000'
v2 = 'HGamEventInfoAuxDyn.catCoup_Moriond2017BDT'
bkg_hists = []
sig_hists = []
data_hist = None
if options.data:
data_hist = anaplot.Get2dVariableHistsFromTrees(
tree_d, key_d, v1, v2, dweight, options)[0]
data_hist.SetLineWidth(2)
data_hist.SetLineColor(1)
data_hist.SetMarkerColor(1)
if options.bkgs:
bkg_hists = anaplot.Get2dVariableHistsFromTrees(trees_b,
keys_b,
v1,
v2,
weight,
options,
scales=scales_b)
bkg_hists, keys_b = anaplot.MergeSamples(bkg_hists, options)
anaplot.PrepareBkgHistosForStack(bkg_hists, options)
if options.signal:
sig_hists = anaplot.Get2dVariableHistsFromTrees(trees_s,
keys_s,
v1,
v2,
weight,
options,
scales=scales_s)
sig_hists, keys_s = anaplot.MergeSamples(sig_hists, options)
sig_hists[-1].SetLineColor(2)
sig_hists[-1].SetMarkerColor(2)
# get the histograms from the files
for c in range(len(CouplingsHelpers.categories)):
#if not categories[c] : continue
lo_bin = c + 1
hi_bin = c + 1
if c == 0: # inclusive case.
lo_bin = 0
hi_bin = 10000 # just to be safe, 10k categories
# MERGE VH DILEP CATEGORIES (23 and 24 --> 23):
if CouplingsHelpers.categories[c] == 'M17_VHdilep_LOW':
hi_bin = hi_bin + 1
# MERGE VH MET HIGH and BSM CATEGORIES (19 and 20 --> 19):
if CouplingsHelpers.categories[c] == 'M17_VHMET_HIGH':
hi_bin = hi_bin + 1
name = 'c%d_%s' % (c, CouplingsHelpers.categories[c])
bkg_projs = []
sig_projs = []
data_proj = None
if options.data:
data_proj = data_hist.ProjectionX('%s_data' % (name), lo_bin,
hi_bin)
PyHelpers.PrintNumberOfEvents(data_proj)
if options.bkgs:
for i, b in enumerate(bkg_hists):
bkg_projs.append(
b.ProjectionX('%s_%s' % (name, keys_b[i]), lo_bin, hi_bin))
PyHelpers.PrintNumberOfEvents(bkg_projs[-1])
if options.signal:
for i, s in enumerate(sig_hists):
sig_projs.append(
s.ProjectionX('%s_%s' % (name, keys_s[i]), lo_bin, hi_bin))
PyHelpers.PrintNumberOfEvents(sig_projs[-1])
cans.append(
anaplot.DrawHistos(v1,
options,
bkg_projs,
sig_projs,
data_proj,
name=name))
cans[-1].SetName(anaplot.CleanUpName(name))
anaplot.UpdateCanvases(options, cans)
if not options.batch:
raw_input('Press enter to exit')
anaplot.doSaving(options, cans)
# Do this afterwards, to make sure the outdir exists.
f = ROOT.TFile('%s/couplings.root' % (options.outdir), 'RECREATE')
for can in cans:
for i in can.GetListOfPrimitives():
if i.GetName()[-6:] == '_error':
continue
if 'stack' in i.GetName():
if not issubclass(type(i), ROOT.THStack):
continue
for j in range(i.GetNhists()):
print 'writing from stack:', i.GetHists()[j].GetName()
i.GetHists()[j].Write()
if issubclass(type(i), ROOT.TH1):
print 'writing:', i.GetName()
i.Write()
f.Close()
print 'done.'
return
digitFigs = [[] for i in range(10)]
nCol = 5 #int(np.sqrt(len(digitFigs)))
nRow = 2 #int(len(digitFigs)/nCol)+1
pxls = outputNum
pxlW = int(np.sqrt(pxls))
pxls = pxlW * pxlW
dpi = 72
zoom = 8 # 28 pxl * zoom
pltInchW = pxlW / dpi * nCol * zoom
if (None != decoder): #and (None!=encoder):
for i in range(sampleLoop):
digitId = 0
for j in range(10):
encode = randomState.randn(inputNum, 1)
encode = np.maximum(0.2, np.minimum(0.8, encode))
output = decoder.Get_OutputValues(encode)
digitFigs[j] = output.transpose().reshape(pxlW, pxlW) * 255
print("({}): code -> Output".format(i))
if pathExists: imgFn = "{}vdoImg_{}.png".format(imgPath, i)
else: imgFn = ""
pltFn.Plot_Images(np.array(digitFigs), nRow, nCol,
["Test DeCoder Fake Image"], imgFn, pltInchW)
aviFn = "{}{}".format(imgPath, "Decoder.avi")
durationSec = 0.5 #min(2.0, 10/sampleLoop)
if ru.ImageFilesToAvi(imgPath, aviFn, durationSec):
rfi.OpenFile(aviFn)
#%%
#使用 mnist_expanded.pkl.gz(250000筆） 準確率提高到 0.97
fn = ".datamnist.pkl.gz" #".datamnist_expanded.pkl.gz"
lstTrain, lstV, lstT = mnist_loader.load_data_wrapper(fn)
lstTrain = list(lstTrain)
lstV = list(lstV)
lstT = list(lstT)
fnNetworkData1 = ".\\{}_NetData_DontDelete.txt".format(
rn.RvNeuralNetwork.__name__)
fnNetworkData2 = ".\\{}_NetData_DropOut.txt".format(
rn.RvNeuralNetwork.__name__)
fnNetworkData3 = ".\\{}_NetData_CnvLyr.txt".format(rn.RvNeuralNetwork.__name__)
#
accur1, t1 = pltFn.Predict_Digits_FromNetworkFile(fnNetworkData1, lstT, False)
accur2, t2 = pltFn.Predict_Digits_FromNetworkFile(fnNetworkData2, lstT, False)
accur3, t3 = pltFn.Predict_Digits_FromNetworkFile(fnNetworkData3, lstT)
print("FullConnected Layer:\n Accu:{}, Time:{:.3} sec\n".format(accur1, t1))
print("DropOut Layer:\n Accu:{}, Time:{:.3} sec\n".format(accur2, t2))
print("Convolution Layer:\n Accu:{}, Time:{:.3} sec\n".format(accur3, t3))
"""
if (os.path.isfile(fnNetworkData1)):
print("\n讀取網路參數檔案以新建網路 - Create_NetworkData():\n")
net = rn.RvNeuralNetwork.Create_Network(fnNetworkData1)
# 預測所有測試集------------------------------------------------
correctNum,n_test = net.Evaluate_Accuracy(lstT)
print("\n預測結果: 正確率({:.3f}), {}/{}(正確/總數)".
format(correctNum/n_test, correctNum,n_test))
def main(options, args):
style = plotfunc.SetupStyle()
style.SetPadRightMargin(0.16)
files_b, trees_b, keys_b = anaplot.GetTreesFromFiles(
options.bkgs, treename=options.treename, xAODInit=options.xAODInit)
files_s, trees_s, keys_s = anaplot.GetTreesFromFiles(
options.signal, treename=options.treename, xAODInit=options.xAODInit)
files_d, trees_d, keys_d = anaplot.GetTreesFromFiles(
options.data, treename=options.treename, xAODInit=options.xAODInit)
scales_b = anaplot.GetScales(files_b, trees_b, keys_b, options)
scales_s = anaplot.GetScales(files_s, trees_s, keys_s, options)
weight = options.weight
if ''.join(options.cuts + options.truthcuts):
weight = (weight + '*(%s)' %
(' && '.join(options.cuts +
options.truthcuts).lstrip('& ').rstrip('& '))
).lstrip('*')
dweight = '' # weight value (and cuts) applied to data
if ''.join(options.cuts + options.blindcut):
dweight = '(' + ' && '.join(options.cuts + options.blindcut).lstrip(
'& ').rstrip('& ') + ')'
cans = []
for v in options.variables:
if v not in options.histformat.keys() or len(
options.histformat[v]) < 4:
print 'Warning: need to set the label of %s using the histformat option.' % (
v)
# get the histograms from the files
for vi, v1 in enumerate(options.variables):
labelv1 = v1
if v1 in options.histformat.keys() and len(
options.histformat[v1]) >= 4:
labelv1 = options.histformat[v1][3]
for vj, v2 in enumerate(options.variables):
if v2 == v1: continue
if vj < vi: continue
labelv2 = v2
if v2 in options.histformat.keys() and len(
options.histformat[v2]) >= 4:
labelv2 = options.histformat[v2][3]
bkg_hists = []
sig_hists = []
data_hist = None
if options.data:
data_hists = anaplot.Get2dVariableHistsFromTrees(trees_d,
keys_d,
v1,
v2,
dweight,
options,
files=files_d)
data_hist = anaplot.MergeSamples(data_hists,
options,
requireFullyMerged=True)[0]
canname = anaplot.CleanUpName('%s_%s_%s' % (v1, v2, 'data'))
cans.append(ROOT.TCanvas(canname, canname, 600, 500))
data_hist.SetMinimum(-0.00001)
plotfunc.AddHistogram(cans[-1], data_hist, drawopt='colz')
plotfunc.SetAxisLabels(cans[-1], labelv1, labelv2)
if options.bkgs:
bkg_hists = anaplot.Get2dVariableHistsFromTrees(
trees_b,
keys_b,
v1,
v2,
weight,
options,
scales=scales_b,
files=files_b)
bkg_hists = anaplot.MergeSamples(bkg_hists, options)
for i, h in enumerate(bkg_hists):
canname = anaplot.CleanUpName('%s_%s_%s' %
(v1, v2, keys_b[i]))
cans.append(ROOT.TCanvas(canname, canname, 600, 500))
h.SetMinimum(-0.00001)
plotfunc.AddHistogram(cans[-1], h, drawopt='colz')
plotfunc.SetAxisLabels(cans[-1], labelv1, labelv2)
if options.signal:
sig_hists = anaplot.Get2dVariableHistsFromTrees(
trees_s,
keys_s,
v1,
v2,
weight,
options,
scales=scales_s,
files=files_s)
sig_hists = anaplot.MergeSamples(sig_hists, options)
for i, h in enumerate(sig_hists):
canname = anaplot.CleanUpName('%s_%s_%s' %
(v1, v2, keys_s[i]))
cans.append(ROOT.TCanvas(canname, canname, 600, 500))
h.SetMinimum(-0.00001)
plotfunc.AddHistogram(cans[-1], h, drawopt='colz')
plotfunc.SetAxisLabels(cans[-1], labelv1, labelv2)
if options.afterburner:
for can in cans:
options.afterburner(can)
anaplot.UpdateCanvases(cans, options)
if options.xAODInit:
ROOT.xAOD.ClearTransientTrees()
if not options.batch:
import code
code.interact(banner='Pausing... Press Contol-D to exit.',
local=locals())
anaplot.doSaving(options, cans)
print 'done.'
return
#%%
label = "Max Risk Adjusted Return Portfolio"
mu = port.mu
cov = port.cov
returns = port.returns
ax = plf.plot_frontier(
w_frontier=w_5,
mu=mu,
cov=cov,
returns=returns,
rm=rm,
rf=0,
alpha=0.01,
cmap="viridis",
w=w1,
label="Portfolio",
marker="*",
s=16,
c="r",
height=6,
width=10,
ax=None,
)
# ax = plf.plot_pie(w=w1, title='Portafolio', height=6, width=10, cmap="tab20",
# ax=None)
# # # w0 = pd.DataFrame(np.ones((100,1))/100)
# # # ax = plf.plot_pie(w=w0, title='Portafolio', height=6, width=10, cmap = "tab20", ax=None)
#######Training data: 2015
#EnergyData
date_start = '6/1/15 12:00 AM'
date_end = '6/30/15 11:59 PM'
std_inv = 60 #in minutes
#Read data
data = DataFunctions.read_PSB_csv(date_start, date_end, 5, 20)
conv_critical, crac_critical, crac_normal, conv_normal, HVAC_critical, Elev, HVAC_normal, elec_total = DataFunctions.PSB_elec_split(
data)
H_t = DataFunctions.fix_data(conv_normal)
H_t = DataFunctions.fix_energy_intervals(H_t, 5, std_inv)
PlotFunctions.Plot_single(H_t)
#schedules
X_sch_t = DataFunctions.compile_features(H_t, date_start, std_inv)
#Weather Data
weather_file = r'/home/sseslab/Documents/SLC PSB data/WBB Weather Data/WBB_2015_June.csv'
weather_train = DataFunctions.read_weather_csv(weather_file, date_start,
date_end)
weather_train = DataFunctions.fix_weather_intervals(weather_train, 5, std_inv)
weather_min, weather_max = DataFunctions.get_normalize_params(weather_train)
#weather_train = DataFunctions.normalize_vector(weather_train, weather_min, weather_max)
weather_train = DataFunctions.interpolate_nans(weather_train)
X_t, Y_t, X_e, Y_e, train_list, test_list = InterpolateFunctions.train_test_split(
X_sch_t, H_t)
best = InterpolateFunctions.imputate_optimize(X_t, Y_t)
H_e = numpy.reshape(H_e, (H_e.shape[0] * 24, 1))
Y_lstm = numpy.reshape(Y_lstm, (Y_lstm.shape[0] * 24, 1))
Y_lstm2 = numpy.reshape(Y_lstm2, (Y_lstm2.shape[0] * 24, 1))
t_train = numpy.arange(0, len(H_t))
t_test = numpy.arange(len(H_t), len(H_t) + len(Y_lstm2))
t_array = numpy.arange(0, len(Y_lstm2))
e_deep = (MathFunctions.rms_flat(Y_lstm2 -
H_e)) / (MathFunctions.rms_flat(H_e))
e_deep2 = (MathFunctions.rms_flat(Y_lstm2 -
H_e)) / (MathFunctions.rms_flat(H_t))
print e_deep
print e_deep2
PlotFunctions.Plot_double(t_array, H_e, t_array, Y_hour, 'Actual conv power',
'LSTM conv power', 'k-', 'r-', "fig_77i1a.eps")
#####################
###Build 1-min model:
def objective(params):
optimize_model = build_lstm_v1.lstm_model_110(params, train_data2.shape[2],
60)
loss_out = NNFunctions.model_optimizer_101(optimize_model, train_data2,
H_t2, val_data2, H_val2, 5)
return {'loss': loss_out, 'status': STATUS_OK}
trials = Trials()
best2 = fmin(objective, space2, algo=tpe.suggest, trials=trials, max_evals=1)
def main(options, args):
plotfunc.SetupStyle()
# The nominal case, where you use an algorithm inside the predefined file
macroFileName = 'PicoXaodSkimAlgos.h'
algName = options.alg
# If a colon is used, it is to specify a different file path
if ':' in options.alg:
macroFileName = options.alg.split(':')[0]
algName = options.alg.split(':')[-1]
isLoaded = ROOT.gROOT.LoadMacro(macroFileName)
if (isLoaded < 0):
print 'Error! Macro compilation of file %s failed. See error msgs.' % (
macroFileName)
sys.exit()
else:
print 'Loaded macro file %s' % (macroFileName)
print 'Skimming using the \"%s\" algorithm...' % (algName)
# Here "b" stands for simulation
files_b, trees_b, keys_b = anaplot.GetTreesFromFiles(
options.bkgs, treename=options.treename)
files_d, trees_d, keys_d = anaplot.GetTreesFromFiles(
options.data, treename=options.treename)
simcuts = ''
if ''.join(options.cuts + options.truthcuts):
simcuts = '(' + ' && '.join(options.cuts + options.truthcuts) + ')'
dcuts = '' # cuts applied to data
if ''.join(options.cuts + options.blindcut):
dcuts = '(' + ' && '.join(options.cuts + options.blindcut) + ')'
if not os.path.exists(options.outdir):
os.makedirs(options.outdir)
for k in keys_d:
outfilename = os.path.basename(files_d[k].GetName()).replace(
'.root', '')
outfilename = '%s_pico' % (outfilename)
outfilename = anaplot.CleanUpName(outfilename,
originalIsDirectoryName=True,
forFileName=True)
histname = anaplot.CleanUpName(k, originalIsDirectoryName=True)
print 'Making picoXaod for %s' % (k)
getattr(ROOT, algName)(files_d[k], trees_d[k], histname, dcuts,
','.join(options.variables), options.outdir,
outfilename)
for k in keys_b:
outfilename = os.path.basename(files_b[k].GetName()).replace(
'.root', '')
outfilename = '%s_pico' % (outfilename)
outfilename = anaplot.CleanUpName(outfilename,
originalIsDirectoryName=True,
forFileName=True)
histname = anaplot.CleanUpName(k, originalIsDirectoryName=True)
print 'Making picoXaod for %s' % (k)
getattr(ROOT, algName)(files_b[k], trees_b[k], histname, simcuts,
','.join(options.variables), options.outdir,
outfilename)
print 'done.'
return
def main_singleCategory(options, args):
if options.category in [30, 31]:
print 'Error! Too few AF2 stats. Not going to do it.'
return
cans = []
options.outdir = ''
if options.functions:
flist = options.functions.split(',')
options.outdir += '_'.join(flist)
elif options.family == 'official':
#flist = ['Exponential','ExpPoly2','ExpPoly3','Bern4','Bern5','Pow','Pow2','Laurent0','Laurent1','Laurent2']
flist = [
'Exponential', 'ExpPoly2', 'Bern4', 'Bern5', 'Pow', 'Pow2',
'Laurent1', 'Laurent2'
]
options.outdir += 'official_functions'
elif options.family == 'selected':
flist = [Tools.selected[Tools.categories[options.category]]]
options.outdir += 'selected_functions'
print flist
elif options.family == 'ExpPoly':
flist = ['Exponential'] + list('ExpPoly%d' % (d) for d in range(2, 4))
options.outdir += 'exppoly_family'
elif options.family == 'Laurent':
flist = list('Laurent%d' % (d) for d in range(0, 3))
options.outdir += 'Laurent_family'
elif options.family == 'PolyOverX4':
flist = ['1/x^4'] + list('poly%d/x^4' % (d) for d in range(1, 6))
options.outdir += 'PolyOverX4_family'
elif options.family == 'Bernstein':
flist = list('Bern%d' % (d) for d in range(4, 6))
options.outdir += 'Bern_family'
elif options.family == 'PowerSum':
flist = ['Pow', 'Pow2']
options.outdir += 'PowerSum_family'
else:
flist = [
'Exponential', # bad
'ExpPoly2',
'ExpPoly3',
]
options.outdir += '_'.join(flist)
options.outdir += '_c%02d_%s' % (options.category,
Tools.categories[options.category])
functions = []
Tools.PopulateFunctionList(functions, flist)
Tools.LinkFunctionsForFtest(functions)
if len(functions) == 0:
print 'Error! no functions loaded!'
import sys
sys.exit()
for f in functions:
f.SetCategory(options.category)
f.SetFileName(options.file)
f.SetSignalWS(options.signalws)
f.Initialize()
##
## Plot the data and af2 (fit bkg-only just before this.) SpuriousSignal_05_M17_ggH_1J_BSM
##
for f in functions:
f.function.fitTo(f.data, *(Tools.args_bkgonly))
cans.append(
plotfunc.RatioCanvas(
"TemplateFit_%02d_%s" %
(options.category, Tools.categories[options.category]),
"main plot", 600, 500))
functions[0].af2hist.SetMarkerSize(0)
#rebin = Tools.RebinUntilSmallErrors(functions[0].af2hist,0,Tools.lower_range,Tools.upper_range,errormax=0.3)
rebin = 5
functions[0].af2hist.Rebin(rebin)
binwidth = functions[0].af2hist.GetBinWidth(1)
bins = int((Tools.upper_range - Tools.lower_range) / float(binwidth))
functions[0].af2hist.SetTitle('#gamma#gamma MC')
functions[0].af2hist.SetLineWidth(2)
if options.family == 'selected':
functions[0].af2hist.SetLineColor(ROOT.kGray + 2)
functions[0].af2hist.SetFillColor(0)
plotfunc.AddHistogram(cans[-1], functions[0].af2hist, drawopt='')
for i, f in enumerate(functions):
f.obsVar.setBins(int(bins))
f.bins = bins
f.frame = f.obsVar.frame()
if i:
f.af2hist.Rebin(rebin)
f.af2_rebinned = ROOT.RooDataHist('af2_rebinned', '',
ROOT.RooArgList(f.obsVar), f.af2hist,
1.)
#f.af2_rebinned.plotOn(f.frame,ROOT.RooFit.Range("lower,upper"))
#f.function.plotOn(f.frame,ROOT.RooFit.NormRange("lower,upper"),ROOT.RooFit.Range("all"))
f.chisquare = Tools.GetChiSquare_ForSpuriousSignal(
f.frame, f.af2_rebinned, f.function, f.ndof)
f.pvalue_chi2 = ROOT.TMath.Prob(f.chisquare * (f.bins - 1 - f.ndof),
f.bins - 1 - f.ndof)
print 'Original chi2: %2.6f p-value: %2.6f' % (f.chisquare,
f.pvalue_chi2)
f.minNll = 0
curve = f.frame.getCurve()
curve.SetMarkerSize(0)
curve.SetLineWidth(1)
curve.SetTitle(f.name)
curve.SetLineWidth(2)
# resid = f.frame.residHist(); resid.SetMarkerSize(0)
# plotfunc.AddRatioManual(cans[-1],curve,resid,drawopt1='l',drawopt2='l')
pull = f.frame.pullHist()
pull.SetMarkerSize(0.7)
# for special
if options.family == 'selected':
color = {
'Pow': ROOT.kBlack + 0,
'Exponential': ROOT.kRed + 1,
'ExpPoly2': ROOT.kBlue + 1,
'Bern3': ROOT.kGreen + 1,
'Bern4': ROOT.kMagenta + 1,
'Bern5': ROOT.kOrange + 1,
}.get(f.name)
pull.SetMarkerColor(color)
pull.SetLineColor(color)
curve.SetLineColor(color)
curve.SetFillColor(0)
plotfunc.AddRatioManual(cans[-1],
curve,
pull,
drawopt1='l',
drawopt2='p')
f.chisquare_toy = Tools.ChiSquareToys_ForSpuriousSignal(
f, options.outdir)
if not options.family == 'selected':
plotfunc.SetColors(cans[-1])
plotfunc.FormatCanvasAxes(cans[-1])
plotfunc.SetXaxisRanges(cans[-1], Tools.lower_range, Tools.upper_range)
plotfunc.SetAxisLabels(cans[-1], 'm_{#gamma#gamma} [GeV]', 'entries',
'pull')
the_text = [
plotfunc.GetAtlasInternalText(),
plotfunc.GetSqrtsText(13) + ', ' + plotfunc.GetLuminosityText(36.1),
Tools.CategoryNames[Tools.categories[options.category]]
]
plotfunc.DrawText(cans[-1],
the_text,
0.19,
0.70,
0.59,
0.91,
totalentries=3)
plotfunc.MakeLegend(cans[-1], 0.60, 0.70, 0.90, 0.91, totalentries=3)
#plotfunc.GetTopPad(cans[-1]).GetPrimitive('legend').AddEntry(0,'^{ }background-only fit','')
#list(plotfunc.GetTopPad(cans[-1]).GetPrimitive('legend').GetListOfPrimitives())[-1].SetLabel('^{ }background-only fit')
plotfunc.SetYaxisRanges(plotfunc.GetBotPad(cans[-1]), -4, 4)
taxisfunc.AutoFixYaxis(plotfunc.GetTopPad(cans[-1]),
ignorelegend=False,
minzero=True)
for can in cans[:-1]:
plotfunc.FormatCanvasAxes(can)
anaplot.UpdateCanvases(cans, options)
if not options.batch:
raw_input('Press enter to exit')
anaplot.doSaving(options, cans)
return
else:
H1 = H_t.copy()
small_list, large_list = InterpolateFunctions.interpolate_main(
train_data, H_t, start_day, end_day, cons_points)
H_t = InterpolateFunctions.interp_linear(
H_t, small_list) #H_t is beign changed as numpy arrays are mutable
H1 = InterpolateFunctions.interp_linear(H1, small_list)
H_t = H_t[:, None] #changing numpy array shape to fit the function
train_interp, dummy1, dummy2 = DataFunctions.normalize_103(
train_data, train_data, train_data)
Y_t, Y_NN = InterpolateFunctions.interp_LSTM(train_interp, H_t, large_list)
PlotFunctions.Plot_interpolate(H1[s0 * 24:start_day * 24],
Y_t[start_day * 24:end_day * 24],
Y_NN[start_day * 24:end_day * 24],
H1[start_day * 24:end_day * 24],
H1[end_day * 24:s1 * 24])
e_interp = InterpolateFunctions.interpolate_calculate_rms(
H1[start_day * 24:end_day * 24], Y_t[start_day * 24:end_day * 24])
e_NN = InterpolateFunctions.interpolate_calculate_rms(
H1[start_day * 24:end_day * 24], Y_NN[start_day * 24:end_day * 24])
print e_interp
print e_NN
H_t = Y_t.copy()
numpy.save('H1_file_MD1.npy', H_t)
#Aggregating data on a daily basis
conv_hour_to_day = 24
H_mean_t, H_sum_t, H_min_t, H_max_t = DataFunctions.aggregate_data(
Y_NN = NN_savemodel.predict(X3)
#Y_NN = numpy.reshape(Y_NN, (Y_NN.shape[0]*24, 1))
e_NN = (MathFunctions.rms_flat(Y_NN - H_e)) / (MathFunctions.rms_flat(H_e))
e_NN2 = (MathFunctions.rms_flat(Y_NN - H_e)) / (MathFunctions.rms_flat(H_t))
print e_NN
print e_NN2
print "R2: "
print r2_score(Y_lstm2, H_e)
#Calculate p-value
a1, a2 = pearsonr(Y_lstm2, H_e)
print "LSTM rho-value: "
print a1
b1, b2 = pearsonr(Y_NN, H_e)
print "NN rho-value"
print b1
####Plotting
PlotFunctions.Plot_double(t_array, H_e, t_array, Y_lstm2, 'Actual conv power',
'LSTM conv power', 'k-', 'r-', "fig_77a.eps")
PlotFunctions.Plot_triple(t_train, H_t, t_test, Y_lstm2, t_test, H_e,
'Training Data', 'LSTM predictions',
'Test Data (actual)', 'k-', 'r-', 'b-',
"fig_77b.eps")
PlotFunctions.Plot_quadruple(t_train, H_t, t_test, Y_lstm2, t_test, Y_NN,
t_test, H_e, 'Training Data', 'LSTM predictions',
'MLP Predictions', 'Test Data (actual)', 'k-',
'r-', 'y-', 'b-', "fig_77d.eps")
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Train', 'Validation'], loc='upper right')
plt.savefig(path_save_pic + "performance.png")
plt.show()
#end
params_post = model.get_weights()
print("Model evaluation on test data: loss and accuracy\n",
model.evaluate(Xtest,Ytest, verbose = 2))
if (plot['distributions']):
numLayers = len(model.layers)
pfn.jointPlotter(numLayers, path_save_pic, params_pre,
params_post, plot_hist = False)
#end
if (plot['network']):
plotNet = npl.plotNet(params_post, path_save_pic,
trained = True, asGraph = False)
plotNet.plotNetFunction()
#end
streams.check_create_directory(path_save_model + r'\{}'.format(dataset_id))
model.save(path_save_model + r'\\' + dataset_id + \
r'\\model_' + dataset_id + ".h5")
model.save(path_save_model + r'\{}\model_{}.h5'.format(dataset_id,
dataset_id))
return model, history.history['acc']
#end
def Main():
#Load MNIST ****************************************************************
#mnist.pkl.gz(50000） Accuracy 0.96
#mnist_expanded.pkl.gz(250000） Accuracy 0.97
fn = "..\\data\\mnist.pkl.gz" #".datamnist_expanded.pkl.gz"
lstTrain, lstV, lstT = mnist_loader.load_data_wrapper(fn)
lstTrain = list(lstTrain)
lstV = list(lstV)
lstT = list(lstT)
# *********************************************************************
path = "..\\TmpLogs\\"
if not os.path.isdir(path):
os.mkdir(path)
fnNetworkData = "{}{}_NetData".format(path, rn.RvNeuralNetwork.__name__)
fnSaved = ""
#Hyper pameters -------------------------------------------
loop = 10 # loop effect，10, 30 all above 0.95
stepNum = 10 # stepNum effect, 10->0.9, 100->0.5
learnRate = 0.1 # learnRate and lmbda will affect each other
lmbda = 5.0 #add lmbda(Regularization) to solve overfitting
# Training ***********************************************************
# Ask DoTraining-
DoTraining = ri.Ask_YesNo("Execute Training?", "y")
if DoTraining:
"""
[784,50,10],
loop=10, 0.9695
loop=100, 0.9725
"""
# 建立 RvNeuralNetWork----------------------------------------------
inputNeusNum = len(lstTrain[0][0])
lyr1NeuNum = 50
lyr2NeuNum = len(lstTrain[0][1])
lyrsNeus = [inputNeusNum, lyr1NeuNum]
lyrsNeus = ri.Ask_Add_Array_Int("Input new layer Neurons num.",
lyrsNeus, lyr1NeuNum)
lyrsNeus.append(lyr2NeuNum)
#net = rn.RvNeuralNetwork( \
# rn.RvNeuralNetwork.LayersNeurons_To_RvNeuralLayers(lyrsNeus))
#net = rn.RvNeuralNetwork.Class_Create_LayersNeurons(lyrsNeus)
#net = rn.RvNeuralNetwork(lyrsNeus) # ([784,50,10])
cnvLyrSId = 0
lyrObjs = []
# Add RvConvolutionLayer--------------------------------
EnableCovolutionLayer = ri.Ask_YesNo("Add ConvolutionLayer?", "y")
if EnableCovolutionLayer:
lyrsNeus, cnvLyrSId, cnvLyr = Add_ConvLayer(
lyrObjs, lyrsNeus, inputNeusNum, cnvLyrSId)
# Create Layer Object array -------------------------------
for iLyr in range(cnvLyrSId, len(lyrsNeus) - 1):
lyrObjs.append(
rn.RvNeuralLayer([lyrsNeus[iLyr], lyrsNeus[iLyr + 1]]))
net = rn.RvNeuralNetwork(lyrObjs)
# Ask nmtivation ------------------_----------
enumActivation = ri.Ask_Enum("Select Activation method.",
nm.EnumActivation,
nm.EnumActivation.afReLU)
for lyr in net.NeuralLayers:
lyr.ClassActivation, lyr.ClassCost = \
nm.Get_ClassActivation(enumActivation)
net.Motoring_TrainningProcess = rn.Debug
net.NetEnableDropOut = ri.Ask_YesNo("Execute DropOut?", "n")
if net.NetEnableDropOut:
enumDropOut = ri.Ask_Enum("Select DropOut Method.",
nm.EnumDropOutMethod,
drpOut.eoSmallActivation)
rn.gDropOutRatio = ri.Ask_Input_Float("Input DropOut ratio.",
rn.gDropOutRatio)
net.Set_DropOutMethod(enumDropOut, rn.gDropOutRatio)
monitoring = ri.Ask_YesNo("Watch training process?", "y")
net.Motoring_TrainningProcess = monitoring
# Caculate proper hyper pameters ---
DoEvaluate_ProperParams = ri.Ask_YesNo(
"Auto-caculating proper hyper pameters?", "n")
if DoEvaluate_ProperParams:
loop, stepNum, learnRate, lmbda = rf.Evaluate_BestParam_lmbda(
net, net.Train, lstTrain[:1000], lstV[:500], loop, stepNum,
learnRate, lmbda)
loop, stepNum, learnRate, lmbda = rf.Evaluate_BestParam_learnRate(
net, net.Train, lstTrain[:1000], lstV[:500], loop, stepNum,
learnRate, lmbda)
else:
loop, stepNum, learnRate, lmbda = rf.Ask_Input_SGD(
loop, stepNum, learnRate, lmbda)
print(
"Hyper pameters: Loop({}), stepNum({}), learnRatio({}), lmbda({})\n"
.format(loop, stepNum, learnRate, lmbda))
start = time.time()
# 開始網路訓練-
net.Train(lstTrain, loop, stepNum, learnRate, lstV, lmbda)
dT = time.time() - start
fnSaved = rf.Save_NetworkDataFile(net, fnNetworkData, loop, stepNum,
learnRate, lmbda, dT)
# Ask DoPredict----------------------------------------------------
DoPredict = True
if DoPredict:
if (os.path.isfile(fnSaved)):
fn1 = fnSaved
else:
fn1 = ".\\{}_NetData_DontDelete.txt".format(
rn.RvNeuralNetwork.__name__)
rn.Debug_Plot = True #ri.Ask_YesNo("Plot Digits?", "n")
# Predict_Digits(net, lstT)
pltFn.Predict_Digits_FromNetworkFile(fn1, lstT, rn.Debug_Plot)
elif choice == 2:
X_t, Y_t, X_e, Y_e, train_list, test_list = InterpolateFunctions.train_test_split(
train_data, H_t)
best = InterpolateFunctions.imputate_optimize(X_t, Y_t)
Y_p = NNFun_PSB.PSB_model_DL(X_t, Y_t, X_e, best)
H_t = InterpolateFunctions.organize_pred(H_t, Y_p, test_list)
#numpy.save('Ht_file_HVAC1.npy', H_t)
else:
H1 = H_t.copy()
small_list, large_list = InterpolateFunctions.interpolate_main(
train_data, H_t, start_day, end_day, cons_points)
H_t = InterpolateFunctions.interp_linear(
H_t, small_list) #H_t is beign changed as numpy arrays are mutable
H1 = InterpolateFunctions.interp_linear(H1, small_list)
PlotFunctions.Plot_interp_params()
H_t = H_t[:, None] #changing numpy array shape to fit the function
train_interp, dummy1, dummy2 = DataFunctions.normalize_103(
train_data, train_data, train_data)
Y_t, Y_NN = InterpolateFunctions.interp_LSTM(train_interp, H_t, large_list)
PlotFunctions.Plot_interpolate(H1[s0 * 24:start_day * 24],
Y_t[start_day * 24:end_day * 24],
Y_NN[start_day * 24:end_day * 24],
H1[start_day * 24:end_day * 24],
H1[end_day * 24:s1 * 24])
e_interp = InterpolateFunctions.interpolate_calculate_rms(
H1[start_day * 24:end_day * 24], Y_t[start_day * 24:end_day * 24])
e_NN = InterpolateFunctions.interpolate_calculate_rms(
H1[start_day * 24:end_day * 24], Y_NN[start_day * 24:end_day * 24])
#!/usr/bin/env python
import ROOT
import math
import PlotFunctions as plotfunc
import PyAnalysisPlotting as anaplot
import TAxisFunctions as taxisfunc
plotfunc.SetupStyle()
import Tools
import ChiSquareTools
import FunctionsModule
import os
ROOT.gROOT.LoadMacro('RooFitFunctions.h')
ROOT.RooMsgService.instance().setGlobalKillBelow(ROOT.RooFit.FATAL)
ROOT.RooMsgService.instance().setSilentMode(True)
fcns = {
'official': [
'Exponential', 'ExpPoly2', 'ExpPoly3', 'Bernstein_4', 'Bernstein_5',
'Pow'
],
'selected':
None, # to be filled in main_singleCategory
'ftest':
None, # to be filled in main_singleCategory
'ExpPoly': ['Exponential'] + list('ExpPoly%d' % (d) for d in range(2, 4)),
'Laurent':
list('Laurent%d' % (d) for d in range(0, 3)),
'PolyOverX4': ['1/x^4'] + list('poly%d/x^4' % (d) for d in range(1, 6)),
def main(options,args) :
mystyle = plotfunc.SetupStyle()
files_b,trees_b,keys_b = anaplot.GetTreesFromFiles(options.bkgs ,treename=options.treename,xAODInit=options.xAODInit)
files_s,trees_s,keys_s = anaplot.GetTreesFromFiles(options.signal,treename=options.treename,xAODInit=options.xAODInit)
files_d,trees_d,keys_d = anaplot.GetTreesFromFiles(options.data ,treename=options.treename,xAODInit=options.xAODInit)
scales_b = anaplot.GetScales(files_b,trees_b,keys_b,options)
scales_s = anaplot.GetScales(files_s,trees_s,keys_s,options)
weight = options.weight
if ''.join(options.cuts+options.truthcuts) :
weight = (weight+'*(%s)'%(' && '.join(options.cuts+options.truthcuts).lstrip('& ').rstrip('& '))).lstrip('*')
dweight = '' # weight value (and cuts) applied to data
if ''.join(options.cuts+options.blindcut) :
dweight = '('+' && '.join(options.cuts+options.blindcut).lstrip('& ').rstrip('& ')+')'
cans = []
# get the histograms from the files
for v in options.variables :
bkg_hists = []
sig_hists = []
data_hist = None
if options.data :
data_hists = anaplot.GetVariableHistsFromTrees(trees_d,keys_d,v,dweight,options,files=files_d)
data_hist = anaplot.MergeSamples(data_hists,options,requireFullyMerged=True)[0]
anaplot.PrepareDataHistos(data_hists,options)
if options.bkgs :
bkg_hists = anaplot.GetVariableHistsFromTrees(trees_b,keys_b,v,weight,options,scales=scales_b,files=files_b)
bkg_hists = anaplot.MergeSamples(bkg_hists,options)
anaplot.PrepareBkgHistosForStack(bkg_hists,options)
if options.signal :
sig_hists = anaplot.GetVariableHistsFromTrees(trees_s,keys_s,v,weight,options,scales=scales_s,files=files_s)
sig_hists = anaplot.MergeSamples(sig_hists,options)
anaplot.PrepareSignalHistos(sig_hists,options)
if options.normalize :
for hist in [data_hist] + bkg_hists + sig_hists :
if not hist : continue
hist.Scale(1/float(hist.Integral()))
if options.customnormalize :
options.customnormalize(v,sig_hists=sig_hists,bkg_hists=bkg_hists,data_hist=data_hist)
cans.append(anaplot.DrawHistos(v,options,bkg_hists,sig_hists,data_hist))
if options.afterburner :
for can in cans :
options.afterburner(can)
anaplot.UpdateCanvases(cans,options)
if options.xAODInit :
ROOT.xAOD.ClearTransientTrees()
if not options.batch :
import code
code.interact(banner='Pausing... Press Contol-D to exit.',local=locals())
anaplot.doSaving(options,cans)
print 'done.'
return
def main_singleCategory(options, args):
if options.category in [19, 23]: # ggH_0J_Cen is 0
print 'Error! This category has been merged away.'
return
cans = []
options.outdir = ''
if options.analysis == 'couplings2017':
category_name = Tools.categories_couplings2017[options.category]
category_title = Tools.CategoryNames_couplings2017[category_name]
fcns['selected'] = [Tools.selected_couplings2017[category_name]]
fcns['ftest'] = ChiSquareTools.ftest[category_name]
background_label = '#gamma#gamma'
lumi = 36.1
elif options.analysis == 'ysy':
category_name = Tools.categories_ysy[options.category]
category_title = Tools.CategoryNames_ysy[category_name]
fcns['selected'] = [Tools.selected_ysy[category_name]]
fcns['ftest'] = ChiSquareTools.ftest[category_name]
background_label = 'll#gamma'
lumi = 139
else:
print('Error - do not understand analysis name %s' %
(options.analysis))
import sys
sys.exit()
family_name = {
'official': 'official_functions',
'selected': 'selected_function',
}.get(options.family, options.family + '_family')
# A list of options is provided
if options.functions:
flist = options.functions.split(',')
options.outdir += '_'.join(flist)
# Otherwise a family must be provided:
else:
flist = fcns.get(options.family)
options.outdir += family_name
options.outdir += '_c%02d_%s' % (options.category, category_name)
functions = []
FunctionsModule.PopulateFunctionList(functions, flist, options.lower,
options.upper)
ChiSquareTools.LinkFunctionsForFtest(functions)
if len(functions) == 0:
print 'Error! no functions loaded!'
import sys
sys.exit()
for f in functions:
f.SetAnalysis(options.analysis)
f.SetCategory(options.category)
f.SetFileName(options.file)
f.Initialize()
# rebin = 5
# if options.category > 16 :
# rebin = 1
# if options.category > 23 : # ttH categories
# rebin = 10
# Rebin to get 1 bin per GeV
rebin = int(functions[0].datahist.GetNbinsX() / 55)
print 'Proceeding with Data histogram:', functions[0].datahist
functions[0].datahist.Rebin(rebin)
for i, f in enumerate(functions):
if i:
f.datahist.Rebin(rebin)
f.datasb_rebinned = ROOT.RooDataHist('data_real_rebinned', '',
ROOT.RooArgList(f.obsVar),
f.datahist, 1.)
if not os.path.exists(options.outdir):
os.makedirs(options.outdir)
ftest_text = ''
cans.append(
plotfunc.RatioCanvas(
"Ftests_%02d_%s" % (options.category, category_name), "main plot",
600, 500))
functions[0].datahist.SetBinErrorOption(ROOT.TH1.kPoisson)
plotfunc.AddHistogram(cans[-1], functions[0].datahist)
##
## Ftests to sideband
##
for f in functions:
if not hasattr(f, 'ftest_function'):
continue
nbins_blind = int(f.datasb_rebinned.numEntries() - 10)
print 'Sideband numEntries:', f.datasb_rebinned.numEntries(
), 'nbins:', nbins_blind
# ndof_bins = nbins_blind-f.ndof-1
# ndof_bins_2 = nbins_blind-f.ftest_function.ndof-1
ndof_bins = nbins_blind - f.ndof
ndof_bins_2 = nbins_blind - f.ftest_function.ndof
#
# Fitting data sidebands - first function
#
print 'Fitting data sidebands'
print f.PrintParameters()
ChiSquareTools.FitForChi2_DataSidebands(f)
print f.PrintParameters()
print 'Fitting data sidebands done'
ftest_text += f.PrintParameters() + '\n'
chi2 = ROOT.GetChiSquare(f.obsVar, f.function_ext, f.datasb_rebinned,
ndof_bins)
pvalue_chi2 = ROOT.TMath.Prob(chi2 * (ndof_bins), ndof_bins)
#
# Plotting stuff - first function
#
Tools.ClearRooPlot(f.frame)
ChiSquareTools.NormalizeToSideband(f)
f.datasb_rebinned.plotOn(
f.frame, ROOT.RooFit.DataError(ROOT.RooAbsData.Poisson))
print 'about to plot'
#f.workspace.var('a1').setVal(-1.6)
print f.PrintParameters()
f.function_ext.plotOn(f.frame, *(Tools.plotOptions_sb_all))
print f.PrintParameters()
print 'about to plot done'
curve = f.frame.getCurve()
curve.SetMarkerSize(0)
curve.SetLineWidth(2)
curve.SetTitle('%s, p(#chi^{2}) = %2.1f%%' %
(f.name, pvalue_chi2 * 100.))
curve.SetLineColor(ROOT.kRed + 1)
curve.SetFillColor(0)
pull = f.frame.pullHist()
pull.SetMarkerSize(0.7)
pull.SetLineColor(ROOT.kRed + 1)
pull.SetMarkerColor(ROOT.kRed + 1)
plotfunc.AddRatioManual(cans[-1],
curve,
pull,
drawopt1='l',
drawopt2='p')
#
# Fitting data sidebands - first function
#
print 'Fitting data sidebands (other function)'
print f.ftest_function.PrintParameters()
ChiSquareTools.FitForChi2_DataSidebands(f.ftest_function)
print f.ftest_function.PrintParameters()
print 'Fitting data sidebands (other function) done'
ftest_text += f.ftest_function.PrintParameters() + '\n'
chi2_2 = ROOT.GetChiSquare(f.obsVar, f.ftest_function.function_ext,
f.datasb_rebinned, ndof_bins_2)
pvalue_chi2_2 = ROOT.TMath.Prob(chi2_2 * (ndof_bins_2), ndof_bins_2)
#
# Plotting stuff - other function
#
Tools.ClearRooPlot(f.frame)
ChiSquareTools.NormalizeToSideband(f.ftest_function)
f.datasb_rebinned.plotOn(f.frame)
f.ftest_function.function_ext.plotOn(f.frame,
*(Tools.plotOptions_sb_all))
curve = f.frame.getCurve()
curve.SetMarkerSize(0)
curve.SetLineWidth(2)
curve.SetTitle('%s, p(#chi^{2}) = %2.1f%%' %
(f.ftest_function.name, pvalue_chi2_2 * 100.))
curve.SetLineColor(ROOT.kAzure - 2)
curve.SetFillColor(0)
pull = f.frame.pullHist()
pull.SetMarkerSize(0.7)
pull.SetLineColor(ROOT.kAzure - 2)
pull.SetMarkerColor(ROOT.kAzure - 2)
plotfunc.AddRatioManual(cans[-1],
curve,
pull,
drawopt1='l',
drawopt2='p')
ftest = ChiSquareTools.GetF(chi2, chi2_2, ndof_bins, ndof_bins_2)
ftest_text += 'chi2/ndf: %2.5f\n' % (chi2)
ftest_text += 'chi2_2/ndf: %2.5f\n' % (chi2_2)
ftest_text += 'chi2: %2.5f\n' % (chi2 * ndof_bins)
ftest_text += 'chi2_2: %2.5f\n' % (chi2_2 * ndof_bins_2)
ftest_text += 'ftest: %2.5f\n' % (ftest)
#p_ftest = 1.0 - ROOT.TMath.FDistI(ftest,1,ndof_bins_2)
p_ftest = 1.0 - ROOT.TMath.FDistI(ftest, ndof_bins - ndof_bins_2,
ndof_bins_2)
ftest_text += 'p_ftest: %2.5f\n' % (p_ftest)
#
# Throw Toys
#
print 'Running toy Ftests'
fisher_dist = ChiSquareTools.ToyFtest(f, f.ftest_function, ftest,
options.outdir, options.ntoys)
print 'Running toy Ftests done'
if fisher_dist.Integral(0, 100000):
ftest_text += 'p_ftest_toys: %2.5f\n' % (
fisher_dist.Integral(fisher_dist.FindBin(ftest), 100000) /
float(fisher_dist.Integral(0, 100000)))
# chi2_lowerSideBand = ROOT.RooChi2Var("chi2_low","chi2_low",f.function_ext,f.data_realdata,ROOT.RooFit.DataError(ROOT.RooAbsData.Poisson),ROOT.RooFit.Range("lower"));
# chi2_upperSideBand = ROOT.RooChi2Var("chi2_upp","chi2_upp",f.function_ext,f.data_realdata,ROOT.RooFit.DataError(ROOT.RooAbsData.Poisson),ROOT.RooFit.Range("upper"));
# chi2_all = ROOT.RooChi2Var("chi2_all","chi2_all",f.function_ext,f.data_realdata,ROOT.RooFit.Range("lower,upper"));
# chi2_lowerSideBand = ROOT.RooChi2Var("chi2_low","chi2_low",f.function_ext,f.data_realdata,ROOT.RooFit.Range("lower"));
# chi2_upperSideBand = ROOT.RooChi2Var("chi2_upp","chi2_upp",f.function_ext,f.data_realdata,ROOT.RooFit.Range("upper"));
# print 'chi2_all .getValV()',chi2_all.getValV()
# print 'chi2_lowerSideBand.getValV()',chi2_lowerSideBand.getValV()
# print 'chi2_upperSideBand.getValV()',chi2_upperSideBand.getValV()
# print 'total:',(chi2_lowerSideBand.getValV()+chi2_upperSideBand.getValV()) / float(f.bins-1-f.ndof)
# Get the chi2 from the background-only fit
# print 'f.frame.chiSquare(1+f.ndof-10)',f.frame.chiSquare(1+f.ndof)*(f.bins-1-f.ndof)
# f.chisquare_sb = f.frame.chiSquare(1+f.ndof-10) # n-1 bins MINUS 10 BINS
# f.pvalue_chi2_sb = ROOT.TMath.Prob(f.chisquare_sb*(f.bins-1-f.ndof-10),f.bins-1-f.ndof-10)
# print 'chisquare:',f.chisquare_sb,f.pvalue_chi2_sb
continue
curve = f.frame.getCurve()
curve.SetMarkerSize(0)
curve.SetLineWidth(1)
curve.SetTitle(f.name)
curve.SetLineWidth(2)
# resid = f.frame.residHist(); resid.SetMarkerSize(0)
# plotfunc.AddRatioManual(cans[-1],curve,resid,drawopt1='l',drawopt2='l')
pull = f.frame.pullHist()
pull.SetMarkerSize(0.7)
# for special
if options.family == 'selected':
color = {
'Pow': ROOT.kBlack + 0,
'Exponential': ROOT.kRed + 1,
'ExpPoly2': ROOT.kBlue + 1,
'Bern3': ROOT.kGreen + 1,
'Bern4': ROOT.kMagenta + 1,
'Bern5': ROOT.kOrange + 1,
}.get(f.name)
pull.SetMarkerColor(color)
pull.SetLineColor(color)
curve.SetLineColor(color)
curve.SetFillColor(0)
plotfunc.AddRatioManual(cans[-1],
curve,
pull,
drawopt1='l',
drawopt2='p')
plotfunc.SetAxisLabels(cans[-1], 'm_{#gamma#gamma} [GeV]', 'entries',
'pull')
the_text = [
plotfunc.GetAtlasInternalText(),
plotfunc.GetSqrtsText(13) + ', ' + plotfunc.GetLuminosityText(36.1),
category_title,
'1-p(F_{%d%d}) = %2.1f%%' %
(functions[0].ndof, functions[0].ftest_function.ndof, p_ftest * 100)
]
plotfunc.DrawText(cans[-1],
the_text,
0.19,
0.63,
0.59,
0.91,
totalentries=4)
plotfunc.MakeLegend(cans[-1], 0.57, 0.63, 0.90, 0.91, totalentries=4)
plotfunc.SetYaxisRanges(plotfunc.GetBotPad(cans[-1]), -4, 4)
plotfunc.SetXaxisRanges(cans[-1], functions[0].lower_range,
functions[0].upper_range)
taxisfunc.AutoFixYaxis(plotfunc.GetTopPad(cans[-1]), forcemin=0.001)
print ftest_text
a = open('%s/ftests.txt' % (options.outdir), 'w')
a.write(options.file + '\n')
a.write(ftest_text + '\n')
a.close()
for can in cans:
plotfunc.FormatCanvasAxes(can)
anaplot.UpdateCanvases(cans, options)
if not options.batch:
raw_input('Press enter to exit')
anaplot.doSaving(options, cans)
return
