def saveWFs():
from ROOT import TChain
# 1 P3KJR 9472 5254 C1P3D2 610 2115.519087 0.831066 -0.000310 1
# 1 P3KJR 9544 14210 C1P1D2 582 2083.442207 4.322209 -0.000296 2
# 1 P3KJR 10614 8717 C1P1D2 582 2167.199298 1.929810 -0.000396 19
# 1 P3KJR 13345 31547 C1P1D3 580 2013.263961 14.821248 -0.000213 46
tt = TChain("skimTree")
tt.Add("%s/latSkimDS%d*.root" % (dsi.latDir, 1))
chan = 582
ds = 1
truncLo, truncHi = 0, 2
if ds == 6 or ds == 2: truncLo = 4
# wf 1 - passing DCR
tCut1 = "trapENFCal > 2000 && trapENFCal < 2200 && channel==%d && avse>-1 && dcr99<0" % chan
n = tt.Draw("Entry$:Iteration$", tCut1, "goff")
evt, itr = tt.GetV1(), tt.GetV2()
evtList = [[int(evt[i]), int(itr[i])] for i in range(n)]
nEnt = len(set([evt[i] for i in range(n)]))
passWF = []
for i, (iE, iH) in enumerate(evtList):
tt.GetEntry(iE)
run = tt.run
hitE = tt.trapENFCal.at(iH)
dcr99 = tt.dcr99.at(iH)
wf = tt.MGTWaveforms.at(iH)
signal = wl.processWaveform(wf, truncLo, truncHi)
wf1 = signal.GetWaveBLSub()
ts1 = signal.GetTS()
passWF.append((ts1, wf1, hitE, dcr99))
print("%d / %d Run %d chan %d trapENF %.1f" %
(i + 1, len(evtList), run, chan, hitE))
# wf 2 - failing DCR
tCut2 = "trapENFCal > 1500 && channel==%d && dcr99>0.001" % chan
n = tt.Draw("Entry$:Iteration$", tCut2, "goff")
evt, itr = tt.GetV1(), tt.GetV2()
evtList = [[int(evt[i]), int(itr[i])] for i in range(n)]
nEnt = len(set([evt[i] for i in range(n)]))
failWF = []
for i, (iE, iH) in enumerate(evtList):
tt.GetEntry(iE)
run = tt.run
hitE = tt.trapENFCal.at(iH)
dcr99 = tt.dcr99.at(iH)
wf = tt.MGTWaveforms.at(iH)
signal = wl.processWaveform(wf, truncLo, truncHi)
wf2 = signal.GetWaveBLSub()
ts2 = signal.GetTS()
failWF.append((ts2, wf2, hitE, dcr99))
print("%d / %d Run %d chan %d trapENF %.1f" %
(i + 1, len(evtList), run, chan, hitE))
np.savez("../data/dcr-example.npz", passWF, failWF)
def PlotTemplateWaveforms(waveTree, calDict):
""" Check the wf's grabbed from an M1 calibration run. """
# plt.style.use('mjWaveforms') # see ~/.matplotlib/stylelib
# optional: redefine calDict with only one or two channels for debugging
# calDict = {672:18}
# Make a figure
fig = plt.figure(figsize=(9, 7), facecolor='w')
ax = plt.subplot(111)
ax.set_xlabel("time (ns)")
ax.set_ylabel("ADC")
jet = plt.get_cmap('jet')
cNorm = colors.Normalize(vmin=578, vmax=672)
scalarMap = cmx.ScalarMappable(norm=cNorm, cmap=jet)
# Plot multiple waveforms
for chan in calDict:
waveTree.GetEntry(calDict[chan])
waveMGT = waveTree.event.GetWaveform(waveTree.itr)
signal = wl.processWaveform(waveMGT)
waveRaw = signal.GetWaveRaw()
waveOff = signal.GetOffset() / 1e9 # seconds
waveTS = signal.GetTS()
colorVal = scalarMap.to_rgba(chan)
label = ('%d' % chan)
ax.plot(waveTS, waveRaw, color=colorVal, label=label)
# make a legend and show the plot.
handles, labels = ax.get_legend_handles_labels()
ax.legend(handles, labels, loc='upper right')
plt.show()
def animate(iList):
entry = gatTree.GetEntryNumber(iList)
gatTree.LoadTree(entry)
gatTree.GetEntry(entry)
nChans = gatTree.channel.size()
numPass = gatTree.Draw("channel", theCut, "GOFF", 1, iList)
chans = gatTree.GetV1()
chanList = list(set(int(chans[n]) for n in range(numPass)))
# Loop over hits passing cuts
hitList = (iH for iH in range(nChans)
if gatTree.channel.at(iH) in chanList
) # a 'generator expression'
for iH in hitList:
wf = gatTree.MGTWaveforms.at(iH)
iEvent = entry
run = gatTree.run
chan = gatTree.channel.at(iH)
energy = gatTree.trapENFCal.at(iH)
dcr = gatTree.dcr99.at(iH)
# fs = gatTree.fitSlo.at(iH)
# rn = gatTree.riseNoise.at(iH)
signal = wl.processWaveform(wf)
waveRaw = signal.GetWaveRaw()
waveTS = signal.GetTS()
baseline = np.sum(waveRaw[:50]) / 50
# fill the figure
p1.set_ydata(waveRaw)
p1.set_xdata(waveTS)
# plt.title("Run %d Ch %d Entry %d trapENFCal %.1f fitSlo %.2f riseNoise %.2f" % (run,chan,iList,energy, fs, rn))
plt.title("Run %d Ch %d Entry %d trapENFCal %.1f dcr99 %.8f" %
(run, chan, iList, energy, dcr))
# dynamically scale the axes
xmin, xmax = np.amin(waveTS), np.amax(waveTS)
# a1.set_xlim([xmin,xmax])
a1.set_xlim([9000, xmax])
# if energy > 1000:
# a1.set_ylim(0,1000)
# else:
ymin, ymax = np.amin(waveRaw), np.amax(waveRaw)
# a1.set_ylim([ymin-abs(0.1*ymin),ymax+abs(0.1*ymax)])
a1.set_ylim([ymax - abs(0.35 * ymax), ymax + abs(0.2 * ymax)])
# a1.set_ylim([5000,6000])
# print(progress)
# print("%d / %d Run %d nCh %d chan %d trapE %.1f samp %d" % (iList,nList,run,nChans,chan,energy,wf.GetLength()))
if iList % 500 == 0 and iList != 0:
print("%d / %d entries saved (%.2f %% done)." %
(iList, nList, 100 * (float(iList) / nList)))
return p1,
def getTrap():
ds, run, chan = 1, 11085, 580 # C1P1D3
# trigger eff vals from old study: mu = 1.448, sig = 0.305
# 2016-9-15-P3LQG_Run60001538 C2P1D3 P42748B
# ds, run, chan = 4, 60001538, 1110
from ROOT import GATDataSet
gds = GATDataSet(run)
tt = gds.GetGatifiedChain()
bt = gds.GetBuiltChain()
tt.AddFriend(bt)
tCut = "channel==%d" % chan
n = tt.Draw("Entry$:Iteration$", tCut, "goff")
evt, itr = tt.GetV1(), tt.GetV2()
evtList = [[int(evt[i]), int(itr[i])] for i in range(n)]
trapOutput = []
pEvt = -1
for i, (iE, iH) in enumerate(evtList):
if iE != pEvt:
tt.GetEntry(iE)
pEvt = iE
hitE = tt.trapENFCal.at(iH)
wf = bt.event.GetWaveform(iH)
sig = wl.processWaveform(wf)
waveRaw = sig.GetWaveBLSub()
waveTS = sig.GetTS()
# mimic onboard energy trapezoid
eTrap = wl.trapFilter(waveRaw, 400, 180, 0)
nPad = len(waveRaw) - len(eTrap)
eTrap = np.pad(eTrap, (nPad, 0), 'constant')
eTrapTS = np.arange(0, len(eTrap) * 10., 10)
# trap timestamps shouldn't change, don't recalculate them
if i == 0:
pTS = eTrapTS
if not np.array_equal(eTrapTS, pTS):
print("Warning, the timestamps changed. Uggggg")
return
# print(i, iE, iH, hitE)
trapOutput.append(eTrap)
# np.savez("../data/trapOutput.npz",trapOutput,eTrapTS) # original ds4 run
np.savez("../data/trapOutput-2.npz", trapOutput, eTrapTS) # new ds1 run
def main(argv):
""" Interactive-fit or 'rapid'-fit waveforms that pass a given TCut.
BUG: Doesn't always work with a TChain. Add input files together
with hadd and use a single TFile.
"""
scanSpeed = 0.2
iList = -1
opt1 = ""
intMode, batMode = False, False
if (len(argv) >= 1): opt1 = argv[0]
if "-i" in (opt1):
intMode = True
print "Interactive mode selected."
if "-b" in (opt1):
batMode = True
print "Batch mode selected. A new file will be created."
# Load template waveform (created by gen-template.py)
npzfile = np.load("./data/genTemplateWF.npz")
temp, tempTS, tempE, tempST = npzfile['arr_0'] + 1, npzfile[
'arr_1'], npzfile['arr_2'], npzfile['arr_3'] * 10
# Set cuts
# theCut = inputFile.Get("cutUsedHere").GetTitle()
# DS3 "big DC" cut - PRELIMINARY
theCut = "trapENFCal > 0.8 && gain==0 && mHClean==1 && isGood && !muVeto && !wfDCBits && !isLNFill1 && !isLNFill2 && trapETailMin < 0 && channel!=596 && channel!=676 && channel!=676 && channel!=612 && channel!=1104 && channel!=1200 && channel!=1334 && channel!=1336 && tOffset < 10 && waveS5/TMath::Power(trapENFCal,1/4) < 1200 && (waveS3-waveS2)/trapENFCal > 100 && (waveS3-waveS2)/trapENFCal < 300 && !(channel==692 && (run==16974 || run==16975 || run==16976 || run==16977 || run==16978 || run==16979)) && butterTime < 11000"
# theCut += " && trapENFCal > 1.5 && trapENFCal < 2.1"
# theCut += " && trapENFCal < 20 && trapENFCal > 2 && run > 17480"
# theCut += " && kvorrT/trapENFCal > 2.2 && trapENFCal > 2 && trapENFCal < 10"
# Set file I/O and create entry lists
startT = time.clock()
inFile = "~/project/wavelet-skim/waveletSkimDS3_1.root"
# inFile = "~/project/wavelet-skim/hadd/waveletSkimDS3.root"
outFile = "~/project/fit-skim/fitSkimDS3_1.root"
inputFile = TFile(inFile)
waveTree = inputFile.Get("skimTree")
print "Found", waveTree.GetEntries(
), "input entries. Using cut:\n", theCut, "\n"
waveTree.Draw(">>elist", theCut, "entrylist")
elist = gDirectory.Get("elist")
waveTree.SetEntryList(elist)
nList = elist.GetN()
print "Found", nList, "entries passing cuts."
stopT = time.clock()
print "Data loading time (s): ", (stopT - startT)
# In batch mode ONLY, create an output file+tree & append new branches
outputFile = TFile()
outTree = TTree()
if batMode:
outputFile = TFile(outFile, "RECREATE")
print "Attempting tree copy to", outFile
outTree = waveTree.CopyTree("")
outTree.Write()
print "Wrote", outTree.GetEntries(), "entries."
cutUsed = TNamed("cutUsedHere", theCut)
cutUsed.Write()
fitStart = std.vector("double")()
fitE = std.vector("double")()
fitSlo = std.vector("double")()
fitStartSD = std.vector("double")()
fitESD = std.vector("double")()
fitSloSD = std.vector("double")()
fitChi2NDF = std.vector("double")()
fitLnLike = std.vector("double")()
tExp1 = std.vector("double")()
tExp2 = std.vector("double")()
tPol0 = std.vector("double")()
tPol1 = std.vector("double")()
tPol2 = std.vector("double")()
tPol3 = std.vector("double")()
baseAvg = std.vector("double")()
baseNoise = std.vector("double")()
bFitStart = outTree.Branch("fitStart", fitStart)
bFitE = outTree.Branch("fitE", fitE)
bFitSlo = outTree.Branch("fitSlo", fitSlo)
bFitStart_sd = outTree.Branch("fitStartSD", fitStartSD)
bFitE_sd = outTree.Branch("fitESD", fitESD)
bFitSlo_sd = outTree.Branch("fitSloSD", fitSloSD)
bFitChi2NDF = outTree.Branch("fitChi2NDF", fitChi2NDF)
bFitLnLike = outTree.Branch("fitLnLike", fitLnLike)
bTExp1 = outTree.Branch("tExp1", tExp1)
bTExp2 = outTree.Branch("tExp2", tExp2)
bTPol0 = outTree.Branch("tPol0", tPol0)
bTPol1 = outTree.Branch("tPol1", tPol1)
bTPol2 = outTree.Branch("tPol2", tPol2)
bTPol3 = outTree.Branch("tPol3", tPol3)
bBaseAvg = outTree.Branch("baseAvg", baseAvg)
bBaseNoise = outTree.Branch("baseNoise", baseNoise)
# Make a figure
# with PdfPages('multipage_pdf.pdf') as pdf:
fig = plt.figure(figsize=(11, 7), facecolor='w')
p1 = plt.subplot2grid((6, 7), (0, 0), colspan=4, rowspan=2) # original
p2 = plt.subplot2grid((6, 7), (2, 0), colspan=4, rowspan=3) # rising edge
p3 = plt.subplot2grid((6, 7), (5, 0), colspan=4, rowspan=1) # residual
p4 = plt.subplot2grid((6, 7), (0, 4), colspan=3, rowspan=2) # trace 1
p5 = plt.subplot2grid((6, 7), (2, 4), colspan=3, rowspan=2,
sharex=p4) # trace 2
p6 = plt.subplot2grid((6, 7), (4, 4), colspan=3, rowspan=2,
sharex=p4) # trace 3
if not batMode: plt.show(block=False)
# Setup multiprocessing
manager = Manager()
returnDict = manager.dict()
# Loop over events
while (True):
saveMe = False
iList += 1
if intMode == True and iList != 0:
value = raw_input()
if value == 'q': break
if value == 's': saveMe = True
if value == 'p': iList -= 2 # previous
if (value.isdigit()): iList = int(value) # go to entry
if iList >= elist.GetN(): break
entry = waveTree.GetEntryNumber(iList)
waveTree.LoadTree(entry)
waveTree.GetEntry(entry)
nChans = waveTree.channel.size()
numPass = waveTree.Draw("channel", theCut, "GOFF", 1, iList)
chans = waveTree.GetV1()
chanList = list(set(int(chans[n]) for n in xrange(numPass)))
fitStart.assign(nChans, -9999)
fitE.assign(nChans, -9999)
fitSlo.assign(nChans, -9999)
fitStartSD.assign(nChans, -9999)
fitESD.assign(nChans, -9999)
fitSloSD.assign(nChans, -9999)
fitChi2NDF.assign(nChans, -9999)
fitLnLike.assign(nChans, -9999)
tExp1.assign(nChans, -9999)
tExp2.assign(nChans, -9999)
tPol0.assign(nChans, -9999)
tPol1.assign(nChans, -9999)
tPol2.assign(nChans, -9999)
tPol3.assign(nChans, -9999)
baseAvg.assign(nChans, -9999)
baseNoise.assign(nChans, -9999)
# Loop over hits passing cuts
hitList = (iH for iH in xrange(nChans)
if waveTree.channel.at(iH) in chanList
) # a 'generator expression'
for iH in hitList:
# Load waveform for this hit
run = waveTree.run
chan = waveTree.channel.at(iH)
dataE = waveTree.trapENFCal.at(iH)
dataST = waveTree.butterTime.at(iH) # replace with blrwfFMR50?
toe = waveTree.kvorrT.at(iH) / dataE
print "%d / %d Run %d nCh %d chan %d trapENF %.1f t/e %.1f" % (
iList, nList, run, nChans, chan, dataE, toe)
signal = wl.processWaveform(waveTree.MGTWaveforms.at(iH),
opt='full')
waveBLSub = signal.GetWaveBLSub()
waveFilt = signal.GetWaveFilt()
waveTS = signal.GetTS()
dataBaseline, dataNoise = signal.GetBaseNoise()
# Denoise the data waveform (take only lowest-frequency components)
wp = pywt.WaveletPacket(data=waveBLSub,
wavelet='haar',
mode='symmetric',
maxlevel=3)
new_wp = pywt.WaveletPacket(data=None,
wavelet='haar',
mode='symmetric')
new_wp['aaa'] = wp['aaa'].data
waveDenoised = new_wp.reconstruct(update=False)
# Window the fit around rising edge - start time calculator method
loWin, hiWin = dataST - 1000, dataST + 4000 # ns
if loWin < waveTS[0] or hiWin > waveTS[-1]:
print "Window out of range! dataST: %.1f loWin %.1f hiWin %.1f" % (
dataST, loWin, hiWin)
idx = np.where((waveTS >= loWin) & (waveTS <= hiWin))
data = waveBLSub[idx]
# data = waveDenoised[idx]
dataTS = waveTS[idx]
# Pack into lists
rawList = [waveBLSub, waveTS, dataE, dataST]
dataList = [data, dataTS, dataE, dataST, loWin, hiWin]
tempList = [temp, tempTS, tempE, tempST]
# Optionally save something to a file
if saveMe:
print "Saved entry", iList, iH
np.savez("./data/tailSlopeInputs.npz", rawList, tempList)
# Recreate the guess and the guess's rising edge
guessFull, guessFullTS = wm.MakeModel(dataList,
tempList,
[dataST, dataE, 1.],
opt="full")
guess, guessTS = wm.MakeModel(dataList,
tempList, [dataST, dataE, 1.],
opt="!fancy")
# Make an "almost complete" guess - no MCMC
# st, en, slo = dataST-100, dataE, 5
# InterpFn = interpolate.interp1d(tempTS, temp, kind="linear", copy="False", assume_sorted="True")
# model, modelTS = wm.MakeModel(dataList, tempList, [st,en,slo], fn=InterpFn)
# Fit with MCMC and get best-fit parameters
numSteps, burnIn = 3000, 1800 # default: 10000, 5000. fast: 3000, 1800 long test: 20000,10000
wfModel = wm.TemplateModel(dataList, dataNoise, tempList)
p = Process(target=RunMCMC,
args=(wfModel, numSteps, burnIn, returnDict))
p.start()
p.join()
startTimeTr = returnDict["startTimeTr"]
energyTr = returnDict["energyTr"]
slownessTr = returnDict["slownessTr"]
st = np.median(startTimeTr[burnIn:])
en = np.median(energyTr[burnIn:])
slo = np.median(slownessTr[burnIn:])
InterpFn = interpolate.interp1d(tempTS,
temp,
kind="linear",
copy="False",
assume_sorted="True")
model, modelTS = wm.MakeModel(dataList,
tempList, [st, en, slo],
fn=InterpFn)
# Save some extra parameters for the ROOT output
# Calculate residual, Chi2/NDF, likelihood, etc.
st_std = np.std(startTimeTr[burnIn:])
en_std = np.std(energyTr[burnIn:])
slo_std = np.std(slownessTr[burnIn:])
residual = model - data
frac = (np.power(data - model, 2)) / np.abs(model)
chi2NDF = np.sum(frac) / len(model)
inv_sigma2 = 1.0 / (dataNoise**2)
lnLike = -0.5 * (np.sum((data - model)**2 * inv_sigma2 -
np.log(inv_sigma2)))
# ** Do a separate & simple fit of the tail slope **
# TODO: Add this to process-waveforms.py
idxMax = np.where(
guessFull == guessFull.max()) # returns an array/tuple
idxMax = idxMax[0][0] # "cast" to int
tail, tailTS = waveDenoised[idxMax:], waveTS[idxMax:]
popt, _ = curve_fit(wl.tailModelPol, tailTS, tail) # poly fit
pol0, pol1, pol2, pol3 = popt[0], popt[1], popt[2], popt[3]
a, b = dataE, 72000
popt2, _ = curve_fit(wl.tailModelExp, tailTS, tail,
p0=[a, b]) # expo fit
e1, e2 = popt2[0], popt2[1]
# Assign values to output vectors and fill branches
fitStart[iH], fitStartSD[iH] = st, st_std
fitE[iH], fitESD[iH] = en, en_std
fitSlo[iH], fitSloSD[iH] = slo, slo_std
fitChi2NDF[iH] = chi2NDF
fitLnLike[iH] = lnLike
tExp1[iH], tExp2[iH] = e1, e2
tPol0[iH], tPol1[iH], tPol2[iH], tPol3[iH] = pol0, pol1, pol2, pol3
baseAvg[iH] = dataBaseline
baseNoise[iH] = dataNoise
if batMode:
bFitStart.Fill()
bFitE.Fill()
bFitSlo.Fill()
bFitStart_sd.Fill()
bFitE_sd.Fill()
bFitSlo_sd.Fill()
bFitChi2NDF.Fill()
bFitLnLike.Fill()
bTExp1.Fill()
bTExp2.Fill()
bTPol0.Fill()
bTPol1.Fill()
bTPol2.Fill()
bTPol3.Fill()
bBaseAvg.Fill()
bBaseNoise.Fill()
if iList % 5000 == 0:
outTree.Write("", TObject.kOverwrite)
print "%d / %d entries saved (%.2f %% done)." % (
iList, nList, 100 * (float(iList) / nList))
# If not in batch mode, fill the figure
if batMode: continue
p1.cla()
p1.set_ylabel("ADC")
p1.set_title(
"Run %d Channel %d Entry %d\ntrapENFCal %.1f T/E %.1f ST %.1f"
% (run, chan, iList, dataE, toe, dataST))
p1.plot(waveTS, waveBLSub, color='blue')
p1.plot(waveTS, waveDenoised, color='red', alpha=0.8)
p1.plot(guessFullTS, guessFull, color='orange', linewidth=2)
p1.axvline(x=dataST, color='green', linewidth=2)
p1.axvline(x=loWin, color='black')
p1.axvline(x=hiWin, color='black')
p1.plot(tailTS,
wl.tailModelPol(tailTS, *popt),
color='cyan',
linewidth=1) # tail poly fit
p1.plot(tailTS,
wl.tailModelExp(tailTS, *popt2),
color='cyan',
linewidth=1) # tail expo fit
p2.cla()
p2.plot(dataTS, data, color='blue', label='Data')
p2.plot(guessTS, guess, color='orange', label='Guess')
# special: plot the values of the trace after burn-in
# to see how the model is covering the "money-zone"/rising edge after it's converged.
# for i in range(burnIn,numSteps):
# st_tr, en_tr, slo_tr = M.trace('startTime')[i], M.trace('energy')[i], M.trace('slowness')[i]
# trace, traceTS = wm.MakeModel(dataList, tempList, [st_tr,en_tr,slo_tr], fn=InterpFn)
# p2.plot(traceTS, trace, color='red',alpha=0.1,linewidth=2)
p2.plot(modelTS,
model,
color='red',
linewidth=3,
alpha=0.7,
label='Best Fit')
p2.legend(loc=4)
p3.cla()
p3.set_xlabel("Time [ns]", x=0.95, ha='right')
p3.set_ylabel("Residual [ADC]")
p3.plot(modelTS, residual, color='red')
p3.axhline(y=0, color='blue', alpha=0.3)
p3.axhline(y=dataNoise, color='blue', alpha=0.3)
p3.axhline(y=-1.0 * dataNoise, color='blue', alpha=0.3)
p4.cla()
minST = tempST - tempTS[-1] + hiWin
maxST = tempST - tempTS[0] + loWin
p4.set_title(
"startTime %.1f Energy %.2f\nSlow %.1f chi2/ndf %.1f Min %d Max %d"
% (st, en, slo, chi2NDF, minST, maxST))
p4.plot(startTimeTr[:])
p4.set_ylabel('startTime')
p4.axvline(x=burnIn, color='red', alpha=0.5)
p5.cla()
p5.plot(energyTr[:])
p5.set_ylabel('energy')
p5.axvline(x=burnIn, color='red', alpha=0.5)
p6.cla()
p6.plot(slownessTr[:])
p6.set_ylabel('slowness')
p6.axvline(x=burnIn, color='red', alpha=0.5)
plt.tight_layout()
plt.subplots_adjust(hspace=0.35)
plt.pause(scanSpeed)
# pdf.savefig()
# End loop over events
if batMode:
outTree.Write("", TObject.kOverwrite)
print "Wrote", outTree.GetBranch(
"channel").GetEntries(), "entries in the copied tree,"
print "and wrote", bFitStart.GetEntries(
), "entries in the new branches."
def main(argv):
# gROOT.ProcessLine("gErrorIgnoreLevel = 3001;") # suppress ROOT error messages
startT = time.clock()
print "Started:", time.strftime('%X %x %Z')
intMode, batMode, rangeMode, fileMode, dsNum, subNum, runNum = False, False, False, False, -1, -1, -1
for i, opt in enumerate(argv):
if opt == "-i":
intMode = True
print "Interactive mode selected. Use \"p\" for previous and \"q\" to exit."
if opt == "-r":
rangeMode = True
dsNum, subNum = int(argv[i + 1]), int(argv[i + 2])
print "Scanning DS-%d range %d" % (dsNum, subNum)
if opt == "-f":
fileMode = True
dsNum, runNum = int(argv[i + 1]), int(argv[i + 2])
if opt == "-b":
batMode = True
import matplotlib
if os.environ.get('DISPLAY', '') == '':
print('No display found. Using non-interactive Agg backend')
matplotlib.use('Agg')
print "Batch mode selected. A new file will be created."
import matplotlib.pyplot as plt
# Set file I/O and cuts
inFile = "./data/waveSkimDS4_test.root"
outFile = "./data/waveletSkimDS4_test.root"
if (rangeMode):
inFile = "~/project/v2-waveskim/waveSkimDS%d_%d.root" % (dsNum, subNum)
outFile = "~/project/v2-processwfs/waveletSkimDS%d_%d.root" % (dsNum,
subNum)
if (fileMode):
# inFile = "~/project/cal-wave-skim/waveSkimDS%d_run%d.root" % (dsNum,runNum)
# outFile = "~/project/cal-wavelet-skim/waveletSkimDS%d_run%d.root" % (dsNum,runNum)
inFile = "./waveSkimDS%d_run%d.root" % (dsNum, runNum)
outFile = "./waveletSkimDS%d_run%d.root" % (dsNum, runNum)
inputFile = TFile(inFile)
waveTree = inputFile.Get("skimTree")
print "Found", waveTree.GetEntries(), "input entries."
theCut = inputFile.Get("theCut").GetTitle()
# theCut = "trapENFCal > 0.8 && gain==0 && mHClean==1 && isGood && !muVeto && !isLNFill1 && !isLNFill2 && P!=0 && D!=0 && trapETailMin<0"
# theCut += " && Entry$ < 2000"
# theCut += " && trapENFCal > 20"
print "Using cut:\n", theCut, "\n"
print "Attempting entrylist draw ..."
waveTree.Draw(">>elist", theCut, "GOFF entrylist")
elist = gDirectory.Get("elist")
waveTree.SetEntryList(elist)
nList = elist.GetN()
print "Found", nList, "entries passing cuts."
# In batch mode ONLY, create an output file+tree & append new branches
outputFile = TFile()
outTree = TTree()
if (batMode == True):
outputFile = TFile(outFile, "RECREATE")
print "Attempting tree copy to", outFile
outTree = waveTree.CopyTree("")
outTree.Write()
print "Wrote", outTree.GetEntries(), "entries."
cutUsed = TNamed("cutUsedHere", theCut)
cutUsed.Write()
waveS0 = std.vector("double")()
waveS1 = std.vector("double")()
waveS2 = std.vector("double")()
waveS3 = std.vector("double")()
waveS4 = std.vector("double")()
waveS5 = std.vector("double")()
wpar4 = std.vector("double")()
waveEnt = std.vector("double")()
butterMax = std.vector("double")()
butterTime = std.vector("double")()
tOffset = std.vector("double")()
lastZeroTime = std.vector("double")()
pol0 = std.vector("double")()
pol1 = std.vector("double")()
pol2 = std.vector("double")()
pol3 = std.vector("double")()
exp0 = std.vector("double")()
exp1 = std.vector("double")()
rt10 = std.vector("double")()
rt20 = std.vector("double")()
rt50 = std.vector("double")()
rt90 = std.vector("double")()
bWaveS0 = outTree.Branch("waveS0", waveS0)
bWaveS1 = outTree.Branch("waveS1", waveS1)
bWaveS2 = outTree.Branch("waveS2", waveS2)
bWaveS3 = outTree.Branch("waveS3", waveS3)
bWaveS4 = outTree.Branch("waveS4", waveS4)
bWaveS5 = outTree.Branch("waveS5", waveS5)
bWPar4 = outTree.Branch("wpar4", wpar4)
bWaveEnt = outTree.Branch("waveEnt", waveEnt)
bButterMax = outTree.Branch("butterMax", butterMax)
bButterTime = outTree.Branch("butterTime", butterTime)
bTOffset = outTree.Branch("tOffset", tOffset)
bLastZeroTime = outTree.Branch("lastZeroTime", lastZeroTime)
bPol0 = outTree.Branch("pol0", pol0)
bPol1 = outTree.Branch("pol1", pol1)
bPol2 = outTree.Branch("pol2", pol2)
bPol3 = outTree.Branch("pol3", pol3)
bExp0 = outTree.Branch("exp0", exp0)
bExp1 = outTree.Branch("exp1", exp1)
bRt10 = outTree.Branch("rt10", rt10)
bRt20 = outTree.Branch("rt20", rt20)
bRt50 = outTree.Branch("rt50", rt50)
bRt90 = outTree.Branch("rt90", rt90)
# Make a figure
fig = plt.figure(figsize=(7, 7), facecolor='w')
p1 = plt.subplot(211)
p2 = plt.subplot(212)
if not batMode: plt.show(block=False)
# Begin loop over events
iList = -1
while True:
iList += 1
if intMode == True and iList != 0:
value = raw_input()
if value == 'q': break # quit
if value == 'p': iList -= 2 # go to previous
if (value.isdigit()):
iList = int(value) # go to entry number
elif intMode == False and batMode == False:
plt.pause(0.001) # rapid-draw mode
if iList >= nList: break # bail out, goose!
entry = waveTree.GetEntryNumber(iList)
waveTree.LoadTree(entry)
waveTree.GetEntry(entry)
nChans = waveTree.channel.size()
waveS1.assign(nChans, -999)
waveS0.assign(nChans, -999)
waveS1.assign(nChans, -999)
waveS2.assign(nChans, -999)
waveS3.assign(nChans, -999)
waveS4.assign(nChans, -999)
waveS5.assign(nChans, -999)
wpar4.assign(nChans, -999)
waveEnt.assign(nChans, -999)
butterMax.assign(nChans, -999)
butterTime.assign(nChans, -999)
tOffset.assign(nChans, -999)
lastZeroTime.assign(nChans, -999)
pol0.assign(nChans, -999)
pol1.assign(nChans, -999)
pol2.assign(nChans, -999)
pol3.assign(nChans, -999)
exp0.assign(nChans, -999)
exp1.assign(nChans, -999)
rt10.assign(nChans, -999)
rt20.assign(nChans, -999)
rt50.assign(nChans, -999)
rt90.assign(nChans, -999)
# Loop over hits passing cuts
numPass = waveTree.Draw("channel", theCut, "GOFF", 1, iList)
chans = waveTree.GetV1()
chanList = list(set(int(chans[n]) for n in xrange(numPass)))
hitList = (iH for iH in xrange(nChans)
if waveTree.channel.at(iH) in chanList
) # a 'generator expression'
for iH in hitList:
run = waveTree.run
iEvent = waveTree.iEvent
chan = waveTree.channel.at(iH)
energy = waveTree.trapENFCal.at(iH)
wf = waveTree.MGTWaveforms.at(iH)
startTime = waveTree.triggerTrapt0.at(iH)
blrwfFMR50 = waveTree.blrwfFMR50.at(iH)
if wf.GetID() != chan:
print "Warning!! Vector matching failed! iList %d run %d iEvent %d" % (
iList, run, iEvent)
break
signal = wl.processWaveform(wf, opt='full')
waveBLSub = signal.GetWaveBLSub()
waveFilt = signal.GetWaveFilt()
waveTS = signal.GetTS()
waveletYOrig, waveletYTrans = signal.GetWavelet()
# waveFTX, waveFTY, waveFTPow = signal.GetFourier()
# waveTrap = signal.GetTrapezoid()
# waveTrapF = signal.GetFiltTrapezoid()
# waveTrapX = np.linspace(0, len(waveTrap), num=len(waveTrap))
# _,waveletFilt = wl.waveletTransform(waveFilt,level=3) # testing other levels on the filtered WF
# Wavelet cut parameters
waveS0[iH] = np.sum(waveletYTrans[0:1, 1:-1])
waveS1[iH] = np.sum(waveletYTrans[0:1, 1:33])
waveS2[iH] = np.sum(waveletYTrans[0:1, 33:65])
waveS3[iH] = np.sum(waveletYTrans[0:1, 65:97])
waveS4[iH] = np.sum(waveletYTrans[0:1, 97:-1])
waveS5[iH] = np.sum(waveletYTrans[2:-1, 1:-1])
wpar4[iH] = np.amax(waveletYTrans[0:1, 1:-1])
# Waveform entropy parameters
d1 = 2. * np.multiply(
waveletYTrans[0:1, 1:65],
np.log(waveletYTrans[0:1, 1:65] / waveS0[iH] / 2.0))
d2 = np.multiply(waveletYTrans[0:1, 65:-1],
np.log(waveletYTrans[0:1, 65:-1] / waveS0[iH]))
waveEnt[iH] = np.abs(np.sum(d1)) + np.abs(np.sum(d2))
# Smoothed derivative params
waveFiltDeriv = wl.wfDerivative(waveFilt)
butterMax[iH] = np.amax(waveFiltDeriv)
butterTime[iH] = np.argmax(
waveFiltDeriv[100:]) * 10 + signal.GetOffset() + 1000
tOffset[iH] = signal.GetOffset()
# print "max %.3f ts %.0f ns offset %.0f ns" % (butterMax[iH], butterTime[iH], tOffset[iH])
# Make a super denoised waveform
wp = pywt.WaveletPacket(data=waveBLSub,
wavelet='haar',
mode='symmetric',
maxlevel=3)
new_wp = pywt.WaveletPacket(data=None,
wavelet='haar',
mode='symmetric')
new_wp['aaa'] = wp['aaa'].data
superDenoisedWF = new_wp.reconstruct(update=False)
mgtSDWF = wl.MGTWFFromNpArray(superDenoisedWF)
# Try to get start time by finding the super denoised last zero crossing
lastZero = 0
zeros = np.asarray(np.where(superDenoisedWF < 0.1))
if (zeros.size > 0): lastZero = zeros[0, -1]
lastZeroTime[iH] = waveTS[lastZero]
# Time point calculator
timePointCalc = MGWFTimePointCalculator()
timePointCalc.AddPoint(.1)
timePointCalc.AddPoint(.2)
timePointCalc.AddPoint(.5)
timePointCalc.AddPoint(.9)
timePointCalc.FindTimePoints(mgtSDWF)
rt10[iH] = timePointCalc.GetFromStartRiseTime(0) * 10
rt20[iH] = timePointCalc.GetFromStartRiseTime(1) * 10
rt50[iH] = timePointCalc.GetFromStartRiseTime(2) * 10
rt90[iH] = timePointCalc.GetFromStartRiseTime(3) * 10
# print "lastZero %.2f startTime %.2f 10 %.2f 20 %.2f 50 %.2f 90 %.2f" % (lastZeroTime[iH], startTime,rt10, rt20, rt50, rt90)
# Fit tail slope (2 methods). Guard against fit fails
idxMax = np.where(waveTS > rt90[iH]) # returns an array/tuple
idxMax = idxMax[0][0] # "cast" to int
tail, tailTS = waveBLSub[idxMax:], waveTS[idxMax:]
try:
popt1, _ = curve_fit(wl.tailModelPol, tailTS, tail)
pol0[iH], pol1[iH], pol2[iH], pol3[iH] = popt1[0], popt1[
1], popt1[2], popt1[3]
except:
pass
try:
popt2, _ = curve_fit(wl.tailModelExp,
tailTS,
tail,
p0=[energy, 72000])
exp0[iH], exp1[iH] = popt2[0], popt2[1]
except:
# print "curve_fit tailModelExp failed, run %i event %i channel %i" % (run, iList, chan)
pass
# Make a plot
if not batMode:
print "i %d run %d iEvent(i) %d iH(j+1) %d/%d chan %d energy %.2f bt %.2f" % (
iList, run, iEvent, iH + 1, nChans, chan, energy,
butterTime[iH])
# Waveform plot
p1.cla()
p1.plot(waveTS, waveBLSub, color='blue')
p1.plot(waveTS, superDenoisedWF, color='red')
p1.plot(tailTS, wl.tailModelPol(tailTS, *popt1), color='cyan')
p1.plot(tailTS,
wl.tailModelExp(tailTS, *popt2),
color='orange')
p1.set_xlabel("Time (ns)")
p1.set_ylabel("ADC")
p1.set_title(
"Run %d Ch %d Energy %.2f \n S5 %.2f (S3-S2)/E %.2f (S3-S4)/E %.2f"
% (run, chan, energy, waveS5[iH],
((waveS3[iH] - waveS2[iH]) / energy),
((waveS3[iH] - waveS4[iH]) / energy)))
# Wavelet plot
p2.cla()
p2.imshow(waveletYTrans,
interpolation='nearest',
aspect="auto",
origin="lower",
extent=[0, 1, 0, len(waveletYTrans)],
cmap='jet')
plt.tight_layout()
plt.subplots_adjust(hspace=.2, top=0.92)
plt.pause(0.00001)
# End loop over hits
if (batMode == True):
bWaveS0.Fill()
bWaveS1.Fill()
bWaveS2.Fill()
bWaveS3.Fill()
bWaveS4.Fill()
bWaveS5.Fill()
bWPar4.Fill()
bWaveEnt.Fill()
bButterMax.Fill()
bButterTime.Fill()
bTOffset.Fill()
bLastZeroTime.Fill()
bPol0.Fill()
bPol1.Fill()
bPol2.Fill()
bPol3.Fill()
bExp0.Fill()
bExp1.Fill()
bRt10.Fill()
bRt20.Fill()
bRt50.Fill()
bRt90.Fill()
if iList % 5000 == 0:
outTree.Write("", TObject.kOverwrite)
print "%d / %d entries saved (%.2f %% done)." % (
iList, nList, 100 * (float(iList) / nList))
# End loop over events
if (batMode == True):
outTree.Write("", TObject.kOverwrite)
print "Wrote", outTree.GetBranch(
"channel").GetEntries(), "entries in the copied tree,"
print "and wrote", bWaveS0.GetEntries(), "entries in the new branches."
stopT = time.clock()
print "Process time (min): ", (stopT - startT) / 60
def main(argv):
scanSpeed = 0.0001
opt1, opt2 = "", ""
intMode, printWF = False, False
if (len(argv) >= 1): opt1 = argv[0]
if (len(argv) >= 2): opt2 = argv[1]
if "-i" in (opt1, opt2):
intMode = True
print "Interactive mode selected."
if "-s" in (opt1, opt2):
printWF = True
print "Saving WF plots to current directory."
# Set input file
inputFile = TFile("~/project/match-skim/waveletSkimDS5_run23920.root")
waveTree = inputFile.Get("skimTree")
# Set cut
theCut = inputFile.Get("cutUsedHere").GetTitle()
theCut += " && waveS5/trapENFCal < 1200 && trapENFCal > 2 && trapENFCal < 5"
waveTree.Draw(">>elist", theCut, "entrylist")
elist = gDirectory.Get("elist")
waveTree.SetEntryList(elist)
nList = elist.GetN()
print "Using cut:\n", theCut, "\n"
print "Found", nList, "entries passing cuts."
# Make a figure
fig = plt.figure(figsize=(8, 7), facecolor='w')
p1 = plt.subplot(211)
p2 = plt.subplot(212)
plt.show(block=False)
# Make a huge floating template waveform
samp, r, z, tempE, tempStart, smooth = 5000, 0, 15, 10, 2500, 100
tempOrig, tempOrigTS = wl.MakeSiggenWaveform(samp, r, z, tempE, tempStart,
smooth)
# Loop over events
iList = -1
while (True):
iList += 1
if intMode == True and iList != 0:
value = raw_input()
if value == 'q': break
if value == 'p': iList -= 2 # previous
if (value.isdigit()): iList = int(value) # go to entry
if iList >= elist.GetN(): break
entry = waveTree.GetEntryNumber(iList)
waveTree.LoadTree(entry)
waveTree.GetEntry(entry)
nChans = waveTree.channel.size()
nWaves = waveTree.MGTWaveforms.size()
numPass = waveTree.Draw("channel", theCut, "GOFF", 1, iList)
chans = waveTree.GetV1()
chanList = list(set(int(chans[n]) for n in xrange(numPass)))
# Loop over hits passing cuts
hitList = (iH for iH in xrange(nChans)
if waveTree.channel.at(iH) in chanList
) # a 'generator expression'
for iH in hitList:
# Load waveform data
run = waveTree.run
chan = waveTree.channel.at(iH)
dataE = waveTree.trapENFCal.at(iH)
dataENM = waveTree.trapENM.at(iH)
dataMaxTime = waveTree.trapENMSample.at(iH) * 10. - 4000
# dataMaxTime = waveTree.blrwfFMR50.at(iH)
signal = wl.processWaveform(waveTree.MGTWaveforms.at(iH))
data = signal.GetWaveBLSub()
dataTS = signal.GetTS()
print "%d / %d Run %d nCh %d chan %d trapENF %.1f" % (
iList, nList, run, nChans, chan, dataE)
# Draw a trapezoid filter, append with 1000 samples of zeros
# trap = np.zeros(1000)
# trap = np.append(trap,wl.trapezoidalFilter(data))
# 1. Signal-template convolution. Time domain, technically not quite a full matched filter?
temp, tempTS = flipAndAlign(tempOrig, tempOrigTS, dataTS,
dataMaxTime, dataENM, tempE)
smf = gaussian_filter(temp * data, sigma=float(10))
# 2. crazy stuff that doesn't work
deriv = wl.wfDerivative(data)
wp = pywt.WaveletPacket(data=deriv,
wavelet='haar',
mode='symmetric',
maxlevel=3)
new_wp = pywt.WaveletPacket(data=None,
wavelet='haar',
mode='symmetric')
new_wp['aaa'] = wp['aaa'].data
denoised = new_wp.reconstruct(update=False)
# 3. LIGO-inspired matched filter
# cmap = cm.get_cmap('Spectral') # then in plot: color=cmap(fraction)
p1.cla()
p1.set_ylabel("ADC")
p1.plot(dataTS, data, color='blue')
p1.plot(tempTS, temp, color='red')
# p1.plot(tempTS,trap,color='green')
p1.plot(tempTS, smf, color='red')
p2.cla()
p2.plot(dataTS, denoised, color='blue')
plt.pause(scanSpeed)
def main(argv):
""" Interactive-fit or 'rapid'-fit waveforms that pass a given TCut.
BUG: Doesn't always work with a TChain. Add input files together
with hadd and use a single TFile.
"""
scanSpeed = 1.
iList = -1
opt1 = ""
intMode = False
if (len(argv) >= 1): opt1 = argv[0]
if "-i" in (opt1): intMode = True
# Set input data and cuts
inputFile = TFile(
"~/project/match-skim/waveletSkimDS5_run23920.root") # calibration
# inputFile = TFile("~/project/v2-processwfs/waveletSkimDS5_90.root") # noisy BG
waveTree = inputFile.Get("skimTree")
theCut = inputFile.Get("cutUsedHere").GetTitle()
theCut += " && waveS5/trapENFCal < 1200 && trapENFCal > 1.5 && trapENFCal < 3"
# Print cut and events passing cut
waveTree.Draw(">>elist", theCut, "entrylist")
elist = gDirectory.Get("elist")
waveTree.SetEntryList(elist)
nList = elist.GetN()
print "Using cut:\n", theCut, "\n"
print "Found", waveTree.GetEntries(), "input entries."
print "Found", nList, "entries passing cuts."
# Make a figure
fig = plt.figure(figsize=(15, 10), facecolor='w')
p1 = plt.subplot2grid((6, 7), (0, 0), colspan=4, rowspan=2) # original
p2 = plt.subplot2grid((6, 7), (2, 0), colspan=4, rowspan=3) # rising edge
p3 = plt.subplot2grid((6, 7), (5, 0), colspan=4, rowspan=1) # residual
p4 = plt.subplot2grid((6, 7), (0, 4), colspan=3, rowspan=2) # trace 1
p5 = plt.subplot2grid((6, 7), (2, 4), colspan=3, rowspan=2) # trace 2
p6 = plt.subplot2grid((6, 7), (4, 4), colspan=3, rowspan=2) # trace 3
plt.show(block=False)
# Make template(s)
# npzfile = np.load("./data/genTemplateWF.npz") # gen-template.py
# temp, tempTS, tempE, tempST = npzfile['arr_0']+1, npzfile['arr_1'], npzfile['arr_2'], npzfile['arr_3']*10
samp, r, z, tempE, tempST, smooth = 2016, 0, 15, 3938, 1000, 100 # gen-template.py
# samp, r, z, tempE, tempST, smooth = 5000, 0, 15, 10, 2500, 100 # huge template
# samp, r, z, tempE, tempST, smooth = 2016, 0, 15, 10, 1000, 100 # regular size template
# samp, r, z, tempE, tempST, smooth = 2016, 30, 30, 10, 1000, 100 # regular size temp, slower rise
# samp, r, z, tempE, tempST, smooth = 500, 0, 15, 10, 100, 100 # small template
temp, tempTS = wl.MakeSiggenWaveform(samp, r, z, tempE, tempST, smooth)
tempST = tempST * 10 # convert to ns
# Loop over events
while True:
saveMe = False
iList += 1
if intMode == True and iList != 0:
value = raw_input()
if value == 'q': break
if value == 's': saveMe = True
if value == 'p': iList -= 2 # previous
if (value.isdigit()): iList = int(value) # go to entry
if iList >= elist.GetN(): break
entry = waveTree.GetEntryNumber(iList)
waveTree.LoadTree(entry)
waveTree.GetEntry(entry)
nChans = waveTree.channel.size()
numPass = waveTree.Draw("channel", theCut, "GOFF", 1, iList)
chans = waveTree.GetV1()
chanList = list(set(int(chans[n]) for n in xrange(numPass)))
# Loop over hits passing cuts
hitList = (iH for iH in xrange(nChans)
if waveTree.channel.at(iH) in chanList
) # a 'generator expression'
for iH in hitList:
# ------------------------------------------------------------------------
# Load waveform for this hit
run = waveTree.run
chan = waveTree.channel.at(iH)
dataE = waveTree.trapENFCal.at(iH)
dataST = waveTree.butterTime.at(iH) # replace with blrwfFMR50?
toe = waveTree.kvorrT.at(iH) / dataE
print "%d / %d Run %d nCh %d chan %d trapENF %.1f t/e %.1f" % (
iList, nList, run, nChans, chan, dataE, toe)
signal = wl.processWaveform(waveTree.MGTWaveforms.at(iH),
opt='full')
waveBLSub = signal.GetWaveBLSub()
waveFilt = signal.GetWaveFilt()
waveTS = signal.GetTS()
baseAvg, dataNoise = signal.GetBaseNoise()
# Denoise the data waveform (take only lowest-frequency components)
wp = pywt.WaveletPacket(data=waveBLSub,
wavelet='haar',
mode='symmetric',
maxlevel=3)
new_wp = pywt.WaveletPacket(data=None,
wavelet='haar',
mode='symmetric')
new_wp['aaa'] = wp['aaa'].data
waveDenoised = new_wp.reconstruct(update=False)
# Window the fit around rising edge - start time calculator method
loWin, hiWin = dataST - 1000, dataST + 4000 # ns
if loWin < waveTS[0] or hiWin > waveTS[-1]:
print "Window out of range! dataST: %.1f loWin %.1f hiWin %.1f" % (
dataST, loWin, hiWin)
idx = np.where((waveTS >= loWin) & (waveTS <= hiWin))
data = waveBLSub[idx]
# data = waveDenoised[idx]
dataTS = waveTS[idx]
# Pack into lists
rawList = [waveBLSub, waveTS, dataE, dataST]
dataList = [data, dataTS, dataE, dataST, loWin, hiWin, dataNoise]
tempList = [temp, tempTS, tempE, tempST]
# Optionally save something to a file
if saveMe:
np.savez("./data/tailSlopeInputs.npz", rawList, tempList)
# Recreate the guess and the guess's rising edge
guessFull, guessFullTS = wm.MakeModel(dataList,
tempList,
[dataST, dataE, 1.],
opt="full")
guess, guessTS = wm.MakeModel(dataList,
tempList, [dataST, dataE, 1.],
opt="!fancy")
# Make an "almost complete" guess - no fitting
# st, en, slo = dataST-100, dataE, 5
InterpFn = interpolate.interp1d(tempTS,
temp,
kind="linear",
copy="False",
assume_sorted="True")
# model, modelTS = wm.MakeModel(dataList, tempList, [st,en,slo], fn=InterpFn)
# Fit with MCMC and get best-fit parameters
# numSteps, burnIn = 10000, 5000 # default - 10000, 5000. try 3000, 2000. long test: 20000,10000
# myModel = TemplateModel( dataList, dataNoise, tempList )
# waveModel = pymc.Model( myModel )
# M = pymc.MCMC( waveModel )
# M.use_step_method(pymc.Metropolis, M.startTime, proposal_sd=100., proposal_distribution='Normal')
# M.use_step_method(pymc.Metropolis, M.energy, proposal_sd=1., proposal_distribution='Normal')
# M.use_step_method(pymc.Metropolis, M.slowness, proposal_sd=100., proposal_distribution='Normal')
# M.sample(iter=numSteps, verbose=0) # do the fit
# st = np.median(M.trace('startTime')[burnIn:])
# en = np.median(M.trace('energy')[burnIn:])
# slo = np.median(M.trace('slowness')[burnIn:])
# InterpFn = interpolate.interp1d(tempTS, temp, kind="linear", copy="False", assume_sorted="True")
# model, modelTS = wm.MakeModel(dataList, tempList, [st,en,slo], fn=InterpFn)
# Fit with SciPy minimizer(s) and get best-fit parameters
# how long does it take?
start = time.time()
MakeTracesGlobal(
) # creates 3 global arrays: startTrace, enTrace, sloTrace
datas = [dataList, tempList, InterpFn]
# floats = [1.]
# result = op.basinhopping(findLnLike, floats, T=1000, stepsize=15, niter_success=2, minimizer_kwargs={"args":datas})
# print "slowness:",result["x"],"dataE",dataE,"dataST",dataST
# floats = [dataST, dataE, result["x"]]
# MakeTracesGlobal() # resets traces
floats = [dataST, dataE, 20.]
result = op.minimize(findLnLike,
floats,
args=datas,
method="Nelder-Mead")
if not result["success"]: print result["message"]
st, en, slo = result["x"]
model, modelTS = wm.MakeModel(dataList,
tempList, [st, en, slo],
fn=InterpFn)
stop = time.time()
print "fitting took", stop - start
# Calculate residual, Chi2/NDF, likelihood, etc.
residual = model - data
frac = (np.power(data - model, 2)) / np.abs(model)
chi2NDF = np.sum(frac) / len(model)
inv_sigma2 = 1.0 / (dataNoise**2)
lnLike = -0.5 * (np.sum((data - model)**2 * inv_sigma2 -
np.log(inv_sigma2)))
# Fill the figure
p1.cla()
p1.set_ylabel("ADC")
p1.set_title(
"Run %d Channel %d Entry %d\ntrapENFCal %.1f T/E %.1f ST %.1f"
% (run, chan, iList, dataE, toe, dataST))
p1.plot(waveTS, waveBLSub, color='blue')
p1.plot(waveTS, waveDenoised, color='red', alpha=0.8)
p1.plot(guessFullTS, guessFull, color='orange', linewidth=2)
p1.axvline(x=dataST, color='green', linewidth=2)
p1.axvline(x=loWin, color='black')
p1.axvline(x=hiWin, color='black')
p2.cla()
p2.plot(dataTS, data, color='blue', label='Data')
p2.plot(guessTS, guess, color='orange', label='Guess')
p2.plot(modelTS, model, color='red', linewidth=3, label='Best Fit')
p2.legend(loc=4)
p3.cla()
p3.set_xlabel("Time [ns]", x=0.95, ha='right')
p3.set_ylabel("Residual [ADC]")
p3.plot(guessTS, residual, color='red')
p3.axhline(y=0, color='blue', alpha=0.3)
p3.axhline(y=dataNoise, color='blue', alpha=0.3)
p3.axhline(y=-1.0 * dataNoise, color='blue', alpha=0.3)
p4.cla()
p4.set_title("startTime %.1f Energy %.2f Slow %.1f" %
(st, en, slo))
p4.plot(stTrace[1:])
p4.set_ylabel('startTime')
p5.cla()
p5.plot(enTrace[1:])
p5.set_ylabel('energy')
p6.cla()
p6.plot(sloTrace[1:])
p6.set_ylabel('slowness')
plt.tight_layout()
plt.subplots_adjust(hspace=0.35)
plt.pause(scanSpeed)
def main(argv):
scanSpeed = 0.0001
opt1, opt2 = "", ""
intMode, printWF = False, False
if (len(argv) >= 1): opt1 = argv[0]
if (len(argv) >= 2): opt2 = argv[1]
if "-i" in (opt1, opt2):
intMode = True
print "Interactive mode selected."
if "-s" in (opt1, opt2):
printWF = True
print "Saving WF plots to current directory."
# Set input file and cut
inputFile = TFile("~/project/match-skim/waveletSkimDS5_run23920.root")
# inputFile = TFile("~/project/v2-processwfs/waveletSkimDS5_90.root")
waveTree = inputFile.Get("skimTree")
theCut = inputFile.Get("cutUsedHere").GetTitle()
theCut += " && waveS5/trapENFCal < 1200 && trapENFCal < 5"
# theCut += " && trapENFCal > 1.5 && trapENFCal < 2"
waveTree.Draw(">>elist", theCut, "entrylist")
elist = gDirectory.Get("elist")
waveTree.SetEntryList(elist)
nList = elist.GetN()
print "Using cut:\n", theCut, "\n"
print "Found", nList, "entries passing cuts."
# Make a figure
fig = plt.figure(figsize=(15, 10), facecolor='w')
# p1 = plt.subplot(231)
# p2 = plt.subplot(232)
# p3 = plt.subplot(233)
# p4 = plt.subplot(234)
# p5 = plt.subplot(235)
# p6 = plt.subplot(236)
p1 = plt.subplot2grid((2, 3), (0, 0), colspan=2)
p3 = plt.subplot2grid((2, 3), (0, 2))
p4 = plt.subplot2grid((2, 3), (1, 0))
p5 = plt.subplot2grid((2, 3), (1, 1))
p6 = plt.subplot2grid((2, 3), (1, 2))
plt.show(block=False)
# Make templates
samp, r, z, tempE, tempStart, smooth = 5000, 0, 15, 10, 2500, 100 # huge template
tempHuge, tempHugeTS = wl.MakeSiggenWaveform(samp, r, z, tempE, tempStart,
smooth)
samp, r, z, tempE, tempStart, smooth = 2016, 0, 15, 10, 1000, 100 # regular size template
# samp, r, z, tempE, tempStart, smooth = 2016, 30, 30, 10, 1000, 100 # regular size temp, slower rise
# samp, r, z, tempE, tempStart, smooth = 500, 0, 15, 10, 100, 100 # small template
tempOrig, tempOrigTS = wl.MakeSiggenWaveform(samp, r, z, tempE, tempStart,
smooth)
# Load stuff from DS1 forced acq. runs
npzfile = np.load("./data/fft_forcedAcqDS1.npz")
avgPsd, xPsd, avgPwrSpec, xPwrSpec, data_forceAcq, data_fft = npzfile[
'arr_0'], npzfile['arr_1'], npzfile['arr_2'], npzfile[
'arr_3'], npzfile['arr_4'], npzfile['arr_5']
# Sampling frequency - 100MHz
fs = 1e8
# Loop over events
iList = -1
while (True):
iList += 1
if intMode == True and iList != 0:
value = raw_input()
if value == 'q': break
if value == 'p': iList -= 2 # previous
if (value.isdigit()): iList = int(value) # go to entry
if iList >= elist.GetN(): break
entry = waveTree.GetEntryNumber(iList)
waveTree.LoadTree(entry)
waveTree.GetEntry(entry)
nChans = waveTree.channel.size()
nWaves = waveTree.MGTWaveforms.size()
numPass = waveTree.Draw("channel", theCut, "GOFF", 1, iList)
chans = waveTree.GetV1()
chanList = list(set(int(chans[n]) for n in xrange(numPass)))
# Loop over hits passing cuts
hitList = (iH for iH in xrange(nChans)
if waveTree.channel.at(iH) in chanList
) # a 'generator expression'
for iH in hitList:
# ------------------------------------------------------------------------
# Load data
run = waveTree.run
chan = waveTree.channel.at(iH)
dataE = waveTree.trapENFCal.at(iH)
dataENM = waveTree.trapENM.at(iH)
dataMaxTime = waveTree.trapENMSample.at(iH) * 10. - 4000
signal = wl.processWaveform(waveTree.MGTWaveforms.at(iH))
data = signal.GetWaveBLSub()
time = signal.GetTS()
print "%d / %d Run %d nCh %d chan %d trapENF %.1f len %d" % (
iList, nList, run, nChans, chan, dataE, len(data))
# Load template
template, temp_time = tempOrig, tempOrigTS
template = template * (dataENM / tempE)
# template = wl.wfDerivative(template)
# Plot data and template
p1.cla()
p1.plot(time, data, color='blue')
p1.set_xlabel('Time (s)')
p1.set_ylabel('Voltage (arb)')
p1.set_title("Run %i Ch %i E %.2f ENM %.2f" %
(run, chan, dataE, dataENM))
# p2.cla()
p1.plot(temp_time, template, color='red')
# p2.set_xlabel('Time (s)')
# p2.set_ylabel('Voltage (arb)')
# p2.set_title('Template')
# Plot ASD of data and template
# (amplitude spectral density (ASD), which is the square root of the PSD)
p3.cla()
power_data, freq_psd = plt.psd(data,
Fs=fs,
NFFT=2048,
pad_to=2048,
visible=False)
p3.loglog(freq_psd, np.sqrt(power_data), color='blue')
power_temp, freq = plt.psd(template,
Fs=fs,
NFFT=2048,
pad_to=2048,
visible=False) # check this.
p3.loglog(freq, np.sqrt(power_temp), color='red')
p3.loglog(xPsd, np.sqrt(avgPsd),
color='green') # DS1 forced acq. results
p3.set_xlabel('Frequency (Hz)')
p3.set_ylabel('ASD')
p3.grid('on')
# Apply a bandpass filter to the data and plot it
# NOTE: This is tough. There might be a feature
# due to rising edges, between 3e6 and 5e6 Hz.
# But that's only from data. Doing it w/ siggen templates yielded squat.
# B,A = sig.butter(2, [3e6/(1e8/2),5e6/(1e8/2)], btype='bandpass')
B, A = sig.butter(2, 1e6 / (1e8 / 2),
btype='lowpass') # nice lowpass filter
data_LowPass = sig.lfilter(B, A, data)
B1, A1 = sig.butter(2, [1e5 / (1e8 / 2), 1e6 / (1e8 / 2)],
btype='bandpass') # attempt at bandpass
data_BanPass = sig.lfilter(B1, A1, data)
p4.cla()
p4.plot(time, data_LowPass, label='lowpass')
p4.plot(time, data_BanPass, label='bandpass')
p4.set_title('Band passed data')
p4.set_xlabel('Time (s)')
p4.legend(loc=4)
# Plot time-domain cross-correlation
# correlated_raw = np.correlate(data, template, 'same') # ligo used 'valid'. 'full' or 'same'
# correlated_passed = np.correlate(data_pass, template, 'same')
# p5.plot(np.arange(0, (correlated_raw.size*1.)/fs, 1.0/fs),correlated_raw)
# p5.set_title('Time domain cross-correlation')
# p5.set_xlabel('Offest between data and template (s)')
# p5.plot(np.arange(0, (correlated_passed.size*1.)/fs, 1.0/fs), correlated_passed,color='red')
# p5.set_xlabel('Offset between data and template (s)')
# correlated = np.correlate(data,temp,'full')
# correlated = np.correlate(data_pass,temp,'full')
# x_corr = np.arange(0, (correlated.size*1.)/fs, 1.0/fs)
# -- clint -- Do a simple convolution and smooth it
temp, tempTS = flipAndAlign(tempHuge, tempHugeTS, time,
dataMaxTime, dataENM, tempE)
# smf = gaussian_filter(temp * data,sigma=float( 10 ))
smf = gaussian_filter(temp * data_LowPass, sigma=float(5))
p5.cla()
p5.plot(time, data, color='blue', alpha=0.4)
p5.plot(time, data_LowPass, color='green')
p5.plot(tempTS, temp, color='red')
p5.plot(tempTS, smf, color='red')
# p5.plot(tempTS,correlated) # use the 'same' option
# p5.plot(x_corr,correlated)
# Matched (Optimal) Filter, freq. domain
# Take the FFT of the data
# data_fft = np.fft.fft(data)
# Take the FFT of the low pass data
data_fft = np.fft.fft(data_LowPass)
# Pad template and take FFT
zero_pad = np.zeros(data.size - template.size)
template_padded = np.append(template, zero_pad)
template_fft = np.fft.fft(template_padded)
# Match FFT frequency bins to PSD frequency bins
# power_noise, freq_psd_noise = plt.psd(data_forceAcq, Fs=fs, NFFT=2048, pad_to=2048, visible=False)
# power_noise, freq_psd_noise = plt.psd(data[:256], Fs=fs, visible=False) # 512 or 256?
power_noise, freq_psd_noise = avgPsd, xPsd # from file
# just for fun, calculate the integral of the 3 different noise spectra
# forceAcqSingleBL = np.sum(power_noise)
# data_bl_psd,_ = plt.psd(data[:512], Fs=fs, NFFT=512, visible=False)
# dataBL = np.sum(data_bl_psd)
# avgForceAcqSpec = np.sum(avgPsd)
datafreq = np.fft.fftfreq(data.size) * fs
power_vec = np.interp(datafreq, freq_psd_noise, power_noise)
# Apply the optimal matched filter
optimal = data_fft * template_fft.conjugate() / power_vec
optimal_time = 2 * np.fft.ifft(optimal)
# Normalize the matched filter output
df = np.abs(datafreq[1] - datafreq[0]) # freq. bin size
sigmasq = 2 * (template_fft * template_fft.conjugate() /
power_vec).sum() * df
sigma = np.sqrt(np.abs(sigmasq))
SNR = abs(optimal_time) / (sigma)
opE = np.amax(SNR)
opEE = opE / dataE
# Plot the result
p6.cla()
p6.plot(time, SNR)
p6.set_title('Optimal Matched Filter, opE %.1f opE/E %.2f' %
(opE, opEE))
p6.set_xlabel('Offset time (s)')
p6.set_ylabel('SNR')
plt.tight_layout()
plt.pause(scanSpeed)
def main(argv):
ds = 5
tt = TChain("skimTree")
tt.Add("~/project/cal/lat/*.root")
# i'm just pasting the output of view-v2 here.
# thesis plot, fast vs slow
# trapENF 16.0, iE 20958, iH 0 # fast
# trapENF 16.0, iE 33039, iH 0 # slow
evtList = [[20958, 0], [33039, 0]]
cols = ['k', 'r', 'g', 'm']
labels = ['Fast, 16.0 keV', 'Slow, 16.0 keV']
alphas = [1, 0.7]
for i, (iE, iH) in enumerate(evtList):
tt.GetEntry(iE)
run = tt.run
chan = tt.channel.at(iH)
hitE = tt.trapENFCal.at(iH)
wf = tt.MGTWaveforms.at(iH)
truncLo, truncHi = 0, 2
if ds == 6 or ds == 2: truncLo = 4
signal = wl.processWaveform(wf, truncLo, truncHi)
waveBLSub = signal.GetWaveBLSub()
waveTS = signal.GetTS()
print("%d / %d Run %d chan %d trapENF %.1f" %
(i, len(evtList), run, chan, hitE))
# plt.plot(waveTS, waveBLSub, "-", lw=1, c=cols[i], label="%.1f keV" % hitE)
plt.plot(waveTS,
waveBLSub,
"-",
lw=2,
c=cols[i],
alpha=alphas[i],
label=labels[i])
# # standard energy trapezoid
# eTrap = wl.trapFilter(waveBLSub,400,250,-7200)
#
# nPad = len(waveBLSub)-len(eTrap)
# eTrap = np.pad(eTrap, (nPad,0), 'constant')
# eTrapTS = np.arange(0, len(eTrap)*10., 10)
# ePickoff = eTrapTS[nPad + 400 + 200]
# # plt.axvline(ePickoff, c='m')
#
# # trigger trap
# tTrap = wl.trapFilter(waveBLSub,100,150,-7200)
# tTrap = np.pad(tTrap, (nPad,0), 'constant')
# tTrapTS = np.arange(0, len(tTrap)*10., 10)
#
# # low pass filter
# B, A = butter(2,1e6/(1e8/2), btype='lowpass')
# waveLP = lfilter(B, A, waveBLSub)
#
# plt.cla()
# plt.plot(waveTS, waveBLSub, 'b', label='Raw WF, %.2f keV' % (hitE))
# # plt.plot(waveTS, waveLP, 'r', alpha=0.7, label='Low-pass filter')
plt.ylim(ymin=-10)
plt.xlabel("Time (ns)", ha='right', x=1)
plt.ylabel("Voltage (ADC)", ha='right', y=1)
plt.legend(loc=4)
# plt.pause(0.0001) # scan speed, does plt.show(block=False) automatically
# plt.show()
plt.savefig('../plots/fast-and-slow.pdf')
def checkLAT():
""" ./check-files.py [-c] -l """
# make sure nLat matches nSplit files
# repeat the checkWave checks
print("Checking lats. Cal?", checkCal)
fileList = []
# bkg
if not checkCal:
dsMap = bkg.dsMap()
for ds in dsMap:
for sub in range(dsMap[ds] + 1):
sList = dsi.getSplitList(
"%s/splitSkimDS%d_%d*" % (dsi.splitDir, ds, sub), sub)
latList = dsi.getSplitList(
"%s/latSkimDS%d_%d*" % (dsi.latDir, ds, sub), sub)
if len(sList) != len(latList):
print(
"Error: ds %d sub %d. Found %d split files but %d lat files."
% (ds, sub, len(sList), len(latList)))
tmpList = [f for idx, f in sorted(latList.items())]
fileList.extend(tmpList)
# cal
else:
dsMap = cal.GetKeys()
for key in dsMap:
ds = int(key[2])
if skipDS6Cal and ds == 6: continue
for sub in range(cal.GetIdxs(key)):
cRuns = cal.GetCalList(key, sub)
for run in cRuns:
sList = dsi.getSplitList(
"%s/splitSkimDS%d_run%d*" % (dsi.calSplitDir, ds, run),
run)
latList = dsi.getSplitList(
"%s/latSkimDS%d_run%d*" % (dsi.calLatDir, ds, run),
run)
if len(sList) != len(latList):
print(
"Error: ds %d sub %d run %d. Found %d split files but %d lat files."
% (ds, sub, run, len(sList), len(latList)))
tmpList = [f for idx, f in sorted(latList.items())]
fileList.extend(tmpList)
if testMode:
fileList = []
ds = 1
# dsMap = bkg.dsMap()
# for sub in range(dsMap[ds]+1):
# latList = dsi.getSplitList("%s/latSkimDS%d_%d*" % (dsi.latDir, ds, sub), sub)
# tmpList = [f for idx, f in sorted(latList.items())]
# fileList.extend(tmpList)
dsMap = cal.GetKeys()
key = dsMap[1]
for sub in range(cal.GetIdxs(key)):
cRuns = cal.GetCalList(key, sub)
for run in cRuns:
latList = dsi.getSplitList(
"%s/latSkimDS%d_run%d*" % (dsi.calLatDir, ds, run), run)
fileList.extend([f for idx, f in sorted(latList.items())])
# loop over files, repeating the same checks in checkWave
for idx, fname in enumerate(fileList[:fLimit]):
f = TFile(fname)
t = f.Get("skimTree")
n = t.GetEntries()
if n == 0:
print("No entries in file", fname)
continue
brSingle, brVector = [], []
for br in t.GetListOfBranches():
if "vector" in br.GetClassName():
brVector.append(br.GetName())
else:
brSingle.append(br.GetName())
if n < 20:
eList = np.arange(0, n, 1)
else:
eList = np.arange(0, 20, 1)
eList = np.append(eList, np.arange(n - 20, n, 1))
for i in eList:
t.GetEntry(i)
# make sure individual entries are accessible (no segfaults)
for br in brSingle:
val = getattr(t, br)
# print(val)
for br in brVector:
try:
nCh = getattr(t, br).size()
except AttributeError:
print("Error:", br)
for j in range(nCh):
val = getattr(t, br)[j]
# print(br,nCh,val)
# make sure we can process waveforms
for j in range(nCh):
wf = t.MGTWaveforms.at(j)
ch = t.channel.at(j)
# be absolutely sure you're matching the right waveform to this hit
if wf.GetID() != ch:
print(
"ERROR -- Vector matching failed. entry %d, file: %s" %
(i, fname))
# run the LAT routine to convert into numpy arrays
truncLo, truncHi = 0, 2
if ds == 6 or ds == 2: truncLo = 4
signal = wl.processWaveform(wf, truncLo, truncHi)
if verbose:
print("%d/%d %s nEnt %d" % (idx, len(
fileList[:fLimit]), fname.split("/")[-1], t.GetEntries()))
f.Close()
def main(argv):
""" Bug: Doesn't always work with a TChain.
Add input files together with hadd and use a single TFile.
"""
scanSpeed = 0.2
opt1, opt2 = "", ""
intMode, printWF = False, False
if (len(argv) >= 1): opt1 = argv[0]
if (len(argv) >= 2): opt2 = argv[1]
if "-i" in (opt1, opt2):
intMode = True
print "Interactive mode selected."
if "-s" in (opt1, opt2):
printWF = True
print "Saving WF plots to current directory."
# Set input file
# DS5 calibration run (lots of all pulses)
# inputFile = TFile("~/project/match-skim/waveletSkimDS5_run23884.root")
# Simulated noise pulses from Brian
inputFile = TFile("./data/SimWF_Baseline.root")
waveTree = inputFile.Get("skimTree")
# ext. pulsing runs (P2D2, chan 674) https://majorana.npl.washington.edu/elog/Run+Elog/1093
# gatFile = TFile("~/project/gat-blt-data/gatified/mjd_run7219.root")
# bltFile = TFile("~/project/gat-blt-data/built/OR_run7219.root")
# ext. trigger runs https://majorana.npl.washington.edu/elog/Run+Elog/1207
# gatFile = TFile("~/project/gat-blt-data/gatified/mjd_run9913.root")
# bltFile = TFile("~/project/gat-blt-data/built/OR_run9913.root")
# waveTree = gatFile.Get("mjdTree")
# bltTree = bltFile.Get("MGTree")
# waveTree.AddFriend(bltTree)
print "Found",waveTree.GetEntries(),"input entries."
# Set cuts
# theCut = inputFile.Get("cutUsedHere").GetTitle()
# theCut = "trapENFCal > 0.8 && gain==0 && mH==1 && isGood && !muVeto && !wfDCBits && !isLNFill1 && !isLNFill2 && trapETailMin < 0 && channel!=596 && channel!=676 && channel!=676 && channel!=612 && channel!=1104 && channel!=1200 && channel!=1334 && channel!=1336"
# theCut = "channel==674"
theCut = ""
print "Using cut:\n",theCut,"\n"
waveTree.Draw(">>elist", theCut, "entrylist")
elist = gDirectory.Get("elist")
waveTree.SetEntryList(elist)
nList = elist.GetN()
print "Found",nList,"entries passing cuts."
# Make a figure
fig = plt.figure(figsize=(8,5), facecolor='w')
# p1 = plt.subplot2grid((6,7), (0,0), colspan=4, rowspan=2) # original
p1 = plt.subplot(111)
plt.show(block=False)
# Make a template waveform
samp, r, z, ene, t0, smooth = 2016, 0, 15, 10, 1000, 100
temp, tempTS = MakeSiggenWaveform(samp,r,z,ene,t0,smooth)
# Loop over events
iList = -1
while(True):
iList += 1
if intMode==True and iList !=0:
value = raw_input()
if value=='q': break
if value=='p': iList -= 2 # previous
if (value.isdigit()): iList = int(value) # go to entry
if iList >= elist.GetN(): break
entry = waveTree.GetEntryNumber(iList);
waveTree.LoadTree(entry)
waveTree.GetEntry(entry)
# nChans = waveTree.channel.size()
nChans = waveTree.MGTWaveforms.size()
# numPass = waveTree.Draw("channel",theCut,"GOFF",1,iList)
# chans = waveTree.GetV1()
# chanList = list(set(int(chans[n]) for n in xrange(numPass)))
# event = bltTree.event # grab waveforms from built data (comment this out if we're using clint-skim)
# Loop over hits passing cuts
# hitList = (iH for iH in xrange(nChans) if waveTree.channel.at(iH) in chanList) # a 'generator expression'
# for iH in hitList:
for iH in range(nChans):
# run = waveTree.run
# chan = waveTree.channel.at(iH)
# dataE = waveTree.trapENFCal.at(iH)
signal = wl.processWaveform(waveTree.MGTWaveforms.at(iH)) # use when we have skim
# signal = wl.processWaveform(event.GetWaveform(iH)) # use when we have gat+blt data
waveBLSub = signal.GetWaveBLSub()
waveFilt = signal.GetWaveFilt()
waveTS = signal.GetTS() * 10.
# print "%d / %d Run %d nCh %d chan %d trapENF %.1f" % (iList,nList,run,nChans,chan,dataE)
# Generate a siggen WF
samp, r, z, ene, t0, smooth = 2016, 0, 15, 10, 1000, 100
sigg, siggTS = MakeSiggenWaveform(samp,r,z,ene,t0,smooth)
data = sigg + waveBLSub
# Fill the figure
p1.cla()
p1.set_ylabel("ADC")
# p1.set_title("Run %d Channel %d Entry %d trapENFCal %.1f" % (run,chan,iList,dataE))
p1.plot(waveTS,data,color='blue')
p1.plot(tempTS,temp,color='red')
plt.pause(scanSpeed)
def main(argv):
startT = time.clock()
inDir, outDir = ".", "."
inFile, inFileName, outFileName = TFile(), "", ""
gatTree = TTree()
dsNum, subNum, runNum, theCut, inFileName = -1, -1, -1, "", ""
saveLAT, savePacket, saveWave = False, False, False
if len(argv) == 0: return
for i, opt in enumerate(argv):
if opt == "-f":
# dsNum, subNum, inDir, outDir = int(argv[i+1]), int(argv[i+2]), str(argv[i+3]), str(argv[i+4])
inDir, inFileName, outDir = str(argv[i + 1]), str(
argv[i + 2]), str(argv[i + 3])
# inFileName = "waveSkimDS{}_{}.root".format(dsNum, subNum)
if opt == "-r":
inDir, inFileName, outDir = str(argv[i + 1]), str(
argv[i + 2]), str(argv[i + 3])
# dsNum, runNum, inDir, outDir = int(argv[i+1]), int(argv[i+2]), str(argv[i+3]), str(argv[i+4])
# inFileName = "waveSkimDS{}_run{}.root".format(dsNum, runNum)
# inFileName = inDir + inFile
print("Scanning File: {}".format(inFileName))
inFile = TFile("%s/%s" % (inDir, inFileName))
gatTree = inFile.Get("skimTree")
theCut = inFile.Get("theCut").GetTitle()
# Make files smaller for tests
# theCut += " && sumEHL > 236 && sumEHL < 240 && mHL==2 && trapENFCal < 5"
# Select only pulsers
# theCut += " && EventDC1Bits > 0"
print "Using cut:\n", theCut
gatTree.Draw(">>elist", theCut, "entrylist")
elist = gDirectory.Get("elist")
gatTree.SetEntryList(elist)
nList = elist.GetN()
print "Found", gatTree.GetEntries(), "input entries."
print "Found", nList, "entries passing cuts."
# Gimmicky but works... this bypasses creating the branches...
gatTree.GetEntry(0)
# Mess of various branches
channel = std.vector("int")()
trapENFCal = std.vector("double")()
trapENM = std.vector("double")()
# Create map of branches to put into dataframe
# This map is only for branches that we want to keep!
keepMapBase = {
'trapENFCal': gatTree.trapENFCal,
'trapENM': gatTree.trapENM,
"channel": gatTree.channel,
"run": gatTree.run,
"mHL": gatTree.mHL
}
# Combine dictionaries, if keepMapLAT is empty it won't add any branches
keepMap = dict(keepMapBase)
# keepMap.update(keepMapLAT)
dataList = []
print 'Writing to: ', '%s/proc%s.h5' % (outDir, inFileName.split('.')[0])
iList, removeNBeg, removeNEnd = -1, 500, 500
# Loop over events
while True:
iList += 1
if iList >= nList: break
# if iList >= 5000: break
entry = gatTree.GetEntryNumber(iList)
gatTree.LoadTree(entry)
gatTree.GetEntry(entry)
nChans = gatTree.channel.size()
numPass = gatTree.Draw("channel", theCut, "GOFF", 1, iList)
chans = gatTree.GetV1()
chanList = list(set(int(chans[n]) for n in xrange(numPass)))
hitList = (iH for iH in xrange(nChans)
if gatTree.channel.at(iH) in chanList
) # a 'generator expression'
for iH in hitList:
dataMap = {}
wf = gatTree.MGTWaveforms.at(iH)
signal = wl.processWaveform(wf, removeNBeg, removeNEnd)
wave = np.array(signal.GetWaveRaw(), dtype=np.int16)
for key, branch in keepMap.items():
# Save branches that aren't vector<Template> (so far only run and mHL)
if key == 'run' or key == 'mHL': dataMap[key] = int(branch)
elif key == 'channel': dataMap[key] = int(branch.at(iH))
else: dataMap[key] = float(branch.at(iH))
dataMap['waveform'] = wave.tolist()
dataList.append(dataMap)
if iList % 5000 == 0 and iList != 0:
print "%d / %d entries saved (%.2f %% done), time: %s" % (
iList, nList, 100 *
(float(iList) / nList), time.strftime('%X %x %Z'))
df = pd.DataFrame.from_dict(dataList)
print(df.head())
print(df.info())
print(np.unique(df.channel))
print(np.unique(df.mHL))
print(np.unique(df.run))
# Suppress stupid warning
warnings.filterwarnings(action="ignore",
module="pandas",
message="^\nyour performance")
# Chunk write like a sucker
chunksize = 50000
start = 0
end = chunksize - 1
i = 0
# for i in len(df):
while end < df.shape[0]:
# chunk = df.iloc[(i*chunksize):min((i+1)*chunksize,len(df))]
chunk = df.iloc[start:end]
try:
chunk.to_hdf('{}/proc{}_{}.h5'.format(outDir,
inFileName.split('.')[0], i),
key='skimTree',
data_columns=[
'trapENFCal', 'trapENM', 'channel', 'mHL',
'waveform'
],
format='fixed',
mode='w',
complevel=9)
except (Exception) as e:
print e
print chunk
print chunk.info()
start += chunksize
end += chunksize
i += 1
# df.to_hdf('%s/proc%s.h5' % (outDir,inFileName.split('.')[0]), key="skimTree", data_columns=['trapENFCal', 'trapENM','channel','mHL','waveform'], format='fixed', mode='w', complevel=9)
stopT = time.clock()
print("Stopped:", time.strftime('%X %x %Z'), "\nProcess time (min):",
(stopT - startT) / 60)
print(float(nList) / ((stopT - startT) / 60.), "entries per minute.")
def main(argv):
quickDraw = False
for i, opt in enumerate(argv):
if opt == "-q":
print("Quick draw mode selected.")
quickDraw = True
# ds = "5A"
# tt.Add("~/project/cal/lat/*.root")
# tt.Add("~/project/cal/lat/latSkimDS1_run13774_0.root")
# tCut = "trapENFCal > 238 && trapENFCal < 239" # 238 kev wf, thesis plot
# tCut = "trapENFCal >= 1.0 && trapENFCal < 1.2 && channel!=598 && trapENFCal > threshKeV+3*threshSigma && fitSlo < 100" # 1 kev wf, thesis plot
# tCut = "trapENFCal > 16 && trapENFCal < 16.1 && fitSlo < 100 && tOffset<10"
# tCut = "trapENFCal > 16 && trapENFCal < 16.1 && fitSlo > 100"
# don't open the final files as tchains, it made a big segfault??
# tf = TFile("~/project/bkg/cut/final/final_DS%s.root" % ds)
# tt = tf.Get("skimTree")
# tCut = "trapENFCal >= 1 && trapENFCal < 4"
# tCut = "trapENFCal < 1.5 && tOffset > 1000" # try to grab a retrigger waveform
# ds = 5
tt = TChain("skimTree")
tt.Add("~/project/bkg/cut/final95/final95*.root")
tCut = "tOffset > 4000" # for sure retriggers
# tCut = "tOffset > 200 && tOffset < 2000"
n = tt.Draw("Entry$:Iteration$", tCut, "goff")
evt, itr = tt.GetV1(), tt.GetV2()
evtList = [[int(evt[i]), int(itr[i])] for i in range(n)]
nEnt = len(set([evt[i] for i in range(n)]))
print("Found %d total entries, %d passing cut: %s" %
(tt.GetEntries(), nEnt, tCut))
dsr = bkg.dsRanges()
i, pEvt = -1, -1
while (True):
i += 1
if not quickDraw and i != 0:
val = input()
if val == "q": break
if val == "p": i -= 2
if val.isdigit(): i = int(val)
if val == "s":
pltName = "../plots/wf-%d.pdf" % i
print("Saving figure:", pltName)
plt.savefig(pltName)
if i >= len(evtList): break
iE, iH = evtList[i]
if iE != pEvt:
tt.GetEntry(iE)
pEvt = iE
run = tt.run
ds = bkg.GetDSNum(run)
chan = tt.channel.at(iH)
hitE = tt.trapENFCal.at(iH)
tOff = tt.tOffset.at(iH)
wf = tt.MGTWaveforms.at(iH)
truncLo, truncHi = 0, 2
if ds == 6 or ds == 2: truncLo = 4
signal = wl.processWaveform(wf, truncLo, truncHi)
# waveform
waveBLSub = signal.GetWaveBLSub()
waveTS = signal.GetTS()
print("%d / %d Run %d chan %d trapENF %.1f, iE %d, iH %d" %
(i, len(evtList), run, chan, hitE, iE, iH))
# standard energy trapezoid
eTrap = wl.trapFilter(waveBLSub, 400, 250, -7200)
nPad = len(waveBLSub) - len(eTrap)
eTrap = np.pad(eTrap, (nPad, 0), 'constant')
eTrapTS = np.arange(0, len(eTrap) * 10., 10)
ePickoff = eTrapTS[nPad + 400 + 200]
# plt.axvline(ePickoff, c='m')
# trigger trap
tTrap = wl.trapFilter(waveBLSub, 100, 150, -7200)
tTrap = np.pad(tTrap, (nPad, 0), 'constant')
tTrapTS = np.arange(0, len(tTrap) * 10., 10)
# low pass filter
B, A = butter(2, 1e6 / (1e8 / 2), btype='lowpass')
waveLP = lfilter(B, A, waveBLSub)
# wavelet denoised
wp = pywt.WaveletPacket(waveBLSub, 'db2', 'symmetric', maxlevel=4)
new_wp = pywt.WaveletPacket(data=None, wavelet='db2', mode='symmetric')
new_wp['aaa'] = wp['aaa'].data
waveDenoised = new_wp.reconstruct(update=False)
# resize in a smart way
diff = len(waveDenoised) - len(waveBLSub)
if diff > 0: waveDenoised = waveDenoised[diff:]
plt.cla()
plt.plot(waveTS,
waveBLSub,
'b',
lw=2,
label='Raw WF, %.2f keV' % (hitE))
# plt.plot(waveTS, waveBLSub, 'b', alpha=0.2, label='Raw WF, %.2f keV' % (hitE))
# plt.plot(waveTS, waveLP, 'r', alpha=0.7, label='Low-pass filter')
plt.plot(waveTS,
waveDenoised,
"r",
lw=2,
alpha=0.5,
label="Denoised WF")
plt.plot(np.nan, np.nan, "-w", label="tOffset: %d" % tOff)
plt.xlabel("Time (ns)", ha='right', x=1)
plt.ylabel("Voltage (ADC)", ha='right', y=1)
plt.legend(loc=4)
plt.tight_layout()
plt.pause(
0.0001) # scan speed, does plt.show(block=False) automatically
def main(argv):
scanSpeed = 1.0
opt1, opt2 = "", ""
intMode, printWF = False, False
if (len(argv) >= 1): opt1 = argv[0]
if (len(argv) >= 2): opt2 = argv[1]
if "-i" in (opt1, opt2):
intMode = True
print "Interactive mode selected."
if "-s" in (opt1, opt2):
printWF = True
print "Saving WF plots to current directory."
# Set input file
inputFile = TFile(
"./data/SimWF_Baseline.root") # simulated noise pulses from Brian
waveTree = inputFile.Get("skimTree")
theCut = ""
# Make a figure
fig = plt.figure(figsize=(8, 5), facecolor='w')
# fig, (ax_orig, ax_noise, ax_corr) = plt.subplots(3, 1, sharex=True)
# p1 = plt.subplot2grid((6,7), (0,0), colspan=4, rowspan=2) # original
p1 = plt.subplot(111)
plt.show(block=False)
# Make a template waveform
samp, r, z, ene, t0, smooth = 5000, 0, 15, 10, 2500, 100 # huge floating waveform
tempOrig, tempOrigTS = wl.MakeSiggenWaveform(samp, r, z, ene, t0, smooth)
# Loop over events
iEnt = -1
while (True):
iEnt += 1
if intMode == True and iEnt != 0:
value = raw_input()
if value == 'q': break
if value == 'p': iEnt -= 2 # previous
if (value.isdigit()): iEnt = int(value) # go to entry
if iEnt >= waveTree.GetEntriesFast(): break
waveTree.GetEntry(iEnt)
# Make a fake "data" pulse
signal = wl.processWaveform(
waveTree.MGTWaveforms.at(0)) # brian only has 1 pulse per event
noise = signal.GetWaveBLSub()
dataTS = signal.GetTS() * 10.
samp, r, z, ene, t0, smooth = 2016, 0, 15, 10, 1000, 100 # realistic waveform
sigg, siggTS = wl.MakeSiggenWaveform(samp, r, z, ene, t0, smooth)
data = sigg + noise
# how long does this take?
start = time.time()
wp = pywt.WaveletPacket(data=data,
wavelet='haar',
mode='symmetric',
maxlevel=3)
new_wp = pywt.WaveletPacket(data=None,
wavelet='haar',
mode='symmetric')
new_wp['aaa'] = wp['aaa'].data
newWave = new_wp.reconstruct(update=False)
stop = time.time()
print "denoising took", stop - start
# what about this part?
start = time.time()
deriv = wl.wfDerivative(newWave)
stop = time.time()
print "derivative took", stop - start
p1.cla()
p1.set_ylabel("ADC")
p1.plot(dataTS, data, color='blue')
p1.plot(siggTS, sigg, color='red')
def procSim(nMax=None):
"""
Get sum and hit spectra w/ threshold cut, without ch. 598 (it's noisy.)
Here we use "mHT" and "sumET" exclusively.
Use !EventDC1Bits and trapENFCal > 0.7, all hits. (Skim file was generated w/ 'dontSkipAnything')
Save detailed info for events with hits < 2630 keV.
"""
from ROOT import TFile, TTree, MGTWaveform
# fig = plt.figure(figsize=(12,7), facecolor='w')
# p0 = plt.subplot2grid((6,10), (0,0), colspan=10, rowspan=3) # waveform fit
# p1 = plt.subplot2grid((6,10), (3,0), colspan=10, rowspan=1) # residual
# p2 = plt.subplot2grid((6,10), (4,0), colspan=2, rowspan=2) # traces
# p3 = plt.subplot2grid((6,10), (4,2), colspan=2, rowspan=2)
# p4 = plt.subplot2grid((6,10), (4,4), colspan=2, rowspan=2)
# p5 = plt.subplot2grid((6,10), (4,6), colspan=2, rowspan=2)
# p6 = plt.subplot2grid((6,10), (4,8), colspan=2, rowspan=2)
# Use the force acquisition baseline + SigGen waveforms
inDir = os.environ['LATDIR'] + '/data'
f1 = TFile('{}/waveSkimDS0_run4201.root'.format(inDir))
skimTree = f1.Get('skimTree')
dfSigGen = pd.read_hdf('{}/SigGen_WFs_P42574A.h5'.format(inDir))
# sigList = [0, 20, 40, 60]
sigList = [0]
# Truncates to 2017 samples, cuz we in 2017
truncLo, truncHi = 8, 6
dataList = []
print('Total Entries: ', skimTree.GetEntries())
maxVal = 0
if nMax:
maxVal = min(nMax, skimTree.GetEntries())
else:
maxVal = skimTree.GetEntries()
for idx in range(maxVal):
if idx % 100 == 0:
print("Processed Entry ", idx)
skimTree.GetEntry(idx)
if skimTree.mH > 1: continue # Only care about mH1 for noise
channel = skimTree.channel.at(0)
if channel != 624: continue
wf = MGTWaveform()
wf = skimTree.MGTWaveforms.at(0)
signal = wl.processWaveform(wf, truncLo, truncHi)
data = signal.GetWaveRaw()
data_blSub = signal.GetWaveBLSub()
dataTS = signal.GetTS()
dataBL, dataNoise = signal.GetBaseNoise()
# Generate one fake WF at each amplitude/r/z combination (30 total)
for index, row in dfSigGen.iterrows():
# Skip the waveforms that are far
# if row['r'] != 34.0: continue
# if row['z'] != 50.0: continue
for sigma in sigList:
dataMap = {}
if sigma == 0:
simWF = row['waveform']
else:
simWF = gaussian_filter(row['waveform'], sigma)
sig = addSigToBaseline(data, simWF)
sigBL = addSigToBaseline(data_blSub, simWF)
# Dummy variables
# fitAmp, TSMax = row['amp'], 1200*10
dataMap = procEvent(
sig, dataTS, sigBL, dataBL,
'{}_A{}R{}Z{}'.format(idx, row['amp'], row['r'], row['z']))
dataMap['Amp'] = row['amp']
dataMap['R'] = row['r']
dataMap['Z'] = row['z']
dataMap['Sigma'] = sigma
# dataMap['waveform'] = sigBL.tolist()
dataList.append(dataMap)
df = pd.DataFrame.from_dict(dataList)
print(df.head())
df.to_hdf('{}/SimPSA_P42574A.h5'.format(inDir), 'skimTree')
def main(argv):
"""Interactive-draw or rapid-draw waveforms that pass a given TCut.
Bug: Doesn't always work with a TChain. Add input files together
with hadd and use a single TFile.
"""
scanSpeed = 0.2
opt1, opt2 = "", ""
intMode, printWF, warpMode = False, False, False
if (len(argv) >= 1): opt1 = argv[0]
if (len(argv) >= 2): opt2 = argv[1]
if "-i" in (opt1, opt2):
intMode = True
print("Interactive mode selected.")
if "-s" in (opt1, opt2):
printWF = True
print("Saving WF plots to current directory.")
if "-w" in (opt1, opt2):
warpMode = True
import matplotlib
matplotlib.use('TkAgg')
print("Warp-speed drawing, don't trust figure axes.")
import matplotlib.pyplot as plt
plt.style.use('../pltReports.mplstyle')
from matplotlib.colors import LogNorm, Normalize
# Set input file and cuts
dsNum = 5
# waveTree = TChain("skimTree") # NOTE: Can't always recognize MGTWaveforms branch
# waveTree.Add("~/project/lat/latSkimDS1*")
inputFile = TFile(
"/Users/wisecg/project/cal/lat/latSkimDS5_run20044_0.root")
# inputFile = TFile("~/project/cal-waves/waveSkimDS2_run14857.root")
# inputFile = TFile("~/project/cal-waves/waveSkimDS5_run23895.root")
waveTree = inputFile.Get("skimTree")
print("Found", waveTree.GetEntries(), "input entries.")
theCut = inputFile.Get("theCut").GetTitle()
# theCut += " && trapENFCal 1.1"
# theCut = " trapENFCal > 0.5 && trapENFCal < 10"
# Print cut and events passing cut
print("Using cut:\n", theCut, "\n")
waveTree.Draw(">>elist", theCut, "entrylist")
elist = gDirectory.Get("elist")
waveTree.SetEntryList(elist)
nList = elist.GetN()
print("Found", nList, "entries passing cuts.")
# Make a figure (only setting data in the loop is faster)
fig = plt.figure()
a1 = plt.subplot(111)
a1.set_xlabel("Time (ns)", ha='right', x=1)
a1.set_ylabel("Voltage (ADC)", ha='right', y=1)
p1, = a1.plot(np.ones(1),
np.ones(1),
color='blue',
alpha=0.8,
label='data')
p2, = a1.plot(np.ones(1),
np.ones(1),
color='magenta',
linewidth=1.5,
label='denoised')
p3, = a1.plot(np.ones(1),
np.ones(1),
color='red',
linewidth=3,
label='lowpass')
# a1.legend(loc=4)
plt.show(block=False)
# Loop over events
iList = -1
while (True):
iList += 1
if intMode == True and iList != 0:
value = input()
if value == 'q': break
if value == 'p': iList -= 2 # previous
if (value.isdigit()): iList = int(value) # go to entry
if iList >= elist.GetN(): break
entry = waveTree.GetEntryNumber(iList)
waveTree.LoadTree(entry)
waveTree.GetEntry(entry)
nChans = waveTree.channel.size()
nWFs = waveTree.MGTWaveforms.size()
if (nWFs == 0):
print("Error - nWFs:", nWFs, "nChans", nChans)
continue
numPass = waveTree.Draw("channel", theCut, "GOFF", 1, iList)
chans = waveTree.GetV1()
chanList = list(set(int(chans[n]) for n in range(numPass)))
# Loop over hits passing cuts
hitList = (iH for iH in range(nChans)
if waveTree.channel.at(iH) in chanList)
for iH in hitList:
run = waveTree.run
chan = waveTree.channel.at(iH)
energy = waveTree.trapENFCal.at(iH)
wf = waveTree.MGTWaveforms.at(iH)
truncLo, truncHi = 0, 2
if dsNum == 6 or dsNum == 2: truncLo = 4
signal = wl.processWaveform(wf, truncLo, truncHi)
waveBLSub = signal.GetWaveBLSub()
# waveRaw = signal.GetWaveRaw()
waveTS = signal.GetTS()
print("%d / %d Run %d nCh %d chan %d trapENF %.1f" %
(iList, nList, run, nChans, chan, energy))
# standard energy trapezoid
eTrap = wl.trapFilter(waveBLSub, 400, 250, -7200)
nPad = len(waveBLSub) - len(eTrap)
eTrap = np.pad(eTrap, (nPad, 0), 'constant')
eTrapTS = np.arange(0, len(eTrap) * 10., 10)
ePickoff = eTrapTS[nPad + 400 + 200]
# plt.axvline(ePickoff, c='m')
# trigger trap
tTrap = wl.trapFilter(waveBLSub, 100, 150, -7200)
tTrap = np.pad(tTrap, (nPad, 0), 'constant')
tTrapTS = np.arange(0, len(tTrap) * 10., 10)
# -- fill the figure --
p1.set_ydata(waveBLSub)
p1.set_xdata(waveTS)
p2.set_ydata(tTrap)
p2.set_ydata(tTrapTS)
p3.set_ydata(eTrap)
p3.set_xdata(eTrapTS)
xmin, xmax = np.amin(waveTS), np.amax(waveTS)
ymin, ymax = np.amin(waveBLSub), np.amax(waveBLSub)
a1.set_xlim([xmin, xmax])
a1.set_ylim([ymin - abs(0.1 * ymin), ymax + abs(0.1 * ymax)])
# plt.title("Run %d Channel %d Entry %d trapENFCal %.1f" % (run,chan,iList,energy))
if warpMode:
# superfast, requires TkAgg backend, doesn't update axes
a1.draw_artist(a1.patch)
a1.draw_artist(p1)
fig.canvas.blit(a1.bbox)
fig.canvas.flush_events()
else:
plt.pause(scanSpeed)
if (printWF):
plt.savefig("./plots/wave-%d-%d-%d.pdf" % (run, iList, chan))
def main(argv):
print("=======================================")
print("LAT started:",time.strftime('%X %x %Z'))
startT = time.clock()
# gROOT.ProcessLine("gErrorIgnoreLevel = 3001;") # suppress ROOT error messages
global batMode
intMode, batMode, rangeMode, fileMode, gatMode, singleMode, pathMode, cutMode = False, False, False, False, False, False, False, False
dontUseTCuts = False
dsNum, subNum, runNum, plotNum = -1, -1, -1, 1
pathToInput, pathToOutput, manualInput, manualOutput, customPar = ".", ".", "", "", ""
if len(argv)==0: return
for i,opt in enumerate(argv):
if opt == "-r":
rangeMode, dsNum, subNum = True, int(argv[i+1]), int(argv[i+2])
print("Scanning DS-%d sub-range %d" % (dsNum, subNum))
if opt == "-p":
pathMode, manualInput, manualOutput = True, argv[i+1], argv[i+2]
print("Manually set input/output files:\nInput: %s\nOutput: %s" % (manualInput, manualOutput))
if opt == "-d":
pathToInput, pathToOutput = argv[i+1], argv[i+2]
print("Custom paths: Input %s, Output %s" % (pathToInput,pathToOutput))
if opt == "-f":
fileMode, dsNum, runNum = True, int(argv[i+1]), int(argv[i+2])
print("Scanning DS-%d, run %d" % (dsNum, runNum))
if opt == "-g":
gatMode, runNum = True, int(argv[i+1])
print("GATDataSet mode. Scanning run %d" % (runNum))
if opt == "-s":
singleMode, pathToInput = True, argv[i+1]
print("Single file mode. Scanning {}".format(pathToInput))
if opt == "-i":
intMode, plotNum = True, int(argv[i+1])
print("Interactive mode selected. Use \"p\" for previous and \"q\" to exit.")
if opt == "-x":
dontUseTCuts = True
print("DC TCuts deactivated. Retaining all events ...")
if opt == "-c":
cutMode, customPar = True, str(argv[i+1])
print("Using custom cut parameter: {}".format(customPar))
if opt == "-b":
batMode = True
import matplotlib
# if os.environ.get('DISPLAY','') == '':
# print('No display found. Using non-interactive Agg backend')
matplotlib.use('Agg')
print("Batch mode selected. A new file will be created.")
import matplotlib.pyplot as plt
from matplotlib import gridspec
import matplotlib.ticker as mtick
plt.style.use('pltTalks.mplstyle')
from matplotlib.colors import LogNorm, Normalize
# File I/O
inFile, outFile, bltFile = TFile(), TFile(), TFile()
gatTree, bltTree, out = TTree(), TTree(), TTree()
theCut, inPath, outPath = "", "", ""
# Set input and output files
if rangeMode:
inPath = "%s/waveSkimDS%d_%d.root" % (pathToInput, dsNum, subNum)
outPath = "%s/latSkimDS%d_%d.root" % (pathToOutput, dsNum, subNum)
if fileMode:
inPath = "%s/waveSkimDS%d_run%d.root" % (pathToInput, dsNum, runNum)
outPath = "%s/latSkimDS%d_run%d.root" % (pathToOutput, dsNum, runNum)
if pathMode:
inPath, outPath = manualInput, manualOutput
if gatMode:
ds = GATDataSet()
gatPath = ds.GetPathToRun(runNum,GATDataSet.kGatified)
bltPath = ds.GetPathToRun(runNum,GATDataSet.kBuilt)
outPath = "%s/lat_run%d.root" % (pathToOutput, runNum)
if pathMode and gatMode:
outPath = manualOutput
# Initialize trees
if rangeMode or fileMode or pathMode:
inFile = TFile(inPath)
gatTree = inFile.Get("skimTree")
print(gatTree.GetEntries(),"entries in input tree.")
elif gatMode:
inFile = TFile(gatPath)
bltFile = TFile(bltPath)
gatTree = inFile.Get("mjdTree")
bltTree = bltFile.Get("MGTree")
gatTree.AddFriend(bltTree)
if singleMode:
inFile = TFile(pathToInput)
gatTree = inFile.Get("skimTree")
# apply cut to tree
if (rangeMode or fileMode or pathMode) and not dontUseTCuts:
try:
theCut = inFile.Get("theCut").GetTitle()
except ReferenceError:
theCut = ""
if cutMode:
# theCut += customPar
# theCut = "(channel==672 || channel==674) && mH==2" # sync chan: 672, extp chan: 674
# theCut += " && fitSlo < 10"
# theCut = "trapENFCal > 1 && trapENFCal < 10 && riseNoise > 2"
theCut = "trapENFCal > 20 && trapENFCal < 100 && riseNoise > 2"
print("WARNING: Custom cut in use! : ",theCut)
gatTree.Draw(">>elist", theCut, "entrylist")
elist = gDirectory.Get("elist")
gatTree.SetEntryList(elist)
nList = elist.GetN()
print("Using cut:\n",theCut)
print("Found",gatTree.GetEntries(),"input entries.")
print("Found",nList,"entries passing cuts.")
# Output: In batch mode (-b) only, create an output file+tree & append new branches.
if batMode and not intMode:
outFile = TFile(outPath, "RECREATE")
print("Attempting tree copy to",outPath)
out = gatTree.CopyTree("")
out.Write()
print("Wrote",out.GetEntries(),"entries.")
cutUsed = TNamed("theCut",theCut)
cutUsed.Write()
waveS1, waveS2 = std.vector("double")(), std.vector("double")()
waveS3, waveS4, waveS5 = std.vector("double")(), std.vector("double")(), std.vector("double")()
bcMax, bcMin = std.vector("double")(), std.vector("double")()
bandMax, bandTime = std.vector("double")(), std.vector("double")()
den10, den50, den90 = std.vector("double")(), std.vector("double")(), std.vector("double")()
oppie = std.vector("double")()
fitMu, fitAmp, fitSlo = std.vector("double")(), std.vector("double")(), std.vector("double")()
fitTau, fitBL = std.vector("double")(), std.vector("double")()
matchMax, matchWidth, matchTime = std.vector("double")(), std.vector("double")(), std.vector("double")()
pol0, pol1, pol2, pol3 = std.vector("double")(), std.vector("double")(), std.vector("double")(), std.vector("double")()
fails, fitChi2, fitLL = std.vector("int")(), std.vector("double")(), std.vector("double")()
riseNoise = std.vector("double")()
t0_SLE, t0_ALE, lat, latF = std.vector("double")(), std.vector("double")(), std.vector("double")(), std.vector("double")()
latAF, latFC, latAFC = std.vector("double")(), std.vector("double")(), std.vector("double")()
nMS = std.vector("int")()
tE50, latE50, wfStd = std.vector("double")(), std.vector("double")(), std.vector("double")()
wfAvgBL, wfRMSBL = std.vector("double")(), std.vector("double")()
fitErr = std.vector("int")()
# It's not possible to put the "out.Branch" call into a class initializer (waveLibs::latBranch). You suck, ROOT.
b1, b2 = out.Branch("waveS1",waveS1), out.Branch("waveS2",waveS2)
b3, b4, b5 = out.Branch("waveS3",waveS3), out.Branch("waveS4",waveS4), out.Branch("waveS5",waveS5)
b7, b8 = out.Branch("bcMax",bcMax), out.Branch("bcMin",bcMin)
b9, b10 = out.Branch("bandMax",bandMax), out.Branch("bandTime",bandTime)
b11, b12, b13 = out.Branch("den10",den10), out.Branch("den50",den50), out.Branch("den90",den90)
b14 = out.Branch("oppie",oppie)
b15, b16, b17 = out.Branch("fitMu", fitMu), out.Branch("fitAmp", fitAmp), out.Branch("fitSlo", fitSlo)
b18, b19 = out.Branch("fitTau",fitTau), out.Branch("fitBL",fitBL)
b20, b21, b22 = out.Branch("matchMax", matchMax), out.Branch("matchWidth", matchWidth), out.Branch("matchTime", matchTime)
b23, b24, b25, b26 = out.Branch("pol0", pol0), out.Branch("pol1", pol1), out.Branch("pol2", pol2), out.Branch("pol3", pol3)
b27, b28, b29 = out.Branch("fails",fails), out.Branch("fitChi2",fitChi2), out.Branch("fitLL",fitLL)
b30 = out.Branch("riseNoise",riseNoise)
b31, b32, b33, b34 = out.Branch("t0_SLE",t0_SLE), out.Branch("t0_ALE",t0_ALE), out.Branch("lat",lat), out.Branch("latF",latF)
b35, b36, b37 = out.Branch("latAF",latAF), out.Branch("latFC",latFC), out.Branch("latAFC",latAFC)
b38 = out.Branch("nMS",nMS)
b39, b40, b41 = out.Branch("tE50", tE50), out.Branch("latE50", latE50), out.Branch("wfStd", wfStd)
b42, b43 = out.Branch("wfAvgBL", wfAvgBL), out.Branch("wfRMSBL", wfRMSBL)
b44 = out.Branch("fitErr",fitErr)
# make a dictionary that can be iterated over (avoids code repetition in the loop)
brDict = {
"waveS1":[waveS1, b1], "waveS2":[waveS2, b2],
"waveS3":[waveS3, b3], "waveS4":[waveS4, b4], "waveS5":[waveS5, b5],
"bcMax":[bcMax, b7], "bcMin":[bcMin, b8],
"bandMax":[bandMax, b9], "bandTime":[bandTime, b10],
"den10":[den10, b11], "den50":[den50, b12], "den90":[den90, b13],
"oppie":[oppie, b14],
"fitMu":[fitMu, b15], "fitAmp":[fitAmp, b16], "fitSlo":[fitSlo, b17],
"fitTau":[fitTau, b18], "fitBL":[fitBL,b19],
"matchMax":[matchMax, b20], "matchWidth":[matchWidth, b21], "matchTime":[matchTime, b22],
"pol0":[pol0, b23], "pol1":[pol1, b24], "pol2":[pol2, b25], "pol3":[pol3, b26],
"fails":[fails,b27], "fitChi2":[fitChi2,b28], "fitLL":[fitLL,b29],
"riseNoise":[riseNoise,b30],
"t0_SLE":[t0_SLE,b31], "t0_ALE":[t0_ALE,b32], "lat":[lat,b33], "latF":[latF,b34],
"latAF":[latAF,b35], "latFC":[latFC,b36], "latAFC":[latAFC,b37],
"nMS":[nMS,b38], "tE50":[tE50,b39], "latE50":[latE50,b40], "wfStd":[wfStd,b41],
"wfAvgBL":[wfAvgBL,b42], "wfRMSBL":[wfRMSBL,b43],
"fitErr":[fitErr,b44]
}
# Make a figure (-i option: select different plots)
# fig = plt.figure(figsize=(12,9), facecolor='w')
fig = plt.figure()
if plotNum==0 or plotNum==7 or plotNum==8:
p0 = plt.subplot(111) # 0-raw waveform, 7-new trap filters
elif plotNum==1 or plotNum==2:
p0 = plt.subplot(211) # 1-wavelet, 2-time points, bandpass filters, tail slope
p1 = plt.subplot(212)
elif plotNum==3:
p0 = plt.subplot2grid((2,5), (0,0), colspan=3) # oppie / freq-domain matched filter
p1 = plt.subplot2grid((2,5), (0,3), colspan=2)
p2 = plt.subplot2grid((2,5), (1,0), colspan=3)
elif plotNum==4:
p0 = plt.subplot(111) # time-domain matched filter
elif plotNum==5:
p0 = plt.subplot(111) # bandpass / bandTime
elif plotNum==6:
p0 = plt.subplot2grid((6,10), (0,0), colspan=10, rowspan=3) # waveform fit
p1 = plt.subplot2grid((6,10), (3,0), colspan=10, rowspan=1) # residual
p2 = plt.subplot2grid((6,10), (4,0), colspan=2, rowspan=2) # traces
p3 = plt.subplot2grid((6,10), (4,2), colspan=2, rowspan=2)
p4 = plt.subplot2grid((6,10), (4,4), colspan=2, rowspan=2)
p5 = plt.subplot2grid((6,10), (4,6), colspan=2, rowspan=2)
p6 = plt.subplot2grid((6,10), (4,8), colspan=2, rowspan=2)
elif plotNum==9:
p0 = plt.subplot2grid((5,1), (0,0)) # 9- wpt on wf fit residual
p1 = plt.subplot2grid((5,1), (1,0), rowspan=2)
p2 = plt.subplot2grid((5,1), (3,0), rowspan=2)
if not batMode: plt.show(block=False)
# Load a fast signal template - used w/ the freq-domain matched filter
# print("Generating signal template ...")
tSamp, tR, tZ, tAmp, tST, tSlo = 5000, 0, 15, 100, 2500, 10
# tOrig, tOrigTS = wl.MakeSiggenWaveform(tSamp,tR,tZ,tAmp,tST,tSlo) # Damn you to hell, PDSF
templateFile = np.load("%s/data/lat_template.npz" % os.environ['LATDIR'])
if dsNum==2 or dsNum==6: templateFile = np.load("%s/data/lat_ds2template.npz" % os.environ['LATDIR'])
tOrig, tOrigTS = templateFile['arr_0'], templateFile['arr_1']
# Load stuff from DS1 forced acq. runs
npzfile = np.load("%s/data/fft_forcedAcqDS1.npz" % os.environ['LATDIR'])
noise_asd, noise_xFreq, avgPwrSpec, xPwrSpec, data_forceAcq, data_fft = npzfile['arr_0'],npzfile['arr_1'],npzfile['arr_2'],npzfile['arr_3'],npzfile['arr_4'],npzfile['arr_5']
# Loop over events
print("Starting event loop ...")
iList = -1
while True:
iList += 1
if intMode==True and iList != 0:
value = input()
if value=='q': break # quit
if value=='p': iList -= 2 # go to previous
if (value.isdigit()):
iList = int(value) # go to entry number
elif intMode==False and batMode==False:
plt.pause(0.00001) # rapid-draw mode
if iList >= nList: break # bail out, goose!
entry = gatTree.GetEntryNumber(iList);
gatTree.LoadTree(entry)
gatTree.GetEntry(entry)
nChans = gatTree.channel.size()
event = MGTEvent()
if gatMode: event = bltTree.event
# Reset all branch vectors
# NOTE: The events sometimes contain 'straggler' hits that do not pass the
# given TCut. This line sets ALL the new parameters to -88888 by default.
# If you see this value in a plot, then you must be including hits that
# passed the cut in wave-skim but did not pass the (different?) cut in LAT.
for key in brDict: brDict[key][0].assign(nChans,-88888)
brDict["fails"][0].assign(nChans,0) # set error code to 'true' by default
errorCode = [0,0,0,0]
# Loop over hits passing cuts
numPass = gatTree.Draw("channel",theCut,"GOFF",1,iList)
chans = gatTree.GetV1()
chanList = list(set(int(chans[n]) for n in range(numPass)))
hitList = (iH for iH in range(nChans) if gatTree.channel.at(iH) in chanList) # a 'generator expression'
for iH in hitList:
# ------------------------------------------------------------------------
# Waveform processing
# load data
run = gatTree.run
chan = gatTree.channel.at(iH)
dataENFCal = gatTree.trapENFCal.at(iH)
dataENM = gatTree.trapENM.at(iH)
dataTSMax = gatTree.trapENMSample.at(iH)*10. - 4000
wf = MGTWaveform()
iEvent = 0
if gatMode:
wf = event.GetWaveform(iH)
iEvent = entry
else:
wf = gatTree.MGTWaveforms.at(iH)
iEvent = gatTree.iEvent
# print("%d: run %d chan %d trapENFCal %.2f" % (iList, run, chan, dataENFCal))
# be absolutely sure you're matching the right waveform to this hit
if wf.GetID() != chan:
print("ERROR -- Vector matching failed. iList %d run %d iEvent %d" % (iList,run,iEvent))
return
# Let's start the show - grab a waveform.
# Remove first 4 samples when we have multisampling
# Remove last 2 samples to get rid of the ADC spike at the end of all wf's.
truncLo, truncHi = 0, 2
if dsNum==6 or dsNum==2: truncLo = 4
signal = wl.processWaveform(wf,truncLo,truncHi)
data = signal.GetWaveRaw()
data_blSub = signal.GetWaveBLSub()
dataTS = signal.GetTS()
dataBL,dataNoise = signal.GetBaseNoise()
# wavelet packet transform
wp = pywt.WaveletPacket(data_blSub, 'db2', 'symmetric', maxlevel=4)
nodes = wp.get_level(4, order='freq')
wpCoeff = np.array([n.data for n in nodes],'d')
wpCoeff = abs(wpCoeff)
# wavelet parameters
# First get length of wavelet on the time axis, the scale axis will always be the same
# due to the number of levels in the wavelet
wpLength = len(wpCoeff[1,:])
waveS1[iH] = np.sum(wpCoeff[0:1,1:wpLength//4+1]) # python3 : floor division (//) returns an int
waveS2[iH] = np.sum(wpCoeff[0:1,wpLength//4+1:wpLength//2+1])
waveS3[iH] = np.sum(wpCoeff[0:1,wpLength//2+1:3*wpLength//4+1])
waveS4[iH] = np.sum(wpCoeff[0:1,3*wpLength//4+1:-1])
waveS5[iH] = np.sum(wpCoeff[2:-1,1:-1])
S6 = np.sum(wpCoeff[2:9,1:wpLength//4+1])
S7 = np.sum(wpCoeff[2:9,wpLength//4+1:wpLength//2+1])
S8 = np.sum(wpCoeff[2:9,wpLength//2+1:3*wpLength//4+1])
S9 = np.sum(wpCoeff[2:9,3*wpLength//4+1:-1])
S10 = np.sum(wpCoeff[9:,1:wpLength//4+1])
S11 = np.sum(wpCoeff[9:,wpLength//4+1:wpLength//2+1])
S12 = np.sum(wpCoeff[9:,wpLength//2+1:3*wpLength//4+1])
S13 = np.sum(wpCoeff[9:,3*wpLength//4+1:-1])
sumList = [S6, S7, S8, S9, S10, S11, S12, S13]
bcMax[iH] = np.max(sumList)
bcMin[iH] = 1. if np.min(sumList) < 1 else np.min(sumList)
# reconstruct waveform w/ only lowest frequency.
new_wp = pywt.WaveletPacket(data=None, wavelet='db2', mode='symmetric')
new_wp['aaa'] = wp['aaa'].data
data_wlDenoised = new_wp.reconstruct(update=False)
# resize in a smart way
diff = len(data_wlDenoised) - len(data_blSub)
if diff > 0: data_wlDenoised = data_wlDenoised[diff:]
# waveform high/lowpass filters - parameters are a little arbitrary
B1,A1 = butter(2, [1e5/(1e8/2),1e6/(1e8/2)], btype='bandpass')
data_bPass = lfilter(B1, A1, data_blSub)
# used in the multisite tagger
B2, A2 = butter(1, 0.08)
data_filt = filtfilt(B2, A2, data_blSub)
data_filtDeriv = wl.wfDerivative(data_filt)
filtAmp = np.amax(data_filtDeriv) # scale the max to match the amplitude
data_filtDeriv = data_filtDeriv * (dataENM / filtAmp)
B3, A3 = butter(2,1e6/(1e8/2), btype='lowpass')
data_lPass = lfilter(B3, A3, data_blSub)
idx = np.where((dataTS > dataTS[0]+100) & (dataTS < dataTS[-1]-100))
windowingOffset = dataTS[idx][0] - dataTS[0]
bandMax[iH] = np.amax(data_bPass[idx])
bandTime[iH] = dataTS[ np.argmax(data_bPass[idx])] - windowingOffset
# timepoints of low-pass waveforms
tpc = MGWFTimePointCalculator();
tpc.AddPoint(.2)
tpc.AddPoint(.5)
tpc.AddPoint(.9)
mgtLowPass = wl.MGTWFFromNpArray(data_lPass)
tpc.FindTimePoints(mgtLowPass)
den10[iH] = tpc.GetFromStartRiseTime(0)*10
den50[iH] = tpc.GetFromStartRiseTime(1)*10
den90[iH] = tpc.GetFromStartRiseTime(2)*10
# ================ xgauss waveform fitting ================
amp, mu, sig, tau, bl = dataENM, dataTSMax, 600., -72000., dataBL
floats = np.asarray([amp, mu, sig, tau, bl])
temp = xgModelWF(dataTS, floats)
if not batMode: MakeTracesGlobal()
# get the noise of the denoised wf
denoisedNoise,_,_ = wl.baselineParameters(data_wlDenoised)
# NOTE: fit is to wavelet-denoised data, BECAUSE there are no HF components in the model,
# AND we'll still calculate fitChi2 w/r/t the data, not the denoised data.
# datas = [dataTS, data, dataNoise] # fit data
datas = [dataTS, data_wlDenoised + dataBL, denoisedNoise] # fit wavelet-denoised data w/ Bl added back in
# Set bounds - A,mu,sig,tau,bl.
# bnd = ((None,None),(None,None),(None,None),(None,None),(None,None)) # often gets caught at sig=0
bnd = ((None,None),(None,None),(2.,None),(-72001.,-71999.),(None,None)) # gets caught much less often.
# L-BGFS-B with numerical gradient.
start = time.clock()
result = op.minimize(lnLike, floats, args=datas, method="L-BFGS-B", options=None, bounds=bnd)
fitSpeed = time.clock() - start
fitErr[iH] = 0
if not result["success"]:
# print("fit fail: ", result["message"])
fitErr[iH] = 1
errorCode[0] = 1
amp, mu, sig, tau, bl = result["x"]
# save parameters
fitMu[iH], fitAmp[iH], fitSlo[iH], fitTau[iH], fitBL[iH] = mu, amp, sig, tau, bl
floats = np.asarray([amp, mu, sig, tau, bl])
fit = xgModelWF(dataTS, floats)
# print("%d/%d iH %d e %-10.2f fs %-8.2f f %d" % (iList, nList, iH, dataENFCal, fitSlo[iH], fitErr[iH]))
# log-likelihood of this fit
fitLL[iH] = result["fun"]
# chi-square of this fit
# Textbook is (observed - expected)^2 / expected,
# but we'll follow MGWFCalculateChiSquare.cc and do (observed - expected)^2 / NDF.
# NOTE: we're doing the chi2 against the DATA, though the FIT is to the DENOISED DATA.
fitChi2[iH] = np.sum(np.square(data-fit)) / (len(data)-1)/dataNoise
# get wavelet coeff's for rising edge only. normalize to bcMin
# view this w/ plot 1
# find the window of rising edge
fit_blSub = fit - bl
fitMaxTime = dataTS[np.argmax(fit_blSub)]
fitStartTime = dataTS[0]
idx = np.where(fit_blSub < 0.1)
if len(dataTS[idx] > 0): fitStartTime = dataTS[idx][-1]
fitRiseTime50 = (fitMaxTime + fitStartTime)/2.
# bcMin is 32 samples long in the x-direction.
# if we make the window half as wide, it'll have the same # of coeff's as bcMin.
# this is still 'cheating' since we're not summing over the same rows.
numXRows = wpCoeff.shape[1]
wpCtrRise = int((fitRiseTime50 - dataTS[0]) / (dataTS[-1] - dataTS[0]) * numXRows)
wpLoRise = wpCtrRise - 8
if wpLoRise < 0: wpLoRise = 0
wpHiRise = wpCtrRise + 8
if wpHiRise > numXRows: wpHiRise = numXRows
# sum all HF wavelet components for this edge.
riseNoise[iH] = np.sum(wpCoeff[2:-1,wpLoRise:wpHiRise]) / bcMin[iH]
# print("%d %d %d %d e %-5.2f bmax %-6.2f bmin %-6.2f mu %-5.2f a %-5.2f s %-5.2f bl %-5.2f rn %.2f" % (run,iList,iH,chan,dataENFCal,bcMax[iH],bcMin[iH],fitMu[iH],fitAmp[iH],fitSlo[iH],fitBL[iH],riseNoise[iH]))
# =========================================================
# optimal matched filter (freq. domain)
# we use the pysiggen fast template (not the fit result) to keep this independent of the wf fitter.
# pull in the template, shift it, and make sure it's the same length as the data
guessTS = tOrigTS - 15000.
idx = np.where((guessTS > -5) & (guessTS < dataTS[-1]))
guessTS, guess = guessTS[idx], tOrig[idx]
if len(guess)!=len(data):
if len(guess)>len(data):
guess, guessTS = guess[0:len(data)], guessTS[0:len(data)]
else:
guess = np.pad(guess, (0,len(data)-len(guess)), 'edge')
guessTS = np.pad(guessTS, (0,len(data)-len(guessTS)), 'edge')
data_fft = np.fft.fft(data_blSub) # can also try taking fft of the low-pass data
temp_fft = np.fft.fft(guess)
datafreq = np.fft.fftfreq(data.size) * 1e8
power_vec = np.interp(datafreq, noise_xFreq, noise_asd) # load power spectra from file
# Apply the filter
optimal = data_fft * temp_fft.conjugate() / power_vec
optimal_time = 2 * np.fft.ifft(optimal)
# Normalize the output
df = np.abs(datafreq[1] - datafreq[0]) # freq. bin size
sigmasq = 2 * (temp_fft * temp_fft.conjugate() / power_vec).sum() * df
sigma = np.sqrt(np.abs(sigmasq))
SNR = abs(optimal_time) / (sigma)
oppie[iH] = np.amax(SNR)
# time-domain matched filter. use the baseline-subtracted wf as data, and fit_blSub too.
# make a longer best-fit waveform s/t it can be shifted L/R.
matchTS = np.append(dataTS, np.arange(dataTS[-1], dataTS[-1] + 20000, 10)) # add 2000 samples
match = xgModelWF(matchTS, [amp, mu+10000., sig, tau, bl]) # shift mu accordingly
match = match[::-1] - bl # time flip and subtract off bl
# line up the max of the 'match' (flipped wf) with the max of the best-fit wf
# this kills the 1-1 matching between matchTS and dataTS (each TS has some offset)
matchMaxTime = matchTS[np.argmax(match)]
matchTS = matchTS + (fitMaxTime - matchMaxTime)
# resize match, matchTS to have same # samples as data, dataTS.
# this is the only case we really care about
# ("too early" and "too late" also happen, but the shift is larger than the trigger walk, making it unphysical)
if matchTS[0] <= dataTS[0] and matchTS[-1] >= dataTS[-1]:
idx = np.where((matchTS >= dataTS[0]) & (matchTS <= dataTS[-1]))
match, matchTS = match[idx], matchTS[idx]
sizeDiff = len(dataTS)-len(matchTS)
if sizeDiff < 0:
match, matchTS = match[:sizeDiff], matchTS[:sizeDiff]
elif sizeDiff > 0:
match = np.hstack((match, np.zeros(sizeDiff)))
matchTS = np.hstack((matchTS, dataTS[-1*sizeDiff:]))
if len(match) != len(data):
print("FIXME: match filter array manip is still broken.")
# compute match filter parameters
matchMax[iH], matchWidth[iH], matchTime[iH] = -888, -888, -888
if len(match)==len(data):
smoothMF = gaussian_filter(match * data_blSub, sigma=5.)
matchMax[iH] = np.amax(smoothMF)
matchTime[iH] = matchTS[ np.argmax(smoothMF) ]
idx = np.where(smoothMF > matchMax[iH]/2.)
if len(matchTS[idx]>1):
matchWidth[iH] = matchTS[idx][-1] - matchTS[idx][0]
# Fit tail slope to polynomial. Guard against fit fails
idx = np.where(dataTS >= fitMaxTime)
tail, tailTS = data[idx], dataTS[idx]
popt1,popt2 = 0,0
try:
popt1,_ = op.curve_fit(wl.tailModelPol, tailTS, tail)
pol0[iH], pol1[iH], pol2[iH], pol3[iH] = popt1[0], popt1[1], popt1[2], popt1[3]
except:
# print("curve_fit tailModelPol failed, run %i event %i channel %i" % (run, iList, chan))
errorCode[2] = 1
pass
# =========================================================
# new trap filters.
# params: t0_SLE, t0_ALE, lat, latF, latAF, latFC, latAFC
# calculate trapezoids
# standard trapezoid - prone to walking, less sensitive to noise. use to find energy
eTrap = wl.trapFilter(data_blSub, 400, 250, 7200.)
eTrapTS = np.arange(0, len(eTrap)*10., 10)
eTrapInterp = interpolate.interp1d(eTrapTS, eTrap)
# short trapezoid - triggers more quickly, sensitive to noise. use to find t0
sTrap = wl.trapFilter(data_blSub, 100, 150, 7200.)
sTrapTS = np.arange(0, len(sTrap)*10., 10)
# asymmetric trapezoid - used to find the t0 only
aTrap = wl.asymTrapFilter(data_blSub, 4, 10, 200, True) # (0.04us, 0.1us, 2.0us)
aTrapTS = np.arange(0, len(aTrap)*10., 10)
# find leading edges (t0 times)
# limit the range from 0 to 10us, and use an ADC threshold of 1.0 as suggested by DCR
t0_SLE[iH],_ = wl.walkBackT0(sTrap, eTrapTS[-1]+7000-4000-2000, 1., 0, 1000) # (in ns) finds leading edge from short trap
t0_ALE[iH],_ = wl.walkBackT0(aTrap, eTrapTS[-1]+7000-4000-2000, 1., 0, 1000) # (in ns) finds leading edge from asymmetric trap
# standard energy trapezoid w/ a baseline padded waveform
data_pad = np.pad(data_blSub,(200,0),'symmetric')
pTrap = wl.trapFilter(data_pad, 400, 250, 7200.)
pTrapTS = np.linspace(0, len(pTrap)*10, len(pTrap))
pTrapInterp = interpolate.interp1d(pTrapTS, pTrap)
# calculate energy parameters
# standard amplitude. basically trapEM, but w/o NL correction if the input WF doesn't have it.
lat[iH] = np.amax(eTrap)
# Calculate DCR suggested amplitude, using the 50% to the left and right of the maximum point
t0_F50,t0fail1 = wl.walkBackT0(pTrap, thresh=lat[iH]*0.5, rmin=0, rmax=len(pTrap)-1)
t0_B50,t0fail2 = wl.walkBackT0(pTrap, thresh=lat[iH]*0.5, rmin=0, rmax=len(pTrap)-1, forward=True)
t0_E50 = (t0_F50 + t0_B50)/2.0
#TODO -- if it's necessary due to the trigger walk, we could potentially add a way to recursively increase the threshold until a timepoint is found, however it will still always fail for most noise events
if not t0fail1 or not t0fail2:
latE50[iH] = 0 # Set amplitude to 0 if one of the evaluations failed
else:
latE50[iH] = pTrapInterp(t0_E50) # Maybe I should call this latDCR50 to confuse people
tE50[iH] = t0_B50 - t0_F50 # Save the difference between the middle points, can be used as a cut later
# standard amplitude with t0 from the shorter traps
# If either fixed pickoff time (t0) is < 0, use the first sample as the amplitude (energy).
latF[iH] = eTrapInterp( np.amax([t0_SLE[iH]-7000+4000+2000, 0.]) ) # This should be ~trapEF
latAF[iH] = eTrapInterp( np.amax([t0_ALE[iH]-7000+4000+2000, 0.]) )
# amplitude from padded trapezoid, with t0 from short traps and a correction function
# function is under development. currently: f() = exp(p0 + p1*E), p0 ~ 7.8, p1 ~ -0.45 and -0.66
# functional walk back distance is *either* the minimum of the function value, or 5500 (standard value)
# t0_corr = -7000+6000+2000 # no correction
t0_corr = -7000+6000+2000 - np.amin([np.exp(7.8 - 0.45*lat[iH]),1000.])
t0A_corr = -7000+6000+2000 - np.amin([np.exp(7.8 - 0.66*lat[iH]),1000.])
latFC[iH] = pTrapInterp( np.amax([t0_SLE[iH] + t0_corr, 0.]) )
latAFC[iH] = pTrapInterp( np.amax([t0_ALE[iH] + t0A_corr, 0.]) )
# =========================================================
# the genius multisite event tagger - plot 8
# decide a threshold
dIdx = np.argmax(data_filtDeriv)
dMax = data_filtDeriv[dIdx]
dRMS,_,_ = wl.baselineParameters(data_filtDeriv)
# msThresh = np.amax([dMax * .2, dRMS * 5.])
# msThresh = dMax * .15
msThresh = 50. # I don't know. this seems like a good value
# run peak detect algorithm
maxtab,_ = wl.peakdet(data_filtDeriv, msThresh)
# profit
msList = []
for iMax in range(len(maxtab)):
idx = int(maxtab[iMax][0])
val = maxtab[iMax][1]
msList.append(dataTS[idx])
# print("%d idx %d TS %d val %.2f thresh %.2f" % (iList, idx, dataTS[idx], val, msThresh))
nMS[iH] = len(maxtab)
# =========================================================
# wfStd analysis
wfAvgBL[iH] = dataBL
wfRMSBL[iH] = dataNoise
wfStd[iH] = np.std(data[5:-5])
# ------------------------------------------------------------------------
# End waveform processing.
# Calculate error code
fails[iH] = 0
for i,j in enumerate(errorCode):
if j==1: fails[iH] += int(j)<<i
# print("fails:",fails[iH])
# Make plots!
if batMode: continue
if plotNum==0: # raw data
p0.cla()
p0.plot(dataTS,data,'b')
p0.set_title("Run %d Entry %d Channel %d ENFCal %.2f" % (run,iList,chan,dataENFCal))
p0.set_xlabel("Time (ns)", ha='right', x=1.)
p0.set_ylabel("Voltage (ADC)", ha='right', y=1.)
if plotNum==1: # wavelet plot
p0.cla()
p0.margins(x=0)
p0.plot(dataTS,data_blSub,color='blue',label='data (%.2f keV)' % dataENFCal)
p0.plot(dataTS,data_wlDenoised,color='cyan',label='denoised',alpha=0.7)
p0.axvline(fitRiseTime50,color='green',label='fit 50%',linewidth=2)
p0.plot(dataTS,fit_blSub,color='red',label='bestfit',linewidth=2)
# p0.set_title("Run %d Entry %d Channel %d ENFCal %.2f flo %.0f fhi %.0f fhi-flo %.0f" % (run,iList,chan,dataENFCal,fitStartTime,fitMaxTime,fitMaxTime-fitStartTime))
p0.legend(loc='best')
p0.set_xlabel("Time (ns)", ha='right', x=1.)
p0.set_ylabel("Voltage (ADC)", ha='right', y=1.)
p1.cla()
p1.imshow(wpCoeff, interpolation='nearest', aspect="auto", origin="lower",extent=[0, 1, 0, len(wpCoeff)],cmap='viridis')
p1.axvline(float(wpLoRise)/numXRows,color='orange',linewidth=2)
p1.axvline(float(wpHiRise)/numXRows,color='orange',linewidth=2)
# p1.set_title("waveS5 %.2f bcMax %.2f bcMin %.2f riseNoise %.2f" % (waveS5[iH], bcMax[iH], bcMin[iH], riseNoise[iH]))
# p1.set_xlabel("Time (%wf)", ha='right', x=1.)
p1.set_ylabel("WPT Coefficients", ha='right', y=1.)
if plotNum==2: # time points, bandpass filters, tail slope
p0.cla()
p0.plot(dataTS,data,color='blue',label='data')
p0.axvline(den10[iH],color='black',label='lpTP')
p0.axvline(den50[iH],color='black')
p0.axvline(den90[iH],color='black')
p0.plot(dataTS,fit,color='magenta',label='bestfit')
if errorCode[2]!=1: p0.plot(tailTS, wl.tailModelPol(tailTS, *popt1), color='orange',linewidth=2, label='tailPol')
p0.legend(loc='best')
p0.set_title("Run %d Entry %d Channel %d ENFCal %.2f" % (run,iEvent,chan,dataENFCal))
p1.cla()
p1.plot(dataTS,data_lPass,color='blue',label='lowpass')
p1.plot(dataTS,data_filtDeriv,color='green',label='filtDeriv')
p1.plot(dataTS,data_filt,color='black',label='filtfilt')
p1.plot(dataTS,data_bPass,color='red',label='bpass')
p1.axvline(bandTime[iH],color='orange',label='bandTime')
p1.legend(loc='best')
if plotNum==3: # freq-domain matched filter
p0.cla()
p0.plot(dataTS,data,'b')
p0.plot(dataTS,temp,'r')
p0.plot(dataTS,fit,color='cyan')
p0.set_title("Run %d Entry %d Channel %d ENFCal %.2f" % (run,iEvent,chan,dataENFCal))
data_asd, data_xFreq = plt.psd(data, Fs=1e8, NFFT=2048, pad_to=2048, visible=False)
temp_asd, temp_xFreq = plt.psd(temp, Fs=1e8, NFFT=2048, pad_to=2048, visible=False)
p1.cla()
p1.loglog(data_xFreq, np.sqrt(data_asd), 'b')
p1.loglog(noise_xFreq, np.sqrt(noise_asd), 'g')
p1.loglog(temp_xFreq, np.sqrt(temp_asd), 'r')
p1.set_xlabel('Frequency (Hz)')
p1.set_ylabel('ASD')
p1.grid('on')
p2.cla()
p2.plot(dataTS, SNR)
p2.set_title('oppie %.1f' % (oppie[iH]))
p2.set_xlabel('Offset time (s)')
p2.set_ylabel('SNR')
if plotNum==4: # time domain match filter plot
p0.cla()
p0.plot(dataTS,data_blSub,color='blue',label='data',alpha=0.7)
p0.plot(dataTS,fit_blSub,color='red',label='bestfit',linewidth=3)
p0.axvline(matchTime[iH],color='orange',label='matchTime',linewidth=2)
p0.plot(matchTS,smoothMF,color='magenta',label='smoothMF',linewidth=3)
p0.plot(matchTS,match,color='cyan',label='match',linewidth=3)
p0.set_xlabel('Time (s)')
p0.set_ylabel('Voltage (arb)')
p0.legend(loc='best')
p0.set_title("Run %d Entry %d Channel %d ENFCal %.2f matchMax %.2f matchTime %.2f matchWidth %.2f" % (run,iEvent,chan,dataENFCal,matchMax[iH],matchTime[iH],matchWidth[iH]))
if plotNum==5: # bandTime plot
p0.cla()
p0.plot(dataTS,data_blSub,color='blue',label='data',alpha=0.7)
p0.plot(dataTS,data_lPass,color='magenta',label='lowpass',linewidth=4)
p0.plot(dataTS,data_bPass,color='red',label='bpass',linewidth=4)
p0.axvline(bandTime[iH],color='orange',label='bandTime',linewidth=4)
p0.legend(loc='best')
p0.set_xlabel('Time (ns)')
p0.set_ylabel('ADC (arb)')
p0.set_title("Run %d Entry %d Channel %d ENFCal %.2f" % (run,iEvent,chan,dataENFCal))
if plotNum==6: # waveform fit plot
p0.cla()
p0.plot(dataTS,data,color='blue',label='data')
# p0.plot(dataTS,data_wlDenoised,color='cyan',label='wlDenoised',alpha=0.5)
p0.plot(dataTS,temp,color='orange',label='xgauss guess')
p0.plot(dataTS,fit,color='red',label='xgauss fit')
p0.set_title("Run %d evt %d chan %d trapENFCal %.1f trapENM %.1f deltaBL %.1f\n amp %.2f mu %.2f sig %.2f tau %.2f chi2 %.2f spd %.3f" % (run,iList,chan,dataENFCal,dataENM,dataBL-bl,amp,mu,sig,tau,fitChi2[iH],fitSpeed))
p0.legend(loc='best')
p1.cla()
p1.plot(dataTS,data-fit,color='blue',label='residual')
p1.legend(loc='best')
p2.cla()
p2.plot(ampTr[1:],label='amp',color='red')
p2.legend(loc='best')
p3.cla()
p3.plot(muTr[1:],label='mu',color='green')
p3.legend(loc='best')
p4.cla()
p4.plot(sigTr[1:],label='sig',color='blue')
p4.yaxis.set_major_formatter(mtick.FormatStrFormatter('%.1e'))
p4.legend(loc='best')
p5.cla()
p5.plot(tauTr[1:],label='tau',color='black')
p5.legend(loc='best')
p6.cla()
p6.plot(blTr[1:],label='bl',color='magenta')
p6.legend(loc='best')
print(gatTree.fitSlo.at(iH), sig)
if plotNum==7: # new traps plot
p0.cla()
p0.plot(dataTS, data_blSub, color='blue', label='data')
p0.plot(sTrapTS, sTrap, color='red', label='sTrap')
p0.axvline(t0_SLE[iH], color='red')
p0.plot(aTrapTS, aTrap, color='orange', label='aTrap')
p0.axvline(t0_ALE[iH], color='orange')
p0.plot(eTrapTS, eTrap, color='green', label='eTrap')
p0.axhline(lat[iH],color='green')
p0.plot(pTrapTS, pTrap, color='magenta', label='pTrap')
p0.axhline(latAFC[iH], color='magenta')
p0.axhline(latE50[iH], color='cyan')
p0.set_title("trapENFCal %.2f trapENM %.2f || latEM %.2f latEF %.2f latEAF %.2f latEFC %.2f latEAFC %.2f latE50 %.2f" % (dataENFCal,dataENM,lat[iH],latF[iH],latAF[iH],latFC[iH],latAFC[iH], latE50[iH]))
p0.legend(loc='best')
if plotNum==8: # multisite tag plot
p0.cla()
p0.plot(dataTS, data_blSub, color='blue', label='data')
p0.plot(dataTS, data_filtDeriv, color='red', label='filtDeriv')
for mse in msList: p0.axvline(mse, color='green')
p0.axhline(msThresh,color='red')
p0.legend()
if plotNum==9: # wavelet vs wf fit residual plot
# wavelet packet transform on wf fit residual
fitResid = data-fit
wpRes = pywt.WaveletPacket(fitResid, 'db2', 'symmetric', maxlevel=4)
nodesRes = wpRes.get_level(4, order='freq')
wpCoeffRes = np.array([n.data for n in nodesRes], 'd')
wpCoeffRes = abs(wpCoeffRes)
R6 = np.sum(wpCoeffRes[2:9,1:wpLength//4+1])
R7 = np.sum(wpCoeffRes[2:9,wpLength//4+1:wpLength//2+1])
R8 = np.sum(wpCoeffRes[2:9,wpLength//2+1:3*wpLength//4+1])
R9 = np.sum(wpCoeffRes[2:9,3*wpLength//4+1:-1])
R10 = np.sum(wpCoeffRes[9:,1:wpLength//4+1])
R11 = np.sum(wpCoeffRes[9:,wpLength//4+1:wpLength//2+1])
R12 = np.sum(wpCoeffRes[9:,wpLength//2+1:3*wpLength//4+1])
R13 = np.sum(wpCoeffRes[9:,3*wpLength//4+1:-1])
RsumList = [R6, R7, R8, R9, R10, R11, R12, R13]
bcMinRes = 1. if np.min(RsumList) < 1 else np.min(RsumList)
riseNoiseRes = np.sum(wpCoeffRes[2:-1,wpLoRise:wpHiRise]) / bcMinRes
rnCut = 1.1762 + 0.00116 * np.log(1 + np.exp((dataENFCal-7.312)/0.341))
p0.cla()
p0.margins(x=0)
p0.plot(dataTS,data_blSub,color='blue',label='data')
# p0.plot(dataTS,data_wlDenoised,color='cyan',label='denoised',alpha=0.7)
# p0.axvline(fitRiseTime50,color='green',label='fit 50%',linewidth=2)
p0.plot(dataTS,fit_blSub,color='red',label='bestfit',linewidth=2)
# p0.set_title("Run %d Entry %d Channel %d ENFCal %.2f flo %.0f fhi %.0f fhi-flo %.0f" % (run,iList,chan,dataENFCal,fitStartTime,fitMaxTime,fitMaxTime-fitStartTime))
# p0.legend(loc='best')
p0.set_title("Run %d Entry %d Channel %d ENFCal %.2f flo %.0f fhi %.0f fhi-flo %.0f approxFitE %.2f" % (run,iList,chan,dataENFCal,fitStartTime,fitMaxTime,fitMaxTime-fitStartTime,fitAmp[iH]*0.4))
p1.cla()
p1.plot(dataTS,fitResid,color='blue')
p2.cla()
p2.set_title("riseNoise %.2f rnCut %.2f riseNoiseRes %.2f bcMinRes %.2f bcMin %.2f max %.2f" % (riseNoise[iH],rnCut,riseNoiseRes,bcMinRes,bcMin[iH],wpCoeffRes.max()))
p2.imshow(wpCoeffRes, interpolation='nearest', aspect="auto", origin="lower",extent=[0, 1, 0, len(wpCoeff)],cmap='viridis')
plt.tight_layout()
plt.pause(0.000001)
# ------------------------------------------------------------------------
# End loop over hits, fill branches
if batMode:
for key in brDict:
brDict[key][1].Fill()
if iList%5000 == 0 and iList!=0:
out.Write("",TObject.kOverwrite)
print("%d / %d entries saved (%.2f %% done), time: %s" % (iList,nList,100*(float(iList)/nList),time.strftime('%X %x %Z')))
# End loop over events
if batMode and not intMode:
out.Write("",TObject.kOverwrite)
print("Wrote",out.GetBranch("channel").GetEntries(),"entries in the copied tree,")
print("and wrote",b1.GetEntries(),"entries in the new branches.")
stopT = time.clock()
print("Stopped:",time.strftime('%X %x %Z'),"\nProcess time (min):",(stopT - startT)/60)
print(float(nList)/((stopT-startT)/60.),"entries per minute.")
def main(argv):
""" Get a sweet minimizer + time domain match filter working. Also graduate. """
scanSpeed = 1.
iList = -1
opt1 = ""
intMode = False
if (len(argv) >= 1): opt1 = argv[0]
if "-i" in (opt1): intMode = True
# Set input data and cuts
inputFile = TFile("~/project/match-skim/waveletSkimDS5_run23920.root") # calibration
# inputFile = TFile("~/project/v2-processwfs/waveletSkimDS5_90.root") # noisy BG
waveTree = inputFile.Get("skimTree")
theCut = inputFile.Get("cutUsedHere").GetTitle()
# theCut += " && waveS5/trapENFCal < 1200 && trapENFCal < 10"
theCut += " && trapENFCal < 30"
# Print cut and events passing cut
waveTree.Draw(">>elist", theCut, "entrylist")
elist = gDirectory.Get("elist")
waveTree.SetEntryList(elist)
nList = elist.GetN()
print "Using cut:\n",theCut,"\n"
print "Found",waveTree.GetEntries(),"input entries."
print "Found",nList,"entries passing cuts."
# Make a figure
fig = plt.figure(figsize=(15,10), facecolor='w')
p1 = plt.subplot2grid((6,7), (0,0), colspan=4, rowspan=3) # data & fit
p2 = plt.subplot2grid((6,7), (3,0), colspan=4, rowspan=3) # matched filter
p3 = plt.subplot2grid((6,7), (0,4), colspan=3, rowspan=2) # trace 1
p4 = plt.subplot2grid((6,7), (2,4), colspan=3, rowspan=2) # trace 2
p5 = plt.subplot2grid((6,7), (4,4), colspan=3, rowspan=2) # trace 3
plt.show(block=False)
# Make template(s)
tSamp, tR, tZ, tAmp, tST, tSlo = 5000, 0, 15, 100, 2500, 10
tOrig, tOrigTS = wl.MakeSiggenWaveform(tSamp,tR,tZ,tAmp,tST,tSlo)
# Loop over events
while True:
saveMe = False
iList += 1
if intMode==True and iList != 0:
value = raw_input()
if value=='q': break
if value=='s': saveMe=True
if value=='p': iList -= 2 # previous
if (value.isdigit()): iList = int(value) # go to entry
if iList >= elist.GetN(): break
entry = waveTree.GetEntryNumber(iList);
waveTree.LoadTree(entry)
waveTree.GetEntry(entry)
nChans = waveTree.channel.size()
numPass = waveTree.Draw("channel",theCut,"GOFF",1,iList)
chans = waveTree.GetV1()
chanList = list(set(int(chans[n]) for n in xrange(numPass)))
# Loop over hits passing cuts
hitList = (iH for iH in xrange(nChans) if waveTree.channel.at(iH) in chanList) # a 'generator expression'
for iH in hitList:
# ------------------------------------------------------------------------
# Load data
run = waveTree.run
chan = waveTree.channel.at(iH)
dataE = waveTree.trapENFCal.at(iH)
dataENM = waveTree.trapENM.at(iH)
# dataMax = waveTree.trapENMSample.at(iH)*10. - 4000
dataMax = waveTree.rt90.at(iH)
signal = wl.processWaveform(waveTree.MGTWaveforms.at(iH))
data = signal.GetWaveBLSub()
dataTS = signal.GetTS()
dataBL, dataNoise = signal.GetBaseNoise()
print "%d / %d Run %d nCh %d chan %d trapENF %.1f len %d" % (iList,nList,run,nChans,chan,dataE,len(data))
# Load and scale template
temp, tempTS = tOrig, tOrigTS
temp = temp * (dataENM / tAmp) # scale by amplitudes (not energies)
tempMax = np.argmax(temp) * 10 # convert to ns
tempTS = tempTS - (tempMax - dataMax) # align @ max of rising edge
tempMax = tempTS[np.argmax(temp)] # get the new max TS after the shifting
tempE = dataE # set 'calibrated' energy of template equal to data's trapENFCal.
# Lowpass or bandpass filter the data
# NOTE: This is tough. There might be a feature due to rising edges, between 3e6 and 5e6 Hz.
# But that's only from data. Doing it w/ siggen templates yielded squat.
# B,A = sig.butter(2, [3e6/(1e8/2),5e6/(1e8/2)], btype='bandpass')
B,A = sig.butter(2,1e6/(1e8/2),btype='lowpass') # nice lowpass filter
data_LowPass = sig.lfilter(B, A, data)
B1,A1 = sig.butter(2, [1e5/(1e8/2),1e6/(1e8/2)], btype='bandpass') # attempt at bandpass
data_BanPass = sig.lfilter(B1, A1, data)
# -- Run minimizers
# Set window and parameters
loWin, hiWin = dataTS[0], dataTS[-1]
mt, en, slo = dataMax-5, dataE+1, 20. # guesses
# print "guesses: mt %.0f en %.3f slo %.2f" % (mt, en, slo)
# pack into lists
InterpFn = interpolate.interp1d(tempTS, temp, kind="linear", copy="False", assume_sorted="True")
dataList = [data, dataTS, dataE, dataMax, loWin, hiWin, dataNoise]
tempList = [temp, tempTS, tempE, tempMax]
datas = [dataList, tempList, InterpFn]
floats = [mt,en,slo]
guess, guessTS = wm.MakeModel(dataList, tempList, floats, fn=InterpFn)
start = time.clock()
MakeTracesGlobal()
fitFail = False # you should output this to ROOT
# 0. brute - trying to make a quick initial guess for the slowness parameter
# FIXME: this keeps returning the lowest allowed value for slowness.
# bruteArgs = [en, datas]
# ranges = (slice(dataMax-500,dataMax+300,100), slice(1,100,10))
# resbrute = op.brute(bruteLnLike, ranges, args=(bruteArgs,), full_output=False, finish=None, disp=False)
# mt, slo = resbrute[0], resbrute[1]
# print "brute results: mt %.0f en %.3f slo %.2f" % (mt, en, slo)
# floats = [mt, en, slo]
# 1. nelder-mead (~0.5 sec). nobody does it better. works almost like magic.
result = op.minimize(findLnLike, floats, args=datas, method="Nelder-Mead")#,options={"maxiter":10})
if not result["success"]:
print "fit 'fail': ", result["message"]
fitFail = True
mt, en, slo = result["x"]
# print "nelder results: mt %.0f en %.3f slo %.2f" % (mt, en, slo)
nelder, nelderTS = wm.MakeModel(dataList, tempList, [mt,en,slo], fn=InterpFn)
nelderStop = time.clock()
# 2. powell "polish step" (~0.1 sec). Sometimes screws up Nelder's good results!
# floats = [mt,en,slo]
# powell = op.minimize(findLnLike, floats, args=datas, method="Powell")
# if not powell["success"]:
# fitFail= True
# print powell["message"]
# mt, en, slo = powell["x"]
# # print "powell results: mt %.0f en %.3f slo %.2f" % (mt, en, slo)
# model, modelTS = wm.MakeModel(dataList, tempList, [mt,en,slo], fn=InterpFn)
#
# powellStop = time.clock()
# print "Time: nelder %.3f powell %.3f total %.3f" % (nelderStop-start, powellStop-nelderStop, powellStop-start)
# take absolute values for parameters
mt, en, slo = abs(mt), abs(en), abs(slo)
model, modelTS = wm.MakeModel(dataList, tempList, [mt,en,slo], fn=InterpFn)
# compute a trap energy off the model (need pinghan's calibration constants)
trap = np.zeros(1000)
trap = np.append(trap,wl.trapezoidalFilter(model))
trapTS = modelTS
trapMax = np.amax(trap)
# -- Time domain match filter
match, matchTS = wm.MakeModel(dataList, tempList, [mt,en,slo], opt="nowindow")
match = np.flip(match,0)
# line up the max of the match with the max of the best-fit signal
modelMaxTime = modelTS[np.argmax(model)]
matchMaxTime = matchTS[np.argmax(match)]
matchTS = matchTS + (modelMaxTime - matchMaxTime)
idx = np.where((matchTS >= dataTS[0]-5) & (matchTS <= dataTS[-1]+5))
match, matchTS = match[idx], matchTS[idx]
# make sure match is same length as data, then compute convolution and parameters
matchMax, matchWidth, matchArea = 0, 9999, 0
smoothMF = np.zeros(len(dataTS))
if len(match)!=len(data):
print "array mismatch: len match %i len data %i " % (len(match),len(data))
else:
# I can't decide which of these is better.
# smoothMF = gaussian_filter(match * data, sigma=float(5))
smoothMF = gaussian_filter(match * data_LowPass,sigma=float(5))
# computer AWESOME match filter parameters
matchMax = np.amax(smoothMF)
idx = np.where(smoothMF > matchMax/2.)
matchWidth = matchTS[idx][-1] - matchTS[idx][0]
matchArea = np.sum(match[idx]) / float(len(match[idx]))
# Plot data and template
p1.cla()
p1.plot(dataTS,data,'b',label='data')
idx2 = np.where((tempTS >= dataTS[0]-5) & (tempTS <= dataTS[-1]+5))
p1.plot(tempTS[idx2],temp[idx2],'r',label='template')
# p1.plot(dataTS, data_LowPass, label='lowpass')
# p1.plot(dataTS, data_BanPass, label='bandpass')
p1.plot(trapTS,trap,color='k',label='bestfit-trap')
p1.plot(nelderTS,nelder,color='cyan',label='bestfit')
# p1.plot(modelTS,model,color='magenta',label='powells')
p1.set_xlabel('Time (s)')
p1.set_ylabel('Voltage (arb)')
p1.set_title("Run %i Ch %i E %.2f ENM %.2f trapMax %.2f" % (run,chan,dataE,dataENM,trapMax))
p1.legend(loc=4)
p2.cla()
p2.plot(dataTS,data,'b',alpha=0.8,label='data')
# p2.plot(guessTS,guess,'y',alpha=0.8,label='guess')
p2.plot(matchTS,match,'k',label='match')
p2.plot(modelTS,model,color='orange',label='model')
p2.plot(dataTS,data_LowPass,'r',label='lowpass')
if len(match)==len(data): p2.plot(matchTS,smoothMF,'g',label='smoothMF')
p2.axvline(matchTS[idx][-1],color='green',alpha=0.7)
p2.axvline(matchTS[idx][0],color='green',alpha=0.7)
p2.set_title("mMax %.2f mWidth %.2f mAreaNorm %.2f mM/mW %.4f" % (matchMax,matchWidth,matchArea,matchMax/matchWidth))
p2.legend(loc=4)
# plot traces
p3.cla()
p3.set_title("maxTime %.1f Energy %.2f Slow %.1f" % (mt,en,slo))
p3.plot(mtTrace[1:])
p3.set_ylabel('maxTime')
p4.cla()
p4.plot(enTrace[1:])
p4.set_ylabel('energy')
p5.cla()
p5.plot(sloTrace[1:])
p5.set_ylabel('slowness')
plt.tight_layout()
plt.pause(scanSpeed)
def main(argv):
scanSpeed = 0.0001
opt1, opt2 = "", ""
intMode, batMode = False, False
if (len(argv) >= 1): opt1 = argv[0]
if (len(argv) >= 2): opt2 = argv[1]
if "-i" in (opt1, opt2):
intMode = True
print "Interactive mode selected."
if "-b" in (opt1,opt2):
batMode = True
print "Batch mode selected."
# Input: forced acquisition runs
gatFile = TFile("~/project/mjddatadir/gatified/mjd_run13071.root")
bltFile = TFile("~/project/mjddatadir/built/OR_run13071.root")
waveTree = gatFile.Get("mjdTree")
bltTree = bltFile.Get("MGTree")
waveTree.AddFriend(bltTree)
print "Found",waveTree.GetEntries(),"input entries."
# Set cuts
theCut = "rawWFMax < 200 && rawWFMin > -20"
theCut += " && Entry$ < 10000"
waveTree.Draw(">>elist", theCut, "entrylist")
elist = gDirectory.Get("elist")
waveTree.SetEntryList(elist)
nList = elist.GetN()
print "Using cut:\n",theCut,"\n"
print "Found",nList,"entries passing cuts."
# Loop figure (diagnostic)
fig1 = plt.figure(figsize=(8,5), facecolor='w')
p1 = plt.subplot(211)
p2 = plt.subplot(212)
if not batMode: plt.show(block=False)
# Quantities to write to file
avgPsd, xPsd = 0, 0 # pyplot.psd
avgPwrSpec, xPwrSpec = 0, 0 # wl.fourierTransform
data_forceAcq = 0 # single baseline
data_fft = 0 # fft of single baseline
nWF = 0
ents = elist.GetN()
# Loop over events
iList = -1
while(True):
iList += 1
if intMode==True and iList !=0:
value = raw_input()
if value=='q': break
if value=='p': iList -= 2 # previous
if (value.isdigit()): iList = int(value) # go to entry
if iList >= ents: break
# print progress
if (np.fabs((float(iList)/ents)%0.1) < 0.001):
print "%i done" % (100*iList/ents), np.fabs(float(iList)/ents)%0.1
entry = waveTree.GetEntryNumber(iList)
waveTree.LoadTree(entry)
waveTree.GetEntry(entry)
nChans = waveTree.channel.size()
numPass = waveTree.Draw("channel",theCut,"GOFF",1,iList)
chans = waveTree.GetV1()
chanList = list(set(int(chans[n]) for n in xrange(numPass)))
event = bltTree.event # grab waveforms from built data
# Loop over hits passing cuts
hitList = (iH for iH in xrange(nChans) if waveTree.channel.at(iH) in chanList) # a 'generator expression'
for iH in hitList:
# Get signal
signal = wl.processWaveform(event.GetWaveform(iH)) # use when we have gat+blt data
data = signal.GetWaveBLSub()
dataTS = signal.GetTS() * 10.
# Get power spectrum
xPwrSpec,_,ypwr = wl.fourierTransform(data)
avgPwrSpec += ypwr
nWF += 1
# Get full FFT (real, cplx)
data_fft = np.fft.fft(data)
data_forceAcq = data
# power spectral density
psd, xPsd = plt.psd(data, Fs=1e8, NFFT=2048, pad_to=2048, visible=True)
# avgPsd += np.sqrt(psd)
avgPsd += psd
# Fill the figure
if not batMode:
p1.cla()
p1.plot(dataTS,data,color='blue')
p2.cla()
plt.yscale('log')
# p2.plot(xPwrSpec,avgPwrSpec,color='blue')
p2.loglog(xPsd,avgPsd,color='blue')
plt.pause(scanSpeed)
# Plot average power spectrum
fig2 = plt.figure(figsize=(8,5), facecolor='w')
p3 = plt.subplot(111)
plt.yscale('log')
# plt.xscale('log')
p3.set_xlabel("frequency")
avgPwrSpec /= nWF
idx = np.where(xPwrSpec > 10000) # low frequency cutoff
p3.plot(xPwrSpec[idx],avgPwrSpec[idx],color='blue')
fig2.savefig("./plots/powerSpectrum_run13071_linscale.pdf")
# Get noise power density
print "noise power density N_0: ", np.sum(avgPwrSpec[idx])
# Plot power spectral density
avgPsd /= nWF
p3.cla()
p3.loglog(xPsd,avgPsd,color='blue')
fig2.savefig("./plots/psd_run13071.pdf")
# Save stuff to file for optimal matched filter (ligo-mjd.py)
np.savez("./data/fft_forcedAcqDS1.npz",avgPsd,xPsd,avgPwrSpec,xPwrSpec,data_forceAcq,data_fft)
def main(argv):
intMode, batMode = True, False
for i,opt in enumerate(argv):
if opt == "-b":
intMode, batMode = False, True
# set D and R
nBL = 512
D = 100
R = 80
# lo = 80
gFile = TFile("~/project/v2-waveskim/waveSkimDS5_90.root")
gatTree = gFile.Get("skimTree")
# Select a NOISE POPULATION
theCut = gFile.Get("theCut").GetTitle()
theCut += " && trapENFCal < 2 && channel!=692 && channel!=1232"
theCut += " && Entry$ < 10000"
gatTree.Draw(">>elist", theCut, "entrylist")
elist = gDirectory.Get("elist")
gatTree.SetEntryList(elist)
nList = elist.GetN()
print "Using cut:\n",theCut
print "Found",gatTree.GetEntries(),"input entries."
print "Found",nList,"entries passing cuts."
nWF = 0
rhoNoise = np.zeros([D,D])
for iList in range(nList):
entry = gatTree.GetEntryNumber(iList);
gatTree.LoadTree(entry)
gatTree.GetEntry(entry)
nChans = gatTree.channel.size()
numPass = gatTree.Draw("channel",theCut,"GOFF",1,iList)
chans = gatTree.GetV1()
chanList = list(set(int(chans[n]) for n in xrange(numPass)))
hitList = (iH for iH in xrange(nChans) if gatTree.channel.at(iH) in chanList)
for iH in hitList:
run = gatTree.run
iEvent = gatTree.iEvent
chan = gatTree.channel.at(iH)
dataENF = gatTree.trapENFCal.at(iH)
wf = gatTree.MGTWaveforms.at(iH)
# print "%d: run %d chan %d trapENFCal %.2f" % (iList, run, chan, dataENF)
signal = wl.processWaveform(wf)
data = signal.GetWaveBLSub()
data = data[:nBL]
binned, nBin = BinData(data,D)
for vec in binned:
out = np.outer(vec,vec)
rhoNoise = np.add(rhoNoise, out)
nWF += nBin
# fig = plt.figure(figsize=(9,7), facecolor='w')
# plt.imshow(rhoNoise,cmap='jet')
# plt.colorbar()
# plt.tight_layout()
# plt.show()
# ==========================================================
# ============ Construct the KLJ/Ralston filter ============
rhoNoise /= nWF
eigenvals, eigenvectors = lin.eig(rhoNoise) # eigenvectors are returned normalized.
eigenDict = {}
for i in range(len(eigenvals)):
eigenDict[eigenvals[i]] = eigenvectors[i]
# Sort the states by the np.absolute value (hopefully that's same as sorting by noise power)
eigenSorted = sorted(eigenvals, key=lambda eigenval: np.absolute(eigenval), reverse=True)
# for eig in eigenSorted: print np.absolute(eig), eig
# print len(eigenSorted)
# Now construct filters for different ranks R:
filterDict = {}
for i in range(len(eigenSorted)):
# for i in range(R+1): # don't calculate too many levels
evec = eigenDict[eigenSorted[i]]
pia = np.outer(evec,evec)
filterDict[i] = pia
piNoise = filterDict[R]
for i in range(R+1,D): piNoise += filterDict[i]
piSignal = np.identity(D) - piNoise
# piSignal = piNoise # crazy test
# fig = plt.figure(figsize=(9,7), facecolor='w')
# plt.imshow(piSignal,cmap='jet')
# plt.colorbar()
# plt.tight_layout()
# plt.show()
# ==========================================================
# Now that we've got the filter matrix, loop back over the data.
gFile.Close() # it's f*****g stupid I have to close and reopen.
gFile = TFile("~/project/v2-waveskim/waveSkimDS5_90.root")
gatTree = gFile.Get("skimTree")
# Select a SIGNAL POPULATION
theCut = gFile.Get("theCut").GetTitle()
theCut += " && trapENFCal > 2 && trapENFCal < 10"
gatTree.Draw(">>elist", theCut, "entrylist")
elist = gDirectory.Get("elist")
gatTree.SetEntryList(elist)
nList = elist.GetN()
print "Second pass using cut:\n",theCut
print "Found",gatTree.GetEntries(),"input entries."
print "Found",nList,"entries passing cuts."
fig = plt.figure(figsize=(8,6),facecolor='w')
p1 = plt.subplot(111)
# p2 = plt.subplot(212)
plt.show(block=False)
# Begin loop over events
iList = -1
while True:
iList += 1
if intMode and iList != 0:
value = raw_input()
if value=='q': break # quit
if value=='p': iList -= 2 # go to previous
if (value.isdigit()):
iList = int(value) # go to entry number
elif not intMode and batMode:
plt.pause(0.001) # rapid-draw mode
if iList >= nList: break # bail out, goose!
entry = gatTree.GetEntryNumber(iList);
gatTree.LoadTree(entry)
gatTree.GetEntry(entry)
nChans = gatTree.channel.size()
numPass = gatTree.Draw("channel",theCut,"GOFF",1,iList)
chans = gatTree.GetV1()
chanList = list(set(int(chans[n]) for n in xrange(numPass)))
hitList = (iH for iH in xrange(nChans) if gatTree.channel.at(iH) in chanList) # a 'generator expression'
for iH in hitList:
run = gatTree.run
iEvent = gatTree.iEvent
chan = gatTree.channel.at(iH)
dataENF = gatTree.trapENFCal.at(iH)
wf = gatTree.MGTWaveforms.at(iH)
print "%d: run %d chan %d trapENFCal %.2f" % (iList, run, chan, dataENF)
signal = wl.processWaveform(wf)
data = signal.GetWaveBLSub()
dataTS = signal.GetTS()
# partition the data into 'bins' - filter is applied to each, individually
# filter matrix is D x D.
# inside bins, timing resolution of order D * (sampling frequency) is expected.
# apply the filter to the subspaces
# transform back
binned, nBin = BinData(data,D)
filtBins = []
for vec in binned:
filtBins.append( piSignal.dot(vec) ) # dot product
filtData = np.zeros(len(data))
for i in range(len(filtData)):
j, k = int(i/D), i%D
filtData[i] = filtBins[j][k]
# show the data
p1.cla()
p1.plot(dataTS,filtData,color='red',label='filt')
p1.plot(dataTS,data,color='blue',alpha=0.7,label='data')
p1.legend(loc=4)
plt.tight_layout()
plt.pause(0.0000001)
def checkWave():
""" ./check-files.py [-c] -w """
print("Checking waves. Cal?", checkCal)
fileList = []
# bkg
if not checkCal:
dsMap = bkg.dsMap()
for ds in dsMap:
for sub in range(dsMap[ds] + 1):
fname = "%s/waveSkimDS%d_%d.root" % (dsi.waveDir, ds, sub)
if os.path.isfile(fname):
fileList.append(fname)
else:
print("File not found:", fname)
continue
# cal
else:
for key in cal.GetKeys():
ds = int(key[2])
if skipDS6Cal and ds == 6: continue
for cIdx in range(cal.GetIdxs(key)): # 0-indexed
cRuns = cal.GetCalList(key, cIdx)
for run in cRuns:
fname = "%s/waveSkimDS%d_run%d.root" % (dsi.calWaveDir, ds,
run)
if os.path.isfile(fname):
fileList.append(fname)
else:
print("File not found:", fname)
continue
if testMode:
fileList = []
ds = 1
dsMap = bkg.dsMap()
for sub in range(dsMap[ds] + 1):
fname = "%s/waveSkimDS%d_%d.root" % (dsi.waveDir, ds, sub)
fileList.append(fname)
# check first and last few entries of every file
for idx, fname in enumerate(fileList[:fLimit]):
f = TFile(fname)
t = f.Get("skimTree")
n = t.GetEntries()
if n == 0:
print("No entries in file", fname)
continue
brSingle, brVector = [], []
for br in t.GetListOfBranches():
if "vector" in br.GetClassName():
brVector.append(br.GetName())
else:
brSingle.append(br.GetName())
if n < 20:
eList = np.arange(0, n, 1)
else:
eList = np.arange(0, 20, 1)
eList = np.append(eList, np.arange(n - 20, n, 1))
for i in eList:
t.GetEntry(i)
# make sure individual entries are accessible (no segfaults)
for br in brSingle:
val = getattr(t, br)
# print(val)
for br in brVector:
try:
nCh = getattr(t, br).size()
except AttributeError:
print("Error:", br)
for j in range(nCh):
val = getattr(t, br)[j]
# print(br,nCh,val)
# make sure we can process waveforms
for j in range(nCh):
wf = t.MGTWaveforms.at(j)
ch = t.channel.at(j)
# be absolutely sure you're matching the right waveform to this hit
if wf.GetID() != ch:
print(
"ERROR -- Vector matching failed. entry %d, file: %s" %
(i, fname))
# run the LAT routine to convert into numpy arrays
truncLo, truncHi = 0, 2
if ds == 6 or ds == 2: truncLo = 4
signal = wl.processWaveform(wf, truncLo, truncHi)
if verbose:
print("%d/%d %s nEnt %d" % (idx, len(
fileList[:fLimit]), fname.split("/")[-1], t.GetEntries()))
f.Close()
def main(argv):
""" Interactive-fit or 'rapid'-fit waveforms that pass a given TCut.
BUG: Doesn't always work with a TChain. Add input files together
with hadd and use a single TFile.
"""
scanSpeed = 0.2
iList = -1
opt1, opt2 = "", "-"
intMode = False
if (len(argv) >= 1): opt1 = argv[0]
if (len(argv) >= 2): opt2 = argv[1]
if "-i" in (opt1, opt2):
intMode = True
print "Interactive mode selected."
startT = time.clock()
# Load template waveforms
templateFile = TFile("./data/wave-m1cal.root")
waveTemplates = templateFile.Get("waveTree")
calDict = {640:28, 672:18, 610:16, 580:19, 582:34, 648:38, 600:21, 578:39, 592:27, 664:55, 626:62, 692:8, 598:22, 690:52, 632:9, 608:7}
# PlotTemplateWaveforms(waveTemplates,calDict)
# return
# Set input data and cuts
inputFile = TFile("~/project/wavelet-skim/hadd/waveletSkimDS3.root")
# inputFile = TFile("~/project/wavelet-skim/waveletSkimDS3_13.root")
waveTree = inputFile.Get("skimTree")
print "Found",waveTree.GetEntries(),"input entries."
# theCut = inputFile.Get("cutUsedHere").GetTitle()
# DS3 big cut
theCut = "trapENFCal > 0.8 && gain==0 && mHClean==1 && isGood && !muVeto && !wfDCBits && !isLNFill1 && !isLNFill2 && trapETailMin < 0 && channel!=596 && channel!=676 && channel!=676 && channel!=612 && channel!=1104 && channel!=1200 && channel!=1334 && channel!=1336 && tOffset < 10 && waveS5/TMath::Power(trapENFCal,1/4) < 1200 && (waveS3-waveS2)/trapENFCal > 100 && (waveS3-waveS2)/trapENFCal < 300 && !(channel==692 && (run==16974 || run==16975 || run==16976 || run==16977 || run==16978 || run==16979)) && butterTime < 11000"
# theCut += " && trapENFCal > 2 && trapENFCal < 2.1 && channel==578 && run==16868" # template fast WF (2kev)
theCut += " && trapENFCal > 7 && trapENFCal < 10"
# theCut += " && trapENFCal < 20 && trapENFCal > 2 && run > 17480"
# theCut += " && kvorrT/trapENFCal > 2.2 && trapENFCal > 2 && trapENFCal < 10"
# Print cut and events passing cut
print "Using cut:\n",theCut,"\n"
waveTree.Draw(">>elist", theCut, "entrylist")
elist = gDirectory.Get("elist")
waveTree.SetEntryList(elist)
nList = elist.GetN()
print "Found",nList,"entries passing cuts."
stopT=time.clock()
print "Data loading time (s): ", (stopT-startT)
# Make a figure
fig = plt.figure(figsize=(11,7), facecolor='w')
p1 = plt.subplot2grid((6,7), (0,0), colspan=4, rowspan=2) # original
p2 = plt.subplot2grid((6,7), (2,0), colspan=4, rowspan=3) # rising edge
p3 = plt.subplot2grid((6,7), (5,0), colspan=4, rowspan=1) # residual
p4 = plt.subplot2grid((6,7), (0,4), colspan=3, rowspan=2 ) # trace 1
p5 = plt.subplot2grid((6,7), (2,4), colspan=3, rowspan=2, sharex=p4) # trace 2
p6 = plt.subplot2grid((6,7), (4,4), colspan=3, rowspan=2, sharex=p4) # trace 3
plt.show(block=False)
# Loop over events
while(True):
saveMe = False
iList += 1
if intMode==True and iList != 0:
value = raw_input()
if value=='q': break
if value=='s': saveMe=True
if value=='p': iList -= 2 # previous
if (value.isdigit()): iList = int(value) # go to entry
if iList >= elist.GetN(): break
entry = waveTree.GetEntryNumber(iList);
waveTree.LoadTree(entry)
waveTree.GetEntry(entry)
nChans = waveTree.channel.size()
numPass = waveTree.Draw("channel",theCut,"GOFF",1,iList)
chans = waveTree.GetV1()
chanList = list(set(int(chans[n]) for n in xrange(numPass)))
# Loop over hits passing cuts
hitList = (iH for iH in xrange(nChans) if waveTree.channel.at(iH) in chanList) # a 'generator expression'
for iH in hitList:
# Load waveform for this hit
run = waveTree.run
chan = waveTree.channel.at(iH)
dataE = waveTree.trapENFCal.at(iH)
dataST = waveTree.butterTime.at(iH) # replace with blrwfFMR50?
toe = waveTree.kvorrT.at(iH)/dataE
print "%d / %d Run %d nCh %d chan %d trapENF %.1f t/e %.1f" % (iList,nList,run,nChans,chan,dataE,toe)
signal = wl.processWaveform(waveTree.MGTWaveforms.at(iH),opt='full')
waveBLSub = signal.GetWaveBLSub()
waveFilt = signal.GetWaveFilt()
waveTS = signal.GetTS()
baseAvg, dataNoise = signal.GetBaseNoise()
# Denoise the data waveform (take only lowest-frequency components)
wp = pywt.WaveletPacket(data=waveBLSub, wavelet='haar', mode='symmetric',maxlevel=3)
new_wp = pywt.WaveletPacket(data=None, wavelet='haar', mode='symmetric')
new_wp['aaa'] = wp['aaa'].data
waveDenoised = new_wp.reconstruct(update=False)
# Load channel template waveform
try: tempEntry = calDict[chan]
except:
print "ain't got channel",chan
continue
waveTemplates.GetEntry(calDict[chan])
template = wl.processWaveform(waveTemplates.event.GetWaveform(waveTemplates.itr),opt='full')
temp = template.GetWaveBLSub()
tempTS = template.GetTS()
tempE = waveTemplates.trapENFCal
tempST = waveTemplates.blrwfFMR50
# Window the fit around rising edge - start time calculator method
loWin, hiWin = dataST - 1000, dataST + 4000 # ns
if loWin < waveTS[0] or hiWin > waveTS[-1]:
print "Window out of range! dataST: %.1f loWin %.1f hiWin %.1f" % (dataST,loWin,hiWin)
idx = np.where((waveTS >= loWin) & (waveTS <= hiWin))
data = waveBLSub[idx]
# data = waveDenoised[idx]
dataTS = waveTS[idx]
# Pack into lists
rawList = [waveBLSub, waveTS, dataE, dataST]
dataList = [data, dataTS, dataE, dataST, loWin, hiWin]
tempList = [temp, tempTS, tempE, tempST]
# Optionally save to a file
if saveMe:
print "Saved entry",iList,iH
np.savez("./data/optimumInputs.npz",rawList,tempList)
# np.savez("./data/rawInputs.npz",rawList)
# Recreate the guess and the guess's rising edge
guessFull, guessFullTS = wm.MakeModel(dataList, tempList, [dataST,dataE,1.], opt="full")
guess, guessTS = wm.MakeModel(dataList, tempList, [dataST,dataE,1.], opt="!fancy")
# Make an "almost complete" guess - no MCMC
# st, en, slo = dataST-100, dataE, 5
# InterpFn = interpolate.interp1d(tempTS, temp, kind="linear", copy="False", assume_sorted="True")
# model, modelTS = wm.MakeModel(dataList, tempList, [st,en,slo], fn=InterpFn)
# Fit with MCMC and get best-fit parameters
numSteps, burnIn = 2000, 1000 # default - 10000, 5000. try 3000, 2000
myModel = wm.TemplateModel( dataList, dataNoise, tempList )
waveModel = pymc.Model( myModel )
M = pymc.MCMC( waveModel )
M.use_step_method(pymc.Metropolis, M.startTime, proposal_sd=100., proposal_distribution='Normal')
M.use_step_method(pymc.Metropolis, M.energy, proposal_sd=1., proposal_distribution='Normal')
M.use_step_method(pymc.Metropolis, M.slowness, proposal_sd=100., proposal_distribution='Normal')
M.sample(iter=numSteps, verbose=0) # do the fit
st = np.median(M.trace('startTime')[burnIn:])
en = np.median(M.trace('energy')[burnIn:])
slo = np.median(M.trace('slowness')[burnIn:])
InterpFn = interpolate.interp1d(tempTS, temp, kind="linear", copy="False", assume_sorted="True")
model, modelTS = wm.MakeModel(dataList, tempList, [st,en,slo], fn=InterpFn)
# Calculate residual, Chi2/NDF
residual = model - data
frac = (np.power(data - model, 2)) / np.abs(model)
chi2NDF = np.sum(frac) / len(model)
print "chi2/NDF:",chi2NDF
# ** Do a separate & simple fit of the tail slope **
# TODO: Add this to process-waveforms.py
idxMax = np.where(guessFull == guessFull.max()) # returns an array/tuple
idxMax = idxMax[0][0] # "cast" to int
tail, tailTS = waveDenoised[idxMax:], waveTS[idxMax:]
popt,_ = curve_fit(wl.tailModelPol, tailTS, tail) # poly fit
a, b = dataE, 72000
popt2,_ = curve_fit(wl.tailModelExp, tailTS, tail, p0=[a,b]) # expo fit
# Fill the figure
p1.cla()
p1.set_ylabel("ADC")
p1.set_title("Run %d Channel %d Entry %d\ntrapENFCal %.1f T/E %.1f ST %.1f" % (run,chan,iList,dataE,toe,dataST))
p1.plot(waveTS,waveBLSub,color='blue')
p1.plot(waveTS,waveDenoised,color='red',alpha=0.8)
p1.plot(guessFullTS,guessFull,color='orange',linewidth=2)
p1.axvline(x = dataST, color='green',linewidth=2)
p1.axvline(x = loWin, color='black')
p1.axvline(x = hiWin, color='black')
p1.plot(tailTS, wl.tailModelPol(tailTS, *popt), color='cyan', linewidth=1) # tail poly fit
p1.plot(tailTS, wl.tailModelExp(tailTS, *popt2), color='cyan', linewidth=1) # tail expo fit
p2.cla()
p2.plot(dataTS,data,color='blue',label='Data')
p2.plot(guessTS,guess,color='orange',label='Guess')
p2.plot(modelTS,model,color='red',linewidth=3,alpha=0.7,label='Fit')
p2.legend(loc=4)
p3.cla()
p3.set_xlabel("Time [ns]", x=0.95, ha='right')
p3.set_ylabel("Residual [ADC]")
p3.plot(modelTS,residual, color='red')
p3.axhline(y = 0, color='blue', alpha=0.3)
p3.axhline(y = dataNoise, color='blue', alpha=0.3)
p3.axhline(y = -1.0*dataNoise, color='blue', alpha=0.3)
p4.cla()
minST = tempST - tempTS[-1] + hiWin
maxST = tempST - tempTS[0] + loWin
p4.set_title("startTime %.1f Energy %.2f\nSlow %.1f chi2/ndf %.1f Min %d Max %d" % (st,en,slo,chi2NDF,minST,maxST))
p4.plot(M.trace('startTime')[:])
p4.set_ylabel('startTime')
p5.cla()
p5.plot(M.trace('energy')[:])
p5.set_ylabel('energy')
p6.cla()
p6.plot(M.trace('slowness')[:])
p6.set_ylabel('slowness')
plt.tight_layout()
plt.subplots_adjust(hspace=0.35)
plt.pause(scanSpeed)
def updateFile(dsNum, cal):
""" ./lat2.py -upd [options]
Cruel twist of fate: Not gonna use the calibration stuff above rn.
Am gonna use this to update each dataset's LAT files with Ralph's
sigma parameter.
11/16/2017 - This has big problems when used on the PDSF queue. It randomly corrupts files and
causes segfaults when the tree is read. For some reason it works fine when run on login nodes.
Short term fix for wfStd is to do it with the login nodes.
If we ever implement LAT2 energy calibration, we will have to deal with this issue.
"""
import ROOT
from ROOT import std
filePath = bgDir + "/latSkimDS%d*.root" % dsNum
if cal == True:
filePath = calDir + "/latSkimDS%d*.root" % dsNum
print "Globbing", filePath
files = glob.glob(filePath)
# print files
# return
# comment this block out for normal running
# skip ahead to where these jobs died last
# smallList = []
# foundLeftOff = False
# for fileName in sorted(files):
# if dsNum==0:
# if foundLeftOff:
# smallList.append(fileName)
# if fileName == calDir + "/latSkimDS0_run3439_0.root":
# foundLeftOff = True
# elif dsNum==1:
# if foundLeftOff:
# smallList.append(fileName)
# if fileName == calDir + "/latSkimDS1_run12729_2.root":
# foundLeftOff = True
# print len(files), len(smallList)
# files = smallList
# run over just one file
# files = ["/global/homes/w/wisecg/project/cal-lat/latSkimDS0_run4841_0.root"]
for fileName in sorted(files):
start = time.clock()
print "Now scanning", fileName
f = TFile(fileName, "UPDATE")
tree = f.Get("skimTree")
nEnt = tree.GetEntries()
# wipe the wfstd branch if it already exists before creating a new one
b0 = tree.GetListOfBranches().FindObject("wfstd")
if isinstance(b0, ROOT.TBranchElement):
tree.GetListOfBranches().Remove(b0)
tree.Write()
wfstd = std.vector("double")()
b1 = tree.Branch("wfstd", wfstd)
# loop over events
for iList in range(nEnt):
tree.GetEntry(iList)
nChans = tree.channel.size()
nWFs = tree.MGTWaveforms.size()
if (nChans != nWFs):
print "Wrong num entries. Bailing!"
exit(1)
wfstd.assign(nChans, -88888)
# loop over hits
for iH in range(nWFs):
run = tree.run
chan = tree.channel.at(iH)
energy = tree.trapENFCal.at(iH)
wf = tree.MGTWaveforms.at(iH)
signal = wl.processWaveform(wf, 0, 0)
waveRaw = signal.GetWaveRaw()
waveTS = signal.GetTS()
# print "%d / %d Run %d nCh %d chan %d trapENF %.1f" % (iList,nList,run,nChans,chan,energy)
wfstd[iH] = np.std(waveRaw[5:-5])
maxAdc = max(waveRaw[5:-5])
minAdc = min(waveRaw[5:-5])
nBins = int(maxAdc - minAdc)
# End loop over hits, fill branches
b1.Fill()
# End loop over events
tree.Write("", ROOT.TObject.kOverwrite)
print " - Tree entries: %d Branch entries: %d Time (min): %.2f" % (
tree.GetEntries(), b1.GetEntries(), (time.clock() - start) / 60.)
f.Close()
def main(argv):
""" Interactive-fit or 'rapid'-fit waveforms that pass a given TCut.
BUG: Doesn't always work with a TChain. Add input files together
with hadd and use a single TFile.
"""
scanSpeed = 0.2
iList = -1
opt1, opt2 = "", "-"
intMode = False
if (len(argv) >= 1): opt1 = argv[0]
if (len(argv) >= 2): opt2 = argv[1]
if "-i" in (opt1, opt2):
intMode = True
print "Interactive mode selected."
# if opt2.isdigit:
# iList = int(opt2) - 1
startT = time.clock()
# Load template waveforms
templateFile = TFile("./data/wave-m1cal.root")
waveTemplates = templateFile.Get("waveTree")
calDict = {
640: 28,
672: 18,
610: 16,
580: 19,
582: 34,
648: 38,
600: 21,
578: 39,
592: 27,
664: 55,
626: 62,
692: 8,
598: 22,
690: 52,
632: 9,
608: 7
}
# PlotTemplateWaveforms(waveTemplates,calDict)
# Set input data and cuts
# waveTree = TChain("skimTree") # BUG: Can't always recognize MGTWaveforms branch
# waveTree.Add("~/project/wavelet-skim/waveletSkimDS3*")
# inputFile = TFile("~/project/wavelet-skim/hadd/waveletSkimDS3.root")
inputFile = TFile("~/project/wavelet-skim/waveletSkimDS3_13.root")
waveTree = inputFile.Get("skimTree")
print "Found", waveTree.GetEntries(), "input entries."
# theCut = inputFile.Get("cutUsedHere").GetTitle()
# theCut = "trapENFCal > 0.8 && gain==0 && mH==1 && isGood && !muVeto && !wfDCBits && !isLNFill1 && !isLNFill2 && trapETailMin < 0 && channel!=596 && channel!=676 && channel!=676 && channel!=612 && channel!=1104 && channel!=1200 && channel!=1334 && channel!=1336"
# DS3 big cut
theCut = "trapENFCal > 0.8 && gain==0 && mHClean==1 && isGood && !muVeto && !wfDCBits && !isLNFill1 && !isLNFill2 && trapETailMin < 0 && channel!=596 && channel!=676 && channel!=676 && channel!=612 && channel!=1104 && channel!=1200 && channel!=1334 && channel!=1336 && tOffset < 10 && waveS5/TMath::Power(trapENFCal,1/4) < 1200 && (waveS3-waveS2)/trapENFCal > 100 && (waveS3-waveS2)/trapENFCal < 300 && !(channel==692 && (run==16974 || run==16975 || run==16976 || run==16977 || run==16978 || run==16979))"
# theCut += " && trapENFCal > 2 && trapENFCal < 2.1 && channel==578 && run==16868" # template fast WF (2kev)
theCut += " && trapENFCal > 3 && trapENFCal < 4"
# theCut += " && trapENFCal < 10 && trapENFCal > 1"
# Print cut and events passing cut
print "Using cut:\n", theCut, "\n"
waveTree.Draw(">>elist", theCut, "entrylist")
elist = gDirectory.Get("elist")
waveTree.SetEntryList(elist)
nList = elist.GetN()
print "Found", nList, "entries passing cuts."
stopT = time.clock()
print "Data loading time (s): ", (stopT - startT)
# Make a figure
fig = plt.figure(figsize=(8, 7), facecolor='w')
gs = gridspec.GridSpec(3, 1, height_ratios=[2, 3, 1])
a1 = plt.subplot(gs[0])
a2 = plt.subplot(gs[1])
a3 = plt.subplot(gs[2])
plt.show(block=False)
# Loop over events
while (True):
iList += 1
if intMode == True and iList != 0:
value = raw_input()
if value == 'q': break
if value == 'p': iList -= 2 # previous
if (value.isdigit()): iList = int(value) # go to entry
if iList >= elist.GetN(): break
entry = waveTree.GetEntryNumber(iList)
waveTree.LoadTree(entry)
waveTree.GetEntry(entry)
nChans = waveTree.channel.size()
numPass = waveTree.Draw("channel", theCut, "GOFF", 1, iList)
chans = waveTree.GetV1()
chanList = list(set(int(chans[n]) for n in xrange(numPass)))
# Loop over hits passing cuts
hitList = (iH for iH in xrange(nChans)
if waveTree.channel.at(iH) in chanList
) # a 'generator expression'
for iH in hitList:
# Load waveform for this hit
run = waveTree.run
chan = waveTree.channel.at(iH)
energy = waveTree.trapENFCal.at(iH)
print "%d / %d Run %d nCh %d chan %d trapENF %.1f" % (
iList, nList, run, nChans, chan, energy)
signal = wl.processWaveform(waveTree.MGTWaveforms.at(iH),
opt='full')
waveBLSub = signal.GetWaveBLSub()
waveFilt = signal.GetWaveFilt()
waveTS = signal.GetTS()
baseAvg, noiseAvg = signal.GetBaseNoise()
riseTime = waveTree.butterTime.at(iH) # replace with blrwfFMR50?
# Load channel template waveform
try:
tempEntry = calDict[chan]
except:
print "ain't got channel", chan
continue
waveTemplates.GetEntry(calDict[chan])
template = wl.processWaveform(waveTemplates.event.GetWaveform(
waveTemplates.itr),
opt='full')
templateENF = waveTemplates.trapENFCal
templateT50 = waveTemplates.blrwfFMR50
tempTS = template.GetTS()
tempBLSub = template.GetWaveBLSub()
# Guess scaling parameters & fill arrays
scaleGuess = energy / templateENF
tDiffGuess = riseTime - templateT50
tempGuess = tempBLSub * scaleGuess
tempTSGuess = tempTS + tDiffGuess
tempGuessRT = templateT50 + tDiffGuess
# Fit around rising edge - rise time calculator method
lo, hi = 300, 700 # samples before and after the start time
loWin = riseTime / 10 - lo
hiWin = riseTime / 10 + hi
waveEndsRT = np.concatenate(
(np.arange(0, loWin), np.arange(hiWin, waveBLSub.size)),
axis=0)
waveEdgeRT = np.delete(waveBLSub, waveEndsRT)
tsEdgeRT = np.delete(waveTS, waveEndsRT)
# Fit with MCMC
numSteps, burnin = 10000, 4000
# numSteps, burnin = 1000, 400
waveModel = pymc.Model(
wm.WaveformModel(waveEdgeRT, tsEdgeRT, tempBLSub, tempTS,
tDiffGuess, scaleGuess, noiseAvg))
M = pymc.MCMC(waveModel)
M.use_step_method(pymc.Metropolis,
M.riseTime,
proposal_sd=4.,
proposal_distribution='Normal')
M.use_step_method(pymc.Metropolis,
M.slowness,
proposal_sd=0.1,
proposal_distribution='Normal')
M.use_step_method(pymc.Metropolis,
M.scale,
proposal_sd=0.1,
proposal_distribution='Normal')
M.sample(iter=numSteps, verbose=0) # do the fit
rt = np.median(M.trace('riseTime')[burnin:])
slowness = np.median(M.trace('slowness')[burnin:])
scale = np.median(M.trace('scale')[burnin:])
# Recreate the guess, the best-fit signal, and the residual
templateTSRT = tempTS + rt
lo, hi = tsEdgeRT[0], tsEdgeRT[-1] + 10
idx = np.where((templateTSRT >= lo) & (templateTSRT <= hi))
scaled = tempBLSub[idx] * scale
smoothed = gaussian_filter(scaled, sigma=float(slowness))
smoothedTS = templateTSRT[idx]
guess = tempBLSub[idx] * scaleGuess
guessTS = templateTSRT[idx]
residual1 = smoothed - guess
residual2 = smoothed - waveEdgeRT
residual3 = smoothed - waveFilt[idx]
# Calculate the Chi2 of the fit
# Fill the figure
a1.cla()
a1.set_ylabel("ADC")
a1.set_title("Run %d Channel %d Entry %d trapENFCal %.1f" %
(run, chan, iList, energy))
a1.plot(waveTS, waveBLSub, color='blue')
a1.plot(waveTS, waveFilt, color='green', alpha=0.6)
a1.plot(tempTSGuess, tempGuess, color='orange')
a1.axvline(x=riseTime, color='red', linewidth=3)
a1.axvline(x=tempGuessRT, color='green', alpha=0.8)
a1.axvline(x=loWin * 10, color='cyan')
a1.axvline(x=hiWin * 10, color='cyan')
a2.cla()
a2.plot(tsEdgeRT, waveEdgeRT, color='blue')
a2.plot(smoothedTS, smoothed, color='red', linewidth=4)
a2.plot(guessTS, guess, color='orange')
a3.cla()
a3.set_xlabel("Time [ns]", x=0.95, ha='right')
a3.set_ylabel("Residual [ADC]")
a3.plot(smoothedTS, residual1, color='red')
# a3.plot(smoothedTS,residual2, color='cyan')
# a3.plot(smoothedTS,residual3, color='cyan')
a3.axhline(y=0, color='blue', alpha=0.3)
plt.tight_layout()
plt.subplots_adjust(hspace=0.15)
plt.pause(scanSpeed)
