def processData(directory,
defaultTemplate,
isVerbose=False,
GUI=False,
progress_callback=(),
dataset=(),
mainDirectory=(),
continuing=False,
filterMethod=mF.wienerFilter):
"""
Loop through .npz files in the directory and create filter coefficient files for each
INPUTS:
directory - path for the folder containing the data
defaultTemplate - numpy array with template coefficients (200 coefficients)
isVerbose - prints progress to terminal
GUI - True if using in optimalFilterGUI
progress_callback - percent complete callback for GUI, ignored if GUI=False
dataset - index of data folder being run on, ignored if GUI = False
mainDirectory - directory containing folders, ignored if GUI=False
continuing - flag for if continuing from a previous calculation, ignored if GUI=False
filterMethod - filter calculation method. Must have same input structure as functions
in makeFilter.py
"""
# start time
startTime = time.time()
# make default Filter (renormalize and flip)
defaultFilter = -defaultTemplate[0:50] / np.dot(defaultTemplate[0:50],
defaultTemplate[0:50])
defaultFilter = defaultFilter[::-1]
# parse filterMethod name
filterName = str(filterMethod).split(' ')[1]
# set flag for log file output when unexpected files in the directory
logFileFlag = 0
# check before deleting files
continuingFlag = 0
if os.path.isfile(os.path.join(directory, "log_file.txt")) and not GUI:
answer = query_yes_no("Are you continuing a stopped calculation?")
if answer is True:
continuingFlag = 1
else:
answer = query_yes_no(
"Are you sure that you want to delete previous filter "
"calculations?")
if answer is False:
return
if GUI:
continuingFlag = continuing
# delete old log and filter coefficients if exists if told to do so
if not continuingFlag:
if os.path.isfile(os.path.join(directory, "log_file.txt")):
os.remove(os.path.join(directory, "log_file.txt"))
if os.path.isfile(
os.path.join(directory, filterName + '_coefficients.txt')):
os.remove(os.path.join(directory,
filterName + '_coefficients.txt'))
if os.path.isfile(os.path.join(directory,
'template_coefficients.txt')):
os.remove(os.path.join(directory, 'template_coefficients.txt'))
if os.path.isfile(os.path.join(directory, 'filter_type.txt')):
os.remove(os.path.join(directory, 'filter_type.txt'))
if os.path.isfile(os.path.join(directory, 'noise_data.txt')):
os.remove(os.path.join(directory, 'noise_data.txt'))
if os.path.isfile(os.path.join(directory, 'rough_templates.txt')):
os.remove(os.path.join(directory, 'rough_templates.txt'))
if os.path.isfile(os.path.join(directory, 'file_list.txt')):
os.remove(os.path.join(directory, 'file_list.txt'))
if os.path.isfile(os.path.join(directory,
filterName + '_fourier.txt')):
os.remove(os.path.join(directory, filterName + '_fourier.txt'))
# get .npz files into list
fileList = []
for item in os.listdir(directory):
if os.path.isfile(os.path.join(directory,
item)) and item.endswith('.npz'):
fileList.append(item)
# get channel numbers for each file
channelNumber = []
timeStamp = []
badNameInd = []
for index, fileName in enumerate(fileList):
# extract channel number from filename (assuming snap_X_resIDX_DATE-time format)
if fileName.split('_')[2][0:5] == 'resID':
channelNumber.append(int(fileName.split('_')[2][5:]))
timeString = fileName.split('_')[3]
timeString = timeString.split('-')[0] + timeString.split(
'-')[1].split('.')[0]
timeStamp.append(int(timeString))
else:
logFileFlag = 1
badNameInd.append(index)
if not continuingFlag:
with open(os.path.join(directory, "log_file.txt"),
'a') as logfile:
message = "Removed '{0}' from file list due to incorrect name " +\
"format" + os.linesep
logfile.write(message.format(fileName))
# remove filenames with incorrect formats
fileList = [
element for i, element in enumerate(fileList) if i not in badNameInd
]
# sort file list by channel number
sortedIndices = np.argsort(channelNumber)
fileList = [fileList[i] for i in sortedIndices]
timeStamp = [timeStamp[i] for i in sortedIndices]
channelNumber = [channelNumber[i] for i in sortedIndices]
# find duplicate channel numbers and remove all but the most recent data file
goodIndicies = []
uniqueChannelNumbers = list(set(channelNumber))
uniqueChannelNumbers.sort()
for uniqueChannel in uniqueChannelNumbers:
indexList = []
timeStampList = []
for channelInd, channel in enumerate(channelNumber):
# create list of duplicate channels and their time stamps
if channel == uniqueChannel:
indexList.append(channelInd)
timeStampList.append(timeStamp[channelInd])
# sort timestamps
sortedIndices = np.argsort(timeStampList)
# print warning about duplicates to logfile
if len(sortedIndices) > 1 and not continuingFlag:
with open(os.path.join(directory, "log_file.txt"), 'a') as logfile:
for ind in sortedIndices[:-1]:
logFileFlag = 1
message = "Removed '{0}' from file list due to duplicate channel " + \
"number " + os.linesep
logfile.write(message.format(fileList[indexList[ind]]))
# append channel number index with the largest timestamp to the good index list
goodIndicies.append(indexList[sortedIndices[-1]])
# warn about unexpected files
if logFileFlag and not continuingFlag:
print(
"Unexpected files in the current directory. Check the log file to make "
"sure the program removed the right ones!")
with open(os.path.join(directory, "log_file.txt"), 'a') as logfile:
logfile.write(os.linesep)
# print progress to terminal
if isVerbose and not GUI:
sys.stdout.write("Percent of filters created: 0.0% " + os.linesep)
sys.stdout.flush()
if GUI:
progress_callback.emit((0.0, dataset))
# pick filenames removing duplicate channel numbers
fileList = [fileList[i] for i in goodIndicies]
# uncomment these lines for debugging particular files
# indicies = range(0,11)
# fileList = [fileList[i] for i in indicies]
# start at a particular file if continuing calculation
if continuingFlag:
filters = np.atleast_2d(
np.loadtxt(
os.path.join(directory, filterName + '_coefficients.txt')))
num = filters.shape[0]
if num == len(fileList):
print('No more files to itterate over')
if GUI:
return dataset, float(time.time() - startTime), True, 100
else:
return
else:
fileListOriginal = fileList
fileList = fileList[num:]
with open(os.path.join(directory, 'file_list.txt'), 'wb') as fp:
pickle.dump(fileListOriginal, fp)
else:
# save file list if not continuing from previous calculation
with open(os.path.join(directory, 'file_list.txt'), 'wb') as fp:
pickle.dump(fileList, fp)
# initialize arrays
filterArray = np.zeros((1, 50))
templateArray = np.zeros((1, 200))
roughTemplateArray = np.zeros((1, 200))
typeArray = np.zeros(1)
noiseArray = np.zeros((1, 101))
filterNoiseArray = np.zeros((1, 26))
# loop through all the phase stream data files in the directories
for index, fileName in enumerate(fileList):
startLoop = time.time()
finishedLoad = startLoop
finishedTemp = startLoop
finishedFilt = startLoop
# reinitialize noise flag
noiseFlag = 0
# load data
try:
errorFlag = 0
# load data
rawData = np.load(os.path.join(directory, fileName))
key = rawData.keys()
rawData = rawData[key[0]]
except:
with open(os.path.join(directory, "log_file.txt"), 'a') as logfile:
message = "{1}: File '{0}' data failed to load. Using default " + \
"template and filter " + os.linesep
logfile.write(message.format(fileName, index))
# use default filter and templates
templateArray = defaultTemplate
filterArray = defaultFilter
# find fourier transform of filter for comparison
filterNoiseArray = np.abs(np.fft.rfft(filterArray))**2
# save data
with open(
os.path.join(directory, filterName + '_coefficients.txt'),
'a') as filters:
np.savetxt(filters, np.atleast_2d(filterArray))
with open(os.path.join(directory, 'template_coefficients.txt'),
'a') as templates:
np.savetxt(templates, np.atleast_2d(templateArray))
with open(os.path.join(directory, 'rough_templates.txt'),
'a') as rough:
np.savetxt(rough, np.atleast_2d(roughTemplateArray))
with open(os.path.join(directory, 'filter_type.txt'),
'a') as types:
np.savetxt(types, np.atleast_1d(typeArray))
with open(os.path.join(directory, 'noise_data.txt'), 'a') as noise:
np.savetxt(noise, np.atleast_2d(noiseArray))
with open(os.path.join(directory, filterName + '_fourier.txt'),
'a') as noise:
np.savetxt(noise, np.atleast_2d(filterNoiseArray))
continue
finishedLoad = time.time()
# make template
try:
template, _, noiseDict, templateList, _ = mT.makeTemplate(
rawData,
nSigmaTrig=5.,
numOffsCorrIters=2,
defaultFilter=defaultFilter)
roughTemplateArray = templateList[-1]
noiseFlag = 1
# check for bad template
# (fall times greater than 7 and less than 50, assuming 50 points given)
if (np.trapz(template[5:50]) > -5
or np.trapz(template[5:50]) < -31.6
or templateList[-1][5] != -1):
errorFlag = 1
raise ValueError('proccessData: template not correct')
# add template coefficients to array
templateArray = template
templateFlag = 1
except:
# add template coefficients to array
templateArray = defaultTemplate
roughTemplateArray = np.zeros(np.shape(defaultTemplate))
templateFlag = 0
finishedTemp = time.time()
# make filter
try:
if templateFlag:
filterCoef = filterMethod(template,
noiseDict['spectrum'],
nTaps=50)
elif noiseFlag:
filterCoef = filterMethod(defaultTemplate,
noiseDict['spectrum'],
nTaps=50)
else:
data = mT.hpFilter(rawData)
noiseDict = mNS.makeNoiseSpectrum(data,
window=200,
filt=defaultFilter)
filterCoef = filterMethod(defaultTemplate,
noiseDict['spectrum'],
nTaps=50)
# add filter coefficients to array
filterArray = filterCoef
noiseArray = noiseDict['spectrum']
# find fourier transform of filter for comparison
filterNoiseArray = np.abs(np.fft.rfft(filterArray))**2
filterFlag = 1
except:
# add filter coefficients to array
if templateFlag:
filterArray = -template[:50] / np.dot(template[:50],
template[:50])
filterArray = filterArray[::-1]
else:
filterArray = defaultFilter
noiseArray = np.zeros(101)
# find fourier transform of filter for comparison
filterNoiseArray = np.abs(np.fft.rfft(filterArray))**2
filterFlag = 0
finishedFilt = time.time()
# log results and categorize them in the type array
with open(os.path.join(directory, "log_file.txt"), 'a') as logfile:
if templateFlag == 0 and filterFlag == 0:
message = "{1}: File '{0}' template and filter calculation failed. " + \
"Using default template as filter :: Timing info: " + \
"{2}, {3}, {4}" + os.linesep
logfile.write(
message.format(fileName, index,
round(finishedLoad - startLoop, 2),
round(finishedTemp - finishedLoad, 2),
round(finishedFilt - finishedTemp, 2)))
typeArray = 0
elif templateFlag == 1 and filterFlag == 0:
message = "{1}: File '{0}' filter calculation failed. Using " + \
"calculated template as filter :: Timing info: {2}, {3}, {4}" +\
os.linesep
logfile.write(
message.format(fileName, index,
round(finishedLoad - startLoop, 2),
round(finishedTemp - finishedLoad, 2),
round(finishedFilt - finishedTemp, 2)))
typeArray = 1
elif templateFlag == 0 and filterFlag == 1:
message = "{1}: File '{0}' template calculation failed. Using " + \
"default template with noise as filter :: Timing info: " + \
"{2}, {3}, {4}" + os.linesep
logfile.write(
message.format(fileName, index,
round(finishedLoad - startLoop, 2),
round(finishedTemp - finishedLoad, 2),
round(finishedFilt - finishedTemp, 2)))
typeArray = 2
else:
message = "{1}: File '{0}' calculation successful :: Timing info: " + \
"{2}, {3}, {4}" + os.linesep
logfile.write(
message.format(fileName, index,
round(finishedLoad - startLoop, 2),
round(finishedTemp - finishedLoad, 2),
round(finishedFilt - finishedTemp, 2)))
typeArray = 3
# write new data to file
with open(os.path.join(directory, filterName + '_coefficients.txt'),
'a') as filters:
np.savetxt(filters, np.atleast_2d(filterArray))
with open(os.path.join(directory, 'template_coefficients.txt'),
'a') as templates:
np.savetxt(templates, np.atleast_2d(templateArray))
with open(os.path.join(directory, 'rough_templates.txt'),
'a') as rough:
np.savetxt(rough, np.atleast_2d(roughTemplateArray))
with open(os.path.join(directory, 'filter_type.txt'), 'a') as types:
np.savetxt(types, np.atleast_1d(typeArray))
with open(os.path.join(directory, 'noise_data.txt'), 'a') as noise:
np.savetxt(noise, np.atleast_2d(noiseArray))
with open(os.path.join(directory, filterName + '_fourier.txt'),
'a') as noise:
np.savetxt(noise, np.atleast_2d(filterNoiseArray))
# determine full filelist
if continuingFlag:
index0 = index + len(fileListOriginal) - len(fileList)
fileList0 = fileListOriginal
else:
index0 = index
fileList0 = fileList
# print progress to terminal
if isVerbose and not GUI:
if index != len(fileList) - 1:
if continuingFlag:
index0 = index + len(fileListOriginal) - len(fileList)
else:
index0 = index
perc = round(float(index0 + 1) / (len(fileList0)) * 100, 1)
message = "Percent of filters created: %.1f%% " + os.linesep
sys.stdout.write(message % (perc))
sys.stdout.flush()
else:
print("Percent of filters created: 100% ")
# if GUI is running check to see if program should end and display progress
if GUI:
perc = round(float(index0 + 1) / (len(fileList0)) * 100, 1)
progress_callback.emit((perc, dataset))
working = False
while not working:
try:
killArray = np.loadtxt(
os.path.join(mainDirectory,
'kill_processes' + str(dataset) + '.txt'))
if killArray:
return dataset, float(
(time.time() - startTime)), False, perc
working = True
except:
time.sleep(0.5)
# count number of each type of filter
typeArray = np.loadtxt(os.path.join(directory, 'filter_type.txt'))
typeArray = np.atleast_1d(typeArray)
unique, counts = np.unique(typeArray, return_counts=True)
countdict = dict(zip(unique, counts))
if 0 not in countdict.keys():
countdict[0] = 0
if 1 not in countdict.keys():
countdict[1] = 0
if 2 not in countdict.keys():
countdict[2] = 0
if 3 not in countdict.keys():
countdict[3] = 0
# print final results
if not GUI:
print("{0}% of pixels using optimal filters".format(
round(countdict[3] / float(len(typeArray)) * 100, 2)))
print("{0}% of pixels using default template with noise as filter".
format(round(countdict[2] / float(len(typeArray)) * 100, 2)))
print("{0}% of pixels using calculated template as filter".format(
round(countdict[1] / float(len(typeArray)) * 100, 2)))
print("{0}% of pixels using default template as filter".format(
round(countdict[0] / float(len(typeArray)) * 100, 2)))
endTime = time.time()
# log final results
with open(os.path.join(directory, "log_file.txt"), 'a') as logfile:
logfile.write(os.linesep + "File list that was itterated over: " +
os.linesep)
for fileName in fileList0:
logfile.write(("{0} " + os.linesep).format(fileName))
logfile.write(os.linesep + " computation time: {0} minutes".format(
(endTime - startTime) / 60.0))
logfile.write((os.linesep + "{0}% of pixels using optimal filters " +
os.linesep).format(
round(countdict[3] / float(len(typeArray)) * 100,
2)))
logfile.write(
("{0}% of pixels using default template with noise as filter " +
os.linesep).format(
round(countdict[2] / float(len(typeArray)) * 100, 2)))
logfile.write(("{0}% of pixels using calculated template as filter " +
os.linesep).format(
round(countdict[1] / float(len(typeArray)) * 100,
2)))
logfile.write(("{0}% of pixels using default template as filter " +
os.linesep).format(
round(countdict[0] / float(len(typeArray)) * 100,
2)))
# return some stuff if GUI is running
if GUI:
return dataset, float((endTime - startTime)), True, 100
nPointsBefore=100.
riseTime=2e-6
fallTime=50e-6
tmax=nPointsBefore/sampleRate
t0=tmax+riseTime*np.log(riseTime/(riseTime+fallTime))
#get fake poissonian distributed pulse data
rawdata, rawtime = mAD.makePoissonData(totalTime=2*131.072e-3,isVerbose=True,maxSignalToNoise=2)
if isPlotPoisson:
fig1=plt.figure(0)
plt.plot(rawtime,rawdata)
plt.show()
#calculate templates
finalTemplate, time , _, templateList, _ = mT.makeTemplate(rawdata,nSigmaTrig=4.,numOffsCorrIters=2,isVerbose=True,isPlot=isPlot)
roughTemplate = templateList[0]
#make fitted template
fittedTemplate, startFit, riseFit, fallFit = mT.makeFittedTemplate(finalTemplate,time,riseGuess=3.e-6,fallGuess=55.e-6)
#calculate real template
realTemplate = mAD.makePulse(time,t0,riseTime,fallTime)
realTemplate = mT.hpFilter(realTemplate)
realTemplate /= np.abs(realTemplate[np.argmax(np.abs(realTemplate))])
if isPlotRes:
fig, (ax0, ax1) = plt.subplots(nrows=2, sharex=True)
h1, = ax0.plot(time*1e6,roughTemplate,'r',linewidth=2); h1Label='initial template'
h2, = ax0.plot(time*1e6,finalTemplate,'g',linewidth=2); h2Label='offset corrected template'
h3, = ax0.plot(time*1e6,realTemplate,'b', linewidth=2); h3Label='real pulse shape'
reload(mF)
reload(tP)
##### Test on real data #####
if False:
isPlot=False
isVerbose=True
#get data
rawData = eRD.parseQDRPhaseSnap(os.path.join(os.getcwd(),'TestData/20140915/redLaser'),pixelNum=0,roachNum=0,steps=30)
#make fake data
#rawData, rawTime = mAD.makePoissonData(totalTime=10*131.072e-3,amplitudes='random',maxSignalToNoise=10,isVerbose=isVerbose)
#rawData*=2
print "data extracted"
#make template
template, time, noiseSpectrumDict, _, _ = mT.makeTemplate(rawData,nSigmaTrig=5.,numOffsCorrIters=3,isVerbose=isVerbose,isPlot=isPlot, sigPass=.5)
print "template made"
#noiseSpectrumDict['noiseSpectrum']=np.ones(len(noiseSpectrumDict['noiseSpectrum']))
#noiseSpectrumDict['noiseSpectrum']=np.ones(len(noiseSpectrumDict['noiseSpectrum']))*3e3/np.abs(noiseSpectrumDict['noiseFreqs'])
#noiseSpectrumDict['noiseSpectrum'][0]=2*noiseSpectrumDict['noiseSpectrum'][1]
#noiseSpectrumDict['noiseSpectrum'][np.abs(noiseSpectrumDict['noiseFreqs'])>210000]=5e-5
#make matched filter
matchedFilter=mF.makeMatchedFilter(template, noiseSpectrumDict['noiseSpectrum'], nTaps=50, tempOffs=75)
coef, _ = opt.curve_fit(lambda x, a, t0 : a*exp(-x/t0), time[len(time)*1/5:len(time)*4/5],template[len(time)*1/5:len(time)*4/5], [-1 , 30e-6])
fallFit=coef[1]
superMatchedFilter=mF.makeSuperMatchedFilter(template, noiseSpectrumDict['noiseSpectrum'], fallFit, nTaps=50, tempOffs=75)
print "filters made"
#convolve with filter
firCoeffs /= len(peakIndices)
return firCoeffs
if __name__=='__main__':
testMatched = False
testWiener = True
if testMatched:
#test makeMatchedFilter
template = np.zeros(800)
template[100:800] = 10.*np.exp(-np.arange(0,700)/30.)
data,time = mAD.makePoissonData(totalTime=1000.e-3, maxSignalToNoise=0.0)
noiseCov = noise.covFromData(data-np.mean(data), 100)['covMatrix']
plt.show()
if testWiener:
templateData, time = mAD.makePoissonData(totalTime=10*131.072e-3)
finalTemplate, time, _,_,_ = mkt.makeTemplate(templateData)
filterData, time = mAD.makePoissonData(totalTime=10*131.072e-3,maxSignalToNoise=10) #new data set for Wiener filter raw pulses
_,_,_,_,peakIndices = mkt.makeTemplate(filterData)
wienerFilt = makeAvgCausalWiener(finalTemplate, filterData, peakIndices)
plt.plot(wienerFilt)
plt.plot(finalTemplate)
#plt.plot(filterData[peakIndices[0]-100:peakIndices[0]+800]/5)
plt.show()
