def testSecondBoundary2(self):
run = 'PAL2012'
sundownDate = '20121211'
obsDate = '20121212'
seq = '121229'
t0 = 123
t1 = 133
nSigma = 1
stride = 1
threshold = 15
fn = FileName.FileName(run, sundownDate, obsDate+"-"+seq)
cosmic = Cosmic(fn, beginTime=t0, endTime=t1,
loggingLevel=logging.CRITICAL)
fc = cosmic.findCosmics(stride=stride,
threshold=threshold,
populationMax=2000,
nSigma=nSigma)
print "len if timeHgValues=",len(fc['timeHgValues'])
plt.clf()
t = np.arange(len(fc['timeHgValues']))/1e6
plt.plot(t,fc['timeHgValues'])
plt.ylabel("number of photons per tick")
plt.xlabel("time after %.1f (sec)"%t0)
plt.title("run=%s obsDate=%s seq=%s t0=%d t1=%d"%(run,obsDate,seq,t0,t1))
plt.grid(axis='x')
#plt.yscale('symlog', linthreshy=0.9)
plt.yscale('log')
plt.ylim(ymin=-0.1)
plt.savefig(inspect.stack()[0][3]+".png")
def findv1(self):
populationMax=2000
ySum = np.zeros(populationMax)
frameSum = 'none'
seq5 = self.s['seq5'].split()
for seq in seq5:
print "seq=",seq
outfileName = "cosmicTimeList-"+seq+".pkl"
if not os.path.exists(outfileName):
fn = FileName(self.s['run'],
self.s['sundownDate'],
self.s['obsDate']+"-"+str(seq))
cosmic = Cosmic(fn,
beginTime=self.s['beginTime'],
endTime=self.s['endTime'],
loggingLevel = logging.INFO)
fc = cosmic.findCosmics(stride=int(self.s['stride']),
threshold=int(self.s['threshold']),
populationMax=populationMax,
nSigma=float(self.s['nSigma']))
outfile = open(outfileName, "wb")
pickle.dump(fc['cosmicTimeList'],outfile)
pickle.dump(fc['binContents'],outfile)
outfile.close()
cfn = "cosmicMask-%s.h5"%seq
ObsFile.writeCosmicIntervalToFile(fc['interval'],1.0e6, cfn,
self.s['beginTime'],
self.s['endTime'],
int(self.s['stride']),
int(self.s['threshold']),
float(self.s['nSigma']),
populationMax)
del cosmic
def testCosmicTimeMasking(self):
"""
tests the cosmic time masking function
"""
ymax = sys.float_info.max/100.0
run = 'PAL2012'
sundownDate = '20121211'
obsDate = '20121212'
seq = '121229'
fn = FileName.FileName(run, sundownDate, obsDate+"-"+seq)
beginTime = 123.247400
endTime = 123.248400
cosmic = Cosmic(fn, beginTime=beginTime, endTime=endTime)
stride = 10
threshold = 100
populationMax=2000
nSigma=10
dictionary0 = cosmic.findCosmics(stride=stride,
threshold=threshold,
populationMax=populationMax,
nSigma=nSigma)
interval = dictionary0['interval']
plt.clf()
plt.plot(dictionary0['timeHgValues'])
for oneInt in interval:
x0 = (oneInt[0]-beginTime)*cosmic.file.ticksPerSec
x1 = (oneInt[1]-beginTime)*cosmic.file.ticksPerSec
plt.fill_between((x0,x1),(0,0),(ymax,ymax),alpha=0.2,color='red')
plt.title("timeHgValues stride=%d threshold=%d nSigma=%d"%(stride,threshold,nSigma))
plt.xlabel("ticks after t=%f seconds"%beginTime)
plt.yscale("symlog",linthreshy=0.5)
plt.ylim(0,dictionary0['timeHgValues'].max())
plt.savefig("testCosmicTimeMasking-0.png")
ObsFile.ObsFile.writeCosmicIntervalToFile(interval,
cosmic.file.ticksPerSec,
'junk.h5')
cosmic.file.loadCosmicMask('junk.h5')
dictionary1 = cosmic.findCosmics(populationMax=1000)
plt.clf()
hist = dictionary0['populationHg'][0]
bins = np.arange(len(hist))
plt.plot(bins, hist)
plt.ylabel("no mask")
plt.xscale('log')
plt.yscale('log')
hist = dictionary1['populationHg'][0]
bins = np.arange(len(hist))
plt.plot(bins, hist, color='g')
plt.ylabel("cosmic mask")
plt.xscale('log')
plt.yscale('log')
plt.savefig("testCosmicTimeMasking.png")
def test_findCosmics(self):
run = 'PAL2012'
obsDate = '20121211'
seq = '20121212-113212'
fn = FileName.FileName(run, obsDate, seq)
cosmic = Cosmic(fn, beginTime= 222, endTime=232,\
nBinsPerSec = 10, loggingLevel=logging.FATAL)
stride = 10
threshold = 20
populationMax = 2000
fc = cosmic.findCosmics(stride, threshold, populationMax)
cosmicTimeList = fc['cosmicTimeList']
# 4 events found on 2013-09-27
self.assertEquals(4,len(cosmicTimeList))
def testExponAndGaussFit(self):
run = 'PAL2012'
sundownDate = '20121211'
obsDate = '20121212'
seq = '131254'
fn = FileName(run, sundownDate, obsDate+"-"+seq)
beginTime=79.248482
endTime = beginTime+200*1e-6
cosmic = Cosmic(fn, beginTime=beginTime,endTime=endTime)
dictionary = cosmic.findCosmics(populationMax=1000)
hist = np.array(dictionary['timeHgValues'])
bins = np.arange(len(hist))
plt.plot(bins, hist)
plt.savefig(inspect.stack()[0][3]+".png")
def testPopulationFromTimeHgValues1(self):
"""
test one specific example with stride of 4
"""
timeHgValues = np.array([0,0,10,20,0,0,5,6,7,8,9,1])
populationMax = 40
stride = 4
threshold = 15
d = Cosmic.populationFromTimeHgValues(timeHgValues, populationMax,
stride, threshold)
populationTrue = np.zeros(populationMax)
populationTrue[11] = 1
populationTrue[25] = 1
populationTrue[26] = 1
populationTrue[30] = 2
self.assertEquals(None, np.testing.assert_array_equal(
d['populationHg'][0],
populationTrue))
self.assertEquals(None, np.testing.assert_array_equal(
d['cosmicTimeList'],
np.array([0,2,6,8])))
self.assertEquals(None, np.testing.assert_array_equal(
d['binContents'],
np.array([30,30,26,25])))
def test_pfthv2(self):
"""
Demonstrate the static method populationFromTimeHgValues
Put 1 photon in 10 successive time ticks (starting at tick 15),
and 0 photons in the rest of the time ticks.
For a stride of 10, the method samples 7 time bins. The bins overlap
by stride/2.
The populationHg returned reports that 4 of these bins have zero photons,
2 of the bins have 5 photons, and 1 bin has 10 photons.
"""
length = 40
timeHgValues = np.zeros(length, dtype=np.int64)
for i in range(15,25,1):
timeHgValues[i] = 1
populationMax = 40
stride = 10
threshold = 4
pfthv = Cosmic.populationFromTimeHgValues(timeHgValues, populationMax,
stride, threshold)
self.assertEquals(populationMax, len(pfthv['populationHg'][0]))
self.assertEquals(4, pfthv['populationHg'][0][0])
self.assertEquals(2, pfthv['populationHg'][0][5])
self.assertEquals(1, pfthv['populationHg'][0][10])
def testPopulationFromTimeHgValues3(self):
"""
test one specific example with stride of 5
and a number of timeHgValues that is not a multiple of stride
"""
timeHgValues = np.array([0,0,10,20,0,0,5,6,7,8,9,1,99])
populationMax = 40
stride = 5
threshold = 15
d = Cosmic.populationFromTimeHgValues(timeHgValues, populationMax,
stride, threshold)
populationTrue = np.zeros(populationMax)
populationTrue[26] = 1
populationTrue[30] = 1
populationTrue[35] = 1
self.assertEquals(None, np.testing.assert_array_equal(
d['populationHg'][0],
populationTrue))
self.assertEquals(None, np.testing.assert_array_equal(
d['cosmicTimeList'],
np.array([0,2,5])))
self.assertEquals(None, np.testing.assert_array_equal(
d['binContents'],
np.array([30,35,26])))
def testSecondBoundary(self):
run = 'PAL2012'
sundownDate = '20121211'
obsDate = '20121212'
seq = '121229'
t0 = 123
t1 = 133
nSigma = 1
stride = 1
threshold = 15
fn = FileName.FileName(run, sundownDate, obsDate+"-"+seq)
tsum = None
for t in range(t0,t1,1):
beginTime = t-15e-5
endTime = t+15e-5
cosmic = Cosmic(fn, beginTime=beginTime, endTime=endTime,
loggingLevel=logging.CRITICAL)
fc = cosmic.findCosmics(stride=stride,
threshold=threshold,
populationMax=2000,
nSigma=nSigma)
if tsum == None:
tsum = np.array(fc['timeHgValues'])
else:
tsum += np.array(fc['timeHgValues'])
plt.clf()
dt = (endTime-beginTime)
x0 = (t-beginTime)*(len(tsum)-1)/dt
print "dt=",dt
print "x0=",x0
print "len(tsum)=",len(tsum)
x = np.arange(len(tsum))-x0
plt.plot(x,tsum, drawstyle='steps-mid')
ymin = -1
ymax = 1.1*tsum.max()
xAtMax = tsum.argmax()-x0
#plt.vlines(xAtMax, ymin, ymax, linestyles='dotted', label="beginning of second")
plt.text(xAtMax+10,ymax/2, "t at max=%.1f"%xAtMax, ha="left",va="center")
plt.ylim(-1,ymax)
plt.ylabel("number of photons per time tick")
plt.xlabel("ticks since beginning of second")
plt.title("run=%s obsDate=%s seq=%s t0=%d t1=%d"%(run,obsDate,seq,t0,t1))
plt.savefig(inspect.stack()[0][3]+".png")
def run(self):
for obs in self.param['obsSequence']:
fn = FileName.FileName(run=self.param['run'],
date=self.param['sunsetDate'],
tstamp = obs)
cosmic = Cosmic(fn,
loggingLevel=logging.INFO,
beginTime=self.s['beginTime'],
endTime=self.s['endTime'])
fc = cosmic.findCosmics(stride=int(self.s['stride']),
threshold=int(self.s['threshold']),
populationMax=int(self.s['populationMax']),
nSigma=float(self.s['nSigma']),
writeCosmicMask=True)
cosmic = Cosmic(fn,
loggingLevel=logging.INFO,
beginTime=self.s['beginTime'],
endTime=self.s['endTime'],
applyCosmicMask=True)
fcm = cosmic.findCosmics(stride=int(self.s['stride']),
threshold=int(self.s['threshold']),
populationMax=int(self.s['populationMax']),
nSigma=float(self.s['nSigma']),
writeCosmicMask=False)
p = {
'populationHg':fc['populationHg'][0],
'populationHgM':fcm['populationHg'][0],
'pps':fc['pps'],
'ppsM':fcm['pps'],
'ppmMasked':fc['ppmMasked'],
'ppsTime':fc['ppsTime'],
'beginTime':cosmic.beginTime,
'endTime':cosmic.endTime
}
outfileName = "cosmic-summary-"+obs+".pkl"
outfile = open(outfileName, "wb")
pickle.dump(p,outfile)
outfile.close()
def testFitExponRealData(self):
"""
Read timestamps from part of one data file where there was an event.
"""
# We imported FileName at the top of this file. Use this to
# locate the file for this specific observation.
# on turk, the file with the observation is this one:
# /ScienceData/PAL2012/20121211/obs_20121212-1333303.h5
run = 'PAL2012'
sundownDate = '20121211'
obsDate = '20121212'
seq = '133303'
fn = FileName(run, sundownDate, obsDate+"-"+seq)
# The class Cosmic is in ARCONS-pipeline/cosmic/Cosmic.py
cosmic = Cosmic(fn)
t0 = 138595526
t1 = 138595666
dictionary = cosmic.fitExpon(t0,t1)
hist = dictionary['timeHgValues']
bins = np.arange(len(hist))
width = 0.7*(bins[1]-bins[0])
center = (bins[:-1]+bins[1:])/2
#plt.step(center, hist, where = 'post')
plt.plot(bins,hist, label='data')
xFit = np.linspace(0,len(hist),10*len(hist))
pFit = dictionary['pExponFit']
yFit = Cosmic.funcExpon(xFit,*pFit)
plt.plot(xFit,yFit, label='Exponential Fit')
xFit = np.linspace(0,len(hist),10*len(hist))
pFit = dictionary['pGaussFit']
print "pFit=",pFit
yFit = Cosmic.funcGauss(xFit,*pFit)
plt.plot(xFit,yFit, label='Gaussian Fit')
plt.legend()
xLimit = dictionary['xLimit']
plt.axvline(x=xLimit[0], color='magenta', linestyle='-.')
plt.axvline(x=xLimit[1], color='magenta', linestyle='-.')
plt.savefig(inspect.stack()[0][3]+".png")
def testCosmic(self):
run = 'PAL2012'
sundownDate = '20121211'
obsDate = '20121212'
seq = '121229'
fn = FileName.FileName(run, sundownDate, obsDate+"-"+seq)
cosmic = Cosmic(fn, beginTime=123, endTime=125)
dictionary0 = cosmic.findCosmics(threshold=500, populationMax=2000)
print "dictionary0 keyst=",dictionary0.keys()
print "interval=",dictionary0['interval']
hist = dictionary0['populationHg'][0]
bins = np.arange(len(hist))
plt.plot(bins, hist, label="no mask")
plt.xscale('log')
plt.yscale('log')
plt.legend()
plt.savefig(inspect.stack()[0][3]+".png")
def testPopulationFromTimeHgValues0(self):
"""
test one specific example with stride of 1
"""
timeHgValues = np.array([0,10,5,0,0,0,0,10,8,1,0,0,0,0,0,0])
populationMax = 20
stride = 1
threshold = 9
d = Cosmic.populationFromTimeHgValues(timeHgValues, populationMax,
stride, threshold)
self.assertEquals(None, np.testing.assert_array_equal(
d['populationHg'][0],
np.array([11,1,0,0,0,1,0,0,1,0,2,0,0,0,0,0,0,0,0,0])))
self.assertEquals(None, np.testing.assert_array_equal(
d['cosmicTimeList'],
np.array([1,7])))
self.assertEquals(None, np.testing.assert_array_equal(
d['binContents'],
np.array([10,10])))
def test_populationFromTimeHgValues(self):
"""
test the funcion Cosmic.populationFromTimeHgValues.
Make a time stream where in successive time values you have 1,3,1
photons. Look for cosmic events with a threshold of 4. Arrange
the time stream so that one of the large bins (starting at 89995)
overlaps the time stamps with all five photons, but the next one
(starting at 90000) only overlaps 4 photons. With the threshold
set to 4, the bin starting at 89995 is found and the bin starting
at 90000 is not found.
"""
minLength = 1000000
timeHgValues = np.zeros(minLength, dtype=np.int64)
timeHgValues[89999] = 1
timeHgValues[90000] = 3
timeHgValues[90001] = 1
populationMax = 10
stride = 10
threshold = 4
pfthv = Cosmic.populationFromTimeHgValues(
timeHgValues, populationMax, stride, threshold)
cosmicTimeList = pfthv['cosmicTimeList']
#print "cosmicTimeList=",cosmicTimeList
self.assertEquals(cosmicTimeList.size,1)
self.assertEquals(cosmicTimeList[0], 89995)
populationHg = pfthv['populationHg']
#print "populationHg[0]=",populationHg[0]
self.assertEquals(populationHg[0][0], 199996)
self.assertEquals(populationHg[0][1], 1)
self.assertEquals(populationHg[0][2], 0)
self.assertEquals(populationHg[0][3], 0)
self.assertEquals(populationHg[0][4], 1)
self.assertEquals(populationHg[0][5], 1)
seq5 = ['112709', '113212', '113714', '114216', '114718', '115220', '115722', '120224', '120727', '121229', '121732', '122234', '122736', '123238', '123740', '124242', '124744', '125246', '125748', '130250', '130752', '131254', '131756', '132258', '132800', '133303']
#seq5 = ['133303']
nSigma = 7
stride = 10
threshold = 50
populationMax=2000
ySum = np.zeros(populationMax)
frameSum = 'none'
for seq in seq5:
print "seq=",seq
fn = FileName(run, sundownDate, obsDate+"-"+seq)
#cosmic = Cosmic(fn, endTime='exptime')
cosmic = Cosmic(fn, beginTime=0)
fc = cosmic.findCosmics(stride=stride,
threshold=threshold,
populationMax=populationMax,
nSigma=nSigma)
if frameSum == 'none':
frameSum = fc['frameSum']
else:
frameSum += fc['frameSum']
outfile = open("cosmicTimeList-"+seq+".pkl", "wb")
pickle.dump(fc['cosmicTimeList'],outfile)
pickle.dump(fc['binContents'],outfile)
outfile.close()
cfn = "cosmicMax-%s.h5"%seq
ObsFile.writeCosmicIntervalToFile(fc['interval'],1.0e6, cfn)
populationHg = fc['populationHg']
nSigma = 7
threshold = 25
stride = 10
populationMax = 1000
nlist = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37,
38, 39, 40]
nlist2 = [10, 20, 30]
for n in nlist:
beginTime = n-1
endTime = n
cosmic = Cosmic(fn, beginTime, endTime)
dictionary0 = cosmic.findCosmics(nSigma, threshold, stride, populationMax)
interval = dictionary0['interval']
hist0 = dictionary0['populationHg'][0]
bins0 = np.arange(len(hist0))
muNum = (bins0*hist0).sum()
muDen = float(hist0.sum())
mu1 = muNum/muDen
mu1Err = np.sqrt(mu1/muDen)
print "mu1=",mu1," mu1Err=",mu1Err," muNum=",muNum," muDen=",muDen
mean.append(mu1)
meanErr.append(mu1Err)
import pickle
from interval import interval, inf, imath
import logging, os
import pickle
run = 'PAL2013'
sundownDate = '20131204'
obsDate = '20131205'
seq = '033506'
populationMax=1000
beginTime = 0
endTime = 100
stride = 1000
fn = FileName(run, sundownDate, obsDate+"-"+seq)
cosmic = Cosmic(fn, beginTime=beginTime, endTime=endTime, loggingLevel=logging.INFO)
timeHgValues,frameSum = cosmic.getTimeHgAndFrameSum(beginTime,endTime)
lthgv=len(timeHgValues)
print "length of timeHgValues=",lthgv
nStride=lthgv/stride
binnedTimeHgValues = np.reshape(timeHgValues[0:nStride*stride],[nStride,stride]).sum(axis=1)
#plt.xkcd()
dt = cosmic.file.tickDuration*stride
times = cosmic.beginTime + dt*np.arange(nStride)
plt.plot(times,binnedTimeHgValues,
drawstyle='steps-mid',label="all data")
#plt.xlim(xmax=1000)
#plt.xscale('symlog',linthreshx=0.9)
plt.yscale('symlog',linthreshy=0.5)
import logging, os
import pickle
run = 'PAL2013'
sundownDate = '20131204'
obsDate = '20131205'
seq = '033506'
populationMax=1000
beginTime = 0
expTime = 300
fn = FileName(run, sundownDate, obsDate+"-"+seq)
cosmic = Cosmic(fn, beginTime=0, endTime=10, loggingLevel=logging.INFO)
fcd = cosmic.findCosmics(nSigma=10, stride=10, threshold=15)
print "done: fcd.keys=",fcd.keys()
maskedTime = Cosmic.countMaskedBins(fcd['interval'])
maskedPercent = 100*maskedTime/(cosmic.endTime-cosmic.beginTime)
print "maskedPercent=",maskedPercent
cosmic.file.setCosmicMask(fcd['interval'])
fcd2 = cosmic.findCosmics()
#plt.xkcd()
plt.plot(fcd['populationHg'][0],drawstyle='steps-mid',label="all data")
plt.plot(fcd2['populationHg'][0],drawstyle='steps-mid',label="cosmics masked")
plt.xlim(xmax=1000)
plt.xscale('symlog',linthreshx=0.9)
plt.yscale('symlog',linthreshy=0.5)
Makes a plot for the entire file showing the mean for every ten seconds of
data and the cosmic events. This plot is saved as MeanAndCosmicEvents.png
Makes a scatter plot of the number of cosmics masked out and the mean which is
saved as scatter.png
"""
#loads the file
run = 'PAL2012'
sundownDate = '20121211'
obsDate = '20121212'
seq = '121229'
fn = FileName.FileName(run, sundownDate, obsDate+"-"+seq)
#this particular plot is for only ten seconds of data
cosmic = Cosmic(fn, beginTime=123, endTime=133)
dictionary0 = cosmic.findCosmics(nSigma=6, threshold=15, populationMax=1000)
interval = dictionary0['interval']
ObsFile.ObsFile.writeCosmicIntervalToFile(interval,
cosmic.file.ticksPerSec,
'junk.h5')
cosmic.file.loadCosmicMask('junk.h5')
dictionary1 = cosmic.findCosmics(nSigma=6, threshold=15, populationMax=1000)
nSigma = 6
threshold = 15
plt.clf()
hist0 = dictionary0['populationHg'][0]
def testCountMaskedBins(self):
"""count the number of time bins that the interval masks"""
i = interval([1,3]) | interval([10,20])
cmb = Cosmic.countMaskedBins(i)
self.assertEquals(12,cmb)
nSigmas = [10, 10, 1000, 10]
strides = [10, 10, 10, 10]
thresholds = [5000, 30, 30, 15]
styles = ['b-', 'ro', 'g+', 'cx']
populationMax=1000
ySum = np.zeros(populationMax)
frameSum = 'none'
beginTime = 123
endTime = 133
plt.clf()
for nSigma,stride,threshold,style in zip(nSigmas, strides, thresholds, styles):
print "seq=",seq
fn = FileName(run, sundownDate, obsDate+"-"+seq)
#cosmic = Cosmic(fn, endTime='exptime')
cosmic = Cosmic(fn, beginTime=beginTime, endTime=endTime,
loggingLevel=logging.CRITICAL)
fc = cosmic.findCosmics(stride=stride,
threshold=threshold,
populationMax=populationMax,
nSigma=nSigma)
tMasked = Cosmic.countMaskedBins(fc['interval'])
ppmMasked = 1000000*tMasked/(endTime-beginTime)
print "ppmMasked=",ppmMasked
cosmic.file.cosmicMask = fc['interval']
cosmic.file.cosmicMaskIsApplied = True
fc = cosmic.findCosmics(stride=stride,
threshold=threshold,
