def testExponaverage(self):
"""
generate and fit a histogram of an exponential distribution with many
events using the average time to find the fit. The histogram is then
saved in testExponaverage.png
"""
tau = 25.0
nBins = 400
size = 100
taulist = []
for i in range(1000):
x = range(nBins)
yPlot = funcExpon(xPlot, *popt)
timeHgValues = np.zeros(nBins, dtype=np.int64)
timeStamps = expon.rvs(loc=0, scale=tau, size=size)
ts64 = timeStamps.astype(np.uint64)
tsBinner.tsBinner(ts64, timeHgValues)
param = sum(timeStamps)/len(timeStamps)
fit = expon.pdf(x,param)
fit *= size
taulist.append(param)
hist,bins = np.histogram(taulist, bins=20, range=(15,35))
width = 0.7*(bins[1]-bins[0])
center = (bins[:-1]+bins[1:])/2
#plt.bar(center, hist, align = 'center', width = width) produces bar graph
plt.step(center, hist, where = 'post')
plt.savefig(inspect.stack()[0][3]+".png")
def testExponOneEvent(self):
"""
generate and fit an exponential distribution with lifetime of 25
make a plot in testExpon.png
"""
tau = 25.0
nBins = 400
size = 100
x = range(nBins)
timeHgValues = np.zeros(nBins, dtype=np.int64)
timeStamps = expon.rvs(loc=0, scale=tau, size=size)
ts64 = timeStamps.astype(np.uint64)
tsBinner.tsBinner(ts64, timeHgValues)
param = expon.fit(timeStamps)
fit = expon.pdf(x,loc=param[0],scale=param[1])
fit *= size
tvf = timeHgValues.astype(np.double)
tvf[tvf<1] = 1e-3 # the plot looks nicer if zero values are replaced
plt.plot(x, tvf, label="data")
plt.plot(x, fit, label="fit")
plt.yscale('log')
plt.xlim(xmax=100)
plt.ylim(ymin=0.09)
plt.legend()
plt.title("true tau=%.1f fit tau=%.1f"%(tau,param[1]))
plt.savefig(inspect.stack()[0][3]+".png")
def testExponManyEvents(self):
"""
generate and fit an exponential distribution with lifetime of 25
make a plot in testExponManyEvents.png
"""
tau = 25.0
nBins = 400
size = 100
taulist = []
for i in range(1000):
x = range(nBins)
timeHgValues = np.zeros(nBins, dtype=np.int64)
timeStamps = expon.rvs(loc=0, scale=tau, size=size)
ts64 = timeStamps.astype(np.uint64)
tsBinner.tsBinner(ts64, timeHgValues)
param = expon.fit(timeStamps)
fit = expon.pdf(x,loc=param[0],scale=param[1])
fit *= size
print "i=",i," param[1]=",param[1]
taulist.append(param[1])
hist,bins = np.histogram(taulist, bins=20, range=(15,25))
width = 0.7*(bins[1]-bins[0])
center = (bins[:-1]+bins[1:])/2
plt.step(center, hist, where = 'post')
plt.savefig(inspect.stack()[0][3]+".png")
def testExpon(self):
"""
generate and fit an exponential distribution with lifetime of 25
make a plot in testExpon.png
"""
tau = 25.0
nBins = 400
size = 100
x = range(nBins)
timeHgValues = np.zeros(nBins, dtype=np.int64)
timeStamps = expon.rvs(loc=0, scale=tau, size=size)
ts64 = timeStamps.astype(np.uint64)
tsBinner.tsBinner(ts64, timeHgValues)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
# Note: this line casus a RuntimeWarning in optimize.py:301
param = expon.fit(timeStamps)
fit = expon.pdf(x,loc=param[0],scale=param[1])
fit *= size
tvf = timeHgValues.astype(np.double)
#tvf[tvf<1] = 1e-3 # the plot looks nicer if zero values are replaced
plt.plot(x, tvf, label="data")
plt.plot(x, fit, label="fit")
plt.yscale('symlog', linthreshy=0.9)
plt.xlim(xmax=100)
plt.ylim(ymin = -0.1)
plt.legend()
plt.title("true tau=%.1f fit tau=%.1f"%(tau,param[1]))
plt.savefig(inspect.stack()[0][3]+".png")
def testTiming(self):
"""
For binning into nBins=20 bins, for nTs=100,000 time sample values,
repeat c_binner.binner and then tsBinner.tsBinner nTrials=1000 times.
Then run NPBinner.binner
Report the elapsed time (in milliseconds) for the two ways.
Typical speeds are tsbinner=0.17, cbinner=0.53, and NPBinner=1.94
"""
nBins = 20
nTs = 100000
nTrials = 1000
ts = np.zeros(nTs, dtype=np.uint64)
ts += np.random.random_integers(0,1,nTs)
bins = np.zeros(nBins, dtype=np.int)
t0 = time.time()
for i in range(nTrials):
tsBinner.tsBinner(ts,bins)
t1 = time.time()
dtTsBinner = t1-t0
bins = np.zeros(nBins, dtype=np.int)
t0 = time.time()
for i in range(nTrials):
c_binner.binner(ts,bins)
t1 = time.time()
dtCBinner = t1-t0
print "tsBinner=%.3f cBinner=%.3f"%(dtTsBinner, dtCBinner)
bins = np.zeros(nBins, dtype=np.int)
ts32 = ts.astype(np.uint32)
t0 = time.time()
for i in range(nTrials):
#print "i=",i,"/",nTrials
contents = NPBinner.NPBinner.binner(ts32,nBins)
t1 = time.time()
dtNPBinner = t1-t0
print "NPBinner=%.3f"%(dtNPBinner)
t0 = time.time()
for i in range(nTrials):
contents = np.bincount(ts32,minlength=nBins)
t1 = time.time()
dtBincount = t1-t0
print "bincount=%.3f"%(dtBincount)
t0 = time.time()
for i in range(nTrials):
tsBinner.tsBinner32(ts32,bins)
t1 = time.time()
dtTsBinner32 = t1-t0
print "tsBinner32=%.3f"%(dtTsBinner32)
def testTsBinner(self):
"""
confirm that tsBinner.tsBinner gets the correct answer
for one data set
"""
ts = np.array([1,3,5,6], dtype=np.uint64)
bins = np.zeros(10, dtype=np.int)
tsBinner.tsBinner(ts,bins)
# print "bins=",bins
self.assertTrue((bins == self.answer).all())
def makemovie1(self):
run = self.s['run']
sundownDate = self.s['sundownDate']
obsDate = self.s['obsDate']
stride = int(self.s['stride'])
seq5 = self.s['seq5'].split()
for seq in seq5:
inFile = open("cosmicTimeList-%s.pkl"%(seq),"rb")
cosmicTimeList = pickle.load(inFile)
binContents = pickle.load(inFile)
cfn = "cosmicMask-%s.h5"%seq
intervals = ObsFile.readCosmicIntervalFromFile(cfn)
for interval in intervals:
print "interval=",interval
fn = FileName(run, sundownDate,obsDate+"-"+seq)
obsFile = ObsFile(fn.obs())
obsFile.loadTimeAdjustmentFile(fn.timeAdjustments())
i0=interval[0]
i1=interval[1]
intervalTime = i1-i0
dt = intervalTime/2
beginTime = max(0,i0-0.000200)
endTime = beginTime + 0.001
integrationTime = endTime-beginTime
nBins = int(np.round(obsFile.ticksPerSec*(endTime-beginTime)+1))
timeHgValues = np.zeros(nBins, dtype=np.int64)
ymax = sys.float_info.max/100.0
for iRow in range(obsFile.nRow):
for iCol in range(obsFile.nCol):
gtpl = obsFile.getTimedPacketList(iRow,iCol,
beginTime,integrationTime)
ts = (gtpl['timestamps'] - beginTime)*obsFile.ticksPerSec
ts64 = np.round(ts).astype(np.uint64)
tsBinner.tsBinner(ts64, timeHgValues)
plt.clf()
plt.plot(timeHgValues, label="data")
x0 = (i0-beginTime)*obsFile.ticksPerSec
x1 = (i1-beginTime)*obsFile.ticksPerSec
plt.fill_between((x0,x1),(0,0), (ymax,ymax), alpha=0.2, color='red')
plt.yscale("symlog",linthreshy=0.9)
plt.xlim(0,1000)
plt.ylim(-0.1,300)
tick0 = int(np.round(i0*obsFile.ticksPerSec))
plotfn = "cp-%05d-%s-%s-%s-%09d"%(timeHgValues.sum(),run,obsDate,seq,tick0)
plt.title(plotfn)
plt.legend()
plt.savefig(plotfn+".png")
plt.xlabel("nSigma=%d stride=%d threshold=%d"%(int(self.s['nSigma']),int(self.s['stride']),int(self.s['threshold'])))
print "plotfn=",plotfn
os.system("convert -delay 0 `ls -r cp*png` cp.gif")
def reducedChiHist(self):
def funcExpon(x, a, b, c, d):
retval = a*np.exp(-b*(x-d)) + c
retval[x < d] = 0
return retval
tau = 25.0
t0 = 40
nBins = 800
size = 10000
taulist = []
for i in range(100):
timeHgValues = np.zeros(nBins, dtype=np.int64)
timeStamps = expon.rvs(loc=0, scale=tau, size=size) + t0
ts64 = timeStamps.astype(np.uint64)
tsBinner.tsBinner(ts64, timeHgValues)
xPoints = []
yPoints = []
for x in range(nBins):
y = timeHgValues[x]
if y > 2:
xPoints.append(x)
yPoints.append(y)
bGuess = 1/(np.mean(timeStamps))
aGuess = bGuess*(size)
cGuess = 0
dGuess = t0
pGuess = [aGuess, bGuess, cGuess, dGuess]
xArray = np.array(xPoints) #must be in array for not list to use curve_fit
yArray = np.array(yPoints)
ySigma = (yArray ** 1/2)
popt, pcov = curve_fit(funcExpon, xArray, yArray, p0=pGuess, sigma=ySigma)
xPlot = np.linspace(xArray[0], xArray[-1], 1000)
yPlot = funcExpon(xPlot, *popt)
yPlotAtPoints = funcExpon(xArray, *popt)
chi_2 = sum(((yArray-yPlotAtPoints)**2)/ySigma)
red_chi_2 = chi_2/(len(yArray)-len(popt))
print red_chi_2
hist,bins = np.histogram(red_chi_2, bins=20)
width = 0.7*(bins[1]-bins[0])
center = (bins[:-1]+bins[1:])/2
plt.step(center, hist, where = 'post')
plt.savefig(inspect.stack()[0][3]+".png")
def testRound(self):
"""
tests rounding the time stamps so that the histograms don't contain
unexpected zero values
"""
run = 'PAL2012'
sundownDate = '20121211'
obsDate = '20121212'
seq = '115722'
fileName = FileName.FileName(run, sundownDate, obsDate+"-"+seq)
timeAdjustments = fileName.timeAdjustments()
obsFile = ObsFile.ObsFile(fileName.obs())
obsFile.loadTimeAdjustmentFile(timeAdjustments)
firstSec=49.163775
integrationTime = 100.e-6
times = np.array([])
for iRow in range(46):
for iCol in range(44):
gtpl = obsFile.getTimedPacketList(iRow, iCol, firstSec, integrationTime)
ts = gtpl['timestamps']
times = np.append(times,ts)
times -= firstSec
#times *= 1.e6
times *= 1000000
timesOriginal = times.copy()
nBins = 100
hist,edges = np.histogram(times,bins=nBins,range=(0,100))
plt.plot(edges[0:-1],hist, label="no rounding np.histogram")
times = np.round(times)
hist,edges = np.histogram(times,bins=nBins,range=(0,100))
plt.plot(edges[0:-1],hist, label="yes rounding np.histogram")
timeHgValues = np.zeros(nBins, dtype=np.int64)
ts64 = timesOriginal.astype(np.uint64)
tsBinner.tsBinner(ts64,timeHgValues)
plt.plot(edges[0:-1],timeHgValues, 'r+', label="no rounding tsBinner")
timeHgValues = np.zeros(nBins, dtype=np.int64)
ts64 = np.round(timesOriginal).astype(np.uint64)
tsBinner.tsBinner(ts64,timeHgValues)
plt.plot(edges[0:-1],timeHgValues, 'cx', label="yes rounding tsBinner")
plt.legend(loc="upper left").get_frame().set_alpha(0.5)
plt.savefig(inspect.stack()[0][3]+".png")
def displayFits(self):
"""
generates two histograms on the same plot. One uses maximum likelihood to fit
the data while the other uses the average time.
"""
tau = 25.0
nBins = 400
size = 100
taulist = []
taulistavg = []
for i in range(1000):
x = range(nBins)
timeHgValues = np.zeros(nBins, dtype=np.int64)
timeStamps = expon.rvs(loc=0, scale=tau, size=size)
ts64 = timeStamps.astype(np.uint64)
tsBinner.tsBinner(ts64, timeHgValues)
param = sum(timeStamps)/len(timeStamps)
fit = expon.pdf(x,param)
fit *= size
taulistavg.append(param)
for i in range(1000):
x = range(nBins)
timeHgValues = np.zeros(nBins, dtype=np.int64)
timeStamps = expon.rvs(loc=0, scale=tau, size=size)
ts64 = timeStamps.astype(np.uint64)
tsBinner.tsBinner(ts64, timeHgValues)
param = expon.fit(timeStamps)
fit = expon.pdf(x,loc=param[0],scale=param[1])
fit *= size
taulist.append(param[1])
hist,bins = np.histogram(taulistavg, bins=20, range=(15,35))
width = 0.7*(bins[1]-bins[0])
center = (bins[:-1]+bins[1:])/2
plt.step(center, hist, where = 'post', label="averagetime", color='g')
hist,bins = np.histogram(taulist, bins=20, range=(15,35))
width = 0.7*(bins[1]-bins[0])
center = (bins[:-1]+bins[1:])/2
plt.step(center, hist, where = 'post', label="maxlikelihood")
plt.legend()
plt.savefig(inspect.stack()[0][3]+".png")
def printReducedChi(self):
def funcExpon(x, a, b, c, d):
retval = a*np.exp(-b*(x-d)) + c
retval[x < d] = 0
return retval
tau = 25.0
t0 = 40
nBins = 800
size = 10000
taulist = []
for i in range(100):
timeHgValues = np.zeros(nBins, dtype=np.int64)
timeStamps = expon.rvs(loc=0, scale=tau, size=size) + t0
ts64 = timeStamps.astype(np.uint64)
tsBinner.tsBinner(ts64, timeHgValues)
xPoints = []
yPoints = []
for x in range(nBins):
y = timeHgValues[x]
if y > 2:
xPoints.append(x)
yPoints.append(y)
bGuess = 1/(np.mean(timeStamps))
aGuess = bGuess*(size)
cGuess = 0
dGuess = t0
pGuess = [aGuess, bGuess, cGuess, dGuess]
xArray = np.array(xPoints) #must be in array for not list to use curve_fit
yArray = np.array(yPoints)
ySigma = (yArray ** 1/2)
popt, pcov = curve_fit(funcExpon, xArray, yArray, p0=pGuess, sigma=ySigma)
xPlot = np.linspace(xArray[0], xArray[-1], 1000)
yPlot = funcExpon(xPlot, *popt)
yPlotAtPoints = funcExpon(xArray, *popt)
chi_2 = sum(((yArray-yPlotAtPoints)**2)/ySigma)
red_chi_2 = chi_2/(len(yArray)-len(popt))
print red_chi_2
def testExponchisquared(self):
def funcExpon(x, a, b, c, d):
retval = a*np.exp(-b*(x-d)) + c
retval[x < d] = 0
return retval
tau = 25.0
t0 = 40
nBins = 800
size = 10000
taulist = []
for i in range(100):
timeHgValues = np.zeros(nBins, dtype=np.int64)
timeStamps = expon.rvs(loc=0, scale=tau, size=size) + t0
ts64 = timeStamps.astype(np.uint64)
tsBinner.tsBinner(ts64, timeHgValues)
xPoints = []
yPoints = []
for x in range(nBins):
y = timeHgValues[x]
if y > 2:
xPoints.append(x)
yPoints.append(y)
bGuess = 1/(np.mean(timeStamps))
aGuess = bGuess*(size)
cGuess = 0
dGuess = t0
pGuess = [aGuess, bGuess, cGuess, dGuess]
xArray = np.array(xPoints) #must be in array for not list to use curve_fit
yArray = np.array(yPoints)
ySigma = (yArray ** 1/2)
popt, pcov = curve_fit(funcExpon, xArray, yArray, sigma=ySigma, p0=pGuess)
taulist.append(1/popt[1]) #B is 1/tau
if (i == 0):
plt.clf()
plt.plot(xArray, yArray, 'ro')
xPlot = np.linspace(xArray[0], xArray[-1], 1000)
yPlot = funcExpon(xPlot, *popt)
plt.plot(xPlot, yPlot, color='g')
plt.savefig("junk.png") #first iteration
print "popt=", popt
xPlot = np.linspace(xArray[0], xArray[-1], 1000)
yPlot = funcExpon(xPlot, *popt)
plt.clf()
plt.plot(xArray, yArray, 'ro')
plt.plot(xPlot, yPlot, color='g')
#plt.errorbar(yerr, color='g', label='error')
plt.title(inspect.stack()[0][3])
plt.savefig(inspect.stack()[0][3]+".png") #saves plot
plt.clf()
hist,bins = np.histogram(taulist, bins=20)
width = 0.7*(bins[1]-bins[0])
center = (bins[:-1]+bins[1:])/2
plt.step(center, hist, where = 'post', color='g')
plt.title(inspect.stack()[0][3])
plt.savefig("chisquaredhistogram"+".png") #saves histogram
def fitExpon(self, t0, t1):
"""
Fit an exponential to all photons from time t0 to time t1
t0 and t1 are in ticks, 1e6 ticks per second
return a dictionary of: timeStamps,fitParams,chi2
"""
firstSec = int(t0/1e6) # in seconds
integrationTime = 1+int((t1-t0)/1e6) # in seconds
nBins = integrationTime*1e6 # number of microseconds; one bin per microsecond
timeHgValues = np.zeros(nBins, dtype=np.int64)
print "firstSec=",firstSec," integrationTime=",integrationTime
for iRow in range(self.file.nRow):
for iCol in range(self.file.nCol):
timedPacketList = self.file.getTimedPacketList(
iRow, iCol, firstSec=firstSec,
integrationTime=integrationTime)
timeStamps = timedPacketList['timestamps']
if (len(timeStamps) > 0):
# covert the time values to microseconds, and
# make it the type np.uint64
# round per Matt S. suggestion 2013-07-09
#ts64 = (timeStamps).astype(np.uint64)
ts32round = np.round(timeStamps).astype(np.uint32)
tsBinner.tsBinner(ts32round, timeHgValues)
temp = 1e6*(timeStamps-firstSec)
for i in range(len(timeStamps)):
ts32 = ((timeStamps-firstSec)*1e6).astype(np.uint32)
# add these timestamps to the histogram timeHgValues
remain0 = int(t0%1e6)
remain1 = int(t1%1e6)
timeHgValues = timeHgValues[remain0:remain1]
x = np.arange(len(timeHgValues))
y = timeHgValues
xArray = np.arange(0, dtype=np.int64)
yArray = np.arange(0, dtype=np.int64)
for i in range(len(x)):
if y[i] > 2:
xArray = np.append(xArray,i)
yArray = np.append(yArray,y[i])
ySigma = np.sqrt(yArray)
mean = (x*y).sum()/float(y.sum())
bExponGuess = 1/mean
aExponGuess = bExponGuess*timeHgValues.sum()
cExponGuess = 0
dExponGuess = 0
pExponGuess = [aExponGuess, bExponGuess, cExponGuess, dExponGuess]
bGaussGuess = mean
avgx2 = (x*x*y).sum()/float(y.sum())
cGaussGuess = np.sqrt(avgx2-bGaussGuess*bGaussGuess)
aGaussGuess = (timeHgValues.sum()/(cGaussGuess*np.sqrt(2*np.pi)))
pGaussGuess = [aGaussGuess, bGaussGuess, cGaussGuess]
xLimit = [bGaussGuess-4*cGaussGuess, bGaussGuess+4*cGaussGuess]
retval = {'timeHgValues':timeHgValues, 'pExponFit':pExponGuess,
'pGaussFit':pGaussGuess, 'xLimit':xLimit,
'cGaussGuess':cGaussGuess, 'timeStamps':timeStamps}
return retval
i1=interval[1]
intervalTime = i1-i0
dt = intervalTime/2
beginTime = max(0,i0-0.000200)
endTime = beginTime + 0.001
integrationTime = endTime-beginTime
nBins = int(np.round(obsFile.ticksPerSec*(endTime-beginTime)+1))
timeHgValues = np.zeros(nBins, dtype=np.int64)
ymax = sys.float_info.max/100.0
for iRow in range(obsFile.nRow):
for iCol in range(obsFile.nCol):
gtpl = obsFile.getTimedPacketList(iRow,iCol,
beginTime,integrationTime)
ts = (gtpl['timestamps'] - beginTime)*obsFile.ticksPerSec
ts64 = np.round(ts).astype(np.uint64)
tsBinner.tsBinner(ts64, timeHgValues)
plt.clf()
plt.plot(timeHgValues)
x0 = (i0-beginTime)*obsFile.ticksPerSec
x1 = (i1-beginTime)*obsFile.ticksPerSec
plt.fill_between((x0,x1),(0,0), (ymax,ymax), alpha=0.2, color='red')
plt.yscale("symlog",linthreshy=0.9)
plt.xlim(0,1000)
#plt.ylim(-0.1,timeHgValues.max())
plt.ylim(-0.1,300)
tick0 = int(np.round(i0*obsFile.ticksPerSec))
plotfn = "cp-%05d-%s-%s-%s-%09d"%(timeHgValues.sum(),run,obsDate,seq,tick0)
plt.title(plotfn)
plt.savefig(plotfn+".png")
print "plotfn=",plotfn
