def testLoadBeammap():
'''
Test if a remapped beammap is actually remapped
'''
#open an obs file from PAL2012,the sky file for hr9087
#we'll use the embedded beammap file, which has some misplaced pixels
run = 'PAL2012'
date = '20121210'
obsTimestamp = '20121211-051650'
obsFN = FileName(run=run,date=date,tstamp=obsTimestamp)
obsFileName = obsFN.obs()
obs = ObsFile(obsFileName)
beammapFileName = obsFN.beammap()
#load the PixelMap for PAL2012 to know which pixels should be remapped
pixMap = remapPixels.PixelMap(obsFN.pixRemap())
pixMapSourceList,pixMapDestList = pixMap.getRemappedPix()
#load the corrected beammap into the obs
obs.loadBeammapFile(beammapFileName)
#check that each pixel that should be moved is moved
#by comparing the embedded beammap and the loaded corrected one
for source,dest in zip(pixMapSourceList,pixMapDestList):
assert obs.beamImage[dest] == obs.file.root.beammap.beamimage[source]
obs.file.close()
def main():
#load the current PAL2012 beammap from an obs file from that run,
#a sky file for hr9087
run = 'PAL2012'
date = '20121210'
obsTimestamp = '20121211-051650'
obsFN = FileName(run=run,date=date,tstamp=obsTimestamp)
obsFileName = obsFN.obs()
obs = ObsFile(obsFileName)
#Load an existing definition of which pixels to remap where
pixMap = remapPixels.PixelMap(obsFN.pixRemap())
pixMapSourceList,pixMapDestList = pixMap.getRemappedPix()
#create a new h5 file and copy the existing beammap node hierarchy into it
newBeammapPath = 'beamimage_PAL2012_corrected.h5'
newBeammapFile = tables.openFile(newBeammapPath, mode='w')
newBeammapFile.copyNode(obs.file.root.beammap, newparent=newBeammapFile.root, newname='beammap', recursive=True)
newBeammap = newBeammapFile.root.beammap.beamimage
#for each array in the h5, move the pixels according to pixMap
newBeammap[:,:] = pixMap.remapArray(obs.beamImage)
newBeammapFile.root.beammap.resfreq[:,:] = pixMap.remapArray(newBeammapFile.root.beammap.resfreq.read())
newBeammapFile.root.beammap.atten[:,:] = pixMap.remapArray(newBeammapFile.root.beammap.atten.read())
newBeammapFile.flush()
newBeammapFile.close()
obs.file.close()
def populateFlatCals(runPath,lookupPath=None):
if lookupPath is None:
lookupPath = os.environ['MKID_CAL_LOOKUP']
if not os.path.exists(path):
writeNewCalLookupFile(path)
lookup = CalLookupFile(path,mode='a')
for root,dirs,files in os.walk(runPath):
obsFilenames = fnmatch.filter(files,'obs*.h5')
for obsFilename in obsFilenames:
obsPath = os.path.join(root,obsFilename)
try:
obs = util.ObsFile.ObsFile(obsPath)
fn = FileName(obsFile=obsPath)
params = {}
params['obs_run'],params['obs_date'],params['obs_tstamp'] = fn.getComponents()
params['flatSoln_run'] = params['obs_run']
if params['obs_date'] in ['20140923','20140924','20141020','20141021','20141022']:
params['flatSoln_date'] = params['obs_date']
else:
if params['obs_date'] == '20140925':
params['flatSoln_date'] = '20140924'
print params
lookup.updateObs(newParams=params)
except:
pass
def makemovie1(self):
run = self.s['run']
sundownDate = self.s['sundownDate']
obsDate = self.s['obsDate']
stride = int(self.s['stride'])
seq5 = self.s['seq5'].split()
for seq in seq5:
inFile = open("cosmicTimeList-%s.pkl"%(seq),"rb")
cosmicTimeList = pickle.load(inFile)
binContents = pickle.load(inFile)
cfn = "cosmicMask-%s.h5"%seq
intervals = ObsFile.readCosmicIntervalFromFile(cfn)
for interval in intervals:
print "interval=",interval
fn = FileName(run, sundownDate,obsDate+"-"+seq)
obsFile = ObsFile(fn.obs())
obsFile.loadTimeAdjustmentFile(fn.timeAdjustments())
i0=interval[0]
i1=interval[1]
intervalTime = i1-i0
dt = intervalTime/2
beginTime = max(0,i0-0.000200)
endTime = beginTime + 0.001
integrationTime = endTime-beginTime
nBins = int(np.round(obsFile.ticksPerSec*(endTime-beginTime)+1))
timeHgValues = np.zeros(nBins, dtype=np.int64)
ymax = sys.float_info.max/100.0
for iRow in range(obsFile.nRow):
for iCol in range(obsFile.nCol):
gtpl = obsFile.getTimedPacketList(iRow,iCol,
beginTime,integrationTime)
ts = (gtpl['timestamps'] - beginTime)*obsFile.ticksPerSec
ts64 = np.round(ts).astype(np.uint64)
tsBinner.tsBinner(ts64, timeHgValues)
plt.clf()
plt.plot(timeHgValues, label="data")
x0 = (i0-beginTime)*obsFile.ticksPerSec
x1 = (i1-beginTime)*obsFile.ticksPerSec
plt.fill_between((x0,x1),(0,0), (ymax,ymax), alpha=0.2, color='red')
plt.yscale("symlog",linthreshy=0.9)
plt.xlim(0,1000)
plt.ylim(-0.1,300)
tick0 = int(np.round(i0*obsFile.ticksPerSec))
plotfn = "cp-%05d-%s-%s-%s-%09d"%(timeHgValues.sum(),run,obsDate,seq,tick0)
plt.title(plotfn)
plt.legend()
plt.savefig(plotfn+".png")
plt.xlabel("nSigma=%d stride=%d threshold=%d"%(int(self.s['nSigma']),int(self.s['stride']),int(self.s['threshold'])))
print "plotfn=",plotfn
os.system("convert -delay 0 `ls -r cp*png` cp.gif")
def testWritePhotonList(outputFileName=None,firstSec=0,integrationTime=-1,doPixRemap=True):
'''
Test run of obsFile.writePhotonList. fileName can be used
to specify the output file name. If not specified, default
name/location is used.
Now includes test for pixel remapping....
'''
#Details of example obs file to run test on.
# run = 'PAL2012'
# date = '20121207'
# tstamp = '20121208-074649'
# calTstamp='20121208-070505'
# fluxTstamp='20121208-133002'
run = 'PAL2012'
date = '20121208'
tstamp = '20121209-120530'
centroidTstamp = '20121209-120530'
calTstamp='20121209-131132'
fluxTstamp='20121209-020416'
flatTstamp='20121209-021036'
if doPixRemap==True:
pixRemapFileName = FileName(run=run).pixRemap()
else:
pixRemapFileName = None
#Load up the obs file
obsFileName = FileName(run=run, date=date, tstamp=tstamp)
obsFile = ObsFile(obsFileName.obs())
#Load up associated calibrations
obsFile.loadWvlCalFile(FileName(run=run,date=date,tstamp=calTstamp).calSoln())
obsFile.loadFlatCalFile(FileName(run=run,date=date,tstamp=flatTstamp).flatSoln())
obsFile.loadFluxCalFile(FileName(run=run,date=date,tstamp=fluxTstamp).fluxSoln())
obsFile.loadTimeAdjustmentFile(FileName(run=run,date=date,tstamp=tstamp).timeAdjustments())
obsFile.loadHotPixCalFile(FileName(run=run,date=date,tstamp=tstamp).timeMask())
obsFile.loadCentroidListFile(FileName(run=run,date=date,tstamp=tstamp).centroidList())
#And write out the results....
obsFile.writePhotonList(outputFileName,firstSec,integrationTime,
pixRemapFileName=pixRemapFileName)
def populateTimeMasks(runPath,lookupPath=None):
if lookupPath is None:
lookupPath = os.environ['MKID_CAL_LOOKUP']
if not os.path.exists(path):
writeNewCalLookupFile(path)
lookup = CalLookupFile(path,mode='a')
for root,dirs,files in os.walk(runPath):
obsFilenames = fnmatch.filter(files,'obs*.h5')
for obsFilename in obsFilenames:
obsPath = os.path.join(root,obsFilename)
try:
obs = util.ObsFile.ObsFile(obsPath)
fn = FileName(obsFile=obsPath)
params = {}
params['obs_run'],params['obs_date'],params['obs_tstamp'] = fn.getComponents()
params['timeMask_run'],params['timeMask_date'],params['timeMask_tstamp'] = fn.getComponents()
print params
lookup.updateObs(newParams=params)
except:
pass
def populateWaveCals(runPath,lookupPath=None):
if lookupPath is None:
lookupPath = os.environ['MKID_CAL_LOOKUP']
if not os.path.exists(path):
writeNewCalLookupFile(path)
lookup = CalLookupFile(path,mode='a')
for root,dirs,files in os.walk(runPath):
obsFilenames = fnmatch.filter(files,'obs*.h5')
for obsFilename in obsFilenames:
obsPath = os.path.join(root,obsFilename)
try:
obs = util.ObsFile.ObsFile(obsPath)
fn = FileName(obsFile=obsPath)
params = {}
params['obs_run'],params['obs_date'],params['obs_tstamp'] = fn.getComponents()
params['waveSoln_run'] = params['obs_run']
obs.loadBestWvlCalFile()
waveCalPath = obs.wvlCalFileName
waveCalFilename = os.path.basename(waveCalPath)
if waveCalFilename.startswith('master'):
params['waveSoln_isMasterCal'] = True
tstamp = waveCalFilename.split('_')[1].split('.')[0]
params['waveSoln_tstamp'] = tstamp
params['waveSoln_date'] = ''
else:
params['waveSoln_isMasterCal'] = False
tstamp = waveCalFilename.split('_')[1].split('.')[0]
params['waveSoln_tstamp'] = tstamp
dirs = os.path.dirname(os.path.normpath(waveCalPath)).split(os.sep)
params['waveSoln_date'] = dirs[-1]
print params
lookup.updateObs(newParams=params)
except:
pass
npList = []
sigList = []
run = 'PAL2012'
sundownDate = '20121211'
obsDate = '20121212'
for seq in seq5:
inFile = open("cosmicTimeList-%s.pkl"%(seq),"rb")
cosmicTimeList = pickle.load(inFile)
binContents = pickle.load(inFile)
cfn = "cosmicMax-%s.h5"%seq
intervals = ObsFile.readCosmicIntervalFromFile(cfn)
for interval in intervals:
print "interval=",interval
fn = FileName(run, sundownDate,obsDate+"-"+seq)
obsFile = ObsFile(fn.obs())
obsFile.loadTimeAdjustmentFile(fn.timeAdjustments())
i0=interval[0]
i1=interval[1]
intervalTime = i1-i0
dt = intervalTime/2
beginTime = max(0,i0-0.000200)
endTime = beginTime + 0.001
integrationTime = endTime-beginTime
nBins = int(np.round(obsFile.ticksPerSec*(endTime-beginTime)+1))
timeHgValues = np.zeros(nBins, dtype=np.int64)
ymax = sys.float_info.max/100.0
for iRow in range(obsFile.nRow):
for iCol in range(obsFile.nCol):
gtpl = obsFile.getTimedPacketList(iRow,iCol,
def __init__(self,paramFile,plots=False,verbose=False):
"""
Opens flux file, prepares standard spectrum, and calculates flux factors for the file.
Method is provided in param file. If 'relative' is selected, an obs file with standard star defocused over
the entire array is expected, with accompanying sky file to do sky subtraction.
If any other method is provided, 'absolute' will be done by default, wherein a point source is assumed
to be present. The obs file is then broken into spectral frames with photometry (psf or aper) performed
on each frame to generate the ARCONS observed spectrum.
"""
self.verbose=verbose
self.plots = plots
self.params = readDict()
self.params.read_from_file(paramFile)
run = self.params['run']
sunsetDate = self.params['fluxSunsetLocalDate']
self.fluxTstamp = self.params['fluxTimestamp']
skyTstamp = self.params['skyTimestamp']
wvlSunsetDate = self.params['wvlCalSunsetLocalDate']
wvlTimestamp = self.params['wvlCalTimestamp']
flatCalFileName = self.params['flatCalFileName']
needTimeAdjust = self.params['needTimeAdjust']
self.deadtime = float(self.params['deadtime']) #from firmware pulse detection
self.timeSpacingCut = self.params['timeSpacingCut']
bLoadBeammap = self.params.get('bLoadBeammap',False)
self.method = self.params['method']
self.objectName = self.params['object']
self.r = float(self.params['energyResolution'])
self.photometry = self.params['photometry']
self.centroidRow = self.params['centroidRow']
self.centroidCol = self.params['centroidCol']
self.aperture = self.params['apertureRad']
self.annulusInner = self.params['annulusInner']
self.annulusOuter = self.params['annulusOuter']
self.collectingArea = self.params['collectingArea']
self.startTime = self.params['startTime']
self.intTime = self.params['integrationTime']
fluxFN = FileName(run=run,date=sunsetDate,tstamp=self.fluxTstamp)
self.fluxFileName = fluxFN.obs()
self.fluxFile = ObsFile(self.fluxFileName)
if self.plots:
self.plotSavePath = os.environ['MKID_PROC_PATH']+os.sep+'fluxCalSolnFiles'+os.sep+run+os.sep+sunsetDate+os.sep+'plots'+os.sep
if not os.path.exists(self.plotSavePath): os.mkdir(self.plotSavePath)
if self.verbose: print "Created directory %s"%self.plotSavePath
obsFNs = [fluxFN]
self.obsList = [self.fluxFile]
if self.startTime in ['',None]: self.startTime=0
if self.intTime in ['',None]: self.intTime=-1
if self.method=="relative":
try:
print "performing Relative Flux Calibration"
skyFN = FileName(run=run,date=sunsetDate,tstamp=skyTstamp)
self.skyFileName = skyFN.obs()
self.skyFile = ObsFile(self.skyFileName)
obsFNs.append(skyFN)
self.obsList.append(self.skyFile)
except:
print "For relative flux calibration a sky file must be provided in param file"
self.__del__()
else:
self.method='absolute'
print "performing Absolute Flux Calibration"
if self.photometry not in ['aperture','PSF']: self.photometry='PSF' #default to PSF fitting if no valid photometry selected
timeMaskFileNames = [fn.timeMask() for fn in obsFNs]
timeAdjustFileName = FileName(run=run).timeAdjustments()
#make filename for output fluxCalSoln file
self.fluxCalFileName = FileName(run=run,date=sunsetDate,tstamp=self.fluxTstamp).fluxSoln()
print "Creating flux cal: %s"%self.fluxCalFileName
if wvlSunsetDate != '':
wvlCalFileName = FileName(run=run,date=wvlSunsetDate,tstamp=wvlTimestamp).calSoln()
if flatCalFileName =='':
flatCalFileName=FileName(obsFile=self.fluxFile).flatSoln()
#load cal files for flux file and, if necessary, sky file
for iObs,obs in enumerate(self.obsList):
if bLoadBeammap:
print 'loading beammap',os.environ['MKID_BEAMMAP_PATH']
obs.loadBeammapFile(os.environ['MKID_BEAMMAP_PATH'])
if wvlSunsetDate != '':
obs.loadWvlCalFile(wvlCalFileName)
else:
obs.loadBestWvlCalFile()
obs.loadFlatCalFile(flatCalFileName)
obs.setWvlCutoffs(-1,-1)
if needTimeAdjust:
obs.loadTimeAdjustmentFile(timeAdjustFileName)
timeMaskFileName = timeMaskFileNames[iObs]
print timeMaskFileName
if not os.path.exists(timeMaskFileName):
print 'Running hotpix for ',obs
hp.findHotPixels(obsFile=obs,outputFileName=timeMaskFileName,fwhm=np.inf,useLocalStdDev=True)
print "Flux cal/sky file pixel mask saved to %s"%(timeMaskFileName)
obs.loadHotPixCalFile(timeMaskFileName)
if self.verbose: print "Loaded hot pixel file %s"%timeMaskFileName
#get flat cal binning information since flux cal will need to match it
self.wvlBinEdges = self.fluxFile.flatCalFile.root.flatcal.wavelengthBins.read()
self.nWvlBins = self.fluxFile.flatWeights.shape[2]
self.binWidths = np.empty((self.nWvlBins),dtype=float)
self.binCenters = np.empty((self.nWvlBins),dtype=float)
for i in xrange(self.nWvlBins):
self.binWidths[i] = self.wvlBinEdges[i+1]-self.wvlBinEdges[i]
self.binCenters[i] = (self.wvlBinEdges[i]+(self.binWidths[i]/2.0))
if self.method=='relative':
print "Extracting ARCONS flux and sky spectra"
self.loadRelativeSpectrum()
print "Flux Spectrum loaded"
self.loadSkySpectrum()
print "Sky Spectrum loaded"
elif self.method=='absolute':
print "Extracting ARCONS point source spectrum"
self.loadAbsoluteSpectrum()
print "Loading standard spectrum"
try:
self.loadStdSpectrum(self.objectName)
except KeyError:
print "Invalid spectrum object name"
self.__del__()
sys.exit()
print "Generating sensitivity curve"
self.calculateFactors()
print "Sensitivity Curve calculated"
print "Writing fluxCal to file %s"%self.fluxCalFileName
self.writeFactors(self.fluxCalFileName)
if self.plots: self.makePlots()
print "Done"
from util.ObsFile import ObsFile from util.FileName import FileName run = "PAL2014" date = "20141022" timeStamp = '20141023-033821' fn = FileName(run,date,timeStamp) of = ObsFile(fn.obs()) of.loadBeammapFile(fn.beammap()) of.loadBestWvlCalFile() print "wvlCalFileName=",of.wvlCalFileName fn2 = FileName(run,date,"") of.loadFlatCalFile(fn2.flatSoln()) row = 4 col = 4 firstSec = 72 integrationTime = 1 spec = of.getPixelSpectrum(row,col,firstSec,integrationTime) print "spec=",spec del of
def plot_pixel_phaseHeights(self,row=None,col=None):
self.row = row
self.col = col
self.laserCalFile = self.parent.wave_obs
dataDict=self.laserCalFile.getTimedPacketList(self.row,self.col,timeSpacingCut=self.parent.params['danicas_cut'])
peakHeights=np.asarray(dataDict['peakHeights'])*1.0
## less than 'min_amp' per second average count rate
if dataDict['effIntTime']==0.0 or len(peakHeights)<=(dataDict['effIntTime']*self.parent.params['min_count_rate']):
print 'Not enough data for phase histogram'
raise ValueError
baselines=np.asarray(dataDict['baselines'])*1.0
peakHeights-=baselines
biggest_photon = int(min(peakHeights))
n_inbin,phase_bins=np.histogram(peakHeights,bins=np.abs(biggest_photon),range=(biggest_photon,0))
phase_bins=(phase_bins+(phase_bins[1]-phase_bins[0])/2.0)[:-1]
self.n_inbin = n_inbin
self.phase_bins = phase_bins
try:
last_ind = np.where(n_inbin>self.parent.params['min_amp'])[0][-1]
except IndexError:
last_ind=len(n_inbin)-1
## Cut out all the zeros on the right
self.axes.plot(phase_bins,n_inbin, 'b.',label="data")
self.axes.set_xlim(phase_bins[(np.where(n_inbin >= 3))[0][0]],phase_bins[last_ind])
n_inbin = n_inbin[:last_ind]
phase_bins = phase_bins[:last_ind]
#Check if this pixel is in drift file
waveFN = FileName(obsFile=self.laserCalFile)
driftFile=tables.openFile(waveFN.calDriftInfo(),mode='r')
drift_row = driftFile.root.params_drift.driftparams.cols.pixelrow[:]
drift_col = driftFile.root.params_drift.driftparams.cols.pixelcol[:]
args=(np.where(np.multiply(drift_row==self.row, drift_col==self.col)))[0]
if len(args==1):
#pixel has a solution!
fit_params = driftFile.root.params_drift.driftparams.cols.gaussparams[args[0]]
#print fit_params
model = driftFile.root.params_drift.driftparams.attrs.model_type
model_list = {
'parabola': parabola,
'gaussian': gaussian,
'fourgaussian': fourgaussian,
'threegaussian_exp': threegaussian_exp,
'threegaussian_exppow': threegaussian_exppow,
'threegaussian_moyal': threegaussian_moyal,
'threegaussian_power': threegaussian_power,
'threegaussian_lorentzian': threegaussian_lorentzian}
modelArr=model_list[model](p=fit_params,x=phase_bins,return_models=True)
fullModel=np.zeros(len(n_inbin))
for model_part in modelArr:
fullModel = fullModel+model_part
self.axes.plot(phase_bins,model_part, 'k--')
self.axes.plot(phase_bins,fullModel, 'r-',label="model")
self.axes.legend()
self.axes.set_xlabel("phase")
self.axes.set_ylabel("counts")
self.axes.set_title('('+str(self.col)+', '+str(self.row)+') --> '+(self.laserCalFile.beamImage[self.row,self.col]).rsplit('/',1)[0])
#res_blue = (self.drift.r_blue[self.row,self.col,self.closest_time_ind])
#self.axes.text(0.15,0.95,"R ~ "+str(round(res_blue,3)),verticalalignment='center', horizontalalignment='center', transform=self.axes.transAxes)
#print res_blue
driftFile.close()
return True
def plot_pixel_soln(self,row,col,list_of_fits):
#self.fig, axarr = plt.subplots(2, sharex=True)
#self.axes=axarr[0]
#axes2=axarr[1]
self.fig.delaxes(self.axes)
gs = gridspec.GridSpec(2,1,height_ratios=[4,1])
self.fig.subplots_adjust(hspace=0.,wspace=0.)
self.axes=self.fig.add_subplot(gs[0])
QE_wavelengths = self.parent.QE_data[:,0]
QE_energies = [self.parent.params['h'] * self.parent.params['c'] / (x*10. * self.parent.params['ang2m']) for x in QE_wavelengths]
QE_x_offsets = np.asarray(list_of_fits)[:,1]
waveFN = FileName(obsFile=self.parent.wave_obs)
calFile=tables.openFile(waveFN.calSoln(),mode='r')
pixRow=calFile.root.wavecal.calsoln.cols.pixelrow[:]
pixCol=calFile.root.wavecal.calsoln.cols.pixelcol[:]
cal_ind = np.where((pixRow==row) * (pixCol==col))[0][0]
polyfit=calFile.root.wavecal.calsoln.cols.polyfit[cal_ind]
if np.all(polyfit==-1) and np.all(QE_x_offsets==0):
raise ValueError
calFile.close()
driftFile=tables.openFile(waveFN.calDriftInfo(),mode='r')
drift_row = driftFile.root.params_drift.driftparams.cols.pixelrow[:]
drift_col = driftFile.root.params_drift.driftparams.cols.pixelcol[:]
try:
drift_ind=(np.where(np.multiply(drift_row==row, drift_col==col)))[0][0]
fit_params = driftFile.root.params_drift.driftparams.cols.gaussparams[drift_ind]
cal_x_offsets = fit_params[[1,4,7]]
cal_x_offsets[1]+=cal_x_offsets[0]
cal_x_offsets[2]+=cal_x_offsets[1]
if fit_params[8]==0.:
cal_x_offsets[2]=0.
cal_wavelengths = [self.parent.params['bluelambda'],self.parent.params['redlambda'], self.parent.params['irlambda']] #Angstroms
cal_energies = [self.parent.params['h'] * self.parent.params['c'] / (x * self.parent.params['ang2m']) for x in cal_wavelengths] #eV
except IndexError:
cal_wavelengths = [self.parent.params['bluelambda'],self.parent.params['redlambda'], self.parent.params['irlambda']] #Angstroms
cal_energies = [self.parent.params['h'] * self.parent.params['c'] / (x * self.parent.params['ang2m']) for x in cal_wavelengths] #eV
cal_x_offsets = [0,0,0]
driftFile.close()
xArr = np.arange(-5000.,0.,0.1)
if not np.all(polyfit==-1):
modelArr=(parabola(p=polyfit,x=xArr,return_models=True))[0]
self.axes.plot(xArr,modelArr,'k-')
self.axes.plot(cal_x_offsets,cal_energies,'ko', label='Laser Cal')
self.axes.plot(QE_x_offsets,QE_energies,'gx',label='QE wavelength fits')
#Do linear fit of QE data to see if x-intercept is at 0
QE_x_offsets = np.asarray(QE_x_offsets)
QE_energies = np.asarray(QE_energies)
energies = (QE_energies[np.where(QE_x_offsets<0)])
sort_ind = np.argsort(energies)
energies = energies[sort_ind]
x_offsets = QE_x_offsets[np.where(QE_x_offsets<0)]
x_offsets = x_offsets[sort_ind]
if len(x_offsets)>1:
parameter_guess = [0.0,(energies[0]-energies[-1])*1.0/(x_offsets[0]-x_offsets[-1]), 0.]
parameter_lowerlimit=[None,None,0.]
parameter_upperlimit=[None,None,0.]
parameter_fit, redchi2gauss2, mpperr = fitData(x_offsets,energies,parameter_guess,parameter_lowerlimit,parameter_upperlimit,model='parabola')
print 'x_intercept: '+str(-1.*parameter_fit[0]/parameter_fit[1])
else:
print 'x_intercept: Not enough data'
#lin_fit=(parabola(p=parameter_fit,x=xArr,return_models=True))[0]
#self.axes.plot(xArr,lin_fit,'g--')
#Do parabola fit to QE data
if len(x_offsets)>1:
parameter_guess = [0.0,(energies[0]-energies[-1])*1.0/(x_offsets[0]-x_offsets[-1]), 0.]
parameter_lowerlimit=[None,None,None]
parameter_upperlimit=[None,None,None]
parameter_fit, redchi2gauss2, mpperr = fitData(x_offsets,energies,parameter_guess,parameter_lowerlimit,parameter_upperlimit,model='parabola')
print polyfit,' --> ',parameter_fit
parab_fit=(parabola(p=parameter_fit,x=xArr,return_models=True))[0]
self.axes.plot(xArr,parab_fit,'g--')
#Plot Residuals
if not np.all(polyfit==-1):
cal_QE=(parabola(p=polyfit,x=x_offsets,return_models=True))[0]
axes2=self.fig.add_subplot(gs[1],sharex=self.axes)
#axes2 = self.fig.add_axes((.1,.1,.8,.2))
axes2.plot(x_offsets,cal_QE-energies,'.')
axes2.axhline(0,color='black')
#Add labels etc
self.axes.set_ylabel('Energy (eV)')
self.axes.set_xlim(min(min(cal_x_offsets),min(QE_x_offsets))-200.,0.)
self.axes.set_ylim(0.,5.)
if not np.all(polyfit==-1):
axes2.set_xlabel('Phase (ADC/DAC units)')
axes2.set_ylabel('Calibration residuals (eV)')
axes2.set_ylim(-3.,3.)
#self.axes.set_xticklabels(self.axes.get_xticklabels(),visible=False)
plt.setp(self.axes.get_xticklabels(),visible=False)
import matplotlib.pyplot as plt
from util.ObsFile import ObsFile
import os
import hotpix.hotPixels as hp
from util.FileName import FileName
from util.popup import plotArray
import astrometry.CentroidCalc as cc
from photonlist.photlist import writePhotonList
from headers.ArconsHeaders import PulsarPhotonList
if __name__=='__main__':
run = 'PAL2014'
date = '20140924'
tstamp = '20140925-112538'
obsFN = FileName(run=run,date=date,tstamp=tstamp)
obsPath = obsFN.obs()
flatPath = FileName(run=run,date=date).flatSoln()
hotPath = obsFN.timeMask()
centroidPath = obsFN.centroidList()
fluxPath = FileName(run='PAL2012',date='20121211',tstamp='absolute_021727').fluxSoln()
obs = ObsFile(obsPath)
if not os.path.exists(hotPath):
hp.findHotPixels(obsPath,hotPath)
obs.loadHotPixCalFile(hotPath,switchOnMask=False)
obs.loadBestWvlCalFile()
obs.loadFlatCalFile(flatPath)
obs.loadFluxCalFile(fluxPath)
obs.loadCentroidListFile(centroidPath)
ffn = flatCalFilenames[iSeq]
sunsetDate = sunsetDates[iSeq]
for position in ObjPosFile:
if iSeq + 1 == position[0]:
print "finding aperture and sky masks"
guessX = position[1]
guessY = position[2]
apertureMask = circleAperture(guessX, guessY, radius1)
bigMask = circleAperture(guessX, guessY, radius2)
skyMask = bigMask - apertureMask
# print skyMask[0:20][5:25]
for i, ts in enumerate(timestampList):
print "loading", ts
obsFn = FileName(run=run, date=sunsetDate, tstamp=ts).obs()
ob = ObsFile(obsFn)
ob.loadTimeAdjustmentFile(FileName(run=run).timeAdjustments())
index1 = obsFn.find("_")
hotPixFn = "/Scratch/timeMasks/timeMask" + obsFn[index1:]
if not os.path.exists(hotPixFn):
hp.findHotPixels(obsFn, hotPixFn)
print "Flux file pixel mask saved to %s" % (hotPixFn)
ob.loadHotPixCalFile(hotPixFn, switchOnMask=True)
ob.loadWvlCalFile(wfn)
ob.loadFlatCalFile(ffn)
ob.setWvlCutoffs(wvlLowerCutoff, wvlUpperCutoff)
bad_solution_mask = np.zeros((46, 44))
bad_count = 0
for y in range(46):
def testWritePhotonList(outputFileName=None,firstSec=0,integrationTime=-1,doPixRemap=False):
'''
Test run of obsFile.writePhotonList. fileName can be used
to specify the output file name. If not specified, default
name/location is used.
Now includes test for pixel remapping....
'''
#Details of example obs file to run test on.
# run = 'PAL2012'
# date = '20121207'
# tstamp = '20121208-074649'
# calTstamp='20121208-070505'
# fluxTstamp='20121208-133002'
run = 'PAL2013'
date = '20131209'
tstamp = '20131209-115553'
centroidTstamp = tstamp
calTstamp='20131209-132225'
fluxTstamp='ones'
flatTstamp='20131209'
if doPixRemap==True:
pixRemapFileName = FileName(run=run).pixRemap()
else:
pixRemapFileName = None
#Load up the obs file
obsFileName = FileName(run=run, date=date, tstamp=tstamp)
obsFile = ObsFile(obsFileName.obs())
obsFile.loadWvlCalFile(FileName(run=run,date=date,tstamp=calTstamp).calSoln())
obsFile.loadFlatCalFile(FileName(run=run,date=date,tstamp=flatTstamp).flatSoln())
obsFile.loadFluxCalFile(FileName(run=run,date=date,tstamp=fluxTstamp).fluxSoln())
HotPixFile = FileName(run=run,date=date,tstamp=tstamp).timeMask()
if not os.path.exists(HotPixFile): #check if hot pix file already exists
hp.findHotPixels(obsFileName.obs(),HotPixFile)
print "Flux file pixel mask saved to %s"%(HotPixFile)
obsFile.loadHotPixCalFile(HotPixFile)
print "Hot pixel mask loaded %s"%(HotPixFile)
#hz21 location alpha(2000) = 12h 13m 56.42s , delta(2000) = +32d 56' 30.8''
centroid_RA = '12:13:56.42'
centroid_DEC = '32:56:30.8'
CentFile = FileName(run=run,date=date,tstamp=tstamp).centroidList()
#print "checking centroid file", CentFile
#if not os.path.exists(CentFile): #check if cent pix file already exists
cent.centroidCalc(obsFile, centroid_RA, centroid_DEC, outputFileName = CentFile, guessTime=10, integrationTime=10)
# print "Flux file centroid cal saved to %s"%(CentFile)
obsFile.loadCentroidListFile(CentFile)
print "Centroid File loaded %s"%(CentFile)
#Load up associated calibrations
#obsFile.loadTimeAdjustmentFile(FileName(run=run,date=date,tstamp=tstamp).timeAdjustments())
#obsFile.loadHotPixCalFile(FileName(run=run,date=date,tstamp=tstamp).timeMask())
#obsFile.loadCentroidListFile(FileName(run=run,date=date,tstamp=tstamp).centroidList())
#And write out the results....
obsFile.writePhotonList(outputFileName,firstSec,integrationTime,
pixRemapFileName=pixRemapFileName)
#Read the results back in....
#photFile = photList.PhotFile(outputFilename)
print "Wrote photon list file: ", outputFileName
def main():
#open the sky file for hr9087
run = 'PAL2012'
date = '20121210'
wvlCal = '20121211-052230'
obsTimestamp = '20121211-051650'
flatCalDate = '20121211'
flatCalTstamp = '20121212-074700'
obsFN = FileName(run=run,date=date,tstamp=obsTimestamp)
obsFileName = obsFN.obs()
timeMaskFileName = obsFN.timeMask()
wvlFileName = FileName(run=run,date=date,tstamp=wvlCal).calSoln()
flatFileName = FileName(run=run,date=flatCalDate,tstamp=flatCalTstamp).flatSoln()
if not os.path.exists(timeMaskFileName):
print 'Running hotpix for ',obsFileName
hp.findHotPixels(obsFileName,timeMaskFileName)
print "Flux file pixel mask saved to %s"%(timeMaskFileName)
obs = ObsFile(obsFileName)
obs.loadTimeAdjustmentFile(FileName(run='PAL2012').timeAdjustments())
obs.loadWvlCalFile(wvlFileName)
obs.loadFlatCalFile(flatFileName)
obs.loadHotPixCalFile(timeMaskFileName)
#obs.setWvlCutoffs(4000,8000)
#get image before and after flat cal
print 'getting images'
beforeImgDict = obs.getPixelCountImage(weighted=False,fluxWeighted=False,scaleByEffInt=True)
rawCubeDict = obs.getSpectralCube(weighted=False)
rawCube = np.array(rawCubeDict['cube'],dtype=np.double)
effIntTime = rawCubeDict['effIntTime']
maxIntTime = np.max(effIntTime)
#add third dimension for broadcasting
effIntTime3d = np.reshape(effIntTime,np.shape(effIntTime)+(1,))
rawCube *= maxIntTime / effIntTime3d
rawCube[np.isnan(rawCube)] = 0
rawCube[rawCube == np.inf] = 0
beforeImg = np.sum(rawCube,axis=-1)
print 'finished first cube'
flatCubeDict = obs.getSpectralCube(weighted=True)
flatCube = np.array(flatCubeDict['cube'],dtype=np.double)
effIntTime = flatCubeDict['effIntTime']
maxIntTime = np.max(effIntTime)
#add third dimension for broadcasting
effIntTime3d = np.reshape(effIntTime,np.shape(effIntTime)+(1,))
flatCube *= maxIntTime / effIntTime3d
flatCube[np.isnan(flatCube)] = 0
flatCube[flatCube == np.inf] = 0
afterImg = np.sum(flatCube,axis=-1)
plotArray(title='before flatcal',image=beforeImg)
plotArray(title='after flatcal',image=afterImg)
print 'before sdev',np.std(beforeImg[afterImg!=0])
print 'after sdev',np.std(afterImg[afterImg!=0])
np.savez('flatCubeGem.npz',beforeImg=beforeImg,afterImg=afterImg,rawCube=rawCube,flatCube=flatCube)
def main():
run = 'PAL2014'
year = '2014'
initialPath = '/Scratch'
packetMasterLogDir = '/LABTEST/PacketMasterLogs/'
initialPath = os.path.join(initialPath,run)
outPath = FileName(run=run,mkidDataDir='/Scratch/').timeAdjustments()
outFile = tables.openFile(outPath,'w')
timeAdjustGroup = outFile.createGroup('/','timeAdjust','Times to add to timestamps')
firmwareDelayTable = outFile.createTable(timeAdjustGroup,'firmwareDelay',firmwareDelayDescription,'Times to add to all timestamps taken with a firmware bof')
newFirmwareEntry = firmwareDelayTable.row
newFirmwareEntry['firmwareName']='chan_svf_2014_Aug_06_1839.bof'
newFirmwareEntry['firmwareDelay']=-41e-6 #s, subtract 41 us from timestamps
newFirmwareEntry.append()
firmwareDelayTable.flush()
firmwareDelayTable.close()
roachDelayTable = outFile.createTable(timeAdjustGroup,'roachDelays',roachDelaysDescription,'Times to add to each roach\'s timestamps')
for sunsetDatePath in sorted(glob.glob(os.path.join(initialPath,year+'*'))):
sunsetDate = os.path.basename(sunsetDatePath)
for fullObsPath in sorted(glob.glob(os.path.join(sunsetDatePath,'obs*.h5'))):
obsFileName = os.path.basename(fullObsPath)
obsTStamp = obsFileName.split('.')[0].split('_')[1]
print obsFileName
obsFN = FileName(run=run,date=sunsetDate,tstamp=obsTStamp,packetMasterLogDir=packetMasterLogDir)
pmLogFileName = obsFN.packetMasterLog()
try:
ob = ObsFile(fullObsPath)
except:
print 'can\'t open file'
continue
try:
if os.path.getsize(pmLogFileName) <= 0:
continue
except:
print 'can\'t open Packet Master Log ',pmLogFileName
continue
try:
f = open(pmLogFileName,'r')
except:
print 'can\'t open Packet Master Log ',pmLogFileName
continue
lastTstampLines = np.zeros(8)
firstTstampLines = np.zeros(8)
tstampLine = ''
for line in f:
if 'here' in line:
#skip over large log files with debug info
print 'skipping file with "here"'
continue
if line.split(' ')[0] == 'bundle':
tstampLine = line
break
if tstampLine == '':
print 'skipping file without "bundle"'
#didn't find lines with 'bundle' in them
continue
f.seek(0)
for line in f:
if line.split(' ')[0] == 'bundle':
try:
at = float(line.split('at')[1].split()[0])
except:
break
lastTstampLine = at
iRoach = int(line.split('roach')[1].split('took')[0].strip())
lastTstampLines[iRoach] = at
if firstTstampLines[iRoach] == 0:
firstTstampLines[iRoach] = at
packetReceivedUnixTimestamp = float((tstampLine.split('took')[1].split('total')[0].strip()))
firstPacketDelay = packetReceivedUnixTimestamp-int(ob.getFromHeader('unixtime'))
roachSecDelays =np.array(np.floor(lastTstampLines+firstPacketDelay-ob.getFromHeader('exptime')),dtype=np.int)
print 'roach delays',roachSecDelays
if np.all(roachSecDelays >= 0):
newEntry = roachDelayTable.row
newEntry['obsFileName'] = os.path.basename(fullObsPath)
newEntry['roachDelays'] = roachSecDelays
newEntry.append()
roachDelayTable.flush()
else:
print 'obs was aborted midway, delays cannot be calculated'
roachDelayTable.close()
outFile.close()
seq = '121229'
populationMax=1000
# test the theory that that extra photons show up 23 clock ticks
# after the beginning of each second. Plot properties of the photons
offsets = [23, 100, 1234, 54321]
#offsets = [23]
beginTime = 0
expTime = 300
pickleFile = open("csb2.pkl","wb")
fn = FileName(run, sundownDate, obsDate+"-"+seq)
pickle.dump(fn,pickleFile)
obsFile = ObsFile(fn.obs())
obsFile.loadTimeAdjustmentFile(FileName(run=run).timeAdjustments()) # Matt's time fix
timeMaskFile = fn.timeMask()
if os.path.exists(timeMaskFile):
obsFile.loadHotPixCalFile(timeMaskfile, switchOnMask=True)
if False:
iRow0 = 25
iRow1 = iRow0+1
iCol0 = 40
iCol1 = iCol0+1
else:
iRow0 = 0
iRow1 = obsFile.nRow
import astrometry.CentroidCalc as cent
import hotpix.hotPixels as hp
from photonlist import photlist
import matplotlib.pylab as plt
run = 'PAL2013'
date = '20131209'
tstamps = ['20131209-115553','20131209-120100', '20131209-121634', '20131209-122152']
for tstamp in tstamps:
centroidTstamp = tstamp
calTstamp='20131209-132225'
fluxTstamp='ones'
flatTstamp='20131209'
obsFileName = FileName(run=run, date=date, tstamp=tstamp)
obsFile = ObsFile(obsFileName.obs())
frames = []
print '-----------------------'
print tstamp
for i in xrange(30):
print "making frame ", i
frame = obsFile.getFrame(10*i,10)
#plt.matshow(frame,vmin=10000,vmax=40000,cmap=plt.cm.gray)
#plt.show()
frames.append(frame)
makeMovie(frames,frameTitles=np.arange(30)*10,cbar=True,outName='%s.gif'%(tstamp), normMin=400, normMax=4000)
CentFile = FileName(run=run,date=date,tstamp=tstamp).centroidList()
obsFile.loadCentroidListFile(CentFile)
NumFiles = []
for seq in seqs:
NumFiles.append(len(seq))
NumFiles = sum(NumFiles)
print (NumFiles)*expTime/integrationTime,'frames to make'
for iSeq in range(len(seqs)):
timestampList = timestampLists[iSeq]
print timestampList
wfn = wvlCalFilenames[iSeq]
ffn = flatCalFilenames[iSeq]
sunsetDate = sunsetDates[iSeq]
for i,ts in enumerate(timestampList):
print 'loading',ts
obsFn = FileName(run=run,date=sunsetDate,tstamp=ts).obs()
ob = ObsFile(obsFn)
ob.loadTimeAdjustmentFile(FileName(run=run).timeAdjustments())
index1 = obsFn.find('_')
hotPixFn = '/Scratch/timeMasks/timeMask' + obsFn[index1:]
if not os.path.exists(hotPixFn):
hp.findHotPixels(obsFn,hotPixFn)
print "Flux file pixel mask saved to %s"%(hotPixFn)
ob.loadHotPixCalFile(hotPixFn,switchOnMask=True)
ob.loadWvlCalFile(wfn)
ob.loadFlatCalFile(ffn)
ob.setWvlCutoffs(wvlLowerCutoff,wvlUpperCutoff)
bad_solution_mask=np.zeros((46,44))
bad_count=0;
for y in range(46):
#obsTimestamp ='20140925-124500' #early twilight
#obsTimestamp ='20140925-124603' #early twilight
#obsTimestamp ='20140925-124706' #early twilight
#obsTimestamp ='20140925-124809' #early twilight
#obsTimestamp ='20140925-124912' #early twilight
#obsTimestamp ='20140925-125015' #late twilight
#obsTimestamp ='20140925-125118' #late twilight
#obsTimestamp ='20140925-125221' #late twilight
#obsTimestamp ='20140925-125324' #late twilight
#obsTimestamp ='20140925-125427' #late twilight
#date = '20140925'
#obsTimestamp ='20140926-104528' #psr J0337
obsFN = FileName(run=run,date=date,tstamp=obsTimestamp)
obsFileName = obsFN.obs()
print obsFileName
timeMaskFileName = obsFN.timeMask()
if wvlCal != '':
wvlFileName = FileName(run=run,date=date,tstamp=wvlCal).calSoln()
flatFileName = FileName(run=run,date=flatCalDate,tstamp=flatCalTstamp).flatSoln()
illumFileName = FileName(run=run,date=flatCalDate,tstamp=flatCalTstamp).illumSoln()
if not os.path.exists(timeMaskFileName):
print 'Running hotpix for ',obsFileName
hp.findHotPixels(inputFileName=obsFileName,outputFileName=timeMaskFileName)
print "Flux file pixel mask saved to %s"%(timeMaskFileName)
obs = ObsFile(obsFileName)