def testLoadBeammap():
    '''
    Test if a remapped beammap is actually remapped
    '''
    #open an obs file from PAL2012,the sky file for hr9087
    #we'll use the embedded beammap file, which has some misplaced pixels
    run = 'PAL2012'
    date = '20121210'
    obsTimestamp = '20121211-051650'
    obsFN = FileName(run=run,date=date,tstamp=obsTimestamp)
    obsFileName = obsFN.obs()
    obs = ObsFile(obsFileName)
    
    beammapFileName = obsFN.beammap()
    
    #load the PixelMap for PAL2012 to know which pixels should be remapped
    pixMap = remapPixels.PixelMap(obsFN.pixRemap())
    pixMapSourceList,pixMapDestList = pixMap.getRemappedPix()
    
    #load the corrected beammap into the obs
    obs.loadBeammapFile(beammapFileName)

    #check that each pixel that should be moved is moved
    #by comparing the embedded beammap and the loaded corrected one
    for source,dest in zip(pixMapSourceList,pixMapDestList):
        assert obs.beamImage[dest] == obs.file.root.beammap.beamimage[source]

    obs.file.close()
def main():
    #load the current PAL2012 beammap from an obs file from that run,
    #a sky file for hr9087
    run = 'PAL2012'
    date = '20121210'
    obsTimestamp = '20121211-051650'
    obsFN = FileName(run=run,date=date,tstamp=obsTimestamp)
    obsFileName = obsFN.obs()
    obs = ObsFile(obsFileName)

    #Load an existing definition of which pixels to remap where
    pixMap = remapPixels.PixelMap(obsFN.pixRemap())
    pixMapSourceList,pixMapDestList = pixMap.getRemappedPix()

    #create a new h5 file and copy the existing beammap node hierarchy into it
    newBeammapPath = 'beamimage_PAL2012_corrected.h5'
    newBeammapFile = tables.openFile(newBeammapPath, mode='w')
    newBeammapFile.copyNode(obs.file.root.beammap, newparent=newBeammapFile.root, newname='beammap', recursive=True)
    newBeammap = newBeammapFile.root.beammap.beamimage

    #for each array in the h5, move the pixels according to pixMap
    newBeammap[:,:] = pixMap.remapArray(obs.beamImage)
    newBeammapFile.root.beammap.resfreq[:,:] = pixMap.remapArray(newBeammapFile.root.beammap.resfreq.read())
    newBeammapFile.root.beammap.atten[:,:] = pixMap.remapArray(newBeammapFile.root.beammap.atten.read())

    newBeammapFile.flush()
    newBeammapFile.close()
    obs.file.close()
Ejemplo n.º 3
0
def populateFlatCals(runPath,lookupPath=None):
    if lookupPath is None:
        lookupPath = os.environ['MKID_CAL_LOOKUP']
    if not os.path.exists(path):
        writeNewCalLookupFile(path)

    lookup = CalLookupFile(path,mode='a')

    for root,dirs,files in os.walk(runPath):
        obsFilenames = fnmatch.filter(files,'obs*.h5')
        for obsFilename in obsFilenames:
            obsPath = os.path.join(root,obsFilename)
            
            try:
                obs = util.ObsFile.ObsFile(obsPath)
                fn = FileName(obsFile=obsPath)
                params = {}
                params['obs_run'],params['obs_date'],params['obs_tstamp'] = fn.getComponents()
                params['flatSoln_run'] = params['obs_run']

                if params['obs_date'] in ['20140923','20140924','20141020','20141021','20141022']:
                    params['flatSoln_date'] = params['obs_date']
                else:
                    if params['obs_date'] == '20140925':
                        params['flatSoln_date'] = '20140924'
                print params
                lookup.updateObs(newParams=params)
            except:
                pass
Ejemplo n.º 4
0
    def makemovie1(self):
        run = self.s['run']
        sundownDate = self.s['sundownDate']
        obsDate = self.s['obsDate']
        stride = int(self.s['stride'])
        seq5 = self.s['seq5'].split()
        for seq in seq5:
            inFile = open("cosmicTimeList-%s.pkl"%(seq),"rb")
            cosmicTimeList = pickle.load(inFile)
            binContents = pickle.load(inFile)
            cfn = "cosmicMask-%s.h5"%seq
            intervals = ObsFile.readCosmicIntervalFromFile(cfn)
            for interval in intervals:
                print "interval=",interval
                fn = FileName(run, sundownDate,obsDate+"-"+seq)
                obsFile = ObsFile(fn.obs())
                obsFile.loadTimeAdjustmentFile(fn.timeAdjustments())
                i0=interval[0]
                i1=interval[1]
                intervalTime = i1-i0
                dt = intervalTime/2
                beginTime = max(0,i0-0.000200)
                endTime = beginTime + 0.001
                integrationTime = endTime-beginTime
                nBins = int(np.round(obsFile.ticksPerSec*(endTime-beginTime)+1))
                timeHgValues = np.zeros(nBins, dtype=np.int64)
                ymax = sys.float_info.max/100.0
                for iRow in range(obsFile.nRow):
                    for iCol in range(obsFile.nCol):
                        gtpl = obsFile.getTimedPacketList(iRow,iCol,
                                                          beginTime,integrationTime)
                        ts = (gtpl['timestamps'] - beginTime)*obsFile.ticksPerSec
                        ts64 = np.round(ts).astype(np.uint64)
                        tsBinner.tsBinner(ts64, timeHgValues)
                plt.clf()
                plt.plot(timeHgValues, label="data")
                x0 = (i0-beginTime)*obsFile.ticksPerSec
                x1 = (i1-beginTime)*obsFile.ticksPerSec
                plt.fill_between((x0,x1),(0,0), (ymax,ymax), alpha=0.2, color='red')   
                plt.yscale("symlog",linthreshy=0.9)
                plt.xlim(0,1000)
                plt.ylim(-0.1,300)
                tick0 = int(np.round(i0*obsFile.ticksPerSec))
                plotfn = "cp-%05d-%s-%s-%s-%09d"%(timeHgValues.sum(),run,obsDate,seq,tick0)
                plt.title(plotfn)
                plt.legend()
                plt.savefig(plotfn+".png")
                plt.xlabel("nSigma=%d stride=%d threshold=%d"%(int(self.s['nSigma']),int(self.s['stride']),int(self.s['threshold'])))
                print "plotfn=",plotfn
                

        os.system("convert -delay 0 `ls -r cp*png` cp.gif")
def testWritePhotonList(outputFileName=None,firstSec=0,integrationTime=-1,doPixRemap=True):
    '''
    Test run of obsFile.writePhotonList. fileName can be used
    to specify the output file name. If not specified, default
    name/location is used.
    
    Now includes test for pixel remapping....
    '''

    #Details of example obs file to run test on.
#    run = 'PAL2012'
#    date = '20121207'
#    tstamp = '20121208-074649'
#    calTstamp='20121208-070505'
#    fluxTstamp='20121208-133002'
    run = 'PAL2012'
    date = '20121208'
    tstamp = '20121209-120530'
    centroidTstamp = '20121209-120530'
    calTstamp='20121209-131132'
    fluxTstamp='20121209-020416'
    flatTstamp='20121209-021036'
    if doPixRemap==True:
        pixRemapFileName = FileName(run=run).pixRemap()
    else:
        pixRemapFileName = None
    
    #Load up the obs file
    obsFileName = FileName(run=run, date=date, tstamp=tstamp)
    obsFile = ObsFile(obsFileName.obs())
    
    #Load up associated calibrations
    obsFile.loadWvlCalFile(FileName(run=run,date=date,tstamp=calTstamp).calSoln())
    obsFile.loadFlatCalFile(FileName(run=run,date=date,tstamp=flatTstamp).flatSoln())
    obsFile.loadFluxCalFile(FileName(run=run,date=date,tstamp=fluxTstamp).fluxSoln())
    obsFile.loadTimeAdjustmentFile(FileName(run=run,date=date,tstamp=tstamp).timeAdjustments())
    obsFile.loadHotPixCalFile(FileName(run=run,date=date,tstamp=tstamp).timeMask())
    obsFile.loadCentroidListFile(FileName(run=run,date=date,tstamp=tstamp).centroidList())
    
    #And write out the results....
    obsFile.writePhotonList(outputFileName,firstSec,integrationTime,
                            pixRemapFileName=pixRemapFileName)
Ejemplo n.º 6
0
def populateTimeMasks(runPath,lookupPath=None):
    if lookupPath is None:
        lookupPath = os.environ['MKID_CAL_LOOKUP']
    if not os.path.exists(path):
        writeNewCalLookupFile(path)

    lookup = CalLookupFile(path,mode='a')
    for root,dirs,files in os.walk(runPath):
        obsFilenames = fnmatch.filter(files,'obs*.h5')
        for obsFilename in obsFilenames:
            obsPath = os.path.join(root,obsFilename)
            
            try:
                obs = util.ObsFile.ObsFile(obsPath)
                fn = FileName(obsFile=obsPath)
                params = {}
                params['obs_run'],params['obs_date'],params['obs_tstamp'] = fn.getComponents()
                params['timeMask_run'],params['timeMask_date'],params['timeMask_tstamp'] = fn.getComponents()
                print params
                lookup.updateObs(newParams=params)
            except:
                pass
Ejemplo n.º 7
0
def populateWaveCals(runPath,lookupPath=None):
    if lookupPath is None:
        lookupPath = os.environ['MKID_CAL_LOOKUP']
    if not os.path.exists(path):
        writeNewCalLookupFile(path)

    lookup = CalLookupFile(path,mode='a')
    for root,dirs,files in os.walk(runPath):
        obsFilenames = fnmatch.filter(files,'obs*.h5')
        for obsFilename in obsFilenames:
            obsPath = os.path.join(root,obsFilename)
            
            try:
                obs = util.ObsFile.ObsFile(obsPath)
                fn = FileName(obsFile=obsPath)
                params = {}
                params['obs_run'],params['obs_date'],params['obs_tstamp'] = fn.getComponents()
                params['waveSoln_run'] = params['obs_run']

                obs.loadBestWvlCalFile()
                waveCalPath = obs.wvlCalFileName
                waveCalFilename = os.path.basename(waveCalPath)
                if waveCalFilename.startswith('master'):
                    params['waveSoln_isMasterCal'] = True
                    tstamp = waveCalFilename.split('_')[1].split('.')[0]
                    params['waveSoln_tstamp'] = tstamp
                    params['waveSoln_date'] = ''
                else:
                    params['waveSoln_isMasterCal'] = False
                    tstamp = waveCalFilename.split('_')[1].split('.')[0]
                    params['waveSoln_tstamp'] = tstamp
                    dirs = os.path.dirname(os.path.normpath(waveCalPath)).split(os.sep)
                    params['waveSoln_date'] = dirs[-1]
                print params
                lookup.updateObs(newParams=params)
            except:
                pass
Ejemplo n.º 8
0
npList = []
sigList = []
   
run = 'PAL2012'
sundownDate = '20121211'
obsDate = '20121212'

for seq in seq5:
    inFile = open("cosmicTimeList-%s.pkl"%(seq),"rb")
    cosmicTimeList = pickle.load(inFile)
    binContents = pickle.load(inFile)
    cfn = "cosmicMax-%s.h5"%seq
    intervals = ObsFile.readCosmicIntervalFromFile(cfn)
    for interval in intervals:
        print "interval=",interval
        fn = FileName(run, sundownDate,obsDate+"-"+seq)
        obsFile = ObsFile(fn.obs())
        obsFile.loadTimeAdjustmentFile(fn.timeAdjustments())
        i0=interval[0]
        i1=interval[1]
        intervalTime = i1-i0
        dt = intervalTime/2
        beginTime = max(0,i0-0.000200)
        endTime = beginTime + 0.001
        integrationTime = endTime-beginTime
        nBins = int(np.round(obsFile.ticksPerSec*(endTime-beginTime)+1))
        timeHgValues = np.zeros(nBins, dtype=np.int64)
        ymax = sys.float_info.max/100.0
        for iRow in range(obsFile.nRow):
            for iCol in range(obsFile.nCol):
                gtpl = obsFile.getTimedPacketList(iRow,iCol,
Ejemplo n.º 9
0
    def __init__(self,paramFile,plots=False,verbose=False):
        """
        Opens flux file, prepares standard spectrum, and calculates flux factors for the file.
        Method is provided in param file. If 'relative' is selected, an obs file with standard star defocused over
        the entire array is expected, with accompanying sky file to do sky subtraction.
        If any other method is provided, 'absolute' will be done by default, wherein a point source is assumed
        to be present. The obs file is then broken into spectral frames with photometry (psf or aper) performed 
        on each frame to generate the ARCONS observed spectrum.
        """
        self.verbose=verbose
        self.plots = plots

        self.params = readDict()
        self.params.read_from_file(paramFile)
        
        run = self.params['run']
        sunsetDate = self.params['fluxSunsetLocalDate']
        self.fluxTstamp = self.params['fluxTimestamp']
        skyTstamp = self.params['skyTimestamp']
        wvlSunsetDate = self.params['wvlCalSunsetLocalDate']
        wvlTimestamp = self.params['wvlCalTimestamp']
        flatCalFileName = self.params['flatCalFileName']
        needTimeAdjust = self.params['needTimeAdjust']
        self.deadtime = float(self.params['deadtime']) #from firmware pulse detection
        self.timeSpacingCut = self.params['timeSpacingCut']
        bLoadBeammap = self.params.get('bLoadBeammap',False)
        self.method = self.params['method']
        self.objectName = self.params['object']
        self.r = float(self.params['energyResolution'])
        self.photometry = self.params['photometry']
        self.centroidRow = self.params['centroidRow']
        self.centroidCol = self.params['centroidCol']
        self.aperture = self.params['apertureRad']
        self.annulusInner = self.params['annulusInner']
        self.annulusOuter = self.params['annulusOuter']
        self.collectingArea = self.params['collectingArea']
        self.startTime = self.params['startTime']
        self.intTime = self.params['integrationTime']

        fluxFN = FileName(run=run,date=sunsetDate,tstamp=self.fluxTstamp)
        self.fluxFileName = fluxFN.obs()
        self.fluxFile = ObsFile(self.fluxFileName)

        if self.plots:
            self.plotSavePath = os.environ['MKID_PROC_PATH']+os.sep+'fluxCalSolnFiles'+os.sep+run+os.sep+sunsetDate+os.sep+'plots'+os.sep
            if not os.path.exists(self.plotSavePath): os.mkdir(self.plotSavePath)
            if self.verbose: print "Created directory %s"%self.plotSavePath

        obsFNs = [fluxFN]
        self.obsList = [self.fluxFile]

        if self.startTime in ['',None]: self.startTime=0
        if self.intTime in ['',None]: self.intTime=-1

        if self.method=="relative":
            try:
                print "performing Relative Flux Calibration"
                skyFN = FileName(run=run,date=sunsetDate,tstamp=skyTstamp)
                self.skyFileName = skyFN.obs()
                self.skyFile = ObsFile(self.skyFileName)
                obsFNs.append(skyFN)
                self.obsList.append(self.skyFile)
            except:
                print "For relative flux calibration a sky file must be provided in param file"
                self.__del__()
        else:
            self.method='absolute'
            print "performing Absolute Flux Calibration"

        if self.photometry not in ['aperture','PSF']: self.photometry='PSF' #default to PSF fitting if no valid photometry selected

        timeMaskFileNames = [fn.timeMask() for fn in obsFNs]
        timeAdjustFileName = FileName(run=run).timeAdjustments()

        #make filename for output fluxCalSoln file
        self.fluxCalFileName = FileName(run=run,date=sunsetDate,tstamp=self.fluxTstamp).fluxSoln()
        print "Creating flux cal: %s"%self.fluxCalFileName

        if wvlSunsetDate != '':
            wvlCalFileName = FileName(run=run,date=wvlSunsetDate,tstamp=wvlTimestamp).calSoln()
        if flatCalFileName =='':
            flatCalFileName=FileName(obsFile=self.fluxFile).flatSoln()

        #load cal files for flux file and, if necessary, sky file
        for iObs,obs in enumerate(self.obsList):
            if bLoadBeammap:
                print 'loading beammap',os.environ['MKID_BEAMMAP_PATH']
                obs.loadBeammapFile(os.environ['MKID_BEAMMAP_PATH'])
            if wvlSunsetDate != '':
                obs.loadWvlCalFile(wvlCalFileName)
            else:
                obs.loadBestWvlCalFile()

            obs.loadFlatCalFile(flatCalFileName)
            obs.setWvlCutoffs(-1,-1)

            if needTimeAdjust:
                obs.loadTimeAdjustmentFile(timeAdjustFileName)
            timeMaskFileName = timeMaskFileNames[iObs]
            print timeMaskFileName

            if not os.path.exists(timeMaskFileName):
                print 'Running hotpix for ',obs
                hp.findHotPixels(obsFile=obs,outputFileName=timeMaskFileName,fwhm=np.inf,useLocalStdDev=True)
                print "Flux cal/sky file pixel mask saved to %s"%(timeMaskFileName)
            obs.loadHotPixCalFile(timeMaskFileName)
            if self.verbose: print "Loaded hot pixel file %s"%timeMaskFileName

        #get flat cal binning information since flux cal will need to match it
        self.wvlBinEdges = self.fluxFile.flatCalFile.root.flatcal.wavelengthBins.read()
        self.nWvlBins = self.fluxFile.flatWeights.shape[2]
        self.binWidths = np.empty((self.nWvlBins),dtype=float)
        self.binCenters = np.empty((self.nWvlBins),dtype=float)
        for i in xrange(self.nWvlBins):
            self.binWidths[i] = self.wvlBinEdges[i+1]-self.wvlBinEdges[i]
            self.binCenters[i] = (self.wvlBinEdges[i]+(self.binWidths[i]/2.0))

        if self.method=='relative':
            print "Extracting ARCONS flux and sky spectra"
            self.loadRelativeSpectrum()
            print "Flux Spectrum loaded"
            self.loadSkySpectrum()
            print "Sky Spectrum loaded"
        elif self.method=='absolute':
            print "Extracting ARCONS point source spectrum"
            self.loadAbsoluteSpectrum()

        print "Loading standard spectrum"
        try:
            self.loadStdSpectrum(self.objectName)
        except KeyError:
            print "Invalid spectrum object name"
            self.__del__()
            sys.exit()

        print "Generating sensitivity curve"
        self.calculateFactors()
        print "Sensitivity Curve calculated"
        print "Writing fluxCal to file %s"%self.fluxCalFileName
        self.writeFactors(self.fluxCalFileName)
        
        if self.plots: self.makePlots()

        print "Done"
Ejemplo n.º 10
0
from util.ObsFile import ObsFile
from util.FileName import FileName
run = "PAL2014"
date = "20141022"
timeStamp = '20141023-033821'
fn = FileName(run,date,timeStamp)

of = ObsFile(fn.obs())
of.loadBeammapFile(fn.beammap())
of.loadBestWvlCalFile()
print "wvlCalFileName=",of.wvlCalFileName
fn2 = FileName(run,date,"")
of.loadFlatCalFile(fn2.flatSoln())
row = 4
col = 4
firstSec = 72
integrationTime = 1
spec = of.getPixelSpectrum(row,col,firstSec,integrationTime)
print "spec=",spec
del of
Ejemplo n.º 11
0
def plot_pixel_phaseHeights(self,row=None,col=None):

    self.row = row
    self.col = col
    self.laserCalFile = self.parent.wave_obs
    
    dataDict=self.laserCalFile.getTimedPacketList(self.row,self.col,timeSpacingCut=self.parent.params['danicas_cut'])
    peakHeights=np.asarray(dataDict['peakHeights'])*1.0
    ## less than 'min_amp' per second average count rate

    if dataDict['effIntTime']==0.0 or len(peakHeights)<=(dataDict['effIntTime']*self.parent.params['min_count_rate']):
        print 'Not enough data for phase histogram'
        raise ValueError
    baselines=np.asarray(dataDict['baselines'])*1.0
    peakHeights-=baselines
    biggest_photon = int(min(peakHeights))
    n_inbin,phase_bins=np.histogram(peakHeights,bins=np.abs(biggest_photon),range=(biggest_photon,0))
    phase_bins=(phase_bins+(phase_bins[1]-phase_bins[0])/2.0)[:-1]
    self.n_inbin = n_inbin
    self.phase_bins = phase_bins
    try:
        last_ind = np.where(n_inbin>self.parent.params['min_amp'])[0][-1]
    except IndexError:
        last_ind=len(n_inbin)-1
    ## Cut out all the zeros on the right


    self.axes.plot(phase_bins,n_inbin, 'b.',label="data")
    self.axes.set_xlim(phase_bins[(np.where(n_inbin >= 3))[0][0]],phase_bins[last_ind])

    n_inbin = n_inbin[:last_ind]
    phase_bins = phase_bins[:last_ind]

    #Check if this pixel is in drift file
    waveFN = FileName(obsFile=self.laserCalFile)
    driftFile=tables.openFile(waveFN.calDriftInfo(),mode='r')
    drift_row = driftFile.root.params_drift.driftparams.cols.pixelrow[:]    
    drift_col = driftFile.root.params_drift.driftparams.cols.pixelcol[:]    

    args=(np.where(np.multiply(drift_row==self.row, drift_col==self.col)))[0]
    if len(args==1):
        #pixel has a solution!
        fit_params = driftFile.root.params_drift.driftparams.cols.gaussparams[args[0]]
        #print fit_params
        model = driftFile.root.params_drift.driftparams.attrs.model_type
        model_list = {
            'parabola': parabola,
            'gaussian': gaussian,
            'fourgaussian': fourgaussian,
            'threegaussian_exp': threegaussian_exp,
            'threegaussian_exppow': threegaussian_exppow,
            'threegaussian_moyal': threegaussian_moyal,
            'threegaussian_power': threegaussian_power,
            'threegaussian_lorentzian': threegaussian_lorentzian}

        modelArr=model_list[model](p=fit_params,x=phase_bins,return_models=True)
        fullModel=np.zeros(len(n_inbin))
        for model_part in modelArr:
            fullModel = fullModel+model_part
            self.axes.plot(phase_bins,model_part, 'k--')
        self.axes.plot(phase_bins,fullModel, 'r-',label="model")

    self.axes.legend()
    self.axes.set_xlabel("phase")
    self.axes.set_ylabel("counts")
    self.axes.set_title('('+str(self.col)+', '+str(self.row)+') --> '+(self.laserCalFile.beamImage[self.row,self.col]).rsplit('/',1)[0])
    #res_blue = (self.drift.r_blue[self.row,self.col,self.closest_time_ind])
    #self.axes.text(0.15,0.95,"R ~ "+str(round(res_blue,3)),verticalalignment='center', horizontalalignment='center', transform=self.axes.transAxes)
    #print res_blue
    driftFile.close()

    return True
Ejemplo n.º 12
0
def plot_pixel_soln(self,row,col,list_of_fits):
    #self.fig, axarr = plt.subplots(2, sharex=True)
    #self.axes=axarr[0]
    #axes2=axarr[1]
    self.fig.delaxes(self.axes)
    gs = gridspec.GridSpec(2,1,height_ratios=[4,1])
    self.fig.subplots_adjust(hspace=0.,wspace=0.)
    self.axes=self.fig.add_subplot(gs[0])

    QE_wavelengths = self.parent.QE_data[:,0]
    QE_energies = [self.parent.params['h'] * self.parent.params['c'] / (x*10. * self.parent.params['ang2m']) for x in QE_wavelengths]
    QE_x_offsets = np.asarray(list_of_fits)[:,1]
    
    waveFN = FileName(obsFile=self.parent.wave_obs)
    calFile=tables.openFile(waveFN.calSoln(),mode='r')
    pixRow=calFile.root.wavecal.calsoln.cols.pixelrow[:]
    pixCol=calFile.root.wavecal.calsoln.cols.pixelcol[:]
    cal_ind = np.where((pixRow==row) * (pixCol==col))[0][0]
    polyfit=calFile.root.wavecal.calsoln.cols.polyfit[cal_ind]
    
    if np.all(polyfit==-1) and np.all(QE_x_offsets==0):
        raise ValueError
    
    calFile.close()
    driftFile=tables.openFile(waveFN.calDriftInfo(),mode='r')
    drift_row = driftFile.root.params_drift.driftparams.cols.pixelrow[:]    
    drift_col = driftFile.root.params_drift.driftparams.cols.pixelcol[:]  
    try:  
        drift_ind=(np.where(np.multiply(drift_row==row, drift_col==col)))[0][0]
        fit_params = driftFile.root.params_drift.driftparams.cols.gaussparams[drift_ind]
        cal_x_offsets = fit_params[[1,4,7]]
        cal_x_offsets[1]+=cal_x_offsets[0]
        cal_x_offsets[2]+=cal_x_offsets[1]
        if fit_params[8]==0.:
            cal_x_offsets[2]=0.
        cal_wavelengths = [self.parent.params['bluelambda'],self.parent.params['redlambda'], self.parent.params['irlambda']]     #Angstroms
        cal_energies = [self.parent.params['h'] * self.parent.params['c'] / (x * self.parent.params['ang2m']) for x in cal_wavelengths]             #eV
    except IndexError:
        cal_wavelengths = [self.parent.params['bluelambda'],self.parent.params['redlambda'], self.parent.params['irlambda']]     #Angstroms
        cal_energies = [self.parent.params['h'] * self.parent.params['c'] / (x * self.parent.params['ang2m']) for x in cal_wavelengths]             #eV
        cal_x_offsets = [0,0,0]
    driftFile.close()
    

    
    xArr = np.arange(-5000.,0.,0.1)
    if not np.all(polyfit==-1):
        modelArr=(parabola(p=polyfit,x=xArr,return_models=True))[0]
        self.axes.plot(xArr,modelArr,'k-')
    
    self.axes.plot(cal_x_offsets,cal_energies,'ko', label='Laser Cal')
    
    self.axes.plot(QE_x_offsets,QE_energies,'gx',label='QE wavelength fits')
    

    
    #Do linear fit of QE data to see if x-intercept is at 0
    QE_x_offsets = np.asarray(QE_x_offsets)
    QE_energies = np.asarray(QE_energies)
    energies = (QE_energies[np.where(QE_x_offsets<0)])
    sort_ind = np.argsort(energies)
    energies = energies[sort_ind]
    x_offsets = QE_x_offsets[np.where(QE_x_offsets<0)]
    x_offsets = x_offsets[sort_ind]
    if len(x_offsets)>1:
        parameter_guess = [0.0,(energies[0]-energies[-1])*1.0/(x_offsets[0]-x_offsets[-1]), 0.]
        parameter_lowerlimit=[None,None,0.]
        parameter_upperlimit=[None,None,0.]
        parameter_fit, redchi2gauss2, mpperr = fitData(x_offsets,energies,parameter_guess,parameter_lowerlimit,parameter_upperlimit,model='parabola')
        print 'x_intercept: '+str(-1.*parameter_fit[0]/parameter_fit[1])
    else:
        print 'x_intercept: Not enough data'
    #lin_fit=(parabola(p=parameter_fit,x=xArr,return_models=True))[0]
    #self.axes.plot(xArr,lin_fit,'g--')
    
    #Do parabola fit to QE data
    if len(x_offsets)>1:
        parameter_guess = [0.0,(energies[0]-energies[-1])*1.0/(x_offsets[0]-x_offsets[-1]), 0.]
        parameter_lowerlimit=[None,None,None]
        parameter_upperlimit=[None,None,None]
        parameter_fit, redchi2gauss2, mpperr = fitData(x_offsets,energies,parameter_guess,parameter_lowerlimit,parameter_upperlimit,model='parabola')
        print polyfit,' --> ',parameter_fit
        parab_fit=(parabola(p=parameter_fit,x=xArr,return_models=True))[0]
        self.axes.plot(xArr,parab_fit,'g--')
    
    #Plot Residuals
    if not np.all(polyfit==-1):
        cal_QE=(parabola(p=polyfit,x=x_offsets,return_models=True))[0]
        axes2=self.fig.add_subplot(gs[1],sharex=self.axes)
        #axes2 = self.fig.add_axes((.1,.1,.8,.2))
        axes2.plot(x_offsets,cal_QE-energies,'.')
        axes2.axhline(0,color='black')
    
    
    #Add labels etc
    self.axes.set_ylabel('Energy (eV)')
    self.axes.set_xlim(min(min(cal_x_offsets),min(QE_x_offsets))-200.,0.)
    self.axes.set_ylim(0.,5.)
    if not np.all(polyfit==-1):
        axes2.set_xlabel('Phase (ADC/DAC units)')
        axes2.set_ylabel('Calibration residuals (eV)')
        axes2.set_ylim(-3.,3.)
        #self.axes.set_xticklabels(self.axes.get_xticklabels(),visible=False)
        plt.setp(self.axes.get_xticklabels(),visible=False)
Ejemplo n.º 13
0
import matplotlib.pyplot as plt
from util.ObsFile import ObsFile
import os
import hotpix.hotPixels as hp
from util.FileName import FileName
from util.popup import plotArray
import astrometry.CentroidCalc as cc
from photonlist.photlist import writePhotonList
from headers.ArconsHeaders import PulsarPhotonList

if __name__=='__main__':
    run = 'PAL2014'
    date = '20140924'
    tstamp = '20140925-112538'

    obsFN = FileName(run=run,date=date,tstamp=tstamp)
    obsPath = obsFN.obs()
     
    flatPath = FileName(run=run,date=date).flatSoln()
    hotPath = obsFN.timeMask()
    centroidPath = obsFN.centroidList()
    fluxPath = FileName(run='PAL2012',date='20121211',tstamp='absolute_021727').fluxSoln()
    
    obs = ObsFile(obsPath)
    if not os.path.exists(hotPath):
        hp.findHotPixels(obsPath,hotPath)
    obs.loadHotPixCalFile(hotPath,switchOnMask=False)
    obs.loadBestWvlCalFile()
    obs.loadFlatCalFile(flatPath)
    obs.loadFluxCalFile(fluxPath)
    obs.loadCentroidListFile(centroidPath)
    ffn = flatCalFilenames[iSeq]
    sunsetDate = sunsetDates[iSeq]

    for position in ObjPosFile:
        if iSeq + 1 == position[0]:
            print "finding aperture and sky masks"
            guessX = position[1]
            guessY = position[2]
            apertureMask = circleAperture(guessX, guessY, radius1)
            bigMask = circleAperture(guessX, guessY, radius2)
            skyMask = bigMask - apertureMask
    #            print skyMask[0:20][5:25]

    for i, ts in enumerate(timestampList):
        print "loading", ts
        obsFn = FileName(run=run, date=sunsetDate, tstamp=ts).obs()
        ob = ObsFile(obsFn)
        ob.loadTimeAdjustmentFile(FileName(run=run).timeAdjustments())
        index1 = obsFn.find("_")
        hotPixFn = "/Scratch/timeMasks/timeMask" + obsFn[index1:]
        if not os.path.exists(hotPixFn):
            hp.findHotPixels(obsFn, hotPixFn)
            print "Flux file pixel mask saved to %s" % (hotPixFn)
        ob.loadHotPixCalFile(hotPixFn, switchOnMask=True)
        ob.loadWvlCalFile(wfn)
        ob.loadFlatCalFile(ffn)
        ob.setWvlCutoffs(wvlLowerCutoff, wvlUpperCutoff)

        bad_solution_mask = np.zeros((46, 44))
        bad_count = 0
        for y in range(46):
def testWritePhotonList(outputFileName=None,firstSec=0,integrationTime=-1,doPixRemap=False):
    '''
    Test run of obsFile.writePhotonList. fileName can be used
    to specify the output file name. If not specified, default
    name/location is used.
    
    Now includes test for pixel remapping....
    '''

    #Details of example obs file to run test on.
#    run = 'PAL2012'
#    date = '20121207'
#    tstamp = '20121208-074649'
#    calTstamp='20121208-070505'
#    fluxTstamp='20121208-133002'

    run = 'PAL2013'
    date = '20131209'
    tstamp = '20131209-115553'
    centroidTstamp = tstamp
    calTstamp='20131209-132225'
    fluxTstamp='ones'
    flatTstamp='20131209'
    if doPixRemap==True:
        pixRemapFileName = FileName(run=run).pixRemap()
    else:
        pixRemapFileName = None
    
    #Load up the obs file
    obsFileName = FileName(run=run, date=date, tstamp=tstamp)
    obsFile = ObsFile(obsFileName.obs())

    obsFile.loadWvlCalFile(FileName(run=run,date=date,tstamp=calTstamp).calSoln())
    obsFile.loadFlatCalFile(FileName(run=run,date=date,tstamp=flatTstamp).flatSoln())
    obsFile.loadFluxCalFile(FileName(run=run,date=date,tstamp=fluxTstamp).fluxSoln())

    HotPixFile = FileName(run=run,date=date,tstamp=tstamp).timeMask()
    if not os.path.exists(HotPixFile): #check if hot pix file already exists
        hp.findHotPixels(obsFileName.obs(),HotPixFile)
        print "Flux file pixel mask saved to %s"%(HotPixFile)
    obsFile.loadHotPixCalFile(HotPixFile)
    print "Hot pixel mask loaded %s"%(HotPixFile)   

    #hz21 location alpha(2000) = 12h 13m 56.42s , delta(2000) = +32d 56' 30.8''
    centroid_RA = '12:13:56.42'
    centroid_DEC = '32:56:30.8'
    CentFile = FileName(run=run,date=date,tstamp=tstamp).centroidList()
    #print "checking centroid file", CentFile
    #if not os.path.exists(CentFile): #check if cent pix file already exists
    cent.centroidCalc(obsFile, centroid_RA, centroid_DEC, outputFileName = CentFile, guessTime=10, integrationTime=10)
    #    print "Flux file centroid cal saved to %s"%(CentFile)
    obsFile.loadCentroidListFile(CentFile)
    print "Centroid File loaded %s"%(CentFile) 

    #Load up associated calibrations

    #obsFile.loadTimeAdjustmentFile(FileName(run=run,date=date,tstamp=tstamp).timeAdjustments())
    #obsFile.loadHotPixCalFile(FileName(run=run,date=date,tstamp=tstamp).timeMask())
    #obsFile.loadCentroidListFile(FileName(run=run,date=date,tstamp=tstamp).centroidList())
    
    #And write out the results....
    obsFile.writePhotonList(outputFileName,firstSec,integrationTime,
                            pixRemapFileName=pixRemapFileName)
    
    #Read the results back in....
    #photFile = photList.PhotFile(outputFilename)
    
    print "Wrote photon list file: ", outputFileName
def main():

    #open the sky file for hr9087
    run = 'PAL2012'
    date = '20121210'
    wvlCal = '20121211-052230'
    obsTimestamp = '20121211-051650'
    flatCalDate = '20121211'
    flatCalTstamp = '20121212-074700'
    obsFN = FileName(run=run,date=date,tstamp=obsTimestamp)
    obsFileName = obsFN.obs()
    timeMaskFileName = obsFN.timeMask()
    wvlFileName = FileName(run=run,date=date,tstamp=wvlCal).calSoln()
    flatFileName = FileName(run=run,date=flatCalDate,tstamp=flatCalTstamp).flatSoln()
    
    if not os.path.exists(timeMaskFileName):
        print 'Running hotpix for ',obsFileName
        hp.findHotPixels(obsFileName,timeMaskFileName)
        print "Flux file pixel mask saved to %s"%(timeMaskFileName)

    obs = ObsFile(obsFileName)
    obs.loadTimeAdjustmentFile(FileName(run='PAL2012').timeAdjustments())
    obs.loadWvlCalFile(wvlFileName)
    obs.loadFlatCalFile(flatFileName)
    obs.loadHotPixCalFile(timeMaskFileName)

    #obs.setWvlCutoffs(4000,8000)

    #get image before and after flat cal
    print 'getting images'
    beforeImgDict = obs.getPixelCountImage(weighted=False,fluxWeighted=False,scaleByEffInt=True)

    rawCubeDict = obs.getSpectralCube(weighted=False)
    rawCube = np.array(rawCubeDict['cube'],dtype=np.double)
    effIntTime = rawCubeDict['effIntTime']
    maxIntTime = np.max(effIntTime)
    #add third dimension for broadcasting
    effIntTime3d = np.reshape(effIntTime,np.shape(effIntTime)+(1,))
    rawCube *= maxIntTime / effIntTime3d
    rawCube[np.isnan(rawCube)] = 0
    rawCube[rawCube == np.inf] = 0
    beforeImg = np.sum(rawCube,axis=-1)
    print 'finished first cube'

    flatCubeDict = obs.getSpectralCube(weighted=True)
    flatCube = np.array(flatCubeDict['cube'],dtype=np.double)
    effIntTime = flatCubeDict['effIntTime']
    maxIntTime = np.max(effIntTime)
    #add third dimension for broadcasting
    effIntTime3d = np.reshape(effIntTime,np.shape(effIntTime)+(1,))
    flatCube *= maxIntTime / effIntTime3d
    flatCube[np.isnan(flatCube)] = 0
    flatCube[flatCube == np.inf] = 0
    afterImg = np.sum(flatCube,axis=-1)

    plotArray(title='before flatcal',image=beforeImg)
    plotArray(title='after flatcal',image=afterImg)

    print 'before sdev',np.std(beforeImg[afterImg!=0])
    print 'after sdev',np.std(afterImg[afterImg!=0])

    np.savez('flatCubeGem.npz',beforeImg=beforeImg,afterImg=afterImg,rawCube=rawCube,flatCube=flatCube)
def main():

    run = 'PAL2014'
    year = '2014'
    initialPath = '/Scratch'
    packetMasterLogDir = '/LABTEST/PacketMasterLogs/'
    initialPath = os.path.join(initialPath,run)
    outPath = FileName(run=run,mkidDataDir='/Scratch/').timeAdjustments()
    outFile = tables.openFile(outPath,'w')
    timeAdjustGroup = outFile.createGroup('/','timeAdjust','Times to add to timestamps')
    firmwareDelayTable = outFile.createTable(timeAdjustGroup,'firmwareDelay',firmwareDelayDescription,'Times to add to all timestamps taken with a firmware bof')
    newFirmwareEntry = firmwareDelayTable.row
    newFirmwareEntry['firmwareName']='chan_svf_2014_Aug_06_1839.bof'
    newFirmwareEntry['firmwareDelay']=-41e-6 #s, subtract 41 us from timestamps
    newFirmwareEntry.append()
    firmwareDelayTable.flush()
    firmwareDelayTable.close()
    roachDelayTable = outFile.createTable(timeAdjustGroup,'roachDelays',roachDelaysDescription,'Times to add to each roach\'s timestamps')


    for sunsetDatePath in sorted(glob.glob(os.path.join(initialPath,year+'*'))):
        sunsetDate = os.path.basename(sunsetDatePath)
        for fullObsPath in sorted(glob.glob(os.path.join(sunsetDatePath,'obs*.h5'))):
            obsFileName = os.path.basename(fullObsPath)
            obsTStamp = obsFileName.split('.')[0].split('_')[1]
            print obsFileName
        

            obsFN = FileName(run=run,date=sunsetDate,tstamp=obsTStamp,packetMasterLogDir=packetMasterLogDir)
            pmLogFileName = obsFN.packetMasterLog()
            try:
                ob = ObsFile(fullObsPath)
            except:
                print 'can\'t open file'
                continue
                
            try:
                if os.path.getsize(pmLogFileName) <= 0:
                    continue
            except:
                print 'can\'t open Packet Master Log ',pmLogFileName
                continue

            try:
                f = open(pmLogFileName,'r')
            except:
                print 'can\'t open Packet Master Log ',pmLogFileName
                continue
            lastTstampLines = np.zeros(8)
            firstTstampLines = np.zeros(8)

            tstampLine = ''
            for line in f:
                if 'here' in line:
                    #skip over large log files with debug info
                    print 'skipping file with "here"'
                    continue
                if line.split(' ')[0] == 'bundle':
                    tstampLine = line
                    break

            if tstampLine == '':
                print 'skipping file without "bundle"'
                #didn't find lines with 'bundle' in them
                continue

            f.seek(0)
            for line in f:
                if line.split(' ')[0] == 'bundle':
                    try:
                        at = float(line.split('at')[1].split()[0])
                    except:
                        break
                    lastTstampLine = at
                    iRoach = int(line.split('roach')[1].split('took')[0].strip())
                    lastTstampLines[iRoach] = at
                    if firstTstampLines[iRoach] == 0:
                        firstTstampLines[iRoach] = at
                    
            packetReceivedUnixTimestamp = float((tstampLine.split('took')[1].split('total')[0].strip()))
            firstPacketDelay = packetReceivedUnixTimestamp-int(ob.getFromHeader('unixtime'))
                

            roachSecDelays =np.array(np.floor(lastTstampLines+firstPacketDelay-ob.getFromHeader('exptime')),dtype=np.int)
            print 'roach delays',roachSecDelays
            if np.all(roachSecDelays >= 0):
                newEntry = roachDelayTable.row
                newEntry['obsFileName'] = os.path.basename(fullObsPath)
                newEntry['roachDelays'] = roachSecDelays
                newEntry.append()
                roachDelayTable.flush()
            else:
                print 'obs was aborted midway, delays cannot be calculated'
    roachDelayTable.close()
    outFile.close()
seq = '121229'

populationMax=1000


# test the theory that that extra photons show up 23 clock ticks
# after the beginning of each second.  Plot properties of the photons

offsets = [23, 100, 1234, 54321]
#offsets = [23]
beginTime = 0
expTime = 300

pickleFile = open("csb2.pkl","wb")

fn = FileName(run, sundownDate, obsDate+"-"+seq)
pickle.dump(fn,pickleFile)
obsFile = ObsFile(fn.obs())
obsFile.loadTimeAdjustmentFile(FileName(run=run).timeAdjustments()) # Matt's time fix
timeMaskFile = fn.timeMask()
if os.path.exists(timeMaskFile):
    obsFile.loadHotPixCalFile(timeMaskfile, switchOnMask=True)

if False:
    iRow0 = 25
    iRow1 = iRow0+1
    iCol0 = 40
    iCol1 = iCol0+1
else:
    iRow0 = 0
    iRow1 = obsFile.nRow
import astrometry.CentroidCalc as cent
import hotpix.hotPixels as hp
from photonlist import photlist
import matplotlib.pylab as plt

run = 'PAL2013'
date = '20131209'
tstamps = ['20131209-115553','20131209-120100', '20131209-121634', '20131209-122152']

for tstamp in tstamps:
    centroidTstamp = tstamp
    calTstamp='20131209-132225'
    fluxTstamp='ones'
    flatTstamp='20131209'

    obsFileName = FileName(run=run, date=date, tstamp=tstamp)
    obsFile = ObsFile(obsFileName.obs())
    frames = []
    print '-----------------------'
    print tstamp
    for i in xrange(30):
        print "making frame ", i
        frame = obsFile.getFrame(10*i,10)
        #plt.matshow(frame,vmin=10000,vmax=40000,cmap=plt.cm.gray)
        #plt.show()
        frames.append(frame)

    makeMovie(frames,frameTitles=np.arange(30)*10,cbar=True,outName='%s.gif'%(tstamp), normMin=400, normMax=4000)

    CentFile = FileName(run=run,date=date,tstamp=tstamp).centroidList()
    obsFile.loadCentroidListFile(CentFile)
NumFiles = []
for seq in seqs:
    NumFiles.append(len(seq))
NumFiles = sum(NumFiles)
print (NumFiles)*expTime/integrationTime,'frames to make'

for iSeq in range(len(seqs)):
    timestampList = timestampLists[iSeq]
    print timestampList
    wfn = wvlCalFilenames[iSeq]
    ffn = flatCalFilenames[iSeq]
    sunsetDate = sunsetDates[iSeq]
    
    for i,ts in enumerate(timestampList):
        print 'loading',ts
        obsFn = FileName(run=run,date=sunsetDate,tstamp=ts).obs()
        ob = ObsFile(obsFn)
        ob.loadTimeAdjustmentFile(FileName(run=run).timeAdjustments())
        index1 = obsFn.find('_')
        hotPixFn = '/Scratch/timeMasks/timeMask' + obsFn[index1:]
        if not os.path.exists(hotPixFn):
            hp.findHotPixels(obsFn,hotPixFn)
            print "Flux file pixel mask saved to %s"%(hotPixFn)
        ob.loadHotPixCalFile(hotPixFn,switchOnMask=True)
        ob.loadWvlCalFile(wfn)
        ob.loadFlatCalFile(ffn)
        ob.setWvlCutoffs(wvlLowerCutoff,wvlUpperCutoff)

        bad_solution_mask=np.zeros((46,44))
        bad_count=0;
        for y in range(46):
    #obsTimestamp ='20140925-124500' #early twilight
    #obsTimestamp ='20140925-124603' #early twilight
    #obsTimestamp ='20140925-124706' #early twilight
    #obsTimestamp ='20140925-124809' #early twilight
    #obsTimestamp ='20140925-124912' #early twilight
    #obsTimestamp ='20140925-125015' #late twilight
    #obsTimestamp ='20140925-125118' #late twilight
    #obsTimestamp ='20140925-125221' #late twilight
    #obsTimestamp ='20140925-125324' #late twilight
    #obsTimestamp ='20140925-125427' #late twilight

    #date = '20140925'
    #obsTimestamp ='20140926-104528' #psr J0337


    obsFN = FileName(run=run,date=date,tstamp=obsTimestamp)
    obsFileName = obsFN.obs()
    print obsFileName
    timeMaskFileName = obsFN.timeMask()
    if wvlCal != '':
        wvlFileName = FileName(run=run,date=date,tstamp=wvlCal).calSoln()
    flatFileName = FileName(run=run,date=flatCalDate,tstamp=flatCalTstamp).flatSoln()
    illumFileName = FileName(run=run,date=flatCalDate,tstamp=flatCalTstamp).illumSoln()
    
    if not os.path.exists(timeMaskFileName):
        print 'Running hotpix for ',obsFileName
        hp.findHotPixels(inputFileName=obsFileName,outputFileName=timeMaskFileName)
        print "Flux file pixel mask saved to %s"%(timeMaskFileName)


    obs = ObsFile(obsFileName)