コード例 #1
0
def centroidObs(obsPath,centroidPath,centroidRa,centroidDec,haOffset,xGuess,yGuess,hotPath,flatPath):
    obs = ObsFile(obsPath)
    print obsPath,obs.getFromHeader('exptime'),obs
    if not os.path.exists(hotPath):
        hp.findHotPixels(obsFile=obs,outputFileName=hotPath)
    obs.loadHotPixCalFile(hotPath,switchOnMask=False)
    obs.loadBestWvlCalFile()
    obs.loadFlatCalFile(flatPath)
    obs.setWvlCutoffs(3000,8000)
    cc.centroidCalc(obs,centroidRa,centroidDec,guessTime=300,integrationTime=30,secondMaxCountsForDisplay=2000,HA_offset=haOffset,xyapprox=[xGuess,yGuess],outputFileName=centroidPath)
    print 'done centroid',centroidPath
    del obs
コード例 #2
0
def centroidObs(obsPath,centroidPath,centroidRa,centroidDec,haOffset,xGuess,yGuess):
    obs = ObsFile(obsPath)
    print obsPath,obs.getFromHeader('exptime'),obs
    obs.loadAllCals()
#    obs.loadBestWvlCalFile()
#    obs.loadFlatCalFile(flatPath)
    obs.setWvlCutoffs(3000,11000)
#    if not os.path.exists(hotPath):
#        hp.findHotPixels(obsFile=obs,outputFileName=hotPath,display=True,fwhm=2.,boxSize=5, nSigmaHot=4.0,)
#    obs.loadHotPixCalFile(hotPath,switchOnMask=True)
    cc.centroidCalc(obs,centroidRa,centroidDec,guessTime=300,integrationTime=30,secondMaxCountsForDisplay=2000,HA_offset=haOffset,xyapprox=[xGuess,yGuess],outputFileName=centroidPath,usePsfFit=True,radiusOfSearch=8)
    print 'done centroid',centroidPath
    del obs
コード例 #3
0
def main():
    """
    params = []
    paramfile = sys.argv[1]

    f = open(paramfile,'r')
    for line in f:
        params.append(line)
    f.close()
    datadir = params[0].split('=')[1].strip()
    flatdir = params[1].split('=')[1].strip()
    fluxdir = params[2].split('=')[1].strip()
    wvldir = params[3].split('=')[1].strip()
    obsfile = params[4].split('=')[1].strip()
    skyfile = params[5].split('=')[1].strip()
    flatfile = params[6].split('=')[1].strip()
    fluxfile = params[7].split('=')[1].strip()
    wvlfile = params[8].split('=')[1].strip()
    objectName = params[9].split('=')[1].strip()
    fluxCalObject = params[10].split('=')[1].strip()

    obsFileName = os.path.join(datadir, obsfile)
    skyFileName = os.path.join(datadir, skyfile)
    wvlCalFileName = os.path.join(wvldir, wvlfile)
    flatCalFileName = os.path.join(flatdir, flatfile)
    fluxCalFileName = os.path.join(fluxdir, fluxfile)
    """

    if len(sys.argv) >2:
        fileNum = str(sys.argv[2])
    else:
        fileNum = '0'

    #science object parameter file
    params = []
    paramfile = sys.argv[1]
    f = open(paramfile,'r')
    for line in f:
        params.append(line)
    f.close()

    datadir = params[0].split('=')[1].strip()
    flatdir = params[1].split('=')[1].strip()
    wvldir = params[2].split('=')[1].strip()
    obsfile = params[3].split('=')[1].strip()
    skyfile = params[4].split('=')[1].strip()
    flatfile = params[5].split('=')[1].strip()
    wvlfile = params[6].split('=')[1].strip()
    objectName = params[9].split('=')[1].strip()

    wvldir = "/Scratch/waveCalSolnFiles/oldbox_numbers/20121206"

    if len(params)>10:
        xpix = int(params[10].split('=')[1].strip())
        ypix = int(params[11].split('=')[1].strip())
        apertureRadius = int(params[12].split('=')[1].strip())
        startTime = int(params[13].split('=')[1].strip())
        intTime =int(params[14].split('=')[1].strip())

    obsFileName = os.path.join(datadir, obsfile)
    skyFileName = os.path.join(datadir, skyfile)
    wvlCalFileName = os.path.join(wvldir, wvlfile)
    flatCalFileName = os.path.join(flatdir, flatfile)

    obs = ObsFile(obsFileName)
    obs.loadWvlCalFile(wvlCalFileName)
    obs.loadFlatCalFile(flatCalFileName)
    print "analyzing file %s"%(obsFileName)
    print "loaded data file and calibrations\n---------------------\n"

    nRow = obs.nRow
    nCol = obs.nCol
    obsTime = obs.getFromHeader("exptime")
    #wvlBinEdges,obsSpectra = loadSpectra(obs,nCol,nRow)
    #nWvlBins=len(wvlBinEdges)-1

    #print np.shape(obsSpectra)
    #print nRow
    #print nCol
    #print nWvlBins

    #load/generate hot pixel mask file
    HotPixFile = getTimeMaskFileName(obsFileName)
    if not os.path.exists(HotPixFile):
        hp.findHotPixels(obsFileName,HotPixFile)
        print "Flux file pixel mask saved to %s"%(HotPixFile)
    obs.loadHotPixCalFile(HotPixFile)
    print "Hot pixel mask loaded %s"%(HotPixFile)

    #######
    #EVERYTHING BEFORE HERE IS STANDARD FILE/CALFILE LOADING

    
    startWvl = 3000
    #stopWvl = 7000 #for V-band
    stopWvl = 9000 #for R-band

    print "Making spectral cube"
    #for pg0220 first sec should be 80 since object is moving around before this
    #for pg0220A first sec should be 70, integration time is 140
    #for landolt 9542 first sec should be 20, int time is -1
    cubeDict = obs.getSpectralCube(firstSec=startTime, integrationTime=intTime, wvlStart = startWvl, wvlStop = stopWvl, wvlBinEdges = [startWvl,stopWvl], weighted=False)

    cube= np.array(cubeDict['cube'], dtype=np.double)
    wvlBinEdges= cubeDict['wvlBinEdges']
    effIntTime= cubeDict['effIntTime']
    print "median effective integration time = ", np.median(effIntTime)

    nWvlBins=len(wvlBinEdges)-1
    print "cube shape ", np.shape(cube)
    print "effIntTime shape ", np.shape(effIntTime)

    #add third dimension to effIntTime for  broadcasting
    effIntTime = np.reshape(effIntTime,np.shape(effIntTime)+(1,))
    cube /= effIntTime #put cube into counts/s
    
    #calculate midpoints of wvl bins for plotting
    wvls = np.empty((nWvlBins),dtype=float)
    for n in xrange(nWvlBins):
        binsize=wvlBinEdges[n+1]-wvlBinEdges[n]
        wvls[n] = (wvlBinEdges[n]+(binsize/2.0))

    print "wvls ",wvls
    #reshape cube for makeMovie
    movieCube = np.zeros((nWvlBins,np.shape(cube)[0],np.shape(cube)[1]),dtype=float)
    for i in xrange(nWvlBins):
        movieCube[i,:,:] = cube[:,:,i]
        #show individual frames as they are made to debug
        #plt.matshow(movieCube[i],vmin = 0, vmax = 100)
        #plt.show()
    print "movieCube shape ", np.shape(movieCube)
    print "wvls shape ", np.shape(wvls)

    #print cube
    #print "--------------------------"
    #print movieCube

    print "adding frames with wvl below ", stopWvl
    finalframe = np.zeros((1,np.shape(movieCube)[1],np.shape(movieCube)[2]))
    for f in xrange(len(wvls[wvls<stopWvl])):
        print wvls[f]
        finalframe[0]+=movieCube[f]
        plt.matshow(movieCube[f],vmin=0,vmax = 40)
        plt.show()

    movieCube = finalframe

    np.savez('%s_%s.npz'%(objectName,fileNum),stack=movieCube,wvls=wvls)
    print "Saved frame to .npz file"
    
    plt.matshow(movieCube[0],vmin=0,vmax = 40)
    plt.show()
コード例 #4
0
        if not os.path.exists(hotPixFn):
            hp.findHotPixels(obsFn, hotPixFn)
            print "Flux file pixel mask saved to %s" % (hotPixFn)
        ob.loadHotPixCalFile(hotPixFn, switchOnMask=True)
        ob.loadWvlCalFile(wfn)
        ob.loadFlatCalFile(ffn)
        ob.setWvlCutoffs(wvlLowerCutoff, wvlUpperCutoff)

        bad_solution_mask = np.zeros((46, 44))
        bad_count = 0
        for y in range(46):
            for x in range(44):
                if (ob.wvlRangeTable[y][x][1] < wvlUpperCutoff) or (ob.wvlRangeTable[y][x][0] > wvlLowerCutoff):
                    bad_solution_mask[y][x] = 1

        unix = ob.getFromHeader("unixtime")
        startJD = unix / 86400.0 + 2440587.5
        nSecInFile = ob.getFromHeader("exptime")
        deadMask = ob.getDeadPixels()

        y_values, x_values = np.where(
            np.logical_and(bad_solution_mask == 0, np.logical_and(apertureMask != 0, deadMask == 1))
        )
        y_sky, x_sky = np.where(np.logical_and(bad_solution_mask == 0, np.logical_and(skyMask != 0, deadMask == 1)))
        # tic = time()
        for sec in np.arange(0, nSecInFile, integrationTime):
            jd = (
                startJD + sec / (24.0 * 3600.0) + integrationTime / 2.0 / (24.0 * 3600.0)
            )  # add seconds offset to julian date, move jd to center of bin
            print count, jd
            count += 1
コード例 #5
0
def main():

    run = 'PAL2014'
    year = '2014'
    initialPath = '/Scratch'
    packetMasterLogDir = '/LABTEST/PacketMasterLogs/'
    initialPath = os.path.join(initialPath,run)
    outPath = FileName(run=run,mkidDataDir='/Scratch/').timeAdjustments()
    outFile = tables.openFile(outPath,'w')
    timeAdjustGroup = outFile.createGroup('/','timeAdjust','Times to add to timestamps')
    firmwareDelayTable = outFile.createTable(timeAdjustGroup,'firmwareDelay',firmwareDelayDescription,'Times to add to all timestamps taken with a firmware bof')
    newFirmwareEntry = firmwareDelayTable.row
    newFirmwareEntry['firmwareName']='chan_svf_2014_Aug_06_1839.bof'
    newFirmwareEntry['firmwareDelay']=-41e-6 #s, subtract 41 us from timestamps
    newFirmwareEntry.append()
    firmwareDelayTable.flush()
    firmwareDelayTable.close()
    roachDelayTable = outFile.createTable(timeAdjustGroup,'roachDelays',roachDelaysDescription,'Times to add to each roach\'s timestamps')


    for sunsetDatePath in sorted(glob.glob(os.path.join(initialPath,year+'*'))):
        sunsetDate = os.path.basename(sunsetDatePath)
        for fullObsPath in sorted(glob.glob(os.path.join(sunsetDatePath,'obs*.h5'))):
            obsFileName = os.path.basename(fullObsPath)
            obsTStamp = obsFileName.split('.')[0].split('_')[1]
            print obsFileName
        

            obsFN = FileName(run=run,date=sunsetDate,tstamp=obsTStamp,packetMasterLogDir=packetMasterLogDir)
            pmLogFileName = obsFN.packetMasterLog()
            try:
                ob = ObsFile(fullObsPath)
            except:
                print 'can\'t open file'
                continue
                
            try:
                if os.path.getsize(pmLogFileName) <= 0:
                    continue
            except:
                print 'can\'t open Packet Master Log ',pmLogFileName
                continue

            try:
                f = open(pmLogFileName,'r')
            except:
                print 'can\'t open Packet Master Log ',pmLogFileName
                continue
            lastTstampLines = np.zeros(8)
            firstTstampLines = np.zeros(8)

            tstampLine = ''
            for line in f:
                if 'here' in line:
                    #skip over large log files with debug info
                    print 'skipping file with "here"'
                    continue
                if line.split(' ')[0] == 'bundle':
                    tstampLine = line
                    break

            if tstampLine == '':
                print 'skipping file without "bundle"'
                #didn't find lines with 'bundle' in them
                continue

            f.seek(0)
            for line in f:
                if line.split(' ')[0] == 'bundle':
                    try:
                        at = float(line.split('at')[1].split()[0])
                    except:
                        break
                    lastTstampLine = at
                    iRoach = int(line.split('roach')[1].split('took')[0].strip())
                    lastTstampLines[iRoach] = at
                    if firstTstampLines[iRoach] == 0:
                        firstTstampLines[iRoach] = at
                    
            packetReceivedUnixTimestamp = float((tstampLine.split('took')[1].split('total')[0].strip()))
            firstPacketDelay = packetReceivedUnixTimestamp-int(ob.getFromHeader('unixtime'))
                

            roachSecDelays =np.array(np.floor(lastTstampLines+firstPacketDelay-ob.getFromHeader('exptime')),dtype=np.int)
            print 'roach delays',roachSecDelays
            if np.all(roachSecDelays >= 0):
                newEntry = roachDelayTable.row
                newEntry['obsFileName'] = os.path.basename(fullObsPath)
                newEntry['roachDelays'] = roachSecDelays
                newEntry.append()
                roachDelayTable.flush()
            else:
                print 'obs was aborted midway, delays cannot be calculated'
    roachDelayTable.close()
    outFile.close()
コード例 #6
0
class Cosmic:

    def __init__(self, fn, beginTime=0, endTime='exptime',
                 nBinsPerSec=10, flashMergeTime=1.0, 
                 applyCosmicMask = False,
                 loggingLevel=logging.CRITICAL,
                 loggingHandler=logging.StreamHandler()):
        
        """
        Opens fileName in MKID_RAW_PATH, sets roachList
        endTime is exclusive
        """
        self.logger = logging.getLogger("cosmic")
        self.logger.setLevel(loggingLevel)
        formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
        loggingHandler.setFormatter(formatter)
        self.logger.addHandler(loggingHandler)
        self.logger.info("Cosmic:  begin init for obsFile=%s"%fn.obs())
        self.fn = fn
        self.fileName = fn.obs();
        self.file = ObsFile(self.fileName)
        # apply Matt's time fix
        timeAdjustments = self.fn.timeAdjustments()
        if os.path.exists(timeAdjustments):
            self.file.loadTimeAdjustmentFile(timeAdjustments)
        # apply Julian's time masks
        timeMaskFile = self.fn.timeMask();
        if os.path.exists(timeMaskFile):
            self.file.loadHotPixCalFile(timeMaskFile,switchOnMask=True)

        # apply standard mask
        if applyCosmicMask:
            self.file.loadStandardCosmicMask()

        self._setRoachList()
        self._setAllSecs()
        self.exptime = self.file.getFromHeader('exptime')
        if endTime =='exptime':
            self.endTime = float(self.exptime)
        else:
            self.endTime = float(endTime)
        if ( (self.endTime > self.exptime) or (endTime < 0)):
            raise RuntimeError("bad endTime:  endTime=%s exptime=%s" % \
                                   (str(endTime),str(self.exptime)))

        self.beginTime = float(beginTime)
        self.timeHgs = "none"
        self.nBinsPerSec = nBinsPerSec
        self.flashMergeTime = flashMergeTime

        self.times = \
        np.arange(self.beginTime, self.endTime, 1.0/self.nBinsPerSec)

        # for measuring flashes, indexed by roach name
        self.rMean = {}   # mean from meanclip
        self.rSigma = {}  # sigma from meanclip
        self.rNSurvived = {} # number of survivors from meanclip
        self.rNormed = {} # (value-mean)/sigma
        self.flashInterval = {}
        self.logger.info("Cosmic:  end init:  beginTime=%s endTime=%s"%(str(self.beginTime),str(self.endTime)))
    def __del__(self):
        """
        Close any open files
        """
        # print "now in Cosmic.__del__ for ",self.fileName
        try:
            del self.file
        except:
            pass

    def _setRoachList(self):
        self.roachList = []
        for row in self.file.beamImage:
            for name in row:
                roachName = name.split("/")[1]
                if roachName not in self.roachList:
                    self.roachList.append(roachName)
        self.roachList.sort()

    def _setAllSecs(self):
       nRow = self.file.nRow
       nCol = self.file.nCol
       self.allSecs =  \
           dict( ((i,j),None) for i in range(nRow) for j in range(nCol))
       for iRow in np.arange(nRow):
           for iCol in np.arange(nCol):
               self.allSecs[iRow,iCol] = \
                   self.file.file.getNode(self.file.beamImage[iRow][iCol])

    def nPhoton(self, beginTime=0, endTime='expTime'):
        """
        trivial example of counting the number of photons in a file
        """
        nPhoton = 0
        for iRow in range(self.file.nRow):
            for iCol in range(self.file.nCol):
                for iSec in range(self.beginTime, self.endTime):
                    sec = self.allSecs[iRow,iCol][iSec]
                    nPhoton += len(sec)
        return nPhoton

    def findFlashes(self, clipsig=3.0, maxiter=5,\
                        converge_num=0.05, verbose=0, flashsig=6):
        """
        find flashes by looking at the time histograms.  Calculate the
        mean,sigma using meanclip and the parameters clipsig, maxiter, converge_num
        A time bin has a flash if the normalized signal (measured-mean)/sigma
        is larger than flashsig.

        
        """
        # make the blank data structures
        if self.timeHgs == "none":
            self.makeTimeHgs()
            self.flashInterval["all"] = []
        # find the flashes in each roach 
        for roach in self.roachList:
            self.rMean[roach],self.rSigma[roach],self.rNSurvived[roach] = \
                meanclip.meanclip(\
                np.array(self.timeHgs[roach]), clipsig, maxiter, converge_num,\
                    verbose)
            self.rNormed[roach] = \
                (self.timeHgs[roach]-self.rMean[roach])/self.rSigma[roach]

            self.flashInterval[roach] = interval()
            prev = 0
            a = self.rNormed[roach]
            for i in range(len(a)):
                this = a[i] > flashsig
                if (this != prev):
                    if (this):
                        iBegin = i
                    else:
                        self.flashInterval[roach] = \
                        self.flashInterval[roach] | interval[iBegin,i]
                prev = this
            if (prev):
                self.flashInterval[roach] = \
                    self.flashInterval[roach] | interval[iBegin,i]

        # union of all flashes
        self.flashInterval["all"] = interval()
        for roach in self.roachList:
            self.flashInterval["all"] = \
                self.flashInterval["all"] | self.flashInterval[roach]

        # look for gaps smaller than self.flashMergeTime and plug them
        dMax = self.nBinsPerSec*self.flashMergeTime
        extrema = self.flashInterval["all"].extrema
        for i in range(len(extrema)/2-1):
            i0 = extrema[2*i+1][0]
            i1 = extrema[2*(i+1)][0]
            if (i1-i0) <= dMax:
                t0 = self.beginTime + float(i0)/self.nBinsPerSec
                t1 = self.beginTime + float(i1)/self.nBinsPerSec
                self.flashInterval["all"] = \
                    self.flashInterval["all"] | interval[i0,i1]

        # convert to ticks since the beginning of the data file
        rlAll = list(self.roachList)
        rlAll.append("all")
        ticksPerSecond = int(1.0/self.file.tickDuration)
        offset = self.beginTime*ticksPerSecond
        scale = 1.0/(self.file.tickDuration*self.nBinsPerSec)
        for roach in rlAll:
            self.flashInterval[roach] = offset+scale*self.flashInterval[roach]

    def writeFlashesToHdf5(self,overwrite=1):
        """
        write intervals with flashes to the timeMask file
        """
        # get the output file name, and make the directory if you need to
        cmFileName = self.fn.cosmicMask()
        (cmDir,name) = os.path.split(cmFileName)
        if not os.path.exists(cmDir):
            os.makedirs(cmDir)

        # write parameters used to find flashes
        h5f = tables.openFile(cmFileName, 'w')
        fnode = filenode.newNode(h5f, where='/', name='timeMaskHdr')
        fnode.attrs.beginTime      = self.beginTime
        fnode.attrs.endTime        = self.endTime
        fnode.attrs.nBinsPerSec    = self.nBinsPerSec
        fnode.attrs.flashMergeTime = self.flashMergeTime
        fnode.close();

        # write the times where flashes are located
        tbl =  h5f.createTable('/','timeMask',TimeMask.TimeMask,"Time Mask")
        rlAll = list(self.roachList)
        rlAll.append("all")
        for roach in rlAll:
            extrema = self.flashInterval[roach].extrema
            for i in range(len(extrema)/2):
                row = tbl.row
                row['tBegin'] = int(extrema[2*i][0])
                row['tEnd'] = int(extrema[2*i+1][0])
                if (roach == "all"):
                    reason = "Merged Flash"
                else:
                    reason = "Flash in %s" % roach
                row['reason'] = TimeMask.timeMaskReason[reason]
                row.append()
                tbl.flush()
        tbl.close()
        h5f.close()

    def makeTimeHgs(self):
        """
        Fill in the timeHgs variable
        This is a dictionary, indexed by the roach name, of the time histograms
        """
        self.timeHgs = {}
        for iSec in range(self.beginTime, self.endTime):
            self.logger.info("in makeTimeHgs iSec=%4d / %4d" % (iSec,self.endTime))
            hgsThisSec = {}
            for iRow in range(self.file.nRow):
                for iCol in range(self.file.nCol):
                    sec = self.allSecs[iRow,iCol][iSec]
                    if len(sec) > 0:
                        times = sec & self.file.timestampMask
                        hg,edges = \
                        np.histogram(times,bins=self.nBinsPerSec, \
                                     range=(0,1.0/self.file.tickDuration))
                        roachName = \
                        self.file.beamImage[iRow][iCol].split("/")[1]
                        if not hgsThisSec.has_key(roachName):
                            hgsThisSec[roachName] = \
                            np.zeros(self.nBinsPerSec,dtype=np.int64)
                        hgsThisSec[roachName] += hg
            for roachName in hgsThisSec.keys():
                if not self.timeHgs.has_key(roachName):
                    self.timeHgs[roachName] = []
                self.timeHgs[roachName] += list(hgsThisSec[roachName])

    def plotTimeHgs(self):
        """
        Plot the time HGS in plt structure, with legend
        """
        plt.clf()
        plt.figure(1)
        keys = self.timeHgs.keys()
        keys.sort()

        plt.subplot(211)
        for roachName in keys:
            hg = self.timeHgs[roachName]
            plt.plot(self.times, hg,label=roachName)
        plt.legend()
        dt = 1.0/self.nBinsPerSec
        plt.ylabel("photons/%.2f sec" % dt)
        plt.title("Cosmic timeHgs "+ self.fileName)

        plt.subplot(212)
        for roachName in keys:
            plt.plot(self.times, \
                         self.rNormed[roachName],label=roachName)
        plt.xlabel("time (sec)")
        plt.ylim(-23,30)
        dt = 1.0/self.nBinsPerSec
        plt.ylabel("normalized photons/%.2f sec" % dt)

        y = -5
        x0 = self.beginTime + 0.1*(self.endTime-self.beginTime)
        xmax = plt.xlim()[1]
        rlAll = list(self.roachList)
        rlAll.append("all")
        for roach in rlAll:
            print "plot for roach=",roach
            plt.plot([x0,xmax],[y,y], linestyle=":", color="gray")
            plt.text(x0, y, roach, fontsize=8, va="center")
            extrema = self.flashInterval[roach].extrema
            for i in range(len(extrema)/2):
                t0 = (extrema[2*i][0]   - 0.5)*self.file.tickDuration
                t1 = (extrema[2*i+1][0] - 0.5)*self.file.tickDuration
                plt.plot([t0,t1],[y,y],'r', linewidth=4)
            y -= 2

    def findCosmics(self, stride=10, threshold=100, 
                    populationMax=2000, nSigma=5, writeCosmicMask=False,
                    ppsStride=10000):
        """
        Find cosmics ray suspects.  Histogram the number of photons
        recorded at each timeStamp.  When the number of photons in a
        group of stride timeStamps is greater than threshold in second
        iSec, add (iSec,timeStamp) to cosmicTimeLists.  Also keep
        track of the histogram of the number of photons per stride
        timeStamps.

        return a dictionary of 'populationHg', 'cosmicTimeLists',
        'binContents', 'timeHgValues', 'interval', 'frameSum', and 'pps'
      
         populationHg is a histogram of the number of photons in each time bin.
        This is a poisson distribution with a long tail due to cosmic events

        
        cosmicTimeLists is a numpy array  of all the sequences that are
        suspects for cosmic rays


        binContents corresponds to cosmicTimeLists.  For each time in
        cosmicTimeLists, binContents is the number of photons detected
        at that time.
        
        timeHgValues is a histogram of the number of photons in each time
        interval
        
        frameSum is a two dimensional  numpy array of the number of photons
        detected by each pixel

        interval is the interval of data to be masked out

        pps is photons per second, calculated every ppsStride bins.

        """

        self.logger.info("findCosmics: begin stride=%d threshold=%d populationMax=%d nSigma=%d writeCosmicMask=%s"%(stride,threshold,populationMax,nSigma,writeCosmicMask))
        exptime = self.endTime-self.beginTime
        nBins = int(np.round(self.file.ticksPerSec*exptime+1))
        bins = np.arange(0, nBins, 1)
        timeHgValues,frameSum = self.getTimeHgAndFrameSum(self.beginTime,self.endTime)
        remainder = len(timeHgValues)%ppsStride
        if remainder > 0:
            temp = timeHgValues[:-remainder]
        else:
            temp = timeHgValues
        ppsTime = (ppsStride*self.file.tickDuration)
        pps = np.sum(temp.reshape(-1, ppsStride), axis=1)/ppsTime
        self.logger.info("findCosmics:  call populationFromTimeHgValues")
        pfthgv = Cosmic.populationFromTimeHgValues\
            (timeHgValues,populationMax,stride,threshold)
        #now build up all of the intervals in seconds
        self.logger.info("findCosmics:  build up intervals:  nCosmicTime=%d"%len(pfthgv['cosmicTimeList']))
        i = interval()
        iCount = 0
        secondsPerTick = self.file.tickDuration
        for cosmicTime in pfthgv['cosmicTimeList']:
            #t0 = max(0,self.beginTime+(cosmicTime-50)/1.e6)
            #t1 = min(self.endTime,self.beginTime+(cosmicTime+50)/1.e6)
            #intTime = t1-t0            
            t0 = self.beginTime+cosmicTime*secondsPerTick
            dt = stride*secondsPerTick
            t1 = t0+dt
            left = max(self.beginTime, t0-nSigma*dt)
            right = min(self.endTime, t1+2*nSigma*dt)
            i = i | interval[left,right]
            self.logger.debug("findCosmics:  iCount=%d t0=%f t1=%f left=%f right=%f"%(iCount,t0,t1,left,right))
            iCount+=1

        tMasked = Cosmic.countMaskedBins(i)
        ppmMasked = 1000000*tMasked/(self.endTime-self.beginTime)

        retval = {}
        retval['timeHgValues'] = timeHgValues
        retval['populationHg'] = pfthgv['populationHg']
        retval['cosmicTimeList'] = pfthgv['cosmicTimeList']
        retval['binContents'] = pfthgv['binContents']
        retval['frameSum'] = frameSum
        retval['interval'] = i
        retval['ppmMasked'] = ppmMasked
        retval['pps'] = pps
        retval['ppsTime'] = ppsTime
        if writeCosmicMask:
            cfn = self.fn.cosmicMask()
            self.logger.info("findCosmics:  write masks to =%s"%cfn)
            ObsFile.writeCosmicIntervalToFile(i, self.file.ticksPerSec, 
                                              cfn,self.beginTime, self.endTime, 
                                              stride, threshold, nSigma, populationMax)
        self.logger.info("findCosmics:  end with ppm masked=%d"%ppmMasked)
        return retval
    def getTimeHgAndFrameSum(self, beginTime, endTime):
        integrationTime = endTime - beginTime
        nBins = int(np.round(self.file.ticksPerSec*integrationTime+1))
        timeHgValues = np.zeros(nBins, dtype=np.int64)
        frameSum = np.zeros((self.file.nRow,self.file.nCol))
        self.logger.info("get all time stamps for integrationTime=%f"%integrationTime)
        for iRow in range(self.file.nRow):
            #print "Cosmic.findCosmics:  iRow=",iRow
            for iCol in range(self.file.nCol):
                # getTimedPacketList is slow.  Use getPackets instead.
                #gtpl = self.file.getTimedPacketList(iRow,iCol,beginTime, 
                #                                    integrationTime)
                gtpl = self.file.getPackets(iRow,iCol,
                                            beginTime,integrationTime)
                timestamps = gtpl['timestamps']
                if timestamps.size > 0:
                    timestamps = \
                        (timestamps - beginTime)*self.file.ticksPerSec
                    # per Matt S. suggestion 2013-07-09
                    ts32 = np.round(timestamps).astype(np.uint32)
                    tsBinner.tsBinner32(ts32, timeHgValues)
                    frameSum[iRow,iCol] += ts32.size

        return timeHgValues,frameSum
    @staticmethod
    def countMaskedBins(maskInterval):
        retval = 0
        for x in maskInterval:
            retval += x[1]-x[0]
        return retval

    @staticmethod
    def populationFromTimeHgValues(timeHgValues,populationMax,stride,threshold):
        """
        Rebin the timgHgValues histogram by combining stride bins.  If
        stride > 1, then bin a second time after shifting by stride/2
        Create populationHg, a histogram of the number of photons in
        the large bins.  Also, create (and then sort) a list
        cosmicTimeList of the start of bins (in original time units)
        of overpopulated bins that have more than threshold number of
        photons.

        return a dictionary containing populationHg and cosmicTimeList
        """
        popRange = (-0.5,populationMax-0.5)
        if stride==1:
            populationHg = np.histogram(\
                timeHgValues, populationMax, range=popRange)
            cosmicTimeList = np.where(timeHgValues > threshold)[0]
            binContents = np.extract(timeHgValues > threshold, timeHgValues)
        else:
            # rebin the timeHgValues before counting the populations
            length = timeHgValues.size
            remainder = length%stride
            if remainder == 0:
                end = length
            else:
                end = -remainder

            timeHgValuesTrimmed = timeHgValues[0:end]

            timeHgValuesRebinned0 = np.reshape(
                timeHgValuesTrimmed, [length/stride, stride]).sum(axis=1)
            populationHg0 = np.histogram(
                timeHgValuesRebinned0, populationMax, range=popRange)
            cosmicTimeList0 = stride*np.where(\
                timeHgValuesRebinned0 > threshold)[0]
            binContents0 = np.extract(timeHgValuesRebinned0 > threshold,
                                      timeHgValuesRebinned0)

            timeHgValuesRebinned1 = np.reshape(
                timeHgValuesTrimmed[stride/2:-stride/2], 
                [(length-stride)/stride, stride]).sum(axis=1)
            populationHg1 = np.histogram(\
                timeHgValuesRebinned1, populationMax, range=popRange)
            cosmicTimeList1 = (stride/2)+stride*np.where(\
                timeHgValuesRebinned1 > threshold)[0]
            binContents1 = np.extract(timeHgValuesRebinned1 > threshold,
                                      timeHgValuesRebinned1)

            populationHg = (populationHg0[0]+populationHg1[0],\
                                populationHg0[1])
            cosmicTimeList = np.concatenate((cosmicTimeList0,cosmicTimeList1))
            binContents = np.concatenate((binContents0, binContents1))
            args = np.argsort(cosmicTimeList)
            cosmicTimeList = cosmicTimeList[args]
            binContents = binContents[args]
            cosmicTimeList.sort()

        retval = {}
        retval['populationHg'] = populationHg
        retval['cosmicTimeList'] = cosmicTimeList
        retval['binContents'] = binContents
        return retval

    def makeMovies(self,beginTick, endTick, backgroundFrame, accumulate=False):
        tick0 = np.uint64(beginTick)
        tick1 = np.uint64(endTick)
        for iRow in range(cosmic.file.nRow):
            for iCol in range(cosmic.file.nCol):
                gtpl = self.getTimedPacketList(iRow,iCol,sec0,1)
        timestamps = gtpl['timestamps']
        timestamps *= cosmic.file.ticksPerSec
        ts32 = timestamps.astype(np.uint32)
        for ts in ts32:
            tindex = ts-t0
            try:
                listOfPixelsToMark[tindex].append((iRow,iCol))
            except IndexError:
                pass
            for tick in range(t0,t1):
                frames.append(frameSum)
                title = makeTitle(tick,t0,t1)
                titles.append(title)

                mfn0 = "m-%s-%s-%s-%s-%010d-%010d-i.gif"%(run,sundownDate,obsDate,seq,t0,t1)
                utils.makeMovie(frames, titles, outName=mfn0, delay=0.1, colormap=mpl.cm.gray,
                                listOfPixelsToMark=listOfPixelsToMark,
                                pixelMarkColor='red')

        for i in range(len(listOfPixelsToMark)-1):
            listOfPixelsToMark[i+1].extend(listOfPixelsToMark[i])

        mfn1 = "m-%s-%s-%s-%s-%010d-%010d-a.gif"%(run,sundownDate,obsDate,seq,t0,t1)
        utils.makeMovie(frames, titles, outName=mfn1, delay=0.1, colormap=mpl.cm.gray,
                        listOfPixelsToMark=listOfPixelsToMark,
                        pixelMarkColor='green')

    def fitDecayTime(self,t0Sec,lengthSec=200,plotFileName='none'):
        print "hello from fitDecayTime"
        timedPacketList = self.file.getTimedPacketList(
            iRow, iCol, sec0, lengthSec)

    
    def fitExpon(self, t0, t1):
        """
        Fit an exponential to all photons from time t0 to time t1
        t0 and t1 are in ticks, 1e6 ticks per second
        return a dictionary of:  timeStamps,fitParams,chi2

        """
         
        firstSec = int(t0/1e6)  # in seconds
        integrationTime = 1+int((t1-t0)/1e6) # in seconds
        nBins = integrationTime*1e6 # number of microseconds; one bin per microsecond
        timeHgValues = np.zeros(nBins, dtype=np.int64)
        print "firstSec=",firstSec," integrationTime=",integrationTime
        for iRow in range(self.file.nRow):
            for iCol in range(self.file.nCol):
                timedPacketList = self.file.getTimedPacketList(
                    iRow, iCol, firstSec=firstSec, 
                    integrationTime=integrationTime)
                timeStamps = timedPacketList['timestamps']
                if (len(timeStamps) > 0):
                    # covert the time values to microseconds, and
                    # make it the type np.uint64
                    # round per Matt S. suggestion 2013-07-09
                    #ts64 = (timeStamps).astype(np.uint64)
                    ts32round = np.round(timeStamps).astype(np.uint32)
                    tsBinner.tsBinner(ts32round, timeHgValues)
        
                    temp = 1e6*(timeStamps-firstSec)
                    for i in range(len(timeStamps)):
                        ts32 = ((timeStamps-firstSec)*1e6).astype(np.uint32)
                    # add these timestamps to the histogram timeHgValues
        remain0 = int(t0%1e6)
        remain1 = int(t1%1e6)
        timeHgValues = timeHgValues[remain0:remain1]
        x = np.arange(len(timeHgValues))
        y = timeHgValues
        
        xArray = np.arange(0, dtype=np.int64)
        yArray = np.arange(0, dtype=np.int64)

        for i in range(len(x)):
            if y[i] > 2:
                xArray = np.append(xArray,i)
                yArray = np.append(yArray,y[i])
        ySigma = np.sqrt(yArray)


        mean = (x*y).sum()/float(y.sum())
        bExponGuess = 1/mean
        aExponGuess = bExponGuess*timeHgValues.sum()
        cExponGuess = 0
        dExponGuess = 0
        pExponGuess = [aExponGuess, bExponGuess, cExponGuess, dExponGuess]

        bGaussGuess = mean
        avgx2 = (x*x*y).sum()/float(y.sum())        
        cGaussGuess = np.sqrt(avgx2-bGaussGuess*bGaussGuess)
        
        aGaussGuess = (timeHgValues.sum()/(cGaussGuess*np.sqrt(2*np.pi)))
        pGaussGuess = [aGaussGuess, bGaussGuess, cGaussGuess]
        
        xLimit = [bGaussGuess-4*cGaussGuess, bGaussGuess+4*cGaussGuess]
        retval = {'timeHgValues':timeHgValues, 'pExponFit':pExponGuess, 
                  'pGaussFit':pGaussGuess, 'xLimit':xLimit, 
                  'cGaussGuess':cGaussGuess, 'timeStamps':timeStamps}

        return retval

    @staticmethod
    def intervalTime(intervals):
        """
        return the time (in seconds) masked by the intervals
        """
        time = 0
        for interval in intervals:
            time += interval[1]-interval[0]
        return time

    @staticmethod
    def funcExpon(x, a, b, c, d):
        retval = a*np.exp(-b*(x-d)) + c
        retval[x < d] = 0
        return retval

    @staticmethod
    def funcGauss(x, a, b, c):
        return a*np.exp(-(x-b)**2/(2.*c**2))
コード例 #7
0
def main():
    """
    params = []
    paramfile = sys.argv[1]

    f = open(paramfile,'r')
    for line in f:
        params.append(line)
    f.close()
    datadir = params[0].split('=')[1].strip()
    flatdir = params[1].split('=')[1].strip()
    fluxdir = params[2].split('=')[1].strip()
    wvldir = params[3].split('=')[1].strip()
    obsfile = params[4].split('=')[1].strip()
    skyfile = params[5].split('=')[1].strip()
    flatfile = params[6].split('=')[1].strip()
    fluxfile = params[7].split('=')[1].strip()
    wvlfile = params[8].split('=')[1].strip()
    objectName = params[9].split('=')[1].strip()
    fluxCalObject = params[10].split('=')[1].strip()

    obsFileName = os.path.join(datadir, obsfile)
    skyFileName = os.path.join(datadir, skyfile)
    wvlCalFileName = os.path.join(wvldir, wvlfile)
    flatCalFileName = os.path.join(flatdir, flatfile)
    fluxCalFileName = os.path.join(fluxdir, fluxfile)
    """

    if len(sys.argv) > 2:
        fileNum = str(sys.argv[2])
    else:
        fileNum = "0"

    # science object parameter file
    params = []
    paramfile = sys.argv[1]
    f = open(paramfile, "r")
    for line in f:
        params.append(line)
    f.close()

    datadir = params[0].split("=")[1].strip()
    flatdir = params[1].split("=")[1].strip()
    wvldir = params[2].split("=")[1].strip()
    obsfile = params[3].split("=")[1].strip()
    skyfile = params[4].split("=")[1].strip()
    flatfile = params[5].split("=")[1].strip()
    wvlfile = params[6].split("=")[1].strip()
    objectName = params[9].split("=")[1].strip()

    # wvldir = "/Scratch/waveCalSolnFiles/oldbox_numbers/20121205"
    # objectName = "crabNight1"

    if len(params) > 10:
        xpix = int(params[10].split("=")[1].strip())
        ypix = int(params[11].split("=")[1].strip())
        apertureRadius = int(params[12].split("=")[1].strip())
        startTime = int(params[13].split("=")[1].strip())
        intTime = int(params[14].split("=")[1].strip())

    obsFileName = os.path.join(datadir, obsfile)
    skyFileName = os.path.join(datadir, skyfile)
    wvlCalFileName = os.path.join(wvldir, wvlfile)
    flatCalFileName = os.path.join(flatdir, flatfile)

    obs = ObsFile(obsFileName)
    # obs.loadWvlCalFile(wvlCalFileName)
    obs.loadBestWvlCalFile()
    obs.loadFlatCalFile(flatCalFileName)
    print "analyzing file %s" % (obsFileName)
    print "loaded data file and calibrations\n---------------------\n"

    nRow = obs.nRow
    nCol = obs.nCol
    obsTime = obs.getFromHeader("exptime")
    # wvlBinEdges,obsSpectra = loadSpectra(obs,nCol,nRow)
    # nWvlBins=len(wvlBinEdges)-1

    # print np.shape(obsSpectra)
    # print nRow
    # print nCol
    # print nWvlBins

    # Apply Hot pixel masking before getting dead time correction
    # HotPixFile = getTimeMaskFileName(obsFileName)
    HotPixFile = FileName(obsFile=obs).timeMask()
    print "making hot pixel file ", HotPixFile
    if not os.path.exists(HotPixFile):  # check if hot pix file already exists
        hp.findHotPixels(inputFileName=obsFileName, outputFileName=HotPixFile)
        print "Flux file pixel mask saved to %s" % (HotPixFile)
    obs.loadHotPixCalFile(HotPixFile)
    print "Hot pixel mask loaded %s" % (HotPixFile)

    # GET RAW PIXEL COUNT IMAGE TO CALCULATE CORRECTION FACTORS
    print "Making raw cube to get dead time correction"
    cubeDict = obs.getSpectralCube(firstSec=startTime, integrationTime=intTime, weighted=False)

    cube = np.array(cubeDict["cube"], dtype=np.double)
    wvlBinEdges = cubeDict["wvlBinEdges"]
    effIntTime = cubeDict["effIntTime"]
    print "median effective integration time = ", np.median(effIntTime)

    nWvlBins = len(wvlBinEdges) - 1
    print "cube shape ", np.shape(cube)
    print "effIntTime shape ", np.shape(effIntTime)

    # add third dimension to effIntTime for  broadcasting
    effIntTime = np.reshape(effIntTime, np.shape(effIntTime) + (1,))
    # put cube into counts/s in each pixel
    cube /= effIntTime

    # CALCULATE DEADTIME CORRECTION
    # NEED TOTAL COUNTS PER SECOND FOR EACH PIXEL TO DO PROPERLY
    # ASSUMES SAME CORRECTION FACTOR APPLIED FOR EACH WAVELENGTH, MEANING NO WL DEPENDANCE ON DEAD TIME EFFECT
    DTCorr = np.zeros((np.shape(cube)[0], np.shape(cube)[1]), dtype=float)
    for f in range(0, np.shape(cube)[2]):
        print cube[:, :, f]
        print "-----------------------"
        DTCorr += cube[:, :, f]
        print DTCorr
        print "\n=====================\n"
    # Correct for 100 us dead time
    DTCorrNew = DTCorr / (1 - DTCorr * 100e-6)
    CorrFactors = DTCorrNew / DTCorr  # This is what the frames need to be multiplied by to get their true values

    print "Dead time correction factors = "
    print CorrFactors

    # REMAKE CUBE WITH FLAT WEIGHTS AND APPLY DEAD TIME CORRECTION AS WELL
    print "Making Weighted cube"
    # load/generate hot pixel mask file
    # HotPixFile = getTimeMaskFileName(obsFileName)
    HotPixFile = FileName(obsFile=obs).timeMask()
    if not os.path.exists(HotPixFile):  # check if hot pix file already exists
        hp.findHotPixels(inputFileName=obsFileName, outputFileName=HotPixFile)
        print "Flux file pixel mask saved to %s" % (HotPixFile)
    obs.loadHotPixCalFile(HotPixFile)
    print "Hot pixel mask loaded %s" % (HotPixFile)

    cubeDict = obs.getSpectralCube(firstSec=startTime, integrationTime=intTime, weighted=True, fluxWeighted=False)
    # cubeDict = obs.getSpectralCube(firstSec=startTime, integrationTime=intTime, weighted=True, fluxWeighted=True)

    cube = np.array(cubeDict["cube"], dtype=np.double)
    wvlBinEdges = cubeDict["wvlBinEdges"]
    effIntTime = cubeDict["effIntTime"]
    print "median effective integration time = ", np.median(effIntTime)

    nWvlBins = len(wvlBinEdges) - 1
    print "cube shape ", np.shape(cube)
    print "effIntTime shape ", np.shape(effIntTime)

    # add third dimension to effIntTime for broadcasting
    effIntTime = np.reshape(effIntTime, np.shape(effIntTime) + (1,))
    # put cube into counts/s in each pixel
    cube /= effIntTime

    # add third dimension to CorrFactors for broadcasting
    CorrFactors = np.reshape(CorrFactors, np.shape(CorrFactors) + (1,))
    # apply dead time correction factors
    cube *= CorrFactors

    # calculate midpoints of wvl bins for plotting
    wvls = np.empty((nWvlBins), dtype=float)
    for n in xrange(nWvlBins):
        binsize = wvlBinEdges[n + 1] - wvlBinEdges[n]
        wvls[n] = wvlBinEdges[n] + (binsize / 2.0)

    print "wvls ", wvls
    # reshape cube for makeMovie
    movieCube = np.zeros((nWvlBins, np.shape(cube)[0], np.shape(cube)[1]), dtype=float)
    for i in xrange(nWvlBins):
        movieCube[i, :, :] = cube[:, :, i]
        # show individual frames as they are made to debug
        # plt.matshow(movieCube[i],vmin = 0, vmax = 100)
        # plt.show()
    print "movieCube shape ", np.shape(movieCube)
    print "wvls shape ", np.shape(wvls)

    # print cube
    # print "--------------------------"
    # print movieCube

    outdir = "/home/srmeeker/scratch/standards/"
    np.savez(outdir + "%s_raw_%s.npz" % (objectName, fileNum), stack=movieCube, wvls=wvls)
    utils.makeMovie(
        movieCube,
        frameTitles=wvls,
        cbar=True,
        outName=outdir + "%s_raw_%s.gif" % (objectName, fileNum),
        normMin=0,
        normMax=50,
    )

    """
コード例 #8
0
def main():
    #filenames = ['obs_20121212-033825.h5',
    fn = FileName(run='PAL2012',date='20121211',tstamp='20121212-055428').obs()
    ob = ObsFile(fn)
    frame = ob.getPixelCountImage(0,30,weighted=False)
    hotPixMask = hotPixels.findHotPixels(image=frame,firstSec=0,intTime=30,weighted=False)['badflag']
    frame[hotPixMask == 1] = 0
    def plotFrame(fig,axes):
        hMat=axes.matshow(frame,cmap=matplotlib.cm.gnuplot,origin='lower',vmax=np.mean(frame)+3*np.std(frame))
        fig.colorbar(hMat)
    PopUp(plotFunc=plotFrame)
    #d = datetime.date(2012,10,15)#date of table value
    #d2 = datetime.date(2012,12,11)#date of observation
    #dt=(d2-d).total_seconds()#seconds between obs and table value
    ##crabtime ephemeris
    #nu=29.6957720714 #1/us
    #pdot=4.2013878e-13#us/s
    #period=(1.0/nu)+pdot*dt#us

    #goldstone ephemeris
    F0=29.7414493465249770#Hz
    deltaF0=0.0000000055983574
    F1=-3.669005118205e-10#df0/dt Hz/s
    F2=-3.085573120457e-20#d^2f0/dt^2 Hz/s^2
    pEpoch = 54781.604891 #Modified Julian Date corresponding to F0
    pEpoch = pEpoch+2400000.5#convert mjd to jd
    pEpoch *= 24*3600 #in seconds
    #obsDate = ob.getFromHeader('jd')

    unixEpochJD = 2440587.5
    unixSecsInDay = 86400.
    headerUnixtime = ob.getFromHeader('unixtime')
    obsDate = headerUnixtime/unixSecsInDay+unixEpochJD

    startTime = obsDate*24*3600#in seconds
    dt = startTime-pEpoch#seconds since pepoch

    #freq = F0+F1*dt+F2/2*dt**2
    freq = F0+F1*dt
    period = 1.0/freq
    print 'period=',period,'s'


    #period=0.03367660643405371
    #period=0.03367664238573182
    #period=0.03367662440988317

    iRow = 10
    iCol = 14
    integrationTime = 30
    circCol,circRow = circ(iCol,iRow,radius=5)
    firstSec = 0
    
    dt = startTime-pEpoch + firstSec
    freq = F0+F1*dt
    period = 1.0/freq
    print 'period=',period,'s'

    nPhaseBins = 200
    #np.set_printoptions(threshold=np.nan)
    
    jdTimes = np.array([],dtype=np.float64)
    times = np.array([])
    for i in range(len(circCol)):
        iRow = circRow[i]
        iCol = circCol[i]
        timestamps,peaks,baselines = ob.getTimedPacketList(iRow,iCol,firstSec,integrationTime)
        timestamps = np.array(timestamps,dtype=np.float64)
        jdTimestamps = obsDate+timestamps /(24.*3600.)
        jdTimes = np.append(jdTimes,jdTimestamps)
        times = np.append(times,timestamps)

    jdTimes -= 2400000.5 #convert to modified jd
    np.savetxt('crabOpticalSample-20121212-055428.txt',jdTimes)
    periodDays = period/(24.*3600.)
    phaseOffset = .2
    phases = (jdTimes % periodDays)/periodDays+.2
    phases = phases % 1.0
    print len(phases)
    histPhases,phaseBinEdges = np.histogram(phases,bins=nPhaseBins)
    print jdTimes[0:10]
    print times[0:10]
       

    plt.plot(phaseBinEdges[0:-1],histPhases)
    plt.show()
コード例 #9
0
def main():
    """
    params = []
    paramfile = sys.argv[1]

    f = open(paramfile,'r')
    for line in f:
        params.append(line)
    f.close()
    datadir = params[0].split('=')[1].strip()
    flatdir = params[1].split('=')[1].strip()
    fluxdir = params[2].split('=')[1].strip()
    wvldir = params[3].split('=')[1].strip()
    obsfile = params[4].split('=')[1].strip()
    skyfile = params[5].split('=')[1].strip()
    flatfile = params[6].split('=')[1].strip()
    fluxfile = params[7].split('=')[1].strip()
    wvlfile = params[8].split('=')[1].strip()
    objectName = params[9].split('=')[1].strip()
    fluxCalObject = params[10].split('=')[1].strip()

    obsFileName = os.path.join(datadir, obsfile)
    skyFileName = os.path.join(datadir, skyfile)
    wvlCalFileName = os.path.join(wvldir, wvlfile)
    flatCalFileName = os.path.join(flatdir, flatfile)
    fluxCalFileName = os.path.join(fluxdir, fluxfile)
    """

    if len(sys.argv) >3:
        filenum = str('_'+sys.argv[3])
    else:
        filenum = '_0'

    #science object parameter file
    params = []
    paramfile = sys.argv[1]
    f = open(paramfile,'r')
    for line in f:
        params.append(line)
    f.close()

    datadir = params[0].split('=')[1].strip()
    flatdir = params[1].split('=')[1].strip()
    wvldir = params[2].split('=')[1].strip()
    obsfile = params[3].split('=')[1].strip()
    skyfile = params[4].split('=')[1].strip()
    flatfile = params[5].split('=')[1].strip()
    wvlfile = params[6].split('=')[1].strip()
    objectName = params[9].split('=')[1].strip()

    if len(params)>10:
        xpix = params[10].split('=')[1].strip()
        ypix = params[11].split('=')[1].strip()
        apertureRadius = params[12].split('=')[1].strip()


    #flux cal object parameter file

    params2 = []
    param2file = sys.argv[2]
    f = open(param2file,'r')
    for line in f:
        params2.append(line)
    f.close()

    fluxdir = params2[7].split('=')[1].strip()
    fluxfile = params2[8].split('=')[1].strip()
    fluxCalObject = params2[9].split('=')[1].strip()

    obsFileName = os.path.join(datadir, obsfile)
    skyFileName = os.path.join(datadir, skyfile)
    wvlCalFileName = os.path.join(wvldir, wvlfile)
    flatCalFileName = os.path.join(flatdir, flatfile)
    fluxCalFileName = os.path.join(fluxdir, fluxfile)

    print "obsfile = ",obsFileName
    print "skyfile = ",skyFileName
    print "wvlcal = ", wvlCalFileName
    print "flatcal = ", flatCalFileName
    print "fluxcal = ", fluxCalFileName
    print "object = ", objectName
    print "flux cal object = ", fluxCalObject
    print "\n---------------------\n"

    obs = ObsFile(obsFileName)
    obs.loadWvlCalFile(wvlCalFileName)
    obs.loadFlatCalFile(flatCalFileName)
    obs.loadFluxCalFile(fluxCalFileName)
    print "loaded data file and calibrations\n---------------------\n"

    HotPixFile = getTimeMaskFileName(obsFileName)
    if not os.path.exists(HotPixFile):
        hp.findHotPixels(obsFileName,HotPixFile)
        print "Flux file pixel mask saved to %s"%(HotPixFile)
    obs.loadHotPixCalFile(HotPixFile)
    print "Hot pixel mask loaded %s"%(HotPixFile)

    nRow = obs.nRow
    nCol = obs.nCol
    obsTime = obs.getFromHeader("exptime")
    wvlBinEdges,obsSpectra,obsIntTime = loadSpectra(obs,nCol,nRow)
    nWvlBins=len(wvlBinEdges)-1
    #divide spectrum by bin widths
    binWidths = np.empty(nWvlBins)
    for i in xrange(nWvlBins):
        binWidths[i] = wvlBinEdges[i+1]-wvlBinEdges[i]
    obsSpectra = obsSpectra/binWidths
    #divide by effective integration time
    for iRow in xrange(nRow):
        for iCol in xrange(nCol):
            obsSpectra[iRow][iCol] = obsSpectra[iRow][iCol] / obsIntTime[iRow][iCol]

    #print np.shape(obsSpectra)
    #print nRow
    #print nCol
    #print nWvlBins

    medianObsSpectrum = calculateMedian(obsSpectra,nCol,nRow,nWvlBins)
    print "target spectrum loaded\n---------------------\n"

    if skyfile != "None":
        sky = ObsFile(skyFileName)
        sky.loadWvlCalFile(wvlCalFileName)
        sky.loadFlatCalFile(flatCalFileName)
        sky.loadFluxCalFile(fluxCalFileName)
        skyTime = sky.getFromHeader("exptime")

        skyHotPixFile = getTimeMaskFileName(skyFileName)
        if not os.path.exists(skyHotPixFile):
            hp.findHotPixels(skyFileName,skyHotPixFile)
            print "Flux file pixel mask saved to %s"%(skyHotPixFile)
        obs.loadHotPixCalFile(HotPixFile)
        print "Hot pixel mask loaded %s"%(skyHotPixFile)

        wvlBinEdges,skySpectra,skyIntTime = loadSpectra(sky,nCol,nRow)

        skySpectra = skySpectra/binWidths
        #divide by effective integration time
        for iRow in xrange(nRow):
            for iCol in xrange(nCol):
                skySpectra[iRow][iCol] = skySpectra[iRow][iCol] / skyIntTime[iRow][iCol]

        skySpectrum = calculateMedian(skySpectra, nCol, nRow, nWvlBins)
        #skySpectrum = skySpectrum*float(obsTime)/float(skyTime) #scale sky spectrum to target observation time
        print "sky spectrum loaded\n---------------------\n"
    else:
        #if no sky file given, estimate sky spectrum as median spectrum of obs file, assuming object is tiny
        skySpectrum = medianObsSpectrum
        print "sky spectrum estimated as median of target file spectrum\n---------------------\n"

    #subtract sky spectrum from every pixel
    allSkySpectrum = obsSpectra-skySpectrum
    #set any negative values to 0 after sky subtraction
    allSkySpectrum[allSkySpectrum<0]=0

    #take median of remaining sky subtracted spectra to get median object spectrum
    finalSpectrum = calculateMedian(allSkySpectrum,nCol,nRow,nWvlBins)
    
    #load std spectrum for comparison
    realSpectra = loadStd(objectName,wvlBinEdges)
    print "real std spectrum loaded for reference\n---------------------\n"

    #create plots
    plotDir = "/home/srmeeker/ARCONS-pipeline/fluxcal/test/plots"
    plotFileName = "%s_from_%s%s.pdf"%(objectName,fluxCalObject,filenum)
    fullFluxPlotFileName = os.path.join(plotDir,plotFileName)
    pp = PdfPages(fullFluxPlotFileName)
    matplotlib.rcParams['font.size']=6

    #calculate midpoints of wvl bins for plotting
    wvls = np.empty((nWvlBins),dtype=float)
    for n in xrange(nWvlBins):
        binsize=wvlBinEdges[n+1]-wvlBinEdges[n]
        wvls[n] = (wvlBinEdges[n]+(binsize/2.0))

    plt.figure()

    ax1 = plt.subplot(231)
    ax1.set_title('ARCONS median flat/flux cal\'d obs in counts')
    plt.plot(wvls,medianObsSpectrum)
    #plt.show()
    ax2 = plt.subplot(232)
    ax2.set_title('ARCONS median flat/flux cal\'d sky in counts')
    plt.plot(wvls,skySpectrum)
    #plt.show()
    ax5 = plt.subplot(233)
    ax5.set_title('Sensitivity Spectrum')
    plt.plot(wvls,obs.fluxWeights)
    ax3 = plt.subplot(234)
    ax3.set_title('MKID data minus sky in counts')
    plt.plot(wvls,finalSpectrum/max(finalSpectrum))
    ax4 = plt.subplot(235)
    ax4.set_title('Rebinned Std Spectrum of %s'%(objectName))
    plt.plot(wvls,realSpectra)

    #ax7 = plt.subplot(337)
    #ax7.set_title('Flux Cal\'d ARCONS Spectrum of Std')
    #plt.plot(wvls,fluxFactors*subtractedSpectra)

    pp.savefig()
    pp.close()

    #del obs
    #del sky

    print "output plots to %s\n---------------------\n"%(fullFluxPlotFileName)

    txtDir = "/home/srmeeker/ARCONS-pipeline/fluxcal/test/txt"
    txtFileName = "%s_from_%s%s.txt"%(objectName,fluxCalObject,filenum)
    fullFluxTxtFileName = os.path.join(txtDir,txtFileName)

    outarr = np.empty((len(medianObsSpectrum),2),dtype=float)
    outarr[:,0]=wvls
    outarr[:,1]=medianObsSpectrum
    #save sensitivity spectrum to file
    np.savetxt(fullFluxTxtFileName, outarr)
    
    print "output txt file to %s\n---------------------\n"%(fullFluxPlotFileName)
コード例 #10
0
class DisplayStack(QMainWindow):

    def __init__(self):
        # 0) Start up gui
        QWidget.__init__(self, parent=None)
        self.ui = Ui_DisplayStack_gui()
        self.ui.setupUi(self)

        # Initialize Variables
        self.initializeVariables()
     
        # Initialize run and target, load param file
        self.chooseRun()

        '''
        # Browse wavecal button
        self.ui.wavelengthCalibrationSolutionButton.clicked.connect(self.chooseWavelengthFile)
        # Browse flatcal button
        self.ui.flatCalibrationSolutionButton.clicked.connect(self.chooseFlatFile)       
        '''

        # Use wavelength calibration checkbox
        self.ui.wavelengthCalibrationBox.clicked.connect(self.useWaveCal)
        # Use flat calibration checkbox
        self.ui.flatCalibrationBox.clicked.connect(self.useFlatCal)

        # Buttons for obs list creation
        self.ui.addButton.clicked.connect(self.addObservation)
        self.ui.removeButton.clicked.connect(self.removeObservation)
        self.ui.clearButton.clicked.connect(self.createObsList)

        # Start process button
        self.ui.stackButton.clicked.connect(self.stackProcess)


        # Load image stack button
        self.ui.loadStackButton.clicked.connect(self.chooseStack)

    # 1) Initialize Variables
    def initializeVariables(self):
        # Define path names
        self.displayStackPath = '/Scratch/DisplayStack/'
        self.defaultWavelengthPath = '/Scratch/waveCalSolnFiles/'
        self.defaultFlatPath = '/Scratch/flatCalSolnFiles/'

        # Arrays with run names and buttons
        self.runNames = ['LICK2012','PAL2012','PAL2013']
        self.runButtons = [self.ui.lick2012Button, self.ui.pal2012Button, self.ui.pal2013Button]

        # Arrays with target names and buttons
        self.lick2012TargetNames = ['LICK2012_Object']
        self.lick2012TargetButtons = [self.ui.lick2012ObjectButton]
        self.pal2012TargetNames = ['SDSS_J0651', 'SDSS_J0926']
        self.pal2012TargetButtons = [self.ui.sdss0651Button,self.ui.sdss0926Button]
        self.pal2013TargetNames = ['PAL2013_Object']
        self.pal2013TargetButtons = [self.ui.pal2013ObjectButton]
        self.targetNames = [self.lick2012TargetNames, self.pal2012TargetNames, self.pal2013TargetNames]
        self.targetButtons = [self.lick2012TargetButtons, self.pal2012TargetButtons, self.pal2013TargetButtons]

        # Activate run and target buttons
        for iRun in range(len(self.runButtons)):
            self.runButtons[iRun].clicked.connect(self.chooseRun)
            for iTarget in range(len(self.targetButtons[iRun])):
                self.targetButtons[iRun][iTarget].clicked.connect(self.chooseTarget)

        self.ui.sunsetList.itemClicked.connect(self.createObsList)

    def testFunction(self):
        print 'test'

    # 2) Choose Run
    def chooseRun(self):
        for iRun in range(len(self.runButtons)):
            for iTarget in range(len(self.targetButtons[iRun])):
                    self.targetButtons[iRun][iTarget].setVisible(False)     
            if self.runButtons[iRun].isChecked():
                self.runNumber = iRun        
                for iTarget in range(len(self.targetButtons[iRun])):
                    self.targetButtons[iRun][iTarget].setVisible(True)  
        self.run = self.runNames[self.runNumber]      
        self.targetNumber = 0
        self.targetButtons[self.runNumber][self.targetNumber].setChecked(True)
        self.target = self.targetNames[self.runNumber][self.targetNumber]
        self.loadParamFile()
           
    # 3) Choose Target
    def chooseTarget(self):
        for iTarget in range(len(self.targetButtons[self.runNumber])):
            if self.targetButtons[self.runNumber][iTarget].isChecked():
                self.targetNumber = iTarget
        self.target = self.targetNames[self.runNumber][self.targetNumber]
        self.loadParamFile()

    # 4) Load Param File
    def loadParamFile(self):
        try:
            self.paramData = readDict()
            self.paramData.read_from_file(self.displayStackPath + self.run + '/' + self.target + '/' + self.target + '.dict')
            self.obsTimes = np.array(self.paramData['obsTimes'])
            self.utcDates = self.paramData['utcDates']
            self.sunsetDates = self.paramData['sunsetDates']
            self.calTimestamps = self.paramData['calTimestamps']
            self.flatCalDates = self.paramData['flatCalDates']  

            print 'Loading parameter file at ' + self.displayStackPath + self.run + '/' + self.target + '/' + self.target + '.dict'
            self.createSunsetList()
            #self.createObsList()
            self.createWavelengthList()
            self.createFlatList()
            self.paramFileExists = True
        except IOError:           
            print 'No existing parameter file at ' + self.displayStackPath + self.run + '/' + self.target + '/' + self.target + '.dict'
            self.ui.sunsetList.clear()
            self.ui.obsList.clear()
            self.ui.inputList.clear()
            self.ui.wavelengthList.clear()
            self.ui.flatList.clear()
            self.paramFileExists = False

    # 5) Choose Obs File
    # Create list of available sunset dates
    def createSunsetList(self):
        self.ui.sunsetList.clear()
        for iDate in range(len(self.sunsetDates)):
            self.ui.sunsetList.addItem(self.sunsetDates[iDate])
    # Create Initial Obs file list
    def createObsList(self):
        self.ui.obsList.clear()
        self.ui.inputList.clear()
        self.currentSunsetDate = self.sunsetDates[self.ui.sunsetList.row(self.ui.sunsetList.currentItem())]
        self.currentUTCDate = self.utcDates[self.ui.sunsetList.row(self.ui.sunsetList.currentItem())]
        self.singleDayObservations = self.obsTimes[self.ui.sunsetList.row(self.ui.sunsetList.currentItem())]
        for iObs in range(len(self.singleDayObservations)):
            self.ui.obsList.addItem(self.singleDayObservations[iObs])
    # Add observation to input list
    def addObservation(self):
        self.selectedObs = self.ui.obsList.currentItem()
        self.ui.obsList.takeItem(self.ui.obsList.row(self.selectedObs))
        self.ui.inputList.addItem(self.selectedObs)
        self.ui.inputList.sortItems()
    # Remove observation from input list
    def removeObservation(self):
        self.removedObs = self.ui.inputList.currentItem()
        self.ui.inputList.takeItem(self.ui.inputList.row(self.removedObs))
        self.ui.obsList.addItem(self.removedObs)
        self.ui.obsList.sortItems()

    # 6) Load settings
    def loadSettings(self):
        
        self.validSettings = True        

        # Run and target information
        self.run = self.runNames[self.runNumber]
        self.target = self.targetNames[self.runNumber][self.targetNumber]

        # General settings
        self.integrationTime = int(self.ui.integrationTimeLine.text())
        self.useTimeAdjustment = self.ui.timeAdjustmentBox.isChecked()
        self.useHotPixelMasking = self.ui.hotPixelBox.isChecked()

        # Wavelength calibration settings
        self.useWavelengthCalibration = self.ui.wavelengthCalibrationBox.isChecked()
        self.lowerWavelengthCutoff = float(self.ui.lowerWavelengthCutoffLine.text())
        self.upperWavelengthCutoff = float(self.ui.upperWavelengthCutoffLine.text())
               
        # Flat calibration settings
        self.useFlatCalibration = self.ui.flatCalibrationBox.isChecked()
        self.useDeadPixelMasking = self.ui.deadPixelBox.isChecked()    

        self.fileCount = self.ui.inputList.count()  

        self.weighted = self.useFlatCalibration
        self.useRawCounts = not (self.useWavelengthCalibration and self.useFlatCalibration)
        self.scaleByEffInt = self.useHotPixelMasking
            
        if self.ui.sunsetList.currentItem() != None:

            if self.fileCount == 0:
                print 'Please select files to process...'
                self.validSettings = False

            if self.useWavelengthCalibration: 
                if self.ui.wavelengthList.currentItem() == None:
                    print 'Please select wavelength calibration...'
                    self.validSettings = False
                else:
                    self.selectedWvlCal = self.ui.wavelengthList.currentItem().text()
                    self.wvlCalFilename = str(FileName(run=self.run,date=self.currentSunsetDate,tstamp=self.selectedWvlCal).calSoln())

            if self.useFlatCalibration:
                if self.ui.flatList.currentItem() == None:
                    print 'Please select flat calibration...'
                    self.validSettings = False
                else:
                    self.flatCalNight = self.ui.flatList.currentItem().text()
                    self.flatCalFilename = str(FileName(run=self.run,date=self.flatCalNight).flatSoln())
                    
        else:
            print 'Please select sunset night...'
            self.validSettings = False

    # 7) Load hot pixel mask
    def loadHotMask(self):
        self.hotPixelFilename = str(self.displayStackPath + self.run + '/' + self.target + '/HotPixelMasks/hotPixelMask_' + self.obsTS + '.h5')
        if not os.path.exists(self.hotPixelFilename):
            hp.findHotPixels(self.obsFn,self.hotPixelFilename)
            print "Hot pixel mask saved to %s"%(self.hotPixelFilename)
        self.ob.loadHotPixCalFile(self.hotPixelFilename,switchOnMask=True)

    # 8) Create wavelength cal file list
    def createWavelengthList(self):
        self.ui.wavelengthList.clear()
        for iCal in range(len(self.calTimestamps)):
            self.ui.wavelengthList.addItem(self.calTimestamps[iCal])
    # Enable/disable wavecal options
    def useWaveCal(self):
        if self.ui.wavelengthCalibrationBox.isChecked():
            self.ui.lowerWavelengthCutoffLine.setEnabled(True)
            self.ui.lowerWavelengthCutoffLabel.setEnabled(True)
            self.ui.upperWavelengthCutoffLine.setEnabled(True)
            self.ui.upperWavelengthCutoffLabel.setEnabled(True)
            self.ui.wavelengthList.setEnabled(True)          
            self.ui.flatCalibrationBox.setEnabled(True)
            self.ui.flatCalibrationBox.setChecked(True)
            self.ui.deadPixelBox.setEnabled(True)
            self.ui.deadPixelBox.setChecked(True)       
            self.ui.flatList.setEnabled(True)
        else:
            self.ui.lowerWavelengthCutoffLine.setEnabled(False)
            self.ui.lowerWavelengthCutoffLabel.setEnabled(False)
            self.ui.upperWavelengthCutoffLine.setEnabled(False)
            self.ui.upperWavelengthCutoffLabel.setEnabled(False)
            self.ui.wavelengthList.setEnabled(False)
            self.ui.flatCalibrationBox.setEnabled(False)
            self.ui.flatCalibrationBox.setChecked(False)
            self.ui.deadPixelBox.setChecked(False)
            self.ui.deadPixelBox.setEnabled(False)
            self.ui.flatList.setEnabled(False)


    # 9) Create flat cal file list
    def createFlatList(self):
        self.ui.flatList.clear()
        for iCal in range(len(self.flatCalDates)):
            self.ui.flatList.addItem(self.flatCalDates[iCal])
    # Enable/disable flatcal options
    def useFlatCal(self):
        if self.ui.flatCalibrationBox.isChecked():
            self.ui.deadPixelBox.setEnabled(True)
            self.ui.deadPixelBox.setChecked(True)       
            self.ui.flatList.setEnabled(True)
        else:
            self.ui.deadPixelBox.setChecked(False)
            self.ui.deadPixelBox.setEnabled(False)
            self.ui.flatList.setEnabled(False)

    # 10) Load dead pixel mask
    def loadDeadMask(self):
        self.deadPixelFilename = str(self.displayStackPath + self.run + '/' + self.target + '/DeadPixelMasks/deadPixelMask_' + self.obsTS + '.npz')
        if not os.path.exists(self.deadPixelFilename):
            self.deadMask = self.ob.getDeadPixels()
            np.savez(self.deadPixelFilename,deadMask = self.deadMask)
            print "Dead pixel mask saved to %s"%(self.deadPixelFilename)
        else:
            self.deadFile = np.load(self.deadPixelFilename)
            self.deadMask = self.deadFile['deadMask']

    # 12) Create output name
    def createOutputName(self):
        self.rawName = str(self.displayStackPath + self.run + '/' + self.target + '/ImageStacks/' + 'ImageStack_' + self.obsTS + '_' + str(self.integrationTime) + 's')
        if self.useWavelengthCalibration and self.useHotPixelMasking:
            self.outputFilename = str(self.rawName + '_' + str(int(self.lowerWavelengthCutoff)) + '-' + str(int(self.upperWavelengthCutoff)) + '_hp.npz')
        elif self.useWavelengthCalibration and not self.useHotPixelMasking:
            self.outputFilename = str(self.rawName + '_' + str(int(self.lowerWavelengthCutoff)) + '-' + str(int(self.upperWavelengthCutoff)) + '.npz')
        elif not self.waveLengthCalibration and self.useHotPixelMasking:
            self.outputFilename = str(self.rawName + '_hp.npz')
        else:
            self.outputFilename = str(self.rawName + '.npz')

    # Start process for creating image stacks
    def stackProcess(self):

        # Check for valid params file
        if self.paramFileExists:

            # 6) Load settings choosen from gui
            self.loadSettings()
            if self.validSettings:
        
                # Loop through all files in input list
                for iFile in range(self.fileCount):
                    # Create ObsFile instance
                    self.obsTS = str(self.currentUTCDate) + '-' + self.ui.inputList.item(iFile).text()
                    self.obsFn = str(FileName(run=self.run,date=self.currentSunsetDate,tstamp=self.obsTS).obs())
                    print 'Processing file ' + self.obsFn + '...'
                    self.ob = ObsFile(self.obsFn)

                    # Load time adjustment file
                    if self.useTimeAdjustment:
                        print 'Loading time adjustment file...'
                        self.ob.loadTimeAdjustmentFile(FileName(run=self.run).timeAdjustments())
       
                    # 7) Load hot pixel mask
                    if self.useHotPixelMasking:
                        print 'Loading hot pixel mask...'
                        self.loadHotMask()

                    # 8) Load wave cal solution
                    if self.useWavelengthCalibration:
                        print 'Loading wavelength calibration...'
                        self.ob.loadWvlCalFile(self.wvlCalFilename)
                        
                    # 9) Load flatcal solution
                    if self.useFlatCalibration:
                        print 'Loading flat calibration...'
                        self.ob.loadFlatCalFile(self.flatCalFilename)

                    # 10) Load dead pixel mask
                    if self.useDeadPixelMasking:
                        print 'Loading dead pixel mask...'
                        self.loadDeadMask()

                    # 11) Set wavelength cutoffs
                    if self.useWavelengthCalibration:
                        print 'Setting wavelength cutoffs...'
                        self.ob.setWvlCutoffs(self.lowerWavelengthCutoff,self.upperWavelengthCutoff)
        
                    # Timing
                    self.unix = self.ob.getFromHeader('unixtime')
                    self.startJD = self.unix/86400.+2440587.5
                    self.exptime = self.ob.getFromHeader('exptime')
                    
                    self.times = []
                    self.frames = []
                        
                    # 11) Create Image Stack
                    print 'Stacking images...'
                    for iSec in np.arange(0,self.exptime,self.integrationTime):
                        #add seconds offset to julian date, move jd to center of bin
                        self.jd = self.startJD + iSec/(24.*3600.) + self.integrationTime/2./(24.*3600.)
                        'Creating frame for time ' + self.jd
                        self.frameData = self.ob.getPixelCountImage(firstSec=iSec,integrationTime=self.integrationTime,weighted=self.weighted,getRawCount=self.useRawCounts,scaleByEffInt=self.scaleByEffInt)
                        self.frame = self.frameData['image']         
                        if self.useDeadPixelMasking:
                            self.frame[self.deadMask == 0] = np.nan
                        self.frames.append(self.frame)

                    self.cube = np.dstack(self.frames)
                    self.times = np.array(self.times)
            
                    # 12) Create output file
                    self.createOutputName()
                    print 'Saving image stack to ' + self.outputFilename
                    np.savez(self.outputFilename, stack=self.cube, jd=self.times)

        # Invalid params file
        else:
            print 'Invalid parameter file...'

    # Choose an image stack
    def chooseStack(self):
        self.defaultLoadStackDirectory = str(self.displayStackPath + self.run + '/' + self.target + '/ImageStacks')
        self.stackName = QFileDialog.getOpenFileName(parent=None, directory=self.defaultLoadStackDirectory, caption=str("Choose Image Stack"), filter=str("NPZ (*.npz)")) 
        if self.stackName == '':
            print 'No file chosen'
        else:
            self.loadStack()

    # Load that image stack for viewing
    def loadStack(self):
コード例 #11
0
        print 'Loading cal files'
        ob.loadHotPixCalFile(hotPixFn,switchOnMask=True)
        ob.loadWvlCalFile(wfn)
        ob.loadFlatCalFile(ffn)
        ob.setWvlCutoffs(wvlLowerCutoff,wvlUpperCutoff)

        print 'Finding bad wvl pixels'
        bad_solution_mask=np.zeros((46,44))
        bad_count=0;
        for y in range(46):
            for x in range(44):
                if (ob.wvlRangeTable[y][x][1] < 11000):
                    bad_solution_mask[y][x] = 1
                    bad_count = bad_count+1

        startJD = ob.getFromHeader('jd')
        nSecInFile = ob.getFromHeader('exptime')
        #tic = time()
        print 'Finding dead pixels'
        deadMask = ob.getDeadPixels()
        #print 'Dead mask load time = ', time()-tic
        print 'Getting images'
        for sec in range(0,nSecInFile,integrationTime):
            jd = startJD + sec/(24.*3600.)#add seconds offset to julian date
            print count,jd
            count+=1
            times.append(jd)
            titles.append('%.6f'%jd)
            frameData = ob.getPixelCountImage(firstSec=sec,integrationTime=integrationTime,weighted=True)
            frame = frameData['image']         
            showFrame = np.array(frame)
コード例 #12
0
def main():
    """
    params = []
    paramfile = sys.argv[1]

    f = open(paramfile,'r')
    for line in f:
        params.append(line)
    f.close()
    datadir = params[0].split('=')[1].strip()
    flatdir = params[1].split('=')[1].strip()
    fluxdir = params[2].split('=')[1].strip()
    wvldir = params[3].split('=')[1].strip()
    obsfile = params[4].split('=')[1].strip()
    skyfile = params[5].split('=')[1].strip()
    flatfile = params[6].split('=')[1].strip()
    fluxfile = params[7].split('=')[1].strip()
    wvlfile = params[8].split('=')[1].strip()
    objectName = params[9].split('=')[1].strip()
    fluxCalObject = params[10].split('=')[1].strip()

    obsFileName = os.path.join(datadir, obsfile)
    skyFileName = os.path.join(datadir, skyfile)
    wvlCalFileName = os.path.join(wvldir, wvlfile)
    flatCalFileName = os.path.join(flatdir, flatfile)
    fluxCalFileName = os.path.join(fluxdir, fluxfile)
    """

    if len(sys.argv) >3:
        filenum = str('_'+sys.argv[3])
    else:
        filenum = '_0'

    #science object parameter file
    params = []
    paramfile = sys.argv[1]
    f = open(paramfile,'r')
    for line in f:
        params.append(line)
    f.close()

    datadir = params[0].split('=')[1].strip()
    flatdir = params[1].split('=')[1].strip()
    wvldir = params[2].split('=')[1].strip()
    obsfile = params[3].split('=')[1].strip()
    skyfile = params[4].split('=')[1].strip()
    flatfile = params[5].split('=')[1].strip()
    wvlfile = params[6].split('=')[1].strip()
    objectName = params[9].split('=')[1].strip()

    if len(params)>10:
        xpix = int(params[10].split('=')[1].strip())
        ypix = int(params[11].split('=')[1].strip())
        apertureRadius = int(params[12].split('=')[1].strip())


    #flux cal object parameter file

    params2 = []
    param2file = sys.argv[2]
    f = open(param2file,'r')
    for line in f:
        params2.append(line)
    f.close()

    fluxdir = params2[7].split('=')[1].strip()
    fluxfile = params2[8].split('=')[1].strip()
    fluxCalObject = params2[9].split('=')[1].strip()

    obsFileName = os.path.join(datadir, obsfile)
    skyFileName = os.path.join(datadir, skyfile)
    wvlCalFileName = os.path.join(wvldir, wvlfile)
    flatCalFileName = os.path.join(flatdir, flatfile)
    fluxCalFileName = os.path.join(fluxdir, fluxfile)

    print "obsfile = ",obsFileName
    print "skyfile = ",skyFileName
    print "wvlcal = ", wvlCalFileName
    print "flatcal = ", flatCalFileName
    print "fluxcal = ", fluxCalFileName
    print "object = ", objectName
    print "flux cal object = ", fluxCalObject
    print "\n---------------------\n"

    obs = ObsFile(obsFileName)
    obs.loadWvlCalFile(wvlCalFileName)
    obs.loadFlatCalFile(flatCalFileName)
    obs.loadFluxCalFile(fluxCalFileName)
    print "loaded data file and calibrations\n---------------------\n"

    nRow = obs.nRow
    nCol = obs.nCol
    obsTime = obs.getFromHeader("exptime")
    #wvlBinEdges,obsSpectra = loadSpectra(obs,nCol,nRow)
    #nWvlBins=len(wvlBinEdges)-1

    #print np.shape(obsSpectra)
    #print nRow
    #print nCol
    #print nWvlBins

    """
    medianObsSpectrum = calculateMedian(obsSpectra,nCol,nRow,nWvlBins)
    print "target spectrum loaded\n---------------------\n"

    if skyfile != "None":
        sky = ObsFile(skyFileName)
        sky.loadWvlCalFile(wvlCalFileName)
        sky.loadFlatCalFile(flatCalFileName)
        sky.loadFluxCalFile(fluxCalFileName)
        skyTime = sky.getFromHeader("exptime")
        wvlBinEdges,skySpectra = loadSpectra(sky,nCol,nRow)
        skySpectrum = calculateMedian(skySpectra, nCol, nRow, nWvlBins)
        skySpectrum = skySpectrum*float(obsTime)/float(skyTime) #scale sky spectrum to target observation time
        print "sky spectrum loaded\n---------------------\n"
    else:
        #if no sky file given, estimate sky spectrum as median spectrum of obs file, assuming object is tiny
        skySpectrum = calculateMedian(obsSpectra, nCol, nRow, nWvlBins)
        print "sky spectrum estimated as median of target file spectrum\n---------------------\n"

    #subtract sky spectrum from every pixel
    allSkySpectrum = obsSpectra-skySpectrum
    #set any negative values to 0 after sky subtraction
    allSkySpectrum[allSkySpectrum<0]=0

    #take median of remaining sky subtracted spectra to get median object spectrum
    finalSpectrum = calculateMedian(allSkySpectrum,nCol,nRow,nWvlBins)
    """

    #load/generate hot pixel mask file
    HotPixFile = getTimeMaskFileName(obsFileName)
    if not os.path.exists(HotPixFile):
        hp.findHotPixels(obsFileName,HotPixFile)
        print "Flux file pixel mask saved to %s"%(HotPixFile)
    obs.loadHotPixCalFile(HotPixFile)
    print "Hot pixel mask loaded %s"%(HotPixFile)

    print "Making spectrum with Aperture Spectrum in ObsFile"
    #use Aperture Spectrum from obsfile
    medianObsSpectrum, wvlBinEdges = obs.getApertureSpectrum(pixelCol=ypix,pixelRow=xpix,radius1=apertureRadius, radius2 = apertureRadius*2.0,weighted=True, fluxWeighted=True, lowCut=3000, highCut=7000)

    nWvlBins=len(wvlBinEdges)-1

    #load std spectrum for comparison
    try:
        realSpectra = loadStd(objectName,wvlBinEdges)
        print "real std spectrum loaded for reference\n---------------------\n"
        stdTitle = "Rebinned Std Spectrum of %s"%(objectName)
    except KeyError:
        print "Key Error loading MKIDStd"
        realSpectra = np.ones(nWvlBins)
        stdTitle = "No MKIDStd spectrum available for %s"%(objectName)

    #create plots
    plotDir = "/home/srmeeker/ARCONS-pipeline/fluxcal/test/plots"
    plotFileName = "%s_from_%s%s.pdf"%(objectName,fluxCalObject,filenum)
    fullFluxPlotFileName = os.path.join(plotDir,plotFileName)
    pp = PdfPages(fullFluxPlotFileName)
    matplotlib.rcParams['font.size']=6

    #calculate midpoints of wvl bins for plotting
    wvls = np.empty((nWvlBins),dtype=float)
    for n in xrange(nWvlBins):
        binsize=wvlBinEdges[n+1]-wvlBinEdges[n]
        wvls[n] = (wvlBinEdges[n]+(binsize/2.0))

    plt.figure()

    ax1 = plt.subplot(221)
    ax1.set_title('ARCONS median flat/flux cal\'d obs in counts')
    ax1.set_xlim((4000,11000))
    ax1.set_ylim((min(medianObsSpectrum[(wvls>4000) & (wvls<11000)]),max(medianObsSpectrum[(wvls>4000) & (wvls<8000)])))
    plt.plot(wvls,medianObsSpectrum)
    #plt.show()
    #ax2 = plt.subplot(232)
    #ax2.set_title('ARCONS median flat/flux cal\'d sky in counts')
    #plt.plot(wvls,skySpectrum)
    #plt.show()
    ax5 = plt.subplot(223)
    ax5.set_title('Sensitivity Spectrum')
    ax5.set_xlim((3000,13000))
    ax5.set_ylim((0,5))
    plt.plot(wvls,obs.fluxWeights)
    #ax3 = plt.subplot(234)
    #ax3.set_title('MKID data minus sky in counts')
    #plt.plot(wvls,finalSpectrum/max(finalSpectrum))
    ax4 = plt.subplot(222)
    ax4.set_title(stdTitle)
    plt.plot(wvls,realSpectra)


    #ax7 = plt.subplot(337)
    #ax7.set_title('Flux Cal\'d ARCONS Spectrum of Std')
    #plt.plot(wvls,fluxFactors*subtractedSpectra)

    pp.savefig()
    pp.close()

    #del obs
    #del sky

    print "output plots to %s\n---------------------\n"%(fullFluxPlotFileName)

    txtDir = "/home/srmeeker/ARCONS-pipeline/fluxcal/test/txt"
    txtFileName = "%s_from_%s%s.txt"%(objectName,fluxCalObject,filenum)
    fullFluxTxtFileName = os.path.join(txtDir,txtFileName)

    outarr = np.empty((len(medianObsSpectrum),2),dtype=float)
    outarr[:,0]=wvls
    outarr[:,1]=medianObsSpectrum
    #save sensitivity spectrum to file
    np.savetxt(fullFluxTxtFileName, outarr)
    
    print "output txt file to %s\n---------------------\n"%(fullFluxPlotFileName)
コード例 #13
0
class DisplayStack(QMainWindow):

    def __init__(self):
        # Start up gui
        QWidget.__init__(self, parent=None)
        self.ui = Ui_DisplayStack_gui()
        self.ui.setupUi(self)

        # Initialize Variables
        self.initializeVariables()

        # Lists and buttons used for specifying run and target information.
        # Click item in runList to select a run. Run number corresponds to array index.  Load up corresponding target list.
        self.ui.runList.itemClicked.connect(self.selectRun)
        # Click item in targetList to select a target.
        self.ui.targetList.itemClicked.connect(self.selectTarget)
        # Click button in 
        self.ui.targetButton.clicked.connect(self.loadTarget)


        self.ui.sunsetList.itemClicked.connect(self.createObsList)


        # Use wavelength calibration checkbox
        self.ui.wavelengthCalibrationBox.clicked.connect(self.useWaveCal)
        # Use flat calibration checkbox
        self.ui.flatCalibrationBox.clicked.connect(self.useFlatCal)

        # Buttons for obs list creation
        self.ui.addButton.clicked.connect(self.addObservation)
        self.ui.removeButton.clicked.connect(self.removeObservation)
        self.ui.clearButton.clicked.connect(self.createObsList)

        # Start process button
        self.ui.stackButton.clicked.connect(self.stackProcess)


        # Load image stack button
        self.ui.loadStackButton.clicked.connect(self.chooseStack)

    # Initialize Variables
    def initializeVariables(self):
        # Define path names
        self.displayStackPath = '/Scratch/DisplayStack/'
        self.defaultWavelengthPath = '/Scratch/waveCalSolnFiles/PAL2012/master_cals/'
        self.defaultFlatPath = '/Scratch/flatCalSolnFiles/'
       
        # Load and display list of run names from /Scratch/DisplayStack/runList.dict
        self.loadRunData()
        
        # Load list of target names from /Scratch/DisplayStack/runName/runName.dict, for all runs
        self.loadTargetData()


    # Function to load list of run names. Runs on initialization.
    def loadRunData(self):
        # Load run data from /Scratch/DisplayStack/runList.dict
        self.runData = readDict()
        self.runData.read_from_file(self.displayStackPath + '/runList.dict')
        self.runNames = np.array(self.runData['runs'])
        # Populate runList table with run names.
        for iRun in range(len(self.runNames)):
            self.ui.runList.addItem(self.runNames[iRun])

    # Function to load a table of target names.
    def loadTargetData(self):
        self.targetNames = []
        # Cycle through runs and extract target name information from various dictionaries.
        for iRun in range(len(self.runNames)):
            self.targetData = readDict()
            self.targetData.read_from_file(self.displayStackPath + self.runNames[iRun] + '/' + self.runNames[iRun] + '.dict')
            self.iTargets = np.array(self.targetData['targets'])
            self.targetNames.append(self.iTargets)

    # Function to select a run and populate the target list for that particular run.
    def selectRun(self):
        # Clear list of target information for previously selected run.
        self.ui.targetList.clear()
        # Define a run number by the index of the selected run.
        self.runNumber = self.ui.runList.row(self.ui.runList.currentItem())
        # Populate targetList table with target names for selected run.
        for iTarget in range(len(self.targetNames[self.runNumber])):
            self.ui.targetList.addItem(self.targetNames[self.runNumber][iTarget])

    def selectTarget(self):
        self.targetNumber = self.ui.targetList.row(self.ui.targetList.currentItem())

    def loadTarget(self):
        self.run = self.runNames[self.runNumber]
        self.target = self.targetNames[self.runNumber][self.targetNumber]
        try:
            self.paramName = self.displayStackPath + self.run + '/' + self.target + '/' + self.target + '.dict'
            self.paramData = readDict()
            self.paramData.read_from_file(self.paramName)
            self.obsTimes = np.array(self.paramData['obsTimes'])
            self.utcDates = self.paramData['utcDates']
            self.sunsetDates = self.paramData['sunsetDates']
            self.calTimestamps = self.paramData['calTimestamps']
            self.flatCalDates = self.paramData['flatCalDates']
            self.RA = self.paramData['RA']
            self.Dec = self.paramData['Dec']
            self.hourAngleOffset = self.paramData['HA_offset']

            print 'Loading parameter file at ' + self.displayStackPath + self.run + '/' + self.target + '/' + self.target + '.dict'
            self.createSunsetList()
            #self.createObsList()
            self.createWavelengthList()
            self.createFlatList()
            self.paramFileExists = True
        except IOError:           
            print 'No existing parameter file at ' + self.displayStackPath + self.run + '/' + self.target + '/' + self.target + '.dict'
            self.ui.sunsetList.clear()
            self.ui.obsList.clear()
            self.ui.inputList.clear()
            self.ui.wavelengthList.clear()
            self.ui.flatList.clear()
            self.paramFileExists = False
        
    # Choose Obs File
    # Create list of available sunset dates
    def createSunsetList(self):
        self.ui.sunsetList.clear()
        for iDate in range(len(self.sunsetDates)):
            self.ui.sunsetList.addItem(self.sunsetDates[iDate])
    # Create Initial Obs file list
    def createObsList(self):
        self.ui.obsList.clear()
        self.ui.inputList.clear()
        self.currentSunsetDate = self.sunsetDates[self.ui.sunsetList.row(self.ui.sunsetList.currentItem())]
        self.currentUTCDate = self.utcDates[self.ui.sunsetList.row(self.ui.sunsetList.currentItem())]
        self.singleDayObservations = self.obsTimes[self.ui.sunsetList.row(self.ui.sunsetList.currentItem())]
        for iObs in range(len(self.singleDayObservations)):
            self.ui.obsList.addItem(self.singleDayObservations[iObs])
    # Add observation to input list
    def addObservation(self):
        self.selectedObs = self.ui.obsList.currentItem()
        self.ui.obsList.takeItem(self.ui.obsList.row(self.selectedObs))
        self.ui.inputList.addItem(self.selectedObs)
        self.ui.inputList.sortItems()
    # Remove observation from input list
    def removeObservation(self):
        self.removedObs = self.ui.inputList.currentItem()
        self.ui.inputList.takeItem(self.ui.inputList.row(self.removedObs))
        self.ui.obsList.addItem(self.removedObs)
        self.ui.obsList.sortItems()

    # Load settings
    def loadSettings(self):
        
        self.validSettings = True        

        # Run and target information
        self.run = self.runNames[self.runNumber]
        self.target = self.targetNames[self.runNumber][self.targetNumber]

        # General settings
        self.integrationTime = int(self.ui.integrationTimeLine.text())
        self.useTimeAdjustment = self.ui.timeAdjustmentBox.isChecked()
        self.useHotPixelMasking = self.ui.hotPixelBox.isChecked()

        # Wavelength calibration settings
        self.useWavelengthCalibration = self.ui.wavelengthCalibrationBox.isChecked()
        self.useBestWavelengthCalibration = self.ui.bestWavelengthCalibrationBox.isChecked()
        self.lowerWavelengthCutoff = float(self.ui.lowerWavelengthCutoffLine.text())
        self.upperWavelengthCutoff = float(self.ui.upperWavelengthCutoffLine.text())
               
        # Flat calibration settings
        self.useFlatCalibration = self.ui.flatCalibrationBox.isChecked()
        self.useDeadPixelMasking = self.ui.deadPixelBox.isChecked() 

        self.fileCount = self.ui.inputList.count()  

        self.weighted = self.useFlatCalibration
        self.useRawCounts = not (self.useWavelengthCalibration and self.useFlatCalibration)
        self.scaleByEffInt = self.useHotPixelMasking
            
        if self.ui.sunsetList.currentItem() != None:

            if self.fileCount == 0:
                print 'Please select files to process...'
                self.validSettings = False

            if self.useWavelengthCalibration: 
                if self.useBestWavelengthCalibration:
                    print 'Using best wavelength calibration...'
                elif self.ui.wavelengthList.currentItem() == None:
                    print 'Please select wavelength calibration...'
                    self.validSettings = False
                else:
                    self.selectedWvlCal = self.ui.wavelengthList.currentItem().text()
                    print "Chose wvl cal: ", str(self.defaultWavelengthPath+self.selectedWvlCal)
                    self.wvlCalFilename = str(self.defaultWavelengthPath+self.selectedWvlCal)
                    #self.wvlCalFilename = str(FileName(run=self.run,date=self.currentSunsetDate,tstamp=self.selectedWvlCal).calSoln())

            if self.useFlatCalibration:
                if self.ui.flatList.currentItem() == None:
                    print 'Please select flat calibration...'
                    self.validSettings = False
                else:
                    self.flatCalNight = self.ui.flatList.currentItem().text()
                    self.flatCalFilename = str(FileName(run=self.run,date=self.flatCalNight).flatSoln())
                    
        else:
            print 'Please select sunset night...'
            self.validSettings = False

    # Load hot pixel mask
    def loadHotMask(self):
        #self.hotPixelFilename = str(self.displayStackPath + self.run + '/' + self.target + '/HotPixelMasks/hotPixelMask_' + self.obsTS + '.h5')
        self.hotPixelFilename = str(FileName(obsFile = self.ob).timeMask())
        if not os.path.exists(self.hotPixelFilename):
            hp.findHotPixels(obsFile=self.ob,outputFileName=self.hotPixelFilename)
            print "Hot pixel mask saved to %s"%(self.hotPixelFilename)
        self.ob.loadHotPixCalFile(self.hotPixelFilename,switchOnMask=True)

    # Create wavelength cal file list
    def createWavelengthList(self):
        self.ui.wavelengthList.clear()
        for iCal in range(len(self.calTimestamps)):
            self.ui.wavelengthList.addItem(self.calTimestamps[iCal])
    # Enable/disable wavecal options
    def useWaveCal(self):
        if self.ui.wavelengthCalibrationBox.isChecked():
            self.ui.lowerWavelengthCutoffLine.setEnabled(True)
            self.ui.lowerWavelengthCutoffLabel.setEnabled(True)
            self.ui.upperWavelengthCutoffLine.setEnabled(True)
            self.ui.upperWavelengthCutoffLabel.setEnabled(True)
            self.ui.wavelengthList.setEnabled(True)          
            self.ui.flatCalibrationBox.setEnabled(True)
            self.ui.flatCalibrationBox.setChecked(True)
            self.ui.deadPixelBox.setEnabled(True)
            self.ui.deadPixelBox.setChecked(True)       
            self.ui.flatList.setEnabled(True)
            self.ui.bestWavelengthCalibrationBox.setEnabled(True)
            self.ui.bestWavelengthCalibrationBox.setChecked(True)
        else:
            self.ui.lowerWavelengthCutoffLine.setEnabled(False)
            self.ui.lowerWavelengthCutoffLabel.setEnabled(False)
            self.ui.upperWavelengthCutoffLine.setEnabled(False)
            self.ui.upperWavelengthCutoffLabel.setEnabled(False)
            self.ui.wavelengthList.setEnabled(False)
            self.ui.flatCalibrationBox.setEnabled(False)
            self.ui.flatCalibrationBox.setChecked(False)
            self.ui.deadPixelBox.setChecked(False)
            self.ui.deadPixelBox.setEnabled(False)
            self.ui.flatList.setEnabled(False)
            self.ui.bestWavelengthCalibrationBox.setEnabled(False)
            self.ui.bestWavelengthCalibrationBox.setChecked(False)


    # Create flat cal file list
    def createFlatList(self):
        self.ui.flatList.clear()
        for iCal in range(len(self.flatCalDates)):
            self.ui.flatList.addItem(self.flatCalDates[iCal])
    # Enable/disable flatcal options
    def useFlatCal(self):
        if self.ui.flatCalibrationBox.isChecked():
            self.ui.deadPixelBox.setEnabled(True)
            self.ui.deadPixelBox.setChecked(True)       
            self.ui.flatList.setEnabled(True)
        else:
            self.ui.deadPixelBox.setChecked(False)
            self.ui.deadPixelBox.setEnabled(False)
            self.ui.flatList.setEnabled(False)

    # Load dead pixel mask
    def loadDeadMask(self):
        self.deadPixelFilename = str(self.displayStackPath + self.run + '/' + self.target + '/DeadPixelMasks/deadPixelMask_' + self.obsTS + '.npz')
        if not os.path.exists(self.deadPixelFilename):
            self.deadMask = self.ob.getDeadPixels()
            np.savez(self.deadPixelFilename,deadMask = self.deadMask)
            print "Dead pixel mask saved to %s"%(self.deadPixelFilename)
        else:
            self.deadFile = np.load(self.deadPixelFilename)
            self.deadMask = self.deadFile['deadMask']

    # Describe the structure of the header row
    class headerDescription(tables.IsDescription):
        targetName = tables.StringCol(100, dflt='')
        run = tables.StringCol(100, dflt='')
        obsFileName = tables.StringCol(100, dflt='')
        wvlCalFileName = tables.StringCol(100, dflt=np.nan)
        flatCalFileName = tables.StringCol(100, dflt='')
        deadPixFileName = tables.StringCol(100, dflt='')
        hotPixFileName = tables.StringCol(100, dflt='')
        nCol = tables.UInt32Col(dflt=-1)
        nRow = tables.UInt32Col(dflt=-1)
        lowWvlCutoff = tables.Float64Col(dflt=np.nan)
        highWvlCutoff = tables.Float64Col(dflt=np.nan)
        exptime = tables.Float64Col(dflt=np.nan)
        lst = tables.StringCol(100, dflt='')
        integrationTime = tables.Float64Col(dflt=np.nan)
        RA = tables.StringCol(100, dflt='')
        Dec = tables.StringCol(100, dflt='')
        HA_offset = tables.Float64Col(dflt=0.0)


    # Create output name
    def createOutputName(self):
        self.rawName = str(self.displayStackPath + self.run + '/' + self.target + '/ImageStacks/' + 'ImageStack_' + self.obsTS + '_' + str(self.integrationTime) + 's')
        if self.useWavelengthCalibration and self.useHotPixelMasking:
            self.outputFilename = str(self.rawName + '_' + str(int(self.lowerWavelengthCutoff)) + '-' + str(int(self.upperWavelengthCutoff)) + '_hp.h5')
        elif self.useWavelengthCalibration and not self.useHotPixelMasking:
            self.outputFilename = str(self.rawName + '_' + str(int(self.lowerWavelengthCutoff)) + '-' + str(int(self.upperWavelengthCutoff)) + '.h5')
        elif not self.useWavelengthCalibration and self.useHotPixelMasking:
            self.outputFilename = str(self.rawName + '_hp.h5')
        else:
            self.outputFilename = str(self.rawName + '.h5')

    def createH5File(self):
        # Create header and data group and table names
        headerGroupName = 'header'
        headerTableName = 'header'
        dataGroupName = 'stack'
        dataTableName = 'stack'
        timeTableName = 'time'

        # Create lookup names for header information
        runColName = 'run'
        targetColName = 'targetName'
        obsFileColName = 'obsFileName'
        wvlCalFileColName = 'wvlCalFileName'
        flatCalFileColName = 'flatCalFileName'
        nRowColName = 'nRow'
        nColColName = 'nCol'
        RAColName = 'RA'
        DecColName = 'Dec'
        deadPixColName = 'deadPixFileName'
        hotPixColName = 'hotPixFileName'
        lowWvlColName = 'lowWvlCutoff'
        highWvlColName = 'highWvlCutoff'
        expTimeColName = 'exptime'
        lstColName = 'lst'
        integrationTimeColName = 'integrationTime'
        HA_offsetColName = 'HA_offset'


        # Create and h5 output file, create header and data groups
        fileh = tables.openFile(self.outputFilename, mode='w')
        headerGroup = fileh.createGroup("/", headerGroupName, 'Header')
        stackGroup = fileh.createGroup("/", dataGroupName, 'Image Stack')

        # Create row for header information
        headerTable = fileh.createTable(headerGroup, headerTableName, self.headerDescription,
                                        'Header Info')
        header = headerTable.row

        # Fill in the header with possibly useful information.
        header[runColName] = self.run
        header[targetColName] = self.target
        header[obsFileColName] = self.obsFn       
        header[nColColName] = self.numberCols
        header[nRowColName] = self.numberRows
        header[RAColName] = self.RA
        header[DecColName] = self.Dec
        header[expTimeColName] = self.exptime
        header[lstColName] = self.lst
        header[integrationTimeColName] = self.integrationTime
        header[HA_offsetColName] = self.hourAngleOffset
        if self.useDeadPixelMasking:
            header[deadPixColName] = self.deadPixelFilename
        if self.useHotPixelMasking:
            header[hotPixColName] = self.hotPixelFilename
        if self.useWavelengthCalibration:
            header[wvlCalFileColName] = self.ob.wvlCalFileName       
            header[lowWvlColName] = self.lowerWavelengthCutoff
            header[highWvlColName] = self.upperWavelengthCutoff
        if self.useFlatCalibration:
            header[flatCalFileColName] = self.flatCalFilename
        header.append()

        # Create an h5 array for the midtime of each frame in the image cube.
        timeTable = fileh.createCArray(stackGroup, timeTableName, Float64Atom(), (1,len(self.times)))       
        timeTable[:] = self.times
        
        # Create an h5 table for the image cube.
        stackTable = fileh.createCArray(stackGroup, dataTableName, Float64Atom(), (self.numberRows,self.numberCols, self.cube.shape[2]))        
        stackTable[:] = self.cube

        # Flush the h5 output file
        fileh.flush()
        fileh.close()


    # Start process for creating image stacks
    def stackProcess(self):

        # Check for valid params file
        if self.paramFileExists:

            # Load settings choosen from gui
            self.loadSettings()
            if self.validSettings:
        
                # Loop through all files in input list
                for iFile in range(self.fileCount):
                    # Create ObsFile instance
                    self.obsTS = str(self.currentUTCDate) + '-' + self.ui.inputList.item(iFile).text()
                    self.obsFn = str(FileName(run=self.run,date=self.currentSunsetDate,tstamp=self.obsTS).obs())
                    print 'Processing file ' + self.obsFn + '...'
                    self.ob = ObsFile(self.obsFn)

                    self.numberRows = self.ob.nRow
                    self.numberCols = self.ob.nCol

                    # Load time adjustment file
                    if self.useTimeAdjustment:
                        print 'Loading time adjustment file...'
                        self.ob.loadTimeAdjustmentFile(FileName(run=self.run).timeAdjustments())
       
                    # Load hot pixel mask
                    if self.useHotPixelMasking:
                        print 'Loading hot pixel mask...'
                        self.loadHotMask()

                    # Load wave cal solution
                    if self.useWavelengthCalibration:
                        if self.useBestWavelengthCalibration:
                            print 'Loading best wavelength calibration...'
                            self.ob.loadBestWvlCalFile()
                        else:
                            print 'Loading selected wavelength calibration...'
                            self.ob.loadWvlCalFile(self.wvlCalFilename)
                        
                    # Load flatcal solution
                    if self.useFlatCalibration:
                        print 'Loading flat calibration...'
                        self.ob.loadFlatCalFile(self.flatCalFilename)

                    # Load dead pixel mask
                    if self.useDeadPixelMasking:
                        print 'Loading dead pixel mask...'
                        self.loadDeadMask()

                    # Set wavelength cutoffs
                    if self.useWavelengthCalibration:
                        print 'Setting wavelength cutoffs...'
                        self.ob.setWvlCutoffs(self.lowerWavelengthCutoff,self.upperWavelengthCutoff)
        
                    # Timing
                    self.unix = self.ob.getFromHeader('unixtime')
                    self.startJD = self.unix/86400.+2440587.5
                    self.exptime = self.ob.getFromHeader('exptime')
                    self.lst = self.ob.getFromHeader('lst')
                    
                    self.times = []
                    self.frames = []
                        
                    # Create Image Stack
                    print 'Stacking images...'
                    for iSec in np.arange(0,self.exptime,self.integrationTime):
                        #add seconds offset to julian date, move jd to center of bin
                        self.jd = self.startJD + iSec/(24.*3600.) + self.integrationTime/2./(24.*3600.)
                        self.times.append(self.jd)
                        print 'Creating frame for time ' + str(self.jd)
                        self.frameData = self.ob.getPixelCountImage(firstSec=iSec,integrationTime=self.integrationTime,weighted=self.weighted,getRawCount=self.useRawCounts,scaleByEffInt=self.scaleByEffInt)
                        self.frame = self.frameData['image']

                        if self.ui.verticalFlipBox.isChecked():
                            self.frame = np.flipud(self.frame)                   
         
                        if self.useDeadPixelMasking:
                            self.frame[self.deadMask == 0] = np.nan
                        self.frames.append(self.frame)

                    self.cube = np.dstack(self.frames)
                    self.times = np.array(self.times)
            
                    # Create output file
                    self.createOutputName()
                    print 'Saving image stack to ' + self.outputFilename
                    self.createH5File()                 

        # Invalid params file
        else:
            print 'Invalid parameter file...'

    # Choose an image stack
    
    def chooseStack(self):
        self.defaultLoadStackDirectory = str(self.displayStackPath)
        self.stackName = ''
        self.stackName = QFileDialog.getOpenFileName(parent=None, directory=self.defaultLoadStackDirectory, caption=str("Choose Image Stack"), filter=str("H5 (*.h5)")) 
        if self.stackName == '':
            print 'No file chosen'
        else:          
            loadStackApp = LoadImageStack.LoadImageStack(stackName = self.stackName)
            loadStackApp.show()
            loadStackApp.exec_()
コード例 #14
0
tic = time()
print 'Retrieving photon timestamps...'
# Define the area of an observation containing the object.
appMask = aperture(15,8,8)
yValues,xValues = np.where(appMask==0)
# Get a list of timestamps with photons from each pixel in the aperture.
timestamps=[]
for j in range(len(xValues)):
    x=ob.getPixelWvlList(iRow=yValues[j],iCol=xValues[j])
    timestamps = np.append(timestamps,x['timestamps'])
print 'Total retrieval time: ' + str(time()-tic) + 's'

# Use timestamp data to perform periodograms.
timestep = 5.0*10**-4
averagingTime = 2.0
exptime = ob.getFromHeader('exptime')
totalAverageBins = int(exptime/averagingTime)
timestepsPerBin = int(averagingTime/timestep)
# Specify frequencies for which the periodogram algorithm should find transform components.
freqs = np.linspace(1,1000,num=10**4)
angularFreqs=2*np.pi*freqs

binnedCounts, binEdges = np.histogram(timestamps,range=[0,exptime],bins = exptime/timestep)
times = binEdges[0:timestepsPerBin]

periodogramStack=np.zeros((totalAverageBins,len(freqs)))
tic = time()
print 'Calculating individual Fourier Transforms...'
for bin in range(int(totalAverageBins)):
    counts = binnedCounts[bin*timestepsPerBin:(bin+1)*timestepsPerBin]
    scaledCounts = (counts-counts.mean())/counts.std()
コード例 #15
0
out = '/home/mstrader/data/nltt/'+'timestream'+runlabel+'.txt'
outh5 = '/home/mstrader/data/nltt/'+'list'+runlabel+'.h5'


timestampList = [utcDate+'-'+ts for ts in timestampListPost]

files = []
pl = []
times = []
for i,ts in enumerate(timestampList):
    print 'loading',ts
    fn = FileName(run=run,date=sunsetDate,tstamp=ts).photonList()
    obsFn = FileName(run=run,date=sunsetDate,tstamp=ts).obs()
    files.append(tables.openFile(fn,mode='r'))
    ob = ObsFile(obsFn)
    ut = ob.getFromHeader('unixtime')
    pl.append(files[i].root.photons.photons.read())
    times.append(np.array(pl[i]['ArrivalTime'],dtype=np.float64) + ut)


all=np.concatenate(pl)
allTimes = np.concatenate(times)

nlttPSFByPixels = []
nlttTimesByPixels = []
img = np.zeros((46,44))
rawImg = np.zeros((46,44))
for iPixel in range(len(circCol)):
    x = circCol[iPixel]
    y = circRow[iPixel]
    timesInPixel = allTimes[np.logical_and(all['Xpix']==x,all['Ypix']==y)]
コード例 #16
0
        if not os.path.exists(hotPixFn):
            hp.findHotPixels(obsFn,hotPixFn)
            print "Flux file pixel mask saved to %s"%(hotPixFn)
        ob.loadHotPixCalFile(hotPixFn,switchOnMask=True)
        ob.loadWvlCalFile(wfn)
        ob.loadFlatCalFile(ffn)
        ob.setWvlCutoffs(wvlLowerCutoff,wvlUpperCutoff)

        bad_solution_mask=np.zeros((46,44))
        bad_count=0;
        for y in range(46):
            for x in range(44):
                if (ob.wvlRangeTable[y][x][1] < wvlUpperCutoff) or (ob.wvlRangeTable[y][x][0] > wvlLowerCutoff):
                    bad_solution_mask[y][x] = 1

        unix = ob.getFromHeader('unixtime')
        startJD = unix/86400.+2440587.5
        nSecInFile = ob.getFromHeader('exptime')
        #tic = time()
        deadMask = ob.getDeadPixels()

	#print 'Dead mask load time = ', time()-tic
        for sec in np.arange(0,nSecInFile,integrationTime):
            jd = startJD + sec/(24.*3600.) + integrationTime/2./(24.*3600.)#add seconds offset to julian date, move jd to center of bin
            print count,jd
            count+=1
            times.append(jd)
            titles.append('%.6f'%jd)
            frameData = ob.getPixelCountImage(firstSec=sec,integrationTime=integrationTime,weighted=True,scaleByEffInt=True)
            frame = frameData['image']
            showFrame = np.array(frame)
コード例 #17
0
y_values,x_values = np.where(app_mask==0)
unixOffset=0.0

timestamps =[]
tic = time()
for iSeq in range(len(seqs)):
    timestampList = timestampLists[iSeq]
    wfn = wvlCalFilenames[iSeq]
    ffn = flatCalFilenames[iSeq]
    sunsetDate = sunsetDates[iSeq]
    for i,ts in enumerate(timestampList):
        print 'Loading',ts
        obsFn = FileName(run=run,date=sunsetDate,tstamp=ts).obs()
        ob = ObsFile(obsFn)
	ob.loadTimeAdjustmentFile(FileName(run='PAL2012').timeAdjustments())
	exptime = ob.getFromHeader('exptime')
	unixtime= ob.getFromHeader('unixtime')
	if i == 0:
	    unixOffset = unixtime
	exptimes[i]=exptime
	index1 = obsFn.find('_')
	hotPixFn = '/Scratch/timeMasks/timeMask' + obsFn[index1:]
        if not os.path.exists(hotPixFn):
            hp.findHotPixels(obsFn,hotPixFn)
            print "Flux file pixel mask saved to %s"%(hotPixFn)
        ob.loadHotPixCalFile(hotPixFn,switchOnMask=True)
        ob.loadWvlCalFile(wfn)
        ob.loadFlatCalFile(ffn)
	ob.setWvlCutoffs(3000,8000)
	for j in range(len(x_values)):
	    x=ob.getPixelWvlList(iRow=y_values[j],iCol=x_values[j])