def loadObs(self,stackLabel):
        timestampList = self.params[stackLabel+'Sequence']
        run = self.params['run']
        sunsetDate = self.params[stackLabel+'SunsetDate']
        utcDate = self.params[stackLabel+'UtcDate']
        intTime = self.params[stackLabel+'IntTime']
        wvlLowerCutoff = self.params[stackLabel+'WvlLowerCutoff']
        wvlUpperCutoff = self.params[stackLabel+'WvlUpperCutoff']

        calTimestamp = self.params[stackLabel+'WvlTimestamp']
        print stackLabel,calTimestamp
        wvlSolnFileName = FileName(run=run,date=sunsetDate,tstamp=calTimestamp).calSoln()
        wvlCalFileName = FileName(run=run,date=self.params[stackLabel+'WvlSunsetDate'],tstamp=calTimestamp).cal()
        flatSolnFileName = FileName(run=run,date=self.params[stackLabel+'FlatCalSunsetDate'],tstamp=self.params[stackLabel+'FlatCalTimestamp']).flatSoln()

        obsFileNames = [FileName(run=run,date=sunsetDate,tstamp=timestamp).obs() for timestamp in timestampList]
        obList = [ObsFile(obsFn) for obsFn in obsFileNames]
        for ob in obList:
            ob.loadWvlCalFile(wvlSolnFileName)
            ob.loadFlatCalFile(flatSolnFileName)

        self.stackObsFileLists[stackLabel] = obList
        
        cal = ObsFile(wvlCalFileName)
        cal.loadWvlCalFile(wvlSolnFileName)
        cal.loadFlatCalFile(flatSolnFileName)
        self.stackWvlCals[stackLabel] = cal
    def findv1(self):
        populationMax=2000
        ySum = np.zeros(populationMax)
        frameSum = 'none'
        seq5 = self.s['seq5'].split()
        for seq in seq5:
            print "seq=",seq
            outfileName = "cosmicTimeList-"+seq+".pkl"
            if not os.path.exists(outfileName):
                fn = FileName(self.s['run'], 
                              self.s['sundownDate'], 
                              self.s['obsDate']+"-"+str(seq))
                cosmic = Cosmic(fn, 
                                beginTime=self.s['beginTime'],
                                endTime=self.s['endTime'],
                                loggingLevel = logging.INFO)

                fc = cosmic.findCosmics(stride=int(self.s['stride']), 
                                        threshold=int(self.s['threshold']), 
                                        populationMax=populationMax,
                                        nSigma=float(self.s['nSigma']))
                outfile = open(outfileName, "wb")
                pickle.dump(fc['cosmicTimeList'],outfile)
                pickle.dump(fc['binContents'],outfile)
                outfile.close()
                cfn = "cosmicMask-%s.h5"%seq
                ObsFile.writeCosmicIntervalToFile(fc['interval'],1.0e6, cfn,
                                                  self.s['beginTime'],
                                                  self.s['endTime'],
                                                  int(self.s['stride']),
                                                  int(self.s['threshold']),
                                                  float(self.s['nSigma']),
                                                  populationMax)

                del cosmic
def testLoadBeammap():
    '''
    Test if a remapped beammap is actually remapped
    '''
    #open an obs file from PAL2012,the sky file for hr9087
    #we'll use the embedded beammap file, which has some misplaced pixels
    run = 'PAL2012'
    date = '20121210'
    obsTimestamp = '20121211-051650'
    obsFN = FileName(run=run,date=date,tstamp=obsTimestamp)
    obsFileName = obsFN.obs()
    obs = ObsFile(obsFileName)
    
    beammapFileName = obsFN.beammap()
    
    #load the PixelMap for PAL2012 to know which pixels should be remapped
    pixMap = remapPixels.PixelMap(obsFN.pixRemap())
    pixMapSourceList,pixMapDestList = pixMap.getRemappedPix()
    
    #load the corrected beammap into the obs
    obs.loadBeammapFile(beammapFileName)

    #check that each pixel that should be moved is moved
    #by comparing the embedded beammap and the loaded corrected one
    for source,dest in zip(pixMapSourceList,pixMapDestList):
        assert obs.beamImage[dest] == obs.file.root.beammap.beamimage[source]

    obs.file.close()
def main():
    np.set_printoptions(threshold=np.nan)
    testPixelRow = 24
    testPixelCol = 17
    #obs_20120919-131142.h5,obs_20120919-131346.h5
    #create a cal file from a twilight flat
    cal = FlatCal('../../params/flatCal.dict')
    #open another twilight flat as an observation and apply a wavelength cal and the new flat cal
#    run='LICK2012'
#    obsFileName = FileName(run=run,date='20120918',tstamp='20120919-131142').flat()
#    flatCalFileName = FileName(run=run,date='20120918',tstamp='20120919-131448').flatSoln()
#    wvlCalFileName = FileName(run=run,date='20120916',tstamp='20120917-072537').calSoln()

    run = 'PAL2012'
    obsFileName = FileName(run=run,date='20121211',tstamp='20121212-140003').obs()
    flatCalFileName = FileName(run=run,date='20121210',tstamp='').flatSoln()
    wvlCalFileName = FileName(run=run,date='20121210',tstamp='20121211-133056').calSoln()
    flatCalPath = os.path.dirname(flatCalFileName)

    ob = ObsFile(obsFileName)#('obs_20120919-131142.h5')
    ob.loadWvlCalFile(wvlCalFileName)#('calsol_20120917-072537.h5')
    ob.loadFlatCalFile(flatCalFileName)#('flatsol_20120919-131142.h5')

    #plot some uncalibrated and calibrated spectra for one pixel
    fig = plt.figure()
    ax = fig.add_subplot(211)
    ax2 = fig.add_subplot(212)
    print ob.getPixelCount(testPixelRow,testPixelCol)

    #flatSpectrum,wvlBinEdges = ob.getPixelSpectrum(testPixelRow,testPixelCol,weighted=False)
    spectrum,wvlBinEdges = ob.getPixelSpectrum(testPixelRow,testPixelCol,wvlStart=cal.wvlStart,wvlStop=cal.wvlStop,wvlBinWidth=cal.wvlBinWidth,weighted=False,firstSec=0,integrationTime=-1)

    weightedSpectrum,wvlBinEdges = ob.getPixelSpectrum(testPixelRow,testPixelCol,weighted=True)
    #flatSpectrum,wvlBinEdges = cal.flatFile.getPixelSpectrum(testPixelRow,testPixelCol,wvlStart=cal.wvlStart,wvlStop=cal.wvlStop,wvlBinWidth=cal.wvlBinWidth,weighted=False,firstSec=0,integrationTime=-1)
    flatSpectrum = cal.spectra[testPixelRow,testPixelCol]
    x = wvlBinEdges[0:-1]
    ax.plot(x,cal.wvlMedians,label='median spectrum',alpha=.5)
    ax2.plot(x,cal.flatFactors[testPixelRow,testPixelCol,:],label='pixel weights',alpha=.5)
    ax2.set_title('flat weights for pixel %d,%d'%(testPixelRow,testPixelCol))
    ax.plot(x,spectrum+20,label='unweighted spectrum for pixel %d,%d'%(testPixelRow,testPixelCol),alpha=.5)
    ax.plot(x,weightedSpectrum+10,label='weighted %d,%d'%(testPixelRow,testPixelCol),alpha=.5)
    ax.plot(x,flatSpectrum+30,label='flatFile %d,%d'%(testPixelRow,testPixelCol),alpha=.5)

    ax.legend(loc='upper center', bbox_to_anchor=(0.5, 1.3),fancybox=True,ncol=3)
    plt.show()

    #display a time-flattened image of the twilight flat as it is and after using itself as it's flat cal
    #cal.flatFile.loadFlatCalFile(flatCalFileName)#('flatsol_20120919-131142.h5')
    #cal.flatFile.displaySec(weighted=True,integrationTime=-1)
    #ob.displaySec(integrationTime=-1)
    #ob.displaySec(weighted=True,integrationTime=-1)


    for idx in range(0,100,20):
        factors10 = cal.flatFactors[:,:,idx]
        plt.matshow(factors10,vmax=np.mean(factors10)+1.5*np.std(factors10))
        plt.title('Flat weights at %d'%cal.wvlBinEdges[idx])
        plt.colorbar()
        plt.savefig('plots/factors%d.png'%idx)
        plt.show()
def centroidObs(obsPath,centroidPath,centroidRa,centroidDec,haOffset,xGuess,yGuess,hotPath,flatPath):
    obs = ObsFile(obsPath)
    print obsPath,obs.getFromHeader('exptime'),obs
    if not os.path.exists(hotPath):
        hp.findHotPixels(obsFile=obs,outputFileName=hotPath)
    obs.loadHotPixCalFile(hotPath,switchOnMask=False)
    obs.loadBestWvlCalFile()
    obs.loadFlatCalFile(flatPath)
    obs.setWvlCutoffs(3000,8000)
    cc.centroidCalc(obs,centroidRa,centroidDec,guessTime=300,integrationTime=30,secondMaxCountsForDisplay=2000,HA_offset=haOffset,xyapprox=[xGuess,yGuess],outputFileName=centroidPath)
    print 'done centroid',centroidPath
    del obs
    def makemovie1(self):
        run = self.s['run']
        sundownDate = self.s['sundownDate']
        obsDate = self.s['obsDate']
        stride = int(self.s['stride'])
        seq5 = self.s['seq5'].split()
        for seq in seq5:
            inFile = open("cosmicTimeList-%s.pkl"%(seq),"rb")
            cosmicTimeList = pickle.load(inFile)
            binContents = pickle.load(inFile)
            cfn = "cosmicMask-%s.h5"%seq
            intervals = ObsFile.readCosmicIntervalFromFile(cfn)
            for interval in intervals:
                print "interval=",interval
                fn = FileName(run, sundownDate,obsDate+"-"+seq)
                obsFile = ObsFile(fn.obs())
                obsFile.loadTimeAdjustmentFile(fn.timeAdjustments())
                i0=interval[0]
                i1=interval[1]
                intervalTime = i1-i0
                dt = intervalTime/2
                beginTime = max(0,i0-0.000200)
                endTime = beginTime + 0.001
                integrationTime = endTime-beginTime
                nBins = int(np.round(obsFile.ticksPerSec*(endTime-beginTime)+1))
                timeHgValues = np.zeros(nBins, dtype=np.int64)
                ymax = sys.float_info.max/100.0
                for iRow in range(obsFile.nRow):
                    for iCol in range(obsFile.nCol):
                        gtpl = obsFile.getTimedPacketList(iRow,iCol,
                                                          beginTime,integrationTime)
                        ts = (gtpl['timestamps'] - beginTime)*obsFile.ticksPerSec
                        ts64 = np.round(ts).astype(np.uint64)
                        tsBinner.tsBinner(ts64, timeHgValues)
                plt.clf()
                plt.plot(timeHgValues, label="data")
                x0 = (i0-beginTime)*obsFile.ticksPerSec
                x1 = (i1-beginTime)*obsFile.ticksPerSec
                plt.fill_between((x0,x1),(0,0), (ymax,ymax), alpha=0.2, color='red')   
                plt.yscale("symlog",linthreshy=0.9)
                plt.xlim(0,1000)
                plt.ylim(-0.1,300)
                tick0 = int(np.round(i0*obsFile.ticksPerSec))
                plotfn = "cp-%05d-%s-%s-%s-%09d"%(timeHgValues.sum(),run,obsDate,seq,tick0)
                plt.title(plotfn)
                plt.legend()
                plt.savefig(plotfn+".png")
                plt.xlabel("nSigma=%d stride=%d threshold=%d"%(int(self.s['nSigma']),int(self.s['stride']),int(self.s['threshold'])))
                print "plotfn=",plotfn
                

        os.system("convert -delay 0 `ls -r cp*png` cp.gif")
def centroidObs(obsPath,centroidPath,centroidRa,centroidDec,haOffset,xGuess,yGuess,savePath,tstamp):
    obs = ObsFile(obsPath)
#    if not os.path.exists(hotPath):
#        hp.findHotPixels(obsFile=obs,outputFileName=hotPath)
    obs.loadAllCals()
#    obs.loadHotPixCalFile(hotPath,switchOnMask=True)
#    obs.loadBestWvlCalFile()
#    obs.loadFlatCalFile(flatPath)
    obs.setWvlCutoffs(3000,11000)
    obs.loadCentroidListFile(centroidPath)
    
    ctrdFile = obs.centroidListFile
    sliceTimes = ctrdFile.root.centroidlist.times.read()
    xPositions = ctrdFile.root.centroidlist.xPositions.read()
    yPositions = ctrdFile.root.centroidlist.yPositions.read()
    intTime = sliceTimes[1]-sliceTimes[0]
    
    for iTime,time in enumerate(sliceTimes):
        x = xPositions[iTime]
        y = yPositions[iTime]
        title='centroid_{}_{}s'.format(tstamp,time)
        imgDict = obs.getPixelCountImage(firstSec=time,integrationTime=intTime,weighted=True)
        imgPath=os.path.join(savePath,title+'.png')
        pop = PopUp(showMe=False)
        pop.plotArray(imgDict['image'],title=title)
        pop.axes.plot(x,y,color='g',marker='d')
        pop.fig.savefig(imgPath)
        print 'saved to',imgPath
        
    del obs
    def openImage(self):

        timestampList = [self.params['obsUtcDate']+'-'+ts for ts in self.params['obsSequence']] 
        run = self.params['run']
        sunsetDate = self.params['obsSunsetDate']
        utcDate = self.params['obsUtcDate']
        self.intTime = self.params['intTime']
        wvlLowerCutoff = self.params['wvlLowerCutoff']
        wvlUpperCutoff = self.params['wvlUpperCutoff']

        calTimestamp = self.params['wvlTimestamp']
        wfn = FileName(run=run,date=sunsetDate,tstamp=calTimestamp).calSoln()
        calfn = FileName(run=run,date=self.params['wvlSunsetDate'],tstamp=calTimestamp).cal()
        ffn = FileName(run=run,date=self.params['flatCalSunsetDate'],tstamp='').flatSoln()

        obsFns = [FileName(run=run,date=sunsetDate,tstamp=timestamp).obs() for timestamp in timestampList]
        self.obList = [ObsFile(obsFn) for obsFn in obsFns]
        for ob in self.obList:
            print 'Loading ',ob.fullFileName
            ob.loadWvlCalFile(wfn)
            ob.loadFlatCalFile(ffn)

        self.cal = ObsFile(calfn)
        self.cal.loadWvlCalFile(wfn)
        self.cal.loadFlatCalFile(ffn)
        self.loadSpectra()
    def __init__(self, fn, beginTime=0, endTime='exptime',
                 nBinsPerSec=10, flashMergeTime=1.0, 
                 applyCosmicMask = False,
                 loggingLevel=logging.CRITICAL,
                 loggingHandler=logging.StreamHandler()):
        
        """
        Opens fileName in MKID_RAW_PATH, sets roachList
        endTime is exclusive
        """
        self.logger = logging.getLogger("cosmic")
        self.logger.setLevel(loggingLevel)
        formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
        loggingHandler.setFormatter(formatter)
        self.logger.addHandler(loggingHandler)
        self.logger.info("Cosmic:  begin init for obsFile=%s"%fn.obs())
        self.fn = fn
        self.fileName = fn.obs();
        self.file = ObsFile(self.fileName)
        # apply Matt's time fix
        timeAdjustments = self.fn.timeAdjustments()
        if os.path.exists(timeAdjustments):
            self.file.loadTimeAdjustmentFile(timeAdjustments)
        # apply Julian's time masks
        timeMaskFile = self.fn.timeMask();
        if os.path.exists(timeMaskFile):
            self.file.loadHotPixCalFile(timeMaskFile,switchOnMask=True)

        # apply standard mask
        if applyCosmicMask:
            self.file.loadStandardCosmicMask()

        self._setRoachList()
        self._setAllSecs()
        self.exptime = self.file.getFromHeader('exptime')
        if endTime =='exptime':
            self.endTime = float(self.exptime)
        else:
            self.endTime = float(endTime)
        if ( (self.endTime > self.exptime) or (endTime < 0)):
            raise RuntimeError("bad endTime:  endTime=%s exptime=%s" % \
                                   (str(endTime),str(self.exptime)))

        self.beginTime = float(beginTime)
        self.timeHgs = "none"
        self.nBinsPerSec = nBinsPerSec
        self.flashMergeTime = flashMergeTime

        self.times = \
        np.arange(self.beginTime, self.endTime, 1.0/self.nBinsPerSec)

        # for measuring flashes, indexed by roach name
        self.rMean = {}   # mean from meanclip
        self.rSigma = {}  # sigma from meanclip
        self.rNSurvived = {} # number of survivors from meanclip
        self.rNormed = {} # (value-mean)/sigma
        self.flashInterval = {}
        self.logger.info("Cosmic:  end init:  beginTime=%s endTime=%s"%(str(self.beginTime),str(self.endTime)))
def timePL(tstamp,obsPath,centroidPath):
    obs = ObsFile(obsPath)
    obs.loadAllCals()
    obs.setWvlCutoffs(3000,11000)
    obs.loadCentroidListFile(centroidPath)
    writePhotonList(obs,photListDescription=PulsarPhotonList,checkForExisting=False)
    del obs
def main():

    objectName = "hz21"
    fileNum=0
    energyBinWidth = 0.1
    #make bins for 3000 to 13000
    wvlStart = 3000
    wvlStop = 13000
    wvlBinEdges = ObsFile.makeWvlBins(energyBinWidth,wvlStart,wvlStop)
    nWvlBins = len(wvlBinEdges)-1
    binWidths = np.empty(nWvlBins)
    print "Showing bin widths for %i bins"%(nWvlBins)
    for i in xrange(nWvlBins):
        binWidths[i] = wvlBinEdges[i+1]-wvlBinEdges[i]
    print binWidths

    nVirtPixX=250
    nVirtPixY=250
    cube = np.zeros((nVirtPixX,nVirtPixY,nWvlBins),dtype=float)

    for n in xrange(nWvlBins):
        print "Making image for wvls %i to %i"%(wvlBinEdges[n], wvlBinEdges[n+1])
        virtualImage, imageStack, medImage = makeImageStack(fileNames='photons_*.h5', dir=os.getenv('MKID_PROC_PATH', default="/Scratch")+'/photonLists/20131209',
                   detImage=False, saveFileName='stackedImage.pkl', wvlMin=wvlBinEdges[n],
                   wvlMax=wvlBinEdges[n+1], doWeighted=True, medCombine=False, vPlateScale=0.2,
                   nPixRA=nVirtPixX,nPixDec=nVirtPixY)
        print virtualImage
        print virtualImage.image
        print np.shape(virtualImage.image)
        cube[:,:,n] = virtualImage.image
    
    #calculate midpoints of wvl bins for plotting
    wvls = np.empty((nWvlBins),dtype=float)
    for n in xrange(nWvlBins):
        binsize=wvlBinEdges[n+1]-wvlBinEdges[n]
        wvls[n] = (wvlBinEdges[n]+(binsize/2.0))

    print "wvls ",wvls
    #reshape cube for makeMovie
    movieCube = np.zeros((nWvlBins,np.shape(cube)[0],np.shape(cube)[1]),dtype=float)
    for i in xrange(nWvlBins):
        movieCube[i,:,:] = cube[:,:,i]
        #show individual frames as they are made to debug
        #plt.matshow(movieCube[i],vmin = 0, vmax = 100)
        #plt.show()
    print "movieCube shape ", np.shape(movieCube)
    print "wvls shape ", np.shape(wvls)

    #print cube
    #print "--------------------------"
    #print movieCube

    np.savez('%s_raw_%s.npz'%(objectName,fileNum),stack=movieCube,wvls=wvls)
    utils.makeMovie(movieCube,frameTitles=wvls,cbar=True,outName='%s_pl_raw_%s.gif'%(objectName,fileNum), normMin=0, normMax=1000)

    '''
def centroidObs(obsPath,centroidPath,centroidRa,centroidDec,haOffset,xGuess,yGuess):
    obs = ObsFile(obsPath)
    print obsPath,obs.getFromHeader('exptime'),obs
    obs.loadAllCals()
#    obs.loadBestWvlCalFile()
#    obs.loadFlatCalFile(flatPath)
    obs.setWvlCutoffs(3000,11000)
#    if not os.path.exists(hotPath):
#        hp.findHotPixels(obsFile=obs,outputFileName=hotPath,display=True,fwhm=2.,boxSize=5, nSigmaHot=4.0,)
#    obs.loadHotPixCalFile(hotPath,switchOnMask=True)
    cc.centroidCalc(obs,centroidRa,centroidDec,guessTime=300,integrationTime=30,secondMaxCountsForDisplay=2000,HA_offset=haOffset,xyapprox=[xGuess,yGuess],outputFileName=centroidPath,usePsfFit=True,radiusOfSearch=8)
    print 'done centroid',centroidPath
    del obs
def main():
    """
    params = []
    paramfile = sys.argv[1]

    f = open(paramfile,'r')
    for line in f:
        params.append(line)
    f.close()
    datadir = params[0].split('=')[1].strip()
    flatdir = params[1].split('=')[1].strip()
    fluxdir = params[2].split('=')[1].strip()
    wvldir = params[3].split('=')[1].strip()
    obsfile = params[4].split('=')[1].strip()
    skyfile = params[5].split('=')[1].strip()
    flatfile = params[6].split('=')[1].strip()
    fluxfile = params[7].split('=')[1].strip()
    wvlfile = params[8].split('=')[1].strip()
    objectName = params[9].split('=')[1].strip()
    fluxCalObject = params[10].split('=')[1].strip()

    obsFileName = os.path.join(datadir, obsfile)
    skyFileName = os.path.join(datadir, skyfile)
    wvlCalFileName = os.path.join(wvldir, wvlfile)
    flatCalFileName = os.path.join(flatdir, flatfile)
    fluxCalFileName = os.path.join(fluxdir, fluxfile)
    """

    if len(sys.argv) > 2:
        fileNum = str(sys.argv[2])
    else:
        fileNum = "0"

    # science object parameter file
    params = []
    paramfile = sys.argv[1]
    f = open(paramfile, "r")
    for line in f:
        params.append(line)
    f.close()

    datadir = params[0].split("=")[1].strip()
    flatdir = params[1].split("=")[1].strip()
    wvldir = params[2].split("=")[1].strip()
    obsfile = params[3].split("=")[1].strip()
    skyfile = params[4].split("=")[1].strip()
    flatfile = params[5].split("=")[1].strip()
    wvlfile = params[6].split("=")[1].strip()
    objectName = params[9].split("=")[1].strip()

    # wvldir = "/Scratch/waveCalSolnFiles/oldbox_numbers/20121205"
    # objectName = "crabNight1"

    if len(params) > 10:
        xpix = int(params[10].split("=")[1].strip())
        ypix = int(params[11].split("=")[1].strip())
        apertureRadius = int(params[12].split("=")[1].strip())
        startTime = int(params[13].split("=")[1].strip())
        intTime = int(params[14].split("=")[1].strip())

    obsFileName = os.path.join(datadir, obsfile)
    skyFileName = os.path.join(datadir, skyfile)
    wvlCalFileName = os.path.join(wvldir, wvlfile)
    flatCalFileName = os.path.join(flatdir, flatfile)

    obs = ObsFile(obsFileName)
    # obs.loadWvlCalFile(wvlCalFileName)
    obs.loadBestWvlCalFile()
    obs.loadFlatCalFile(flatCalFileName)
    print "analyzing file %s" % (obsFileName)
    print "loaded data file and calibrations\n---------------------\n"

    nRow = obs.nRow
    nCol = obs.nCol
    obsTime = obs.getFromHeader("exptime")
    # wvlBinEdges,obsSpectra = loadSpectra(obs,nCol,nRow)
    # nWvlBins=len(wvlBinEdges)-1

    # print np.shape(obsSpectra)
    # print nRow
    # print nCol
    # print nWvlBins

    # Apply Hot pixel masking before getting dead time correction
    # HotPixFile = getTimeMaskFileName(obsFileName)
    HotPixFile = FileName(obsFile=obs).timeMask()
    print "making hot pixel file ", HotPixFile
    if not os.path.exists(HotPixFile):  # check if hot pix file already exists
        hp.findHotPixels(inputFileName=obsFileName, outputFileName=HotPixFile)
        print "Flux file pixel mask saved to %s" % (HotPixFile)
    obs.loadHotPixCalFile(HotPixFile)
    print "Hot pixel mask loaded %s" % (HotPixFile)

    # GET RAW PIXEL COUNT IMAGE TO CALCULATE CORRECTION FACTORS
    print "Making raw cube to get dead time correction"
    cubeDict = obs.getSpectralCube(firstSec=startTime, integrationTime=intTime, weighted=False)

    cube = np.array(cubeDict["cube"], dtype=np.double)
    wvlBinEdges = cubeDict["wvlBinEdges"]
    effIntTime = cubeDict["effIntTime"]
    print "median effective integration time = ", np.median(effIntTime)

    nWvlBins = len(wvlBinEdges) - 1
    print "cube shape ", np.shape(cube)
    print "effIntTime shape ", np.shape(effIntTime)

    # add third dimension to effIntTime for  broadcasting
    effIntTime = np.reshape(effIntTime, np.shape(effIntTime) + (1,))
    # put cube into counts/s in each pixel
    cube /= effIntTime

    # CALCULATE DEADTIME CORRECTION
    # NEED TOTAL COUNTS PER SECOND FOR EACH PIXEL TO DO PROPERLY
    # ASSUMES SAME CORRECTION FACTOR APPLIED FOR EACH WAVELENGTH, MEANING NO WL DEPENDANCE ON DEAD TIME EFFECT
    DTCorr = np.zeros((np.shape(cube)[0], np.shape(cube)[1]), dtype=float)
    for f in range(0, np.shape(cube)[2]):
        print cube[:, :, f]
        print "-----------------------"
        DTCorr += cube[:, :, f]
        print DTCorr
        print "\n=====================\n"
    # Correct for 100 us dead time
    DTCorrNew = DTCorr / (1 - DTCorr * 100e-6)
    CorrFactors = DTCorrNew / DTCorr  # This is what the frames need to be multiplied by to get their true values

    print "Dead time correction factors = "
    print CorrFactors

    # REMAKE CUBE WITH FLAT WEIGHTS AND APPLY DEAD TIME CORRECTION AS WELL
    print "Making Weighted cube"
    # load/generate hot pixel mask file
    # HotPixFile = getTimeMaskFileName(obsFileName)
    HotPixFile = FileName(obsFile=obs).timeMask()
    if not os.path.exists(HotPixFile):  # check if hot pix file already exists
        hp.findHotPixels(inputFileName=obsFileName, outputFileName=HotPixFile)
        print "Flux file pixel mask saved to %s" % (HotPixFile)
    obs.loadHotPixCalFile(HotPixFile)
    print "Hot pixel mask loaded %s" % (HotPixFile)

    cubeDict = obs.getSpectralCube(firstSec=startTime, integrationTime=intTime, weighted=True, fluxWeighted=False)
    # cubeDict = obs.getSpectralCube(firstSec=startTime, integrationTime=intTime, weighted=True, fluxWeighted=True)

    cube = np.array(cubeDict["cube"], dtype=np.double)
    wvlBinEdges = cubeDict["wvlBinEdges"]
    effIntTime = cubeDict["effIntTime"]
    print "median effective integration time = ", np.median(effIntTime)

    nWvlBins = len(wvlBinEdges) - 1
    print "cube shape ", np.shape(cube)
    print "effIntTime shape ", np.shape(effIntTime)

    # add third dimension to effIntTime for broadcasting
    effIntTime = np.reshape(effIntTime, np.shape(effIntTime) + (1,))
    # put cube into counts/s in each pixel
    cube /= effIntTime

    # add third dimension to CorrFactors for broadcasting
    CorrFactors = np.reshape(CorrFactors, np.shape(CorrFactors) + (1,))
    # apply dead time correction factors
    cube *= CorrFactors

    # calculate midpoints of wvl bins for plotting
    wvls = np.empty((nWvlBins), dtype=float)
    for n in xrange(nWvlBins):
        binsize = wvlBinEdges[n + 1] - wvlBinEdges[n]
        wvls[n] = wvlBinEdges[n] + (binsize / 2.0)

    print "wvls ", wvls
    # reshape cube for makeMovie
    movieCube = np.zeros((nWvlBins, np.shape(cube)[0], np.shape(cube)[1]), dtype=float)
    for i in xrange(nWvlBins):
        movieCube[i, :, :] = cube[:, :, i]
        # show individual frames as they are made to debug
        # plt.matshow(movieCube[i],vmin = 0, vmax = 100)
        # plt.show()
    print "movieCube shape ", np.shape(movieCube)
    print "wvls shape ", np.shape(wvls)

    # print cube
    # print "--------------------------"
    # print movieCube

    outdir = "/home/srmeeker/scratch/standards/"
    np.savez(outdir + "%s_raw_%s.npz" % (objectName, fileNum), stack=movieCube, wvls=wvls)
    utils.makeMovie(
        movieCube,
        frameTitles=wvls,
        cbar=True,
        outName=outdir + "%s_raw_%s.gif" % (objectName, fileNum),
        normMin=0,
        normMax=50,
    )

    """
def main():

    run = 'PAL2014'
    year = '2014'
    initialPath = '/Scratch'
    packetMasterLogDir = '/LABTEST/PacketMasterLogs/'
    initialPath = os.path.join(initialPath,run)
    outPath = FileName(run=run,mkidDataDir='/Scratch/').timeAdjustments()
    outFile = tables.openFile(outPath,'w')
    timeAdjustGroup = outFile.createGroup('/','timeAdjust','Times to add to timestamps')
    firmwareDelayTable = outFile.createTable(timeAdjustGroup,'firmwareDelay',firmwareDelayDescription,'Times to add to all timestamps taken with a firmware bof')
    newFirmwareEntry = firmwareDelayTable.row
    newFirmwareEntry['firmwareName']='chan_svf_2014_Aug_06_1839.bof'
    newFirmwareEntry['firmwareDelay']=-41e-6 #s, subtract 41 us from timestamps
    newFirmwareEntry.append()
    firmwareDelayTable.flush()
    firmwareDelayTable.close()
    roachDelayTable = outFile.createTable(timeAdjustGroup,'roachDelays',roachDelaysDescription,'Times to add to each roach\'s timestamps')


    for sunsetDatePath in sorted(glob.glob(os.path.join(initialPath,year+'*'))):
        sunsetDate = os.path.basename(sunsetDatePath)
        for fullObsPath in sorted(glob.glob(os.path.join(sunsetDatePath,'obs*.h5'))):
            obsFileName = os.path.basename(fullObsPath)
            obsTStamp = obsFileName.split('.')[0].split('_')[1]
            print obsFileName
        

            obsFN = FileName(run=run,date=sunsetDate,tstamp=obsTStamp,packetMasterLogDir=packetMasterLogDir)
            pmLogFileName = obsFN.packetMasterLog()
            try:
                ob = ObsFile(fullObsPath)
            except:
                print 'can\'t open file'
                continue
                
            try:
                if os.path.getsize(pmLogFileName) <= 0:
                    continue
            except:
                print 'can\'t open Packet Master Log ',pmLogFileName
                continue

            try:
                f = open(pmLogFileName,'r')
            except:
                print 'can\'t open Packet Master Log ',pmLogFileName
                continue
            lastTstampLines = np.zeros(8)
            firstTstampLines = np.zeros(8)

            tstampLine = ''
            for line in f:
                if 'here' in line:
                    #skip over large log files with debug info
                    print 'skipping file with "here"'
                    continue
                if line.split(' ')[0] == 'bundle':
                    tstampLine = line
                    break

            if tstampLine == '':
                print 'skipping file without "bundle"'
                #didn't find lines with 'bundle' in them
                continue

            f.seek(0)
            for line in f:
                if line.split(' ')[0] == 'bundle':
                    try:
                        at = float(line.split('at')[1].split()[0])
                    except:
                        break
                    lastTstampLine = at
                    iRoach = int(line.split('roach')[1].split('took')[0].strip())
                    lastTstampLines[iRoach] = at
                    if firstTstampLines[iRoach] == 0:
                        firstTstampLines[iRoach] = at
                    
            packetReceivedUnixTimestamp = float((tstampLine.split('took')[1].split('total')[0].strip()))
            firstPacketDelay = packetReceivedUnixTimestamp-int(ob.getFromHeader('unixtime'))
                

            roachSecDelays =np.array(np.floor(lastTstampLines+firstPacketDelay-ob.getFromHeader('exptime')),dtype=np.int)
            print 'roach delays',roachSecDelays
            if np.all(roachSecDelays >= 0):
                newEntry = roachDelayTable.row
                newEntry['obsFileName'] = os.path.basename(fullObsPath)
                newEntry['roachDelays'] = roachSecDelays
                newEntry.append()
                roachDelayTable.flush()
            else:
                print 'obs was aborted midway, delays cannot be calculated'
    roachDelayTable.close()
    outFile.close()
Beispiel #15
0
inCounts = []
for line in powermeter:
    if '#' in line:
        continue
    inCounts.append(float(line.strip()))
inCounts = np.array(inCounts)

pixToUse = [[0,26],[2,25],[2,27],[2,31],[2,34],[2,39],[3,17],[3,26],[3,32],
            [3,36],[4,26],[4,32],[5,32],[6,22],[6,29],[6,30],[7,25],[7,27],[7,28],
            [7,32],[7,37],[8,30],[10,32],[10,42],[11,30],[12,29],[12,31],[13,29],[13,31],
            [14,27],[14,32],[15,25],[15,26],[15,30],[15,31],[15,32],[16,23],[16,30],
            [17,23],[17,25],[17,26],[17,39],[18,24],[18,28],[19,23],[19,26],[19,28],[19,30]]

cmap = matplotlib.cm.jet
obsFileName = '/Scratch/linearityTestData/obs_20130612-003423.h5'
obs = ObsFile(obsFileName)


row,col=pixToUse[0]
allResolutions = []
allModResolutions = []
allModResolutions2 = []
allCountRates = []
timeBinStarts = timeBinStarts[0:-3]
for (row,col) in pixToUse[0:3]:
    firstSec=timeBinStarts[0]
    intTime=timeBinWidth
    resolutions = []
    modResolutions = []
    modResolutions2 = []
    countRates = []
    def findCosmics(self, stride=10, threshold=100, 
                    populationMax=2000, nSigma=5, writeCosmicMask=False,
                    ppsStride=10000):
        """
        Find cosmics ray suspects.  Histogram the number of photons
        recorded at each timeStamp.  When the number of photons in a
        group of stride timeStamps is greater than threshold in second
        iSec, add (iSec,timeStamp) to cosmicTimeLists.  Also keep
        track of the histogram of the number of photons per stride
        timeStamps.

        return a dictionary of 'populationHg', 'cosmicTimeLists',
        'binContents', 'timeHgValues', 'interval', 'frameSum', and 'pps'
      
         populationHg is a histogram of the number of photons in each time bin.
        This is a poisson distribution with a long tail due to cosmic events

        
        cosmicTimeLists is a numpy array  of all the sequences that are
        suspects for cosmic rays


        binContents corresponds to cosmicTimeLists.  For each time in
        cosmicTimeLists, binContents is the number of photons detected
        at that time.
        
        timeHgValues is a histogram of the number of photons in each time
        interval
        
        frameSum is a two dimensional  numpy array of the number of photons
        detected by each pixel

        interval is the interval of data to be masked out

        pps is photons per second, calculated every ppsStride bins.

        """

        self.logger.info("findCosmics: begin stride=%d threshold=%d populationMax=%d nSigma=%d writeCosmicMask=%s"%(stride,threshold,populationMax,nSigma,writeCosmicMask))
        exptime = self.endTime-self.beginTime
        nBins = int(np.round(self.file.ticksPerSec*exptime+1))
        bins = np.arange(0, nBins, 1)
        timeHgValues,frameSum = self.getTimeHgAndFrameSum(self.beginTime,self.endTime)
        remainder = len(timeHgValues)%ppsStride
        if remainder > 0:
            temp = timeHgValues[:-remainder]
        else:
            temp = timeHgValues
        ppsTime = (ppsStride*self.file.tickDuration)
        pps = np.sum(temp.reshape(-1, ppsStride), axis=1)/ppsTime
        self.logger.info("findCosmics:  call populationFromTimeHgValues")
        pfthgv = Cosmic.populationFromTimeHgValues\
            (timeHgValues,populationMax,stride,threshold)
        #now build up all of the intervals in seconds
        self.logger.info("findCosmics:  build up intervals:  nCosmicTime=%d"%len(pfthgv['cosmicTimeList']))
        i = interval()
        iCount = 0
        secondsPerTick = self.file.tickDuration
        for cosmicTime in pfthgv['cosmicTimeList']:
            #t0 = max(0,self.beginTime+(cosmicTime-50)/1.e6)
            #t1 = min(self.endTime,self.beginTime+(cosmicTime+50)/1.e6)
            #intTime = t1-t0            
            t0 = self.beginTime+cosmicTime*secondsPerTick
            dt = stride*secondsPerTick
            t1 = t0+dt
            left = max(self.beginTime, t0-nSigma*dt)
            right = min(self.endTime, t1+2*nSigma*dt)
            i = i | interval[left,right]
            self.logger.debug("findCosmics:  iCount=%d t0=%f t1=%f left=%f right=%f"%(iCount,t0,t1,left,right))
            iCount+=1

        tMasked = Cosmic.countMaskedBins(i)
        ppmMasked = 1000000*tMasked/(self.endTime-self.beginTime)

        retval = {}
        retval['timeHgValues'] = timeHgValues
        retval['populationHg'] = pfthgv['populationHg']
        retval['cosmicTimeList'] = pfthgv['cosmicTimeList']
        retval['binContents'] = pfthgv['binContents']
        retval['frameSum'] = frameSum
        retval['interval'] = i
        retval['ppmMasked'] = ppmMasked
        retval['pps'] = pps
        retval['ppsTime'] = ppsTime
        if writeCosmicMask:
            cfn = self.fn.cosmicMask()
            self.logger.info("findCosmics:  write masks to =%s"%cfn)
            ObsFile.writeCosmicIntervalToFile(i, self.file.ticksPerSec, 
                                              cfn,self.beginTime, self.endTime, 
                                              stride, threshold, nSigma, populationMax)
        self.logger.info("findCosmics:  end with ppm masked=%d"%ppmMasked)
        return retval
NumFrames = 31
#nFiles = 13 #j0926
#nFiles=25 #crab
nFiles = 1 #try with only 1 file
curves = np.zeros((nFiles,NumFrames),dtype=float)

for k in xrange(nFiles):
    #FileName = '/home/srmeeker/scratch/standards/crabNight2_fit_%s.npz'%(fileNum)
    FileName = '/home/srmeeker/scratch/standards/crabNight1_fit_%s.npz'%(fileNum)
    print FileName
    t = np.load(FileName)

    energyBinWidth = 0.1
    wvlStart = 3000
    wvlStop = 13000
    wvlBinEdges = ObsFile.makeWvlBins(energyBinWidth,wvlStart,wvlStop)
    nWvlBins = len(wvlBinEdges)-1
    binWidths = np.empty(nWvlBins)
    for i in xrange(nWvlBins):
        binWidths[i] = wvlBinEdges[i+1]-wvlBinEdges[i]
    #print binWidths
    params = t['params']
    wvls = t['wvls']
    amps = params[:,1]
    widths = params[:,4]
    xpos = params[:,2]
    ypos = params[:,3]

    #print len(wvls)
    #print len(binWidths)
sunsetDate = '20121208'
utcDate = '20121209'
# Specify which wavelength calibration file to use.
calTimestamp = '20121209-131132'
# Create wavelength and flat cal file names
wvlCalFilename = FileName(run=run,date=sunsetDate,tstamp=calTimestamp).calSoln()
flatCalFilename = FileName(run=run,date=sunsetDate,tstamp=calTimestamp).flatSoln()
# No twilights taken on December 8, using December 7 twilights to do flat cal instead.
flatCalFilename = '/Scratch/flatCalSolnFiles/20121207/flatsol_20121207.h5'

# Run standard ObsFile functions.
# Create ObsFile instance.
tic = time()
print 'Loading obs file and performing calibrations ...'
obsFn = FileName(run=run,date=sunsetDate,tstamp=obsTimestamp).obs()
ob = ObsFile(obsFn)
# Load roach time delay corrections.
ob.loadTimeAdjustmentFile(FileName(run=run).timeAdjustments())
# Search for time mask for given observation file.  If the time mask does not exist, create it.
index1 = obsFn.find('_')
hotPixFn = '/Scratch/timeMasks/timeMask' + obsFn[index1:]
if not os.path.exists(hotPixFn):
    hp.findHotPixels(obsFn,hotPixFn)
    print "Flux file pixel mask saved to %s"%(hotPixFn)        
# Load time mask, wavelength calibration, and flat calibration and set wavelenth cutoffs.
ob.loadHotPixCalFile(hotPixFn,switchOnMask=True)
ob.loadWvlCalFile(wvlCalFilename)
ob.loadFlatCalFile(flatCalFilename)
ob.setWvlCutoffs(3000,5000)
print 'Total load time: ' + str(time()-tic) + 's'
    sunsetDate = sunsetDates[iSeq]

    for position in ObjPosFile:
        if iSeq + 1 == position[0]:
            print "finding aperture and sky masks"
            guessX = position[1]
            guessY = position[2]
            apertureMask = circleAperture(guessX, guessY, radius1)
            bigMask = circleAperture(guessX, guessY, radius2)
            skyMask = bigMask - apertureMask
    #            print skyMask[0:20][5:25]

    for i, ts in enumerate(timestampList):
        print "loading", ts
        obsFn = FileName(run=run, date=sunsetDate, tstamp=ts).obs()
        ob = ObsFile(obsFn)
        ob.loadTimeAdjustmentFile(FileName(run=run).timeAdjustments())
        index1 = obsFn.find("_")
        hotPixFn = "/Scratch/timeMasks/timeMask" + obsFn[index1:]
        if not os.path.exists(hotPixFn):
            hp.findHotPixels(obsFn, hotPixFn)
            print "Flux file pixel mask saved to %s" % (hotPixFn)
        ob.loadHotPixCalFile(hotPixFn, switchOnMask=True)
        ob.loadWvlCalFile(wfn)
        ob.loadFlatCalFile(ffn)
        ob.setWvlCutoffs(wvlLowerCutoff, wvlUpperCutoff)

        bad_solution_mask = np.zeros((46, 44))
        bad_count = 0
        for y in range(46):
            for x in range(44):
 #cosmic = Cosmic(fn, endTime='exptime')
 cosmic = Cosmic(fn, beginTime=0)
 fc = cosmic.findCosmics(stride=stride, 
                         threshold=threshold, 
                         populationMax=populationMax,
                         nSigma=nSigma)
 if frameSum == 'none':
     frameSum = fc['frameSum']
 else:
     frameSum += fc['frameSum']
 outfile = open("cosmicTimeList-"+seq+".pkl", "wb")
 pickle.dump(fc['cosmicTimeList'],outfile)
 pickle.dump(fc['binContents'],outfile)
 outfile.close()
 cfn = "cosmicMax-%s.h5"%seq
 ObsFile.writeCosmicIntervalToFile(fc['interval'],1.0e6, cfn)
 populationHg = fc['populationHg']
 yPlot = populationHg[0].astype(np.float)
 ySum += yPlot
 norm = yPlot.sum()
 yPlot /= norm
 yPlot[yPlot==0] = 1e-3/norm
 xPlot = 0.5*(populationHg[1][1:]+populationHg[1][:-1])
 plt.clf()
 plt.plot(xPlot,yPlot,'-')
 plt.yscale('log')
 plt.ylim(ymin=0.5/norm)
 plt.xlim(1,populationMax)
 plt.xscale('log')
 #poisson = []
 #xValues = populationHg[1][1:]-0.5
def main():

    obsSequence1="""
    035332
    035834
    040336
    040838
    041341
    041843
    042346
    042848
    043351
    043853
    044355
    044857
    045359
    045902
    """

    obsSequence2="""
    050404
    050907
    051409
    051912
    052414
    052917
    053419
    053922
    """

    obsSequence3="""
    054926
    055428
    """


    pulseLabel = 1 #1 for interpulse, 2 for main pulse
    verbose = True
    run = 'PAL2012'
    path = '/Scratch/dataProcessing/crabData2/'
    nIdxToCheck = 81
    nBins = 250
    outFilePath = path+'indPulseProfiles_3sigma_P{}_KS.h5'.format(pulseLabel)
    wvlBinEdges = ObsFile.makeWvlBins(wvlStart=4000,wvlStop=11000)
    wvlBinWidths = np.diff(wvlBinEdges)
    nWvlBins = len(wvlBinWidths)
    peakIdx=167#the index of the phaseBin at the main pulse peak

    obsSequences = [obsSequence1,obsSequence2,obsSequence3]
    #obsSequences = [obsSequence1]
    wvlCals = ['063518','063518','063518']
    flatCals = ['20121211','20121211','20121211']
    fluxCalDates = ['20121206','20121206','20121206']
    fluxCals = ['20121207-072055','20121207-072055','20121207-072055']

    obsUtcDates = ['20121212','20121212','20121212']


    obsFileNames = []
    obsFileNameTimestamps = []
    wvlFileNames = []
    flatFileNames = []
    fluxFileNames = []
    timeMaskFileNames = []
    plFileNames = []
    skyFileNames = []

    for iSeq in range(len(obsSequences)):
        obsSequence = obsSequences[iSeq]
        obsSequence = obsSequence.strip().split()
        obsFileNameTimestamps.append(obsSequence)
        obsUtcDate = obsUtcDates[iSeq]
        sunsetDate = str(int(obsUtcDate)-1)
        obsSequence = [obsUtcDates[iSeq]+'-'+ts for ts in obsSequence]
        plFileNames.append([FileName(run=run,date=sunsetDate,tstamp=ts).crabList() for ts in obsSequence])
        skyFileNames.append([FileName(run=run,date=sunsetDate,tstamp=ts).crabSkyList() for ts in obsSequence])

    np.set_printoptions(precision=11,threshold=np.nan)


    labels=np.array('iBTOA   pulseNumber BTOA    Noise_Offset    Noise_RMS   Max  Mean Index    TestMax  TestMean TestIndex'.split())

    table = np.loadtxt(path+'giantPulseList_P{}_3sigma_indices.txt'.format(pulseLabel),skiprows=1,usecols=range(len(labels)))
    peakDetectionMask = table[:,np.argmax('Max'==labels)]!=0
    table = table[peakDetectionMask]#cut out test_P2 (false) detections

    dimMask = np.ones(len(table))
    #dimMask[13292:22401]=0
    dimMask = dimMask==1

    radioStrength = table[:,5]
    allGiantPulseNumbers = table[:,1]
    if pulseLabel == 2:
        radioStrengthCutoff = 0.175
    elif pulseLabel == 1:
        radioStrengthCutoff = 0.01

    radioCutoffMask = radioStrength >= radioStrengthCutoff
    strongMask = np.logical_and(radioCutoffMask,dimMask)
    table = table[strongMask]

    giantDict = dict()
    for iLabel,label in enumerate(labels):
        giantDict[label] = table[:,np.argmax(label==labels)]

    noiseStrengthLabel = 'TestMax'


    #count number of detections in the test range and the P2 peak range
    nNoiseDetections = np.sum(giantDict[noiseStrengthLabel]!=0)
    print nNoiseDetections,'noise detections'
    nPeakDetections = np.sum(giantDict['Max']!=0)
    print nPeakDetections,'peak detections'
    
    giantPulseNumbers = giantDict['pulseNumber']
    radioMax = giantDict['Max']
    radioMean = giantDict['Mean']
    radioDetectedIndices = giantDict['Index']

    nGRP = len(giantPulseNumbers)
    if verbose:
        print 'Number of GRPs',nGRP
    counts = np.zeros((nGRP,nIdxToCheck))
    skyCounts = np.zeros((nGRP,nIdxToCheck))

    idxOffsets = np.array(np.linspace(-(nIdxToCheck//2),nIdxToCheck//2,nIdxToCheck),dtype=np.int)
    nIdxOffsets = len(idxOffsets)

    histStart = 0.
    histEnd = 1.
    pulsarPeriod = 33e-3 #s, approximately
    indProfiles = np.zeros((nGRP,nIdxOffsets,nBins))
    skyIndProfiles = np.zeros((nGRP,nIdxOffsets,nBins))
    fullSpectra = np.zeros((nIdxOffsets,nWvlBins),dtype=np.double)
    skyFullSpectra = np.zeros((nIdxOffsets,nWvlBins),dtype=np.double)
    peakSpectra = np.zeros((nIdxOffsets,nWvlBins),dtype=np.double)
    skyPeakSpectra = np.zeros((nIdxOffsets,nWvlBins),dtype=np.double)
    plList = [PhotList(fn) for seq in plFileNames for fn in seq]
    skyList = [PhotList(fn) for seq in skyFileNames for fn in seq]
    plMins = []
    plMaxs = []
    for iPL,pl in enumerate(plList):
        minPulseNumber = pl.photTable.cols.pulseNumber[pl.photTable.colindexes['pulseNumber'][0]]
        maxPulseNumber = pl.photTable.cols.pulseNumber[pl.photTable.colindexes['pulseNumber'][-1]]
        print pl.fileName,minPulseNumber,maxPulseNumber
        plMins.append(minPulseNumber)
        plMaxs.append(maxPulseNumber)

    plMins = np.array(plMins)
    plMaxs = np.array(plMaxs)
        
    pulseNumberTable = np.array([gpn+idxOffsets for gpn in giantPulseNumbers])
    giantPulseNumberMask = np.zeros(np.shape(pulseNumberTable))
    validWvlCutoff = 11000 #angstroms
    if verbose:
        print 'filling giant pulse number mask'
    for iGiantPN,giantPN in enumerate(allGiantPulseNumbers):
        if verbose and iGiantPN % 500 == 0:
            print 'mask',iGiantPN,'of',len(allGiantPulseNumbers)
        giantMatches = (pulseNumberTable == giantPN)
        giantPulseNumberMask = np.logical_or(giantPulseNumberMask,giantMatches)

    outFile = tables.openFile(outFilePath,mode='w')

    nLivePixels = []
    for iPL,pl in enumerate(plList):
        pixels = np.unique(pl.photTable.cols.xyPix[:])
        print pixels
        nPixels = len(pixels)
        nLivePixels.append(nPixels)

    nLivePixels = np.array(nLivePixels)
    nSkyLivePixels = []
    for iPL,pl in enumerate(skyList):
        pixels = np.unique(pl.photTable.cols.xyPix[:])
        print pixels
        nPixels = len(pixels)
        nSkyLivePixels.append(nPixels)
    nSkyLivePixels = np.array(nSkyLivePixels)

    pulseNumberList = np.unique(pulseNumberTable.ravel())       
    nPulseNumbers = len(pulseNumberList)                        
    pulseNumberListMask = np.ones(nPulseNumbers)                

    floatAtom = tables.Float64Atom()
    uint8Atom = tables.UInt8Atom()

    nExpectedRows=1e7
    #phaseArrays = [outFile.createEArray(outFile.root,'phases{:03d}'.format(iIdxOffset),floatAtom,(0,),title='phases at pulse number {} w.r.t. GRP'.format(idxOffset),expectedrows=nExpectedRows) for iIdxOffset,idxOffset in enumerate(idxOffsets)]
    #skyPhaseArrays = [outFile.createEArray(outFile.root,'skyPhases{:03d}'.format(iIdxOffset),floatAtom,(0,),title='phases at pulse number {} w.r.t. GRP in sky'.format(idxOffset),expectedrows=nExpectedRows) for iIdxOffset,idxOffset in enumerate(idxOffsets)]
    #wavelengthArrays = [outFile.createEArray(outFile.root,'wavelengths{:03d}'.format(iIdxOffset),floatAtom,(0,),title='wavelengths at pulse number {} w.r.t. GRP'.format(idxOffset),expectedrows=nExpectedRows) for iIdxOffset,idxOffset in enumerate(idxOffsets)]
    #skyWavelengthArrays = [outFile.createEArray(outFile.root,'skyWavelengths{:03d}'.format(iIdxOffset),floatAtom,(0,),title='wavelengths at pulse number {} w.r.t. GRP in sky'.format(idxOffset),expectedrows=nExpectedRows) for iIdxOffset,idxOffset in enumerate(idxOffsets)]
    grpPeakWavelengthArrays = outFile.createVLArray(outFile.root,'grpWavelengths',floatAtom,'photon wavelength list for GRPs at main peak',tables.Filters(complevel=1))
    nongrpPeakWavelengthArrays = outFile.createVLArray(outFile.root,'nongrpWavelengths',floatAtom,'photon wavelength list for nonGRPs at main peak',tables.Filters(complevel=1))

    for iGiantPN,giantPN in enumerate(giantPulseNumbers):
        if verbose and iGiantPN % 100 == 0:
            print iGiantPN
        
        iPL = np.searchsorted(plMaxs,giantPN)

        if iPL >= len(plMins):
            if verbose:
                print iGiantPN,'GRP not found in optical'
            continue
        if plMins[iPL] > giantPN:
            if verbose:
                print iGiantPN,'GRP not found in optical'

            continue

        if plMins[iPL] >= giantPN+idxOffsets[0] or plMaxs[iPL] <= giantPN+idxOffsets[-1]:
            if verbose:
                print iGiantPN,'optical pulses surrounding GRP not found'
            continue

        pl = plList[iPL]
        skyPL = skyList[iPL]
        #grab all photons in the pulseNumber range covered by all idxOffsets for this GRP
        pulseSelectionIndices = pl.photTable.getWhereList('({} <= pulseNumber) & (pulseNumber <= {})'.format(giantPN+idxOffsets[0],giantPN+idxOffsets[-1]))
        skyPulseSelectionIndices = skyPL.photTable.getWhereList('({} <= pulseNumber) & (pulseNumber <= {})'.format(giantPN+idxOffsets[0],giantPN+idxOffsets[-1]))
        photonsInPulseSelection = pl.photTable.readCoordinates(pulseSelectionIndices)
        skyPhotonsInPulseSelection = skyPL.photTable.readCoordinates(skyPulseSelectionIndices)

        nPulsesInSelection = len(np.unique(photonsInPulseSelection['pulseNumber']))
        nSkyPulsesInSelection = len(np.unique(skyPhotonsInPulseSelection['pulseNumber']))
        if  nPulsesInSelection < nIdxOffsets or nSkyPulsesInSelection < nIdxOffsets:
            if verbose:
                print 'only ',nPulsesInSelection,' pulses for ',iGiantPN,giantPN
            continue
        startIdx = 0
        
        nPixels = 1.*nLivePixels[iPL]
        nSkyPixels = 1.*nSkyLivePixels[iPL]
        nongrpPeakWvls = np.array([])
        for iIdxOffset,idxOffset in enumerate(idxOffsets):
            pulseNumber = giantPN+idxOffset
            if giantPulseNumberMask[iGiantPN,iIdxOffset] == False or idxOffset==0:
                photons = photonsInPulseSelection[photonsInPulseSelection['pulseNumber']==pulseNumber]
                skyPhotons = skyPhotonsInPulseSelection[skyPhotonsInPulseSelection['pulseNumber']==pulseNumber]
                phases = photons['phase']
                wavelengths = photons['wavelength']
                skyPhases = skyPhotons['phase']
                skyWavelengths = skyPhotons['wavelength']
                count = 1.*len(photons)/nPixels
                skyCount = 1.*len(skyPhotons)/nSkyPixels
                counts[iGiantPN,iIdxOffset] = count
                skyCounts[iGiantPN,iIdxOffset] = skyCount

                profile,phaseBinEdges = np.histogram(phases,bins=nBins,range=(histStart,histEnd))
                skyProfile,phaseBinEdges = np.histogram(skyPhases,bins=nBins,range=(histStart,histEnd))
                
                indProfiles[iGiantPN,iIdxOffset] = profile/nPixels
                skyIndProfiles[iGiantPN,iIdxOffset] = skyProfile/nSkyPixels

                spectrum,_ = np.histogram(wavelengths,bins=wvlBinEdges)
                spectrum = 1.0*spectrum/(nPixels*wvlBinWidths)#convert to counts per pixel per angstrom
                fullSpectra[iIdxOffset] += spectrum

                skySpectrum,_ = np.histogram(skyWavelengths,bins=wvlBinEdges)
                skySpectrum = 1.0*skySpectrum/(nSkyPixels*wvlBinWidths)#convert to counts per pixel per angstrom
                skyFullSpectra[iIdxOffset] += skySpectrum

                phasesBinned = np.digitize(phases,phaseBinEdges)-1
                peakPhaseMask = np.logical_or(phasesBinned==(peakIdx-1),phasesBinned==peakIdx)
                peakPhaseMask = np.logical_or(peakPhaseMask,phasesBinned==(peakIdx+1))
                peakWvls = wavelengths[peakPhaseMask]
                if idxOffset == 0:
                    grpPeakWavelengthArrays.append(peakWvls)
                else:
                    nongrpPeakWvls = np.append(nongrpPeakWvls,peakWvls)

                peakSpectrum,_ = np.histogram(peakWvls,bins=wvlBinEdges)
                peakSpectrum = 1.0*peakSpectrum/(nPixels*wvlBinWidths)#convert to counts per pixel per angstrom
                peakSpectra[iIdxOffset] += peakSpectrum

                skyPhasesBinned = np.digitize(skyPhases,phaseBinEdges)-1
                skyPeakPhaseMask = np.logical_or(skyPhasesBinned==(peakIdx-1),skyPhasesBinned==peakIdx)
                skyPeakPhaseMask = np.logical_or(skyPeakPhaseMask,skyPhasesBinned==(peakIdx+1))
                skyPeakWvls = skyWavelengths[skyPeakPhaseMask]
                skyPeakSpectrum,_ = np.histogram(skyPeakWvls,bins=wvlBinEdges)
                skyPeakSpectrum = 1.0*skyPeakSpectrum/(nSkyPixels*wvlBinWidths)#convert to counts per pixel per angstrom
                skyPeakSpectra[iIdxOffset] += skyPeakSpectrum
#                phaseArrays[iIdxOffset].append(phases)
#                skyPhaseArrays[iIdxOffset].append(skyPhases)
#                wavelengthArrays[iIdxOffset].append(photons['wavelength'])
#                skyWavelengthArrays[iIdxOffset].append(skyPhotons['wavelength'])
        nongrpPeakWavelengthArrays.append(nongrpPeakWvls)


    if verbose:
        print 'done searching'
    outFile.createArray(outFile.root,'counts',counts)
    outFile.createArray(outFile.root,'skyCounts',skyCounts)
    outFile.createArray(outFile.root,'idxOffsets',idxOffsets)
    outFile.createArray(outFile.root,'radioMax',radioMax)
    outFile.createArray(outFile.root,'radioMean',radioMean)
    outFile.createArray(outFile.root,'indProfiles',indProfiles)
    outFile.createArray(outFile.root,'skyIndProfiles',skyIndProfiles)
    outFile.createArray(outFile.root,'phaseBinEdges',phaseBinEdges)
    outFile.createArray(outFile.root,'giantPulseNumbers',giantPulseNumbers)
    outFile.createArray(outFile.root,'giantPulseNumberMask',giantPulseNumberMask)
    outFile.createArray(outFile.root,'pulseNumberTable',pulseNumberTable)
    outFile.createArray(outFile.root,'radioIndices',radioDetectedIndices)
    outFile.createArray(outFile.root,'nPixels',nLivePixels)
    outFile.createArray(outFile.root,'nSkyPixels',nSkyLivePixels)
    outFile.createArray(outFile.root,'fullSpectra',fullSpectra)
    outFile.createArray(outFile.root,'skyFullSpectra',skyFullSpectra)
    outFile.createArray(outFile.root,'peakSpectra',peakSpectra)
    outFile.createArray(outFile.root,'skyPeakSpectra',skyPeakSpectra)
    outFile.createArray(outFile.root,'wvlBinEdges',wvlBinEdges)

    outFile.flush()
    outFile.close()
    if verbose:
        print 'done saving'
    def __init__(self,paramFile):
        """
        opens flat file,sets wavelength binnning parameters, and calculates flat factors for the file
        """
        self.params = readDict()
        self.params.read_from_file(paramFile)

        run = self.params['run']
        sunsetDate = self.params['sunsetDate']
        flatTstamp = self.params['flatTstamp']
        wvlSunsetDate = self.params['wvlSunsetDate']
        wvlTimestamp = self.params['wvlTimestamp']
        obsSequence = self.params['obsSequence']
        needTimeAdjust = self.params['needTimeAdjust']
        self.deadtime = self.params['deadtime'] #from firmware pulse detection
        self.intTime = self.params['intTime']
        self.timeSpacingCut = self.params['timeSpacingCut']
        self.nSigmaClip = self.params['nSigmaClip']
        self.nNearest = self.params['nNearest']

        obsFNs = [FileName(run=run,date=sunsetDate,tstamp=obsTstamp) for obsTstamp in obsSequence]
        self.obsFileNames = [fn.obs() for fn in obsFNs]
        self.obsList = [ObsFile(obsFileName) for obsFileName in self.obsFileNames]
        timeMaskFileNames = [fn.timeMask() for fn in obsFNs]
        timeAdjustFileName = FileName(run=run).timeAdjustments()

        print len(self.obsFileNames), 'flat files to co-add'
        self.flatCalFileName = FileName(run=run,date=sunsetDate,tstamp=flatTstamp).illumSoln()
        if wvlSunsetDate != '':
            wvlCalFileName = FileName(run=run,date=wvlSunsetDate,tstamp=wvlTimestamp).calSoln()
        for iObs,obs in enumerate(self.obsList):
            if wvlSunsetDate != '':
                obs.loadWvlCalFile(wvlCalFileName)
            else:
                obs.loadBestWvlCalFile()
            if needTimeAdjust:
                obs.loadTimeAdjustmentFile(timeAdjustFileName)
            timeMaskFileName = timeMaskFileNames[iObs]
            print timeMaskFileName
            #Temporary step, remove old hotpix file
            #if os.path.exists(timeMaskFileName):
            #    os.remove(timeMaskFileName)
            if not os.path.exists(timeMaskFileName):
                print 'Running hotpix for ',obs
                hp.findHotPixels(self.obsFileNames[iObs],timeMaskFileName,fwhm=np.inf,useLocalStdDev=True)
                print "Flux file pixel mask saved to %s"%(timeMaskFileName)
            obs.loadHotPixCalFile(timeMaskFileName)
        self.wvlFlags = self.obsList[0].wvlFlagTable

        self.nRow = self.obsList[0].nRow
        self.nCol = self.obsList[0].nCol
        print 'files opened'
        #self.wvlBinWidth = params['wvlBinWidth'] #angstroms
        self.energyBinWidth = self.params['energyBinWidth'] #eV
        self.wvlStart = self.params['wvlStart'] #angstroms
        self.wvlStop = self.params['wvlStop'] #angstroms
        self.wvlBinEdges = ObsFile.makeWvlBins(self.energyBinWidth,self.wvlStart,self.wvlStop)
        self.intTime = self.params['intTime']
        self.countRateCutoff = self.params['countRateCutoff']
        self.fractionOfChunksToTrim = self.params['fractionOfChunksToTrim']
        #wvlBinEdges includes both lower and upper limits, so number of bins is 1 less than number of edges
        self.nWvlBins = len(self.wvlBinEdges)-1
    filt = sys.argv[1]
else:
    filt='V'


#  All examples shown here will use data from the W Uma binary, 1SWASP J0002, found here:
fileName = '/ScienceData/PAL2014/20140923/obs_20140924-080031.h5'


#  We'll select a pixel in the middle of the target for our examples
col = 18
row = 16


#  first, load the data into an ObsFile object
obs = ObsFile(fileName)
print "Loaded obsFile", fileName


#  if we tried to simply load a filter now, it would fail as we haven't defined our spectrum's binning.
#  Uncomment the following to see the error you would get.
'''
obs.loadFilter()
'''


#  Wavelength bins can be provided during the filter loading, but the easiest way is to
#  just load the flat calibration first, as that sets everything to the binning used for
#  flat cal. This file already has all its calibrations associated
#  with it in /Scratch/calLookup/lookup.h5. If a file doesn't have it's data written into 
#  this table the following will not work and calibrations must be loaded manually.
class FluxCal:
    def __init__(self,paramFile,plots=False,verbose=False):
        """
        Opens flux file, prepares standard spectrum, and calculates flux factors for the file.
        Method is provided in param file. If 'relative' is selected, an obs file with standard star defocused over
        the entire array is expected, with accompanying sky file to do sky subtraction.
        If any other method is provided, 'absolute' will be done by default, wherein a point source is assumed
        to be present. The obs file is then broken into spectral frames with photometry (psf or aper) performed 
        on each frame to generate the ARCONS observed spectrum.
        """
        self.verbose=verbose
        self.plots = plots

        self.params = readDict()
        self.params.read_from_file(paramFile)
        
        run = self.params['run']
        sunsetDate = self.params['fluxSunsetLocalDate']
        self.fluxTstamp = self.params['fluxTimestamp']
        skyTstamp = self.params['skyTimestamp']
        wvlSunsetDate = self.params['wvlCalSunsetLocalDate']
        wvlTimestamp = self.params['wvlCalTimestamp']
        flatCalFileName = self.params['flatCalFileName']
        needTimeAdjust = self.params['needTimeAdjust']
        self.deadtime = float(self.params['deadtime']) #from firmware pulse detection
        self.timeSpacingCut = self.params['timeSpacingCut']
        bLoadBeammap = self.params.get('bLoadBeammap',False)
        self.method = self.params['method']
        self.objectName = self.params['object']
        self.r = float(self.params['energyResolution'])
        self.photometry = self.params['photometry']
        self.centroidRow = self.params['centroidRow']
        self.centroidCol = self.params['centroidCol']
        self.aperture = self.params['apertureRad']
        self.annulusInner = self.params['annulusInner']
        self.annulusOuter = self.params['annulusOuter']
        self.collectingArea = self.params['collectingArea']
        self.startTime = self.params['startTime']
        self.intTime = self.params['integrationTime']

        fluxFN = FileName(run=run,date=sunsetDate,tstamp=self.fluxTstamp)
        self.fluxFileName = fluxFN.obs()
        self.fluxFile = ObsFile(self.fluxFileName)

        if self.plots:
            self.plotSavePath = os.environ['MKID_PROC_PATH']+os.sep+'fluxCalSolnFiles'+os.sep+run+os.sep+sunsetDate+os.sep+'plots'+os.sep
            if not os.path.exists(self.plotSavePath): os.mkdir(self.plotSavePath)
            if self.verbose: print "Created directory %s"%self.plotSavePath

        obsFNs = [fluxFN]
        self.obsList = [self.fluxFile]

        if self.startTime in ['',None]: self.startTime=0
        if self.intTime in ['',None]: self.intTime=-1

        if self.method=="relative":
            try:
                print "performing Relative Flux Calibration"
                skyFN = FileName(run=run,date=sunsetDate,tstamp=skyTstamp)
                self.skyFileName = skyFN.obs()
                self.skyFile = ObsFile(self.skyFileName)
                obsFNs.append(skyFN)
                self.obsList.append(self.skyFile)
            except:
                print "For relative flux calibration a sky file must be provided in param file"
                self.__del__()
        else:
            self.method='absolute'
            print "performing Absolute Flux Calibration"

        if self.photometry not in ['aperture','PSF']: self.photometry='PSF' #default to PSF fitting if no valid photometry selected

        timeMaskFileNames = [fn.timeMask() for fn in obsFNs]
        timeAdjustFileName = FileName(run=run).timeAdjustments()

        #make filename for output fluxCalSoln file
        self.fluxCalFileName = FileName(run=run,date=sunsetDate,tstamp=self.fluxTstamp).fluxSoln()
        print "Creating flux cal: %s"%self.fluxCalFileName

        if wvlSunsetDate != '':
            wvlCalFileName = FileName(run=run,date=wvlSunsetDate,tstamp=wvlTimestamp).calSoln()
        if flatCalFileName =='':
            flatCalFileName=FileName(obsFile=self.fluxFile).flatSoln()

        #load cal files for flux file and, if necessary, sky file
        for iObs,obs in enumerate(self.obsList):
            if bLoadBeammap:
                print 'loading beammap',os.environ['MKID_BEAMMAP_PATH']
                obs.loadBeammapFile(os.environ['MKID_BEAMMAP_PATH'])
            if wvlSunsetDate != '':
                obs.loadWvlCalFile(wvlCalFileName)
            else:
                obs.loadBestWvlCalFile()

            obs.loadFlatCalFile(flatCalFileName)
            obs.setWvlCutoffs(-1,-1)

            if needTimeAdjust:
                obs.loadTimeAdjustmentFile(timeAdjustFileName)
            timeMaskFileName = timeMaskFileNames[iObs]
            print timeMaskFileName

            if not os.path.exists(timeMaskFileName):
                print 'Running hotpix for ',obs
                hp.findHotPixels(obsFile=obs,outputFileName=timeMaskFileName,fwhm=np.inf,useLocalStdDev=True)
                print "Flux cal/sky file pixel mask saved to %s"%(timeMaskFileName)
            obs.loadHotPixCalFile(timeMaskFileName)
            if self.verbose: print "Loaded hot pixel file %s"%timeMaskFileName

        #get flat cal binning information since flux cal will need to match it
        self.wvlBinEdges = self.fluxFile.flatCalFile.root.flatcal.wavelengthBins.read()
        self.nWvlBins = self.fluxFile.flatWeights.shape[2]
        self.binWidths = np.empty((self.nWvlBins),dtype=float)
        self.binCenters = np.empty((self.nWvlBins),dtype=float)
        for i in xrange(self.nWvlBins):
            self.binWidths[i] = self.wvlBinEdges[i+1]-self.wvlBinEdges[i]
            self.binCenters[i] = (self.wvlBinEdges[i]+(self.binWidths[i]/2.0))

        if self.method=='relative':
            print "Extracting ARCONS flux and sky spectra"
            self.loadRelativeSpectrum()
            print "Flux Spectrum loaded"
            self.loadSkySpectrum()
            print "Sky Spectrum loaded"
        elif self.method=='absolute':
            print "Extracting ARCONS point source spectrum"
            self.loadAbsoluteSpectrum()

        print "Loading standard spectrum"
        try:
            self.loadStdSpectrum(self.objectName)
        except KeyError:
            print "Invalid spectrum object name"
            self.__del__()
            sys.exit()

        print "Generating sensitivity curve"
        self.calculateFactors()
        print "Sensitivity Curve calculated"
        print "Writing fluxCal to file %s"%self.fluxCalFileName
        self.writeFactors(self.fluxCalFileName)
        
        if self.plots: self.makePlots()

        print "Done"

    def __del__(self):
        try:
            self.fluxFile.close()
            self.calFile.close()
        except AttributeError:#fluxFile was never defined
            pass

    def getDeadTimeCorrection(self, obs): #WRONG RIGHT NOW. NEEDS TO HAVE RAW COUNTS SUMMED, NOT CUBE WHICH EXCLUDES NOISE TAIL
        if self.verbose: print "Making raw cube to get dead time correction"
        cubeDict = obs.getSpectralCube(firstSec=self.startTime, integrationTime=self.intTime, weighted=False)

        cube= np.array(cubeDict['cube'], dtype=np.double)
        wvlBinEdges= cubeDict['wvlBinEdges']
        effIntTime= cubeDict['effIntTime']
        if self.verbose: print "median effective integration time = ", np.median(effIntTime)

        nWvlBins=len(wvlBinEdges)-1
        if self.verbose: print "cube shape ", np.shape(cube)
        if self.verbose: print "effIntTime shape ", np.shape(effIntTime)

        #add third dimension to effIntTime for  broadcasting
        effIntTime = np.reshape(effIntTime,np.shape(effIntTime)+(1,))
        #put cube into counts/s in each pixel
        cube /= effIntTime

        #CALCULATE DEADTIME CORRECTION
        #NEED TOTAL COUNTS PER SECOND FOR EACH PIXEL TO DO PROPERLY
        #ASSUMES SAME CORRECTION FACTOR APPLIED FOR EACH WAVELENGTH, MEANING NO WL DEPENDANCE ON DEAD TIME EFFECT
        DTCorr = np.zeros((np.shape(cube)[0],np.shape(cube)[1]),dtype=float)
        for f in range(0,np.shape(cube)[2]):
            #if self.verbose: print cube[:,:,f]
            #if self.verbose: print '-----------------------'
            DTCorr += cube[:,:,f]
            #if self.verbose: print DTCorr
            #if self.verbose: print '\n=====================\n'
        #Correct for firmware dead time (100us in 2012 ARCONS firmware)
        DTCorrNew=DTCorr/(1-DTCorr*self.deadtime)
        CorrFactors = DTCorrNew/DTCorr #This is what the frames need to be multiplied by to get their true values
        if self.verbose: print "Dead time correction factors: ", CorrFactors
        #add third dimension to CorrFactors for broadcasting
        CorrFactors = np.reshape(CorrFactors,np.shape(CorrFactors)+(1,))
        return CorrFactors

    def loadAbsoluteSpectrum(self):
        '''
        extract the ARCONS measured spectrum of the spectrophotometric standard by breaking data into spectral cube
        and performing photometry (aper or psf) on each spectral frame
        '''
        if self.verbose:print "Making spectral cube"
        cubeDict = self.fluxFile.getSpectralCube(firstSec=self.startTime, integrationTime=self.intTime, weighted=True, fluxWeighted=False)
        cube= np.array(cubeDict['cube'], dtype=np.double)
        effIntTime= cubeDict['effIntTime']
        if self.verbose: print "median effective integration time in flux file cube = ", np.median(effIntTime)
        if self.verbose: print "cube shape ", np.shape(cube)
        if self.verbose: print "effIntTime shape ", np.shape(effIntTime)

        #add third dimension to effIntTime for broadcasting
        effIntTime = np.reshape(effIntTime,np.shape(effIntTime)+(1,))
        #put cube into counts/s in each pixel
        cube /= effIntTime
        #get dead time correction factors
        DTCorr = self.getDeadTimeCorrection(self.fluxFile)
        cube*=DTCorr #cube now in units of counts/s and corrected for dead time

        
        if self.plots and not 'figureHeader' in sys.modules:
            if self.verbose: print "Saving spectral frames as movie..."
            movieCube = np.zeros((self.nWvlBins,np.shape(cube)[0],np.shape(cube)[1]),dtype=float)
            for i in xrange(self.nWvlBins):
                movieCube[i,:,:] = cube[:,:,i]
            makeMovie(movieCube,frameTitles=self.binCenters,cbar=True,outName=self.plotSavePath+'FluxCal_Cube_%s.gif'%(self.objectName), normMin=0, normMax=50)
            if self.verbose: print "Movie saved in %s"%self.plotSavePath
        

        LCplot=False #light curve pop-ups not compatible with FLuxCal plotting 2/18/15
        #if self.photometry=='PSF': LCplot = False
        LC = LightCurve.LightCurve(verbose=self.verbose, showPlot=LCplot)

        self.fluxSpectrum=np.empty((self.nWvlBins),dtype=float)
        self.skySpectrum=np.zeros((self.nWvlBins),dtype=float)

        for i in xrange(self.nWvlBins):
            frame = cube[:,:,i]
            if self.verbose: print "%s photometry on frame %i of cube, central wvl = %f Angstroms"%(self.photometry,i,self.binCenters[i])
            if self.photometry == 'aperture':
                fDict = LC.performPhotometry(self.photometry,frame,[[self.centroidCol,self.centroidRow]],expTime=None,aper_radius = self.aperture, annulus_inner = self.annulusInner, annulus_outer = self.annulusOuter, interpolation="linear")
                self.fluxSpectrum[i] = fDict['flux']
                self.skySpectrum[i] = fDict['skyFlux']
                print "Sky estimate = ", fDict['skyFlux']
            else:
                fDict = LC.performPhotometry(self.photometry,frame,[[self.centroidCol,self.centroidRow]],expTime=None,aper_radius = self.aperture)
                self.fluxSpectrum[i] = fDict['flux']
        
        self.fluxSpectrum=self.fluxSpectrum/self.binWidths/self.collectingArea #spectrum now in counts/s/Angs/cm^2
        self.skySpectrum=self.skySpectrum/self.binWidths/self.collectingArea

        return self.fluxSpectrum, self.skySpectrum

    def loadRelativeSpectrum(self):
        self.fluxSpectra = [[[] for i in xrange(self.nCol)] for j in xrange(self.nRow)]
        self.fluxEffTime = [[[] for i in xrange(self.nCol)] for j in xrange(self.nRow)]
        for iRow in xrange(self.nRow):
            for iCol in xrange(self.nCol):
                count = self.fluxFile.getPixelCount(iRow,iCol)
                fluxDict = self.fluxFile.getPixelSpectrum(iRow,iCol,weighted=True,firstSec=0,integrationTime=-1)
                self.fluxSpectra[iRow][iCol],self.fluxEffTime[iRow][iCol] = fluxDict['spectrum'],fluxDict['effIntTime']
        self.fluxSpectra = np.array(self.fluxSpectra)
        self.fluxEffTime = np.array(self.fluxEffTime)
        DTCorr = self.getDeadTimeCorrection(self.fluxFile)
        #print "Bin widths = ",self.binWidths
        self.fluxSpectra = self.fluxSpectra/self.binWidths/self.fluxEffTime*DTCorr
        self.fluxSpectrum = self.calculateMedian(self.fluxSpectra) #find median of subtracted spectra across whole array
        return self.fluxSpectrum

    def loadSkySpectrum(self):
        self.skySpectra = [[[] for i in xrange(self.nCol)] for j in xrange(self.nRow)]
        self.skyEffTime = [[[] for i in xrange(self.nCol)] for j in xrange(self.nRow)] 
        for iRow in xrange(self.nRow):
            for iCol in xrange(self.nCol):
                count = self.skyFile.getPixelCount(iRow,iCol)
                skyDict = self.skyFile.getPixelSpectrum(iRow,iCol,weighted=True,firstSec=0,integrationTime=-1)
                self.skySpectra[iRow][iCol],self.skyEffTime[iRow][iCol] = skyDict['spectrum'],skyDict['effIntTime']
        self.skySpectra = np.array(self.skySpectra)
        self.skyEffTime = np.array(self.skyEffTime)
        DTCorr = self.getDeadTimeCorrection(self.skyFile)
        self.skySpectra = self.skySpectra/self.binWidths/self.skyEffTime*DTCorr
        self.skySpectrum = self.calculateMedian(self.skySpectra) #find median of subtracted spectra across whole array
        return self.skySpectrum

    def loadStdSpectrum(self, objectName="G158-100"):
        #import the known spectrum of the calibrator and rebin to the histogram parameters given
        #must be imported into array with dtype float so division later does not have error
        std = MKIDStd.MKIDStd()
        a = std.load(objectName)
        a = std.countsToErgs(a) #convert std spectrum to ergs/s/Angs/cm^2 for BB fitting and cleaning
        self.stdWvls = np.array(a[:,0])
        self.stdFlux = np.array(a[:,1]) #std object spectrum in ergs/s/Angs/cm^2

        if self.plots:
            #create figure for plotting standard spectrum modifications      
            self.stdFig = plt.figure()
            self.stdAx = self.stdFig.add_subplot(111)
            plt.xlim(4000,11000)
            plt.plot(self.stdWvls,self.stdFlux*1E15,linewidth=1,color='grey',alpha=0.75)
            
        convX_rev,convY_rev = self.cleanSpectrum(self.stdWvls,self.stdFlux)
        convX = convX_rev[::-1] #convolved spectrum comes back sorted backwards, from long wvls to low which screws up rebinning
        convY = convY_rev[::-1]
        #rebin cleaned spectrum to flat cal's wvlBinEdges
        newa = rebin(convX,convY,self.wvlBinEdges)
        rebinnedWvl = np.array(newa[:,0])
        rebinnedFlux = np.array(newa[:,1])
        if self.plots:
            #plot final resampled spectrum
            plt.plot(convX,convY*1E15,color='blue')
            plt.step(rebinnedWvl,rebinnedFlux*1E15,color = 'black',where='mid')
            plt.legend(['%s Spectrum'%self.objectName,'Blackbody Fit','Gaussian Convolved Spectrum','Rebinned Spectrum'],'upper right', numpoints=1)
            plt.xlabel(ur"Wavelength (\r{A})")
            plt.ylabel(ur"Flux (10$^{-15}$ ergs s$^{-1}$ cm$^{-2}$ \r{A}$^{-1}$)")
            plt.ylim(1,5)
            plt.savefig(self.plotSavePath+'FluxCal_StdSpectrum_%s.eps'%self.objectName,format='eps')
        
        #convert standard spectrum back into counts/s/angstrom/cm^2
        newa = std.ergsToCounts(newa)
        self.binnedSpectrum = np.array(newa[:,1])


    def cleanSpectrum(self,x,y):
        ##=============== BB Fit to extend spectrum beyond 11000 Angstroms ==================
        fraction = 1.0/3.0
        nirX = np.arange(int(x[(1.0-fraction)*len(x)]),20000)
        T, nirY = fitBlackbody(x,y,fraction=fraction,newWvls=nirX,tempGuess=5600)
        
        if self.plots: plt.plot(nirX,nirY*1E15,linestyle='--',linewidth=2, color="black",alpha=0.5)

        extendedWvl = np.concatenate((x,nirX[nirX>max(x)]))
        extendedFlux = np.concatenate((y,nirY[nirX>max(x)]))
        ##======= Gaussian convolution to smooth std spectrum to MKIDs median resolution ========
        newX, newY = gaussianConvolution(extendedWvl,extendedFlux,xEnMin=0.005,xEnMax=6.0,xdE=0.001,fluxUnits = "lambda",r=self.r,plots=False)
        return newX, newY


    def calculateFactors(self):
        """
        Calculate the sensitivity spectrum: the weighting factors that correct the flat calibrated spectra to the real spectra
        
        For relative calibration:
        First subtract sky spectrum from ARCONS observed spectrum. Then take median of this spectrum as it should be identical 
        across the array, assuming the flat cal has done its job. Then divide this into the known spectrum of the object.
        
        For absolute calibration:
        self.fluxSpectra already has sky subtraction included. Simply divide this spectrum into the known standard spectrum.
        """
        self.subtractedSpectrum = self.fluxSpectrum - self.skySpectrum
        self.subtractedSpectrum = np.array(self.subtractedSpectrum,dtype=float) #cast as floats so division does not fail later

        if self.method=='relative':      
            normWvl = 5500 #Angstroms. Choose an arbitrary wvl to normalize the relative correction at
            ind = np.where(self.wvlBinEdges >= normWvl)[0][0]-1
            self.subtractedSpectrum = self.subtractedSpectrum/(self.subtractedSpectrum[ind]) #normalize
            self.binnedSpectrum = self.binnedSpectrum/(self.binnedSpectrum[ind]) #normalize treated Std spectrum while we are at it

        #Calculate FluxCal factors
        self.fluxFactors = self.binnedSpectrum/self.subtractedSpectrum

        #self.fluxFlags = np.zeros(np.shape(self.fluxFactors),dtype='int')
        self.fluxFlags = np.empty(np.shape(self.fluxFactors),dtype='int')
        self.fluxFlags.fill(pipelineFlags.fluxCal['good'])   #Initialise flag array filled with 'good' flags. JvE 5/1/2013.
        #set factors that will cause trouble to 1
        #self.fluxFlags[self.fluxFactors == np.inf] = 1
        self.fluxFlags[self.fluxFactors == np.inf] = pipelineFlags.fluxCal['infWeight']   #Modified to use flag dictionary - JvE 5/1/2013
        self.fluxFactors[self.fluxFactors == np.inf]=1.0
        self.fluxFlags[np.isnan(self.fluxFactors)] = pipelineFlags.fluxCal['nanWeight']   #Modified to use flag dictionary - JvE 5/1/2013
        self.fluxFactors[np.isnan(self.fluxFactors)]=1.0        
        self.fluxFlags[self.fluxFactors <= 0]=pipelineFlags.fluxCal['LEzeroWeight']   #Modified to use flag dictionary - JvE 5/1/2013
        self.fluxFactors[self.fluxFactors <= 0]=1.0


    def calculateMedian(self, spectra):
        spectra2d = np.reshape(spectra,[self.nRow*self.nCol,self.nWvlBins])
        wvlMedian = np.empty(self.nWvlBins,dtype=float)
        for iWvl in xrange(self.nWvlBins):
            spectrum = spectra2d[:,iWvl]
            goodSpectrum = spectrum[spectrum != 0]#dead pixels need to be taken out before calculating medians
            wvlMedian[iWvl] = np.median(goodSpectrum)
        return wvlMedian


    def makePlots(self):
        """
        Output all debugging plots of ARCONS sky and object spectra, known calibrator spectrum, and sensitivity curve
        """
        scratchDir = os.getenv('MKID_PROC_PATH')
        fluxDir = self.plotSavePath
        fluxCalBase = 'FluxCal_%s'%self.objectName       
        plotFileName = fluxCalBase+".pdf"
        fullFluxPlotFileName = os.path.join(fluxDir,plotFileName)
        
        #uncomment to make some plots for the paper. Proper formatting Will also require figureheader to be imported and for movie making to be turned off
        self.paperFig = plt.figure()
        self.paperAx = self.paperFig.add_subplot(111)
        plt.xlim(4000,11000)
        plt.plot(self.binCenters,self.fluxFactors,linewidth=3,color='black')
        plt.xlabel(ur"Wavelength (\r{A})")
        plt.ylabel(ur"Spectral Calibration Curve")
        plt.ylim(0,150)
        plt.savefig(self.plotSavePath+'FluxCal_Sensitivity_%s.eps'%self.objectName,format='eps')

        #save throughput as a .npz file that other code uses when making paper plots
        np.savez(self.plotSavePath+'%s_%s_throughput.npz'%(self.objectName.strip(),self.fluxTstamp),throughput=1.0/self.fluxFactors,wvls=self.binCenters)

        pp = PdfPages(fullFluxPlotFileName)
        #plt.rcParams['font.size'] = 2

        wvls = self.binCenters

        plt.figure()
        ax1 = plt.subplot(111)
        ax1.set_title('ARCONS median flat cal\'d flux in counts')
        plt.plot(wvls,self.fluxSpectrum)
        pp.savefig()

        plt.figure()
        ax2 = plt.subplot(111)
        ax2.set_title('ARCONS median flat cal\'d sky in counts')
        plt.plot(wvls,self.skySpectrum)
        pp.savefig()

        plt.figure()
        ax3 = plt.subplot(111)
        ax3.set_title('Flux data minus sky in counts')
        plt.plot(wvls,self.subtractedSpectrum)
        pp.savefig()

        plt.figure()
        ax4 = plt.subplot(111)
        ax4.set_title('Std Spectrum of %s'%(self.objectName))
        plt.plot(self.stdWvls,self.stdFlux)
        pp.savefig()

        plt.figure()
        ax5 = plt.subplot(111)
        ax5.set_title('Binned Std Spectrum')
        plt.plot(wvls,self.binnedSpectrum)
        pp.savefig()

        plt.figure()
        ax6 = plt.subplot(111)
        ax6.set_title('Median Sensitivity Spectrum')
        ax6.set_xlim((3000,13000))
        #ax6.set_ylim((0,5))
        plt.plot(wvls,self.fluxFactors)
        pp.savefig()

        plt.figure()
        ax7 = plt.subplot(111)
        ax7.set_title('1/Sensitivity (Throughput)')
        ax7.set_xlim((4000,11000))
        plt.plot(wvls,1.0/self.fluxFactors)
        pp.savefig()

        plt.figure()
        ax8 = plt.subplot(111)
        ax8.set_title('Flux Cal\'d ARCONS Spectrum of Std')
        plt.plot(wvls,self.fluxFactors*self.subtractedSpectrum)
        pp.savefig()
        pp.close()
        print "Saved Flux Cal plots to %s"%(fullFluxPlotFileName)
    

    def writeFactors(self,fluxCalFileName):
        """
        Write flux cal weights to h5 file
        """

        if os.path.isabs(fluxCalFileName) == True:
            fullFluxCalFileName = fluxCalFileName
        else:
            scratchDir = os.getenv('MKID_PROC_PATH')
            fluxDir = os.path.join(scratchDir,'fluxCalSolnFiles')
            fullFluxCalFileName = os.path.join(fluxDir,fluxCalFileName)

        try:
            fluxCalFile = tables.openFile(fullFluxCalFileName,mode='w')
        except:
            print 'Error: Couldn\'t create flux cal file, ',fullFluxCalFileName
            return

        calgroup = fluxCalFile.createGroup(fluxCalFile.root,'fluxcal','Table of flux calibration weights by wavelength')
        caltable = tables.Array(calgroup,'weights',object=self.fluxFactors,title='Flux calibration Weights indexed by wavelengthBin')
        flagtable = tables.Array(calgroup,'flags',object=self.fluxFlags,title='Flux cal flags indexed by wavelengthBin. 0 is Good')
        bintable = tables.Array(calgroup,'wavelengthBins',object=self.wvlBinEdges,title='Wavelength bin edges corresponding to third dimension of weights array')
        fluxCalFile.flush()
        fluxCalFile.close()
        print "Finished Flux Cal, written to %s"%(fullFluxCalFileName)


    def cleanSpectrum_old(self,x,y,objectName):
        ''' 
        function to take high resolution spectrum of standard star, extend IR coverage with 
        an exponential tail, then rebin down to ARCONS resolution. This function has since been
        deprecated with the current cleanSpectrum which uses a BB fit to extend IR coverage,
        and does the rebinning using a gaussian convolution. This is left in for reference.
        '''

        #locations and widths of absorption features in Angstroms
        #features = [3890,3970,4099,4340,4860,6564,6883,7619]
        #widths = [50,50,50,50,50,50,50,50]
        #for i in xrange(len(features)):
        #    #check for absorption feature in std spectrum
        #    ind = np.where((x<(features[i]+15)) & (x>(features[i]-15)))[0]
        #    if len(ind)!=0:
        #        ind = ind[len(ind)/2]
        #    #if feature is found (flux is higher on both sides of the specified wavelength where the feature should be)
        #    if y[ind]<y[ind+1] and y[ind]<y[ind-1]:
        #        #cut out width[i] around feature[i]
        #        inds = np.where((x >= features[i]+widths[i]) | (x <= features[i]-widths[i]))
        #        x = x[inds]
        #        y = y[inds]

        #fit a tail to the end of the spectrum to interpolate out to desired wavelength in angstroms
        fraction = 3.0/4.0
        newx = np.arange(int(x[fraction*len(x)]),20000)

        slopeguess = (np.log(y[-1])-np.log(y[fraction*len(x)]))/(x[-1]-x[fraction*len(x)])
        print "Guess at exponential slope is %f"%(slopeguess)
        guess_a, guess_b, guess_c = float(y[fraction*len(x)]), x[fraction*len(x)], slopeguess
        guess = [guess_a, guess_b, guess_c]

        fitx = x[fraction*len(x):]
        fity = y[fraction*len(x):]

        exp_decay = lambda fx, A, x0, t: A * np.exp((fx-x0) * t)

        params, cov = curve_fit(exp_decay, fitx, fity, p0=guess, maxfev=2000)
        A, x0, t= params
        print "A = %s\nx0 = %s\nt = %s\n"%(A, x0, t)
        best_fit = lambda fx: A * np.exp((fx-x0)*t)

        calcx = np.array(newx,dtype=float)
        newy = best_fit(calcx)

        #func = interpolate.splrep(x[fration*len(x):],y[fraction*len(x):],s=smooth)
        #newx = np.arange(int(x[fraction*len(x)]),self.wvlBinEdges[-1])
        #newy = interpolate.splev(newx,func)

        wl = np.concatenate((x,newx[newx>max(x)]))
        flux = np.concatenate((y,newy[newx>max(x)]))

        #new method, rebin data to grid of wavelengths generated from a grid of evenly spaced energy bins
        #R=7.0 at 4500
        #R=E/dE -> dE = R/E
        dE = 0.3936 #eV
        start = 1000 #Angs
        stop = 20000 #Angs
        enBins = ObsFile.makeWvlBins(dE,start,stop)
        rebinned = rebin(wl,flux,enBins)
        re_wl = rebinned[:,0]
        re_flux = rebinned[:,1]
        #plt.plot(re_wl,re_flux,color='r')
        
        re_wl = re_wl[np.isnan(re_flux)==False]
        re_flux = re_flux[np.isnan(re_flux)==False]

        start1 = self.wvlBinEdges[0]
        stop1 = self.wvlBinEdges[-1]
        #regrid downsampled data 

        new_wl = np.arange(start1,stop1)

        #print re_wl
        #print re_flux
        #print new_wl

        #weight=1.0/(re_flux)**(2/1.00)
        print len(re_flux)
        weight = np.ones(len(re_flux))
        #decrease weights near peak
        ind = np.where(re_flux == max(re_flux))[0]
        weight[ind] = 0.3
        for p in [1,2,3]:
            if p==1:
                wt = 0.3
            elif p==2:
                wt = 0.6
            elif p==3:
                wt = 0.7
            try:
                weight[ind+p] = wt
            except IndexError:
                 pass
            try:
                 if ind-p >= 0:
                     weight[ind-p] = wt
            except IndexError:
                pass
        weight[-4:] = 1.0
        #weight = [0.7,1,0.3,0.3,0.5,0.7,1,1,1]
        #print len(weight)
        #weight = re_flux/min(re_flux)
        #weight = 1.0/weight
        #weight = weight/max(weight)
        #print weight
        f = interpolate.splrep(re_wl,re_flux,w=weight,k=3,s=max(re_flux)**1.71)
        new_flux = interpolate.splev(new_wl,f,der=0)
        return new_wl, new_flux
def main():
    """
    params = []
    paramfile = sys.argv[1]

    f = open(paramfile,'r')
    for line in f:
        params.append(line)
    f.close()
    datadir = params[0].split('=')[1].strip()
    flatdir = params[1].split('=')[1].strip()
    fluxdir = params[2].split('=')[1].strip()
    wvldir = params[3].split('=')[1].strip()
    obsfile = params[4].split('=')[1].strip()
    skyfile = params[5].split('=')[1].strip()
    flatfile = params[6].split('=')[1].strip()
    fluxfile = params[7].split('=')[1].strip()
    wvlfile = params[8].split('=')[1].strip()
    objectName = params[9].split('=')[1].strip()
    fluxCalObject = params[10].split('=')[1].strip()

    obsFileName = os.path.join(datadir, obsfile)
    skyFileName = os.path.join(datadir, skyfile)
    wvlCalFileName = os.path.join(wvldir, wvlfile)
    flatCalFileName = os.path.join(flatdir, flatfile)
    fluxCalFileName = os.path.join(fluxdir, fluxfile)
    """

    if len(sys.argv) >2:
        fileNum = str(sys.argv[2])
    else:
        fileNum = '0'

    #science object parameter file
    params = []
    paramfile = sys.argv[1]
    f = open(paramfile,'r')
    for line in f:
        params.append(line)
    f.close()

    datadir = params[0].split('=')[1].strip()
    flatdir = params[1].split('=')[1].strip()
    wvldir = params[2].split('=')[1].strip()
    obsfile = params[3].split('=')[1].strip()
    skyfile = params[4].split('=')[1].strip()
    flatfile = params[5].split('=')[1].strip()
    wvlfile = params[6].split('=')[1].strip()
    objectName = params[9].split('=')[1].strip()

    wvldir = "/Scratch/waveCalSolnFiles/oldbox_numbers/20121206"

    if len(params)>10:
        xpix = int(params[10].split('=')[1].strip())
        ypix = int(params[11].split('=')[1].strip())
        apertureRadius = int(params[12].split('=')[1].strip())
        startTime = int(params[13].split('=')[1].strip())
        intTime =int(params[14].split('=')[1].strip())

    obsFileName = os.path.join(datadir, obsfile)
    skyFileName = os.path.join(datadir, skyfile)
    wvlCalFileName = os.path.join(wvldir, wvlfile)
    flatCalFileName = os.path.join(flatdir, flatfile)

    obs = ObsFile(obsFileName)
    obs.loadWvlCalFile(wvlCalFileName)
    obs.loadFlatCalFile(flatCalFileName)
    print "analyzing file %s"%(obsFileName)
    print "loaded data file and calibrations\n---------------------\n"

    nRow = obs.nRow
    nCol = obs.nCol
    obsTime = obs.getFromHeader("exptime")
    #wvlBinEdges,obsSpectra = loadSpectra(obs,nCol,nRow)
    #nWvlBins=len(wvlBinEdges)-1

    #print np.shape(obsSpectra)
    #print nRow
    #print nCol
    #print nWvlBins

    #load/generate hot pixel mask file
    HotPixFile = getTimeMaskFileName(obsFileName)
    if not os.path.exists(HotPixFile):
        hp.findHotPixels(obsFileName,HotPixFile)
        print "Flux file pixel mask saved to %s"%(HotPixFile)
    obs.loadHotPixCalFile(HotPixFile)
    print "Hot pixel mask loaded %s"%(HotPixFile)

    #######
    #EVERYTHING BEFORE HERE IS STANDARD FILE/CALFILE LOADING

    
    startWvl = 3000
    #stopWvl = 7000 #for V-band
    stopWvl = 9000 #for R-band

    print "Making spectral cube"
    #for pg0220 first sec should be 80 since object is moving around before this
    #for pg0220A first sec should be 70, integration time is 140
    #for landolt 9542 first sec should be 20, int time is -1
    cubeDict = obs.getSpectralCube(firstSec=startTime, integrationTime=intTime, wvlStart = startWvl, wvlStop = stopWvl, wvlBinEdges = [startWvl,stopWvl], weighted=False)

    cube= np.array(cubeDict['cube'], dtype=np.double)
    wvlBinEdges= cubeDict['wvlBinEdges']
    effIntTime= cubeDict['effIntTime']
    print "median effective integration time = ", np.median(effIntTime)

    nWvlBins=len(wvlBinEdges)-1
    print "cube shape ", np.shape(cube)
    print "effIntTime shape ", np.shape(effIntTime)

    #add third dimension to effIntTime for  broadcasting
    effIntTime = np.reshape(effIntTime,np.shape(effIntTime)+(1,))
    cube /= effIntTime #put cube into counts/s
    
    #calculate midpoints of wvl bins for plotting
    wvls = np.empty((nWvlBins),dtype=float)
    for n in xrange(nWvlBins):
        binsize=wvlBinEdges[n+1]-wvlBinEdges[n]
        wvls[n] = (wvlBinEdges[n]+(binsize/2.0))

    print "wvls ",wvls
    #reshape cube for makeMovie
    movieCube = np.zeros((nWvlBins,np.shape(cube)[0],np.shape(cube)[1]),dtype=float)
    for i in xrange(nWvlBins):
        movieCube[i,:,:] = cube[:,:,i]
        #show individual frames as they are made to debug
        #plt.matshow(movieCube[i],vmin = 0, vmax = 100)
        #plt.show()
    print "movieCube shape ", np.shape(movieCube)
    print "wvls shape ", np.shape(wvls)

    #print cube
    #print "--------------------------"
    #print movieCube

    print "adding frames with wvl below ", stopWvl
    finalframe = np.zeros((1,np.shape(movieCube)[1],np.shape(movieCube)[2]))
    for f in xrange(len(wvls[wvls<stopWvl])):
        print wvls[f]
        finalframe[0]+=movieCube[f]
        plt.matshow(movieCube[f],vmin=0,vmax = 40)
        plt.show()

    movieCube = finalframe

    np.savez('%s_%s.npz'%(objectName,fileNum),stack=movieCube,wvls=wvls)
    print "Saved frame to .npz file"
    
    plt.matshow(movieCube[0],vmin=0,vmax = 40)
    plt.show()
    def __init__(self,paramFile,plots=False,verbose=False):
        """
        Opens flux file, prepares standard spectrum, and calculates flux factors for the file.
        Method is provided in param file. If 'relative' is selected, an obs file with standard star defocused over
        the entire array is expected, with accompanying sky file to do sky subtraction.
        If any other method is provided, 'absolute' will be done by default, wherein a point source is assumed
        to be present. The obs file is then broken into spectral frames with photometry (psf or aper) performed 
        on each frame to generate the ARCONS observed spectrum.
        """
        self.verbose=verbose
        self.plots = plots

        self.params = readDict()
        self.params.read_from_file(paramFile)
        
        run = self.params['run']
        sunsetDate = self.params['fluxSunsetLocalDate']
        self.fluxTstamp = self.params['fluxTimestamp']
        skyTstamp = self.params['skyTimestamp']
        wvlSunsetDate = self.params['wvlCalSunsetLocalDate']
        wvlTimestamp = self.params['wvlCalTimestamp']
        flatCalFileName = self.params['flatCalFileName']
        needTimeAdjust = self.params['needTimeAdjust']
        self.deadtime = float(self.params['deadtime']) #from firmware pulse detection
        self.timeSpacingCut = self.params['timeSpacingCut']
        bLoadBeammap = self.params.get('bLoadBeammap',False)
        self.method = self.params['method']
        self.objectName = self.params['object']
        self.r = float(self.params['energyResolution'])
        self.photometry = self.params['photometry']
        self.centroidRow = self.params['centroidRow']
        self.centroidCol = self.params['centroidCol']
        self.aperture = self.params['apertureRad']
        self.annulusInner = self.params['annulusInner']
        self.annulusOuter = self.params['annulusOuter']
        self.collectingArea = self.params['collectingArea']
        self.startTime = self.params['startTime']
        self.intTime = self.params['integrationTime']

        fluxFN = FileName(run=run,date=sunsetDate,tstamp=self.fluxTstamp)
        self.fluxFileName = fluxFN.obs()
        self.fluxFile = ObsFile(self.fluxFileName)

        if self.plots:
            self.plotSavePath = os.environ['MKID_PROC_PATH']+os.sep+'fluxCalSolnFiles'+os.sep+run+os.sep+sunsetDate+os.sep+'plots'+os.sep
            if not os.path.exists(self.plotSavePath): os.mkdir(self.plotSavePath)
            if self.verbose: print "Created directory %s"%self.plotSavePath

        obsFNs = [fluxFN]
        self.obsList = [self.fluxFile]

        if self.startTime in ['',None]: self.startTime=0
        if self.intTime in ['',None]: self.intTime=-1

        if self.method=="relative":
            try:
                print "performing Relative Flux Calibration"
                skyFN = FileName(run=run,date=sunsetDate,tstamp=skyTstamp)
                self.skyFileName = skyFN.obs()
                self.skyFile = ObsFile(self.skyFileName)
                obsFNs.append(skyFN)
                self.obsList.append(self.skyFile)
            except:
                print "For relative flux calibration a sky file must be provided in param file"
                self.__del__()
        else:
            self.method='absolute'
            print "performing Absolute Flux Calibration"

        if self.photometry not in ['aperture','PSF']: self.photometry='PSF' #default to PSF fitting if no valid photometry selected

        timeMaskFileNames = [fn.timeMask() for fn in obsFNs]
        timeAdjustFileName = FileName(run=run).timeAdjustments()

        #make filename for output fluxCalSoln file
        self.fluxCalFileName = FileName(run=run,date=sunsetDate,tstamp=self.fluxTstamp).fluxSoln()
        print "Creating flux cal: %s"%self.fluxCalFileName

        if wvlSunsetDate != '':
            wvlCalFileName = FileName(run=run,date=wvlSunsetDate,tstamp=wvlTimestamp).calSoln()
        if flatCalFileName =='':
            flatCalFileName=FileName(obsFile=self.fluxFile).flatSoln()

        #load cal files for flux file and, if necessary, sky file
        for iObs,obs in enumerate(self.obsList):
            if bLoadBeammap:
                print 'loading beammap',os.environ['MKID_BEAMMAP_PATH']
                obs.loadBeammapFile(os.environ['MKID_BEAMMAP_PATH'])
            if wvlSunsetDate != '':
                obs.loadWvlCalFile(wvlCalFileName)
            else:
                obs.loadBestWvlCalFile()

            obs.loadFlatCalFile(flatCalFileName)
            obs.setWvlCutoffs(-1,-1)

            if needTimeAdjust:
                obs.loadTimeAdjustmentFile(timeAdjustFileName)
            timeMaskFileName = timeMaskFileNames[iObs]
            print timeMaskFileName

            if not os.path.exists(timeMaskFileName):
                print 'Running hotpix for ',obs
                hp.findHotPixels(obsFile=obs,outputFileName=timeMaskFileName,fwhm=np.inf,useLocalStdDev=True)
                print "Flux cal/sky file pixel mask saved to %s"%(timeMaskFileName)
            obs.loadHotPixCalFile(timeMaskFileName)
            if self.verbose: print "Loaded hot pixel file %s"%timeMaskFileName

        #get flat cal binning information since flux cal will need to match it
        self.wvlBinEdges = self.fluxFile.flatCalFile.root.flatcal.wavelengthBins.read()
        self.nWvlBins = self.fluxFile.flatWeights.shape[2]
        self.binWidths = np.empty((self.nWvlBins),dtype=float)
        self.binCenters = np.empty((self.nWvlBins),dtype=float)
        for i in xrange(self.nWvlBins):
            self.binWidths[i] = self.wvlBinEdges[i+1]-self.wvlBinEdges[i]
            self.binCenters[i] = (self.wvlBinEdges[i]+(self.binWidths[i]/2.0))

        if self.method=='relative':
            print "Extracting ARCONS flux and sky spectra"
            self.loadRelativeSpectrum()
            print "Flux Spectrum loaded"
            self.loadSkySpectrum()
            print "Sky Spectrum loaded"
        elif self.method=='absolute':
            print "Extracting ARCONS point source spectrum"
            self.loadAbsoluteSpectrum()

        print "Loading standard spectrum"
        try:
            self.loadStdSpectrum(self.objectName)
        except KeyError:
            print "Invalid spectrum object name"
            self.__del__()
            sys.exit()

        print "Generating sensitivity curve"
        self.calculateFactors()
        print "Sensitivity Curve calculated"
        print "Writing fluxCal to file %s"%self.fluxCalFileName
        self.writeFactors(self.fluxCalFileName)
        
        if self.plots: self.makePlots()

        print "Done"
for seq in seqs:
    NumFiles.append(len(seq))
NumFiles = sum(NumFiles)
print (NumFiles)*expTime/integrationTime,'frames to make'

for iSeq in range(len(seqs)):
    timestampList = timestampLists[iSeq]
    print timestampList
    wfn = wvlCalFilenames[iSeq]
    ffn = flatCalFilenames[iSeq]
    sunsetDate = sunsetDates[iSeq]
    
    for i,ts in enumerate(timestampList):
        print 'loading',ts
        obsFn = FileName(run=run,date=sunsetDate,tstamp=ts).obs()
        ob = ObsFile(obsFn)
        ob.loadTimeAdjustmentFile(FileName(run=run).timeAdjustments())
        index1 = obsFn.find('_')
        hotPixFn = '/Scratch/timeMasks/timeMask' + obsFn[index1:]
        if not os.path.exists(hotPixFn):
            hp.findHotPixels(obsFn,hotPixFn)
            print "Flux file pixel mask saved to %s"%(hotPixFn)
        ob.loadHotPixCalFile(hotPixFn,switchOnMask=True)
        ob.loadWvlCalFile(wfn)
        ob.loadFlatCalFile(ffn)
        ob.setWvlCutoffs(wvlLowerCutoff,wvlUpperCutoff)

        bad_solution_mask=np.zeros((46,44))
        bad_count=0;
        for y in range(46):
            for x in range(44):
stride = 10
threshold = 100
nAboveThreshold = 0
npList = []
sigList = []
   
run = 'PAL2012'
sundownDate = '20121211'
obsDate = '20121212'

for seq in seq5:
    inFile = open("cosmicTimeList-%s.pkl"%(seq),"rb")
    cosmicTimeList = pickle.load(inFile)
    binContents = pickle.load(inFile)
    cfn = "cosmicMax-%s.h5"%seq
    intervals = ObsFile.readCosmicIntervalFromFile(cfn)
    for interval in intervals:
        print "interval=",interval
        fn = FileName(run, sundownDate,obsDate+"-"+seq)
        obsFile = ObsFile(fn.obs())
        obsFile.loadTimeAdjustmentFile(fn.timeAdjustments())
        i0=interval[0]
        i1=interval[1]
        intervalTime = i1-i0
        dt = intervalTime/2
        beginTime = max(0,i0-0.000200)
        endTime = beginTime + 0.001
        integrationTime = endTime-beginTime
        nBins = int(np.round(obsFile.ticksPerSec*(endTime-beginTime)+1))
        timeHgValues = np.zeros(nBins, dtype=np.int64)
        ymax = sys.float_info.max/100.0
def cleanSpectrum(x,y,objectName, wvlBinEdges):
        #locations and widths of absorption features in Angstroms
        #features = [3890,3970,4099,4340,4860,6564,6883,7619]
        #widths = [50,50,50,50,50,50,50,50]
        #for i in xrange(len(features)):
        #    #check for absorption feature in std spectrum
        #    ind = np.where((x<(features[i]+15)) & (x>(features[i]-15)))[0]
        #    if len(ind)!=0:
        #        ind = ind[len(ind)/2]
        #    #if feature is found (flux is higher on both sides of the specified wavelength where the feature should be)
        #    if y[ind]<y[ind+1] and y[ind]<y[ind-1]:
        #        #cut out width[i] around feature[i]
        #        inds = np.where((x >= features[i]+widths[i]) | (x <= features[i]-widths[i]))
        #        x = x[inds]
        #        y = y[inds]

        #fit a tail to the end of the spectrum to interpolate out to desired wavelength in angstroms
        fraction = 2.1/3
        newx = np.arange(int(x[fraction*len(x)]),20000)

        slopeguess = (np.log(y[-1])-np.log(y[fraction*len(x)]))/(x[-1]-x[fraction*len(x)])
        print "Guess at exponential slope is %f"%(slopeguess)
        guess_a, guess_b, guess_c = float(y[fraction*len(x)]), x[fraction*len(x)], slopeguess
        guess = [guess_a, guess_b, guess_c]

        fitx = x[fraction*len(x):]
        fity = y[fraction*len(x):]

        exp_decay = lambda fx, A, x0, t: A * np.exp((fx-x0) * t)

        params, cov = curve_fit(exp_decay, fitx, fity, p0=guess, maxfev=2000)
        A, x0, t= params
        print "A = %s\nx0 = %s\nt = %s\n"%(A, x0, t)
        best_fit = lambda fx: A * np.exp((fx-x0)*t)

        calcx = np.array(newx,dtype=float)
        newy = best_fit(calcx)

        #func = interpolate.splrep(x[fration*len(x):],y[fraction*len(x):],s=smooth)
        #newx = np.arange(int(x[fraction*len(x)]),self.wvlBinEdges[-1])
        #newy = interpolate.splev(newx,func)

        wl = np.concatenate((x,newx[newx>max(x)]))
        flux = np.concatenate((y,newy[newx>max(x)]))

        #new method, rebin data to grid of wavelengths generated from a grid of evenly spaced energy bins
        #R=7.0 at 4500
        #R=E/dE -> dE = R/E
        dE = 0.3936 #eV
        start = 1000 #Angs
        stop = 25000 #Angs
        enBins = ObsFile.makeWvlBins(dE,start,stop)
        rebinned = rebin(wl,flux,enBins)
        re_wl = rebinned[:,0]
        re_flux = rebinned[:,1]
        #plt.plot(re_wl,re_flux,color='r')
        
        re_wl = re_wl[np.isnan(re_flux)==False]
        re_flux = re_flux[np.isnan(re_flux)==False]

        start1 = wvlBinEdges[0]
        stop1 = wvlBinEdges[-1]
        #regrid downsampled data 

        new_wl = np.arange(start1,stop1)

        #print re_wl
        #print re_flux
        #print new_wl

        #weight=1.0/(re_flux)**(2/1.00)
        print len(re_flux)
        weight = np.ones(len(re_flux))
        #decrease weights near peak
        ind = np.where(re_flux == max(re_flux))[0]
        weight[ind] = 0.3
        for p in [1,2,3]:
            if p==1:
                wt = 0.3
            elif p==2:
                wt = 0.6
            elif p==3:
                wt = 0.7
            try:
                weight[ind+p] = wt
            except IndexError:
                 pass
            try:
                 if ind-p >= 0:
                     weight[ind-p] = wt
            except IndexError:
                pass
        #change weights to set how tightly fit must match data points
        #weight[-4:] = 1.0
        weight = [0.7,0.7,0.7,0.7,0.7,0.7,0.7,0.7,0.7]
        #print len(weight)
        #weight = re_flux/min(re_flux)
        #weight = 1.0/weight
        #weight = weight/max(weight)
        print weight
        f = interpolate.splrep(re_wl,re_flux,w=weight,k=3,s=max(re_flux)**200)
        new_flux = interpolate.splev(new_wl,f,der=0)
        return new_wl, new_flux
class Cosmic:

    def __init__(self, fn, beginTime=0, endTime='exptime',
                 nBinsPerSec=10, flashMergeTime=1.0, 
                 applyCosmicMask = False,
                 loggingLevel=logging.CRITICAL,
                 loggingHandler=logging.StreamHandler()):
        
        """
        Opens fileName in MKID_RAW_PATH, sets roachList
        endTime is exclusive
        """
        self.logger = logging.getLogger("cosmic")
        self.logger.setLevel(loggingLevel)
        formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
        loggingHandler.setFormatter(formatter)
        self.logger.addHandler(loggingHandler)
        self.logger.info("Cosmic:  begin init for obsFile=%s"%fn.obs())
        self.fn = fn
        self.fileName = fn.obs();
        self.file = ObsFile(self.fileName)
        # apply Matt's time fix
        timeAdjustments = self.fn.timeAdjustments()
        if os.path.exists(timeAdjustments):
            self.file.loadTimeAdjustmentFile(timeAdjustments)
        # apply Julian's time masks
        timeMaskFile = self.fn.timeMask();
        if os.path.exists(timeMaskFile):
            self.file.loadHotPixCalFile(timeMaskFile,switchOnMask=True)

        # apply standard mask
        if applyCosmicMask:
            self.file.loadStandardCosmicMask()

        self._setRoachList()
        self._setAllSecs()
        self.exptime = self.file.getFromHeader('exptime')
        if endTime =='exptime':
            self.endTime = float(self.exptime)
        else:
            self.endTime = float(endTime)
        if ( (self.endTime > self.exptime) or (endTime < 0)):
            raise RuntimeError("bad endTime:  endTime=%s exptime=%s" % \
                                   (str(endTime),str(self.exptime)))

        self.beginTime = float(beginTime)
        self.timeHgs = "none"
        self.nBinsPerSec = nBinsPerSec
        self.flashMergeTime = flashMergeTime

        self.times = \
        np.arange(self.beginTime, self.endTime, 1.0/self.nBinsPerSec)

        # for measuring flashes, indexed by roach name
        self.rMean = {}   # mean from meanclip
        self.rSigma = {}  # sigma from meanclip
        self.rNSurvived = {} # number of survivors from meanclip
        self.rNormed = {} # (value-mean)/sigma
        self.flashInterval = {}
        self.logger.info("Cosmic:  end init:  beginTime=%s endTime=%s"%(str(self.beginTime),str(self.endTime)))
    def __del__(self):
        """
        Close any open files
        """
        # print "now in Cosmic.__del__ for ",self.fileName
        try:
            del self.file
        except:
            pass

    def _setRoachList(self):
        self.roachList = []
        for row in self.file.beamImage:
            for name in row:
                roachName = name.split("/")[1]
                if roachName not in self.roachList:
                    self.roachList.append(roachName)
        self.roachList.sort()

    def _setAllSecs(self):
       nRow = self.file.nRow
       nCol = self.file.nCol
       self.allSecs =  \
           dict( ((i,j),None) for i in range(nRow) for j in range(nCol))
       for iRow in np.arange(nRow):
           for iCol in np.arange(nCol):
               self.allSecs[iRow,iCol] = \
                   self.file.file.getNode(self.file.beamImage[iRow][iCol])

    def nPhoton(self, beginTime=0, endTime='expTime'):
        """
        trivial example of counting the number of photons in a file
        """
        nPhoton = 0
        for iRow in range(self.file.nRow):
            for iCol in range(self.file.nCol):
                for iSec in range(self.beginTime, self.endTime):
                    sec = self.allSecs[iRow,iCol][iSec]
                    nPhoton += len(sec)
        return nPhoton

    def findFlashes(self, clipsig=3.0, maxiter=5,\
                        converge_num=0.05, verbose=0, flashsig=6):
        """
        find flashes by looking at the time histograms.  Calculate the
        mean,sigma using meanclip and the parameters clipsig, maxiter, converge_num
        A time bin has a flash if the normalized signal (measured-mean)/sigma
        is larger than flashsig.

        
        """
        # make the blank data structures
        if self.timeHgs == "none":
            self.makeTimeHgs()
            self.flashInterval["all"] = []
        # find the flashes in each roach 
        for roach in self.roachList:
            self.rMean[roach],self.rSigma[roach],self.rNSurvived[roach] = \
                meanclip.meanclip(\
                np.array(self.timeHgs[roach]), clipsig, maxiter, converge_num,\
                    verbose)
            self.rNormed[roach] = \
                (self.timeHgs[roach]-self.rMean[roach])/self.rSigma[roach]

            self.flashInterval[roach] = interval()
            prev = 0
            a = self.rNormed[roach]
            for i in range(len(a)):
                this = a[i] > flashsig
                if (this != prev):
                    if (this):
                        iBegin = i
                    else:
                        self.flashInterval[roach] = \
                        self.flashInterval[roach] | interval[iBegin,i]
                prev = this
            if (prev):
                self.flashInterval[roach] = \
                    self.flashInterval[roach] | interval[iBegin,i]

        # union of all flashes
        self.flashInterval["all"] = interval()
        for roach in self.roachList:
            self.flashInterval["all"] = \
                self.flashInterval["all"] | self.flashInterval[roach]

        # look for gaps smaller than self.flashMergeTime and plug them
        dMax = self.nBinsPerSec*self.flashMergeTime
        extrema = self.flashInterval["all"].extrema
        for i in range(len(extrema)/2-1):
            i0 = extrema[2*i+1][0]
            i1 = extrema[2*(i+1)][0]
            if (i1-i0) <= dMax:
                t0 = self.beginTime + float(i0)/self.nBinsPerSec
                t1 = self.beginTime + float(i1)/self.nBinsPerSec
                self.flashInterval["all"] = \
                    self.flashInterval["all"] | interval[i0,i1]

        # convert to ticks since the beginning of the data file
        rlAll = list(self.roachList)
        rlAll.append("all")
        ticksPerSecond = int(1.0/self.file.tickDuration)
        offset = self.beginTime*ticksPerSecond
        scale = 1.0/(self.file.tickDuration*self.nBinsPerSec)
        for roach in rlAll:
            self.flashInterval[roach] = offset+scale*self.flashInterval[roach]

    def writeFlashesToHdf5(self,overwrite=1):
        """
        write intervals with flashes to the timeMask file
        """
        # get the output file name, and make the directory if you need to
        cmFileName = self.fn.cosmicMask()
        (cmDir,name) = os.path.split(cmFileName)
        if not os.path.exists(cmDir):
            os.makedirs(cmDir)

        # write parameters used to find flashes
        h5f = tables.openFile(cmFileName, 'w')
        fnode = filenode.newNode(h5f, where='/', name='timeMaskHdr')
        fnode.attrs.beginTime      = self.beginTime
        fnode.attrs.endTime        = self.endTime
        fnode.attrs.nBinsPerSec    = self.nBinsPerSec
        fnode.attrs.flashMergeTime = self.flashMergeTime
        fnode.close();

        # write the times where flashes are located
        tbl =  h5f.createTable('/','timeMask',TimeMask.TimeMask,"Time Mask")
        rlAll = list(self.roachList)
        rlAll.append("all")
        for roach in rlAll:
            extrema = self.flashInterval[roach].extrema
            for i in range(len(extrema)/2):
                row = tbl.row
                row['tBegin'] = int(extrema[2*i][0])
                row['tEnd'] = int(extrema[2*i+1][0])
                if (roach == "all"):
                    reason = "Merged Flash"
                else:
                    reason = "Flash in %s" % roach
                row['reason'] = TimeMask.timeMaskReason[reason]
                row.append()
                tbl.flush()
        tbl.close()
        h5f.close()

    def makeTimeHgs(self):
        """
        Fill in the timeHgs variable
        This is a dictionary, indexed by the roach name, of the time histograms
        """
        self.timeHgs = {}
        for iSec in range(self.beginTime, self.endTime):
            self.logger.info("in makeTimeHgs iSec=%4d / %4d" % (iSec,self.endTime))
            hgsThisSec = {}
            for iRow in range(self.file.nRow):
                for iCol in range(self.file.nCol):
                    sec = self.allSecs[iRow,iCol][iSec]
                    if len(sec) > 0:
                        times = sec & self.file.timestampMask
                        hg,edges = \
                        np.histogram(times,bins=self.nBinsPerSec, \
                                     range=(0,1.0/self.file.tickDuration))
                        roachName = \
                        self.file.beamImage[iRow][iCol].split("/")[1]
                        if not hgsThisSec.has_key(roachName):
                            hgsThisSec[roachName] = \
                            np.zeros(self.nBinsPerSec,dtype=np.int64)
                        hgsThisSec[roachName] += hg
            for roachName in hgsThisSec.keys():
                if not self.timeHgs.has_key(roachName):
                    self.timeHgs[roachName] = []
                self.timeHgs[roachName] += list(hgsThisSec[roachName])

    def plotTimeHgs(self):
        """
        Plot the time HGS in plt structure, with legend
        """
        plt.clf()
        plt.figure(1)
        keys = self.timeHgs.keys()
        keys.sort()

        plt.subplot(211)
        for roachName in keys:
            hg = self.timeHgs[roachName]
            plt.plot(self.times, hg,label=roachName)
        plt.legend()
        dt = 1.0/self.nBinsPerSec
        plt.ylabel("photons/%.2f sec" % dt)
        plt.title("Cosmic timeHgs "+ self.fileName)

        plt.subplot(212)
        for roachName in keys:
            plt.plot(self.times, \
                         self.rNormed[roachName],label=roachName)
        plt.xlabel("time (sec)")
        plt.ylim(-23,30)
        dt = 1.0/self.nBinsPerSec
        plt.ylabel("normalized photons/%.2f sec" % dt)

        y = -5
        x0 = self.beginTime + 0.1*(self.endTime-self.beginTime)
        xmax = plt.xlim()[1]
        rlAll = list(self.roachList)
        rlAll.append("all")
        for roach in rlAll:
            print "plot for roach=",roach
            plt.plot([x0,xmax],[y,y], linestyle=":", color="gray")
            plt.text(x0, y, roach, fontsize=8, va="center")
            extrema = self.flashInterval[roach].extrema
            for i in range(len(extrema)/2):
                t0 = (extrema[2*i][0]   - 0.5)*self.file.tickDuration
                t1 = (extrema[2*i+1][0] - 0.5)*self.file.tickDuration
                plt.plot([t0,t1],[y,y],'r', linewidth=4)
            y -= 2

    def findCosmics(self, stride=10, threshold=100, 
                    populationMax=2000, nSigma=5, writeCosmicMask=False,
                    ppsStride=10000):
        """
        Find cosmics ray suspects.  Histogram the number of photons
        recorded at each timeStamp.  When the number of photons in a
        group of stride timeStamps is greater than threshold in second
        iSec, add (iSec,timeStamp) to cosmicTimeLists.  Also keep
        track of the histogram of the number of photons per stride
        timeStamps.

        return a dictionary of 'populationHg', 'cosmicTimeLists',
        'binContents', 'timeHgValues', 'interval', 'frameSum', and 'pps'
      
         populationHg is a histogram of the number of photons in each time bin.
        This is a poisson distribution with a long tail due to cosmic events

        
        cosmicTimeLists is a numpy array  of all the sequences that are
        suspects for cosmic rays


        binContents corresponds to cosmicTimeLists.  For each time in
        cosmicTimeLists, binContents is the number of photons detected
        at that time.
        
        timeHgValues is a histogram of the number of photons in each time
        interval
        
        frameSum is a two dimensional  numpy array of the number of photons
        detected by each pixel

        interval is the interval of data to be masked out

        pps is photons per second, calculated every ppsStride bins.

        """

        self.logger.info("findCosmics: begin stride=%d threshold=%d populationMax=%d nSigma=%d writeCosmicMask=%s"%(stride,threshold,populationMax,nSigma,writeCosmicMask))
        exptime = self.endTime-self.beginTime
        nBins = int(np.round(self.file.ticksPerSec*exptime+1))
        bins = np.arange(0, nBins, 1)
        timeHgValues,frameSum = self.getTimeHgAndFrameSum(self.beginTime,self.endTime)
        remainder = len(timeHgValues)%ppsStride
        if remainder > 0:
            temp = timeHgValues[:-remainder]
        else:
            temp = timeHgValues
        ppsTime = (ppsStride*self.file.tickDuration)
        pps = np.sum(temp.reshape(-1, ppsStride), axis=1)/ppsTime
        self.logger.info("findCosmics:  call populationFromTimeHgValues")
        pfthgv = Cosmic.populationFromTimeHgValues\
            (timeHgValues,populationMax,stride,threshold)
        #now build up all of the intervals in seconds
        self.logger.info("findCosmics:  build up intervals:  nCosmicTime=%d"%len(pfthgv['cosmicTimeList']))
        i = interval()
        iCount = 0
        secondsPerTick = self.file.tickDuration
        for cosmicTime in pfthgv['cosmicTimeList']:
            #t0 = max(0,self.beginTime+(cosmicTime-50)/1.e6)
            #t1 = min(self.endTime,self.beginTime+(cosmicTime+50)/1.e6)
            #intTime = t1-t0            
            t0 = self.beginTime+cosmicTime*secondsPerTick
            dt = stride*secondsPerTick
            t1 = t0+dt
            left = max(self.beginTime, t0-nSigma*dt)
            right = min(self.endTime, t1+2*nSigma*dt)
            i = i | interval[left,right]
            self.logger.debug("findCosmics:  iCount=%d t0=%f t1=%f left=%f right=%f"%(iCount,t0,t1,left,right))
            iCount+=1

        tMasked = Cosmic.countMaskedBins(i)
        ppmMasked = 1000000*tMasked/(self.endTime-self.beginTime)

        retval = {}
        retval['timeHgValues'] = timeHgValues
        retval['populationHg'] = pfthgv['populationHg']
        retval['cosmicTimeList'] = pfthgv['cosmicTimeList']
        retval['binContents'] = pfthgv['binContents']
        retval['frameSum'] = frameSum
        retval['interval'] = i
        retval['ppmMasked'] = ppmMasked
        retval['pps'] = pps
        retval['ppsTime'] = ppsTime
        if writeCosmicMask:
            cfn = self.fn.cosmicMask()
            self.logger.info("findCosmics:  write masks to =%s"%cfn)
            ObsFile.writeCosmicIntervalToFile(i, self.file.ticksPerSec, 
                                              cfn,self.beginTime, self.endTime, 
                                              stride, threshold, nSigma, populationMax)
        self.logger.info("findCosmics:  end with ppm masked=%d"%ppmMasked)
        return retval
    def getTimeHgAndFrameSum(self, beginTime, endTime):
        integrationTime = endTime - beginTime
        nBins = int(np.round(self.file.ticksPerSec*integrationTime+1))
        timeHgValues = np.zeros(nBins, dtype=np.int64)
        frameSum = np.zeros((self.file.nRow,self.file.nCol))
        self.logger.info("get all time stamps for integrationTime=%f"%integrationTime)
        for iRow in range(self.file.nRow):
            #print "Cosmic.findCosmics:  iRow=",iRow
            for iCol in range(self.file.nCol):
                # getTimedPacketList is slow.  Use getPackets instead.
                #gtpl = self.file.getTimedPacketList(iRow,iCol,beginTime, 
                #                                    integrationTime)
                gtpl = self.file.getPackets(iRow,iCol,
                                            beginTime,integrationTime)
                timestamps = gtpl['timestamps']
                if timestamps.size > 0:
                    timestamps = \
                        (timestamps - beginTime)*self.file.ticksPerSec
                    # per Matt S. suggestion 2013-07-09
                    ts32 = np.round(timestamps).astype(np.uint32)
                    tsBinner.tsBinner32(ts32, timeHgValues)
                    frameSum[iRow,iCol] += ts32.size

        return timeHgValues,frameSum
    @staticmethod
    def countMaskedBins(maskInterval):
        retval = 0
        for x in maskInterval:
            retval += x[1]-x[0]
        return retval

    @staticmethod
    def populationFromTimeHgValues(timeHgValues,populationMax,stride,threshold):
        """
        Rebin the timgHgValues histogram by combining stride bins.  If
        stride > 1, then bin a second time after shifting by stride/2
        Create populationHg, a histogram of the number of photons in
        the large bins.  Also, create (and then sort) a list
        cosmicTimeList of the start of bins (in original time units)
        of overpopulated bins that have more than threshold number of
        photons.

        return a dictionary containing populationHg and cosmicTimeList
        """
        popRange = (-0.5,populationMax-0.5)
        if stride==1:
            populationHg = np.histogram(\
                timeHgValues, populationMax, range=popRange)
            cosmicTimeList = np.where(timeHgValues > threshold)[0]
            binContents = np.extract(timeHgValues > threshold, timeHgValues)
        else:
            # rebin the timeHgValues before counting the populations
            length = timeHgValues.size
            remainder = length%stride
            if remainder == 0:
                end = length
            else:
                end = -remainder

            timeHgValuesTrimmed = timeHgValues[0:end]

            timeHgValuesRebinned0 = np.reshape(
                timeHgValuesTrimmed, [length/stride, stride]).sum(axis=1)
            populationHg0 = np.histogram(
                timeHgValuesRebinned0, populationMax, range=popRange)
            cosmicTimeList0 = stride*np.where(\
                timeHgValuesRebinned0 > threshold)[0]
            binContents0 = np.extract(timeHgValuesRebinned0 > threshold,
                                      timeHgValuesRebinned0)

            timeHgValuesRebinned1 = np.reshape(
                timeHgValuesTrimmed[stride/2:-stride/2], 
                [(length-stride)/stride, stride]).sum(axis=1)
            populationHg1 = np.histogram(\
                timeHgValuesRebinned1, populationMax, range=popRange)
            cosmicTimeList1 = (stride/2)+stride*np.where(\
                timeHgValuesRebinned1 > threshold)[0]
            binContents1 = np.extract(timeHgValuesRebinned1 > threshold,
                                      timeHgValuesRebinned1)

            populationHg = (populationHg0[0]+populationHg1[0],\
                                populationHg0[1])
            cosmicTimeList = np.concatenate((cosmicTimeList0,cosmicTimeList1))
            binContents = np.concatenate((binContents0, binContents1))
            args = np.argsort(cosmicTimeList)
            cosmicTimeList = cosmicTimeList[args]
            binContents = binContents[args]
            cosmicTimeList.sort()

        retval = {}
        retval['populationHg'] = populationHg
        retval['cosmicTimeList'] = cosmicTimeList
        retval['binContents'] = binContents
        return retval

    def makeMovies(self,beginTick, endTick, backgroundFrame, accumulate=False):
        tick0 = np.uint64(beginTick)
        tick1 = np.uint64(endTick)
        for iRow in range(cosmic.file.nRow):
            for iCol in range(cosmic.file.nCol):
                gtpl = self.getTimedPacketList(iRow,iCol,sec0,1)
        timestamps = gtpl['timestamps']
        timestamps *= cosmic.file.ticksPerSec
        ts32 = timestamps.astype(np.uint32)
        for ts in ts32:
            tindex = ts-t0
            try:
                listOfPixelsToMark[tindex].append((iRow,iCol))
            except IndexError:
                pass
            for tick in range(t0,t1):
                frames.append(frameSum)
                title = makeTitle(tick,t0,t1)
                titles.append(title)

                mfn0 = "m-%s-%s-%s-%s-%010d-%010d-i.gif"%(run,sundownDate,obsDate,seq,t0,t1)
                utils.makeMovie(frames, titles, outName=mfn0, delay=0.1, colormap=mpl.cm.gray,
                                listOfPixelsToMark=listOfPixelsToMark,
                                pixelMarkColor='red')

        for i in range(len(listOfPixelsToMark)-1):
            listOfPixelsToMark[i+1].extend(listOfPixelsToMark[i])

        mfn1 = "m-%s-%s-%s-%s-%010d-%010d-a.gif"%(run,sundownDate,obsDate,seq,t0,t1)
        utils.makeMovie(frames, titles, outName=mfn1, delay=0.1, colormap=mpl.cm.gray,
                        listOfPixelsToMark=listOfPixelsToMark,
                        pixelMarkColor='green')

    def fitDecayTime(self,t0Sec,lengthSec=200,plotFileName='none'):
        print "hello from fitDecayTime"
        timedPacketList = self.file.getTimedPacketList(
            iRow, iCol, sec0, lengthSec)

    
    def fitExpon(self, t0, t1):
        """
        Fit an exponential to all photons from time t0 to time t1
        t0 and t1 are in ticks, 1e6 ticks per second
        return a dictionary of:  timeStamps,fitParams,chi2

        """
         
        firstSec = int(t0/1e6)  # in seconds
        integrationTime = 1+int((t1-t0)/1e6) # in seconds
        nBins = integrationTime*1e6 # number of microseconds; one bin per microsecond
        timeHgValues = np.zeros(nBins, dtype=np.int64)
        print "firstSec=",firstSec," integrationTime=",integrationTime
        for iRow in range(self.file.nRow):
            for iCol in range(self.file.nCol):
                timedPacketList = self.file.getTimedPacketList(
                    iRow, iCol, firstSec=firstSec, 
                    integrationTime=integrationTime)
                timeStamps = timedPacketList['timestamps']
                if (len(timeStamps) > 0):
                    # covert the time values to microseconds, and
                    # make it the type np.uint64
                    # round per Matt S. suggestion 2013-07-09
                    #ts64 = (timeStamps).astype(np.uint64)
                    ts32round = np.round(timeStamps).astype(np.uint32)
                    tsBinner.tsBinner(ts32round, timeHgValues)
        
                    temp = 1e6*(timeStamps-firstSec)
                    for i in range(len(timeStamps)):
                        ts32 = ((timeStamps-firstSec)*1e6).astype(np.uint32)
                    # add these timestamps to the histogram timeHgValues
        remain0 = int(t0%1e6)
        remain1 = int(t1%1e6)
        timeHgValues = timeHgValues[remain0:remain1]
        x = np.arange(len(timeHgValues))
        y = timeHgValues
        
        xArray = np.arange(0, dtype=np.int64)
        yArray = np.arange(0, dtype=np.int64)

        for i in range(len(x)):
            if y[i] > 2:
                xArray = np.append(xArray,i)
                yArray = np.append(yArray,y[i])
        ySigma = np.sqrt(yArray)


        mean = (x*y).sum()/float(y.sum())
        bExponGuess = 1/mean
        aExponGuess = bExponGuess*timeHgValues.sum()
        cExponGuess = 0
        dExponGuess = 0
        pExponGuess = [aExponGuess, bExponGuess, cExponGuess, dExponGuess]

        bGaussGuess = mean
        avgx2 = (x*x*y).sum()/float(y.sum())        
        cGaussGuess = np.sqrt(avgx2-bGaussGuess*bGaussGuess)
        
        aGaussGuess = (timeHgValues.sum()/(cGaussGuess*np.sqrt(2*np.pi)))
        pGaussGuess = [aGaussGuess, bGaussGuess, cGaussGuess]
        
        xLimit = [bGaussGuess-4*cGaussGuess, bGaussGuess+4*cGaussGuess]
        retval = {'timeHgValues':timeHgValues, 'pExponFit':pExponGuess, 
                  'pGaussFit':pGaussGuess, 'xLimit':xLimit, 
                  'cGaussGuess':cGaussGuess, 'timeStamps':timeStamps}

        return retval

    @staticmethod
    def intervalTime(intervals):
        """
        return the time (in seconds) masked by the intervals
        """
        time = 0
        for interval in intervals:
            time += interval[1]-interval[0]
        return time

    @staticmethod
    def funcExpon(x, a, b, c, d):
        retval = a*np.exp(-b*(x-d)) + c
        retval[x < d] = 0
        return retval

    @staticmethod
    def funcGauss(x, a, b, c):
        return a*np.exp(-(x-b)**2/(2.*c**2))