def loadHotMask(self):
     #self.hotPixelFilename = str(self.displayStackPath + self.run + '/' + self.target + '/HotPixelMasks/hotPixelMask_' + self.obsTS + '.h5')
     self.hotPixelFilename = str(FileName(obsFile = self.ob).timeMask())
     if not os.path.exists(self.hotPixelFilename):
         hp.findHotPixels(obsFile=self.ob,outputFileName=self.hotPixelFilename)
         print "Hot pixel mask saved to %s"%(self.hotPixelFilename)
     self.ob.loadHotPixCalFile(self.hotPixelFilename,switchOnMask=True)
def generateObsObjectList(obsFNs,wvlLowerLimit=3000, wvlUpperLimit=13000,beammapFileName = None,loadHotPix=True,loadWvlCal=True,loadFlatCal=True,loadSpectralCal=True):

    obsFiles = []
    for obsFN in obsFNs:
        if type(obsFN) == type(''):     #Full path to obs file
            obs = ObsFile(obsFN)
        else:                           #FileName object
            obs = ObsFile(obsFN.obs())
            
        if beammapFileName is not None and os.path.isfile(beammapFileName):
            obs.loadBeammapFile(beammapFileName)
            
        obs.setWvlCutoffs(wvlLowerLimit=wvlLowerLimit, wvlUpperLimit=wvlUpperLimit)
        if loadHotPix:
            if not os.path.isfile(FileName(obsFile=obs).timeMask()):
                print "generating hp file ", FileName(obsFile=obs).timeMask()
                hp.findHotPixels(obsFile=obs,outputFileName=FileName(obsFile=obs).timeMask())
            obs.loadHotPixCalFile(FileName(obsFile=obs).timeMask(),reasons=['hot pixel','dead pixel'])
        if loadWvlCal:
            obs.loadBestWvlCalFile()
        if loadFlatCal:
            #obs.loadFlatCalFile(FileName(obsFile=obs).flatSoln())
            obs.loadFlatCalFile('/Scratch/flatCalSolnFiles/flatsol_1s.h5')
        if loadSpectralCal:
            pass
            
        obsFiles.append(obs)
        
    return obsFiles
def centroidObs(obsPath,centroidPath,centroidRa,centroidDec,haOffset,xGuess,yGuess,hotPath,flatPath):
    obs = ObsFile(obsPath)
    print obsPath,obs.getFromHeader('exptime'),obs
    if not os.path.exists(hotPath):
        hp.findHotPixels(obsFile=obs,outputFileName=hotPath)
    obs.loadHotPixCalFile(hotPath,switchOnMask=False)
    obs.loadBestWvlCalFile()
    obs.loadFlatCalFile(flatPath)
    obs.setWvlCutoffs(3000,8000)
    cc.centroidCalc(obs,centroidRa,centroidDec,guessTime=300,integrationTime=30,secondMaxCountsForDisplay=2000,HA_offset=haOffset,xyapprox=[xGuess,yGuess],outputFileName=centroidPath)
    print 'done centroid',centroidPath
    del obs
def main():
    """
    params = []
    paramfile = sys.argv[1]

    f = open(paramfile,'r')
    for line in f:
        params.append(line)
    f.close()
    datadir = params[0].split('=')[1].strip()
    flatdir = params[1].split('=')[1].strip()
    fluxdir = params[2].split('=')[1].strip()
    wvldir = params[3].split('=')[1].strip()
    obsfile = params[4].split('=')[1].strip()
    skyfile = params[5].split('=')[1].strip()
    flatfile = params[6].split('=')[1].strip()
    fluxfile = params[7].split('=')[1].strip()
    wvlfile = params[8].split('=')[1].strip()
    objectName = params[9].split('=')[1].strip()
    fluxCalObject = params[10].split('=')[1].strip()

    obsFileName = os.path.join(datadir, obsfile)
    skyFileName = os.path.join(datadir, skyfile)
    wvlCalFileName = os.path.join(wvldir, wvlfile)
    flatCalFileName = os.path.join(flatdir, flatfile)
    fluxCalFileName = os.path.join(fluxdir, fluxfile)
    """

    if len(sys.argv) >3:
        filenum = str('_'+sys.argv[3])
    else:
        filenum = '_0'

    #science object parameter file
    params = []
    paramfile = sys.argv[1]
    f = open(paramfile,'r')
    for line in f:
        params.append(line)
    f.close()

    datadir = params[0].split('=')[1].strip()
    flatdir = params[1].split('=')[1].strip()
    wvldir = params[2].split('=')[1].strip()
    obsfile = params[3].split('=')[1].strip()
    skyfile = params[4].split('=')[1].strip()
    flatfile = params[5].split('=')[1].strip()
    wvlfile = params[6].split('=')[1].strip()
    objectName = params[9].split('=')[1].strip()

    if len(params)>10:
        xpix = params[10].split('=')[1].strip()
        ypix = params[11].split('=')[1].strip()
        apertureRadius = params[12].split('=')[1].strip()


    #flux cal object parameter file

    params2 = []
    param2file = sys.argv[2]
    f = open(param2file,'r')
    for line in f:
        params2.append(line)
    f.close()

    fluxdir = params2[7].split('=')[1].strip()
    fluxfile = params2[8].split('=')[1].strip()
    fluxCalObject = params2[9].split('=')[1].strip()

    obsFileName = os.path.join(datadir, obsfile)
    skyFileName = os.path.join(datadir, skyfile)
    wvlCalFileName = os.path.join(wvldir, wvlfile)
    flatCalFileName = os.path.join(flatdir, flatfile)
    fluxCalFileName = os.path.join(fluxdir, fluxfile)

    print "obsfile = ",obsFileName
    print "skyfile = ",skyFileName
    print "wvlcal = ", wvlCalFileName
    print "flatcal = ", flatCalFileName
    print "fluxcal = ", fluxCalFileName
    print "object = ", objectName
    print "flux cal object = ", fluxCalObject
    print "\n---------------------\n"

    obs = ObsFile(obsFileName)
    obs.loadWvlCalFile(wvlCalFileName)
    obs.loadFlatCalFile(flatCalFileName)
    obs.loadFluxCalFile(fluxCalFileName)
    print "loaded data file and calibrations\n---------------------\n"

    HotPixFile = getTimeMaskFileName(obsFileName)
    if not os.path.exists(HotPixFile):
        hp.findHotPixels(obsFileName,HotPixFile)
        print "Flux file pixel mask saved to %s"%(HotPixFile)
    obs.loadHotPixCalFile(HotPixFile)
    print "Hot pixel mask loaded %s"%(HotPixFile)

    nRow = obs.nRow
    nCol = obs.nCol
    obsTime = obs.getFromHeader("exptime")
    wvlBinEdges,obsSpectra,obsIntTime = loadSpectra(obs,nCol,nRow)
    nWvlBins=len(wvlBinEdges)-1
    #divide spectrum by bin widths
    binWidths = np.empty(nWvlBins)
    for i in xrange(nWvlBins):
        binWidths[i] = wvlBinEdges[i+1]-wvlBinEdges[i]
    obsSpectra = obsSpectra/binWidths
    #divide by effective integration time
    for iRow in xrange(nRow):
        for iCol in xrange(nCol):
            obsSpectra[iRow][iCol] = obsSpectra[iRow][iCol] / obsIntTime[iRow][iCol]

    #print np.shape(obsSpectra)
    #print nRow
    #print nCol
    #print nWvlBins

    medianObsSpectrum = calculateMedian(obsSpectra,nCol,nRow,nWvlBins)
    print "target spectrum loaded\n---------------------\n"

    if skyfile != "None":
        sky = ObsFile(skyFileName)
        sky.loadWvlCalFile(wvlCalFileName)
        sky.loadFlatCalFile(flatCalFileName)
        sky.loadFluxCalFile(fluxCalFileName)
        skyTime = sky.getFromHeader("exptime")

        skyHotPixFile = getTimeMaskFileName(skyFileName)
        if not os.path.exists(skyHotPixFile):
            hp.findHotPixels(skyFileName,skyHotPixFile)
            print "Flux file pixel mask saved to %s"%(skyHotPixFile)
        obs.loadHotPixCalFile(HotPixFile)
        print "Hot pixel mask loaded %s"%(skyHotPixFile)

        wvlBinEdges,skySpectra,skyIntTime = loadSpectra(sky,nCol,nRow)

        skySpectra = skySpectra/binWidths
        #divide by effective integration time
        for iRow in xrange(nRow):
            for iCol in xrange(nCol):
                skySpectra[iRow][iCol] = skySpectra[iRow][iCol] / skyIntTime[iRow][iCol]

        skySpectrum = calculateMedian(skySpectra, nCol, nRow, nWvlBins)
        #skySpectrum = skySpectrum*float(obsTime)/float(skyTime) #scale sky spectrum to target observation time
        print "sky spectrum loaded\n---------------------\n"
    else:
        #if no sky file given, estimate sky spectrum as median spectrum of obs file, assuming object is tiny
        skySpectrum = medianObsSpectrum
        print "sky spectrum estimated as median of target file spectrum\n---------------------\n"

    #subtract sky spectrum from every pixel
    allSkySpectrum = obsSpectra-skySpectrum
    #set any negative values to 0 after sky subtraction
    allSkySpectrum[allSkySpectrum<0]=0

    #take median of remaining sky subtracted spectra to get median object spectrum
    finalSpectrum = calculateMedian(allSkySpectrum,nCol,nRow,nWvlBins)
    
    #load std spectrum for comparison
    realSpectra = loadStd(objectName,wvlBinEdges)
    print "real std spectrum loaded for reference\n---------------------\n"

    #create plots
    plotDir = "/home/srmeeker/ARCONS-pipeline/fluxcal/test/plots"
    plotFileName = "%s_from_%s%s.pdf"%(objectName,fluxCalObject,filenum)
    fullFluxPlotFileName = os.path.join(plotDir,plotFileName)
    pp = PdfPages(fullFluxPlotFileName)
    matplotlib.rcParams['font.size']=6

    #calculate midpoints of wvl bins for plotting
    wvls = np.empty((nWvlBins),dtype=float)
    for n in xrange(nWvlBins):
        binsize=wvlBinEdges[n+1]-wvlBinEdges[n]
        wvls[n] = (wvlBinEdges[n]+(binsize/2.0))

    plt.figure()

    ax1 = plt.subplot(231)
    ax1.set_title('ARCONS median flat/flux cal\'d obs in counts')
    plt.plot(wvls,medianObsSpectrum)
    #plt.show()
    ax2 = plt.subplot(232)
    ax2.set_title('ARCONS median flat/flux cal\'d sky in counts')
    plt.plot(wvls,skySpectrum)
    #plt.show()
    ax5 = plt.subplot(233)
    ax5.set_title('Sensitivity Spectrum')
    plt.plot(wvls,obs.fluxWeights)
    ax3 = plt.subplot(234)
    ax3.set_title('MKID data minus sky in counts')
    plt.plot(wvls,finalSpectrum/max(finalSpectrum))
    ax4 = plt.subplot(235)
    ax4.set_title('Rebinned Std Spectrum of %s'%(objectName))
    plt.plot(wvls,realSpectra)

    #ax7 = plt.subplot(337)
    #ax7.set_title('Flux Cal\'d ARCONS Spectrum of Std')
    #plt.plot(wvls,fluxFactors*subtractedSpectra)

    pp.savefig()
    pp.close()

    #del obs
    #del sky

    print "output plots to %s\n---------------------\n"%(fullFluxPlotFileName)

    txtDir = "/home/srmeeker/ARCONS-pipeline/fluxcal/test/txt"
    txtFileName = "%s_from_%s%s.txt"%(objectName,fluxCalObject,filenum)
    fullFluxTxtFileName = os.path.join(txtDir,txtFileName)

    outarr = np.empty((len(medianObsSpectrum),2),dtype=float)
    outarr[:,0]=wvls
    outarr[:,1]=medianObsSpectrum
    #save sensitivity spectrum to file
    np.savetxt(fullFluxTxtFileName, outarr)
    
    print "output txt file to %s\n---------------------\n"%(fullFluxPlotFileName)
def main():

    obsSequence0 = """
    051516
    052520
    """
    obsSequence1 = """
    033323
    041843
    045902
    """

    obsSequence2 = """
    050404
    054424
    """

    obsSequence3 = """
    054926
    062942
    """

    run = "PAL2012"
    obsSequences = [obsSequence1, obsSequence2, obsSequence3]
    wvlCals = ["063518", "063518", "063518"]
    flatCals = ["20121211", "20121211", "20121211"]
    fluxCalDates = ["20121206", "20121206", "20121206"]
    fluxCals = ["20121207-072055", "20121207-072055", "20121207-072055"]

    # Row coordinate of center of crab pulsar for each obsSequence
    centersRow = [29, 29, 10]
    # Col coordinate of center of crab pulsar for each obsSequence
    centersCol = [29, 30, 14]

    obsUtcDate = "20121212"
    obsUtcDates = ["20121212", "20121212", "20121212"]

    obsFileNames = []
    obsFileNameTimestamps = []
    wvlFileNames = []
    flatFileNames = []
    fluxFileNames = []
    timeMaskFileNames = []

    for iSeq in range(len(obsSequences)):
        obsSequence = obsSequences[iSeq]
        obsSequence = obsSequence.strip().split()
        obsFileNameTimestamps.append(obsSequence)
        obsUtcDate = obsUtcDates[iSeq]
        sunsetDate = str(int(obsUtcDate) - 1)
        obsSequence = [obsUtcDates[iSeq] + "-" + ts for ts in obsSequence]
        obsFileNames.append([FileName(run=run, date=sunsetDate, tstamp=ts).obs() for ts in obsSequence])
        timeMaskFileNames.append([FileName(run=run, date=sunsetDate, tstamp=ts).timeMask() for ts in obsSequence])
        wvlCalTstamp = obsUtcDate + "-" + wvlCals[iSeq]
        wvlFileNames.append(FileName(run=run, date=sunsetDate, tstamp=wvlCalTstamp).calSoln())
        fluxFileNames.append(FileName(run=run, date=fluxCalDates[iSeq], tstamp=fluxCals[iSeq]).fluxSoln())
        flatFileNames.append(FileName(run=run, date=flatCals[iSeq], tstamp="").flatSoln())

    for iSeq, obsSequence in enumerate(obsSequences):
        obsSequence = obsSequence.strip().split()
        print obsSequence
        for iOb, obs in enumerate(obsSequence):
            timeMaskFileName = timeMaskFileNames[iSeq][iOb]
            if not os.path.exists(timeMaskFileName):
                print "Running hotpix for ", obs
                hp.findHotPixels(obsFileNames[iSeq][iOb], timeMaskFileName)
                print "Flux file pixel mask saved to %s" % (timeMaskFileName)

    apertureRadius = 4
    obLists = [[ObsFile(fn) for fn in seq] for seq in obsFileNames]
    tstampFormat = "%H:%M:%S"
    # print 'fileName','headerUnix','headerUTC','logUnix','packetReceivedUnixTime'
    for iSeq, obList in enumerate(obLists):
        for iOb, ob in enumerate(obList):
            print ob.fileName
            centerRow = centersRow[iSeq]
            centerCol = centersCol[iSeq]
            circCol, circRow = circ(centerCol, centerRow)
            ob.loadTimeAdjustmentFile(FileName(run="PAL2012").timeAdjustments())
            ob.loadWvlCalFile(wvlFileNames[iSeq])
            ob.loadFlatCalFile(flatFileNames[iSeq])
            ob.loadFluxCalFile(fluxFileNames[iSeq])
            timeMaskFileName = timeMaskFileNames[iSeq][iOb]
            ob.loadHotPixCalFile(timeMaskFileName)
            ob.setWvlCutoffs(None, None)

    for iSeq, obList in enumerate(obLists):
        for iOb, ob in enumerate(obList):
            print ob.fileName

            centerRow = centersRow[iSeq]
            centerCol = centersCol[iSeq]
            circCol, circRow = circ(centerCol, centerRow)
            imgDict = ob.getPixelCountImage()
            img = imgDict["image"]
            utils.plotArray(img, showMe=False)
            aperture = plt.Circle((centerCol, centerRow), rad, fill=False, color="g")
            aperture2 = plt.Circle((centerCol, centerRow), 2 * rad, fill=False, color="g")
            plt.gca().add_patch(aperture)
            plt.gca().add_patch(aperture2)
            plt.show()
if __name__=='__main__':
    run = 'PAL2014'
    date = '20140924'
    tstamp = '20140925-112538'

    obsFN = FileName(run=run,date=date,tstamp=tstamp)
    obsPath = obsFN.obs()
     
    flatPath = FileName(run=run,date=date).flatSoln()
    hotPath = obsFN.timeMask()
    centroidPath = obsFN.centroidList()
    fluxPath = FileName(run='PAL2012',date='20121211',tstamp='absolute_021727').fluxSoln()
    
    obs = ObsFile(obsPath)
    if not os.path.exists(hotPath):
        hp.findHotPixels(obsPath,hotPath)
    obs.loadHotPixCalFile(hotPath,switchOnMask=False)
    obs.loadBestWvlCalFile()
    obs.loadFlatCalFile(flatPath)
    obs.loadFluxCalFile(fluxPath)
    obs.loadCentroidListFile(centroidPath)
    obs.setWvlCutoffs(3000,11000)

    
    centroidRa = '03:37:43.826'
    centroidDec = '14:15:14.828'
    haOffset = 150.

#    imgDict = obs.getPixelCountImage(integrationTime=60,scaleByEffInt=True)
#    plotArray(imgDict['image'])
    
flatCalFilenames[0] = '/Scratch/flatCalSolnFiles/20121207/flatsol_20121207.h5'
flatCalFilenames[1] = '/Scratch/flatCalSolnFiles/20121207/flatsol_20121207.h5'
fluxCalFileNames = ['/Scratch/fluxCalSolnFiles/20121206/fluxsol_20121207-124034.h5']

obsFn = FileName(run=run,date=sunsetDates[0],tstamp='20121209-120530').obs()
ob = ObsFile(obsFn)
print 'Loading wavelength calibration solution: ' + wvlCalFilenames[0]
ob.loadWvlCalFile(wvlCalFilenames[0])
print 'Loading flat calibration solution: ' + flatCalFilenames[0]
ob.loadFlatCalFile(flatCalFilenames[0])
ob.loadFluxCalFile(fluxCalFileNames[0])

#load/generate hot pixel mask file
HotPixFile = getTimeMaskFileName(obsFn)
if not os.path.exists(HotPixFile):
    hp.findHotPixels(obsFn,HotPixFile)
    print "Flux file pixel mask saved to %s"%(HotPixFile)
ob.loadHotPixCalFile(HotPixFile)
print "Hot pixel mask loaded %s"%(HotPixFile)

frame = ob.getPixelCountImage(firstSec=0,integrationTime=300,weighted=True)
#hotPixMask = hotPixels.checkInterval(image=frame, firstSec=0, intTime=300, weighted=True, display=False)['mask']

#summed_array,bin_edges=ob.getApertureSpectrum(pixelCol=14,pixelRow=8,radius=7)
ob.plotApertureSpectrum(pixelCol=14,pixelRow=8,radius=7,weighted = True,fluxWeighted=True,lowCut=3000,highCut=9000)


'''
h = 6.626068*10**-34
c = 299792458.0
k = 1.3806503*10**-23
def main():

    obsSequence1="""
    033323
    033825
    034327
    034830
    035332
    035834
    040336
    040838
    041341
    041843
    042346
    042848
    043351
    043853
    044355
    044857
    045359
    045902
    """

    obsSequence2="""
    050404
    050907
    051409
    051912
    052414
    052917
    053419
    053922
    """

    obsSequence3="""
    054926
    055428
    055930
    060432
    060934
    061436
    061938
    062440
    062942
    """

    run = 'PAL2012'
    obsSequences = [obsSequence1,obsSequence2,obsSequence3]
    wvlCals = ['063518','063518','063518']
    flatCals = ['20121211','20121211','20121211']
    fluxCalDates = ['20121206','20121206','20121206']
    fluxCals = ['20121207-072055','20121207-072055','20121207-072055']

    #Row coordinate of center of crab pulsar for each obsSequence
    centersRow = [29,29,10]
    #Col coordinate of center of crab pulsar for each obsSequence
    centersCol = [29,30,14]


    obsUtcDate = '20121212'
    obsUtcDates = ['20121212','20121212','20121212']

    obsFileNames = []
    obsFileNameTimestamps = []
    wvlFileNames = []
    flatFileNames = []
    fluxFileNames = []
    timeMaskFileNames = []
    plFileNames = []
    skyFileNames = []
    
    for iSeq in range(len(obsSequences)):
        obsSequence = obsSequences[iSeq]
        obsSequence = obsSequence.strip().split()
        obsFileNameTimestamps.append(obsSequence)
        obsUtcDate = obsUtcDates[iSeq]
        sunsetDate = str(int(obsUtcDate)-1)
        obsSequence = [obsUtcDates[iSeq]+'-'+ts for ts in obsSequence]
        obsFileNames.append([FileName(run=run,date=sunsetDate,tstamp=ts).obs() for ts in obsSequence])
        timeMaskFileNames.append([FileName(run=run,date=sunsetDate,tstamp=ts).timeMask() for ts in obsSequence])
        wvlCalTstamp = obsUtcDate+'-'+wvlCals[iSeq]
        wvlFileNames.append(FileName(run=run,date=sunsetDate,tstamp=wvlCalTstamp).calSoln())
        fluxFileNames.append(FileName(run=run,date=fluxCalDates[iSeq],tstamp=fluxCals[iSeq]).fluxSoln())
        flatFileNames.append(FileName(run=run,date=flatCals[iSeq],tstamp='').flatSoln())
        plFileNames.append([FileName(run=run,date=sunsetDate,tstamp=ts).crabList() for ts in obsSequence])
        skyFileNames.append([FileName(run=run,date=sunsetDate,tstamp=ts).crabSkyList() for ts in obsSequence])

    for iSeq,obsSequence in enumerate(obsSequences):
        obsSequence = obsSequence.strip().split()
        print obsSequence
        for iOb,obs in enumerate(obsSequence):
            timeMaskFileName = timeMaskFileNames[iSeq][iOb]
            if not os.path.exists(timeMaskFileName):
                print 'Running hotpix for ',obs
                hp.findHotPixels(obsFileNames[iSeq][iOb],timeMaskFileName)
                print "Flux file pixel mask saved to %s"%(timeMaskFileName)

    apertureRadius = 5
    obLists = [[ObsFile(fn)for fn in seq ] for seq in obsFileNames]
    tstampFormat = '%H:%M:%S'
    #print 'fileName','headerUnix','headerUTC','logUnix','packetReceivedUnixTime'
    for iSeq,obList in enumerate(obLists):
        for iOb,ob in enumerate(obList):
            print ob.fileName
            centerRow = centersRow[iSeq]
            centerCol = centersCol[iSeq]
            circCol,circRow = circ(centerCol,centerRow,radius=apertureRadius)
            skyCol,skyRow = halfTorus(centerCol,centerRow,radiusInner=apertureRadius,radiusOuter=2*apertureRadius)
            ob.loadTimeAdjustmentFile(FileName(run='PAL2012').timeAdjustments())
            ob.loadWvlCalFile(wvlFileNames[iSeq])
            ob.loadFlatCalFile(flatFileNames[iSeq])
            ob.loadFluxCalFile(fluxFileNames[iSeq])
            timeMaskFileName = timeMaskFileNames[iSeq][iOb]
            ob.loadHotPixCalFile(timeMaskFileName)
            ob.setWvlCutoffs(4000,11000)
            photlist.writePhotonList(ob,rowList=skyRow,colList=skyCol,filename=skyFileNames[iSeq][iOb])
            print 'wrote sky list'
            photlist.writePhotonList(ob,rowList=circRow,colList=circCol,filename=plFileNames[iSeq][iOb])
            print 'wrote crab list'
def main():
    """
    params = []
    paramfile = sys.argv[1]

    f = open(paramfile,'r')
    for line in f:
        params.append(line)
    f.close()
    datadir = params[0].split('=')[1].strip()
    flatdir = params[1].split('=')[1].strip()
    fluxdir = params[2].split('=')[1].strip()
    wvldir = params[3].split('=')[1].strip()
    obsfile = params[4].split('=')[1].strip()
    skyfile = params[5].split('=')[1].strip()
    flatfile = params[6].split('=')[1].strip()
    fluxfile = params[7].split('=')[1].strip()
    wvlfile = params[8].split('=')[1].strip()
    objectName = params[9].split('=')[1].strip()
    fluxCalObject = params[10].split('=')[1].strip()

    obsFileName = os.path.join(datadir, obsfile)
    skyFileName = os.path.join(datadir, skyfile)
    wvlCalFileName = os.path.join(wvldir, wvlfile)
    flatCalFileName = os.path.join(flatdir, flatfile)
    fluxCalFileName = os.path.join(fluxdir, fluxfile)
    """

    if len(sys.argv) >2:
        fileNum = str(sys.argv[2])
    else:
        fileNum = '0'

    #science object parameter file
    params = []
    paramfile = sys.argv[1]
    f = open(paramfile,'r')
    for line in f:
        params.append(line)
    f.close()

    datadir = params[0].split('=')[1].strip()
    flatdir = params[1].split('=')[1].strip()
    wvldir = params[2].split('=')[1].strip()
    obsfile = params[3].split('=')[1].strip()
    skyfile = params[4].split('=')[1].strip()
    flatfile = params[5].split('=')[1].strip()
    wvlfile = params[6].split('=')[1].strip()
    objectName = params[9].split('=')[1].strip()

    wvldir = "/Scratch/waveCalSolnFiles/oldbox_numbers/20121206"

    if len(params)>10:
        xpix = int(params[10].split('=')[1].strip())
        ypix = int(params[11].split('=')[1].strip())
        apertureRadius = int(params[12].split('=')[1].strip())
        startTime = int(params[13].split('=')[1].strip())
        intTime =int(params[14].split('=')[1].strip())

    obsFileName = os.path.join(datadir, obsfile)
    skyFileName = os.path.join(datadir, skyfile)
    wvlCalFileName = os.path.join(wvldir, wvlfile)
    flatCalFileName = os.path.join(flatdir, flatfile)

    obs = ObsFile(obsFileName)
    obs.loadWvlCalFile(wvlCalFileName)
    obs.loadFlatCalFile(flatCalFileName)
    print "analyzing file %s"%(obsFileName)
    print "loaded data file and calibrations\n---------------------\n"

    nRow = obs.nRow
    nCol = obs.nCol
    obsTime = obs.getFromHeader("exptime")
    #wvlBinEdges,obsSpectra = loadSpectra(obs,nCol,nRow)
    #nWvlBins=len(wvlBinEdges)-1

    #print np.shape(obsSpectra)
    #print nRow
    #print nCol
    #print nWvlBins

    #load/generate hot pixel mask file
    HotPixFile = getTimeMaskFileName(obsFileName)
    if not os.path.exists(HotPixFile):
        hp.findHotPixels(obsFileName,HotPixFile)
        print "Flux file pixel mask saved to %s"%(HotPixFile)
    obs.loadHotPixCalFile(HotPixFile)
    print "Hot pixel mask loaded %s"%(HotPixFile)

    #######
    #EVERYTHING BEFORE HERE IS STANDARD FILE/CALFILE LOADING

    
    startWvl = 3000
    #stopWvl = 7000 #for V-band
    stopWvl = 9000 #for R-band

    print "Making spectral cube"
    #for pg0220 first sec should be 80 since object is moving around before this
    #for pg0220A first sec should be 70, integration time is 140
    #for landolt 9542 first sec should be 20, int time is -1
    cubeDict = obs.getSpectralCube(firstSec=startTime, integrationTime=intTime, wvlStart = startWvl, wvlStop = stopWvl, wvlBinEdges = [startWvl,stopWvl], weighted=False)

    cube= np.array(cubeDict['cube'], dtype=np.double)
    wvlBinEdges= cubeDict['wvlBinEdges']
    effIntTime= cubeDict['effIntTime']
    print "median effective integration time = ", np.median(effIntTime)

    nWvlBins=len(wvlBinEdges)-1
    print "cube shape ", np.shape(cube)
    print "effIntTime shape ", np.shape(effIntTime)

    #add third dimension to effIntTime for  broadcasting
    effIntTime = np.reshape(effIntTime,np.shape(effIntTime)+(1,))
    cube /= effIntTime #put cube into counts/s
    
    #calculate midpoints of wvl bins for plotting
    wvls = np.empty((nWvlBins),dtype=float)
    for n in xrange(nWvlBins):
        binsize=wvlBinEdges[n+1]-wvlBinEdges[n]
        wvls[n] = (wvlBinEdges[n]+(binsize/2.0))

    print "wvls ",wvls
    #reshape cube for makeMovie
    movieCube = np.zeros((nWvlBins,np.shape(cube)[0],np.shape(cube)[1]),dtype=float)
    for i in xrange(nWvlBins):
        movieCube[i,:,:] = cube[:,:,i]
        #show individual frames as they are made to debug
        #plt.matshow(movieCube[i],vmin = 0, vmax = 100)
        #plt.show()
    print "movieCube shape ", np.shape(movieCube)
    print "wvls shape ", np.shape(wvls)

    #print cube
    #print "--------------------------"
    #print movieCube

    print "adding frames with wvl below ", stopWvl
    finalframe = np.zeros((1,np.shape(movieCube)[1],np.shape(movieCube)[2]))
    for f in xrange(len(wvls[wvls<stopWvl])):
        print wvls[f]
        finalframe[0]+=movieCube[f]
        plt.matshow(movieCube[f],vmin=0,vmax = 40)
        plt.show()

    movieCube = finalframe

    np.savez('%s_%s.npz'%(objectName,fileNum),stack=movieCube,wvls=wvls)
    print "Saved frame to .npz file"
    
    plt.matshow(movieCube[0],vmin=0,vmax = 40)
    plt.show()
    def __init__(self,paramFile,plots=False,verbose=False):
        """
        Opens flux file, prepares standard spectrum, and calculates flux factors for the file.
        Method is provided in param file. If 'relative' is selected, an obs file with standard star defocused over
        the entire array is expected, with accompanying sky file to do sky subtraction.
        If any other method is provided, 'absolute' will be done by default, wherein a point source is assumed
        to be present. The obs file is then broken into spectral frames with photometry (psf or aper) performed 
        on each frame to generate the ARCONS observed spectrum.
        """
        self.verbose=verbose
        self.plots = plots

        self.params = readDict()
        self.params.read_from_file(paramFile)
        
        run = self.params['run']
        sunsetDate = self.params['fluxSunsetLocalDate']
        self.fluxTstamp = self.params['fluxTimestamp']
        skyTstamp = self.params['skyTimestamp']
        wvlSunsetDate = self.params['wvlCalSunsetLocalDate']
        wvlTimestamp = self.params['wvlCalTimestamp']
        flatCalFileName = self.params['flatCalFileName']
        needTimeAdjust = self.params['needTimeAdjust']
        self.deadtime = float(self.params['deadtime']) #from firmware pulse detection
        self.timeSpacingCut = self.params['timeSpacingCut']
        bLoadBeammap = self.params.get('bLoadBeammap',False)
        self.method = self.params['method']
        self.objectName = self.params['object']
        self.r = float(self.params['energyResolution'])
        self.photometry = self.params['photometry']
        self.centroidRow = self.params['centroidRow']
        self.centroidCol = self.params['centroidCol']
        self.aperture = self.params['apertureRad']
        self.annulusInner = self.params['annulusInner']
        self.annulusOuter = self.params['annulusOuter']
        self.collectingArea = self.params['collectingArea']
        self.startTime = self.params['startTime']
        self.intTime = self.params['integrationTime']

        fluxFN = FileName(run=run,date=sunsetDate,tstamp=self.fluxTstamp)
        self.fluxFileName = fluxFN.obs()
        self.fluxFile = ObsFile(self.fluxFileName)

        if self.plots:
            self.plotSavePath = os.environ['MKID_PROC_PATH']+os.sep+'fluxCalSolnFiles'+os.sep+run+os.sep+sunsetDate+os.sep+'plots'+os.sep
            if not os.path.exists(self.plotSavePath): os.mkdir(self.plotSavePath)
            if self.verbose: print "Created directory %s"%self.plotSavePath

        obsFNs = [fluxFN]
        self.obsList = [self.fluxFile]

        if self.startTime in ['',None]: self.startTime=0
        if self.intTime in ['',None]: self.intTime=-1

        if self.method=="relative":
            try:
                print "performing Relative Flux Calibration"
                skyFN = FileName(run=run,date=sunsetDate,tstamp=skyTstamp)
                self.skyFileName = skyFN.obs()
                self.skyFile = ObsFile(self.skyFileName)
                obsFNs.append(skyFN)
                self.obsList.append(self.skyFile)
            except:
                print "For relative flux calibration a sky file must be provided in param file"
                self.__del__()
        else:
            self.method='absolute'
            print "performing Absolute Flux Calibration"

        if self.photometry not in ['aperture','PSF']: self.photometry='PSF' #default to PSF fitting if no valid photometry selected

        timeMaskFileNames = [fn.timeMask() for fn in obsFNs]
        timeAdjustFileName = FileName(run=run).timeAdjustments()

        #make filename for output fluxCalSoln file
        self.fluxCalFileName = FileName(run=run,date=sunsetDate,tstamp=self.fluxTstamp).fluxSoln()
        print "Creating flux cal: %s"%self.fluxCalFileName

        if wvlSunsetDate != '':
            wvlCalFileName = FileName(run=run,date=wvlSunsetDate,tstamp=wvlTimestamp).calSoln()
        if flatCalFileName =='':
            flatCalFileName=FileName(obsFile=self.fluxFile).flatSoln()

        #load cal files for flux file and, if necessary, sky file
        for iObs,obs in enumerate(self.obsList):
            if bLoadBeammap:
                print 'loading beammap',os.environ['MKID_BEAMMAP_PATH']
                obs.loadBeammapFile(os.environ['MKID_BEAMMAP_PATH'])
            if wvlSunsetDate != '':
                obs.loadWvlCalFile(wvlCalFileName)
            else:
                obs.loadBestWvlCalFile()

            obs.loadFlatCalFile(flatCalFileName)
            obs.setWvlCutoffs(-1,-1)

            if needTimeAdjust:
                obs.loadTimeAdjustmentFile(timeAdjustFileName)
            timeMaskFileName = timeMaskFileNames[iObs]
            print timeMaskFileName

            if not os.path.exists(timeMaskFileName):
                print 'Running hotpix for ',obs
                hp.findHotPixels(obsFile=obs,outputFileName=timeMaskFileName,fwhm=np.inf,useLocalStdDev=True)
                print "Flux cal/sky file pixel mask saved to %s"%(timeMaskFileName)
            obs.loadHotPixCalFile(timeMaskFileName)
            if self.verbose: print "Loaded hot pixel file %s"%timeMaskFileName

        #get flat cal binning information since flux cal will need to match it
        self.wvlBinEdges = self.fluxFile.flatCalFile.root.flatcal.wavelengthBins.read()
        self.nWvlBins = self.fluxFile.flatWeights.shape[2]
        self.binWidths = np.empty((self.nWvlBins),dtype=float)
        self.binCenters = np.empty((self.nWvlBins),dtype=float)
        for i in xrange(self.nWvlBins):
            self.binWidths[i] = self.wvlBinEdges[i+1]-self.wvlBinEdges[i]
            self.binCenters[i] = (self.wvlBinEdges[i]+(self.binWidths[i]/2.0))

        if self.method=='relative':
            print "Extracting ARCONS flux and sky spectra"
            self.loadRelativeSpectrum()
            print "Flux Spectrum loaded"
            self.loadSkySpectrum()
            print "Sky Spectrum loaded"
        elif self.method=='absolute':
            print "Extracting ARCONS point source spectrum"
            self.loadAbsoluteSpectrum()

        print "Loading standard spectrum"
        try:
            self.loadStdSpectrum(self.objectName)
        except KeyError:
            print "Invalid spectrum object name"
            self.__del__()
            sys.exit()

        print "Generating sensitivity curve"
        self.calculateFactors()
        print "Sensitivity Curve calculated"
        print "Writing fluxCal to file %s"%self.fluxCalFileName
        self.writeFactors(self.fluxCalFileName)
        
        if self.plots: self.makePlots()

        print "Done"
def main(testRun=False):

    #INPUTS:
    #    - testRun :    If True, run only a single exposure.

    #Try remapping the pixels - set to None to not do this....
    pixRemapFileName = FileName(run='PAL2012').pixRemap()

    obsSequence0="""
    051516
    052018
    052520
    """
    obsSequence1="""
    033323
    033825
    034327
    034830
    035332
    035834
    040336
    040838
    041341
    041843
    042346
    042848
    043351
    043853
    044355
    044857
    045359
    045902
    """

    obsSequence2="""
    050404
    050907
    051409
    051912
    052414
    052917
    053419
    053922
    """
    # 054424 - repointed mid-exposure, leave out for now.    
    

    obsSequence3="""
    054926
    055428
    055930
    060432
    060934
    061436
    061938
    062440
    062942
    """

    run = 'PAL2012'
    obsSequences = [obsSequence0,obsSequence1,obsSequence2,obsSequence3]
    
    #If test run is requested, override and just run one test exposure.
    if testRun is True:
        obsSequences = ['9999999',
                        '''
                        033323
                        ''',
                        '9999999',
                        '9999999']
    
    wvlCals = ['051341','063518','063518','063518']
    flatCals = ['20121211','20121211','20121211','20121211']
    #fluxCalDates = ['20121206','20121206','20121206','20121206']
    #fluxCals = ['20121207-072055','20121207-072055','20121207-072055','20121207-072055']
    fluxCalDates = ['20121211','20121211','20121211','20121211']
    fluxCals = ['absolute_021727','absolute_021727','absolute_021727','absolute_021727']

    #Row coordinate of center of crab pulsar for each obsSequence
    centersRow = [9,30,29,10]
    #Col coordinate of center of crab pulsar for each obsSequence
    centersCol = [13,30,30,14]

    centerRA = '05:34:31.93830'
    centerDec = '+22:00:52.1758'

    obsUtcDate = '20121212'
    obsUtcDates = ['20121206','20121212','20121212','20121212']

    obsFileNames = []
    obsFileNameTimestamps = []
    wvlFileNames = []
    flatFileNames = []
    fluxFileNames = []
    timeMaskFileNames = []
    centroidFileNames = []
    
    for iSeq in range(len(obsSequences)):
        obsSequence = obsSequences[iSeq]
        obsSequence = obsSequence.strip().split()
        obsFileNameTimestamps.append(obsSequence)
        obsUtcDate = obsUtcDates[iSeq]
        sunsetDate = str(int(obsUtcDate)-1)
        obsSequence = [obsUtcDates[iSeq]+'-'+ts for ts in obsSequence]
        obsFileNames.append([FileName(run=run,date=sunsetDate,tstamp=ts).obs() for ts in obsSequence])
        timeMaskFileNames.append([FileName(run=run,date=sunsetDate,tstamp=ts).timeMask() for ts in obsSequence])
        centroidFileNames.append([FileName(run=run,date=sunsetDate,tstamp=ts).centroidList() for ts in obsSequence])

        wvlCalTstamp = obsUtcDate+'-'+wvlCals[iSeq]
        wvlFileNames.append(FileName(run=run,date=sunsetDate,tstamp=wvlCalTstamp).calSoln())
        fluxFileNames.append(FileName(run=run,date=fluxCalDates[iSeq],tstamp=fluxCals[iSeq]).fluxSoln())
        flatFileNames.append(FileName(run=run,date=flatCals[iSeq]).illumSoln()) # *** CURRENTLY USES ILLUMINATION SOLUTION! ***
        centroidFileNames.append(FileName(run=run,date=sunsetDate,tstamp=ts).centroidList())


#     #Make hot pixel masks if needed
#     for iSeq,obsSequence in enumerate(obsSequences):
#         obsSequence = obsSequence.strip().split()
#         print obsSequence
#         for iOb,obs in enumerate(obsSequence):
#             timeMaskFileName = timeMaskFileNames[iSeq][iOb]
#             if os.path.exists(obsFileNames[iSeq][iOb]) and not os.path.exists(timeMaskFileName):
#                 print 'Running hotpix for ',obs
#                 hp.findHotPixels(obsFileNames[iSeq][iOb],timeMaskFileName)
#                 print "Flux file pixel mask saved to %s"%(timeMaskFileName)
#             else:
#                 print 'Skipping hot pixel mask creation for file '+obsFileNames[iSeq][iOb]


    #apertureRadius = 4
    obLists = [[ObsFile(fn)for fn in seq if os.path.exists(fn)] for seq in obsFileNames]
    tstampFormat = '%H:%M:%S'
    #print 'fileName','headerUnix','headerUTC','logUnix','packetReceivedUnixTime'
   
    print '---------Making hot pixel masks/getting centroids-----------' 
    for iSeq,obList in enumerate(obLists):
        for iOb,ob in enumerate(obList):
            timeAdjFileName = FileName(run='PAL2012').timeAdjustments()
            wvlCalFileName = wvlFileNames[iSeq]
            flatCalFileName = flatFileNames[iSeq]
            fluxCalFileName = fluxFileNames[iSeq]
            timeMaskFileName = FileName(obsFile=ob).timeMask() #timeMaskFileNames[iSeq][iOb]
            centroidFileName = FileName(obsFile=ob).centroidList()
            if not os.path.exists(centroidFileName) or not os.path.exists(timeMaskFileName):
                print 'Loading calibration files:'
                print [os.path.basename(x) for x in [timeAdjFileName,wvlCalFileName,flatCalFileName, \
                        fluxCalFileName,timeMaskFileName, centroidFileName]]
                ob.loadTimeAdjustmentFile(timeAdjFileName)
                ob.loadWvlCalFile(wvlCalFileName)
                ob.loadFlatCalFile(flatCalFileName)
                ob.loadFluxCalFile(fluxCalFileName)
                ob.setWvlCutoffs(3000.,12000.) #Keep most of the wavelength range for the purposes of centroiding/hot pixel detection.

                
            if not os.path.exists(timeMaskFileName):
                print 'Running hotpix for ',ob.fileName
                #Run hot pixel detection on *calibrated* file.
                #But don't use flux weighting, as this scales by a large amount, and throws
                #the poisson statistics way too much. (i.e., we really need to use
                #counts *detected* for shot noise statistics, rather than 
                #estimated counts above the atmosphere....)
                hp.findHotPixels(obsFile=ob,outputFileName=timeMaskFileName,
                                 useRawCounts=False,weighted=True,fluxWeighted=False)
                print "Flux file pixel mask saved to %s"%(timeMaskFileName)
            else:
                print 'Skipping hot pixel mask creation for file '+obsFileNames[iSeq][iOb]
            if not os.path.exists(centroidFileName):
                ob.loadHotPixCalFile(timeMaskFileName)
                print 'Running CentroidCalc for ',ob.fileName
                cc.centroidCalc(ob, centerRA, centerDec, outputFileName=centroidFileName,
                                guessTime=300, integrationTime=30, secondMaxCountsForDisplay=500,
                                xyapprox=[centersCol[iSeq],centersRow[iSeq]])
                print "Centroiding calculations saved to %s"%(centroidFileName)
            else:
                print "Centroid file already exists - skipping: "+centroidFileName
def main():
    """
    params = []
    paramfile = sys.argv[1]

    f = open(paramfile,'r')
    for line in f:
        params.append(line)
    f.close()
    datadir = params[0].split('=')[1].strip()
    flatdir = params[1].split('=')[1].strip()
    fluxdir = params[2].split('=')[1].strip()
    wvldir = params[3].split('=')[1].strip()
    obsfile = params[4].split('=')[1].strip()
    skyfile = params[5].split('=')[1].strip()
    flatfile = params[6].split('=')[1].strip()
    fluxfile = params[7].split('=')[1].strip()
    wvlfile = params[8].split('=')[1].strip()
    objectName = params[9].split('=')[1].strip()
    fluxCalObject = params[10].split('=')[1].strip()

    obsFileName = os.path.join(datadir, obsfile)
    skyFileName = os.path.join(datadir, skyfile)
    wvlCalFileName = os.path.join(wvldir, wvlfile)
    flatCalFileName = os.path.join(flatdir, flatfile)
    fluxCalFileName = os.path.join(fluxdir, fluxfile)
    """

    if len(sys.argv) >3:
        filenum = str('_'+sys.argv[3])
    else:
        filenum = '_0'

    #science object parameter file
    params = []
    paramfile = sys.argv[1]
    f = open(paramfile,'r')
    for line in f:
        params.append(line)
    f.close()

    datadir = params[0].split('=')[1].strip()
    flatdir = params[1].split('=')[1].strip()
    wvldir = params[2].split('=')[1].strip()
    obsfile = params[3].split('=')[1].strip()
    skyfile = params[4].split('=')[1].strip()
    flatfile = params[5].split('=')[1].strip()
    wvlfile = params[6].split('=')[1].strip()
    objectName = params[9].split('=')[1].strip()

    if len(params)>10:
        xpix = int(params[10].split('=')[1].strip())
        ypix = int(params[11].split('=')[1].strip())
        apertureRadius = int(params[12].split('=')[1].strip())


    #flux cal object parameter file

    params2 = []
    param2file = sys.argv[2]
    f = open(param2file,'r')
    for line in f:
        params2.append(line)
    f.close()

    fluxdir = params2[7].split('=')[1].strip()
    fluxfile = params2[8].split('=')[1].strip()
    fluxCalObject = params2[9].split('=')[1].strip()

    obsFileName = os.path.join(datadir, obsfile)
    skyFileName = os.path.join(datadir, skyfile)
    wvlCalFileName = os.path.join(wvldir, wvlfile)
    flatCalFileName = os.path.join(flatdir, flatfile)
    fluxCalFileName = os.path.join(fluxdir, fluxfile)

    print "obsfile = ",obsFileName
    print "skyfile = ",skyFileName
    print "wvlcal = ", wvlCalFileName
    print "flatcal = ", flatCalFileName
    print "fluxcal = ", fluxCalFileName
    print "object = ", objectName
    print "flux cal object = ", fluxCalObject
    print "\n---------------------\n"

    obs = ObsFile(obsFileName)
    obs.loadWvlCalFile(wvlCalFileName)
    obs.loadFlatCalFile(flatCalFileName)
    obs.loadFluxCalFile(fluxCalFileName)
    print "loaded data file and calibrations\n---------------------\n"

    nRow = obs.nRow
    nCol = obs.nCol
    obsTime = obs.getFromHeader("exptime")
    #wvlBinEdges,obsSpectra = loadSpectra(obs,nCol,nRow)
    #nWvlBins=len(wvlBinEdges)-1

    #print np.shape(obsSpectra)
    #print nRow
    #print nCol
    #print nWvlBins

    """
    medianObsSpectrum = calculateMedian(obsSpectra,nCol,nRow,nWvlBins)
    print "target spectrum loaded\n---------------------\n"

    if skyfile != "None":
        sky = ObsFile(skyFileName)
        sky.loadWvlCalFile(wvlCalFileName)
        sky.loadFlatCalFile(flatCalFileName)
        sky.loadFluxCalFile(fluxCalFileName)
        skyTime = sky.getFromHeader("exptime")
        wvlBinEdges,skySpectra = loadSpectra(sky,nCol,nRow)
        skySpectrum = calculateMedian(skySpectra, nCol, nRow, nWvlBins)
        skySpectrum = skySpectrum*float(obsTime)/float(skyTime) #scale sky spectrum to target observation time
        print "sky spectrum loaded\n---------------------\n"
    else:
        #if no sky file given, estimate sky spectrum as median spectrum of obs file, assuming object is tiny
        skySpectrum = calculateMedian(obsSpectra, nCol, nRow, nWvlBins)
        print "sky spectrum estimated as median of target file spectrum\n---------------------\n"

    #subtract sky spectrum from every pixel
    allSkySpectrum = obsSpectra-skySpectrum
    #set any negative values to 0 after sky subtraction
    allSkySpectrum[allSkySpectrum<0]=0

    #take median of remaining sky subtracted spectra to get median object spectrum
    finalSpectrum = calculateMedian(allSkySpectrum,nCol,nRow,nWvlBins)
    """

    #load/generate hot pixel mask file
    HotPixFile = getTimeMaskFileName(obsFileName)
    if not os.path.exists(HotPixFile):
        hp.findHotPixels(obsFileName,HotPixFile)
        print "Flux file pixel mask saved to %s"%(HotPixFile)
    obs.loadHotPixCalFile(HotPixFile)
    print "Hot pixel mask loaded %s"%(HotPixFile)

    print "Making spectrum with Aperture Spectrum in ObsFile"
    #use Aperture Spectrum from obsfile
    medianObsSpectrum, wvlBinEdges = obs.getApertureSpectrum(pixelCol=ypix,pixelRow=xpix,radius1=apertureRadius, radius2 = apertureRadius*2.0,weighted=True, fluxWeighted=True, lowCut=3000, highCut=7000)

    nWvlBins=len(wvlBinEdges)-1

    #load std spectrum for comparison
    try:
        realSpectra = loadStd(objectName,wvlBinEdges)
        print "real std spectrum loaded for reference\n---------------------\n"
        stdTitle = "Rebinned Std Spectrum of %s"%(objectName)
    except KeyError:
        print "Key Error loading MKIDStd"
        realSpectra = np.ones(nWvlBins)
        stdTitle = "No MKIDStd spectrum available for %s"%(objectName)

    #create plots
    plotDir = "/home/srmeeker/ARCONS-pipeline/fluxcal/test/plots"
    plotFileName = "%s_from_%s%s.pdf"%(objectName,fluxCalObject,filenum)
    fullFluxPlotFileName = os.path.join(plotDir,plotFileName)
    pp = PdfPages(fullFluxPlotFileName)
    matplotlib.rcParams['font.size']=6

    #calculate midpoints of wvl bins for plotting
    wvls = np.empty((nWvlBins),dtype=float)
    for n in xrange(nWvlBins):
        binsize=wvlBinEdges[n+1]-wvlBinEdges[n]
        wvls[n] = (wvlBinEdges[n]+(binsize/2.0))

    plt.figure()

    ax1 = plt.subplot(221)
    ax1.set_title('ARCONS median flat/flux cal\'d obs in counts')
    ax1.set_xlim((4000,11000))
    ax1.set_ylim((min(medianObsSpectrum[(wvls>4000) & (wvls<11000)]),max(medianObsSpectrum[(wvls>4000) & (wvls<8000)])))
    plt.plot(wvls,medianObsSpectrum)
    #plt.show()
    #ax2 = plt.subplot(232)
    #ax2.set_title('ARCONS median flat/flux cal\'d sky in counts')
    #plt.plot(wvls,skySpectrum)
    #plt.show()
    ax5 = plt.subplot(223)
    ax5.set_title('Sensitivity Spectrum')
    ax5.set_xlim((3000,13000))
    ax5.set_ylim((0,5))
    plt.plot(wvls,obs.fluxWeights)
    #ax3 = plt.subplot(234)
    #ax3.set_title('MKID data minus sky in counts')
    #plt.plot(wvls,finalSpectrum/max(finalSpectrum))
    ax4 = plt.subplot(222)
    ax4.set_title(stdTitle)
    plt.plot(wvls,realSpectra)


    #ax7 = plt.subplot(337)
    #ax7.set_title('Flux Cal\'d ARCONS Spectrum of Std')
    #plt.plot(wvls,fluxFactors*subtractedSpectra)

    pp.savefig()
    pp.close()

    #del obs
    #del sky

    print "output plots to %s\n---------------------\n"%(fullFluxPlotFileName)

    txtDir = "/home/srmeeker/ARCONS-pipeline/fluxcal/test/txt"
    txtFileName = "%s_from_%s%s.txt"%(objectName,fluxCalObject,filenum)
    fullFluxTxtFileName = os.path.join(txtDir,txtFileName)

    outarr = np.empty((len(medianObsSpectrum),2),dtype=float)
    outarr[:,0]=wvls
    outarr[:,1]=medianObsSpectrum
    #save sensitivity spectrum to file
    np.savetxt(fullFluxTxtFileName, outarr)
    
    print "output txt file to %s\n---------------------\n"%(fullFluxPlotFileName)
def main():

    #open the sky file for hr9087
    run = 'PAL2012'
    date = '20121210'
    wvlCal = '20121211-052230'
    obsTimestamp = '20121211-051650'
    flatCalDate = '20121211'
    flatCalTstamp = '20121212-074700'
    obsFN = FileName(run=run,date=date,tstamp=obsTimestamp)
    obsFileName = obsFN.obs()
    timeMaskFileName = obsFN.timeMask()
    wvlFileName = FileName(run=run,date=date,tstamp=wvlCal).calSoln()
    flatFileName = FileName(run=run,date=flatCalDate,tstamp=flatCalTstamp).flatSoln()
    
    if not os.path.exists(timeMaskFileName):
        print 'Running hotpix for ',obsFileName
        hp.findHotPixels(obsFileName,timeMaskFileName)
        print "Flux file pixel mask saved to %s"%(timeMaskFileName)

    obs = ObsFile(obsFileName)
    obs.loadTimeAdjustmentFile(FileName(run='PAL2012').timeAdjustments())
    obs.loadWvlCalFile(wvlFileName)
    obs.loadFlatCalFile(flatFileName)
    obs.loadHotPixCalFile(timeMaskFileName)

    #obs.setWvlCutoffs(4000,8000)

    #get image before and after flat cal
    print 'getting images'
    beforeImgDict = obs.getPixelCountImage(weighted=False,fluxWeighted=False,scaleByEffInt=True)

    rawCubeDict = obs.getSpectralCube(weighted=False)
    rawCube = np.array(rawCubeDict['cube'],dtype=np.double)
    effIntTime = rawCubeDict['effIntTime']
    maxIntTime = np.max(effIntTime)
    #add third dimension for broadcasting
    effIntTime3d = np.reshape(effIntTime,np.shape(effIntTime)+(1,))
    rawCube *= maxIntTime / effIntTime3d
    rawCube[np.isnan(rawCube)] = 0
    rawCube[rawCube == np.inf] = 0
    beforeImg = np.sum(rawCube,axis=-1)
    print 'finished first cube'

    flatCubeDict = obs.getSpectralCube(weighted=True)
    flatCube = np.array(flatCubeDict['cube'],dtype=np.double)
    effIntTime = flatCubeDict['effIntTime']
    maxIntTime = np.max(effIntTime)
    #add third dimension for broadcasting
    effIntTime3d = np.reshape(effIntTime,np.shape(effIntTime)+(1,))
    flatCube *= maxIntTime / effIntTime3d
    flatCube[np.isnan(flatCube)] = 0
    flatCube[flatCube == np.inf] = 0
    afterImg = np.sum(flatCube,axis=-1)

    plotArray(title='before flatcal',image=beforeImg)
    plotArray(title='after flatcal',image=afterImg)

    print 'before sdev',np.std(beforeImg[afterImg!=0])
    print 'after sdev',np.std(afterImg[afterImg!=0])

    np.savez('flatCubeGem.npz',beforeImg=beforeImg,afterImg=afterImg,rawCube=rawCube,flatCube=flatCube)
    #date = '20140925'
    #obsTimestamp ='20140926-104528' #psr J0337


    obsFN = FileName(run=run,date=date,tstamp=obsTimestamp)
    obsFileName = obsFN.obs()
    print obsFileName
    timeMaskFileName = obsFN.timeMask()
    if wvlCal != '':
        wvlFileName = FileName(run=run,date=date,tstamp=wvlCal).calSoln()
    flatFileName = FileName(run=run,date=flatCalDate,tstamp=flatCalTstamp).flatSoln()
    illumFileName = FileName(run=run,date=flatCalDate,tstamp=flatCalTstamp).illumSoln()
    
    if not os.path.exists(timeMaskFileName):
        print 'Running hotpix for ',obsFileName
        hp.findHotPixels(inputFileName=obsFileName,outputFileName=timeMaskFileName)
        print "Flux file pixel mask saved to %s"%(timeMaskFileName)


    obs = ObsFile(obsFileName)
    exptime = obs.getFromHeader('exptime')
    #obs.loadTimeAdjustmentFile(FileName(run='PAL2012').timeAdjustments())
    if wvlCal != '':
        obs.loadWvlCalFile(wvlFileName)
    else:
        obs.loadBestWvlCalFile()
    obs.loadHotPixCalFile(timeMaskFileName)

    obs.setWvlCutoffs(lowerWvlCut,upperWvlCut)

    obs.loadFlatCalFile(flatFileName)
    def __init__(self,paramFile):
        """
        opens flat file,sets wavelength binnning parameters, and calculates flat factors for the file
        """
        self.params = readDict()
        self.params.read_from_file(paramFile)

        run = self.params['run']
        sunsetDate = self.params['sunsetDate']
        flatTstamp = self.params['flatTstamp']
        wvlSunsetDate = self.params['wvlSunsetDate']
        wvlTimestamp = self.params['wvlTimestamp']
        obsSequence = self.params['obsSequence']
        needTimeAdjust = self.params['needTimeAdjust']
        self.deadtime = self.params['deadtime'] #from firmware pulse detection
        self.intTime = self.params['intTime']
        self.timeSpacingCut = self.params['timeSpacingCut']
        self.nSigmaClip = self.params['nSigmaClip']
        self.nNearest = self.params['nNearest']

        obsFNs = [FileName(run=run,date=sunsetDate,tstamp=obsTstamp) for obsTstamp in obsSequence]
        self.obsFileNames = [fn.obs() for fn in obsFNs]
        self.obsList = [ObsFile(obsFileName) for obsFileName in self.obsFileNames]
        timeMaskFileNames = [fn.timeMask() for fn in obsFNs]
        timeAdjustFileName = FileName(run=run).timeAdjustments()

        print len(self.obsFileNames), 'flat files to co-add'
        self.flatCalFileName = FileName(run=run,date=sunsetDate,tstamp=flatTstamp).illumSoln()
        if wvlSunsetDate != '':
            wvlCalFileName = FileName(run=run,date=wvlSunsetDate,tstamp=wvlTimestamp).calSoln()
        for iObs,obs in enumerate(self.obsList):
            if wvlSunsetDate != '':
                obs.loadWvlCalFile(wvlCalFileName)
            else:
                obs.loadBestWvlCalFile()
            if needTimeAdjust:
                obs.loadTimeAdjustmentFile(timeAdjustFileName)
            timeMaskFileName = timeMaskFileNames[iObs]
            print timeMaskFileName
            #Temporary step, remove old hotpix file
            #if os.path.exists(timeMaskFileName):
            #    os.remove(timeMaskFileName)
            if not os.path.exists(timeMaskFileName):
                print 'Running hotpix for ',obs
                hp.findHotPixels(self.obsFileNames[iObs],timeMaskFileName,fwhm=np.inf,useLocalStdDev=True)
                print "Flux file pixel mask saved to %s"%(timeMaskFileName)
            obs.loadHotPixCalFile(timeMaskFileName)
        self.wvlFlags = self.obsList[0].wvlFlagTable

        self.nRow = self.obsList[0].nRow
        self.nCol = self.obsList[0].nCol
        print 'files opened'
        #self.wvlBinWidth = params['wvlBinWidth'] #angstroms
        self.energyBinWidth = self.params['energyBinWidth'] #eV
        self.wvlStart = self.params['wvlStart'] #angstroms
        self.wvlStop = self.params['wvlStop'] #angstroms
        self.wvlBinEdges = ObsFile.makeWvlBins(self.energyBinWidth,self.wvlStart,self.wvlStop)
        self.intTime = self.params['intTime']
        self.countRateCutoff = self.params['countRateCutoff']
        self.fractionOfChunksToTrim = self.params['fractionOfChunksToTrim']
        #wvlBinEdges includes both lower and upper limits, so number of bins is 1 less than number of edges
        self.nWvlBins = len(self.wvlBinEdges)-1
def testWritePhotonList(outputFileName=None,firstSec=0,integrationTime=-1,doPixRemap=False):
    '''
    Test run of obsFile.writePhotonList. fileName can be used
    to specify the output file name. If not specified, default
    name/location is used.
    
    Now includes test for pixel remapping....
    '''

    #Details of example obs file to run test on.
#    run = 'PAL2012'
#    date = '20121207'
#    tstamp = '20121208-074649'
#    calTstamp='20121208-070505'
#    fluxTstamp='20121208-133002'

    run = 'PAL2013'
    date = '20131209'
    tstamp = '20131209-115553'
    centroidTstamp = tstamp
    calTstamp='20131209-132225'
    fluxTstamp='ones'
    flatTstamp='20131209'
    if doPixRemap==True:
        pixRemapFileName = FileName(run=run).pixRemap()
    else:
        pixRemapFileName = None
    
    #Load up the obs file
    obsFileName = FileName(run=run, date=date, tstamp=tstamp)
    obsFile = ObsFile(obsFileName.obs())

    obsFile.loadWvlCalFile(FileName(run=run,date=date,tstamp=calTstamp).calSoln())
    obsFile.loadFlatCalFile(FileName(run=run,date=date,tstamp=flatTstamp).flatSoln())
    obsFile.loadFluxCalFile(FileName(run=run,date=date,tstamp=fluxTstamp).fluxSoln())

    HotPixFile = FileName(run=run,date=date,tstamp=tstamp).timeMask()
    if not os.path.exists(HotPixFile): #check if hot pix file already exists
        hp.findHotPixels(obsFileName.obs(),HotPixFile)
        print "Flux file pixel mask saved to %s"%(HotPixFile)
    obsFile.loadHotPixCalFile(HotPixFile)
    print "Hot pixel mask loaded %s"%(HotPixFile)   

    #hz21 location alpha(2000) = 12h 13m 56.42s , delta(2000) = +32d 56' 30.8''
    centroid_RA = '12:13:56.42'
    centroid_DEC = '32:56:30.8'
    CentFile = FileName(run=run,date=date,tstamp=tstamp).centroidList()
    #print "checking centroid file", CentFile
    #if not os.path.exists(CentFile): #check if cent pix file already exists
    cent.centroidCalc(obsFile, centroid_RA, centroid_DEC, outputFileName = CentFile, guessTime=10, integrationTime=10)
    #    print "Flux file centroid cal saved to %s"%(CentFile)
    obsFile.loadCentroidListFile(CentFile)
    print "Centroid File loaded %s"%(CentFile) 

    #Load up associated calibrations

    #obsFile.loadTimeAdjustmentFile(FileName(run=run,date=date,tstamp=tstamp).timeAdjustments())
    #obsFile.loadHotPixCalFile(FileName(run=run,date=date,tstamp=tstamp).timeMask())
    #obsFile.loadCentroidListFile(FileName(run=run,date=date,tstamp=tstamp).centroidList())
    
    #And write out the results....
    obsFile.writePhotonList(outputFileName,firstSec,integrationTime,
                            pixRemapFileName=pixRemapFileName)
    
    #Read the results back in....
    #photFile = photList.PhotFile(outputFilename)
    
    print "Wrote photon list file: ", outputFileName
def main():
    """
    params = []
    paramfile = sys.argv[1]

    f = open(paramfile,'r')
    for line in f:
        params.append(line)
    f.close()
    datadir = params[0].split('=')[1].strip()
    flatdir = params[1].split('=')[1].strip()
    fluxdir = params[2].split('=')[1].strip()
    wvldir = params[3].split('=')[1].strip()
    obsfile = params[4].split('=')[1].strip()
    skyfile = params[5].split('=')[1].strip()
    flatfile = params[6].split('=')[1].strip()
    fluxfile = params[7].split('=')[1].strip()
    wvlfile = params[8].split('=')[1].strip()
    objectName = params[9].split('=')[1].strip()
    fluxCalObject = params[10].split('=')[1].strip()

    obsFileName = os.path.join(datadir, obsfile)
    skyFileName = os.path.join(datadir, skyfile)
    wvlCalFileName = os.path.join(wvldir, wvlfile)
    flatCalFileName = os.path.join(flatdir, flatfile)
    fluxCalFileName = os.path.join(fluxdir, fluxfile)
    """

    if len(sys.argv) > 2:
        fileNum = str(sys.argv[2])
    else:
        fileNum = "0"

    # science object parameter file
    params = []
    paramfile = sys.argv[1]
    f = open(paramfile, "r")
    for line in f:
        params.append(line)
    f.close()

    datadir = params[0].split("=")[1].strip()
    flatdir = params[1].split("=")[1].strip()
    wvldir = params[2].split("=")[1].strip()
    obsfile = params[3].split("=")[1].strip()
    skyfile = params[4].split("=")[1].strip()
    flatfile = params[5].split("=")[1].strip()
    wvlfile = params[6].split("=")[1].strip()
    objectName = params[9].split("=")[1].strip()

    # wvldir = "/Scratch/waveCalSolnFiles/oldbox_numbers/20121205"
    # objectName = "crabNight1"

    if len(params) > 10:
        xpix = int(params[10].split("=")[1].strip())
        ypix = int(params[11].split("=")[1].strip())
        apertureRadius = int(params[12].split("=")[1].strip())
        startTime = int(params[13].split("=")[1].strip())
        intTime = int(params[14].split("=")[1].strip())

    obsFileName = os.path.join(datadir, obsfile)
    skyFileName = os.path.join(datadir, skyfile)
    wvlCalFileName = os.path.join(wvldir, wvlfile)
    flatCalFileName = os.path.join(flatdir, flatfile)

    obs = ObsFile(obsFileName)
    # obs.loadWvlCalFile(wvlCalFileName)
    obs.loadBestWvlCalFile()
    obs.loadFlatCalFile(flatCalFileName)
    print "analyzing file %s" % (obsFileName)
    print "loaded data file and calibrations\n---------------------\n"

    nRow = obs.nRow
    nCol = obs.nCol
    obsTime = obs.getFromHeader("exptime")
    # wvlBinEdges,obsSpectra = loadSpectra(obs,nCol,nRow)
    # nWvlBins=len(wvlBinEdges)-1

    # print np.shape(obsSpectra)
    # print nRow
    # print nCol
    # print nWvlBins

    # Apply Hot pixel masking before getting dead time correction
    # HotPixFile = getTimeMaskFileName(obsFileName)
    HotPixFile = FileName(obsFile=obs).timeMask()
    print "making hot pixel file ", HotPixFile
    if not os.path.exists(HotPixFile):  # check if hot pix file already exists
        hp.findHotPixels(inputFileName=obsFileName, outputFileName=HotPixFile)
        print "Flux file pixel mask saved to %s" % (HotPixFile)
    obs.loadHotPixCalFile(HotPixFile)
    print "Hot pixel mask loaded %s" % (HotPixFile)

    # GET RAW PIXEL COUNT IMAGE TO CALCULATE CORRECTION FACTORS
    print "Making raw cube to get dead time correction"
    cubeDict = obs.getSpectralCube(firstSec=startTime, integrationTime=intTime, weighted=False)

    cube = np.array(cubeDict["cube"], dtype=np.double)
    wvlBinEdges = cubeDict["wvlBinEdges"]
    effIntTime = cubeDict["effIntTime"]
    print "median effective integration time = ", np.median(effIntTime)

    nWvlBins = len(wvlBinEdges) - 1
    print "cube shape ", np.shape(cube)
    print "effIntTime shape ", np.shape(effIntTime)

    # add third dimension to effIntTime for  broadcasting
    effIntTime = np.reshape(effIntTime, np.shape(effIntTime) + (1,))
    # put cube into counts/s in each pixel
    cube /= effIntTime

    # CALCULATE DEADTIME CORRECTION
    # NEED TOTAL COUNTS PER SECOND FOR EACH PIXEL TO DO PROPERLY
    # ASSUMES SAME CORRECTION FACTOR APPLIED FOR EACH WAVELENGTH, MEANING NO WL DEPENDANCE ON DEAD TIME EFFECT
    DTCorr = np.zeros((np.shape(cube)[0], np.shape(cube)[1]), dtype=float)
    for f in range(0, np.shape(cube)[2]):
        print cube[:, :, f]
        print "-----------------------"
        DTCorr += cube[:, :, f]
        print DTCorr
        print "\n=====================\n"
    # Correct for 100 us dead time
    DTCorrNew = DTCorr / (1 - DTCorr * 100e-6)
    CorrFactors = DTCorrNew / DTCorr  # This is what the frames need to be multiplied by to get their true values

    print "Dead time correction factors = "
    print CorrFactors

    # REMAKE CUBE WITH FLAT WEIGHTS AND APPLY DEAD TIME CORRECTION AS WELL
    print "Making Weighted cube"
    # load/generate hot pixel mask file
    # HotPixFile = getTimeMaskFileName(obsFileName)
    HotPixFile = FileName(obsFile=obs).timeMask()
    if not os.path.exists(HotPixFile):  # check if hot pix file already exists
        hp.findHotPixels(inputFileName=obsFileName, outputFileName=HotPixFile)
        print "Flux file pixel mask saved to %s" % (HotPixFile)
    obs.loadHotPixCalFile(HotPixFile)
    print "Hot pixel mask loaded %s" % (HotPixFile)

    cubeDict = obs.getSpectralCube(firstSec=startTime, integrationTime=intTime, weighted=True, fluxWeighted=False)
    # cubeDict = obs.getSpectralCube(firstSec=startTime, integrationTime=intTime, weighted=True, fluxWeighted=True)

    cube = np.array(cubeDict["cube"], dtype=np.double)
    wvlBinEdges = cubeDict["wvlBinEdges"]
    effIntTime = cubeDict["effIntTime"]
    print "median effective integration time = ", np.median(effIntTime)

    nWvlBins = len(wvlBinEdges) - 1
    print "cube shape ", np.shape(cube)
    print "effIntTime shape ", np.shape(effIntTime)

    # add third dimension to effIntTime for broadcasting
    effIntTime = np.reshape(effIntTime, np.shape(effIntTime) + (1,))
    # put cube into counts/s in each pixel
    cube /= effIntTime

    # add third dimension to CorrFactors for broadcasting
    CorrFactors = np.reshape(CorrFactors, np.shape(CorrFactors) + (1,))
    # apply dead time correction factors
    cube *= CorrFactors

    # calculate midpoints of wvl bins for plotting
    wvls = np.empty((nWvlBins), dtype=float)
    for n in xrange(nWvlBins):
        binsize = wvlBinEdges[n + 1] - wvlBinEdges[n]
        wvls[n] = wvlBinEdges[n] + (binsize / 2.0)

    print "wvls ", wvls
    # reshape cube for makeMovie
    movieCube = np.zeros((nWvlBins, np.shape(cube)[0], np.shape(cube)[1]), dtype=float)
    for i in xrange(nWvlBins):
        movieCube[i, :, :] = cube[:, :, i]
        # show individual frames as they are made to debug
        # plt.matshow(movieCube[i],vmin = 0, vmax = 100)
        # plt.show()
    print "movieCube shape ", np.shape(movieCube)
    print "wvls shape ", np.shape(wvls)

    # print cube
    # print "--------------------------"
    # print movieCube

    outdir = "/home/srmeeker/scratch/standards/"
    np.savez(outdir + "%s_raw_%s.npz" % (objectName, fileNum), stack=movieCube, wvls=wvls)
    utils.makeMovie(
        movieCube,
        frameTitles=wvls,
        cbar=True,
        outName=outdir + "%s_raw_%s.gif" % (objectName, fileNum),
        normMin=0,
        normMax=50,
    )

    """
def hotPixelsTest(testFileName=FileName(run='PAL2012',date='20121208',tstamp='20121209-044636').obs()):
    '''
    Runs some basic checks for consistency between intermediate output
    masks and the final 'bad time lists'.
    
    To run
        - from Python:
            hotPixelsTest('someObsFile.h5')
        
        - from command line:
            python hotPixelsTest.py someObsFile.h5
    
    
    (No parameter file need be supplied).
    
    ''' 
    
    workingDir = '/Users/vaneyken/Data/UCSB/ARCONS/Palomar2012/hotPixTest2/'
    outputFile = workingDir + 'testoutput.h5'
    paramFile = os.path.join(os.path.dirname(__file__),'../../params/hotPixels.dict')  #/Users/vaneyken/UCSB/ARCONS/pipeline/github/ARCONS-pipeline/params/hotPixels.dict'
    testStartTime = 2   #In seconds
    testEndTime = 4     #In seconds
    timeStep = 2        #In seconds (deliberately equal to start time - end time)
    fwhm = 3.0
    boxSize = 5
    nSigmaHot = 2.5
    nSigmaCold = 2.0

    hp.findHotPixels(paramFile=paramFile, inputFileName=testFileName,
                     outputFileName=outputFile, timeStep=timeStep,
                     startTime=testStartTime, endTime=testEndTime,
                     fwhm=fwhm, boxSize=boxSize, nSigmaHot=nSigmaHot,
                     nSigmaCold=nSigmaCold, display=True)
    
    intermediateOutput = hp.checkInterval(inputFileName=testFileName, display=True,
                                          firstSec=testStartTime,
                                          intTime=testEndTime - testStartTime,
                                          fwhm=fwhm, boxSize=boxSize,
                                          nSigmaHot=nSigmaHot, nSigmaCold=nSigmaCold)
    
    hpOutput = hp.readHotPixels(outputFile)

    intMask = intermediateOutput['mask'] > 0    #Make a Boolean mask - any code > 0 is bad for some reason.
    intervals = hpOutput['intervals']
    reasons = hpOutput['reasons']

    #Find the number of entries for each pixel in both the 'intervals' and the
    #'reasons' arrays.
    nIntervals = np.reshape([len(x) for x in intervals.flat], np.shape(intervals))
    nReasons = np.reshape([len(x) for x in reasons.flat], np.shape(reasons))

    #Union the interval lists for each pixel to give an array of single (multi-component) interval objects:
    uIntervals = np.reshape(np.array([interval.union(x) for x in intervals.flat],
                                     dtype='object'), np.shape(intervals))


    #Create a boolean mask that should be True for all bad (hot/cold/dead/other) pixels within the test time range
    finalMask = np.reshape([(interval(testStartTime, testEndTime) in x) 
                            for x in uIntervals.flat], np.shape(uIntervals))   

    assert np.all(np.equal(intMask, finalMask))
    assert np.all(np.equal(nIntervals, nReasons))

    print
    print "All seems okay. The two plots shown should look identical."
def main():

    obsSequence0="""
    112633
    112731
    113234
    113737
    114240
    114743
    115246
    115749
    120252
    120755
    121258
    121801
    122304
    """

    run = 'PAL2012'
    obsSequences = [obsSequence0]
    wvlCals = ['112506']
    flatCals = ['20121211']
    fluxCalDates = ['20121206']
    fluxCals = ['20121207-072055']

    #Row coordinate of center of crab pulsar for each obsSequence
    centersRow = [30]
    #Col coordinate of center of crab pulsar for each obsSequence
    centersCol = [30]


    obsUtcDates = ['20121208']

    obsFileNames = []
    obsFileNameTimestamps = []
    wvlFileNames = []
    flatFileNames = []
    fluxFileNames = []
    timeMaskFileNames = []
    plFileNames = []
    
    for iSeq in range(len(obsSequences)):
        obsSequence = obsSequences[iSeq]
        obsSequence = obsSequence.strip().split()
        obsFileNameTimestamps.append(obsSequence)
        obsUtcDate = obsUtcDates[iSeq]
        sunsetDate = str(int(obsUtcDate)-1)
        obsSequence = [obsUtcDates[iSeq]+'-'+ts for ts in obsSequence]
        obsFileNames.append([FileName(run=run,date=sunsetDate,tstamp=ts).obs() for ts in obsSequence])
        timeMaskFileNames.append([FileName(run=run,date=sunsetDate,tstamp=ts).timeMask() for ts in obsSequence])
        wvlCalTstamp = obsUtcDate+'-'+wvlCals[iSeq]
        wvlFileNames.append(FileName(run=run,date=sunsetDate,tstamp=wvlCalTstamp).calSoln())
        fluxFileNames.append(FileName(run=run,date=fluxCalDates[iSeq],tstamp=fluxCals[iSeq]).fluxSoln())
        flatFileNames.append(FileName(run=run,date=flatCals[iSeq],tstamp='').flatSoln())
        plFileNames.append([FileName(run=run,date=sunsetDate,tstamp=ts).timedPhotonList() for ts in obsSequence])

    for iSeq,obsSequence in enumerate(obsSequences):
        obsSequence = obsSequence.strip().split()
        print obsSequence
        for iOb,obs in enumerate(obsSequence):
            timeMaskFileName = timeMaskFileNames[iSeq][iOb]
            if not os.path.exists(timeMaskFileName):
                print 'Running hotpix for ',obs
                hp.findHotPixels(obsFileNames[iSeq][iOb],timeMaskFileName)
                print "Flux file pixel mask saved to %s"%(timeMaskFileName)

    apertureRadius = 5
    obLists = [[ObsFile(fn)for fn in seq ] for seq in obsFileNames]
    tstampFormat = '%H:%M:%S'
    #print 'fileName','headerUnix','headerUTC','logUnix','packetReceivedUnixTime'
    for iSeq,obList in enumerate(obLists):
        for iOb,ob in enumerate(obList):
            print ob.fileName
            centerRow = centersRow[iSeq]
            centerCol = centersCol[iSeq]
            circCol,circRow = circ(centerCol,centerRow,radius=apertureRadius)
            ob.loadTimeAdjustmentFile(FileName(run='PAL2012').timeAdjustments())
            ob.loadWvlCalFile(wvlFileNames[iSeq])
            ob.loadFlatCalFile(flatFileNames[iSeq])
            ob.loadFluxCalFile(fluxFileNames[iSeq])
            timeMaskFileName = timeMaskFileNames[iSeq][iOb]
            ob.loadHotPixCalFile(timeMaskFileName)
            ob.setWvlCutoffs(4000,11000)
            photlist.writePhotonList(ob,pixelList=zip(circRow,circCol),filename=plFileNames[iSeq][iOb])
            print 'wrote crab list'
def hotPixelsTest2(startTime=12.3, endTime=23.1, getRawCount=True):
    '''
    Runs a check of the bad pixel time masking. Checks:
        - Number of photons removed from returned image is consistent
        with time masks.
        - That timestamps of all photons in the time-masked image are outside
        the time intervals defined for each pixel in the hot pixel file.
        Outputs a test hot pixel file in the current directory, and runs checks 
        on it.
        
    INPUTS:
        startTime - time from beginning of obs file at which to start the check.
        endTime - time from beginning to obs file at which to end (both in seconds).
        getRawCounts - if True, use raw, non-wavelength calibrated photon counts
                        with no wavelength cutoffs applied.
    '''
    
    #dir = '/Users/vaneyken/Data/UCSB/ARCONS/Palomar2012/hotPixTest2/'
    run = 'PAL2012'
    date = '20121208'
    obsFileName = FileName(run=run,date=date,tstamp='20121209-044636').obs()   #'obs_20121209-044636.h5'
    wvlCalFileName = FileName(run=run, date=date, tstamp='20121209-060704').calSoln()    #'calsol_20121209-060704.h5'
    flatCalFileName = FileName(run=run, date='20121210').flatSoln()    #'flatsol_20121210.h5'
    hotPixFileName = os.path.abspath('test-hotPix_20121209-044636.h5')
    paramFile = os.path.join(os.path.dirname(__file__),'../../params/hotPixels.dict')
    startTime = float(startTime)    #Force these values to floats to make sure
    endTime = float(endTime)        #that getPixelCountImage calls getPixelSpectrum
                                    #and applies wavelength cutoffs consistently.
    if not os.path.exists(hotPixFileName):
        print 'Creating hot pixel file....'
        hp.findHotPixels(paramFile=paramFile, inputFileName=obsFileName,
                         outputFileName=hotPixFileName, timeStep=1,
                         startTime=0, endTime=-1,
                         fwhm=3.0, boxSize=5, nSigmaHot=2.5,
                         nSigmaCold=2.5, display=True)
        print 'Done creating hot pixel file.'
        print
    
    intTime = endTime - startTime
    
    obsFile = of.ObsFile(obsFileName)
    obsFile.loadWvlCalFile(wvlCalFileName)
    obsFile.loadFlatCalFile(flatCalFileName)
    print 'Loading hot pixel file into obsFile...'
    obsFile.loadHotPixCalFile(hotPixFileName)
    obsFile.setWvlCutoffs()
    print 'Getting image with masking...'
    imhp = obsFile.getPixelCountImage(startTime, intTime, weighted=False,
                                      getRawCount=getRawCount)['image']
    print 'Getting image without masking...'
    obsFile.switchOffHotPixTimeMask()
    im = obsFile.getPixelCountImage(startTime, intTime, weighted=False,
                                    getRawCount=getRawCount)['image']
    
    diffim = im - imhp #Should end up containing the total number of photons masked from each pixel.

    
    print 'Displaying images...'
    mpl.ion()
    mpl.matshow(imhp)
    mpl.title('Hot-pixel masked')
    mpl.colorbar()
    mpl.matshow(im)
    mpl.title('Unmasked')
    mpl.colorbar()
    
    print 'Loading local version of hot pixel file for direct inspection...'
    hotPix = hp.readHotPixels(hotPixFileName)
    
    if True:
        print 'Checking for consistency in number of photons removed....'
        for iRow in range(np.shape(diffim)[0]):
            for iCol in range(np.shape(diffim)[1]):
                nMaskedPhotons = 0
                for eachInter in hotPix['intervals'][iRow, iCol]:
                    #Make sure there is only one component per interval
                    assert len(eachInter) == 1
                    #If the interval overlaps with the time range in question then 
                    #add the number of photons in the interval to our running tally.
                    if eachInter[0][0] < endTime and eachInter[0][1] > startTime:
                        firstSec = max(eachInter[0][0], startTime)
                        lastSec = min(eachInter[0][1], endTime)
                        nMaskedPhotons += obsFile.getPixelCount(iRow, iCol, firstSec=firstSec,
                                                                integrationTime=lastSec - firstSec,
                                                                weighted=False, fluxWeighted=False,
                                                                getRawCount=getRawCount)['counts']
                assert nMaskedPhotons == diffim[iRow, iCol]
        print 'Okay.'
    
    print 'Checking timestamps of remaining photons for consistency with exposure masks'
    obsFile.switchOnHotPixTimeMask()       #Switch on hot pixel masking
    for iRow in range(np.shape(diffim)[0]):
        for iCol in range(np.shape(diffim)[1]):
            timeStamps = obsFile.getTimedPacketList(iRow, iCol, firstSec=startTime, integrationTime=intTime)['timestamps']
            #timeStamps = timeStamps[np.logical_and(timeStamps<=endTime, timeStamps>=startTime)]
            badInterval = interval.union(hotPix['intervals'][iRow, iCol])
           
            #The following check would be nice, but doesn't work because getTimedPacketList doesn't do a wavelength cut like getPixelCount does.
            #assert len(timeStamps) == im[iRow,iCol]     #Double check that the number of photons returned matches the number in the masked image
            #
            
            for eachTimestamp in timeStamps:
               assert eachTimestamp not in badInterval   #Check that none of those photons' timestamps are in the masked time range

    print 'Okay.'
    print       
    print 'Done. All looks good.'
for iSeq in range(len(seqs)):
    timestampList = timestampLists[iSeq]
    print timestampList
    wfn = wvlCalFilenames[iSeq]
    ffn = flatCalFilenames[iSeq]
    sunsetDate = sunsetDates[iSeq]
    
    for i,ts in enumerate(timestampList):
        print 'loading',ts
        obsFn = FileName(run=run,date=sunsetDate,tstamp=ts).obs()
        ob = ObsFile(obsFn)
        ob.loadTimeAdjustmentFile(FileName(run=run).timeAdjustments())
        index1 = obsFn.find('_')
        hotPixFn = '/Scratch/timeMasks/timeMask' + obsFn[index1:]
        if not os.path.exists(hotPixFn):
            hp.findHotPixels(obsFn,hotPixFn)
            print "Flux file pixel mask saved to %s"%(hotPixFn)
        ob.loadHotPixCalFile(hotPixFn,switchOnMask=True)
        ob.loadWvlCalFile(wfn)
        ob.loadFlatCalFile(ffn)
        ob.setWvlCutoffs(wvlLowerCutoff,wvlUpperCutoff)

        bad_solution_mask=np.zeros((46,44))
        bad_count=0;
        for y in range(46):
            for x in range(44):
                if (ob.wvlRangeTable[y][x][1] < wvlUpperCutoff) or (ob.wvlRangeTable[y][x][0] > wvlLowerCutoff):
                    bad_solution_mask[y][x] = 1

        unix = ob.getFromHeader('unixtime')
        startJD = unix/86400.+2440587.5
def quantifyBadTime(inputFileName, startTime=0, endTime=-1, 
                    defaultTimeMaskFileName='./testTimeMask.h5',
                    timeStep=1, fwhm=3.0, boxSize=5, nSigmaHot=3.0,
                    nSigmaCold=2.5,maxIter=5,binWidth=3,
                    dispToPickle=False, showHist=False, bkgdPercentile=50,
                    weighted=False,fluxWeighted=False, useRawCounts=True):
    '''
    Function to calculate various metrics for the degree of bad pixel behaviour in a raw 
    raw obs file. Calculates the mean total hot/cold/dead time per good pixel (i.e., per 
    pixel which is not permanently dead, hot, or cold).
    
    Makes a couple of heat maps showing time spent bad in each way for each pixel,
    as well as a histogram of times spent bad for the *temporarily* bad pixels.
    JvE Nov 20 2013.
    
    The parameters for the finding algorithm may need to be tuned a little,
    but the defaults should basically work, in principle. nSigmaHot and nSigmaCold
    are good places to start if things need playing around with.
    
    e.g. call, in principle:
    
        from hotpix import quantifyHotTime as qht
        qht.quantifyHotTime('/Users/vaneyken/Data/UCSB/ARCONS/turkDataCopy/ScienceData/PAL2012/20121208/obs_20121209-120530.h5')
    
    - should be all it needs....
    
    
    INPUTS:
    
        inputFileName - either a raw obs. filename or a time mask file. If the former, 
                        runs a hot pixel search; otherwise uses time mask file instead.
        
        startTime, endTime - start and end times within the obs file to
                        calculate the hot pixels for. (endTime =-1 means to end of file).
        
        defaultTimeMaskFileName - use this filename to output new time mask to
                                (if inputFileName is an obs file)
    
        binWidth - width of time bins for plotting the bad-time histogram (seconds)

        dispToPickle - if not False, saves the data for the histogram plot to a pickle file.
                       If a string, then uses that as the name for the pickle file. Otherwise
                       saves to a default name. Saves a dictionary with four entries, the first
                       three of which are each a flat array of total times spent bad for every
                       pixel (in sec) ('hotTime','coldTime','deadTime'). The fourth, 'duration',
                       is the duration of the input time-mask in seconds.
        
        showHist - if True, plot up a histogram of everything. Currently fails though 
                   if there are no bad-flagged intervals in any of the type categories
                   (or their intersections, hot+cold, cold+dead).
    
    
        Parameters passed on to findHotPixels routine if called (see also documentation
        for that function):
    
        timeStep        #Check for hot pixels every timeStep seconds
    
        fwhm            #Expected full width half max of PSF in pixels. Any pixel
                        #with flux much tighter than this will be flagged as bad.
                        #Larger value => more sensitive hot pixel flagging.
    
        boxSize         #Compare flux in each pixel with median flux in a 
                        #surrounding box of this size on a side.
    
        nSigmaHot       #Require flux in a pixel to be > nSigmaHot std. deviations
                        #above the max expected for a Gaussian PSF in order for it
                        #to be flagged as hot. Larger value => less sensitive flagging.
    
        nSigmaCold      #Require flux to be < nSigmaCold std. deviations below the median 
                        #in a surrounding box in order to be flagged as cold (where std.
                        #deviation is estimated as the square root of the median flux).
    
        maxIter         #Max num. of iterations for the bad pixel detection algorithm.
        
        bkdgPercentile, weighted, fluxWeighted    - See findHotPixels() in hotpix.hotpixels.py


    OUTPUTS:
        
        A bunch of statistics on the different kinds of bad pixel behaviour, and a 
        'heat' plot showing how much time each pixel was bad in the array, for each
        type of behaviour. In theory it can plot a histogram of everything too, but
        currently it breaks if there are no bad intervals within any of the type
        categories (hot only, hot and cold, cold only, cold and dead, dead only...)
    
    '''
    
    defaultPklFileName = 'badPixTimeHist.pickle'
    
    #Check whether the input file is a time mask or a regular obs file.
    absInputFileName = os.path.abspath(inputFileName)   #To avoid weird issues with the way findHotPixels expands paths....
    hdffile = tables.openFile(absInputFileName)
    inputIsTimeMaskFile = '/timeMasks' in hdffile   
    hdffile.close()
    
    #Decide whether to generate a new time mask file or not
    if inputIsTimeMaskFile:
        print 'Input file is a time mask file'
        timeMaskFileName = absInputFileName
    else:
        print 'Assuming input file is an obs. file'
        timeMaskFileName = os.path.abspath(defaultTimeMaskFileName)
        if os.path.exists(timeMaskFileName):
            response=''
            while response != 'u' and response !='r':
                response = raw_input(timeMaskFileName+' already exists - "u" to use this (default) or "r" to regenerate? ')
                response = response.strip().lower()
                if response == '': response = 'u'
        else:
            response = 'r'         #If the file *didn't already exist, pretend the user entered 'r' despite not having been asked.
            
        if response == 'r':
                #Make/regenerate the hot pixel file.
                print 'Making new hot pixel time mask file '+timeMaskFileName
                hp.findHotPixels(inputFileName=absInputFileName,
                                 outputFileName=timeMaskFileName,
                                 startTime=startTime,
                                 endTime=endTime,
                                 timeStep=timeStep, fwhm=fwhm,
                                 boxSize=boxSize, nSigmaHot=nSigmaHot, nSigmaCold=nSigmaCold,
                                 display=True, dispToPickle=dispToPickle,
                                 maxIter=maxIter, bkgdPercentile=bkgdPercentile,
                                 weighted=weighted,fluxWeighted=fluxWeighted,useRawCounts=useRawCounts)
    
    
    #Read in the time mask file and calculate hot, cold, and 'other' bad time per pixel.
    timeMask = hp.readHotPixels(timeMaskFileName)
    hotTime = np.zeros((timeMask.nRow,timeMask.nCol))
    coldTime = np.zeros((timeMask.nRow,timeMask.nCol))
    deadTime = np.zeros((timeMask.nRow,timeMask.nCol))
    otherTime = np.zeros((timeMask.nRow,timeMask.nCol))
    hotIntervals = np.array([])
    coldIntervals = np.array([])
    deadIntervals = np.array([])
    otherIntervals = np.array([])

    reasonStringMap = timeMask.reasonEnum
    for iRow in range(timeMask.nRow):
        for iCol in range(timeMask.nCol):
            for eachInterval,eachReasonCode in zip(timeMask.intervals[iRow,iCol], timeMask.reasons[iRow,iCol]):
                eachReasonString = reasonStringMap(eachReasonCode)   #Convert integer code to human readable string
                intSize = utils.intervalSize(eachInterval)
                if eachReasonString == 'hot pixel':
                    hotTime[iRow,iCol] += intSize
                    hotIntervals = np.append(hotIntervals, intSize)
                elif eachReasonString == 'cold pixel':
                    coldTime[iRow,iCol] += intSize
                    coldIntervals = np.append(coldIntervals, intSize)
                elif eachReasonString == 'dead pixel':
                    deadTime[iRow,iCol] += intSize
                    deadIntervals = np.append(deadIntervals, intSize)
                else:
                    otherTime[iRow,iCol] += intSize
                    otherIntervals = np.append(otherIntervals, intSize)

    
    if np.size(hotIntervals) == 0: hotIntervals = np.array([-1])
    if np.size(coldIntervals) == 0: coldIntervals = np.array([-1])
    if np.size(deadIntervals) == 0: deadIntervals = np.array([-1])
    if np.size(otherIntervals) == 0: otherIntervals = np.array([-1])
        
    totBadTime = hotTime+coldTime+deadTime+otherTime
    
    maskDuration = timeMask.endTime-timeMask.startTime
    
    #Figure out which pixels are hot, cold, permanently hot, temporarily cold, etc. etc.
    nPix = timeMask.nRow * timeMask.nCol
    hotPix = hotTime > 0.1
    coldPix = coldTime > 0.1
    deadPix = deadTime > 0.1
    otherPix = otherTime > 0.1
    multiBehaviourPix = ( (np.array(hotPix,dtype=int)+np.array(coldPix,dtype=int)
                         +np.array(deadPix,dtype=int)+np.array(otherPix,dtype=int)) > 1)
    #assert np.all(deadTime[deadPix] == maskDuration)      #All dead pixels should be permanently dead....
    
    tol = timeStep/10. #Tolerance for the comparison operators below.
    
    permHotPix = hotTime >= maskDuration-tol
    permColdPix = coldTime >= maskDuration-tol
    permDeadPix = deadTime >= maskDuration-tol
    permOtherPix = otherTime >= maskDuration-tol
    permGoodPix = (hotTime+coldTime+deadTime+otherTime < tol)
    permBadPix = permHotPix | permColdPix | permDeadPix | permOtherPix
    
    tempHotPix = (hotTime < maskDuration-tol) & (hotTime > tol)
    tempColdPix = (coldTime < maskDuration-tol) & (coldTime > tol)
    tempDeadPix = (deadTime < maskDuration-tol) & (deadTime > tol)
    tempOtherPix = (otherTime < maskDuration-tol) & (otherTime > tol)
    tempGoodPix = tempHotPix | tempColdPix | tempDeadPix | tempOtherPix     #Bitwise or should work okay with boolean arrays
    tempBadPix = tempGoodPix        #Just to be explicit about it....
    
    nGoodPix = np.sum(permGoodPix | tempGoodPix)        #A 'good pixel' is either temporarily or permanently good
    
    #assert np.sum(tempDeadPix) == 0             #Shouldn't be any temporarily dead pixels. APPARENTLY THERE ARE....
    assert np.all((tempHotPix & permHotPix)==False)     #A pixel can't be permanently AND temporarily hot
    assert np.all((tempColdPix & permColdPix)==False)   #... etc.
    assert np.all((tempOtherPix & permOtherPix)==False)
    assert np.all((tempDeadPix & permDeadPix)==False)

    
    #Print out the results
    print '----------------------------------------------'
    print
    print '# pixels total: ', nPix
    print 'Mask duration (sec): ', maskDuration
    print
    print '% hot pixels: ', float(np.sum(permHotPix+tempHotPix))/nPix * 100.
    print '% cold pixels: ', float(np.sum(permColdPix+tempColdPix))/nPix * 100.
    print '% dead pixels: ', float(np.sum(permDeadPix+tempDeadPix))/nPix * 100.
    print '% other pixels: ', float(np.sum(permOtherPix+tempOtherPix))/nPix * 100.
    print
    print '% permanently hot pixels: ', float(np.sum(permHotPix))/nPix * 100.
    print '% permanently cold pixels: ', float(np.sum(permColdPix))/nPix * 100.
    print '% permanently dead pixels: ', float(np.sum(permDeadPix))/nPix * 100.
    print '% permanently "other" bad pixels: ', float(np.sum(permOtherPix))/nPix * 100.
    print
    print '% temporarily hot pixels: ', float(np.sum(tempHotPix))/nPix * 100.
    print '% temporarily cold pixels: ', float(np.sum(tempColdPix))/nPix * 100.
    print '% temporarily dead pixels: ', float(np.sum(tempDeadPix))/nPix * 100.
    print '% temporarily "other" bad pixels: ', float(np.sum(tempOtherPix))/nPix * 100.
    print
    print '% pixels showing multiple bad behaviours: ', float(np.sum(multiBehaviourPix))/nPix * 100.
    print
    print '% permanently bad pixels: ', float(np.sum(permBadPix))/nPix * 100.
    print '% temporarily bad pixels: ', float(np.sum(tempGoodPix))/nPix * 100.      #Temp. good == temp. bad!
    print '% permanently good pixels: ', float(np.sum(permGoodPix))/nPix * 100.
    print
    print 'Mean temp. hot pixel time per good pixel: ', np.sum(hotTime[tempHotPix])/nGoodPix
    print 'Mean temp. hot pixel time per temp. hot pixel: ', np.sum(hotTime[tempHotPix])/np.sum(tempHotPix)
    print
    print 'Mean temp. cold pixel time per good pixel: ', np.sum(coldTime[tempColdPix])/nGoodPix
    print 'Mean temp. cold pixel time per temp. cold pixel: ', np.sum(coldTime[tempColdPix])/np.sum(tempColdPix)
    print
    print 'Mean temp. "other" bad pixel time per good pixel: ', np.sum(otherTime[tempOtherPix])/nGoodPix
    print 'Mean temp. "other" bad pixel time per temp. "other" pixel: ', np.sum(otherTime[tempOtherPix])/np.sum(tempOtherPix)
    print
    print '(All times in seconds)'
    print
    print 'Done.'
    print
    if np.sum(tempOtherPix) > 0 or np.sum(permOtherPix) > 0:
        print '--------------------------------------------------------'
        print 'WARNING: Pixels flagged for "other" reasons detected - '
        print 'Histogram plot will not account for these!!'
        print '--------------------------------------------------------'

    #Display contour plots of the array of total bad times for each pixel for each type of behaviour
    utils.plotArray(hotTime, plotTitle='Hot time per pixel (sec)', fignum=None, cbar=True)
    utils.plotArray(coldTime, plotTitle='Cold time per pixel (sec)', fignum=None, cbar=True)
    utils.plotArray(otherTime, plotTitle='Other bad time per pixel (sec)', fignum=None, cbar=True)
    utils.plotArray(deadTime, plotTitle='Dead time per pixel (sec)', fignum=None, cbar=True)
    
    #Make histogram of time spent 'bad' for the temporarily bad pixels.
    #Ignore pixels flagged as bad for 'other' reasons (other than hot/cold/dead),
    #of which there should be none at the moment.
    assert np.all(otherPix == False)
    #Find total bad time for pixels which go only one of hot, cold, or dead
    onlyHotBadTime = totBadTime[hotPix & ~coldPix & ~deadPix]
    onlyColdBadTime = totBadTime[~hotPix & coldPix & ~deadPix]
    onlyDeadBadTime = totBadTime[~hotPix & ~coldPix & deadPix]
    #Find total bad time for pixels which alternate between more than one bad state
    hotAndColdBadTime = totBadTime[hotPix & coldPix & ~deadPix]
    hotAndDeadBadTime = totBadTime[hotPix & ~coldPix & deadPix]
    coldAndDeadBadTime = totBadTime[~hotPix & coldPix & deadPix]
    hotAndColdAndDeadBadTime = totBadTime[hotPix & coldPix & deadPix]
    allGoodBadTime = totBadTime[~hotPix & ~coldPix & ~deadPix]
    assert np.sum(allGoodBadTime) == 0

    if dispToPickle is not False:
        #Save to pickle file to feed into a separate plotting script, primarily for 
        #the pipeline paper.
        if type(dispToPickle) is str:
            pklFileName = dispToPickle
        else:
            pklFileName = defaultPklFileName
        pDict = {'hotTime':hotTime,
                 'coldTime':coldTime,
                 'deadTime':deadTime,
                 'onlyHotBadTime':onlyHotBadTime,
                 'onlyColdBadTime':onlyColdBadTime, 
                 'onlyDeadBadTime':onlyDeadBadTime,
                 'hotAndColdBadTime':hotAndColdBadTime,
                 'hotAndDeadBadTime':hotAndDeadBadTime,
                 'coldAndDeadBadTime':coldAndDeadBadTime,
                 'hotAndColdAndDeadBadTime':hotAndColdAndDeadBadTime,
                 'maskDuration':maskDuration}
        #pDict = {"hotTime":hotTime.ravel(),"coldTime":coldTime.ravel(),"deadTime":deadTime.ravel(),
        #         "duration":maskDuration}
        print 'Saving to file: ',pklFileName
        output = open(pklFileName, 'wb')
        pickle.dump(pDict,output)
        output.close()

    assert np.size(hotTime)==nPix and np.size(coldTime)==nPix and np.size(deadTime)==nPix
    assert (len(onlyHotBadTime)+len(onlyColdBadTime)+len(onlyDeadBadTime)+len(hotAndColdBadTime)
            +len(coldAndDeadBadTime)+len(hotAndDeadBadTime)+len(hotAndColdAndDeadBadTime)
            +len(allGoodBadTime))==nPix
       
    if showHist is True: 
        #Be sure it's okay to leave hot+dead pixels out, and hot+cold+dead pixels.
        #assert len(hotAndDeadBadTime)==0 and len(hotAndColdAndDeadBadTime)==0 
        mpl.figure()
        norm = 1. #1./np.size(hotTime)*100.
        dataList = [onlyHotBadTime,hotAndColdBadTime,onlyColdBadTime,coldAndDeadBadTime,onlyDeadBadTime]
        dataList2 = [x if np.size(x)>0 else np.array([-1.0]) for x in dataList]     #-1 is a dummy value for empty arrays, so that pyplot.hist doesn't barf.
        weights = [np.ones_like(x)*norm if np.size(x)>0 else np.array([0]) for x in dataList]
        mpl.hist(dataList2, range=None, #[-0.1,maskDuration+0.001],         #Eliminate data at 0sec and maskDuration sec. (always good or permanently bad)
                 weights=weights, 
                 label=['Hot only','Hot/cold','Cold only','Cold/dead','Dead only'], #,'Hot/dead','Hot/cold/dead'],
                 color=['red','pink','lightgray','lightblue','blue'], #,'green','black'],
                 bins=maskDuration/binWidth,histtype='stepfilled',stacked=True,log=False)         
        mpl.title('Duration of Bad Pixel Behaviour - '+os.path.basename(inputFileName))
        mpl.xlabel('Total time "bad" (sec)')
        mpl.ylabel('Percantage of pixels')
        mpl.legend()
        
        mpl.figure()
        mpl.hist(hotIntervals, bins=maskDuration/binWidth)
        print 'Median hot interval: ', np.median(hotIntervals)
        mpl.xlabel('Duration of hot intervals')
        mpl.ylabel('Number of intervals')
 
        mpl.figure()
        mpl.hist(coldIntervals, bins=maskDuration/binWidth)
        print 'Median cold interval: ', np.median(coldIntervals)
        mpl.xlabel('Duration of cold intervals')
        mpl.ylabel('Number of intervals')
 
        mpl.figure()
        mpl.hist(deadIntervals, bins=maskDuration/binWidth)
        print 'Median dead interval: ', np.median(deadIntervals)
        mpl.xlabel('Duration of dead intervals')
        mpl.ylabel('Number of intervals')
 
        mpl.figure()
        mpl.hist(otherIntervals, bins=maskDuration/binWidth)
        print 'Median "other" interval: ', np.median(otherIntervals)
        mpl.xlabel('Duration of "other" intervals')
        mpl.ylabel('Number of intervals')
        
        
 
    print 'Mask duration (s): ',maskDuration
    print 'Number of pixels: ',nPix
    print 'Fraction at 0s (hot,cold,dead): ', np.array([np.sum(hotTime<tol),np.sum(coldTime<tol),
                                                        np.sum(deadTime<tol)]) / float(nPix)                            
    print 'Fraction at '+str(maskDuration)+'s (hot,cold,dead): ', np.array([np.sum(hotTime>maskDuration-tol),
                                                                        np.sum(coldTime>maskDuration-tol),
                                                                        np.sum(deadTime>maskDuration-tol)])/float(nPix)
def hotPixPlot():
    
    if 1==1:
    
        #File #1 - 2012, SDSS J0926
        run = 'PAL2012'
        date = '20121208'
        tstamp='20121209-120530'
        inputFile = fn.FileName(run=run,date=date,tstamp=tstamp).obs()
        timeMaskFile = fn.FileName(run=run,date=date,tstamp=tstamp).timeMask()
        timeAdjFileName = fn.FileName(run='PAL2012').timeAdjustments()
        #wvlCalFileName = fn.FileName(run=run, date=date, tstamp='20121209-060704').calSoln()    #'calsol_20121209-060704.h5'
        #flatCalFileName = fn.FileName(run=run, date='20121210').flatSoln()    #'flatsol_20121210.h5'
        #fluxCalFileName = fn.FileName(run=run, date='20121211', tstamp='absolute_021727').fluxSoln()
        
        obsFile = ObsFile.ObsFile(inputFile)
        print inputFile
        obsFile.loadTimeAdjustmentFile(timeAdjFileName)
        print timeAdjFileName
        obsFile.loadBestWvlCalFile()
        print obsFile.wvlCalFileName
        #print flatCalFileName
        #obsFile.loadFlatCalFile(flatCalFileName)
        #print fluxCalFileName
        #obsFile.loadFluxCalFile(fluxCalFileName)
        obsFile.setWvlCutoffs()   #Use default wavelength cutoffs
        
        hp.findHotPixels(obsFile=obsFile, outputFileName='badPix1.h5', startTime=54,
                        endTime=55, display=True, dispMinPerc=0, dispMaxPerc=98, maxIter=10,
                        nSigmaHot=4.0, bkgdPercentile=50, fwhm=2.5, diagnosticPlots=False,
                        boxSize=5, weighted=False,fluxWeighted=False, useRawCounts=True,
                        dispToPickle='hotPixData-'+tstamp+'.pkl')
        mpl.title(obsFile.getFromHeader('target'))
        
        #Get the stats for the same image
        qbt.quantifyBadTime(inputFile,startTime=0,endTime=-1,maxIter=10,nSigmaHot=4.0,
                            bkgdPercentile=50,fwhm=2.0,boxSize=5,useRawCounts=True,
                            defaultTimeMaskFileName=timeMaskFile)
        

    
    #File 2
    run = 'PAL2014'
    date = '20141021'  #'20141022' 
    tstamp = '20141022-092629'   #'20141022-032331'               # '20141023-050637'
    inputFile = fn.FileName(run=run,date=date,tstamp=tstamp).obs()
    timeMaskFile = fn.FileName(run=run,date=date,tstamp=tstamp).timeMask()

    #Time adjustment file not needed
    #flatCalFileName = fn.FileName(run=run, date=date).flatSoln()
    #Ignore flux cal for this one (shouldn't make much difference, and don't know if new one is set up yet)
    #fluxCalFileName = fn.FileName(run=run, date=', tstamp='absolute_021727').fluxSoln()
    
    obsFile = ObsFile.ObsFile(inputFile)
    print inputFile
    obsFile.loadBestWvlCalFile()
    print obsFile.wvlCalFileName
    #print flatCalFileName
    #obsFile.loadFlatCalFile(flatCalFileName)
    obsFile.setWvlCutoffs()   #Use default wavelength cutoffs
    
    hp.findHotPixels(obsFile=obsFile, outputFileName='badPix2.h5',
                     startTime=24, endTime=25, display=True, dispMinPerc=0, dispMaxPerc=99, maxIter=10,
                     nSigmaHot=4.0, bkgdPercentile=50, fwhm=2.0, diagnosticPlots=False,
                     boxSize=5, weighted=False,fluxWeighted=False, useRawCounts=True,
                     dispToPickle='hotPixData-'+tstamp+'.pkl')
        
    mpl.title(obsFile.getFromHeader('target'))
    
    #Get the stats for the same image
    qbt.quantifyBadTime(inputFile,startTime=0,endTime=-1,maxIter=10,nSigmaHot=4.0,
                        bkgdPercentile=50,fwhm=2.0,boxSize=5,useRawCounts=True,
                        defaultTimeMaskFileName=timeMaskFile)
        
    return obsFile
#        plt.show()
        
        startJD = ob.getFromHeader('jd')
        nSecInFile = ob.getFromHeader('exptime')
        deadMask = ob.getDeadPixels()
        for sec in range(0,nSecInFile,integrationTime):
            jd = startJD + sec/(24.*3600.)#add seconds offset to julian date
            print count,jd
            count+=1
            times.append(jd)
            titles.append('%.6f'%jd)
            #make 30s flat-fielded,hot pixel masked image
            #hotPixMaskRaw = hotPixels.findHotPixels(obsFile=ob,firstSec=sec,intTime=integrationTime)['badflag']
            #hotPixMask = hotPixels.findHotPixels(obsFile=ob,firstSec=sec,intTime=integrationTime,weighted=True)['badflag']
            frame = ob.getPixelCountImage(firstSec=sec,integrationTime=integrationTime,weighted=True)
            hotPixMask = hotPixels.findHotPixels(image=frame,firstSec=sec,intTime=integrationTime,weighted=True)['badflag']
#            plt.show()

            showFrame = np.array(frame)
#            utils.plotArray(showFrame,cbar=True,normMax=np.mean(showFrame)+3*np.std(showFrame))
#            showFrame[hotPixMaskRaw == 1] = 0
#            utils.plotArray(showFrame,cbar=True)
            showFrame[hotPixMask == 1] = 0
#            utils.plotArray(showFrame,cbar=True)
            showframes.append(showFrame)

#            frame[hotPixMaskRaw == 1] = np.nan
            frame[hotPixMask == 1] = np.nan
            frame[deadMask == 0] = np.nan
            frames.append(frame)