Esempio n. 1
0
 def testRandomDither(self):
     """
     Test the random dither pattern.
     """
     maxDither = .5
     data = np.zeros(600,
                     dtype=list(zip(['fieldRA', 'fieldDec'],
                                    [float, float])))
     # Set seed so the test is stable
     np.random.seed(42)
     # Restrict dithers to area where wraparound is not a problem for
     # comparisons.
     data['fieldRA'] = np.degrees(
         np.random.rand(600) * (np.pi) + np.pi / 2.0)
     data['fieldDec'] = np.degrees(
         np.random.rand(600) * np.pi / 2.0 - np.pi / 4.0)
     stacker = stackers.RandomDitherFieldPerVisitStacker(
         maxDither=maxDither)
     data = stacker.run(data)
     diffsra = (data['fieldRA'] -
                data['randomDitherFieldPerVisitRa']) * np.cos(
                    np.radians(data['fieldDec']))
     diffsdec = data['fieldDec'] - data['randomDitherFieldPerVisitDec']
     # Check dithers within expected range.
     self._tDitherRange(diffsra, diffsdec, data['fieldRA'],
                        data['fieldDec'], maxDither)
Esempio n. 2
0
    def testEQ(self):
        """
        Test that stackers can be compared
        """
        s1 = stackers.ParallaxFactorStacker()
        s2 = stackers.ParallaxFactorStacker()
        assert (s1 == s2)

        s1 = stackers.RandomDitherFieldPerVisitStacker()
        s2 = stackers.RandomDitherFieldPerVisitStacker()
        assert (s1 == s2)

        # Test if they have numpy array atributes
        s1.ack = np.arange(10)
        s2.ack = np.arange(10)
        assert (s1 == s2)

        # Change the array and test
        s1.ack += 1
        assert (s1 != s2)

        s2 = stackers.RandomDitherFieldPerVisitStacker(decCol='blah')
        assert (s1 != s2)
Esempio n. 3
0
    def testOut(self):
        """
        Check that the metric bundle can generate the expected output
        """
        nside = 8
        slicer = slicers.HealpixSlicer(nside=nside)
        metric = metrics.MeanMetric(col='airmass')
        sql = 'filter="r"'
        stacker1 = stackers.RandomDitherFieldPerVisitStacker()
        stacker2 = stackers.GalacticStacker()
        map1 = maps.GalCoordsMap()
        map2 = maps.StellarDensityMap()

        metricB = metricBundles.MetricBundle(metric,
                                             slicer,
                                             sql,
                                             stackerList=[stacker1, stacker2],
                                             mapsList=[map1, map2])
        database = os.path.join(getPackageDir('sims_data'), 'OpSimData',
                                'astro-lsst-01_2014.db')

        opsdb = db.OpsimDatabaseV4(database=database)
        resultsDb = db.ResultsDb(outDir=self.outDir)

        bgroup = metricBundles.MetricBundleGroup({0: metricB},
                                                 opsdb,
                                                 outDir=self.outDir,
                                                 resultsDb=resultsDb)
        bgroup.runAll()
        bgroup.plotAll()
        bgroup.writeAll()

        opsdb.close()

        outThumbs = glob.glob(os.path.join(self.outDir, 'thumb*'))
        outNpz = glob.glob(os.path.join(self.outDir, '*.npz'))
        outPdf = glob.glob(os.path.join(self.outDir, '*.pdf'))

        # By default, make 3 plots for healpix
        assert (len(outThumbs) == 3)
        assert (len(outPdf) == 3)
        assert (len(outNpz) == 1)
    def testOut(self):
        """
        Check that the metric bundle can generate the expected output
        """
        nside = 8
        slicer = slicers.HealpixSlicer(nside=nside)
        metric = metrics.MeanMetric(col='airmass')
        sql = 'filter="r"'
        stacker1 = stackers.RandomDitherFieldPerVisitStacker()
        stacker2 = stackers.GalacticStacker()
        map1 = maps.GalCoordsMap()
        map2 = maps.StellarDensityMap()

        metricB = metricBundles.MetricBundle(metric,
                                             slicer,
                                             sql,
                                             stackerList=[stacker1, stacker2])
        filepath = os.path.join(os.getenv('SIMS_MAF_DIR'), 'tests/')

        database = os.path.join(filepath, 'opsimblitz1_1133_sqlite.db')
        opsdb = db.OpsimDatabase(database=database)
        resultsDb = db.ResultsDb(outDir=self.outDir)

        bgroup = metricBundles.MetricBundleGroup({0: metricB},
                                                 opsdb,
                                                 outDir=self.outDir,
                                                 resultsDb=resultsDb)
        bgroup.runAll()
        bgroup.plotAll()
        bgroup.writeAll()

        outThumbs = glob.glob(os.path.join(self.outDir, 'thumb*'))
        outNpz = glob.glob(os.path.join(self.outDir, '*.npz'))
        outPdf = glob.glob(os.path.join(self.outDir, '*.pdf'))

        # By default, make 3 plots for healpix
        assert (len(outThumbs) == 3)
        assert (len(outPdf) == 3)
        assert (len(outNpz) == 1)
Esempio n. 5
0
def coaddM5Analysis(path,
                    dbfile,
                    runName,
                    slair=False,
                    WFDandDDFs=False,
                    noDithOnly=False,
                    bestDithOnly=False,
                    someDithOnly=False,
                    specifiedDith=None,
                    nside=128,
                    filterBand='r',
                    includeDustExtinction=False,
                    saveunMaskedCoaddData=False,
                    pixelRadiusForMasking=5,
                    cutOffYear=None,
                    plotSkymap=True,
                    plotCartview=True,
                    unmaskedColorMin=None,
                    unmaskedColorMax=None,
                    maskedColorMin=None,
                    maskedColorMax=None,
                    nTicks=5,
                    plotPowerSpectrum=True,
                    showPlots=True,
                    saveFigs=True,
                    almAnalysis=True,
                    raRange=[-50, 50],
                    decRange=[-65, 5],
                    saveMaskedCoaddData=True):
    """

    Analyze the artifacts induced in the coadded 5sigma depth due to imperfect observing strategy.
      - Creates an output directory for subdirectories containing the specified things to save.
      - Creates, shows, and saves comparison plots.
      - Returns the metricBundle object containing the calculated coadded depth, and the output directory name.

    Required Parameters
    -------------------
      * path: str: path to the main directory where output directory is to be saved.
      * dbfile: str: path to the OpSim output file, e.g. to a copy of enigma_1189
      * runName: str: run name tag to identify the output of specified OpSim output, e.g. 'enigma1189' 

    Optional Parameters
    -------------------
      * slair: boolean: set to True if analysis on a SLAIR output.
                        Default: False
      * WFDandDDFs: boolean: set to True if want to consider both WFD survet and DDFs. Otherwise will only work
                             with WFD. Default: False
      * noDithOnly: boolean: set to True if only want to consider the undithered survey. Default: False
      * bestDithOnly: boolean: set to True if only want to consider RandomDitherFieldPerVisit.
                               Default: False
      * someDithOnly: boolean: set to True if only want to consider undithered and a few dithered surveys. 
                               Default: False
      * specifiedDith: str: specific dither strategy to run.
                            Default: None
      * nside: int: HEALpix resolution parameter. Default: 128
      * filterBand: str: any one of 'u', 'g', 'r', 'i', 'z', 'y'. Default: 'r'
      * includeDustExtinction: boolean: set to include dust extinction. Default: False
      * saveunMaskedCoaddData: boolean: set to True to save data before border masking. Default: False
      * pixelRadiusForMasking: int: number of pixels to mask along the shallow border. Default: 5

      * cutOffYear: int: year cut to restrict analysis to only a subset of the survey. 
                         Must range from 1 to 9, or None for the full survey analysis (10 yrs).
                         Default: None
      * plotSkymap: boolean: set to True if want to plot skymaps. Default: True
      * plotCartview: boolean: set to True if want to plot cartview plots. Default: False
      * unmaskedColorMin: float: lower limit on the colorscale for unmasked skymaps. Default: None
      * unmaskedColorMax: float: upper limit on the colorscale for unmasked skymaps. Default: None

      * maskedColorMin: float: lower limit on the colorscale for border-masked skymaps. Default: None
      * maskedColorMax: float: upper limit on the colorscale for border-masked skymaps. Default: None
      * nTicks: int: (number of ticks - 1) on the skymap colorbar. Default: 5
      * plotPowerSpectrum: boolean: set to True if want to plot powerspectra. Default: True

      * showPlots: boolean: set to True if want to show figures. Default: True
      * saveFigs: boolean: set to True if want to save figures. Default: True
      
      * almAnalysis: boolean: set to True to perform the alm analysis. Default: True
      * raRange: float array: range of right ascention (in degrees) to consider in alm  cartview plot;
                              applicable when almAnalysis=True. Default: [-50,50]
      * decRange: float array: range of declination (in degrees) to consider in alm cartview plot; 
                               applicable when almAnalysis=True. Default: [-65,5]
      * saveMaskedCoaddData: boolean: set to True to save the coadded depth data after the border
                                      masking. Default: True

    """
    # ------------------------------------------------------------------------
    # read in the database
    if slair:
        # slair database
        opsdb = db.Database(dbfile, defaultTable='observations')
    else:
        # OpSim database
        opsdb = db.OpsimDatabase(dbfile)

    # ------------------------------------------------------------------------
    # set up the outDir
    zeropt_tag = ''
    if cutOffYear is not None: zeropt_tag = '%syearCut' % cutOffYear
    else: zeropt_tag = 'fullSurveyPeriod'

    if includeDustExtinction: dust_tag = 'withDustExtinction'
    else: dust_tag = 'noDustExtinction'

    regionType = ''
    if WFDandDDFs: regionType = 'WFDandDDFs_'

    outDir = 'coaddM5Analysis_%snside%s_%s_%spixelRadiusForMasking_%sBand_%s_%s_directory' % (
        regionType, nside, dust_tag, pixelRadiusForMasking, filterBand,
        runName, zeropt_tag)
    print('# outDir: %s' % outDir)
    resultsDb = db.ResultsDb(outDir=outDir)

    # ------------------------------------------------------------------------
    # set up the sql constraint
    if WFDandDDFs:
        if cutOffYear is not None:
            nightCutOff = (cutOffYear) * 365.25
            sqlconstraint = 'night<=%s and filter=="%s"' % (nightCutOff,
                                                            filterBand)
        else:
            sqlconstraint = 'filter=="%s"' % filterBand
    else:
        # set up the propID and units on the ra, dec
        if slair:  # no prop ID; only WFD is simulated.
            wfdWhere = ''
            raDecInDeg = True
        else:
            propIds, propTags = opsdb.fetchPropInfo()
            wfdWhere = '%s and ' % opsdb.createSQLWhere('WFD', propTags)
            raDecInDeg = opsdb.raDecInDeg
        # set up the year cutoff
        if cutOffYear is not None:
            nightCutOff = (cutOffYear) * 365.25
            sqlconstraint = '%snight<=%s and filter=="%s"' % (
                wfdWhere, nightCutOff, filterBand)
        else:
            sqlconstraint = '%sfilter=="%s"' % (wfdWhere, filterBand)
    print('# sqlconstraint: %s' % sqlconstraint)

    # ------------------------------------------------------------------------
    # setup all the slicers
    slicer = {}
    stackerList = {}

    if specifiedDith is not None:  # would like to add all the stackers first and then keep only the one that is specified
        bestDithOnly, noDithOnly = False, False

    if bestDithOnly:
        stackerList['RandomDitherFieldPerVisit'] = [
            mafStackers.RandomDitherFieldPerVisitStacker(degrees=raDecInDeg,
                                                         randomSeed=1000)
        ]
        slicer['RandomDitherFieldPerVisit'] = slicers.HealpixSlicer(
            lonCol='randomDitherFieldPerVisitRa',
            latCol='randomDitherFieldPerVisitDec',
            latLonDeg=raDecInDeg,
            nside=nside,
            useCache=False)
    else:
        if slair:
            slicer['NoDither'] = slicers.HealpixSlicer(lonCol='RA',
                                                       latCol='dec',
                                                       latLonDeg=raDecInDeg,
                                                       nside=nside,
                                                       useCache=False)
        else:
            slicer['NoDither'] = slicers.HealpixSlicer(lonCol='fieldRA',
                                                       latCol='fieldDec',
                                                       latLonDeg=raDecInDeg,
                                                       nside=nside,
                                                       useCache=False)
        if someDithOnly and not noDithOnly:
            #stackerList['RepulsiveRandomDitherFieldPerVisit'] = [myStackers.RepulsiveRandomDitherFieldPerVisitStacker(degrees=raDecInDeg,
            #                                                                                                          randomSeed=1000)]
            #slicer['RepulsiveRandomDitherFieldPerVisit'] = slicers.HealpixSlicer(lonCol='repulsiveRandomDitherFieldPerVisitRa',
            #                                                                    latCol='repulsiveRandomDitherFieldPerVisitDec',
            #                                                                    latLonDeg=raDecInDeg, nside=nside,
            #                                                                    useCache=False)
            slicer['SequentialHexDitherFieldPerNight'] = slicers.HealpixSlicer(
                lonCol='hexDitherFieldPerNightRa',
                latCol='hexDitherFieldPerNightDec',
                latLonDeg=raDecInDeg,
                nside=nside,
                useCache=False)
            slicer['PentagonDitherPerSeason'] = slicers.HealpixSlicer(
                lonCol='pentagonDitherPerSeasonRa',
                latCol='pentagonDitherPerSeasonDec',
                latLonDeg=raDecInDeg,
                nside=nside,
                useCache=False)
        elif not noDithOnly:
            # random dithers on different timescales
            stackerList['RandomDitherPerNight'] = [
                mafStackers.RandomDitherPerNightStacker(degrees=raDecInDeg,
                                                        randomSeed=1000)
            ]
            stackerList['RandomDitherFieldPerNight'] = [
                mafStackers.RandomDitherFieldPerNightStacker(
                    degrees=raDecInDeg, randomSeed=1000)
            ]
            stackerList['RandomDitherFieldPerVisit'] = [
                mafStackers.RandomDitherFieldPerVisitStacker(
                    degrees=raDecInDeg, randomSeed=1000)
            ]

            # rep random dithers on different timescales
            #stackerList['RepulsiveRandomDitherPerNight'] = [myStackers.RepulsiveRandomDitherPerNightStacker(degrees=raDecInDeg,
            #                                                                                                randomSeed=1000)]
            #stackerList['RepulsiveRandomDitherFieldPerNight'] = [myStackers.RepulsiveRandomDitherFieldPerNightStacker(degrees=raDecInDeg,
            #                                                                                                          randomSeed=1000)]
            #stackerList['RepulsiveRandomDitherFieldPerVisit'] = [myStackers.RepulsiveRandomDitherFieldPerVisitStacker(degrees=raDecInDeg,
            #                                                                                                          randomSeed=1000)]
            # set up slicers for different dithers
            # random dithers on different timescales
            slicer['RandomDitherPerNight'] = slicers.HealpixSlicer(
                lonCol='randomDitherPerNightRa',
                latCol='randomDitherPerNightDec',
                latLonDeg=raDecInDeg,
                nside=nside,
                useCache=False)
            slicer['RandomDitherFieldPerNight'] = slicers.HealpixSlicer(
                lonCol='randomDitherFieldPerNightRa',
                latCol='randomDitherFieldPerNightDec',
                latLonDeg=raDecInDeg,
                nside=nside,
                useCache=False)
            slicer['RandomDitherFieldPerVisit'] = slicers.HealpixSlicer(
                lonCol='randomDitherFieldPerVisitRa',
                latCol='randomDitherFieldPerVisitDec',
                latLonDeg=raDecInDeg,
                nside=nside,
                useCache=False)
            # rep random dithers on different timescales
            #slicer['RepulsiveRandomDitherPerNight'] = slicers.HealpixSlicer(lonCol='repulsiveRandomDitherPerNightRa',
            #                                                               latCol='repulsiveRandomDitherPerNightDec',
            #                                                               latLonDeg=raDecInDeg, nside=nside, useCache=False)
            #slicer['RepulsiveRandomDitherFieldPerNight'] = slicers.HealpixSlicer(lonCol='repulsiveRandomDitherFieldPerNightRa',
            #                                                                    latCol='repulsiveRandomDitherFieldPerNightDec',
            #                                                                    latLonDeg=raDecInDeg, nside=nside,
            #                                                                    useCache=False)
            #slicer['RepulsiveRandomDitherFieldPerVisit'] = slicers.HealpixSlicer(lonCol='repulsiveRandomDitherFieldPerVisitRa',
            #                                                                    latCol='repulsiveRandomDitherFieldPerVisitDec',
            #                                                                    latLonDeg=raDecInDeg, nside=nside,
            #                                                                    useCache=False)
            # spiral dithers on different timescales
            slicer['FermatSpiralDitherPerNight'] = slicers.HealpixSlicer(
                lonCol='fermatSpiralDitherPerNightRa',
                latCol='fermatSpiralDitherPerNightDec',
                latLonDeg=raDecInDeg,
                nside=nside,
                useCache=False)
            slicer['FermatSpiralDitherFieldPerNight'] = slicers.HealpixSlicer(
                lonCol='fermatSpiralDitherFieldPerNightRa',
                latCol='fermatSpiralDitherFieldPerNightDec',
                latLonDeg=raDecInDeg,
                nside=nside,
                useCache=False)
            slicer['FermatSpiralDitherFieldPerVisit'] = slicers.HealpixSlicer(
                lonCol='fermatSpiralDitherFieldPerVisitRa',
                latCol='fermatSpiralDitherFieldPerVisitDec',
                latLonDeg=raDecInDeg,
                nside=nside,
                useCache=False)
            # hex dithers on different timescales
            slicer['SequentialHexDitherPerNight'] = slicers.HealpixSlicer(
                lonCol='hexDitherPerNightRa',
                latCol='hexDitherPerNightDec',
                latLonDeg=raDecInDeg,
                nside=nside,
                useCache=False)
            slicer['SequentialHexDitherFieldPerNight'] = slicers.HealpixSlicer(
                lonCol='hexDitherFieldPerNightRa',
                latCol='hexDitherFieldPerNightDec',
                latLonDeg=raDecInDeg,
                nside=nside,
                useCache=False)
            slicer['SequentialHexDitherFieldPerVisit'] = slicers.HealpixSlicer(
                lonCol='hexDitherFieldPerVisitRa',
                latCol='hexDitherFieldPerVisitDec',
                latLonDeg=raDecInDeg,
                nside=nside,
                useCache=False)
            # per season dithers
            slicer['PentagonDitherPerSeason'] = slicers.HealpixSlicer(
                lonCol='pentagonDitherPerSeasonRa',
                latCol='pentagonDitherPerSeasonDec',
                latLonDeg=raDecInDeg,
                nside=nside,
                useCache=False)
            slicer['PentagonDiamondDitherPerSeason'] = slicers.HealpixSlicer(
                lonCol='pentagonDiamondDitherPerSeasonRa',
                latCol='pentagonDiamondDitherPerSeasonDec',
                latLonDeg=raDecInDeg,
                nside=nside,
                useCache=False)
            slicer['SpiralDitherPerSeason'] = slicers.HealpixSlicer(
                lonCol='spiralDitherPerSeasonRa',
                latCol='spiralDitherPerSeasonDec',
                latLonDeg=raDecInDeg,
                nside=nside,
                useCache=False)
    if specifiedDith is not None:
        stackerList_, slicer_ = {}, {}
        if specifiedDith in slicer.keys():
            if specifiedDith.__contains__(
                    'Random'
            ):  # only Random dithers have a stacker object for rand seed specification
                stackerList_[specifiedDith] = stackerList[specifiedDith]
            slicer_[specifiedDith] = slicer[specifiedDith]
        else:
            raise ValueError(
                'Invalid value for specifiedDith: %s. Allowed values include one of the following:\n%s'
                % (specifiedDith, slicer.keys()))
        stackerList, slicer = stackerList_, slicer_

    # ------------------------------------------------------------------------
    if slair:
        m5Col = 'fivesigmadepth'
    else:
        m5Col = 'fiveSigmaDepth'
    # set up the metric
    if includeDustExtinction:
        # include dust extinction when calculating the co-added depth
        coaddMetric = metrics.ExgalM5(m5Col=m5Col, lsstFilter=filterBand)
    else:
        coaddMetric = metrics.Coaddm5Metric(m5col=m5col)
    dustMap = maps.DustMap(
        interp=False, nside=nside
    )  # include dustMap; actual in/exclusion of dust is handled by the galaxyCountMetric

    # ------------------------------------------------------------------------
    # set up the bundle
    coaddBundle = {}
    for dither in slicer:
        if dither in stackerList:
            coaddBundle[dither] = metricBundles.MetricBundle(
                coaddMetric,
                slicer[dither],
                sqlconstraint,
                stackerList=stackerList[dither],
                runName=runName,
                metadata=dither,
                mapsList=[dustMap])
        else:
            coaddBundle[dither] = metricBundles.MetricBundle(
                coaddMetric,
                slicer[dither],
                sqlconstraint,
                runName=runName,
                metadata=dither,
                mapsList=[dustMap])

    # ------------------------------------------------------------------------
    # run the analysis
    if includeDustExtinction:
        print('\n# Running coaddBundle with dust extinction ...')
    else:
        print('\n# Running coaddBundle without dust extinction ...')
    cGroup = metricBundles.MetricBundleGroup(coaddBundle,
                                             opsdb,
                                             outDir=outDir,
                                             resultsDb=resultsDb,
                                             saveEarly=False)
    cGroup.runAll()

    # ------------------------------------------------------------------------
    # plot and save the data
    plotBundleMaps(path,
                   outDir,
                   coaddBundle,
                   dataLabel='$%s$-band Coadded Depth' % filterBand,
                   filterBand=filterBand,
                   dataName='%s-band Coadded Depth' % filterBand,
                   skymap=plotSkymap,
                   powerSpectrum=plotPowerSpectrum,
                   cartview=plotCartview,
                   colorMin=unmaskedColorMin,
                   colorMax=unmaskedColorMax,
                   nTicks=nTicks,
                   showPlots=showPlots,
                   saveFigs=saveFigs,
                   outDirNameForSavedFigs='coaddM5Plots_unmaskedBorders')
    print('\n# Done saving plots without border masking.\n')

    # ------------------------------------------------------------------------
    plotHandler = plots.PlotHandler(outDir=outDir,
                                    resultsDb=resultsDb,
                                    thumbnail=False,
                                    savefig=False)

    print(
        '# Number of pixels in the survey region (before masking the border):')
    for dither in coaddBundle:
        print(
            '  %s: %s' %
            (dither,
             len(np.where(coaddBundle[dither].metricValues.mask == False)[0])))

    # ------------------------------------------------------------------------
    # save the unmasked data?
    if saveunMaskedCoaddData:
        outDir_new = 'unmaskedCoaddData'
        if not os.path.exists('%s%s/%s' % (path, outDir, outDir_new)):
            os.makedirs('%s%s/%s' % (path, outDir, outDir_new))
        saveBundleData_npzFormat('%s%s/%s' % (path, outDir, outDir_new),
                                 coaddBundle, 'coaddM5Data_unmasked',
                                 filterBand)

    # ------------------------------------------------------------------------
    # mask the edges
    print('\n# Masking the edges for coadd ...')
    coaddBundle = maskingAlgorithmGeneralized(
        coaddBundle,
        plotHandler,
        dataLabel='$%s$-band Coadded Depth' % filterBand,
        nside=nside,
        pixelRadius=pixelRadiusForMasking,
        plotIntermediatePlots=False,
        plotFinalPlots=False,
        printFinalInfo=True)
    if (pixelRadiusForMasking > 0):
        # plot and save the masked data
        plotBundleMaps(path,
                       outDir,
                       coaddBundle,
                       dataLabel='$%s$-band Coadded Depth' % filterBand,
                       filterBand=filterBand,
                       dataName='%s-band Coadded Depth' % filterBand,
                       skymap=plotSkymap,
                       powerSpectrum=plotPowerSpectrum,
                       cartview=plotCartview,
                       colorMin=maskedColorMin,
                       colorMax=maskedColorMax,
                       nTicks=nTicks,
                       showPlots=showPlots,
                       saveFigs=saveFigs,
                       outDirNameForSavedFigs='coaddM5Plots_maskedBorders')
        print('\n# Done saving plots with border masking. \n')

    # ------------------------------------------------------------------------
    # Calculate total power
    summarymetric = metrics.TotalPowerMetric()
    for dither in coaddBundle:
        coaddBundle[dither].setSummaryMetrics(summarymetric)
        coaddBundle[dither].computeSummaryStats()
        print('# Total power for %s case is %f.' %
              (dither, coaddBundle[dither].summaryValues['TotalPower']))
    print('')

    # ------------------------------------------------------------------------
    # run the alm analysis
    if almAnalysis:
        almPlots(path,
                 outDir,
                 copy.deepcopy(coaddBundle),
                 nside=nside,
                 filterband=filterBand,
                 raRange=raRange,
                 decRange=decRange,
                 showPlots=showPlots)
    # ------------------------------------------------------------------------
    # save the masked data?
    if saveMaskedCoaddData and (pixelRadiusForMasking > 0):
        outDir_new = 'maskedCoaddData'
        if not os.path.exists('%s%s/%s' % (path, outDir, outDir_new)):
            os.makedirs('%s%s/%s' % (path, outDir, outDir_new))
        saveBundleData_npzFormat('%s%s/%s' % (path, outDir, outDir_new),
                                 coaddBundle, 'coaddM5Data_masked', filterBand)

    # ------------------------------------------------------------------------
    # plot comparison plots
    if len(coaddBundle.keys()) > 1:  # more than one key
        # set up the directory
        outDir_comp = 'coaddM5ComparisonPlots'
        if not os.path.exists('%s%s/%s' % (path, outDir, outDir_comp)):
            os.makedirs('%s%s/%s' % (path, outDir, outDir_comp))
        # ------------------------------------------------------------------------
        # plot for the power spectra
        cl = {}
        for dither in plotColor:
            if dither in coaddBundle:
                cl[dither] = hp.anafast(hp.remove_dipole(
                    coaddBundle[dither].metricValues.filled(
                        coaddBundle[dither].slicer.badval)),
                                        lmax=500)
                ell = np.arange(np.size(cl[dither]))
                plt.plot(ell, (cl[dither] * ell * (ell + 1)) / 2.0 / np.pi,
                         color=plotColor[dither],
                         linestyle='-',
                         label=dither)
        plt.xlabel(r'$\ell$')
        plt.ylabel(r'$\ell(\ell+1)C_\ell/(2\pi)$')
        plt.xlim(0, 500)
        fig = plt.gcf()
        fig.set_size_inches(12.5, 10.5)
        leg = plt.legend(labelspacing=0.001)
        for legobj in leg.legendHandles:
            legobj.set_linewidth(4.0)
        filename = 'powerspectrum_comparison_all.png'
        plt.savefig('%s%s/%s/%s' % (path, outDir, outDir_comp, filename),
                    bbox_inches='tight',
                    format='png')
        plt.show()

        # create the histogram
        scale = hp.nside2pixarea(nside, degrees=True)

        def tickFormatter(y, pos):
            return '%d' % (y * scale)  # convert pixel count to area

        binsize = 0.01
        for dither in plotColor:
            if dither in coaddBundle:
                ind = np.where(
                    coaddBundle[dither].metricValues.mask == False)[0]
                binAll = int(
                    (max(coaddBundle[dither].metricValues.data[ind]) -
                     min(coaddBundle[dither].metricValues.data[ind])) /
                    binsize)
                plt.hist(coaddBundle[dither].metricValues.data[ind],
                         bins=binAll,
                         label=dither,
                         histtype='step',
                         color=plotColor[dither])
        ax = plt.gca()
        ymin, ymax = ax.get_ylim()
        nYticks = 10.
        wantedYMax = ymax * scale
        wantedYMax = 10. * np.ceil(float(wantedYMax) / 10.)
        increment = 5. * np.ceil(float(wantedYMax / nYticks) / 5.)
        wantedArray = np.arange(0, wantedYMax, increment)
        ax.yaxis.set_ticks(wantedArray / scale)
        ax.yaxis.set_major_formatter(FuncFormatter(tickFormatter))
        plt.xlabel('$%s$-band Coadded Depth' % filterBand)
        plt.ylabel('Area (deg$^2$)')
        fig = plt.gcf()
        fig.set_size_inches(12.5, 10.5)
        leg = plt.legend(labelspacing=0.001, loc=2)
        for legobj in leg.legendHandles:
            legobj.set_linewidth(2.0)
        filename = 'histogram_comparison.png'
        plt.savefig('%s%s/%s/%s' % (path, outDir, outDir_comp, filename),
                    bbox_inches='tight',
                    format='png')
        plt.show()
        # ------------------------------------------------------------------------
        # plot power spectra for the separte panel
        totKeys = len(list(coaddBundle.keys()))
        if (totKeys > 1):
            plt.clf()
            nCols = 2
            nRows = int(np.ceil(float(totKeys) / nCols))
            fig, ax = plt.subplots(nRows, nCols)
            plotRow = 0
            plotCol = 0
            for dither in list(plotColor.keys()):
                if dither in list(coaddBundle.keys()):
                    ell = np.arange(np.size(cl[dither]))
                    ax[plotRow, plotCol].plot(ell, (cl[dither] * ell *
                                                    (ell + 1)) / 2.0 / np.pi,
                                              color=plotColor[dither],
                                              label=dither)
                    if (plotRow == nRows - 1):
                        ax[plotRow, plotCol].set_xlabel(r'$\ell$')
                    ax[plotRow,
                       plotCol].set_ylabel(r'$\ell(\ell+1)C_\ell/(2\pi)$')
                    ax[plotRow,
                       plotCol].yaxis.set_major_locator(MaxNLocator(3))
                    if (dither != 'NoDither'):
                        ax[plotRow, plotCol].set_ylim(0, 0.0035)
                    ax[plotRow, plotCol].set_xlim(0, 500)
                    plotRow += 1
                    if (plotRow > nRows - 1):
                        plotRow = 0
                        plotCol += 1
            fig.set_size_inches(20, int(nRows * 30 / 7.))
            filename = 'powerspectrum_sepPanels.png'
            plt.savefig('%s%s/%s/%s' % (path, outDir, outDir_comp, filename),
                        bbox_inches='tight',
                        format='png')
            plt.show()
    return coaddBundle, outDir
Esempio n. 6
0
def save_csv_dithers(dbs_path,
                     outDir,
                     db_files_only=None,
                     rot_rand_seed=42,
                     trans_rand_seed=42,
                     print_progress=True,
                     show_diagnostic_plots=False,
                     save_plots=False):
    """
    
    The goal here is to calculate the translational and rotational dithers for
    various cadences and save the output as a csv file.  These dithers are largely
    the same as in DC1/DC2: 
        - Translational dithers:
            - WFD: large random offsets (as large as 1.75 deg) applied after every visit.
            - DD: small random offsets (as large as 7 arcmin) applied after every visit.
            - Else: no dithers, so `fieldRA`, `fieldDec` are returned.
        - Rotational dithers:
            - All surveys (WFD, DD, else): random between -90, 90 degrees applied after
                                           every filter change. (Break from DC2: Some visits
                                           dont get dithered since they are forced outside
                                           the rotator range.
                                           See RotStacker info for details.)
             
    Supports OpSim V3/V4 outputs.
    
    Required Inputs
    ---------------
    * dbs_path: str: path to the directory that contains the .db files; could have non-.db files.
    * outDir: str: path to the directory where the output should be saved.
    
    Optional Inputs
    ---------------
    * db_files_only: list of str: list of names of the db files to run.
                                  Default: None. Runs over all the files in db_path.
    * rot_rand_seed: int: seed for random number generator for rotational dithers.
                          Default: 42
    * trans_rand_seed: int: seed for random number generator for translational dithers.
                            Default: 42
    * print_progress: bool: set to False to not print progress.
                            Default: True
    * show_diagnostic_plots: bool: set to True to show histogram of added dithers.
                                   Default: False
    * save_plots: bool: set to True to save the histogram for descDithers in outDir.
                        Default: False
                                   
    Saved file format
    -----------------
    .csv file with four columns:
        obsIDcol, 'descDitheredRA', 'descDitheredDec', 'descDitheredRotTelPos'
    where
        obsIDcol = 'observationId' for V4 outputs and 'obsHistID' for V3 outputs.
    
    Saved filename = descDithers_<database name>.csv
    
    """
    startTime_0 = time.time()
    readme = '##############################\n%s' % (datetime.date.isoformat(
        datetime.date.today()))
    readme += '\nRunning with lsst.sims.maf.__version__: %s' % lsst.sims.maf.__version__
    readme += '\n\nsave_csv_dithers run:\ndbs_path= %s\n' % dbs_path
    readme += 'outDir: %s' % outDir
    readme += 'db_files_only: %s' % db_files_only
    readme += 'rot_rand_seed=%s\ntrans_rand_seed=%s' % (rot_rand_seed,
                                                        trans_rand_seed)
    readme += 'print_progress=%s\show_diagnostic_plots=%s\n' % (
        print_progress, show_diagnostic_plots)

    dbfiles = [f for f in os.listdir(dbs_path)
               if f.endswith('db')]  # select db files
    if print_progress: print('Found files: %s\n' % dbfiles)

    if db_files_only is not None:
        dbfiles = [f for f in dbfiles if f in db_files_only]  # select db files

    readme += '\nReading for files: %s\n\n' % dbfiles
    if print_progress and db_files_only is not None:
        print('Running over: %s\n' % dbfiles)

    for i, dbfile in enumerate(dbfiles):  # loop over all the db files
        startTime = time.time()
        if (i != 0): readme = ''
        readme += '%s' % dbfile

        if print_progress: print('Starting: %s\n' % dbfile)

        opsdb = db.OpsimDatabase('%s/%s' %
                                 (dbs_path, dbfile))  # connect to the database

        # specify the column names to get from the db file
        colnames = [
            'proposalId', 'observationId', 'fieldRA', 'fieldDec', 'rotTelPos'
        ]
        propIDcol, obsIDcol = 'proposalId', 'observationId'

        if (opsdb.opsimVersion == 'V3'):
            # V3 outputs have somewhat different column names
            colnames = [
                'propID', 'obsHistID', 'fieldRA', 'fieldDec', 'rotTelPos'
            ]
            propIDcol, obsIDcol = 'propID', 'obsHistID'

        # get the data
        simdata = opsdb.fetchMetricData(colnames=colnames, sqlconstraint=None)

        # set up to run the stackers that add columns for translational and rotational dithers.
        metric = metrics.PassMetric(
        )  # want to access the database; no analysis needed
        slicer = slicers.OneDSlicer(
            sliceColName='night', binsize=1,
            verbose=print_progress)  # essentially accessing all nights
        sqlconstraint = None

        resultsDb = db.ResultsDb(outDir=outDir)
        ################################################################################################
        # set up metric bundle to run stackers for large translational dithers + rotational dithers
        if print_progress:
            print('Setting up for WFD translational dithers + rot dithers.')
        bgroup = {}
        stackerList = [
            stackers.RandomDitherFieldPerVisitStacker(
                degrees=opsdb.raDecInDeg, randomSeed=trans_rand_seed),
            stackers.RandomRotDitherPerFilterChangeStacker(
                degrees=opsdb.raDecInDeg, randomSeed=rot_rand_seed)
        ]

        bundle = metricBundles.MetricBundle(metric,
                                            slicer,
                                            sqlconstraint=sqlconstraint,
                                            stackerList=stackerList)
        bgroup['WFD'] = metricBundles.MetricBundleGroup({0: bundle},
                                                        opsdb,
                                                        outDir=outDir,
                                                        resultsDb=resultsDb,
                                                        saveEarly=False,
                                                        verbose=print_progress)
        # run the bundle
        bgroup['WFD'].runAll()

        # set up the bundle for small translational dithers
        if print_progress: print('\nSetting up for DD translational dithers.')
        chipSize = 1.75 * 2 / 15
        chipMaxDither = chipSize / 2.
        stackerList = [
            stackers.RandomDitherFieldPerVisitStacker(
                maxDither=chipMaxDither,
                degrees=opsdb.raDecInDeg,
                randomSeed=trans_rand_seed)
        ]
        bundle = metricBundles.MetricBundle(metric,
                                            slicer,
                                            sqlconstraint=sqlconstraint,
                                            stackerList=stackerList)

        bgroup['DD'] = metricBundles.MetricBundleGroup({0: bundle},
                                                       opsdb,
                                                       outDir=outDir,
                                                       resultsDb=resultsDb,
                                                       saveEarly=False,
                                                       verbose=print_progress)
        # run the bundle
        bgroup['DD'].runAll()

        ################################################################################################
        # access the relevant columns
        dithered_RA, dithered_Dec = {}, {}
        for key in bgroup:
            dithered_RA[key] = bgroup[key].simData[
                'randomDitherFieldPerVisitRa']
            dithered_Dec[key] = bgroup[key].simData[
                'randomDitherFieldPerVisitDec']

        dithered_rotTelPos = bgroup['WFD'].simData[
            'randomDitherPerFilterChangeRotTelPos']

        ################################################################################################
        # diagnostic plots
        if show_diagnostic_plots:
            # histograms of dithers
            fig, axes = plt.subplots(nrows=1, ncols=3)

            for key in bgroup:
                # ra
                axes[0].hist(dithered_RA[key] - simdata['fieldRA'],
                             label='%s dithers: delRA' % key,
                             histtype='step',
                             lw=2,
                             bins=30)

                # dec
                axes[1].hist(dithered_Dec[key] - simdata['fieldDec'],
                             label='%s dithers: delDec' % key,
                             histtype='step',
                             lw=2)

            # tel pos
            axes[2].hist(dithered_rotTelPos - simdata['rotTelPos'],
                         label='rot dithers: rotTelPos',
                         histtype='step',
                         lw=2)
            for ax in axes:
                ax.ticklabel_format(style='sci', axis='y', scilimits=(0, 0))
                ax.set_ylabel('Counts')

            axes[0].legend()
            axes[1].legend()

            if opsdb.raDecInDeg: unitlabel = 'degrees'
            else: unitlabel = 'radians'

            axes[0].set_xlabel('delRA (%s)' % unitlabel)
            axes[1].set_xlabel('delDec (%s)' % unitlabel)
            axes[2].set_xlabel('delRotTelPos (%s)' % unitlabel)

            plt.title(dbfile)
            fig.set_size_inches(20, 5)

        ################################################################################################
        # initiate the final arrays as undithered fieldRA, fieldDec as nonWFD, nonDDF should remain unchanged
        descDitheredRA = simdata['fieldRA'].copy()
        descDitheredDec = simdata['fieldDec'].copy()
        descDitheredRot = simdata['rotTelPos'].copy()

        # need to find the indices for WFD vs. DD observations since we are adding different
        # translational dithers for WFD/DDF visits + none for other surveys
        propIds, propTags = opsdb.fetchPropInfo()
        # ok work with WFD visits now
        ind_WFD = np.where(simdata[propIDcol] == propTags['WFD'])[0]
        if print_progress:
            tot = len(simdata)
            print('Total visits: ', tot)
            print('propTags: ', propTags)
            print('%s WFD visits out of total %s' % (len(ind_WFD), tot))

        descDitheredRA[ind_WFD] = dithered_RA['WFD'][ind_WFD]
        descDitheredDec[ind_WFD] = dithered_Dec['WFD'][ind_WFD]

        # work with DD visits now
        ind_DD = np.where(simdata[propIDcol] == propTags['DD'])[0]
        if print_progress:
            print('%s DD visits out of total %s' % (len(ind_DD), tot))

        descDitheredRA[ind_DD] = dithered_RA['DD'][ind_DD]
        descDitheredDec[ind_DD] = dithered_Dec['DD'][ind_DD]

        # add rotational dithers to everything
        descDitheredRot = dithered_rotTelPos

        ###############################################################
        # diagnostic plots
        if show_diagnostic_plots or save_plots:
            # histograms of desc dithered positions
            fig, axes = plt.subplots(nrows=1, ncols=3)

            _, bins, _ = axes[0].hist(descDitheredRA,
                                      label='descDitheredRA',
                                      histtype='step',
                                      lw=2)
            axes[0].hist(simdata['fieldRA'],
                         label='fieldRA',
                         histtype='step',
                         lw=2,
                         bins=bins)

            _, bins, _ = axes[1].hist(descDitheredDec,
                                      label='descDitheredDec',
                                      histtype='step',
                                      lw=2)
            axes[1].hist(simdata['fieldDec'],
                         label='fieldDec',
                         histtype='step',
                         lw=2,
                         bins=bins)

            _, bins, _ = axes[2].hist(descDitheredRot,
                                      label='descDitheredRot',
                                      histtype='step',
                                      lw=2)
            axes[2].hist(simdata['rotTelPos'],
                         label='rotTelPos',
                         histtype='step',
                         lw=2,
                         bins=bins)

            if opsdb.raDecInDeg: xlabel = 'degrees'
            else: xlabel = 'radians'

            for ax in axes:
                ax.legend()
                ax.set_xlabel(xlabel)
                ax.set_ylabel('Counts')

            plt.suptitle(dbfile)
            fig.set_size_inches(20, 5)

            if save_plots:
                filename = 'hist_descDithers_%s.png' % (dbfile.split('.db')[0])
                plt.savefig('%s/%s' % (outDir, filename),
                            format='png',
                            bbox_inches='tight')
                readme += '\nSaved hist for descDithers in %s.' % filename

                if print_progress:
                    print('\nSaved hist plot in %s' % filename)

            if show_diagnostic_plots:
                plt.show()
            else:
                plt.close('all')

        ###############################################################
        # save the columns as a csv file.
        d = {
            obsIDcol: simdata[obsIDcol],
            'descDitheredRA': descDitheredRA,
            'descDitheredDec': descDitheredDec,
            'descDitheredRotTelPos': descDitheredRot
        }

        filename = 'descDithers_%s.csv' % (dbfile.split('.db')[0])
        pd.DataFrame(d).to_csv('%s/%s' % (outDir, filename), index=False)

        readme += '\nSaved the dithers in %s' % filename
        readme += '\nTime taken: %.2f (min)\n\n' % (
            (time.time() - startTime) / 60.)

        if print_progress:
            print('\nSaved the dithers in %s' % filename)
            print('Time taken: %.2f (min)\n\n' %
                  ((time.time() - startTime) / 60.))

        readme_file = open('%s/readme.txt' % (outDir), 'a')
        readme_file.write(readme)
        readme_file.close()

    # mark the end in the readme.
    readme_file = open('%s/readme.txt' % (outDir), 'a')
    readme_file.write('All done. Total time taken: %.2f (min)\n\n' %
                      ((time.time() - startTime_0) / 60.))
    readme_file.close()