def testAddRun(self): """Test adding a run to the tracking database.""" trackingdb = db.TrackingDb(database=self.trackingDb) trackId = trackingdb.addRun(opsimRun=self.opsimRun, opsimComment=self.opsimComment, mafComment=self.mafComment, mafDir=self.mafDir, mafDate=self.mafDate, opsimDate=self.opsimDate) tdb = db.Database(database=self.trackingDb, dbTables={'runs': ['runs', 'mafRunId']}) res = tdb.queryDatabase('runs', 'select * from runs') self.assertEqual(res['mafRunId'][0], trackId) # Try adding this run again. Should just return previous trackId without adding. trackId2 = trackingdb.addRun(opsimRun=self.opsimRun, opsimComment=self.opsimComment, mafComment=self.mafComment, mafDir=self.mafDir, mafDate=self.mafDate, opsimDate=self.opsimDate) self.assertEqual(trackId, trackId2) # Test will add run, if we use 'override=True'. Also works to use None's. trackId3 = trackingdb.addRun(opsimRun=None, opsimComment=None, mafComment=None, mafDir=self.mafDir, override=True, mafDate=self.mafDate, opsimDate=self.opsimDate) self.assertNotEqual(trackId, trackId3) trackingdb.close()
def test_testAddRun(self): """Test adding a run to the tracking database.""" tempdir = tempfile.mkdtemp(prefix='trackDb') trackingDbFile = os.path.join(tempdir, 'tracking.db') trackingdb = db.TrackingDb(database=trackingDbFile) trackId = trackingdb.addRun(opsimGroup=self.opsimGroup, opsimRun=self.opsimRun, opsimComment=self.opsimComment, opsimVersion=self.opsimVersion, opsimDate=self.opsimDate, mafComment=self.mafComment, mafDir=self.mafDir, mafVersion=self.mafVersion, mafDate=self.mafDate, dbFile=self.dbFile) tdb = db.Database(database=trackingDbFile) res = tdb.query_arbitrary('select * from runs') self.assertEqual(res['mafRunId'][0], trackId) # Try adding this run again. Should return previous trackId. trackId2 = trackingdb.addRun(mafDir=self.mafDir) self.assertEqual(trackId, trackId2) # Test will add additional run, with new trackId. trackId3 = trackingdb.addRun(mafDir='test2') self.assertNotEqual(trackId, trackId3) trackingdb.close() tdb.close() shutil.rmtree(tempdir)
def run_glance(outDir, dbname): conn = db.Database(dbname, defaultTable='observations') resultsDb = db.ResultsDb(outDir=outDir) mbg = MetricBundleGroup(bd, conn, outDir=outDir, resultsDb=resultsDb) mbg.runAll() mbg.plotAll() conn.close()
def testBaseDatabase(self): """Test base database class.""" # Test instantation with no dbTables info (and no defaults). basedb = db.Database(database=self.database, driver=self.driver) self.assertEqual(basedb.tables, None) # Test instantiation with some tables. basedb = db.Database(database=self.database, driver=self.driver, dbTables={'obsHistTable':['ObsHistory', 'obsHistID'], 'fieldTable':['Field', 'fieldID'], 'obsHistoryProposalTable':['Obshistory_Proposal', 'obsHistory_propID']}) self.assertEqual(set(basedb.tables.keys()), set(['obsHistTable', 'obsHistoryProposalTable', 'fieldTable'])) # Test general query with a simple query. query = 'select fieldID, fieldRA, fieldDec from Field where fieldDec>0' data = basedb.queryDatabase('fieldTable', query) self.assertEqual(data.dtype.names, ('fieldID', 'fieldRA', 'fieldDec'))
def connectDb(dbfile): version = db.testOpsimVersion(dbfile) if version == "Unknown": opsdb = db.Database(dbfile) colmap = batches.ColMapDict('barebones') elif version == "V3": opsdb = db.OpsimDatabaseV3(dbfile) colmap = batches.ColMapDict('OpsimV3') elif version == "V4": opsdb = db.OpsimDatabaseV4(dbfile) colmap = batches.ColMapDict('OpsimV4') return opsdb, colmap
def testDelRun(self): """Test removing a run from the tracking database.""" trackingdb = db.TrackingDb(database=self.trackingDb) tdb = db.Database(database=self.trackingDb, dbTables={'runs': ['runs', 'mafRunId']}) # Add a run. trackId = trackingdb.addRun(mafDir=self.mafDir) res = tdb.queryDatabase('runs', 'select * from runs') self.assertEqual(res['mafRunId'][0], trackId) # Test removal works. trackingdb.delRun(trackId) res = tdb.queryDatabase('runs', 'select * from runs') self.assertTrue(len(res) == 0) # Test cannot remove run which does not exist. self.assertRaises(Exception, trackingdb.delRun, trackId) trackingdb.close()
def runGlance(dbfile, outDir='Glance', runName='runname', camera='LSST'): conn = db.Database(dbfile, defaultTable='observations') colmap = {'ra': 'RA', 'dec': 'dec', 'mjd': 'mjd', 'exptime': 'exptime', 'visittime': 'exptime', 'alt': 'alt', 'az': 'az', 'filter': 'filter', 'fiveSigmaDepth': 'fivesigmadepth', 'night': 'night', 'slewtime': 'slewtime', 'seeingGeom': 'FWHM_geometric', 'rotSkyPos': 'rotSkyPos', 'raDecDeg': True, 'slewdist': None, 'note': 'note'} gb = glanceBatch(colmap=colmap, slicer_camera=camera) resultsDb = db.ResultsDb(outDir=outDir) group = metricBundles.MetricBundleGroup(gb, conn, outDir=outDir, resultsDb=resultsDb) group.runAll() group.plotAll()
def __init__(self, database=None): """ Instantiate the (multi-run) layout visualization class. Parameters ---------- database :str Path to the sqlite tracking database file. If not set, looks for 'trackingDb_sqlite.db' file in current directory. """ if database is None: database = os.path.join(os.getcwd(), 'trackingDb_sqlite.db') # Read in the results database. database = db.Database(database=database, longstrings=True, dbTables={'runs': ['runs', 'mafRunId']}) self.runs = database.queryDatabase('runs', 'select * from runs') self.runs = self.sortRuns(self.runs) self.runsPage = {}
def test_testDelRun(self): """Test removing a run from the tracking database.""" tempdir = tempfile.mkdtemp(prefix='trackDb') trackingDbFile = os.path.join(tempdir, 'tracking.db') trackingdb = db.TrackingDb(database=trackingDbFile) tdb = db.Database(database=trackingDbFile) # Add two runs. trackId = trackingdb.addRun(mafDir=self.mafDir) trackId2 = trackingdb.addRun(mafDir=self.mafDir + 'test2') res = tdb.query_arbitrary('select * from runs') self.assertEqual(res['mafRunId'][0], trackId) # Test removal works. trackingdb.delRun(trackId) res = tdb.query_arbitrary('select * from runs') self.assertTrue(len(res) == 1) # Test cannot remove run which does not exist. self.assertRaises(Exception, trackingdb.delRun, trackId) trackingdb.close() tdb.close() shutil.rmtree(tempdir)
def testBaseDatabase(self): """Test base database class.""" # Test instantiation connects to expected tables. basedb = db.Database(database=self.database, driver=self.driver) expectedTables = ['Config', 'ScheduledDowntime', 'SlewMaxSpeeds', 'Field', 'Session', 'SummaryAllProps', 'ObsExposures', 'SlewActivities', 'TargetExposures', 'ObsHistory', 'SlewFinalState', 'TargetHistory', 'ObsProposalHistory', 'SlewHistory', 'TargetProposalHistory', 'Proposal', 'ProposalField', 'SlewInitialState', 'UnscheduledDowntime'] self.assertEqual(set(basedb.tableNames), set(expectedTables)) # Test general query with a simple query. query = 'select fieldId, ra, dec from Field where dec>0 limit 3' data = basedb.query_arbitrary(query) self.assertEqual(len(data), 3) # Test query columns with a simple query. data = basedb.query_columns('Field', colnames=['fieldId', 'ra', 'dec'], numLimit=3) self.assertEqual(data.dtype.names, ('fieldId', 'ra', 'dec')) self.assertEqual(len(data), 3)
def __init__(self, database=None): """ Instantiate the (multi-run) layout visualization class. Parameters ---------- database :str Path to the sqlite tracking database file. If not set, looks for 'trackingDb_sqlite.db' file in current directory. """ if database is None: database = os.path.join(os.getcwd(), 'trackingDb_sqlite.db') # Read in the results database. tdb = db.Database(database=database, longstrings=True) cols = [ 'mafRunId', 'opsimRun', 'opsimGroup', 'mafComment', 'opsimComment', 'dbFile', 'mafDir', 'opsimVersion', 'opsimDate', 'mafVersion', 'mafDate' ] self.runs = tdb.query_columns('runs', colnames=cols) self.runs = self.sortRuns(self.runs) self.runsPage = {}
def testAddRun(self): """Test adding a run to the tracking database.""" trackingdb = db.TrackingDb(database=self.trackingDb) trackId = trackingdb.addRun(opsimGroup=self.opsimGroup, opsimRun=self.opsimRun, opsimComment=self.opsimComment, opsimVersion=self.opsimVersion, opsimDate=self.opsimDate, mafComment=self.mafComment, mafDir=self.mafDir, mafVersion=self.mafVersion, mafDate=self.mafDate, dbFile=self.dbFile) tdb = db.Database(database=self.trackingDb, dbTables={'runs': ['runs', 'mafRunId']}) res = tdb.queryDatabase('runs', 'select * from runs') self.assertEqual(res['mafRunId'][0], trackId) # Try adding this run again. Should return previous trackId. trackId2 = trackingdb.addRun(mafDir=self.mafDir) self.assertEqual(trackId, trackId2) # Test will add additional run, with new trackId. trackId3 = trackingdb.addRun(mafDir='test2') self.assertNotEqual(trackId, trackId3) trackingdb.close()
def genCatalog(visits, starsDbAddress, offsets=None, lsstFilter='r', raBlockSize=20., decBlockSize=10., nPatches=16, radiusFoV=1.8, verbose=True, seed=42, uncertFloor=0.005, obsFile='observations.dat', truthFile='starInfo.dat'): """ Generate a catalog of observed stellar magnitudes. visits: A numpy array with the properties of the visits. Expected to have Opsim-like values starsDbAddress: a sqlAlchemy address pointing to a database that contains properties of stars used as input. offsets: A list of instatiated classes that will apply offsets to the stars lsstFilter: Which filter to use for the observed stars obsFile: File to write the observed stellar magnitudes to truthFile: File to write the true stellar magnitudes to raBlockSize/decBlockSize: Size of chucks to use when looping over the sky nPatches: Number of patches to divide the FoV into. Must be an integer squared radiusFoV: Radius of the telescope field of view in degrees seed: random number seed uncertFloor: value to add in quadrature to magnitude uncertainties """ if offsets is None: # Maybe change this to just run with a default SNR offset warnings.warn("Warning, no offsets set, returning without running") return # Open files for writing ofile = open(obsFile, 'w') tfile = open(truthFile, 'w') print('#PatchID, StarID, ObservedMag, MagUncertainty, Radius, HPID', file=ofile) print('#StarID, TrueMag, ra, dec', file=tfile) # Set up connection to stars db: msrgbDB = db.Database(starsDbAddress, dbTables={'stars': ['stars', 'id']}) starCols = ['id', 'rmag', 'gmag', 'ra', 'decl'] if lsstFilter + 'mag' not in starCols: starCols.append(lsstFilter + 'mag') # Loop over the sky raBlocks = np.arange(0., 2. * np.pi, np.radians(raBlockSize)) decBlocks = np.arange(-np.pi, np.pi, np.radians(decBlockSize)) # List to keep track of which starIDs have been written to truth file idsUsed = [] nObs = 0 # For computing what the 'expected' uncertainty on the observation will be magUncert = OffsetSNR(lsstFilter=lsstFilter) # set the radius for the kdtree x0, y0, z0 = (1, 0, 0) x1, y1, z1 = treexyz(np.radians(radiusFoV), 0) treeRadius = np.sqrt((x1 - x0)**2 + (y1 - y0)**2 + (z1 - z0)**2) for raBlock in raBlocks: for decBlock in decBlocks: # The idea here is that this could be run in parallel or not and is still repeatable # b/c the seed will always be the same for each block. np.random.seed(seed) seed += 1 #This should make it possible to run in parallel and maintain repeatability. visitsIn = visits[np.where( (visits['ra'] >= raBlock) & (visits['ra'] < raBlock + np.radians(raBlockSize)) & (visits['dec'] >= decBlock) & (visits['dec'] < decBlock + np.radians(decBlockSize)))] if np.size(visitsIn) > 0: # Fetch the stars in this block+the radiusFoV decPad = radiusFoV raPad = radiusFoV # Need to deal with wrap around effects. decMin = capDec(np.degrees(decBlock) - decPad) decMax = capDec(np.degrees(decBlock) + decBlockSize + decPad) sqlwhere = 'decl > %f and decl < %f ' % (decMin, decMax) raMin = np.degrees(raBlock) - raPad / np.cos( np.radians(decMin)) raMax = np.degrees(raBlock) + raBlockSize + raPad / np.cos( np.radians(decMin)) if (wrapRA(raMin) != raMin) & (wrapRA(raMax) != raMax): # near a pole, just grab all the stars sqlwhere += '' else: raMin = wrapRA(raMin) raMax = wrapRA(raMax) # no wrap if raMin < raMax: sqlwhere += 'and ra < %f and ra > %f ' % (raMax, raMin) # One side wrapped else: sqlwhere += 'and (ra > %f or ra < %f)' % (raMin, raMax) print('quering stars with: ' + sqlwhere) stars = msrgbDB.tables['stars'].query_columns_Array( colnames=starCols, constraint=sqlwhere) print('got %i stars' % stars.size) # Add all the columns I will want here, then I only do # one numpy stack per block rather than lots of stacks per visit # Ugh, feels like writing fortran though... newcols = [ 'x', 'y', 'radius', 'patchID', 'subPatch', 'hpID', 'hp1', 'hp2', 'hp3', 'hp4' ] newtypes = [float, float, float, int, int, int] stars = rfn.merge_arrays([ stars, np.zeros(stars.size, dtype=list(zip(newcols, newtypes))) ], flatten=True, usemask=False) # Build a KDTree for the stars starTree = buildTree(np.radians(stars['ra']), np.radians(stars['decl'])) newIDs = np.in1d(stars['id'], idsUsed, invert=True, assume_unique=True) newIDs = np.arange(stars['id'].size)[newIDs] for newID in newIDs: print('%i, %f, %f, %f' % (stars['id'][newID], stars['%smag' % lsstFilter][newID], stars['ra'][newID], stars['decl'][newID]), file=tfile) idsUsed.extend(stars['id'][newIDs].tolist()) for visit in visitsIn: dmags = {} # Calc x,y, radius for each star, crop off stars outside the FoV # XXX - plan to replace with code to see where each star falls and get chipID. vx, vy, vz = treexyz(visit['ra'], visit['dec']) indices = starTree.query_ball_point((vx, vy, vz), treeRadius) starsIn = stars[indices] starsIn = starsProject(starsIn, visit) # Assign patchIDs and healpix IDs starsIn = assignPatches(starsIn, visit, nPatches=nPatches) #maybe make dmags a dict on stars, so that it's faster to append them all? # Apply the offsets that have been configured for offset in offsets: dmags[offset.newkey] = offset.run(starsIn, visit, dmags=dmags) # Total up all the dmag's to make the observed magnitude keys = list(dmags.keys()) obsMag = starsIn['%smag' % lsstFilter] for key in keys: obsMag += dmags[key] nObs += starsIn.size # Calculate the uncertainty in the observed mag: magErr = (magUncert.calcMagErrors( obsMag, errOnly=True, m5=visit['fiveSigmaDepth'])**2 + uncertFloor**2)**0.5 # patchID, starID, observed Mag, mag uncertainty, radius, healpixIDs for star, obsmag, magE in zip(starsIn, obsMag, magErr): print("%i, %i, %f, %f, %f, %i "%( \ star['patchID'],star['id'], obsmag, magE, star['radius'], 0), file=ofile) #star['hpID']) # Note the new starID's and print those to a truth file # starID true mag # XXX--might be better to just collect these and print at the end so that they can be sorted? # XXX - even better, just save them to a numpy save file, sorted, since the obs are the only # thing that really needs to go to ASCII! # Calc and print a patch file. Note this is slightly ambiguous since the clouds can have structure # patchID magOffset # Look at the distribution of dmags # patchID, starID, dmag1, dmag2, dmag3... ofile.close() tfile.close() return nObs, len(idsUsed)
def coaddM5Analysis(path, dbfile, runName, slair=False, WFDandDDFs=False, noDithOnly=False, bestDithOnly=False, someDithOnly=False, specifiedDith=None, nside=128, filterBand='r', includeDustExtinction=False, saveunMaskedCoaddData=False, pixelRadiusForMasking=5, cutOffYear=None, plotSkymap=True, plotCartview=True, unmaskedColorMin=None, unmaskedColorMax=None, maskedColorMin=None, maskedColorMax=None, nTicks=5, plotPowerSpectrum=True, showPlots=True, saveFigs=True, almAnalysis=True, raRange=[-50, 50], decRange=[-65, 5], saveMaskedCoaddData=True): """ Analyze the artifacts induced in the coadded 5sigma depth due to imperfect observing strategy. - Creates an output directory for subdirectories containing the specified things to save. - Creates, shows, and saves comparison plots. - Returns the metricBundle object containing the calculated coadded depth, and the output directory name. Required Parameters ------------------- * path: str: path to the main directory where output directory is to be saved. * dbfile: str: path to the OpSim output file, e.g. to a copy of enigma_1189 * runName: str: run name tag to identify the output of specified OpSim output, e.g. 'enigma1189' Optional Parameters ------------------- * slair: boolean: set to True if analysis on a SLAIR output. Default: False * WFDandDDFs: boolean: set to True if want to consider both WFD survet and DDFs. Otherwise will only work with WFD. Default: False * noDithOnly: boolean: set to True if only want to consider the undithered survey. Default: False * bestDithOnly: boolean: set to True if only want to consider RandomDitherFieldPerVisit. Default: False * someDithOnly: boolean: set to True if only want to consider undithered and a few dithered surveys. Default: False * specifiedDith: str: specific dither strategy to run. Default: None * nside: int: HEALpix resolution parameter. Default: 128 * filterBand: str: any one of 'u', 'g', 'r', 'i', 'z', 'y'. Default: 'r' * includeDustExtinction: boolean: set to include dust extinction. Default: False * saveunMaskedCoaddData: boolean: set to True to save data before border masking. Default: False * pixelRadiusForMasking: int: number of pixels to mask along the shallow border. Default: 5 * cutOffYear: int: year cut to restrict analysis to only a subset of the survey. Must range from 1 to 9, or None for the full survey analysis (10 yrs). Default: None * plotSkymap: boolean: set to True if want to plot skymaps. Default: True * plotCartview: boolean: set to True if want to plot cartview plots. Default: False * unmaskedColorMin: float: lower limit on the colorscale for unmasked skymaps. Default: None * unmaskedColorMax: float: upper limit on the colorscale for unmasked skymaps. Default: None * maskedColorMin: float: lower limit on the colorscale for border-masked skymaps. Default: None * maskedColorMax: float: upper limit on the colorscale for border-masked skymaps. Default: None * nTicks: int: (number of ticks - 1) on the skymap colorbar. Default: 5 * plotPowerSpectrum: boolean: set to True if want to plot powerspectra. Default: True * showPlots: boolean: set to True if want to show figures. Default: True * saveFigs: boolean: set to True if want to save figures. Default: True * almAnalysis: boolean: set to True to perform the alm analysis. Default: True * raRange: float array: range of right ascention (in degrees) to consider in alm cartview plot; applicable when almAnalysis=True. Default: [-50,50] * decRange: float array: range of declination (in degrees) to consider in alm cartview plot; applicable when almAnalysis=True. Default: [-65,5] * saveMaskedCoaddData: boolean: set to True to save the coadded depth data after the border masking. Default: True """ # ------------------------------------------------------------------------ # read in the database if slair: # slair database opsdb = db.Database(dbfile, defaultTable='observations') else: # OpSim database opsdb = db.OpsimDatabase(dbfile) # ------------------------------------------------------------------------ # set up the outDir zeropt_tag = '' if cutOffYear is not None: zeropt_tag = '%syearCut' % cutOffYear else: zeropt_tag = 'fullSurveyPeriod' if includeDustExtinction: dust_tag = 'withDustExtinction' else: dust_tag = 'noDustExtinction' regionType = '' if WFDandDDFs: regionType = 'WFDandDDFs_' outDir = 'coaddM5Analysis_%snside%s_%s_%spixelRadiusForMasking_%sBand_%s_%s_directory' % ( regionType, nside, dust_tag, pixelRadiusForMasking, filterBand, runName, zeropt_tag) print('# outDir: %s' % outDir) resultsDb = db.ResultsDb(outDir=outDir) # ------------------------------------------------------------------------ # set up the sql constraint if WFDandDDFs: if cutOffYear is not None: nightCutOff = (cutOffYear) * 365.25 sqlconstraint = 'night<=%s and filter=="%s"' % (nightCutOff, filterBand) else: sqlconstraint = 'filter=="%s"' % filterBand else: # set up the propID and units on the ra, dec if slair: # no prop ID; only WFD is simulated. wfdWhere = '' raDecInDeg = True else: propIds, propTags = opsdb.fetchPropInfo() wfdWhere = '%s and ' % opsdb.createSQLWhere('WFD', propTags) raDecInDeg = opsdb.raDecInDeg # set up the year cutoff if cutOffYear is not None: nightCutOff = (cutOffYear) * 365.25 sqlconstraint = '%snight<=%s and filter=="%s"' % ( wfdWhere, nightCutOff, filterBand) else: sqlconstraint = '%sfilter=="%s"' % (wfdWhere, filterBand) print('# sqlconstraint: %s' % sqlconstraint) # ------------------------------------------------------------------------ # setup all the slicers slicer = {} stackerList = {} if specifiedDith is not None: # would like to add all the stackers first and then keep only the one that is specified bestDithOnly, noDithOnly = False, False if bestDithOnly: stackerList['RandomDitherFieldPerVisit'] = [ mafStackers.RandomDitherFieldPerVisitStacker(degrees=raDecInDeg, randomSeed=1000) ] slicer['RandomDitherFieldPerVisit'] = slicers.HealpixSlicer( lonCol='randomDitherFieldPerVisitRa', latCol='randomDitherFieldPerVisitDec', latLonDeg=raDecInDeg, nside=nside, useCache=False) else: if slair: slicer['NoDither'] = slicers.HealpixSlicer(lonCol='RA', latCol='dec', latLonDeg=raDecInDeg, nside=nside, useCache=False) else: slicer['NoDither'] = slicers.HealpixSlicer(lonCol='fieldRA', latCol='fieldDec', latLonDeg=raDecInDeg, nside=nside, useCache=False) if someDithOnly and not noDithOnly: #stackerList['RepulsiveRandomDitherFieldPerVisit'] = [myStackers.RepulsiveRandomDitherFieldPerVisitStacker(degrees=raDecInDeg, # randomSeed=1000)] #slicer['RepulsiveRandomDitherFieldPerVisit'] = slicers.HealpixSlicer(lonCol='repulsiveRandomDitherFieldPerVisitRa', # latCol='repulsiveRandomDitherFieldPerVisitDec', # latLonDeg=raDecInDeg, nside=nside, # useCache=False) slicer['SequentialHexDitherFieldPerNight'] = slicers.HealpixSlicer( lonCol='hexDitherFieldPerNightRa', latCol='hexDitherFieldPerNightDec', latLonDeg=raDecInDeg, nside=nside, useCache=False) slicer['PentagonDitherPerSeason'] = slicers.HealpixSlicer( lonCol='pentagonDitherPerSeasonRa', latCol='pentagonDitherPerSeasonDec', latLonDeg=raDecInDeg, nside=nside, useCache=False) elif not noDithOnly: # random dithers on different timescales stackerList['RandomDitherPerNight'] = [ mafStackers.RandomDitherPerNightStacker(degrees=raDecInDeg, randomSeed=1000) ] stackerList['RandomDitherFieldPerNight'] = [ mafStackers.RandomDitherFieldPerNightStacker( degrees=raDecInDeg, randomSeed=1000) ] stackerList['RandomDitherFieldPerVisit'] = [ mafStackers.RandomDitherFieldPerVisitStacker( degrees=raDecInDeg, randomSeed=1000) ] # rep random dithers on different timescales #stackerList['RepulsiveRandomDitherPerNight'] = [myStackers.RepulsiveRandomDitherPerNightStacker(degrees=raDecInDeg, # randomSeed=1000)] #stackerList['RepulsiveRandomDitherFieldPerNight'] = [myStackers.RepulsiveRandomDitherFieldPerNightStacker(degrees=raDecInDeg, # randomSeed=1000)] #stackerList['RepulsiveRandomDitherFieldPerVisit'] = [myStackers.RepulsiveRandomDitherFieldPerVisitStacker(degrees=raDecInDeg, # randomSeed=1000)] # set up slicers for different dithers # random dithers on different timescales slicer['RandomDitherPerNight'] = slicers.HealpixSlicer( lonCol='randomDitherPerNightRa', latCol='randomDitherPerNightDec', latLonDeg=raDecInDeg, nside=nside, useCache=False) slicer['RandomDitherFieldPerNight'] = slicers.HealpixSlicer( lonCol='randomDitherFieldPerNightRa', latCol='randomDitherFieldPerNightDec', latLonDeg=raDecInDeg, nside=nside, useCache=False) slicer['RandomDitherFieldPerVisit'] = slicers.HealpixSlicer( lonCol='randomDitherFieldPerVisitRa', latCol='randomDitherFieldPerVisitDec', latLonDeg=raDecInDeg, nside=nside, useCache=False) # rep random dithers on different timescales #slicer['RepulsiveRandomDitherPerNight'] = slicers.HealpixSlicer(lonCol='repulsiveRandomDitherPerNightRa', # latCol='repulsiveRandomDitherPerNightDec', # latLonDeg=raDecInDeg, nside=nside, useCache=False) #slicer['RepulsiveRandomDitherFieldPerNight'] = slicers.HealpixSlicer(lonCol='repulsiveRandomDitherFieldPerNightRa', # latCol='repulsiveRandomDitherFieldPerNightDec', # latLonDeg=raDecInDeg, nside=nside, # useCache=False) #slicer['RepulsiveRandomDitherFieldPerVisit'] = slicers.HealpixSlicer(lonCol='repulsiveRandomDitherFieldPerVisitRa', # latCol='repulsiveRandomDitherFieldPerVisitDec', # latLonDeg=raDecInDeg, nside=nside, # useCache=False) # spiral dithers on different timescales slicer['FermatSpiralDitherPerNight'] = slicers.HealpixSlicer( lonCol='fermatSpiralDitherPerNightRa', latCol='fermatSpiralDitherPerNightDec', latLonDeg=raDecInDeg, nside=nside, useCache=False) slicer['FermatSpiralDitherFieldPerNight'] = slicers.HealpixSlicer( lonCol='fermatSpiralDitherFieldPerNightRa', latCol='fermatSpiralDitherFieldPerNightDec', latLonDeg=raDecInDeg, nside=nside, useCache=False) slicer['FermatSpiralDitherFieldPerVisit'] = slicers.HealpixSlicer( lonCol='fermatSpiralDitherFieldPerVisitRa', latCol='fermatSpiralDitherFieldPerVisitDec', latLonDeg=raDecInDeg, nside=nside, useCache=False) # hex dithers on different timescales slicer['SequentialHexDitherPerNight'] = slicers.HealpixSlicer( lonCol='hexDitherPerNightRa', latCol='hexDitherPerNightDec', latLonDeg=raDecInDeg, nside=nside, useCache=False) slicer['SequentialHexDitherFieldPerNight'] = slicers.HealpixSlicer( lonCol='hexDitherFieldPerNightRa', latCol='hexDitherFieldPerNightDec', latLonDeg=raDecInDeg, nside=nside, useCache=False) slicer['SequentialHexDitherFieldPerVisit'] = slicers.HealpixSlicer( lonCol='hexDitherFieldPerVisitRa', latCol='hexDitherFieldPerVisitDec', latLonDeg=raDecInDeg, nside=nside, useCache=False) # per season dithers slicer['PentagonDitherPerSeason'] = slicers.HealpixSlicer( lonCol='pentagonDitherPerSeasonRa', latCol='pentagonDitherPerSeasonDec', latLonDeg=raDecInDeg, nside=nside, useCache=False) slicer['PentagonDiamondDitherPerSeason'] = slicers.HealpixSlicer( lonCol='pentagonDiamondDitherPerSeasonRa', latCol='pentagonDiamondDitherPerSeasonDec', latLonDeg=raDecInDeg, nside=nside, useCache=False) slicer['SpiralDitherPerSeason'] = slicers.HealpixSlicer( lonCol='spiralDitherPerSeasonRa', latCol='spiralDitherPerSeasonDec', latLonDeg=raDecInDeg, nside=nside, useCache=False) if specifiedDith is not None: stackerList_, slicer_ = {}, {} if specifiedDith in slicer.keys(): if specifiedDith.__contains__( 'Random' ): # only Random dithers have a stacker object for rand seed specification stackerList_[specifiedDith] = stackerList[specifiedDith] slicer_[specifiedDith] = slicer[specifiedDith] else: raise ValueError( 'Invalid value for specifiedDith: %s. Allowed values include one of the following:\n%s' % (specifiedDith, slicer.keys())) stackerList, slicer = stackerList_, slicer_ # ------------------------------------------------------------------------ if slair: m5Col = 'fivesigmadepth' else: m5Col = 'fiveSigmaDepth' # set up the metric if includeDustExtinction: # include dust extinction when calculating the co-added depth coaddMetric = metrics.ExgalM5(m5Col=m5Col, lsstFilter=filterBand) else: coaddMetric = metrics.Coaddm5Metric(m5col=m5col) dustMap = maps.DustMap( interp=False, nside=nside ) # include dustMap; actual in/exclusion of dust is handled by the galaxyCountMetric # ------------------------------------------------------------------------ # set up the bundle coaddBundle = {} for dither in slicer: if dither in stackerList: coaddBundle[dither] = metricBundles.MetricBundle( coaddMetric, slicer[dither], sqlconstraint, stackerList=stackerList[dither], runName=runName, metadata=dither, mapsList=[dustMap]) else: coaddBundle[dither] = metricBundles.MetricBundle( coaddMetric, slicer[dither], sqlconstraint, runName=runName, metadata=dither, mapsList=[dustMap]) # ------------------------------------------------------------------------ # run the analysis if includeDustExtinction: print('\n# Running coaddBundle with dust extinction ...') else: print('\n# Running coaddBundle without dust extinction ...') cGroup = metricBundles.MetricBundleGroup(coaddBundle, opsdb, outDir=outDir, resultsDb=resultsDb, saveEarly=False) cGroup.runAll() # ------------------------------------------------------------------------ # plot and save the data plotBundleMaps(path, outDir, coaddBundle, dataLabel='$%s$-band Coadded Depth' % filterBand, filterBand=filterBand, dataName='%s-band Coadded Depth' % filterBand, skymap=plotSkymap, powerSpectrum=plotPowerSpectrum, cartview=plotCartview, colorMin=unmaskedColorMin, colorMax=unmaskedColorMax, nTicks=nTicks, showPlots=showPlots, saveFigs=saveFigs, outDirNameForSavedFigs='coaddM5Plots_unmaskedBorders') print('\n# Done saving plots without border masking.\n') # ------------------------------------------------------------------------ plotHandler = plots.PlotHandler(outDir=outDir, resultsDb=resultsDb, thumbnail=False, savefig=False) print( '# Number of pixels in the survey region (before masking the border):') for dither in coaddBundle: print( ' %s: %s' % (dither, len(np.where(coaddBundle[dither].metricValues.mask == False)[0]))) # ------------------------------------------------------------------------ # save the unmasked data? if saveunMaskedCoaddData: outDir_new = 'unmaskedCoaddData' if not os.path.exists('%s%s/%s' % (path, outDir, outDir_new)): os.makedirs('%s%s/%s' % (path, outDir, outDir_new)) saveBundleData_npzFormat('%s%s/%s' % (path, outDir, outDir_new), coaddBundle, 'coaddM5Data_unmasked', filterBand) # ------------------------------------------------------------------------ # mask the edges print('\n# Masking the edges for coadd ...') coaddBundle = maskingAlgorithmGeneralized( coaddBundle, plotHandler, dataLabel='$%s$-band Coadded Depth' % filterBand, nside=nside, pixelRadius=pixelRadiusForMasking, plotIntermediatePlots=False, plotFinalPlots=False, printFinalInfo=True) if (pixelRadiusForMasking > 0): # plot and save the masked data plotBundleMaps(path, outDir, coaddBundle, dataLabel='$%s$-band Coadded Depth' % filterBand, filterBand=filterBand, dataName='%s-band Coadded Depth' % filterBand, skymap=plotSkymap, powerSpectrum=plotPowerSpectrum, cartview=plotCartview, colorMin=maskedColorMin, colorMax=maskedColorMax, nTicks=nTicks, showPlots=showPlots, saveFigs=saveFigs, outDirNameForSavedFigs='coaddM5Plots_maskedBorders') print('\n# Done saving plots with border masking. \n') # ------------------------------------------------------------------------ # Calculate total power summarymetric = metrics.TotalPowerMetric() for dither in coaddBundle: coaddBundle[dither].setSummaryMetrics(summarymetric) coaddBundle[dither].computeSummaryStats() print('# Total power for %s case is %f.' % (dither, coaddBundle[dither].summaryValues['TotalPower'])) print('') # ------------------------------------------------------------------------ # run the alm analysis if almAnalysis: almPlots(path, outDir, copy.deepcopy(coaddBundle), nside=nside, filterband=filterBand, raRange=raRange, decRange=decRange, showPlots=showPlots) # ------------------------------------------------------------------------ # save the masked data? if saveMaskedCoaddData and (pixelRadiusForMasking > 0): outDir_new = 'maskedCoaddData' if not os.path.exists('%s%s/%s' % (path, outDir, outDir_new)): os.makedirs('%s%s/%s' % (path, outDir, outDir_new)) saveBundleData_npzFormat('%s%s/%s' % (path, outDir, outDir_new), coaddBundle, 'coaddM5Data_masked', filterBand) # ------------------------------------------------------------------------ # plot comparison plots if len(coaddBundle.keys()) > 1: # more than one key # set up the directory outDir_comp = 'coaddM5ComparisonPlots' if not os.path.exists('%s%s/%s' % (path, outDir, outDir_comp)): os.makedirs('%s%s/%s' % (path, outDir, outDir_comp)) # ------------------------------------------------------------------------ # plot for the power spectra cl = {} for dither in plotColor: if dither in coaddBundle: cl[dither] = hp.anafast(hp.remove_dipole( coaddBundle[dither].metricValues.filled( coaddBundle[dither].slicer.badval)), lmax=500) ell = np.arange(np.size(cl[dither])) plt.plot(ell, (cl[dither] * ell * (ell + 1)) / 2.0 / np.pi, color=plotColor[dither], linestyle='-', label=dither) plt.xlabel(r'$\ell$') plt.ylabel(r'$\ell(\ell+1)C_\ell/(2\pi)$') plt.xlim(0, 500) fig = plt.gcf() fig.set_size_inches(12.5, 10.5) leg = plt.legend(labelspacing=0.001) for legobj in leg.legendHandles: legobj.set_linewidth(4.0) filename = 'powerspectrum_comparison_all.png' plt.savefig('%s%s/%s/%s' % (path, outDir, outDir_comp, filename), bbox_inches='tight', format='png') plt.show() # create the histogram scale = hp.nside2pixarea(nside, degrees=True) def tickFormatter(y, pos): return '%d' % (y * scale) # convert pixel count to area binsize = 0.01 for dither in plotColor: if dither in coaddBundle: ind = np.where( coaddBundle[dither].metricValues.mask == False)[0] binAll = int( (max(coaddBundle[dither].metricValues.data[ind]) - min(coaddBundle[dither].metricValues.data[ind])) / binsize) plt.hist(coaddBundle[dither].metricValues.data[ind], bins=binAll, label=dither, histtype='step', color=plotColor[dither]) ax = plt.gca() ymin, ymax = ax.get_ylim() nYticks = 10. wantedYMax = ymax * scale wantedYMax = 10. * np.ceil(float(wantedYMax) / 10.) increment = 5. * np.ceil(float(wantedYMax / nYticks) / 5.) wantedArray = np.arange(0, wantedYMax, increment) ax.yaxis.set_ticks(wantedArray / scale) ax.yaxis.set_major_formatter(FuncFormatter(tickFormatter)) plt.xlabel('$%s$-band Coadded Depth' % filterBand) plt.ylabel('Area (deg$^2$)') fig = plt.gcf() fig.set_size_inches(12.5, 10.5) leg = plt.legend(labelspacing=0.001, loc=2) for legobj in leg.legendHandles: legobj.set_linewidth(2.0) filename = 'histogram_comparison.png' plt.savefig('%s%s/%s/%s' % (path, outDir, outDir_comp, filename), bbox_inches='tight', format='png') plt.show() # ------------------------------------------------------------------------ # plot power spectra for the separte panel totKeys = len(list(coaddBundle.keys())) if (totKeys > 1): plt.clf() nCols = 2 nRows = int(np.ceil(float(totKeys) / nCols)) fig, ax = plt.subplots(nRows, nCols) plotRow = 0 plotCol = 0 for dither in list(plotColor.keys()): if dither in list(coaddBundle.keys()): ell = np.arange(np.size(cl[dither])) ax[plotRow, plotCol].plot(ell, (cl[dither] * ell * (ell + 1)) / 2.0 / np.pi, color=plotColor[dither], label=dither) if (plotRow == nRows - 1): ax[plotRow, plotCol].set_xlabel(r'$\ell$') ax[plotRow, plotCol].set_ylabel(r'$\ell(\ell+1)C_\ell/(2\pi)$') ax[plotRow, plotCol].yaxis.set_major_locator(MaxNLocator(3)) if (dither != 'NoDither'): ax[plotRow, plotCol].set_ylim(0, 0.0035) ax[plotRow, plotCol].set_xlim(0, 500) plotRow += 1 if (plotRow > nRows - 1): plotRow = 0 plotCol += 1 fig.set_size_inches(20, int(nRows * 30 / 7.)) filename = 'powerspectrum_sepPanels.png' plt.savefig('%s%s/%s/%s' % (path, outDir, outDir_comp, filename), bbox_inches='tight', format='png') plt.show() return coaddBundle, outDir