def testAddRun(self):
"""Test adding a run to the tracking database."""
trackingdb = db.TrackingDb(database=self.trackingDb)
trackId = trackingdb.addRun(opsimRun=self.opsimRun,
opsimComment=self.opsimComment,
mafComment=self.mafComment,
mafDir=self.mafDir,
mafDate=self.mafDate,
opsimDate=self.opsimDate)
tdb = db.Database(database=self.trackingDb,
dbTables={'runs': ['runs', 'mafRunId']})
res = tdb.queryDatabase('runs', 'select * from runs')
self.assertEqual(res['mafRunId'][0], trackId)
# Try adding this run again. Should just return previous trackId without adding.
trackId2 = trackingdb.addRun(opsimRun=self.opsimRun,
opsimComment=self.opsimComment,
mafComment=self.mafComment,
mafDir=self.mafDir,
mafDate=self.mafDate,
opsimDate=self.opsimDate)
self.assertEqual(trackId, trackId2)
# Test will add run, if we use 'override=True'. Also works to use None's.
trackId3 = trackingdb.addRun(opsimRun=None,
opsimComment=None,
mafComment=None,
mafDir=self.mafDir,
override=True,
mafDate=self.mafDate,
opsimDate=self.opsimDate)
self.assertNotEqual(trackId, trackId3)
trackingdb.close()
def test_testAddRun(self):
"""Test adding a run to the tracking database."""
tempdir = tempfile.mkdtemp(prefix='trackDb')
trackingDbFile = os.path.join(tempdir, 'tracking.db')
trackingdb = db.TrackingDb(database=trackingDbFile)
trackId = trackingdb.addRun(opsimGroup=self.opsimGroup,
opsimRun=self.opsimRun,
opsimComment=self.opsimComment,
opsimVersion=self.opsimVersion,
opsimDate=self.opsimDate,
mafComment=self.mafComment,
mafDir=self.mafDir,
mafVersion=self.mafVersion,
mafDate=self.mafDate,
dbFile=self.dbFile)
tdb = db.Database(database=trackingDbFile)
res = tdb.query_arbitrary('select * from runs')
self.assertEqual(res['mafRunId'][0], trackId)
# Try adding this run again. Should return previous trackId.
trackId2 = trackingdb.addRun(mafDir=self.mafDir)
self.assertEqual(trackId, trackId2)
# Test will add additional run, with new trackId.
trackId3 = trackingdb.addRun(mafDir='test2')
self.assertNotEqual(trackId, trackId3)
trackingdb.close()
tdb.close()
shutil.rmtree(tempdir)
def run_glance(outDir, dbname):
conn = db.Database(dbname, defaultTable='observations')
resultsDb = db.ResultsDb(outDir=outDir)
mbg = MetricBundleGroup(bd, conn, outDir=outDir, resultsDb=resultsDb)
mbg.runAll()
mbg.plotAll()
conn.close()
def testBaseDatabase(self):
"""Test base database class."""
# Test instantation with no dbTables info (and no defaults).
basedb = db.Database(database=self.database, driver=self.driver)
self.assertEqual(basedb.tables, None)
# Test instantiation with some tables.
basedb = db.Database(database=self.database, driver=self.driver,
dbTables={'obsHistTable':['ObsHistory', 'obsHistID'],
'fieldTable':['Field', 'fieldID'],
'obsHistoryProposalTable':['Obshistory_Proposal',
'obsHistory_propID']})
self.assertEqual(set(basedb.tables.keys()),
set(['obsHistTable',
'obsHistoryProposalTable', 'fieldTable']))
# Test general query with a simple query.
query = 'select fieldID, fieldRA, fieldDec from Field where fieldDec>0'
data = basedb.queryDatabase('fieldTable', query)
self.assertEqual(data.dtype.names, ('fieldID', 'fieldRA', 'fieldDec'))
def connectDb(dbfile):
version = db.testOpsimVersion(dbfile)
if version == "Unknown":
opsdb = db.Database(dbfile)
colmap = batches.ColMapDict('barebones')
elif version == "V3":
opsdb = db.OpsimDatabaseV3(dbfile)
colmap = batches.ColMapDict('OpsimV3')
elif version == "V4":
opsdb = db.OpsimDatabaseV4(dbfile)
colmap = batches.ColMapDict('OpsimV4')
return opsdb, colmap
def testDelRun(self):
"""Test removing a run from the tracking database."""
trackingdb = db.TrackingDb(database=self.trackingDb)
tdb = db.Database(database=self.trackingDb,
dbTables={'runs': ['runs', 'mafRunId']})
# Add a run.
trackId = trackingdb.addRun(mafDir=self.mafDir)
res = tdb.queryDatabase('runs', 'select * from runs')
self.assertEqual(res['mafRunId'][0], trackId)
# Test removal works.
trackingdb.delRun(trackId)
res = tdb.queryDatabase('runs', 'select * from runs')
self.assertTrue(len(res) == 0)
# Test cannot remove run which does not exist.
self.assertRaises(Exception, trackingdb.delRun, trackId)
trackingdb.close()
def runGlance(dbfile, outDir='Glance', runName='runname', camera='LSST'):
conn = db.Database(dbfile, defaultTable='observations')
colmap = {'ra': 'RA', 'dec': 'dec', 'mjd': 'mjd',
'exptime': 'exptime', 'visittime': 'exptime', 'alt': 'alt',
'az': 'az', 'filter': 'filter', 'fiveSigmaDepth': 'fivesigmadepth',
'night': 'night', 'slewtime': 'slewtime', 'seeingGeom': 'FWHM_geometric',
'rotSkyPos': 'rotSkyPos', 'raDecDeg': True, 'slewdist': None,
'note': 'note'}
gb = glanceBatch(colmap=colmap, slicer_camera=camera)
resultsDb = db.ResultsDb(outDir=outDir)
group = metricBundles.MetricBundleGroup(gb, conn, outDir=outDir, resultsDb=resultsDb)
group.runAll()
group.plotAll()
def __init__(self, database=None):
"""
Instantiate the (multi-run) layout visualization class.
Parameters
----------
database :str
Path to the sqlite tracking database file.
If not set, looks for 'trackingDb_sqlite.db' file in current directory.
"""
if database is None:
database = os.path.join(os.getcwd(), 'trackingDb_sqlite.db')
# Read in the results database.
database = db.Database(database=database,
longstrings=True,
dbTables={'runs': ['runs', 'mafRunId']})
self.runs = database.queryDatabase('runs', 'select * from runs')
self.runs = self.sortRuns(self.runs)
self.runsPage = {}
def test_testDelRun(self):
"""Test removing a run from the tracking database."""
tempdir = tempfile.mkdtemp(prefix='trackDb')
trackingDbFile = os.path.join(tempdir, 'tracking.db')
trackingdb = db.TrackingDb(database=trackingDbFile)
tdb = db.Database(database=trackingDbFile)
# Add two runs.
trackId = trackingdb.addRun(mafDir=self.mafDir)
trackId2 = trackingdb.addRun(mafDir=self.mafDir + 'test2')
res = tdb.query_arbitrary('select * from runs')
self.assertEqual(res['mafRunId'][0], trackId)
# Test removal works.
trackingdb.delRun(trackId)
res = tdb.query_arbitrary('select * from runs')
self.assertTrue(len(res) == 1)
# Test cannot remove run which does not exist.
self.assertRaises(Exception, trackingdb.delRun, trackId)
trackingdb.close()
tdb.close()
shutil.rmtree(tempdir)
def testBaseDatabase(self):
"""Test base database class."""
# Test instantiation connects to expected tables.
basedb = db.Database(database=self.database, driver=self.driver)
expectedTables = ['Config', 'ScheduledDowntime', 'SlewMaxSpeeds',
'Field', 'Session', 'SummaryAllProps', 'ObsExposures',
'SlewActivities', 'TargetExposures', 'ObsHistory',
'SlewFinalState', 'TargetHistory', 'ObsProposalHistory',
'SlewHistory', 'TargetProposalHistory', 'Proposal',
'ProposalField',
'SlewInitialState', 'UnscheduledDowntime']
self.assertEqual(set(basedb.tableNames),
set(expectedTables))
# Test general query with a simple query.
query = 'select fieldId, ra, dec from Field where dec>0 limit 3'
data = basedb.query_arbitrary(query)
self.assertEqual(len(data), 3)
# Test query columns with a simple query.
data = basedb.query_columns('Field', colnames=['fieldId', 'ra', 'dec'], numLimit=3)
self.assertEqual(data.dtype.names, ('fieldId', 'ra', 'dec'))
self.assertEqual(len(data), 3)
def __init__(self, database=None):
"""
Instantiate the (multi-run) layout visualization class.
Parameters
----------
database :str
Path to the sqlite tracking database file.
If not set, looks for 'trackingDb_sqlite.db' file in current directory.
"""
if database is None:
database = os.path.join(os.getcwd(), 'trackingDb_sqlite.db')
# Read in the results database.
tdb = db.Database(database=database, longstrings=True)
cols = [
'mafRunId', 'opsimRun', 'opsimGroup', 'mafComment', 'opsimComment',
'dbFile', 'mafDir', 'opsimVersion', 'opsimDate', 'mafVersion',
'mafDate'
]
self.runs = tdb.query_columns('runs', colnames=cols)
self.runs = self.sortRuns(self.runs)
self.runsPage = {}
def testAddRun(self):
"""Test adding a run to the tracking database."""
trackingdb = db.TrackingDb(database=self.trackingDb)
trackId = trackingdb.addRun(opsimGroup=self.opsimGroup,
opsimRun=self.opsimRun,
opsimComment=self.opsimComment,
opsimVersion=self.opsimVersion,
opsimDate=self.opsimDate,
mafComment=self.mafComment,
mafDir=self.mafDir,
mafVersion=self.mafVersion,
mafDate=self.mafDate,
dbFile=self.dbFile)
tdb = db.Database(database=self.trackingDb,
dbTables={'runs': ['runs', 'mafRunId']})
res = tdb.queryDatabase('runs', 'select * from runs')
self.assertEqual(res['mafRunId'][0], trackId)
# Try adding this run again. Should return previous trackId.
trackId2 = trackingdb.addRun(mafDir=self.mafDir)
self.assertEqual(trackId, trackId2)
# Test will add additional run, with new trackId.
trackId3 = trackingdb.addRun(mafDir='test2')
self.assertNotEqual(trackId, trackId3)
trackingdb.close()
def genCatalog(visits,
starsDbAddress,
offsets=None,
lsstFilter='r',
raBlockSize=20.,
decBlockSize=10.,
nPatches=16,
radiusFoV=1.8,
verbose=True,
seed=42,
uncertFloor=0.005,
obsFile='observations.dat',
truthFile='starInfo.dat'):
"""
Generate a catalog of observed stellar magnitudes.
visits: A numpy array with the properties of the visits. Expected to have Opsim-like values
starsDbAddress: a sqlAlchemy address pointing to a database that contains properties of stars used as input.
offsets: A list of instatiated classes that will apply offsets to the stars
lsstFilter: Which filter to use for the observed stars
obsFile: File to write the observed stellar magnitudes to
truthFile: File to write the true stellar magnitudes to
raBlockSize/decBlockSize: Size of chucks to use when looping over the sky
nPatches: Number of patches to divide the FoV into. Must be an integer squared
radiusFoV: Radius of the telescope field of view in degrees
seed: random number seed
uncertFloor: value to add in quadrature to magnitude uncertainties
"""
if offsets is None:
# Maybe change this to just run with a default SNR offset
warnings.warn("Warning, no offsets set, returning without running")
return
# Open files for writing
ofile = open(obsFile, 'w')
tfile = open(truthFile, 'w')
print('#PatchID, StarID, ObservedMag, MagUncertainty, Radius, HPID',
file=ofile)
print('#StarID, TrueMag, ra, dec', file=tfile)
# Set up connection to stars db:
msrgbDB = db.Database(starsDbAddress, dbTables={'stars': ['stars', 'id']})
starCols = ['id', 'rmag', 'gmag', 'ra', 'decl']
if lsstFilter + 'mag' not in starCols:
starCols.append(lsstFilter + 'mag')
# Loop over the sky
raBlocks = np.arange(0., 2. * np.pi, np.radians(raBlockSize))
decBlocks = np.arange(-np.pi, np.pi, np.radians(decBlockSize))
# List to keep track of which starIDs have been written to truth file
idsUsed = []
nObs = 0
# For computing what the 'expected' uncertainty on the observation will be
magUncert = OffsetSNR(lsstFilter=lsstFilter)
# set the radius for the kdtree
x0, y0, z0 = (1, 0, 0)
x1, y1, z1 = treexyz(np.radians(radiusFoV), 0)
treeRadius = np.sqrt((x1 - x0)**2 + (y1 - y0)**2 + (z1 - z0)**2)
for raBlock in raBlocks:
for decBlock in decBlocks:
# The idea here is that this could be run in parallel or not and is still repeatable
# b/c the seed will always be the same for each block.
np.random.seed(seed)
seed += 1 #This should make it possible to run in parallel and maintain repeatability.
visitsIn = visits[np.where(
(visits['ra'] >= raBlock)
& (visits['ra'] < raBlock + np.radians(raBlockSize))
& (visits['dec'] >= decBlock)
& (visits['dec'] < decBlock + np.radians(decBlockSize)))]
if np.size(visitsIn) > 0:
# Fetch the stars in this block+the radiusFoV
decPad = radiusFoV
raPad = radiusFoV
# Need to deal with wrap around effects.
decMin = capDec(np.degrees(decBlock) - decPad)
decMax = capDec(np.degrees(decBlock) + decBlockSize + decPad)
sqlwhere = 'decl > %f and decl < %f ' % (decMin, decMax)
raMin = np.degrees(raBlock) - raPad / np.cos(
np.radians(decMin))
raMax = np.degrees(raBlock) + raBlockSize + raPad / np.cos(
np.radians(decMin))
if (wrapRA(raMin) != raMin) & (wrapRA(raMax) != raMax):
# near a pole, just grab all the stars
sqlwhere += ''
else:
raMin = wrapRA(raMin)
raMax = wrapRA(raMax)
# no wrap
if raMin < raMax:
sqlwhere += 'and ra < %f and ra > %f ' % (raMax, raMin)
# One side wrapped
else:
sqlwhere += 'and (ra > %f or ra < %f)' % (raMin, raMax)
print('quering stars with: ' + sqlwhere)
stars = msrgbDB.tables['stars'].query_columns_Array(
colnames=starCols, constraint=sqlwhere)
print('got %i stars' % stars.size)
# Add all the columns I will want here, then I only do
# one numpy stack per block rather than lots of stacks per visit
# Ugh, feels like writing fortran though...
newcols = [
'x', 'y', 'radius', 'patchID', 'subPatch', 'hpID', 'hp1',
'hp2', 'hp3', 'hp4'
]
newtypes = [float, float, float, int, int, int]
stars = rfn.merge_arrays([
stars,
np.zeros(stars.size, dtype=list(zip(newcols, newtypes)))
],
flatten=True,
usemask=False)
# Build a KDTree for the stars
starTree = buildTree(np.radians(stars['ra']),
np.radians(stars['decl']))
newIDs = np.in1d(stars['id'],
idsUsed,
invert=True,
assume_unique=True)
newIDs = np.arange(stars['id'].size)[newIDs]
for newID in newIDs:
print('%i, %f, %f, %f' %
(stars['id'][newID],
stars['%smag' % lsstFilter][newID],
stars['ra'][newID], stars['decl'][newID]),
file=tfile)
idsUsed.extend(stars['id'][newIDs].tolist())
for visit in visitsIn:
dmags = {}
# Calc x,y, radius for each star, crop off stars outside the FoV
# XXX - plan to replace with code to see where each star falls and get chipID.
vx, vy, vz = treexyz(visit['ra'], visit['dec'])
indices = starTree.query_ball_point((vx, vy, vz), treeRadius)
starsIn = stars[indices]
starsIn = starsProject(starsIn, visit)
# Assign patchIDs and healpix IDs
starsIn = assignPatches(starsIn, visit, nPatches=nPatches)
#maybe make dmags a dict on stars, so that it's faster to append them all?
# Apply the offsets that have been configured
for offset in offsets:
dmags[offset.newkey] = offset.run(starsIn,
visit,
dmags=dmags)
# Total up all the dmag's to make the observed magnitude
keys = list(dmags.keys())
obsMag = starsIn['%smag' % lsstFilter]
for key in keys:
obsMag += dmags[key]
nObs += starsIn.size
# Calculate the uncertainty in the observed mag:
magErr = (magUncert.calcMagErrors(
obsMag, errOnly=True, m5=visit['fiveSigmaDepth'])**2 +
uncertFloor**2)**0.5
# patchID, starID, observed Mag, mag uncertainty, radius, healpixIDs
for star, obsmag, magE in zip(starsIn, obsMag, magErr):
print("%i, %i, %f, %f, %f, %i "%( \
star['patchID'],star['id'], obsmag, magE,
star['radius'], 0), file=ofile) #star['hpID'])
# Note the new starID's and print those to a truth file
# starID true mag
# XXX--might be better to just collect these and print at the end so that they can be sorted?
# XXX - even better, just save them to a numpy save file, sorted, since the obs are the only
# thing that really needs to go to ASCII!
# Calc and print a patch file. Note this is slightly ambiguous since the clouds can have structure
# patchID magOffset
# Look at the distribution of dmags
# patchID, starID, dmag1, dmag2, dmag3...
ofile.close()
tfile.close()
return nObs, len(idsUsed)
def coaddM5Analysis(path,
dbfile,
runName,
slair=False,
WFDandDDFs=False,
noDithOnly=False,
bestDithOnly=False,
someDithOnly=False,
specifiedDith=None,
nside=128,
filterBand='r',
includeDustExtinction=False,
saveunMaskedCoaddData=False,
pixelRadiusForMasking=5,
cutOffYear=None,
plotSkymap=True,
plotCartview=True,
unmaskedColorMin=None,
unmaskedColorMax=None,
maskedColorMin=None,
maskedColorMax=None,
nTicks=5,
plotPowerSpectrum=True,
showPlots=True,
saveFigs=True,
almAnalysis=True,
raRange=[-50, 50],
decRange=[-65, 5],
saveMaskedCoaddData=True):
"""
Analyze the artifacts induced in the coadded 5sigma depth due to imperfect observing strategy.
- Creates an output directory for subdirectories containing the specified things to save.
- Creates, shows, and saves comparison plots.
- Returns the metricBundle object containing the calculated coadded depth, and the output directory name.
Required Parameters
-------------------
* path: str: path to the main directory where output directory is to be saved.
* dbfile: str: path to the OpSim output file, e.g. to a copy of enigma_1189
* runName: str: run name tag to identify the output of specified OpSim output, e.g. 'enigma1189'
Optional Parameters
-------------------
* slair: boolean: set to True if analysis on a SLAIR output.
Default: False
* WFDandDDFs: boolean: set to True if want to consider both WFD survet and DDFs. Otherwise will only work
with WFD. Default: False
* noDithOnly: boolean: set to True if only want to consider the undithered survey. Default: False
* bestDithOnly: boolean: set to True if only want to consider RandomDitherFieldPerVisit.
Default: False
* someDithOnly: boolean: set to True if only want to consider undithered and a few dithered surveys.
Default: False
* specifiedDith: str: specific dither strategy to run.
Default: None
* nside: int: HEALpix resolution parameter. Default: 128
* filterBand: str: any one of 'u', 'g', 'r', 'i', 'z', 'y'. Default: 'r'
* includeDustExtinction: boolean: set to include dust extinction. Default: False
* saveunMaskedCoaddData: boolean: set to True to save data before border masking. Default: False
* pixelRadiusForMasking: int: number of pixels to mask along the shallow border. Default: 5
* cutOffYear: int: year cut to restrict analysis to only a subset of the survey.
Must range from 1 to 9, or None for the full survey analysis (10 yrs).
Default: None
* plotSkymap: boolean: set to True if want to plot skymaps. Default: True
* plotCartview: boolean: set to True if want to plot cartview plots. Default: False
* unmaskedColorMin: float: lower limit on the colorscale for unmasked skymaps. Default: None
* unmaskedColorMax: float: upper limit on the colorscale for unmasked skymaps. Default: None
* maskedColorMin: float: lower limit on the colorscale for border-masked skymaps. Default: None
* maskedColorMax: float: upper limit on the colorscale for border-masked skymaps. Default: None
* nTicks: int: (number of ticks - 1) on the skymap colorbar. Default: 5
* plotPowerSpectrum: boolean: set to True if want to plot powerspectra. Default: True
* showPlots: boolean: set to True if want to show figures. Default: True
* saveFigs: boolean: set to True if want to save figures. Default: True
* almAnalysis: boolean: set to True to perform the alm analysis. Default: True
* raRange: float array: range of right ascention (in degrees) to consider in alm cartview plot;
applicable when almAnalysis=True. Default: [-50,50]
* decRange: float array: range of declination (in degrees) to consider in alm cartview plot;
applicable when almAnalysis=True. Default: [-65,5]
* saveMaskedCoaddData: boolean: set to True to save the coadded depth data after the border
masking. Default: True
"""
# ------------------------------------------------------------------------
# read in the database
if slair:
# slair database
opsdb = db.Database(dbfile, defaultTable='observations')
else:
# OpSim database
opsdb = db.OpsimDatabase(dbfile)
# ------------------------------------------------------------------------
# set up the outDir
zeropt_tag = ''
if cutOffYear is not None: zeropt_tag = '%syearCut' % cutOffYear
else: zeropt_tag = 'fullSurveyPeriod'
if includeDustExtinction: dust_tag = 'withDustExtinction'
else: dust_tag = 'noDustExtinction'
regionType = ''
if WFDandDDFs: regionType = 'WFDandDDFs_'
outDir = 'coaddM5Analysis_%snside%s_%s_%spixelRadiusForMasking_%sBand_%s_%s_directory' % (
regionType, nside, dust_tag, pixelRadiusForMasking, filterBand,
runName, zeropt_tag)
print('# outDir: %s' % outDir)
resultsDb = db.ResultsDb(outDir=outDir)
# ------------------------------------------------------------------------
# set up the sql constraint
if WFDandDDFs:
if cutOffYear is not None:
nightCutOff = (cutOffYear) * 365.25
sqlconstraint = 'night<=%s and filter=="%s"' % (nightCutOff,
filterBand)
else:
sqlconstraint = 'filter=="%s"' % filterBand
else:
# set up the propID and units on the ra, dec
if slair: # no prop ID; only WFD is simulated.
wfdWhere = ''
raDecInDeg = True
else:
propIds, propTags = opsdb.fetchPropInfo()
wfdWhere = '%s and ' % opsdb.createSQLWhere('WFD', propTags)
raDecInDeg = opsdb.raDecInDeg
# set up the year cutoff
if cutOffYear is not None:
nightCutOff = (cutOffYear) * 365.25
sqlconstraint = '%snight<=%s and filter=="%s"' % (
wfdWhere, nightCutOff, filterBand)
else:
sqlconstraint = '%sfilter=="%s"' % (wfdWhere, filterBand)
print('# sqlconstraint: %s' % sqlconstraint)
# ------------------------------------------------------------------------
# setup all the slicers
slicer = {}
stackerList = {}
if specifiedDith is not None: # would like to add all the stackers first and then keep only the one that is specified
bestDithOnly, noDithOnly = False, False
if bestDithOnly:
stackerList['RandomDitherFieldPerVisit'] = [
mafStackers.RandomDitherFieldPerVisitStacker(degrees=raDecInDeg,
randomSeed=1000)
]
slicer['RandomDitherFieldPerVisit'] = slicers.HealpixSlicer(
lonCol='randomDitherFieldPerVisitRa',
latCol='randomDitherFieldPerVisitDec',
latLonDeg=raDecInDeg,
nside=nside,
useCache=False)
else:
if slair:
slicer['NoDither'] = slicers.HealpixSlicer(lonCol='RA',
latCol='dec',
latLonDeg=raDecInDeg,
nside=nside,
useCache=False)
else:
slicer['NoDither'] = slicers.HealpixSlicer(lonCol='fieldRA',
latCol='fieldDec',
latLonDeg=raDecInDeg,
nside=nside,
useCache=False)
if someDithOnly and not noDithOnly:
#stackerList['RepulsiveRandomDitherFieldPerVisit'] = [myStackers.RepulsiveRandomDitherFieldPerVisitStacker(degrees=raDecInDeg,
# randomSeed=1000)]
#slicer['RepulsiveRandomDitherFieldPerVisit'] = slicers.HealpixSlicer(lonCol='repulsiveRandomDitherFieldPerVisitRa',
# latCol='repulsiveRandomDitherFieldPerVisitDec',
# latLonDeg=raDecInDeg, nside=nside,
# useCache=False)
slicer['SequentialHexDitherFieldPerNight'] = slicers.HealpixSlicer(
lonCol='hexDitherFieldPerNightRa',
latCol='hexDitherFieldPerNightDec',
latLonDeg=raDecInDeg,
nside=nside,
useCache=False)
slicer['PentagonDitherPerSeason'] = slicers.HealpixSlicer(
lonCol='pentagonDitherPerSeasonRa',
latCol='pentagonDitherPerSeasonDec',
latLonDeg=raDecInDeg,
nside=nside,
useCache=False)
elif not noDithOnly:
# random dithers on different timescales
stackerList['RandomDitherPerNight'] = [
mafStackers.RandomDitherPerNightStacker(degrees=raDecInDeg,
randomSeed=1000)
]
stackerList['RandomDitherFieldPerNight'] = [
mafStackers.RandomDitherFieldPerNightStacker(
degrees=raDecInDeg, randomSeed=1000)
]
stackerList['RandomDitherFieldPerVisit'] = [
mafStackers.RandomDitherFieldPerVisitStacker(
degrees=raDecInDeg, randomSeed=1000)
]
# rep random dithers on different timescales
#stackerList['RepulsiveRandomDitherPerNight'] = [myStackers.RepulsiveRandomDitherPerNightStacker(degrees=raDecInDeg,
# randomSeed=1000)]
#stackerList['RepulsiveRandomDitherFieldPerNight'] = [myStackers.RepulsiveRandomDitherFieldPerNightStacker(degrees=raDecInDeg,
# randomSeed=1000)]
#stackerList['RepulsiveRandomDitherFieldPerVisit'] = [myStackers.RepulsiveRandomDitherFieldPerVisitStacker(degrees=raDecInDeg,
# randomSeed=1000)]
# set up slicers for different dithers
# random dithers on different timescales
slicer['RandomDitherPerNight'] = slicers.HealpixSlicer(
lonCol='randomDitherPerNightRa',
latCol='randomDitherPerNightDec',
latLonDeg=raDecInDeg,
nside=nside,
useCache=False)
slicer['RandomDitherFieldPerNight'] = slicers.HealpixSlicer(
lonCol='randomDitherFieldPerNightRa',
latCol='randomDitherFieldPerNightDec',
latLonDeg=raDecInDeg,
nside=nside,
useCache=False)
slicer['RandomDitherFieldPerVisit'] = slicers.HealpixSlicer(
lonCol='randomDitherFieldPerVisitRa',
latCol='randomDitherFieldPerVisitDec',
latLonDeg=raDecInDeg,
nside=nside,
useCache=False)
# rep random dithers on different timescales
#slicer['RepulsiveRandomDitherPerNight'] = slicers.HealpixSlicer(lonCol='repulsiveRandomDitherPerNightRa',
# latCol='repulsiveRandomDitherPerNightDec',
# latLonDeg=raDecInDeg, nside=nside, useCache=False)
#slicer['RepulsiveRandomDitherFieldPerNight'] = slicers.HealpixSlicer(lonCol='repulsiveRandomDitherFieldPerNightRa',
# latCol='repulsiveRandomDitherFieldPerNightDec',
# latLonDeg=raDecInDeg, nside=nside,
# useCache=False)
#slicer['RepulsiveRandomDitherFieldPerVisit'] = slicers.HealpixSlicer(lonCol='repulsiveRandomDitherFieldPerVisitRa',
# latCol='repulsiveRandomDitherFieldPerVisitDec',
# latLonDeg=raDecInDeg, nside=nside,
# useCache=False)
# spiral dithers on different timescales
slicer['FermatSpiralDitherPerNight'] = slicers.HealpixSlicer(
lonCol='fermatSpiralDitherPerNightRa',
latCol='fermatSpiralDitherPerNightDec',
latLonDeg=raDecInDeg,
nside=nside,
useCache=False)
slicer['FermatSpiralDitherFieldPerNight'] = slicers.HealpixSlicer(
lonCol='fermatSpiralDitherFieldPerNightRa',
latCol='fermatSpiralDitherFieldPerNightDec',
latLonDeg=raDecInDeg,
nside=nside,
useCache=False)
slicer['FermatSpiralDitherFieldPerVisit'] = slicers.HealpixSlicer(
lonCol='fermatSpiralDitherFieldPerVisitRa',
latCol='fermatSpiralDitherFieldPerVisitDec',
latLonDeg=raDecInDeg,
nside=nside,
useCache=False)
# hex dithers on different timescales
slicer['SequentialHexDitherPerNight'] = slicers.HealpixSlicer(
lonCol='hexDitherPerNightRa',
latCol='hexDitherPerNightDec',
latLonDeg=raDecInDeg,
nside=nside,
useCache=False)
slicer['SequentialHexDitherFieldPerNight'] = slicers.HealpixSlicer(
lonCol='hexDitherFieldPerNightRa',
latCol='hexDitherFieldPerNightDec',
latLonDeg=raDecInDeg,
nside=nside,
useCache=False)
slicer['SequentialHexDitherFieldPerVisit'] = slicers.HealpixSlicer(
lonCol='hexDitherFieldPerVisitRa',
latCol='hexDitherFieldPerVisitDec',
latLonDeg=raDecInDeg,
nside=nside,
useCache=False)
# per season dithers
slicer['PentagonDitherPerSeason'] = slicers.HealpixSlicer(
lonCol='pentagonDitherPerSeasonRa',
latCol='pentagonDitherPerSeasonDec',
latLonDeg=raDecInDeg,
nside=nside,
useCache=False)
slicer['PentagonDiamondDitherPerSeason'] = slicers.HealpixSlicer(
lonCol='pentagonDiamondDitherPerSeasonRa',
latCol='pentagonDiamondDitherPerSeasonDec',
latLonDeg=raDecInDeg,
nside=nside,
useCache=False)
slicer['SpiralDitherPerSeason'] = slicers.HealpixSlicer(
lonCol='spiralDitherPerSeasonRa',
latCol='spiralDitherPerSeasonDec',
latLonDeg=raDecInDeg,
nside=nside,
useCache=False)
if specifiedDith is not None:
stackerList_, slicer_ = {}, {}
if specifiedDith in slicer.keys():
if specifiedDith.__contains__(
'Random'
): # only Random dithers have a stacker object for rand seed specification
stackerList_[specifiedDith] = stackerList[specifiedDith]
slicer_[specifiedDith] = slicer[specifiedDith]
else:
raise ValueError(
'Invalid value for specifiedDith: %s. Allowed values include one of the following:\n%s'
% (specifiedDith, slicer.keys()))
stackerList, slicer = stackerList_, slicer_
# ------------------------------------------------------------------------
if slair:
m5Col = 'fivesigmadepth'
else:
m5Col = 'fiveSigmaDepth'
# set up the metric
if includeDustExtinction:
# include dust extinction when calculating the co-added depth
coaddMetric = metrics.ExgalM5(m5Col=m5Col, lsstFilter=filterBand)
else:
coaddMetric = metrics.Coaddm5Metric(m5col=m5col)
dustMap = maps.DustMap(
interp=False, nside=nside
) # include dustMap; actual in/exclusion of dust is handled by the galaxyCountMetric
# ------------------------------------------------------------------------
# set up the bundle
coaddBundle = {}
for dither in slicer:
if dither in stackerList:
coaddBundle[dither] = metricBundles.MetricBundle(
coaddMetric,
slicer[dither],
sqlconstraint,
stackerList=stackerList[dither],
runName=runName,
metadata=dither,
mapsList=[dustMap])
else:
coaddBundle[dither] = metricBundles.MetricBundle(
coaddMetric,
slicer[dither],
sqlconstraint,
runName=runName,
metadata=dither,
mapsList=[dustMap])
# ------------------------------------------------------------------------
# run the analysis
if includeDustExtinction:
print('\n# Running coaddBundle with dust extinction ...')
else:
print('\n# Running coaddBundle without dust extinction ...')
cGroup = metricBundles.MetricBundleGroup(coaddBundle,
opsdb,
outDir=outDir,
resultsDb=resultsDb,
saveEarly=False)
cGroup.runAll()
# ------------------------------------------------------------------------
# plot and save the data
plotBundleMaps(path,
outDir,
coaddBundle,
dataLabel='$%s$-band Coadded Depth' % filterBand,
filterBand=filterBand,
dataName='%s-band Coadded Depth' % filterBand,
skymap=plotSkymap,
powerSpectrum=plotPowerSpectrum,
cartview=plotCartview,
colorMin=unmaskedColorMin,
colorMax=unmaskedColorMax,
nTicks=nTicks,
showPlots=showPlots,
saveFigs=saveFigs,
outDirNameForSavedFigs='coaddM5Plots_unmaskedBorders')
print('\n# Done saving plots without border masking.\n')
# ------------------------------------------------------------------------
plotHandler = plots.PlotHandler(outDir=outDir,
resultsDb=resultsDb,
thumbnail=False,
savefig=False)
print(
'# Number of pixels in the survey region (before masking the border):')
for dither in coaddBundle:
print(
' %s: %s' %
(dither,
len(np.where(coaddBundle[dither].metricValues.mask == False)[0])))
# ------------------------------------------------------------------------
# save the unmasked data?
if saveunMaskedCoaddData:
outDir_new = 'unmaskedCoaddData'
if not os.path.exists('%s%s/%s' % (path, outDir, outDir_new)):
os.makedirs('%s%s/%s' % (path, outDir, outDir_new))
saveBundleData_npzFormat('%s%s/%s' % (path, outDir, outDir_new),
coaddBundle, 'coaddM5Data_unmasked',
filterBand)
# ------------------------------------------------------------------------
# mask the edges
print('\n# Masking the edges for coadd ...')
coaddBundle = maskingAlgorithmGeneralized(
coaddBundle,
plotHandler,
dataLabel='$%s$-band Coadded Depth' % filterBand,
nside=nside,
pixelRadius=pixelRadiusForMasking,
plotIntermediatePlots=False,
plotFinalPlots=False,
printFinalInfo=True)
if (pixelRadiusForMasking > 0):
# plot and save the masked data
plotBundleMaps(path,
outDir,
coaddBundle,
dataLabel='$%s$-band Coadded Depth' % filterBand,
filterBand=filterBand,
dataName='%s-band Coadded Depth' % filterBand,
skymap=plotSkymap,
powerSpectrum=plotPowerSpectrum,
cartview=plotCartview,
colorMin=maskedColorMin,
colorMax=maskedColorMax,
nTicks=nTicks,
showPlots=showPlots,
saveFigs=saveFigs,
outDirNameForSavedFigs='coaddM5Plots_maskedBorders')
print('\n# Done saving plots with border masking. \n')
# ------------------------------------------------------------------------
# Calculate total power
summarymetric = metrics.TotalPowerMetric()
for dither in coaddBundle:
coaddBundle[dither].setSummaryMetrics(summarymetric)
coaddBundle[dither].computeSummaryStats()
print('# Total power for %s case is %f.' %
(dither, coaddBundle[dither].summaryValues['TotalPower']))
print('')
# ------------------------------------------------------------------------
# run the alm analysis
if almAnalysis:
almPlots(path,
outDir,
copy.deepcopy(coaddBundle),
nside=nside,
filterband=filterBand,
raRange=raRange,
decRange=decRange,
showPlots=showPlots)
# ------------------------------------------------------------------------
# save the masked data?
if saveMaskedCoaddData and (pixelRadiusForMasking > 0):
outDir_new = 'maskedCoaddData'
if not os.path.exists('%s%s/%s' % (path, outDir, outDir_new)):
os.makedirs('%s%s/%s' % (path, outDir, outDir_new))
saveBundleData_npzFormat('%s%s/%s' % (path, outDir, outDir_new),
coaddBundle, 'coaddM5Data_masked', filterBand)
# ------------------------------------------------------------------------
# plot comparison plots
if len(coaddBundle.keys()) > 1: # more than one key
# set up the directory
outDir_comp = 'coaddM5ComparisonPlots'
if not os.path.exists('%s%s/%s' % (path, outDir, outDir_comp)):
os.makedirs('%s%s/%s' % (path, outDir, outDir_comp))
# ------------------------------------------------------------------------
# plot for the power spectra
cl = {}
for dither in plotColor:
if dither in coaddBundle:
cl[dither] = hp.anafast(hp.remove_dipole(
coaddBundle[dither].metricValues.filled(
coaddBundle[dither].slicer.badval)),
lmax=500)
ell = np.arange(np.size(cl[dither]))
plt.plot(ell, (cl[dither] * ell * (ell + 1)) / 2.0 / np.pi,
color=plotColor[dither],
linestyle='-',
label=dither)
plt.xlabel(r'$\ell$')
plt.ylabel(r'$\ell(\ell+1)C_\ell/(2\pi)$')
plt.xlim(0, 500)
fig = plt.gcf()
fig.set_size_inches(12.5, 10.5)
leg = plt.legend(labelspacing=0.001)
for legobj in leg.legendHandles:
legobj.set_linewidth(4.0)
filename = 'powerspectrum_comparison_all.png'
plt.savefig('%s%s/%s/%s' % (path, outDir, outDir_comp, filename),
bbox_inches='tight',
format='png')
plt.show()
# create the histogram
scale = hp.nside2pixarea(nside, degrees=True)
def tickFormatter(y, pos):
return '%d' % (y * scale) # convert pixel count to area
binsize = 0.01
for dither in plotColor:
if dither in coaddBundle:
ind = np.where(
coaddBundle[dither].metricValues.mask == False)[0]
binAll = int(
(max(coaddBundle[dither].metricValues.data[ind]) -
min(coaddBundle[dither].metricValues.data[ind])) /
binsize)
plt.hist(coaddBundle[dither].metricValues.data[ind],
bins=binAll,
label=dither,
histtype='step',
color=plotColor[dither])
ax = plt.gca()
ymin, ymax = ax.get_ylim()
nYticks = 10.
wantedYMax = ymax * scale
wantedYMax = 10. * np.ceil(float(wantedYMax) / 10.)
increment = 5. * np.ceil(float(wantedYMax / nYticks) / 5.)
wantedArray = np.arange(0, wantedYMax, increment)
ax.yaxis.set_ticks(wantedArray / scale)
ax.yaxis.set_major_formatter(FuncFormatter(tickFormatter))
plt.xlabel('$%s$-band Coadded Depth' % filterBand)
plt.ylabel('Area (deg$^2$)')
fig = plt.gcf()
fig.set_size_inches(12.5, 10.5)
leg = plt.legend(labelspacing=0.001, loc=2)
for legobj in leg.legendHandles:
legobj.set_linewidth(2.0)
filename = 'histogram_comparison.png'
plt.savefig('%s%s/%s/%s' % (path, outDir, outDir_comp, filename),
bbox_inches='tight',
format='png')
plt.show()
# ------------------------------------------------------------------------
# plot power spectra for the separte panel
totKeys = len(list(coaddBundle.keys()))
if (totKeys > 1):
plt.clf()
nCols = 2
nRows = int(np.ceil(float(totKeys) / nCols))
fig, ax = plt.subplots(nRows, nCols)
plotRow = 0
plotCol = 0
for dither in list(plotColor.keys()):
if dither in list(coaddBundle.keys()):
ell = np.arange(np.size(cl[dither]))
ax[plotRow, plotCol].plot(ell, (cl[dither] * ell *
(ell + 1)) / 2.0 / np.pi,
color=plotColor[dither],
label=dither)
if (plotRow == nRows - 1):
ax[plotRow, plotCol].set_xlabel(r'$\ell$')
ax[plotRow,
plotCol].set_ylabel(r'$\ell(\ell+1)C_\ell/(2\pi)$')
ax[plotRow,
plotCol].yaxis.set_major_locator(MaxNLocator(3))
if (dither != 'NoDither'):
ax[plotRow, plotCol].set_ylim(0, 0.0035)
ax[plotRow, plotCol].set_xlim(0, 500)
plotRow += 1
if (plotRow > nRows - 1):
plotRow = 0
plotCol += 1
fig.set_size_inches(20, int(nRows * 30 / 7.))
filename = 'powerspectrum_sepPanels.png'
plt.savefig('%s%s/%s/%s' % (path, outDir, outDir_comp, filename),
bbox_inches='tight',
format='png')
plt.show()
return coaddBundle, outDir
