def load_and_run():
dbFile = 'baseline_nexp2_v1.7_10yrs.db'
opsimdb = db.OpsimDatabase(dbFile)
runName = dbFile.replace('.db', '')
nside = 64
slicer = slicers.HealpixSlicer(nside=nside)
metric = SNNSNMetric(verbose=False) #, zlim_coeff=0.98)
bundleList = []
#sql = ''
sql = '(note = "%s")' % ('DD:COSMOS')
bundleList.append(
metricBundles.MetricBundle(metric, slicer, sql, runName=runName))
outDir = 'temp'
resultsDb = db.ResultsDb(outDir=outDir)
bundleDict = metricBundles.makeBundlesDictFromList(bundleList)
bgroup = metricBundles.MetricBundleGroup(bundleDict,
opsimdb,
outDir=outDir,
resultsDb=resultsDb)
bgroup.runAll()
bgroup.plotAll()
def tdcBatch(colmap=None, runName='opsim', nside=64, accuracyThreshold=0.04,
extraSql=None, extraMetadata=None):
# The options to add additional sql constraints are removed for now.
if colmap is None:
colmap = ColMapDict('fbs')
# Calculate a subset of DESC WFD-related metrics.
displayDict = {'group': 'Strong Lensing'}
displayDict['subgroup'] = 'Lens Time Delay'
subsetPlots = [plots.HealpixSkyMap(), plots.HealpixHistogram()]
summaryMetrics = [metrics.MeanMetric(), metrics.MedianMetric(), metrics.RmsMetric()]
# Ideally need a way to do better on calculating the summary metrics for the high accuracy area.
slicer = slicers.HealpixSlicer(nside=nside)
tdcMetric = metrics.TdcMetric(metricName='TDC', nightCol=colmap['night'],
expTimeCol=colmap['exptime'], mjdCol=colmap['mjd'])
bundle = mb.MetricBundle(tdcMetric, slicer, constraint=extraSql, metadata=extraMetadata,
displayDict=displayDict, plotFuncs=subsetPlots,
summaryMetrics=summaryMetrics)
bundleList = [bundle]
# Set the runName for all bundles and return the bundleDict.
for b in bundleList:
b.setRunName(runName)
return mb.makeBundlesDictFromList(bundleList)
def altazLambert(colmap=None,
runName='opsim',
extraSql=None,
extraMetadata=None,
metricName='Nvisits as function of Alt/Az'):
"""Generate a set of metrics measuring the number visits as a function of alt/az
plotted on a LambertSkyMap.
Parameters
----------
colmap : dict, opt
A dictionary with a mapping of column names. Default will use OpsimV4 column names.
runName : str, opt
The name of the simulated survey. Default is "opsim".
extraSql : str, opt
Additional constraint to add to any sql constraints (e.g. 'propId=1' or 'fieldID=522').
Default None, for no additional constraints.
extraMetadata : str, opt
Additional metadata to add before any below (i.e. "WFD"). Default is None.
metricName : str, opt
Unique name to assign to metric
Returns
-------
metricBundleDict
"""
colmap, slicer, metric = basicSetup(metricName=metricName, colmap=colmap)
# Set up basic all and per filter sql constraints.
filterlist, colors, orders, sqls, metadata = filterList(
all=True, extraSql=extraSql, extraMetadata=extraMetadata)
bundleList = []
plotFunc = plots.LambertSkyMap()
for f in filterlist:
if f is 'all':
subgroup = 'All Observations'
else:
subgroup = 'Per filter'
displayDict = {
'group': 'Alt/Az',
'order': orders[f],
'subgroup': subgroup,
'caption': 'Alt/Az pointing distribution for filter %s' % f
}
bundle = mb.MetricBundle(metric,
slicer,
sqls[f],
runName=runName,
metadata=metadata[f],
plotFuncs=[plotFunc],
displayDict=displayDict)
bundleList.append(bundle)
for b in bundleList:
b.setRunName(runName)
return mb.makeBundlesDictFromList(bundleList)
def nvisitsPerNight(colmap=None,
runName='opsim',
binNights=1,
sqlConstraint=None,
metadata=None):
"""Count the number of visits per night through the survey.
Parameters
----------
colmap : dict or None, opt
A dictionary with a mapping of column names. Default will use OpsimV4 column names.
runName : str, opt
The name of the simulated survey. Default is "opsim".
binNights : int, opt
Number of nights to count in each bin. Default = 1, count number of visits in each night.
sqlConstraint : str or None, opt
Additional constraint to add to any sql constraints (e.g. 'propId=1' or 'fieldID=522').
Default None, for no additional constraints.
metadata : str or None, opt
Additional metadata to add before any below (i.e. "WFD"). Default is None.
Returns
-------
metricBundleDict
"""
if colmap is None:
colmap = ColMapDict('opsimV4')
subgroup = metadata
if subgroup is None:
subgroup = 'All visits'
metadataCaption = metadata
if metadata is None:
if sqlConstraint is not None:
metadataCaption = sqlConstraint
else:
metadataCaption = 'all visits'
bundleList = []
displayDict = {'group': 'Per Night', 'subgroup': subgroup}
displayDict['caption'] = 'Number of visits per night for %s.' % (
metadataCaption)
displayDict['order'] = 0
metric = metrics.CountMetric(colmap['mjd'], metricName='Nvisits')
slicer = slicers.OneDSlicer(sliceColName=colmap['mjd'],
binsize=int(binNights))
bundle = mb.MetricBundle(metric,
slicer,
sqlConstraint,
metadata=metadata,
displayDict=displayDict,
summaryMetrics=standardSummary())
bundleList.append(bundle)
# Set the runName for all bundles and return the bundleDict.
for b in bundleList:
b.setRunName(runName)
return mb.makeBundlesDictFromList(bundleList)
def filtersPerNight(colmap=None, runName='opsim', nights=1, extraSql=None, extraMetadata=None):
"""Generate a set of metrics measuring the number and rate of filter changes over a given span of nights.
Parameters
----------
colmap : dict, opt
A dictionary with a mapping of column names. Default will use OpsimV4 column names.
run_name : str, opt
The name of the simulated survey. Default is "opsim".
nights : int, opt
Size of night bin to use when calculating metrics. Default is 1.
extraSql : str, opt
Additional constraint to add to any sql constraints (e.g. 'propId=1' or 'fieldID=522').
Default None, for no additional constraints.
extraMetadata : str, opt
Additional metadata to add before any below (i.e. "WFD"). Default is None.
Returns
-------
metricBundleDict
"""
if colmap is None:
colmap = ColMapDict('opsimV4')
bundleList = []
# Set up sql and metadata, if passed any additional information.
sql = ''
metadata = 'Per'
if nights == 1:
metadata += ' Night'
else:
metadata += ' %s Nights' % nights
metacaption = metadata.lower()
if (extraSql is not None) and (len(extraSql) > 0):
sql = extraSql
if extraMetadata is None:
metadata += ' %s' % extraSql
metacaption += ', with %s selection' % extraSql
if extraMetadata is not None:
metadata += ' %s' % extraMetadata
metacaption += ', %s only' % extraMetadata
metacaption += '.'
displayDict = {'group': 'Filter Changes', 'subgroup': metadata}
summaryStats = standardSummary()
slicer = slicers.OneDSlicer(sliceColName=colmap['night'], binsize=nights)
metricList, captionList = setupMetrics(colmap)
for m, caption in zip(metricList, captionList):
displayDict['caption'] = caption + metacaption
bundle = mb.MetricBundle(m, slicer, sql, runName=runName, metadata=metadata,
displayDict=displayDict,
summaryMetrics=summaryStats)
bundleList.append(bundle)
for b in bundleList:
b.setRunName(runName)
return mb.makeBundlesDictFromList(bundleList)
def makeBundleList(dbFile, runName=None, nside=128, benchmark='design',
lonCol='fieldRA', latCol='fieldDec'):
"""
make a list of metricBundle objects to look at the scientific performance
of an opsim run.
"""
# List to hold everything we're going to make
bundleList = []
filtorder = {'u':1,'g':2,'r':3,'i':4,'z':5,'y':6}
plotList = [plotters.HealpixSkyMap(), plotters.HealpixHistogram()]
slicer=slicers.HealpixSlicer(nside=nside)
plotDict = {'cmap':'jet'}
metricList = []
metricList.append(metrics.IntraNightGapsMetric())
metricList.append(metrics.InterNightGapsMetric())
metricList.append(metrics.IntraNightGapsMetric(reduceFunc=np.size,
metricName='Number of in-night pairs'))
filters = ['u','g','r','i','z','y']
summaryStats = [metrics.MedianMetric()]
ddList = [ {'group':'Median gap, in-night'}, {'group':'Median gap, between nights'},
{'group': 'In-night Pairs'} ]
for f in filters:
sql = "filter='%s'"%f
for metric,dd in zip(metricList,ddList):
displayDict = dd
displayDict['order'] = filtorder[f]
bundleList.append(metricBundles.MetricBundle(metric,slicer,sql, plotFuncs=plotList,
plotDict=plotDict, summaryMetrics=summaryStats,
displayDict=displayDict))
for metric,dd in zip(metricList,ddList):
sql = ''
displayDict = dd
displayDict['order'] = 7
bundleList.append(metricBundles.MetricBundle(metric,slicer,sql, plotFuncs=plotList,
plotDict=plotDict, summaryMetrics=summaryStats,
displayDict=displayDict))
slicer = slicers.OneDSlicer(sliceColName='night', binsize=1)
metric = metrics.UniqueRatioMetric(col='fieldID')
sql='filter="g" or filter="r" or filter="i" or filter="z"'
displayDict = {'group':'Fields Per Night'}
bundleList.append(metricBundles.MetricBundle(metric,slicer,sql,
summaryMetrics=summaryStats,
displayDict=displayDict))
metric = metrics.CountUniqueMetric(col='fieldID')
bundleList.append(metricBundles.MetricBundle(metric,slicer,sql,
summaryMetrics=summaryStats,
displayDict=displayDict))
return metricBundles.makeBundlesDictFromList(bundleList)
def filtersWholeSurveyBatch(colmap=None,
runName='opsim',
extraSql=None,
extraMetadata=None):
"""Generate a set of metrics measuring the number and rate of filter changes over the entire survey.
Parameters
----------
colmap : dict, opt
A dictionary with a mapping of column names. Default will use OpsimV4 column names.
run_name : str, opt
The name of the simulated survey. Default is "opsim".
extraSql : str, opt
Additional constraint to add to any sql constraints (e.g. 'propId=1' or 'fieldID=522').
Default None, for no additional constraints.
extraMetadata : str, opt
Additional metadata to add before any below (i.e. "WFD"). Default is None.
Returns
-------
metricBundleDict
"""
if colmap is None:
colmap = ColMapDict('opsimV4')
bundleList = []
# Set up sql and metadata, if passed any additional information.
sql = ''
metadata = 'Whole Survey'
metacaption = 'over the whole survey'
if (extraSql is not None) and (len(extraSql) > 0):
sql = extraSql
if extraMetadata is None:
metadata += ' %s' % extraSql
metacaption += ', with %s selction' % extraSql
if extraMetadata is not None:
metadata += ' %s' % extraMetadata
metacaption += ', %s only' % (extraMetadata)
metacaption += '.'
displayDict = {'group': 'Filter Changes', 'subgroup': metadata}
slicer = slicers.UniSlicer()
metricList, captionList = setupMetrics(colmap)
for m, caption in zip(metricList, captionList):
displayDict['caption'] = caption + metacaption
bundle = mb.MetricBundle(m,
slicer,
sql,
runName=runName,
metadata=metadata,
displayDict=displayDict)
bundleList.append(bundle)
for b in bundleList:
b.setRunName(runName)
return mb.makeBundlesDictFromList(bundleList)
def runSlices(opsimName, metadata, simdata, bins, args, verbose=False):
"""
Set up and run the movie slicer, and at each step,
setup the healpix slicer and run the metrics, creating and saving the plots.
"""
# Set up movie slicer
movieslicer = setupMovieSlicer(simdata, binsize = args.movieStepsize, cumulative=args.cumulative)
start_date = movieslicer[0]['slicePoint']['binLeft']
sliceformat = '%s0%dd' %('%', int(np.log10(len(movieslicer)))+1)
# Run through the movie slicer slicePoints:
for i, movieslice in enumerate(movieslicer):
t = time.time()
slicenumber = sliceformat %(i)
"""
# Set up plot label.
timeInterval = '%.2f to %.2f' %(movieslice['slicePoint']['binLeft']-start_date,
movieslice['slicePoint']['binRight']-start_date)
"""
# Or add simple view of time to plot label.
times_from_start = movieslice['slicePoint']['binRight'] - (int(bins[0]) + 0.16 - 0.5)
# Opsim years are 365 days (not 365.25)
years = int(times_from_start/365)
days = times_from_start - years*365
plotlabel = 'Year %d Day %.4f' %(years, days)
metricList, plotDictList = setupMetrics(opsimName, metadata,
start_date, movieslice['slicePoint']['binRight'],
cumulative=args.cumulative, plotlabel=plotlabel,
verbose=verbose)
# Identify the subset of simdata in the movieslicer 'data slice'
simdatasubset = simdata[movieslice['idxs']]
# Set up healpix slicer on subset of simdata provided by movieslicer
hs = setupHealpixSlicer(args)
bundles = []
for metric, plotDict in zip(metricList, plotDictList):
bundles.append(mB.MetricBundle(metric, hs, sqlconstraint=args.sqlConstraint,
metadata=metadata, runName=opsimName, plotDict=plotDict,
plotFuncs=[plots.HealpixSkyMap()]))
# Remove (default) stackers from bundles, because we've already run them above on the original data.
for mb in bundles:
mb.stackerList = []
bundledict = mB.makeBundlesDictFromList(bundles)
# Set up metricBundleGroup to handle metrics calculation + plotting
bg = mB.MetricBundleGroup(bundledict, opsDb, outDir=args.outDir, resultsDb=None, saveEarly=False)
# Calculate metric data values for simdatasubset (this also sets up indexing in the slicer)
bg.setCurrent(args.sqlConstraint)
bg.runCurrent(constraint=args.sqlConstraint, simData=simdatasubset)
# Plot data for this slice of the movie, adding slicenumber as a suffix for output plots
bg.plotAll(outfileSuffix=slicenumber, closefigs=True, dpi=72, thumbnail=False, figformat='png')
# Write the data -- uncomment if you want to do this.
# sm.writeAll(outfileSuffix=slicenumber)
if verbose:
dt, t = dtime(t)
print('Ran and plotted slice %s of movieslicer in %f s' %(slicenumber, dt))
def fOBatch(colmap=None, runName='opsim', extraSql=None, extraMetadata=None, nside=64,
benchmarkArea=18000, benchmarkNvisits=825):
# Allow user to add dithering.
if colmap is None:
colmap = ColMapDict('opsimV4')
bundleList = []
sql = ''
metadata = 'All visits'
# Add additional sql constraint (such as wfdWhere) and metadata, if provided.
if (extraSql is not None) and (len(extraSql) > 0):
sql = extraSql
if extraMetadata is None:
metadata = extraSql.replace('filter =', '').replace('filter=', '')
metadata = metadata.replace('"', '').replace("'", '')
if extraMetadata is not None:
metadata = extraMetadata
subgroup = metadata
raCol = colmap['ra']
decCol = colmap['dec']
degrees = colmap['raDecDeg']
# Set up fO metric.
slicer = slicers.HealpixSlicer(nside=nside, lonCol=raCol, latCol=decCol, latLonDeg=degrees)
displayDict = {'group': 'FO metrics', 'subgroup': subgroup, 'order': 0}
# Configure the count metric which is what is used for f0 slicer.
metric = metrics.CountMetric(col=colmap['mjd'], metricName='fO')
plotDict = {'xlabel': 'Number of Visits', 'Asky': benchmarkArea,
'Nvisit': benchmarkNvisits, 'xMin': 0, 'xMax': 1500}
summaryMetrics = [metrics.fOArea(nside=nside, norm=False, metricName='fOArea: Nvisits (#)',
Asky=benchmarkArea, Nvisit=benchmarkNvisits),
metrics.fOArea(nside=nside, norm=True, metricName='fOArea: Nvisits/benchmark',
Asky=benchmarkArea, Nvisit=benchmarkNvisits),
metrics.fONv(nside=nside, norm=False, metricName='fONv: Area (sqdeg)',
Asky=benchmarkArea, Nvisit=benchmarkNvisits),
metrics.fONv(nside=nside, norm=True, metricName='fONv: Area/benchmark',
Asky=benchmarkArea, Nvisit=benchmarkNvisits)]
caption = 'The FO metric evaluates the overall efficiency of observing. '
caption += ('fOArea: Nvisits = %.1f sq degrees receive at least this many visits out of %d. '
% (benchmarkArea, benchmarkNvisits))
caption += ('fONv: Area = this many square degrees out of %.1f receive at least %d visits.'
% (benchmarkArea, benchmarkNvisits))
displayDict['caption'] = caption
bundle = mb.MetricBundle(metric, slicer, sql, plotDict=plotDict,
displayDict=displayDict, summaryMetrics=summaryMetrics,
plotFuncs=[plots.FOPlot()], metadata=metadata)
bundleList.append(bundle)
# Set the runName for all bundles and return the bundleDict.
for b in bundleList:
b.setRunName(runName)
return mb.makeBundlesDictFromList(bundleList)
def slewAngles(colmap=None, runName='opsim', sqlConstraint=None):
"""Generate a set of slew statistics focused on the angles of each component (dome and telescope).
These slew statistics must be run on the SlewFinalState or SlewInitialState table in opsimv4,
and on the SlewState table in opsimv3.
Parameters
----------
colmap : dict or None, opt
A dictionary with a mapping of column names. Default will use OpsimV4 column names.
runName : str, opt
The name of the simulated survey. Default is "opsim".
sqlConstraint : str or None, opt
SQL constraint to apply to metrics. Note this runs on Slew*State table, so constraints
should generally be based on slew_slewCount.
Returns
-------
metricBundleDict
"""
if colmap is None:
colmap = ColMapDict('opsimV4')
bundleList = []
# All of these metrics are run with a unislicer.
slicer = slicers.UniSlicer()
# For each angle, we will compute mean/median/min/max and rms.
# Note that these angles can range over more than 360 degrees, because of cable wrap.
# This is why we're not using the Angle metrics - here 380 degrees is NOT the same as 20 deg.
# Stats for angle:
angles = ['Tel Alt', 'Tel Az', 'Rot Tel Pos']
displayDict = {
'group': 'Slew',
'subgroup': 'Slew Angles',
'order': -1,
'caption': None
}
for angle in angles:
metadata = angle
metriclist = standardMetrics(colmap[angle], replace_colname='')
metriclist += [metrics.RmsMetric(colmap[angle], metricName='RMS')]
for metric in metriclist:
displayDict['caption'] = '%s %s' % (metric.name, angle)
displayDict['order'] += 1
bundle = mb.MetricBundle(metric,
slicer,
sqlConstraint,
displayDict=displayDict,
metadata=metadata)
bundleList.append(bundle)
for b in bundleList:
b.setRunName(runName)
return mb.makeBundlesDictFromList(bundleList)
def nvisitsPerNight(colmap=None, runName='opsim', binNights=1,
extraSql=None, extraMetadata=None, subgroup=None):
"""Count the number of visits per night through the survey.
Parameters
----------
colmap : dict or None, opt
A dictionary with a mapping of column names. Default will use OpsimV4 column names.
runName : str, opt
The name of the simulated survey. Default is "opsim".
binNights : int, opt
Number of nights to count in each bin. Default = 1, count number of visits in each night.
extraSql : str or None, opt
Additional constraint to add to any sql constraints (e.g. 'propId=1' or 'fieldID=522').
Default None, for no additional constraints.
extraMetadata : str or None, opt
Additional metadata to add before any below (i.e. "WFD"). Default is None.
subgroup : str or None, opt
Use this for the 'subgroup' in the displayDict, instead of metadata. Default is None.
Returns
-------
metricBundleDict
"""
if colmap is None:
colmap = ColMapDict('opsimV4')
subgroup = subgroup
if subgroup is None:
subgroup = extraMetadata
if subgroup is None:
subgroup = 'All visits'
metadataCaption = extraMetadata
if extraMetadata is None:
if extraSql is not None:
metadataCaption = extraSql
else:
metadataCaption = 'all visits'
bundleList = []
displayDict = {'group': 'Nvisits Per Night', 'subgroup': subgroup}
displayDict['caption'] = 'Number of visits per night for %s.' % (metadataCaption)
displayDict['order'] = 0
metric = metrics.CountMetric(colmap['mjd'], metricName='Nvisits')
slicer = slicers.OneDSlicer(sliceColName=colmap['night'], binsize=binNights)
bundle = mb.MetricBundle(metric, slicer, extraSql, metadata=metadataCaption,
displayDict=displayDict, summaryMetrics=standardSummary())
bundleList.append(bundle)
# Set the runName for all bundles and return the bundleDict.
for b in bundleList:
b.setRunName(runName)
return mb.makeBundlesDictFromList(bundleList)
def hourglassPlots(colmap=None, runName='opsim', nyears=10, extraSql=None, extraMetadata=None):
"""Run the hourglass metric, for each individual year.
Parameters
----------
colmap : dict, opt
A dictionary with a mapping of column names. Default will use OpsimV4 column names.
run_name : str, opt
The name of the simulated survey. Default is "opsim".
nyears : int (10), opt
How many years to attempt to make hourglass plots for. Default is 10.
extraSql : str, opt
Add an extra sql constraint before running metrics. Default None.
extraMetadata : str, opt
Add an extra piece of metadata before running metrics. Default None.
"""
if colmap is None:
colmap = ColMapDict('opsimV4')
bundleList = []
sql = ''
metadata = ''
# Add additional sql constraint (such as wfdWhere) and metadata, if provided.
if (extraSql is not None) and (len(extraSql) > 0):
sql = extraSql
if extraMetadata is None:
metadata = extraSql.replace('filter =', '').replace('filter=', '')
metadata = metadata.replace('"', '').replace("'", '')
if extraMetadata is not None:
metadata = extraMetadata
years = list(range(nyears + 1))
displayDict = {'group': 'Hourglass'}
for year in years[1:]:
displayDict['subgroup'] = 'Year %d' % year
displayDict['caption'] = 'Visualization of the filter usage of the telescope. ' \
'The black wavy line indicates lunar phase; the red and blue ' \
'solid lines indicate nautical and civil twilight.'
sqlconstraint = 'night > %i and night <= %i' % (365.25 * (year - 1), 365.25 * year)
if len(sql) > 0:
sqlconstraint = '(%s) and (%s)' % (sqlconstraint, sql)
md = metadata + ' year %i-%i' % (year - 1, year)
slicer = slicers.HourglassSlicer()
metric = metrics.HourglassMetric(nightCol=colmap['night'], mjdCol=colmap['mjd'],
metricName='Hourglass')
bundle = mb.MetricBundle(metric, slicer, constraint=sqlconstraint, metadata=md,
displayDict=displayDict)
bundleList.append(bundle)
# Set the runName for all bundles and return the bundleDict.
for b in bundleList:
b.setRunName(runName)
return mb.makeBundlesDictFromList(bundleList)
def slewSpeeds(colmap=None, runName='opsim', sqlConstraint=None):
"""Generate a set of slew statistics focused on the speeds of each component (dome and telescope).
These slew statistics must be run on the SlewMaxSpeeds table in opsimv4 and opsimv3.
Parameters
----------
colmap : dict or None, opt
A dictionary with a mapping of column names. Default will use OpsimV4 column names.
Note that for these metrics, the column names are distinctly different in v3/v4.
runName : str, opt
The name of the simulated survey. Default is "opsim".
sqlConstraint : str or None, opt
SQL constraint to apply to metrics. Note this runs on Slew*State table, so constraints
should generally be based on slew_slewCount.
Returns
-------
metricBundleDict
"""
if colmap is None:
colmap = ColMapDict('opsimV4')
bundleList = []
# All of these metrics run with a unislicer, on all the slew data.
slicer = slicers.UniSlicer()
speeds = ['Dome Alt Speed', 'Dome Az Speed', 'Tel Alt Speed', 'Tel Az Speed', 'Rotator Speed']
displayDict = {'group': 'Slew', 'subgroup': 'Slew Speeds', 'order': -1, 'caption': None}
for speed in speeds:
metadata = combineMetadata(speed, sqlConstraint)
metric = metrics.AbsMaxMetric(col=colmap[speed], metricName='Max (Abs)')
displayDict['caption'] = 'Maximum absolute value of %s.' % speed
displayDict['order'] += 1
bundle = mb.MetricBundle(metric, slicer, sqlConstraint, displayDict=displayDict, metadata=metadata)
bundleList.append(bundle)
metric = metrics.AbsMeanMetric(col=colmap[speed], metricName='Mean (Abs)')
displayDict['caption'] = 'Mean absolute value of %s.' % speed
displayDict['order'] += 1
bundle = mb.MetricBundle(metric, slicer, sqlConstraint, displayDict=displayDict, metadata=metadata)
bundleList.append(bundle)
metric = metrics.AbsMaxPercentMetric(col=colmap[speed], metricName='% @ Max')
displayDict['caption'] = 'Percent of slews at the maximum %s (absolute value).' % speed
displayDict['order'] += 1
bundle = mb.MetricBundle(metric, slicer, sqlConstraint, displayDict=displayDict, metadata=metadata)
bundleList.append(bundle)
for b in bundleList:
b.setRunName(runName)
return mb.makeBundlesDictFromList(bundleList)
def meanRADec(colmap=None, runName='opsim', extraSql=None, extraMetadata=None):
"""Plot the range of RA/Dec as a function of night.
Parameters
----------
colmap : dict, opt
A dictionary with a mapping of column names. Default will use OpsimV4 column names.
runName : str, opt
The name of the simulated survey. Default is "opsim".
extraSql : str, opt
Additional constraint to add to any sql constraints (e.g. 'night<365')
Default None, for no additional constraints.
extraMetadata : str, opt
Additional metadata to add before any below (i.e. "WFD"). Default is None.
"""
if colmap is None:
colmap = ColMapDict('opsimV4')
bundleList = []
plotBundles = []
group = 'RA Dec coverage'
subgroup = 'All visits'
if extraMetadata is not None:
subgroup = extraMetadata
displayDict = {'group': group, 'subgroup': subgroup, 'order': 0}
ra_metrics = [metrics.MeanAngleMetric(colmap['ra']), metrics.FullRangeAngleMetric(colmap['ra'])]
dec_metrics = [metrics.MeanMetric(colmap['dec']), metrics.MinMetric(colmap['dec']),
metrics.MaxMetric(colmap['dec'])]
for m in ra_metrics:
slicer = slicers.OneDSlicer(sliceColName=colmap['night'], binsize=1)
if not colmap['raDecDeg']:
plotDict = {'yMin': np.radians(-5), 'yMax': np.radians(365)}
else:
plotDict = {'yMin': -5, 'yMax': 365}
bundle = mb.MetricBundle(m, slicer, extraSql, metadata=extraMetadata,
displayDict=displayDict, plotDict=plotDict)
bundleList.append(bundle)
for m in dec_metrics:
slicer = slicers.OneDSlicer(sliceColName=colmap['night'], binsize=1)
bundle = mb.MetricBundle(m, slicer, extraSql, metadata=extraMetadata,
displayDict=displayDict)
bundleList.append(bundle)
# Set the runName for all bundles and return the bundleDict.
for b in bundleList:
b.setRunName(runName)
return mb.makeBundlesDictFromList(bundleList), plotBundles
def filtersWholeSurvey(colmap=None, runName='opsim', extraSql=None, extraMetadata=None):
"""Generate a set of metrics measuring the number and rate of filter changes over the entire survey.
Parameters
----------
colmap : dict, opt
A dictionary with a mapping of column names. Default will use OpsimV4 column names.
run_name : str, opt
The name of the simulated survey. Default is "opsim".
extraSql : str, opt
Additional constraint to add to any sql constraints (e.g. 'propId=1' or 'fieldID=522').
Default None, for no additional constraints.
extraMetadata : str, opt
Additional metadata to add before any below (i.e. "WFD"). Default is None.
Returns
-------
metricBundleDict
"""
if colmap is None:
colmap = ColMapDict('opsimV4')
bundleList = []
# Set up sql and metadata, if passed any additional information.
sql = ''
metadata = 'Whole Survey'
metacaption = 'over the whole survey'
if (extraSql is not None) and (len(extraSql) > 0):
sql = extraSql
if extraMetadata is None:
metadata += ' %s' % extraSql
metacaption += ', with %s selction' % extraSql
if extraMetadata is not None:
metadata += ' %s' % extraMetadata
metacaption += ', %s only' % (extraMetadata)
metacaption += '.'
displayDict = {'group': 'Filter Changes', 'subgroup': metadata}
slicer = slicers.UniSlicer()
metricList, captionList = setupMetrics(colmap)
for m, caption in zip(metricList, captionList):
displayDict['caption'] = caption + metacaption
bundle = mb.MetricBundle(m, slicer, sql, runName=runName, metadata=metadata,
displayDict=displayDict)
bundleList.append(bundle)
for b in bundleList:
b.setRunName(runName)
return mb.makeBundlesDictFromList(bundleList)
def slewAngles(colmap=None, runName='opsim', sqlConstraint=None):
"""Generate a set of slew statistics focused on the angles of each component (dome and telescope).
These slew statistics must be run on the SlewFinalState or SlewInitialState table in opsimv4,
and on the SlewState table in opsimv3.
Parameters
----------
colmap : dict or None, opt
A dictionary with a mapping of column names. Default will use OpsimV4 column names.
runName : str, opt
The name of the simulated survey. Default is "opsim".
sqlConstraint : str or None, opt
SQL constraint to apply to metrics. Note this runs on Slew*State table, so constraints
should generally be based on slew_slewCount.
Returns
-------
metricBundleDict
"""
if colmap is None:
colmap = ColMapDict('opsimV4')
bundleList = []
# All of these metrics are run with a unislicer.
slicer = slicers.UniSlicer()
# For each angle, we will compute mean/median/min/max and rms.
# Note that these angles can range over more than 360 degrees, because of cable wrap.
# This is why we're not using the Angle metrics - here 380 degrees is NOT the same as 20 deg.
# Stats for angle:
angles = ['Tel Alt', 'Tel Az', 'Rot Tel Pos']
displayDict = {'group': 'Slew', 'subgroup': 'Slew Angles', 'order': -1, 'caption': None}
for angle in angles:
metadata = combineMetadata(angle, sqlConstraint)
metriclist = standardMetrics(colmap[angle], replace_colname='')
metriclist += [metrics.RmsMetric(colmap[angle], metricName='RMS')]
for metric in metriclist:
displayDict['caption'] = '%s %s' % (metric.name, angle)
displayDict['order'] += 1
bundle = mb.MetricBundle(metric, slicer, sqlConstraint,
displayDict=displayDict, metadata=metadata)
bundleList.append(bundle)
for b in bundleList:
b.setRunName(runName)
return mb.makeBundlesDictFromList(bundleList)
def testRunRegularToo(self):
"""
Test that a binned slicer and a regular slicer can run together
"""
bundleList = []
metric = metrics.AccumulateM5Metric(bins=[0.5, 1.5, 2.5])
slicer = slicers.HealpixSlicer(nside=16)
sql = ''
bundleList.append(metricBundle.MetricBundle(metric, slicer, sql))
metric = metrics.Coaddm5Metric()
slicer = slicers.HealpixSlicer(nside=16)
bundleList.append(metricBundle.MetricBundle(metric, slicer, sql))
bd = metricBundle.makeBundlesDictFromList(bundleList)
mbg = metricBundle.MetricBundleGroup(bd, None, saveEarly=False)
mbg.setCurrent('')
mbg.runCurrent('', simData=self.simData)
assert (np.array_equal(bundleList[0].metricValues[:, 1].compressed(),
bundleList[1].metricValues.compressed()))
def testRunRegularToo(self):
"""
Test that a binned slicer and a regular slicer can run together
"""
bundleList = []
metric = metrics.AccumulateM5Metric(bins=[0.5, 1.5, 2.5])
slicer = slicers.HealpixSlicer(nside=16)
sql = ''
bundleList.append(metricBundle.MetricBundle(metric, slicer, sql))
metric = metrics.Coaddm5Metric()
slicer = slicers.HealpixSlicer(nside=16)
bundleList.append(metricBundle.MetricBundle(metric, slicer, sql))
for bundle in bundleList:
bundle.stackerList = []
bd = metricBundle.makeBundlesDictFromList(bundleList)
mbg = metricBundle.MetricBundleGroup(bd, None, saveEarly=False)
mbg.setCurrent('')
mbg.runCurrent('', simData=self.simData)
assert(np.array_equal(bundleList[0].metricValues[:, 1].compressed(),
bundleList[1].metricValues.compressed()))
def go(nside=64, rmag=20., SedTemplate='flat', DoRun=False, LFilters = [], \
LNightMax=[], nightMax=1e4, \
CustomPlotLimits=True, \
RunOne=False, MaxRuns=1e3, \
SpatialClip=95.):
# runNames = ['enigma_1189', 'ops2_1093']
# runNames
#runNames = ['ops2_1092', 'kraken_1038', 'kraken_1034', 'ops2_1098']
#runNames = ['kraken_1038', 'kraken_1034', 'ops2_1098']
# 2015-12-23 - put kraken_1038 at the end, it seems to run
# extremely slowly...
runNames = ['enigma_1189', 'ops2_1098', 'kraken_1034', 'kraken_1038']
runNames = ['ops2_1092', 'kraken_1033', 'enigma_1271']
# UPDATE - ops2_1092 ran quite quickly on nside=32... rerun on 64
runNames = ['ops2_1092', 'enigma_1189', 'enigma_1271', 'kraken_1038']
# UPDATE 2015-12-28 -- run with single-filter choices, compare
# enigma to ops2_1092
# WIC 2015-12-28 -- try with single-filter and all then small subset
runNames = ['ops2_1092', 'ops2_1092', 'ops2_1092', 'enigma_1189', 'enigma_1189', 'enigma_1189']
LFilters = ["u", "y", '', "u", "y", '']
LNightMax = [1e4, 1e4, 730, 1e4, 1e4, 730]
# WIC 2015-12-28 - 23:00 - try using a different SED template,
# just go with single filters for now
#
# DO WE NEED THIS??
# WIC 2015-12-28 - 22:00; much to my surprise, that took less than
# half an hour to go all the way through. Try again, this time using slightly more filters.
runNames = ['ops2_1092', 'enigma_1189', 'ops2_1092', 'enigma_1189']
LFilters = ['', '', 'griz', 'griz'] # (griz was not recognized)
# WIC 2015-12-29 - set up for a master-run with all cases, this time with plotting limits
# Break the specifications across lines to make subdivision easier
# Subsets by time first, then by filter, finally the whole shebang
# (Yes the inversion of the first two is deliberate.)
runNames = ['enigma_1189', 'ops2_1092', 'ops2_1092', 'enigma_1189', \
'ops2_1092', 'enigma_1189', 'ops2_1092', 'enigma_1189', \
'ops2_1092', 'enigma_1189']
LFilters = ['', '', '', '', \
'u', 'u', 'y', 'y', \
'', '']
LNightMax = [365, 365, 730, 730, \
1e4, 1e4, 1e4, 1e4, \
1e4, 1e4]
# List of upper limits to parallax and proper motion error. For parallax, 3.0 mas is probably good
LUpperParallax = []
LUpperPropmotion = []
if CustomPlotLimits:
LUpperParallax = [10, 10, 10, 10, \
10, 10, 40, 40, \
3.0, 3.0 ]
# For proper motion, it's a little tricky to say because the
# regular case is so pathological for the field. Try the following:
LUpperPropmotion = [40, 40, 5, 20, \
3.5, 20, 3.5, 20, \
0.5, 5]
print "runAstrom.go INFO - will run the following:"
for iSho in range(len(runNames)):
print "%i: %-12s, %1s, %i" % (iSho, runNames[iSho], LFilters[iSho], LNightMax[iSho])
print "==========================="
# print runNames
# if not DoRun:
# print "Set DoRun=True to actually run this."
# print len(LFilters), len(runNames), len(LFilters) == len(runNames)
# return
#'kraken_1038', 'kraken_1034', 'ops2_1098']
# nside = 64
slicer = slicers.HealpixSlicer(nside=nside)
# Make it so we don't bother with the silly power spectra
plotFuncs = [plots.HealpixSkyMap(), plots.HealpixHistogram()]
# WIC - back up the plotting arguments with a default value
plotFuncsPristine = copy.deepcopy(plotFuncs)
# WIC - the only way this will make sense to me is if I make a
# dictionary of plot arguments. Let's try it...
DPlotArgs = {}
for plotArg in ['parallax', 'propmotion', 'coverage', 'HAdegen']:
DPlotArgs[plotArg] = copy.deepcopy(plotFuncs)
if CustomPlotLimits:
# All spatial maps use percentile clipping
for plotMetric in DPlotArgs.keys():
DPlotArgs[plotMetric][0].defaultPlotDict['percentileClip'] = SpatialClip
# Some limits common to spatial maps and histograms
for iPl in range(0,2):
DPlotArgs['propmotion'][iPl].defaultPlotDict['logScale'] = True
# Standardized range for the histograms for new parallax metrics
DPlotArgs['coverage'][1].defaultPlotDict['xMin'] = 0.
DPlotArgs['coverage'][1].defaultPlotDict['xMax'] = 1.
DPlotArgs['HAdegen'][1].defaultPlotDict['xMin'] = -1.
DPlotArgs['HAdegen'][1].defaultPlotDict['xMax'] = 1.
# Standardize at least the lower bound of the histogram in
# both the proper motion and parallax errors. Upper limit we
# can customize with a loop.
DPlotArgs['propmotion'][1].defaultPlotDict['xMin'] = 1e-2 # should not be zero if log scale!!
DPlotArgs['parallax'][1].defaultPlotDict['xMin'] = 0.
# WIC - try changing the plot dictionary
if not DoRun:
plotFuncs[0].defaultPlotDict['logScale'] = True
print DPlotArgs['propmotion'][0].defaultPlotDict
print DPlotArgs['propmotion'][1].defaultPlotDict
return
# The old runs have the seeing in finSeeing
seeingCol = 'finSeeing'
# Try it out for a 20th mag star with a flat SED (can change mag
# or to OBAFGKM)
# rmag = 20. ## NOW AN ARGUMENT
#SedTemplate='flat'
# Use all the observations. Can change if you want a different
# time span
sqlconstraint = ''
# list of sqlconstraints now used, which gets handled within the loop.
# run some summary stats on everything
summaryMetrics = [metrics.MedianMetric()]
tStart = time.time()
# Running one, or the whole lot?
RunMax = len(runNames)
# allow user to set a different number (say, 2)
if MaxRuns < RunMax and MaxRuns > 0:
RunMax = int(MaxRuns)
# the following keyword overrides
if RunOne:
RunMax = 1
print "Starting runs. RunMax = %i" % (RunMax)
for iRun in range(RunMax):
run = runNames[iRun][:]
# for run in runNames:
# Open the OpSim database
timeStartIteration = time.time()
# Some syntax added to test for existence of the database
dbFil = run+'_sqlite.db'
if not os.access(dbFil, os.R_OK):
print "runAstrom.go FATAL - cannot acces db file %s" % (dbFil)
print "runAstrom.go FATAL - skipping run %s" % (run)
continue
else:
deltaT = time.time()-tStart
print "runAstrom.go INFO - ##################################"
print "runAstrom.go INFO - starting run %s with nside=%i after %.2f minutes" \
% (run, nside, deltaT/60.)
opsdb = db.OpsimDatabase(run+'_sqlite.db')
# Set SQL constraint appropriate for each filter in the
# list. If we supplied a list of filters, use it for
sqlconstraint = ''
ThisFilter = 'ugrizy'
if len(LFilters) == len(runNames):
# Only change the filter if one was actually supplied!
if len(LFilters[iRun]) == 1:
ThisFilter = LFilters[iRun]
sqlconstraint = 'filter = "%s"' % (ThisFilter)
# If nightmax was supplied, use it
ThisNightMax = int(nightMax) # copy not view
if len(LNightMax) == len(runNames):
# Only update nightmax if one was given
try:
ThisNightMax = int(LNightMax[iRun]) # This might be redundant with the fmt statement below.
if len(sqlconstraint) < 1:
sqlconstraint = 'night < %i' % (ThisNightMax)
else:
sqlconstraint = '%s and night < %i' % (sqlconstraint, ThisNightMax)
except:
print "runAstrom.go WARN - run %i problem with NightMax" % (iRun)
dumdum = 1.
# Set where the output should go - include the filter!!
sMag = '%.1f' % (rmag)
sMag = sMag.replace(".","p")
outDir = '%s_nside%i_%s_n%i_r%s' % (run, nside, ThisFilter, ThisNightMax, sMag)
# Ensure we'll be able to find this later on...
if CustomPlotLimits:
outDir = '%s_lims' % (outDir)
# From this point onwards, stuff actually gets run. This is
# the place to output what will actually happen next.
print "runAstrom.go INFO - about to run:"
print "runAstrom.go INFO - sqlconstraint: %s ; run name %s ; nside %i" % (sqlconstraint, run, nside)
print "runAstrom.go INFO - output directory will be %s" % (outDir)
if not DoRun:
continue
resultsDb = db.ResultsDb(outDir=outDir)
bundleList = []
# WIC - to make this at least somewhat uniform, build the plot
# functions including arguments out of our copies above.
plotFuncsPropmotion = copy.deepcopy(DPlotArgs['propmotion'])
plotFuncsParallax = copy.deepcopy(DPlotArgs['parallax'])
plotFuncsCoverage = copy.deepcopy(DPlotArgs['coverage'])
plotFuncsHAdegen = copy.deepcopy(DPlotArgs['HAdegen'])
# if using custom plot limits, will want to include the limits
# for proper motion and parallax too... programming a bit defensively
# here, including an extra check (rather than just the length of the lists
# above).
if CustomPlotLimits:
if len(LUpperParallax) == len(runNames):
plotFuncsParallax[1].defaultPlotDict['xMax'] = float(LUpperParallax[iRun])
if len(LUpperPropmotion) == len(runNames):
plotFuncsPropmotion[1].defaultPlotDict['xMax'] = float(LUpperPropmotion[iRun])
# Configure the metrics
metric = metrics.ParallaxMetric(rmag=rmag, seeingCol=seeingCol, SedTemplate=SedTemplate)
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, runName=run,
# plotFuncs=plotFuncs, \
plotFuncs = plotFuncsParallax, \
summaryMetrics=summaryMetrics)
bundleList.append(bundle)
metric=metrics.ProperMotionMetric(rmag=rmag, seeingCol=seeingCol, SedTemplate=SedTemplate)
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, runName=run,
# plotFuncs=plotFuncs, \
plotFuncs=plotFuncsPropmotion, \
summaryMetrics=summaryMetrics)
bundleList.append(bundle)
metric = metrics.ParallaxCoverageMetric(rmag=rmag, seeingCol=seeingCol, SedTemplate=SedTemplate)
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, runName=run,
# plotFuncs=plotFuncs, \
plotFuncs=plotFuncsCoverage, \
summaryMetrics=summaryMetrics)
bundleList.append(bundle)
metric = metrics.ParallaxHADegenMetric(rmag=rmag, seeingCol=seeingCol, SedTemplate=SedTemplate)
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, runName=run,
# plotFuncs=plotFuncs, \
plotFuncs=plotFuncsHAdegen, \
summaryMetrics=summaryMetrics)
bundleList.append(bundle)
# Run everything and make plots
bundleDict = metricBundles.makeBundlesDictFromList(bundleList)
bgroup = metricBundles.MetricBundleGroup(bundleDict, opsdb, outDir=outDir, resultsDb=resultsDb)
# try:
bgroup.runAll()
print "runAstrom.go INFO - bundles took %.2f minutes" \
% ((time.time() - timeStartIteration) / 60.)
# except KeyboardInterrupt:
# print "runAstrom.go FATAL - keyboard interrupt detected. Halting."
# return
bgroup.plotAll()
print "runAstrom.go INFO - bundles + plotting took %.2f minutes" \
% ((time.time() - timeStartIteration) / 60.)
print "Finished entire set. %i runs took %.2f minutes." % (iRun + 1, (time.time()-tStart)/60.)
def intraNight(colmap=None,
runName='opsim',
nside=64,
extraSql=None,
extraMetadata=None):
"""Generate a set of statistics about the pair/triplet/etc. rate within a night.
Parameters
----------
colmap : dict or None, opt
A dictionary with a mapping of column names. Default will use OpsimV4 column names.
runName : str, opt
The name of the simulated survey. Default is "opsim".
nside : int, opt
Nside for the healpix slicer. Default 64.
extraSql : str or None, opt
Additional sql constraint to apply to all metrics.
extraMetadata : str or None, opt
Additional metadata to apply to all results.
Returns
-------
metricBundleDict
"""
if colmap is None:
colmap = ColMapDict('opsimV4')
metadata = extraMetadata
if extraSql is None:
extraSql = ''
if extraSql is not None and len(extraSql) > 0:
if metadata is None:
metadata = extraSql
bundleList = []
standardStats = standardSummary()
subsetPlots = [plots.HealpixSkyMap(), plots.HealpixHistogram()]
# Look for the fraction of visits in gri where there are pairs within dtMin/dtMax.
displayDict = {
'group': 'IntraNight',
'subgroup': 'Pairs',
'caption': None,
'order': 0
}
sql = '(%s) and (filter="g" or filter="r" or filter="i")' % extraSql
md = 'gri'
if metadata is not None:
md += metadata
dtMin = 15.0
dtMax = 60.0
metric = metrics.PairFractionMetric(
mjdCol=colmap['mjd'],
minGap=dtMin,
maxGap=dtMax,
metricName='Fraction of visits in pairs (%.0f-%.0f min)' %
(dtMin, dtMax))
slicer = slicers.HealpixSlicer(nside=nside,
latCol=colmap['dec'],
lonCol=colmap['ra'],
latLonDeg=colmap['raDecDeg'])
displayDict['caption'] = 'Fraction of %s visits that have a paired visit' \
'between %.1f and %.1f minutes away. ' % (metadata, dtMin, dtMax)
displayDict[
'caption'] += 'If all visits were in pairs, this fraction would be 1.'
displayDict['order'] += 1
bundle = mb.MetricBundle(metric,
slicer,
sql,
metadata=md,
summaryMetrics=standardStats,
plotFuncs=subsetPlots,
displayDict=displayDict)
bundleList.append(bundle)
# Look at the fraction of visits which have another visit within dtMax, gri.
dtMax = 50.0
metric = metrics.NRevisitsMetric(
mjdCol=colmap['mjd'],
dT=dtMax,
normed=True,
metricName='Fraction of visits with a revisit < %.0f min' % dtMax)
displayDict['caption'] = 'Fraction of %s visits that have another visit ' \
'within %.1f min. ' % (metadata, dtMax)
displayDict[
'caption'] += 'If all visits were in pairs (only), this fraction would be 0.5.'
displayDict['order'] += 1
bundle = mb.MetricBundle(metric,
slicer,
sql,
metadata=md,
summaryMetrics=standardStats,
plotFuncs=subsetPlots,
displayDict=displayDict)
bundleList.append(bundle)
# Histogram of the time between revisits (all filters) within two hours.
binMin = 0
binMax = 120.
binsize = 5.
bins_metric = np.arange(binMin / 60.0 / 24.0,
(binMax + binsize) / 60. / 24.,
binsize / 60. / 24.)
bins_plot = bins_metric * 24.0 * 60.0
metric = metrics.TgapsMetric(bins=bins_metric,
timesCol=colmap['mjd'],
metricName='DeltaT Histogram')
slicer = slicers.HealpixSlicer(nside=nside,
latCol=colmap['dec'],
lonCol=colmap['ra'],
latLonDeg=colmap['raDecDeg'])
plotDict = {'bins': bins_plot, 'xlabel': 'dT (minutes)'}
displayDict['caption'] = 'Histogram of the time between consecutive visits to a given point ' \
'on the sky, considering visits between %.1f and %.1f minutes' % (binMin,
binMax)
displayDict['order'] += 1
plotFunc = plots.SummaryHistogram()
bundle = mb.MetricBundle(metric,
slicer,
extraSql,
plotDict=plotDict,
displayDict=displayDict,
metadata=metadata,
plotFuncs=[plotFunc])
bundleList.append(bundle)
# Set the runName for all bundles and return the bundleDict.
for b in bundleList:
b.setRunName(runName)
return mb.makeBundlesDictFromList(bundleList)
def interNight(colmap=None,
runName='opsim',
nside=64,
extraSql=None,
extraMetadata=None):
"""Generate a set of statistics about the spacing between nights with observations.
Parameters
----------
colmap : dict or None, opt
A dictionary with a mapping of column names. Default will use OpsimV4 column names.
runName : str, opt
The name of the simulated survey. Default is "opsim".
nside : int, opt
Nside for the healpix slicer. Default 64.
extraSql : str or None, opt
Additional sql constraint to apply to all metrics.
extraMetadata : str or None, opt
Additional metadata to use for all outputs.
Returns
-------
metricBundleDict
"""
if colmap is None:
colmap = ColMapDict('opsimV4')
bundleList = []
metadata = extraMetadata
if extraSql is None:
extraSql = ''
if extraSql is not None and len(extraSql) > 0:
if metadata is None:
metadata = extraSql
filterlist = ('u', 'g', 'r', 'i', 'z', 'y', 'all')
colors = {
'u': 'cyan',
'g': 'g',
'r': 'y',
'i': 'r',
'z': 'm',
'y': 'b',
'all': 'k'
}
filterorder = {'u': 1, 'g': 2, 'r': 3, 'i': 4, 'z': 5, 'y': 6, 'all': 0}
displayDict = {
'group': 'InterNight',
'subgroup': 'Night gaps',
'caption': None,
'order': 0
}
bins = np.arange(1, 20.5, 1)
metric = metrics.NightgapsMetric(bins=bins,
nightCol=colmap['night'],
metricName='DeltaNight Histogram')
slicer = slicers.HealpixSlicer(nside=nside,
latCol=colmap['dec'],
lonCol=colmap['ra'],
latLonDeg=colmap['raDecDeg'])
plotDict = {'bins': bins, 'xlabel': 'dT (nights)'}
displayDict['caption'] = 'Histogram of the number of nights between consecutive visits to a ' \
'given point on the sky, considering separations between %d and %d.' \
% (bins.min(), bins.max())
plotFunc = plots.SummaryHistogram()
bundle = mb.MetricBundle(metric,
slicer,
extraSql,
plotDict=plotDict,
displayDict=displayDict,
metadata=metadata,
plotFuncs=[plotFunc])
bundleList.append(bundle)
standardStats = standardSummary()
subsetPlots = [plots.HealpixSkyMap(), plots.HealpixHistogram()]
# Median inter-night gap (each and all filters)
metric = metrics.InterNightGapsMetric(metricName='Median Inter-Night Gap',
reduceFunc=np.median)
slicer = slicers.HealpixSlicer(nside=nside,
latCol=colmap['dec'],
lonCol=colmap['ra'],
latLonDeg=colmap['raDecDeg'])
for f in filterlist:
if f is not 'all':
sql = '(%s) and filter = "%s"' % (extraSql, f)
md = '%s band' % f
else:
sql = extraSql
md = 'all bands'
if metadata is not None:
md += metadata
displayDict[
'caption'] = 'Median gap between nights with observations, %s.' % md
displayDict['order'] = filterorder[f]
plotDict = {'color': colors[f]}
bundle = mb.MetricBundle(metric,
slicer,
sql,
metadata=md,
displayDict=displayDict,
plotFuncs=subsetPlots,
plotDict=plotDict,
summaryMetrics=standardStats)
bundleList.append(bundle)
# Maximum inter-night gap (in each and all filters).
metric = metrics.InterNightGapsMetric(metricName='Max Inter-Night Gap',
reduceFunc=np.max)
slicer = slicers.HealpixSlicer(nside=nside,
latCol=colmap['dec'],
lonCol=colmap['ra'],
latLonDeg=colmap['raDecDeg'])
for f in filterlist:
if f is not 'all':
sql = '(%s) and filter = "%s"' % (extraSql, f)
md = '%s band' % f
else:
sql = extraSql
md = 'all bands'
if metadata is not None:
md += metadata
displayDict[
'caption'] = 'Maximum gap between nights with observations, %s.' % md
displayDict['order'] = filterorder[f] - 10
plotDict = {'color': colors[f], 'percentileClip': 95., 'binsize': 5}
bundle = mb.MetricBundle(metric,
slicer,
sql,
metadata=md,
displayDict=displayDict,
plotFuncs=subsetPlots,
plotDict=plotDict,
summaryMetrics=standardStats)
bundleList.append(bundle)
# Set the runName for all bundles and return the bundleDict.
for b in bundleList:
b.setRunName(runName)
return mb.makeBundlesDictFromList(bundleList)
def makeBundleList(dbFile, runName=None, nside=64, benchmark='design',
lonCol='fieldRA', latCol='fieldDec', seeingCol='seeingFwhmGeom'):
"""
make a list of metricBundle objects to look at the scientific performance
of an opsim run.
"""
# List to hold everything we're going to make
bundleList = []
# List to hold metrics that shouldn't be saved
noSaveBundleList = []
# Connect to the databse
opsimdb = db.OpsimDatabaseV4(dbFile)
if runName is None:
runName = os.path.basename(dbFile).replace('_sqlite.db', '')
# Fetch the proposal ID values from the database
propids, propTags = opsimdb.fetchPropInfo()
# Fetch the telescope location from config
lat, lon, height = opsimdb.fetchLatLonHeight()
# Add metadata regarding dithering/non-dithered.
commonname = ''.join([a for a in lonCol if a in latCol])
if commonname == 'field':
slicermetadata = ' (non-dithered)'
else:
slicermetadata = ' (%s)' % (commonname)
# Construct a WFD SQL where clause so multiple propIDs can query by WFD:
wfdWhere = opsimdb.createSQLWhere('WFD', propTags)
print('#FYI: WFD "where" clause: %s' % (wfdWhere))
ddWhere = opsimdb.createSQLWhere('DD', propTags)
print('#FYI: DD "where" clause: %s' % (ddWhere))
# Set up benchmark values, scaled to length of opsim run.
runLength = opsimdb.fetchRunLength()
if benchmark == 'requested':
# Fetch design values for seeing/skybrightness/single visit depth.
benchmarkVals = utils.scaleBenchmarks(runLength, benchmark='design')
# Update nvisits with requested visits from config files.
benchmarkVals['nvisits'] = opsimdb.fetchRequestedNvisits(propId=propTags['WFD'])
# Calculate expected coadded depth.
benchmarkVals['coaddedDepth'] = utils.calcCoaddedDepth(benchmarkVals['nvisits'],
benchmarkVals['singleVisitDepth'])
elif (benchmark == 'stretch') or (benchmark == 'design'):
# Calculate benchmarks for stretch or design.
benchmarkVals = utils.scaleBenchmarks(runLength, benchmark=benchmark)
benchmarkVals['coaddedDepth'] = utils.calcCoaddedDepth(benchmarkVals['nvisits'],
benchmarkVals['singleVisitDepth'])
else:
raise ValueError('Could not recognize benchmark value %s, use design, stretch or requested.'
% (benchmark))
# Check that nvisits is not set to zero (for very short run length).
for f in benchmarkVals['nvisits']:
if benchmarkVals['nvisits'][f] == 0:
print('Updating benchmark nvisits value in %s to be nonzero' % (f))
benchmarkVals['nvisits'][f] = 1
# Set values for min/max range of nvisits for All/WFD and DD plots. These are somewhat arbitrary.
nvisitsRange = {}
nvisitsRange['all'] = {'u': [20, 80], 'g': [50, 150], 'r': [100, 250],
'i': [100, 250], 'z': [100, 300], 'y': [100, 300]}
nvisitsRange['DD'] = {'u': [6000, 10000], 'g': [2500, 5000], 'r': [5000, 8000],
'i': [5000, 8000], 'z': [7000, 10000], 'y': [5000, 8000]}
# Scale these ranges for the runLength.
scale = runLength / 10.0
for prop in nvisitsRange:
for f in nvisitsRange[prop]:
for i in [0, 1]:
nvisitsRange[prop][f][i] = int(np.floor(nvisitsRange[prop][f][i] * scale))
# Filter list, and map of colors (for plots) to filters.
filters = ['u', 'g', 'r', 'i', 'z', 'y']
colors = {'u': 'cyan', 'g': 'g', 'r': 'y', 'i': 'r', 'z': 'm', 'y': 'k'}
filtorder = {'u': 1, 'g': 2, 'r': 3, 'i': 4, 'z': 5, 'y': 6}
# Easy way to run through all fi
# Set up a list of common summary stats
commonSummary = [metrics.MeanMetric(), metrics.RobustRmsMetric(), metrics.MedianMetric(),
metrics.PercentileMetric(metricName='25th%ile', percentile=25),
metrics.PercentileMetric(metricName='75th%ile', percentile=75),
metrics.MinMetric(), metrics.MaxMetric()]
allStats = commonSummary
# Set up some 'group' labels
reqgroup = 'A: Required SRD metrics'
depthgroup = 'B: Depth per filter'
uniformitygroup = 'C: Uniformity'
airmassgroup = 'D: Airmass distribution'
seeinggroup = 'E: Seeing distribution'
transgroup = 'F: Transients'
sngroup = 'G: SN Ia'
altAzGroup = 'H: Alt Az'
rangeGroup = 'I: Range of Dates'
intergroup = 'J: Inter-Night'
phaseGroup = 'K: Max Phase Gap'
NEOGroup = 'L: NEO Detection'
# Set up an object to track the metricBundles that we want to combine into merged plots.
mergedHistDict = {}
# Set the histogram merge function.
mergeFunc = plots.HealpixHistogram()
keys = ['NVisits', 'coaddm5', 'NormEffTime', 'Minseeing', 'seeingAboveLimit', 'minAirmass',
'fracAboveAirmass']
for key in keys:
mergedHistDict[key] = plots.PlotBundle(plotFunc=mergeFunc)
##
# Calculate the fO metrics for all proposals and WFD only.
order = 0
for prop in ('All prop', 'WFD only'):
if prop == 'All prop':
metadata = 'All Visits' + slicermetadata
sqlconstraint = ''
if prop == 'WFD only':
metadata = 'WFD only' + slicermetadata
sqlconstraint = '%s' % (wfdWhere)
# Configure the count metric which is what is used for f0 slicer.
m1 = metrics.CountMetric(col='observationStartMJD', metricName='fO')
plotDict = {'xlabel': 'Number of Visits', 'Asky': benchmarkVals['Area'],
'Nvisit': benchmarkVals['nvisitsTotal'], 'xMin': 0, 'xMax': 1500}
summaryMetrics = [metrics.fOArea(nside=nside, norm=False, metricName='fOArea: Nvisits (#)',
Asky=benchmarkVals['Area'], Nvisit=benchmarkVals['nvisitsTotal']),
metrics.fOArea(nside=nside, norm=True, metricName='fOArea: Nvisits/benchmark',
Asky=benchmarkVals['Area'], Nvisit=benchmarkVals['nvisitsTotal']),
metrics.fONv(nside=nside, norm=False, metricName='fONv: Area (sqdeg)',
Asky=benchmarkVals['Area'], Nvisit=benchmarkVals['nvisitsTotal']),
metrics.fONv(nside=nside, norm=True, metricName='fONv: Area/benchmark',
Asky=benchmarkVals['Area'], Nvisit=benchmarkVals['nvisitsTotal'])]
caption = 'The FO metric evaluates the overall efficiency of observing. '
caption += ('fOArea: Nvisits = %.1f sq degrees receive at least this many visits out of %d. '
% (benchmarkVals['Area'], benchmarkVals['nvisitsTotal']))
caption += ('fONv: Area = this many square degrees out of %.1f receive at least %d visits.'
% (benchmarkVals['Area'], benchmarkVals['nvisitsTotal']))
displayDict = {'group': reqgroup, 'subgroup': 'F0', 'displayOrder': order, 'caption': caption}
order += 1
slicer = slicers.HealpixSlicer(nside=nside, lonCol=lonCol, latCol=latCol)
bundle = metricBundles.MetricBundle(m1, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, summaryMetrics=summaryMetrics,
plotFuncs=[plots.FOPlot()],
runName=runName, metadata=metadata)
bundleList.append(bundle)
###
# Calculate the Rapid Revisit Metrics.
order = 0
metadata = 'All Visits' + slicermetadata
sqlconstraint = ''
dTmin = 40.0 # seconds
dTmax = 30.0*60. # seconds
minNvisit = 100
pixArea = float(hp.nside2pixarea(nside, degrees=True))
scale = pixArea * hp.nside2npix(nside)
cutoff1 = 0.15
extraStats1 = [metrics.FracBelowMetric(cutoff=cutoff1, scale=scale, metricName='Area (sq deg)')]
extraStats1.extend(commonSummary)
slicer = slicers.HealpixSlicer(nside=nside, lonCol=lonCol, latCol=latCol)
m1 = metrics.RapidRevisitMetric(metricName='RapidRevisitUniformity',
dTmin=dTmin / 60.0 / 60.0 / 24.0, dTmax=dTmax / 60.0 / 60.0 / 24.0,
minNvisits=minNvisit)
plotDict = {'xMin': 0, 'xMax': 1}
summaryStats = extraStats1
caption = 'Deviation from uniformity for short revisit timescales, between %s and %s seconds, ' % (
dTmin, dTmax)
caption += 'for pointings with at least %d visits in this time range. ' % (minNvisit)
caption += 'Summary statistic "Area" below indicates the area on the sky which has a '
caption += 'deviation from uniformity of < %.2f.' % (cutoff1)
displayDict = {'group': reqgroup, 'subgroup': 'Rapid Revisit', 'displayOrder': order,
'caption': caption}
bundle = metricBundles.MetricBundle(m1, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, summaryMetrics=summaryStats,
runName=runName, metadata=metadata)
bundleList.append(bundle)
order += 1
dTmax = dTmax/60.0 # need time in minutes for Nrevisits metric
m2 = metrics.NRevisitsMetric(dT=dTmax)
plotDict = {'xMin': 0.1, 'xMax': 2000, 'logScale': True}
cutoff2 = 800
extraStats2 = [metrics.FracAboveMetric(cutoff=cutoff2, scale=scale, metricName='Area (sq deg)')]
extraStats2.extend(commonSummary)
caption = 'Number of consecutive visits with return times faster than %.1f minutes, ' % (dTmax)
caption += 'in any filter, all proposals. '
caption += 'Summary statistic "Area" below indicates the area on the sky which has more than '
caption += '%d revisits within this time window.' % (cutoff2)
summaryStats = extraStats2
displayDict = {'group': reqgroup, 'subgroup': 'Rapid Revisit', 'displayOrder': order,
'caption': caption}
bundle = metricBundles.MetricBundle(m2, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, summaryMetrics=summaryStats,
runName=runName, metadata=metadata)
bundleList.append(bundle)
order += 1
m3 = metrics.NRevisitsMetric(dT=dTmax, normed=True)
plotDict = {'xMin': 0, 'xMax': 1, 'cbarFormat': '%.1f'}
cutoff3 = 0.6
extraStats3 = [metrics.FracAboveMetric(cutoff=cutoff3, scale=scale, metricName='Area (sq deg)')]
extraStats3.extend(commonSummary)
summaryStats = extraStats3
caption = 'Fraction of total visits where consecutive visits have return times faster '
caption += 'than %.1f minutes, in any filter, all proposals. ' % (dTmax)
caption += 'Summary statistic "Area" below indicates the area on the sky which has more '
caption += 'than %d revisits within this time window.' % (cutoff3)
displayDict = {'group': reqgroup, 'subgroup': 'Rapid Revisit', 'displayOrder': order,
'caption': caption}
bundle = metricBundles.MetricBundle(m3, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, summaryMetrics=summaryStats,
runName=runName, metadata=metadata)
bundleList.append(bundle)
order += 1
# And add a histogram of the time between quick revisits.
binMin = 0
binMax = 120.
binsize = 3.
bins_metric = np.arange(binMin / 60.0 / 24.0, (binMax + binsize) / 60. / 24., binsize / 60. / 24.)
bins_plot = bins_metric * 24.0 * 60.0
m1 = metrics.TgapsMetric(bins=bins_metric, metricName='dT visits')
plotDict = {'bins': bins_plot, 'xlabel': 'dT (minutes)'}
caption = ('Histogram of the time between consecutive revisits (<%.1f minutes), over entire sky.'
% (binMax))
displayDict = {'group': reqgroup, 'subgroup': 'Rapid Revisit', 'order': order,
'caption': caption}
slicer = slicers.HealpixSlicer(nside=nside, lonCol=lonCol, latCol=latCol)
plotFunc = plots.SummaryHistogram()
bundle = metricBundles.MetricBundle(m1, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, runName=runName,
metadata=metadata, plotFuncs=[plotFunc])
bundleList.append(bundle)
order += 1
##
# Trigonometric parallax and proper motion @ r=20 and r=24
slicer = slicers.HealpixSlicer(nside=nside, lonCol=lonCol, latCol=latCol)
sqlconstraint = ''
order = 0
metric = metrics.ParallaxMetric(metricName='Parallax 20', rmag=20, seeingCol=seeingCol)
summaryStats = allStats
plotDict = {'cbarFormat': '%.1f', 'xMin': 0, 'xMax': 3}
displayDict = {'group': reqgroup, 'subgroup': 'Parallax', 'order': order,
'caption': 'Parallax precision at r=20. (without refraction).'}
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, summaryMetrics=summaryStats,
runName=runName, metadata=metadata)
bundleList.append(bundle)
order += 1
metric = metrics.ParallaxMetric(metricName='Parallax 24', rmag=24, seeingCol=seeingCol)
plotDict = {'cbarFormat': '%.1f', 'xMin': 0, 'xMax': 10}
displayDict = {'group': reqgroup, 'subgroup': 'Parallax', 'order': order,
'caption': 'Parallax precision at r=24. (without refraction).'}
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, summaryMetrics=summaryStats,
runName=runName, metadata=metadata)
bundleList.append(bundle)
order += 1
metric = metrics.ParallaxMetric(metricName='Parallax Normed', rmag=24, normalize=True,
seeingCol=seeingCol)
plotDict = {'xMin': 0.5, 'xMax': 1.0}
displayDict = {'group': reqgroup, 'subgroup': 'Parallax', 'order': order,
'caption':
'Normalized parallax (normalized to optimum observation cadence, 1=optimal).'}
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, summaryMetrics=summaryStats,
runName=runName, metadata=metadata)
bundleList.append(bundle)
order += 1
metric = metrics.ParallaxCoverageMetric(metricName='Parallax Coverage 20', rmag=20, seeingCol=seeingCol)
plotDict = {}
caption = "Parallax factor coverage for an r=20 star (0 is bad, 0.5-1 is good). "
caption += "One expects the parallax factor coverage to vary because stars on the ecliptic "
caption += "can be observed when they have no parallax offset while stars at the pole are always "
caption += "offset by the full parallax offset."""
displayDict = {'group': reqgroup, 'subgroup': 'Parallax', 'order': order,
'caption': caption}
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, summaryMetrics=summaryStats,
runName=runName, metadata=metadata)
bundleList.append(bundle)
order += 1
metric = metrics.ParallaxCoverageMetric(metricName='Parallax Coverage 24', rmag=24, seeingCol=seeingCol)
plotDict = {}
caption = "Parallax factor coverage for an r=24 star (0 is bad, 0.5-1 is good). "
caption += "One expects the parallax factor coverage to vary because stars on the ecliptic "
caption += "can be observed when they have no parallax offset while stars at the pole are always "
caption += "offset by the full parallax offset."""
displayDict = {'group': reqgroup, 'subgroup': 'Parallax', 'order': order,
'caption': caption}
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, summaryMetrics=summaryStats,
runName=runName, metadata=metadata)
bundleList.append(bundle)
order += 1
metric = metrics.ParallaxDcrDegenMetric(metricName='Parallax-DCR degeneracy 20', rmag=20,
seeingCol=seeingCol)
plotDict = {}
caption = 'Correlation between parallax offset magnitude and hour angle an r=20 star.'
caption += ' (0 is good, near -1 or 1 is bad).'
displayDict = {'group': reqgroup, 'subgroup': 'Parallax', 'order': order,
'caption': caption}
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, summaryMetrics=summaryStats,
runName=runName, metadata=metadata)
bundleList.append(bundle)
order += 1
metric = metrics.ParallaxDcrDegenMetric(metricName='Parallax-DCR degeneracy 24', rmag=24,
seeingCol=seeingCol)
plotDict = {}
caption = 'Correlation between parallax offset magnitude and hour angle an r=24 star.'
caption += ' (0 is good, near -1 or 1 is bad).'
displayDict = {'group': reqgroup, 'subgroup': 'Parallax', 'order': order,
'caption': caption}
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, summaryMetrics=summaryStats,
runName=runName, metadata=metadata)
bundleList.append(bundle)
order += 1
metric = metrics.ProperMotionMetric(metricName='Proper Motion 20', rmag=20, seeingCol=seeingCol)
summaryStats = allStats
plotDict = {'xMin': 0, 'xMax': 3}
displayDict = {'group': reqgroup, 'subgroup': 'Proper Motion', 'order': order,
'caption': 'Proper Motion precision at r=20.'}
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, summaryMetrics=summaryStats,
runName=runName, metadata=metadata)
bundleList.append(bundle)
order += 1
metric = metrics.ProperMotionMetric(rmag=24, metricName='Proper Motion 24', seeingCol=seeingCol)
summaryStats = allStats
plotDict = {'xMin': 0, 'xMax': 10}
displayDict = {'group': reqgroup, 'subgroup': 'Proper Motion', 'order': order,
'caption': 'Proper Motion precision at r=24.'}
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, summaryMetrics=summaryStats,
runName=runName, metadata=metadata)
bundleList.append(bundle)
order += 1
metric = metrics.ProperMotionMetric(rmag=24, normalize=True, metricName='Proper Motion Normed',
seeingCol=seeingCol)
plotDict = {'xMin': 0.2, 'xMax': 0.7}
caption = 'Normalized proper motion at r=24. '
caption += '(normalized to optimum observation cadence - start/end. 1=optimal).'
displayDict = {'group': reqgroup, 'subgroup': 'Proper Motion', 'order': order,
'caption': caption}
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, summaryMetrics=summaryStats,
runName=runName, metadata=metadata)
bundleList.append(bundle)
order += 1
##
# Calculate the time uniformity in each filter, for each year.
order = 0
slicer = slicers.HealpixSlicer(nside=nside, lonCol=lonCol, latCol=latCol)
plotFuncs = [plots.TwoDMap()]
step = 0.5
bins = np.arange(0, 365.25 * 10 + 40, 40) - step
metric = metrics.AccumulateUniformityMetric(bins=bins)
plotDict = {'xlabel': 'Night (days)', 'xextent': [bins.min(
) + step, bins.max() + step], 'cbarTitle': 'Uniformity'}
for f in filters:
sqlconstraint = 'filter = "%s"' % (f)
caption = 'Deviation from uniformity in %s band. ' % f
caption += 'Northern Healpixels are at the top of the image.'
caption += '(0=perfectly uniform, 1=perfectly nonuniform).'
displayDict = {'group': uniformitygroup, 'subgroup': 'per night',
'order': filtorder[f], 'caption': caption}
metadata = '%s band' % (f) + slicermetadata
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, runName=runName, metadata=metadata,
plotFuncs=plotFuncs)
noSaveBundleList.append(bundle)
##
# Depth metrics.
slicer = slicers.HealpixSlicer(nside=nside, lonCol=lonCol, latCol=latCol)
for f in filters:
propCaption = '%s band, all proposals %s' % (f, slicermetadata)
sqlconstraint = 'filter = "%s"' % (f)
metadata = '%s band' % (f) + slicermetadata
# Number of visits.
metric = metrics.CountMetric(col='observationStartMJD', metricName='NVisits')
plotDict = {'xlabel': 'Number of visits',
'xMin': nvisitsRange['all'][f][0],
'xMax': nvisitsRange['all'][f][1],
'colorMin': nvisitsRange['all'][f][0],
'colorMax': nvisitsRange['all'][f][1],
'binsize': 5,
'logScale': True, 'nTicks': 4, 'colorMin': 1}
summaryStats = allStats
displayDict = {'group': depthgroup, 'subgroup': 'Nvisits', 'order': filtorder[f],
'caption': 'Number of visits in filter %s, %s.' % (f, propCaption)}
histMerge = {'color': colors[f], 'label': '%s' % (f),
'binsize': 5,
'xMin': nvisitsRange['all'][f][0], 'xMax': nvisitsRange['all'][f][1],
'legendloc': 'upper right'}
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, runName=runName, metadata=metadata,
summaryMetrics=summaryStats)
mergedHistDict['NVisits'].addBundle(bundle, plotDict=histMerge)
bundleList.append(bundle)
# Coadded depth.
metric = metrics.Coaddm5Metric()
plotDict = {'zp': benchmarkVals['coaddedDepth'][f], 'xMin': -0.8, 'xMax': 0.8,
'xlabel': 'coadded m5 - %.1f' % benchmarkVals['coaddedDepth'][f]}
summaryStats = allStats
histMerge = {'legendloc': 'upper right', 'color': colors[f], 'label': '%s' % f, 'binsize': .02,
'xlabel': 'coadded m5 - benchmark value'}
caption = ('Coadded depth in filter %s, with %s value subtracted (%.1f), %s. '
% (f, benchmark, benchmarkVals['coaddedDepth'][f], propCaption))
caption += 'More positive numbers indicate fainter limiting magnitudes.'
displayDict = {'group': depthgroup, 'subgroup': 'Coadded Depth',
'order': filtorder[f], 'caption': caption}
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, runName=runName, metadata=metadata,
summaryMetrics=summaryStats)
mergedHistDict['coaddm5'].addBundle(bundle, plotDict=histMerge)
bundleList.append(bundle)
# Effective time.
metric = metrics.TeffMetric(metricName='Normalized Effective Time', normed=True,
fiducialDepth=benchmarkVals['singleVisitDepth'])
plotDict = {'xMin': 0.1, 'xMax': 1.1}
summaryStats = allStats
histMerge = {'legendLoc': 'upper right', 'color': colors[f], 'label': '%s' % f, 'binsize': 0.02}
caption = ('"Time Effective" in filter %s, calculated with fiducial single-visit depth of %s mag. '
% (f, benchmarkVals['singleVisitDepth'][f]))
caption += 'Normalized by the fiducial time effective, if every observation was at '
caption += 'the fiducial depth.'
displayDict = {'group': depthgroup, 'subgroup': 'Time Eff.',
'order': filtorder[f], 'caption': caption}
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, runName=runName, metadata=metadata,
summaryMetrics=summaryStats)
mergedHistDict['NormEffTime'].addBundle(bundle, plotDict=histMerge)
bundleList.append(bundle)
# Put in a z=0.5 Type Ia SN, based on Cambridge 2015 workshop notebook.
# Check for 1) detection in any band, 2) detection on the rise in any band,
# 3) good characterization
peaks = {'uPeak': 25.9, 'gPeak': 23.6, 'rPeak': 22.6, 'iPeak': 22.7, 'zPeak': 22.7, 'yPeak': 22.8}
peakTime = 15.
transDuration = peakTime + 30. # Days
metric = metrics.TransientMetric(riseSlope=-2. / peakTime, declineSlope=1.4 / 30.0,
transDuration=transDuration, peakTime=peakTime,
surveyDuration=runLength,
metricName='SNDetection', **peaks)
caption = 'Fraction of z=0.5 type Ia SN that are detected in any filter'
displayDict = {'group': transgroup, 'subgroup': 'Detected', 'caption': caption}
sqlconstraint = ''
metadata = '' + slicermetadata
plotDict = {}
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, runName=runName, metadata=metadata)
bundleList.append(bundle)
metric = metrics.TransientMetric(riseSlope=-2. / peakTime, declineSlope=1.4 / 30.0,
transDuration=transDuration, peakTime=peakTime,
surveyDuration=runLength,
nPrePeak=1, metricName='SNAlert', **peaks)
caption = 'Fraction of z=0.5 type Ia SN that are detected pre-peak in any filter'
displayDict = {'group': transgroup, 'subgroup': 'Detected on the rise', 'caption': caption}
plotDict = {}
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, runName=runName, metadata=metadata)
bundleList.append(bundle)
metric = metrics.TransientMetric(riseSlope=-2. / peakTime, declineSlope=1.4 / 30.,
transDuration=transDuration, peakTime=peakTime,
surveyDuration=runLength, metricName='SNLots',
nFilters=3, nPrePeak=3, nPerLC=2, **peaks)
caption = 'Fraction of z=0.5 type Ia SN that are observed 6 times, 3 pre-peak, '
caption += '3 post-peak, with observations in 3 filters'
displayDict = {'group': transgroup, 'subgroup': 'Well observed', 'caption': caption}
sqlconstraint = 'filter="r" or filter="g" or filter="i" or filter="z" '
plotDict = {}
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, runName=runName, metadata=metadata)
bundleList.append(bundle)
# Good seeing in r/i band metrics, including in first/second years.
order = 0
for tcolor, tlabel, timespan in zip(['k', 'g', 'r'], ['10 years', '1 year', '2 years'],
['', ' and night<=365', ' and night<=730']):
order += 1
for f in (['r', 'i']):
sqlconstraint = 'filter = "%s" %s' % (f, timespan)
propCaption = '%s band, all proposals %s, over %s.' % (f, slicermetadata, tlabel)
metadata = '%s band, %s' % (f, tlabel) + slicermetadata
seeing_limit = 0.7
airmass_limit = 1.2
metric = metrics.MinMetric(col=seeingCol)
summaryStats = allStats
plotDict = {'xMin': 0.35, 'xMax': 1.5, 'color': tcolor}
displayDict = {'group': seeinggroup, 'subgroup': 'Best Seeing',
'order': filtorder[f] * 100 + order,
'caption': 'Minimum FWHMgeom values in %s.' % (propCaption)}
histMerge = {'label': '%s %s' % (f, tlabel), 'color': tcolor,
'binsize': 0.03, 'xMin': 0.35, 'xMax': 1.5, 'legendloc': 'upper right'}
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, runName=runName, metadata=metadata,
summaryMetrics=summaryStats)
mergedHistDict['Minseeing'].addBundle(bundle, plotDict=histMerge)
bundleList.append(bundle)
metric = metrics.FracAboveMetric(col=seeingCol, cutoff=seeing_limit)
summaryStats = allStats
plotDict = {'xMin': 0, 'xMax': 1.1, 'color': tcolor}
displayDict = {'group': seeinggroup, 'subgroup': 'Good seeing fraction',
'order': filtorder[f] * 100 + order,
'caption': 'Fraction of total images with FWHMgeom worse than %.1f, in %s'
% (seeing_limit, propCaption)}
histMerge = {'color': tcolor, 'label': '%s %s' % (f, tlabel),
'binsize': 0.05, 'legendloc': 'upper right'}
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, runName=runName, metadata=metadata,
summaryMetrics=summaryStats)
mergedHistDict['seeingAboveLimit'].addBundle(bundle, plotDict=histMerge)
bundleList.append(bundle)
metric = metrics.MinMetric(col='airmass')
plotDict = {'xMin': 1, 'xMax': 1.5, 'color': tcolor}
summaryStats = allStats
displayDict = {'group': airmassgroup, 'subgroup': 'Best Airmass',
'order': filtorder[f] * 100 + order, 'caption':
'Minimum airmass in %s.' % (propCaption)}
histMerge = {'color': tcolor, 'label': '%s %s' % (f, tlabel),
'binsize': 0.03, 'legendloc': 'upper right'}
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, runName=runName, metadata=metadata,
summaryMetrics=summaryStats)
mergedHistDict['minAirmass'].addBundle(bundle, plotDict=histMerge)
bundleList.append(bundle)
metric = metrics.FracAboveMetric(col='airmass', cutoff=airmass_limit)
plotDict = {'xMin': 0, 'xMax': 1, 'color': tcolor}
summaryStats = allStats
displayDict = {'group': airmassgroup, 'subgroup': 'Low airmass fraction',
'order': filtorder[f] * 100 + order, 'caption':
'Fraction of total images with airmass higher than %.2f, in %s'
% (airmass_limit, propCaption)}
histMerge = {'color': tcolor, 'label': '%s %s' % (
f, tlabel), 'binsize': 0.05, 'legendloc': 'upper right'}
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, runName=runName, metadata=metadata,
summaryMetrics=summaryStats)
mergedHistDict['fracAboveAirmass'].addBundle(bundle, plotDict=histMerge)
bundleList.append(bundle)
# SNe metrics from UK workshop.
peaks = {'uPeak': 25.9, 'gPeak': 23.6, 'rPeak': 22.6, 'iPeak': 22.7, 'zPeak': 22.7, 'yPeak': 22.8}
peakTime = 15.
transDuration = peakTime + 30. # Days
metric = metrics.TransientMetric(riseSlope=-2. / peakTime, declineSlope=1.4 / 30.0,
transDuration=transDuration, peakTime=peakTime,
surveyDuration=runLength,
metricName='SNDetection', **peaks)
caption = 'Fraction of z=0.5 type Ia SN that are detected at any point in their light curve in any filter'
displayDict = {'group': sngroup, 'subgroup': 'Detected', 'caption': caption}
sqlconstraint = ''
plotDict = {}
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, runName=runName)
bundleList.append(bundle)
metric = metrics.TransientMetric(riseSlope=-2. / peakTime, declineSlope=1.4 / 30.0,
transDuration=transDuration, peakTime=peakTime,
surveyDuration=runLength,
nPrePeak=1, metricName='SNAlert', **peaks)
caption = 'Fraction of z=0.5 type Ia SN that are detected pre-peak in any filter'
displayDict = {'group': sngroup, 'subgroup': 'Detected on the rise', 'caption': caption}
plotDict = {}
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, runName=runName)
bundleList.append(bundle)
metric = metrics.TransientMetric(riseSlope=-2. / peakTime, declineSlope=1.4 / 30.,
transDuration=transDuration, peakTime=peakTime,
surveyDuration=runLength, metricName='SNLots',
nFilters=3, nPrePeak=3, nPerLC=2, **peaks)
caption = 'Fraction of z=0.5 type Ia SN that are observed 6 times, 3 pre-peak, '
caption += '3 post-peak, with observations in 3 filters'
displayDict = {'group': sngroup, 'subgroup': 'Well observed', 'caption': caption}
sqlconstraint = 'filter="r" or filter="g" or filter="i" or filter="z" '
plotDict = {}
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, runName=runName)
bundleList.append(bundle)
propIDOrderDict = {}
orderVal = 100
for propID in propids:
propIDOrderDict[propID] = orderVal
orderVal += 100
# Full range of dates:
metric = metrics.FullRangeMetric(col='observationStartMJD')
plotFuncs = [plots.HealpixSkyMap(), plots.HealpixHistogram()]
caption = 'Time span of survey.'
sqlconstraint = ''
plotDict = {}
displayDict = {'group': rangeGroup, 'caption': caption}
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, runName=runName)
bundleList.append(bundle)
for f in filters:
for propid in propids:
displayDict = {'group': rangeGroup, 'subgroup': propids[propid], 'caption': caption,
'order': filtorder[f]}
md = '%s, %s' % (f, propids[propid])
sql = 'filter="%s" and proposalId=%i' % (f, propid)
bundle = metricBundles.MetricBundle(metric, slicer, sql, plotDict=plotDict,
metadata=md, plotFuncs=plotFuncs,
displayDict=displayDict, runName=runName)
bundleList.append(bundle)
# Alt az plots
slicer = slicers.HealpixSlicer(nside=64, latCol='zenithDistance', lonCol='azimuth', useCache=False)
metric = metrics.CountMetric('observationStartMJD', metricName='Nvisits as function of Alt/Az')
plotDict = {}
plotFuncs = [plots.LambertSkyMap()]
displayDict = {'group': altAzGroup, 'caption': 'Alt Az pointing distribution'}
for f in filters:
for propid in propids:
displayDict = {'group': altAzGroup, 'subgroup': propids[propid],
'caption': 'Alt Az pointing distribution',
'order': filtorder[f]}
md = '%s, %s' % (f, propids[propid])
sql = 'filter="%s" and proposalId=%i' % (f, propid)
bundle = metricBundles.MetricBundle(metric, slicer, sql, plotDict=plotDict,
plotFuncs=plotFuncs, metadata=md,
displayDict=displayDict, runName=runName)
bundleList.append(bundle)
sql = ''
md = 'all observations'
displayDict = {'group': altAzGroup, 'subgroup': 'All Observations',
'caption': 'Alt Az pointing distribution'}
bundle = metricBundles.MetricBundle(metric, slicer, sql, plotDict=plotDict,
plotFuncs=plotFuncs, metadata=md,
displayDict=displayDict, runName=runName)
bundleList.append(bundle)
# Median inter-night gap (each and all filters)
slicer = slicers.HealpixSlicer(nside=nside, lonCol=lonCol, latCol=latCol)
metric = metrics.InterNightGapsMetric(metricName='Median Inter-Night Gap')
sqls = ['filter = "%s"' % f for f in filters]
orders = [filtorder[f] for f in filters]
orders.append(0)
sqls.append('')
for sql, order in zip(sqls, orders):
displayDict = {'group': intergroup, 'subgroup': 'Median Gap', 'caption': 'Median gap between days',
'order': order}
bundle = metricBundles.MetricBundle(metric, slicer, sql, displayDict=displayDict, runName=runName)
bundleList.append(bundle)
# Max inter-night gap in r and all bands
dslicer = slicers.HealpixSlicer(nside=nside, lonCol='ditheredRA', latCol='ditheredDec')
metric = metrics.InterNightGapsMetric(metricName='Max Inter-Night Gap', reduceFunc=np.max)
plotDict = {'percentileClip': 95.}
for sql, order in zip(sqls, orders):
displayDict = {'group': intergroup, 'subgroup': 'Max Gap', 'caption': 'Max gap between nights',
'order': order}
bundle = metricBundles.MetricBundle(metric, dslicer, sql, displayDict=displayDict,
plotDict=plotDict, runName=runName)
bundleList.append(bundle)
# largest phase gap for periods
periods = [0.1, 1.0, 10., 100.]
sqls = {'u': 'filter = "u"', 'r': 'filter="r"',
'g,r,i,z': 'filter="g" or filter="r" or filter="i" or filter="z"',
'all': ''}
for sql in sqls:
for period in periods:
displayDict = {'group': phaseGroup,
'subgroup': 'period=%.2f days, filter=%s' % (period, sql),
'caption': 'Maximum phase gaps'}
metric = metrics.PhaseGapMetric(nPeriods=1, periodMin=period, periodMax=period,
metricName='PhaseGap, %.1f' % period)
bundle = metricBundles.MetricBundle(metric, slicer, sqls[sql],
displayDict=displayDict, runName=runName)
bundleList.append(bundle)
# NEO XY plots
slicer = slicers.UniSlicer()
metric = metrics.PassMetric(metricName='NEODistances')
stacker = stackers.NEODistStacker()
stacker2 = stackers.EclipticStacker()
for f in filters:
plotFunc = plots.NeoDistancePlotter(eclipMax=10., eclipMin=-10.)
caption = 'Observations within 10 degrees of the ecliptic. Distance an H=22 NEO would be detected'
displayDict = {'group': NEOGroup, 'subgroup': 'xy', 'order': filtorder[f],
'caption': caption}
plotDict = {}
sqlconstraint = 'filter = "%s"' % (f)
bundle = metricBundles.MetricBundle(metric, slicer,
sqlconstraint, displayDict=displayDict,
stackerList=[stacker, stacker2],
plotDict=plotDict,
plotFuncs=[plotFunc])
noSaveBundleList.append(bundle)
# Solar elongation
sqls = ['filter = "%s"' % f for f in filters]
orders = [filtorder[f] for f in filters]
sqls.append('')
orders.append(0)
for sql, order in zip(sqls, orders):
plotFuncs = [plots.HealpixSkyMap(), plots.HealpixHistogram()]
displayDict = {'group': NEOGroup, 'subgroup': 'Solar Elongation',
'caption': 'Median solar elongation in degrees', 'order': order}
metric = metrics.MedianMetric('solarElong')
slicer = slicers.HealpixSlicer(nside=nside, lonCol=lonCol, latCol=latCol)
bundle = metricBundles.MetricBundle(metric, slicer, sql, displayDict=displayDict,
plotFuncs=plotFuncs)
bundleList.append(bundle)
plotFuncs = [plots.HealpixSkyMap(), plots.HealpixHistogram()]
displayDict = {'group': NEOGroup, 'subgroup': 'Solar Elongation',
'caption': 'Minimum solar elongation in degrees', 'order': order}
metric = metrics.MinMetric('solarElong')
slicer = slicers.HealpixSlicer(nside=nside, lonCol=lonCol, latCol=latCol)
bundle = metricBundles.MetricBundle(metric, slicer, sql, displayDict=displayDict,
plotFuncs=plotFuncs)
bundleList.append(bundle)
return (metricBundles.makeBundlesDictFromList(bundleList), mergedHistDict,
metricBundles.makeBundlesDictFromList(noSaveBundleList))
def slewActivities(colmap=None, runName='opsim', totalSlewN=1, sqlConstraint=None):
"""Generate a set of slew statistics focused on finding the contributions to the overall slew time.
These slew statistics must be run on the SlewActivities table in opsimv4 and opsimv3.
Note that the type of activities listed are different between v3 and v4.
Parameters
----------
colmap : dict or None, opt
A dictionary with a mapping of column names. Default will use OpsimV4 column names.
runName : str, opt
The name of the simulated survey. Default is "opsim".
totalSlewN : int, opt
The total number of slews in the simulated survey.
Used to calculate % of slew activities for each component.
Default is 1.
sqlConstraint : str or None, opt
SQL constraint to apply to metrics. Note this runs on Slew*State table, so constraints
should generally be based on slew_slewCount.
Returns
-------
metricBundleDict
"""
if totalSlewN == 1:
warnings.warn('TotalSlewN should be set (using 1). Percents from activities may be incorrect.')
if colmap is None:
colmap = ColMapDict('opsimV4')
bundleList = []
# All of these metrics run with a unislicer, on all the slew data.
slicer = slicers.UniSlicer()
if 'slewactivities' not in colmap:
raise ValueError("List of slewactivities not in colmap! Will not create slewActivities bundles.")
slewTypeDict = colmap['slewactivities']
displayDict = {'group': 'Slew', 'subgroup': 'Slew Activities', 'order': -1, 'caption': None}
for slewType in slewTypeDict:
metadata = combineMetadata(slewType, sqlConstraint)
tableValue = slewTypeDict[slewType]
# Metrics for all activities of this type.
sql = 'activityDelay>0 and activity="%s"' % tableValue
if sqlConstraint is not None:
sql = '(%s) and (%s)' % (sql, sqlConstraint)
# Percent of slews which include this activity.
metric = metrics.CountRatioMetric(col='activityDelay', normVal=totalSlewN / 100.0,
metricName='ActivePerc')
displayDict['caption'] = 'Percent of total slews which include %s movement.' % slewType
displayDict['order'] += 1
bundle = mb.MetricBundle(metric, slicer, sql, displayDict=displayDict, metadata=metadata)
bundleList.append(bundle)
# Mean time for this activity, in all slews.
metric = metrics.MeanMetric(col='activityDelay', metricName='Ave T(s)')
displayDict['caption'] = 'Mean amount of time (in seconds) for %s movements.' % (slewType)
displayDict['order'] += 1
bundle = mb.MetricBundle(metric, slicer, sql, displayDict=displayDict, metadata=metadata)
bundleList.append(bundle)
# Maximum time for this activity, in all slews.
metric = metrics.MaxMetric(col='activityDelay', metricName='Max T(s)')
displayDict['caption'] = 'Max amount of time (in seconds) for %s movement.' % (slewType)
displayDict['order'] += 1
bundle = mb.MetricBundle(metric, slicer, sql, displayDict=displayDict, metadata=metadata)
bundleList.append(bundle)
# Metrics for activities of this type which are in the critical path.
sql = 'activityDelay>0 and inCriticalPath="True" and activity="%s"' % tableValue
if sqlConstraint is not None:
sql = '(%s) and (%s)' % (sql, sqlConstraint)
# Percent of slews which include this activity in the critical path.
metric = metrics.CountRatioMetric(col='activityDelay', normVal=totalSlewN / 100.0,
metricName='ActivePerc in crit')
displayDict['caption'] = 'Percent of total slew which include %s movement, ' \
'and are in critical path.' % (slewType)
displayDict['order'] += 1
bundle = mb.MetricBundle(metric, slicer, sql, displayDict=displayDict, metadata=metadata)
bundleList.append(bundle)
# Mean time for slews which include this activity, in the critical path.
metric = metrics.MeanMetric(col='activityDelay', metricName='Ave T(s) in crit')
displayDict['caption'] = 'Mean time (in seconds) for %s movements, ' \
'when in critical path.' % (slewType)
displayDict['order'] += 1
bundle = mb.MetricBundle(metric, slicer, sql, displayDict=displayDict, metadata=metadata)
bundleList.append(bundle)
# Total time that this activity was in the critical path.
metric = metrics.SumMetric(col='activityDelay', metricName='Total T(s) in crit')
displayDict['caption'] = 'Total time (in seconds) for %s movements, ' \
'when in critical path.' % (slewType)
displayDict['order'] += 1
bundle = mb.MetricBundle(metric, slicer, sql, displayDict=displayDict, metadata=metadata)
bundleList.append(bundle)
for b in bundleList:
b.setRunName(runName)
return mb.makeBundlesDictFromList(bundleList)
def go(nside=64, rmag=21., SedTemplate='flat', DoRun=False, LFilters = [], \
LNightMax=[], nightMax=1e4, \
CustomPlotLimits=True, \
RunOne=False, MaxRuns=1e3, \
SpatialClip=95., \
seeingCol='FWHMeff', \
sCmap='cubehelix_r', \
checkCorrKind=False, \
wfdPlane=True, \
useGRIZ=False):
# Go to the directory where the sqlite databases are held...
# cd /Users/clarkson/Data/LSST/OpSimRuns/opsim20160411
# WIC 2015-12-29 - set up for a master-run with all cases, this time with plotting limits
# Break the specifications across lines to make subdivision easier
# Subsets by time first, then by filter, finally the whole shebang
# 2016-04-23 - replaced enigma_1189 --> minion_1016
# 2016-04-23 - replaced ops2_1092 --> minion_1020
# (Yes the inversion of the first two is deliberate.)
runNames = ['minion_1016', 'minion_1020', 'minion_1020', 'minion_1016', \
'minion_1020', 'minion_1016', 'minion_1020', 'minion_1016', \
'minion_1020', 'minion_1016']
LFilters = ['', '', '', '', \
'u', 'u', 'y', 'y', \
'', '']
LNightMax = [365, 365, 730, 730, \
1e4, 1e4, 1e4, 1e4, \
1e4, 1e4]
# WIC try again, this time on the new astro_lsst_01_1004 only
if wfdPlane:
LFilters = ['', '', '', 'u', 'y']
LNightMax = [365, 730, 1e4, 1e4, 1e4]
runNames = ['astro_lsst_01_1004' for i in range (len(LFilters)) ]
# WIC 2016-05-01 check correlation
if checkCorrKind:
LFilters = ['', '']
LNightMax = [365, 365]
runNames = ['minion_1016', 'minion_1016']
# Type of correlation used for HA Degen
# checkCorrKind = True
useSpearmanR = [False, True]
if useGRIZ:
runNames=['minion_1016','astro_lsst_01_1004', 'minion_1020']
LFilters = ['griz' for iRun in range(len(runNames)) ]
#LNightMax = [1e4 for iRun in range(len(runNames)) ]
#LNightMax = [730 for iRun in range(len(runNames)) ]
LNightMax = [365 for iRun in range(len(runNames)) ]
# List of upper limits to parallax and proper motion error. For parallax, 3.0 mas is probably good
LUpperParallax = []
LUpperPropmotion = []
if CustomPlotLimits:
LUpperParallax = [10, 10, 10, 10, \
10, 10, 40, 40, \
3.0, 3.0 ]
# For proper motion, it's a little tricky to say because the
# regular case is so pathological for the field. Try the following:
LUpperPropmotion = [40, 40, 5, 20, \
3.5, 20, 3.5, 20, \
0.5, 5]
if len(runNames) < 2:
LUpperPropmotion = [100 for i in range(len(runNames))]
print "runAstrom.go INFO - will run the following:"
for iSho in range(len(runNames)):
sFilThis = ''
# print iSho, len(LFilters)
if iSho <= len(LFilters):
sFilThis = sqlFromFilterString(LFilters[iSho])
print "%i: %-12s, %1s, %i, sqlFilter -- %s" % (iSho, runNames[iSho], LFilters[iSho], LNightMax[iSho], sFilThis)
print "==========================="
print "mag max = %.2f" % (rmag)
print "---------------------------"
# print runNames
# if not DoRun:
# print "Set DoRun=True to actually run this."
# print len(LFilters), len(runNames), len(LFilters) == len(runNames)
# return
#'kraken_1038', 'kraken_1034', 'ops2_1098']
# nside = 64
slicer = slicers.HealpixSlicer(nside=nside)
# Make it so we don't bother with the silly power spectra
plotFuncs = [plots.HealpixSkyMap(), plots.HealpixHistogram()]
# WIC - back up the plotting arguments with a default value
plotFuncsPristine = copy.deepcopy(plotFuncs)
# WIC - the only way this will make sense to me is if I make a
# dictionary of plot arguments. Let's try it...
DPlotArgs = {}
for plotArg in ['parallax', 'propmotion', 'coverage', 'HAdegen']:
DPlotArgs[plotArg] = copy.deepcopy(plotFuncs)
if CustomPlotLimits:
# Use the same color map for all the metrics
for plotMetric in DPlotArgs.keys():
DPlotArgs[plotMetric][0].defaultPlotDict['cmap'] = sCmap
# Apply spatial clipping for all but the HADegen, for which we
# have other limits...
for plotMetric in ['parallax', 'propmotion', 'coverage']:
DPlotArgs[plotMetric][0].defaultPlotDict['percentileClip'] = SpatialClip
# Some limits common to spatial maps and histograms
for iPl in range(0,2):
DPlotArgs['propmotion'][iPl].defaultPlotDict['logScale'] = True
# NOT a loop because we might want to separate out the behavior
# Standardized range for the histograms for new parallax metrics
DPlotArgs['coverage'][1].defaultPlotDict['xMin'] = 0.
DPlotArgs['coverage'][1].defaultPlotDict['xMax'] = 1.
DPlotArgs['HAdegen'][1].defaultPlotDict['xMin'] = -1.
DPlotArgs['HAdegen'][1].defaultPlotDict['xMax'] = 1.
# Standardize the sky map for the HAdegen as well.
DPlotArgs['coverage'][1].defaultPlotDict['xMin'] = 0.
DPlotArgs['coverage'][1].defaultPlotDict['xMax'] = 1.
DPlotArgs['HAdegen'][0].defaultPlotDict['xMin'] = -1.
DPlotArgs['HAdegen'][0].defaultPlotDict['xMax'] = 1.
# Standardize at least the lower bound of the histogram in
# both the proper motion and parallax errors. Upper limit we
# can customize with a loop.
DPlotArgs['propmotion'][1].defaultPlotDict['xMin'] = 1e-2 # should not be zero if log scale!!
DPlotArgs['parallax'][1].defaultPlotDict['xMin'] = 0.
# WIC - try changing the plot dictionary
if not DoRun:
plotFuncs[0].defaultPlotDict['logScale'] = True
print DPlotArgs['propmotion'][0].defaultPlotDict
print DPlotArgs['propmotion'][1].defaultPlotDict
return
# The old runs have the seeing in finSeeing
#seeingCol = 'finSeeing'
### UPDATE THE SEEING COLUMN
#seeingCol = 'FWHMeff' ## Moved up to a command-line argument
# Use all the observations. Can change if you want a different
# time span
# sqlconstraint = ''
# list of sqlconstraints now used, which gets handled within the loop.
# run some summary stats on everything
summaryMetrics = [metrics.MedianMetric()]
tStart = time.time()
# Running one, or the whole lot?
RunMax = len(runNames)
# allow user to set a different number (say, 2)
if MaxRuns < RunMax and MaxRuns > 0:
RunMax = int(MaxRuns)
# the following keyword overrides
if RunOne:
RunMax = 1
print "Starting runs. RunMax = %i" % (RunMax)
for iRun in range(RunMax):
run = runNames[iRun][:]
# for run in runNames:
# Open the OpSim database
timeStartIteration = time.time()
# Some syntax added to test for existence of the database
dbFil = run+'_sqlite.db'
if not os.access(dbFil, os.R_OK):
print "runAstrom.go FATAL - cannot acces db file %s" % (dbFil)
print "runAstrom.go FATAL - skipping run %s" % (run)
continue
else:
deltaT = time.time()-tStart
print "runAstrom.go INFO - ##################################"
print "runAstrom.go INFO - starting run %s with nside=%i after %.2f minutes" \
% (run, nside, deltaT/60.)
opsdb = db.OpsimDatabase(run+'_sqlite.db')
# Set SQL constraint appropriate for each filter in the
# list. If we supplied a list of filters, use it for
sqlconstraint = ''
ThisFilter = 'ugrizy'
if len(LFilters) == len(runNames):
# Only change the filter if one was actually supplied!
if len(LFilters[iRun]) > 0:
ThisFilter = LFilters[iRun]
sqlconstraint = sqlFromFilterString(ThisFilter)
### sqlconstraint = 'filter = "%s"' % (ThisFilter)
# If nightmax was supplied, use it
ThisNightMax = int(nightMax) # copy not view
if len(LNightMax) == len(runNames):
# Only update nightmax if one was given
try:
ThisNightMax = int(LNightMax[iRun]) # This might be redundant with the fmt statement below.
if len(sqlconstraint) < 1:
sqlconstraint = 'night < %i' % (ThisNightMax)
else:
sqlconstraint = '%s and night < %i' % (sqlconstraint, ThisNightMax)
except:
print "runAstrom.go WARN - run %i problem with NightMax" % (iRun)
dumdum = 1.
# Set where the output should go - include the filter!!
sMag = '%.1f' % (rmag)
sMag = sMag.replace(".","p")
outDir = './metricEvals/%s_nside%i_%s_n%i_r%s' % (run, nside, ThisFilter, ThisNightMax, sMag)
# Ensure we'll be able to find this later on...
if CustomPlotLimits:
outDir = '%s_lims' % (outDir)
# if we are testing the kind of correlation used, include that
# in the output here.
if checkCorrKind:
if useSpearmanR[iRun]:
sCorr = 'spearmanR'
else:
sCorr = 'pearsonR'
outDir = '%s_%s' % (outDir, sCorr)
# From this point onwards, stuff actually gets run. This is
# the place to output what will actually happen next.
print "runAstrom.go INFO - about to run:"
print "runAstrom.go INFO - sqlconstraint: %s ; run name %s ; nside %i" % (sqlconstraint, run, nside)
print "runAstrom.go INFO - output directory will be %s" % (outDir)
if not DoRun:
continue
# ensure the output directory actually exists...
if not os.access(outDir, os.R_OK):
print "runAstrom.go INFO - creating output directory %s" % (outDir)
os.makedirs(outDir)
resultsDb = db.ResultsDb(outDir=outDir)
bundleList = []
# WIC - to make this at least somewhat uniform, build the plot
# functions including arguments out of our copies above.
plotFuncsPropmotion = copy.deepcopy(DPlotArgs['propmotion'])
plotFuncsParallax = copy.deepcopy(DPlotArgs['parallax'])
plotFuncsCoverage = copy.deepcopy(DPlotArgs['coverage'])
plotFuncsHAdegen = copy.deepcopy(DPlotArgs['HAdegen'])
# if using custom plot limits, will want to include the limits
# for proper motion and parallax too... programming a bit defensively
# here, including an extra check (rather than just the length of the lists
# above).
if CustomPlotLimits:
if len(LUpperParallax) == len(runNames):
plotFuncsParallax[1].defaultPlotDict['xMax'] = float(LUpperParallax[iRun])
if len(LUpperPropmotion) == len(runNames):
plotFuncsPropmotion[1].defaultPlotDict['xMax'] = float(LUpperPropmotion[iRun])
# Configure the metrics
metric = metrics.ParallaxMetric(rmag=rmag, seeingCol=seeingCol, SedTemplate=SedTemplate)
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, runName=run,
# plotFuncs=plotFuncs, \
plotFuncs = plotFuncsParallax, \
summaryMetrics=summaryMetrics)
bundleList.append(bundle)
metric=metrics.ProperMotionMetric(rmag=rmag, seeingCol=seeingCol, SedTemplate=SedTemplate)
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, runName=run,
# plotFuncs=plotFuncs, \
plotFuncs=plotFuncsPropmotion, \
summaryMetrics=summaryMetrics)
bundleList.append(bundle)
metric = calibrationMetrics.ParallaxCoverageMetric(rmag=rmag, seeingCol=seeingCol, SedTemplate=SedTemplate)
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, runName=run,
# plotFuncs=plotFuncs, \
plotFuncs=plotFuncsCoverage, \
summaryMetrics=summaryMetrics)
bundleList.append(bundle)
# Now for the HA Degen metric. If testing the type of
# correlation, call the metric differently here. Since the
# argument to actually do this is only part of my github fork
# at the moment, we use a different call. Running with default
# arguments (checkCorrKind=False) should then work without
# difficulty.
metric = calibrationMetrics.ParallaxHADegenMetric(rmag=rmag, seeingCol=seeingCol, SedTemplate=SedTemplate)
if checkCorrKind:
metric = calibrationMetrics.ParallaxHADegenMetric(rmag=rmag, seeingCol=seeingCol, SedTemplate=SedTemplate, useSpearmanR=useSpearmanR[iRun])
print "TESTING CORRELATION KIND -- useSpearmanR", useSpearmanR[iRun]
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, runName=run,
# plotFuncs=plotFuncs, \
plotFuncs=plotFuncsHAdegen, \
summaryMetrics=summaryMetrics)
bundleList.append(bundle)
# Run everything and make plots
bundleDict = metricBundles.makeBundlesDictFromList(bundleList)
bgroup = metricBundles.MetricBundleGroup(bundleDict, opsdb, outDir=outDir, resultsDb=resultsDb)
# try:
bgroup.runAll()
print "runAstrom.go INFO - bundles took %.2f minutes" \
% ((time.time() - timeStartIteration) / 60.)
# except KeyboardInterrupt:
# print "runAstrom.go FATAL - keyboard interrupt detected. Halting."
# return
bgroup.plotAll()
print "runAstrom.go INFO - bundles + plotting took %.2f minutes" \
% ((time.time() - timeStartIteration) / 60.)
print "Finished entire set. %i runs took %.2f minutes." % (iRun + 1, (time.time()-tStart)/60.)
def runSlices(opsimName, metadata, simdata, bins, args, verbose=False):
"""
Set up and run the movie slicer, and at each step,
setup the healpix slicer and run the metrics, creating and saving the plots.
"""
# Set up movie slicer
movieslicer = setupMovieSlicer(simdata,
binsize=args.movieStepsize,
cumulative=args.cumulative)
start_date = movieslicer[0]['slicePoint']['binLeft']
sliceformat = '%s0%dd' % ('%', int(np.log10(len(movieslicer))) + 1)
# Run through the movie slicer slicePoints:
for i, movieslice in enumerate(movieslicer):
t = time.time()
slicenumber = sliceformat % (i)
"""
# Set up plot label.
timeInterval = '%.2f to %.2f' %(movieslice['slicePoint']['binLeft']-start_date,
movieslice['slicePoint']['binRight']-start_date)
"""
# Or add simple view of time to plot label.
times_from_start = movieslice['slicePoint']['binRight'] - (
int(bins[0]) + 0.16 - 0.5)
# Opsim years are 365 days (not 365.25)
years = int(times_from_start / 365)
days = times_from_start - years * 365
plotlabel = 'Year %d Day %.4f' % (years, days)
metricList, plotDictList = setupMetrics(
opsimName,
metadata,
start_date,
movieslice['slicePoint']['binRight'],
cumulative=args.cumulative,
plotlabel=plotlabel,
verbose=verbose)
# Identify the subset of simdata in the movieslicer 'data slice'
simdatasubset = simdata[movieslice['idxs']]
# Set up healpix slicer on subset of simdata provided by movieslicer
hs = setupHealpixSlicer(args)
bundles = []
for metric, plotDict in zip(metricList, plotDictList):
bundles.append(
mB.MetricBundle(metric,
hs,
sqlconstraint=args.sqlConstraint,
metadata=metadata,
runName=opsimName,
plotDict=plotDict,
plotFuncs=[plots.HealpixSkyMap()]))
# Remove (default) stackers from bundles, because we've already run them above on the original data.
for mb in bundles:
mb.stackerList = []
bundledict = mB.makeBundlesDictFromList(bundles)
# Set up metricBundleGroup to handle metrics calculation + plotting
bg = mB.MetricBundleGroup(bundledict,
opsDb,
outDir=args.outDir,
resultsDb=None,
saveEarly=False)
# Calculate metric data values for simdatasubset (this also sets up indexing in the slicer)
bg.setCurrent(args.sqlConstraint)
bg.runCurrent(constraint=args.sqlConstraint, simData=simdatasubset)
# Plot data for this slice of the movie, adding slicenumber as a suffix for output plots
bg.plotAll(outfileSuffix=slicenumber,
closefigs=True,
dpi=72,
thumbnail=False,
figformat='png')
# Write the data -- uncomment if you want to do this.
# sm.writeAll(outfileSuffix=slicenumber)
if verbose:
dt, t = dtime(t)
print('Ran and plotted slice %s of movieslicer in %f s' %
(slicenumber, dt))
def glanceBatch(colmap=None,
runName='opsim',
nside=64,
filternames=('u', 'g', 'r', 'i', 'z', 'y'),
nyears=10,
pairnside=32,
sqlConstraint=None):
"""Generate a handy set of metrics that give a quick overview of how well a survey performed.
This is a meta-set of other batches, to some extent.
Parameters
----------
colmap : dict, opt
A dictionary with a mapping of column names. Default will use OpsimV4 column names.
run_name : str, opt
The name of the simulated survey. Default is "opsim".
nside : int, opt
The nside for the healpix slicers. Default 64.
filternames : list of str, opt
The list of individual filters to use when running metrics.
Default is ('u', 'g', 'r', 'i', 'z', 'y').
There is always an all-visits version of the metrics run as well.
nyears : int (10)
How many years to attempt to make hourglass plots for
pairnside : int (32)
nside to use for the pair fraction metric (it's slow, so nice to use lower resolution)
sqlConstraint : str or None, opt
Additional SQL constraint to apply to all metrics.
Returns
-------
metricBundleDict
"""
if isinstance(colmap, str):
raise ValueError('colmap must be a dictionary, not a string')
if colmap is None:
colmap = ColMapDict('opsimV4')
bundleList = []
if sqlConstraint is None:
sqlC = ''
else:
sqlC = '(%s) and' % sqlConstraint
sql_per_filt = [
'%s %s="%s"' % (sqlC, colmap['filter'], filtername)
for filtername in filternames
]
sql_per_and_all_filters = [sqlConstraint] + sql_per_filt
standardStats = standardSummary()
subsetPlots = [plots.HealpixSkyMap(), plots.HealpixHistogram()]
# Super basic things
displayDict = {'group': 'Basic Stats', 'order': 1}
sql = sqlConstraint
slicer = slicers.UniSlicer()
# Length of Survey
metric = metrics.FullRangeMetric(col=colmap['mjd'],
metricName='Length of Survey (days)')
bundle = metricBundles.MetricBundle(metric,
slicer,
sql,
displayDict=displayDict)
bundleList.append(bundle)
# Total number of filter changes
metric = metrics.NChangesMetric(col=colmap['filter'],
orderBy=colmap['mjd'])
bundle = metricBundles.MetricBundle(metric,
slicer,
sql,
displayDict=displayDict)
bundleList.append(bundle)
# Total open shutter fraction
metric = metrics.OpenShutterFractionMetric(
slewTimeCol=colmap['slewtime'],
expTimeCol=colmap['exptime'],
visitTimeCol=colmap['visittime'])
bundle = metricBundles.MetricBundle(metric,
slicer,
sql,
displayDict=displayDict)
bundleList.append(bundle)
# Total effective exposure time
metric = metrics.TeffMetric(m5Col=colmap['fiveSigmaDepth'],
filterCol=colmap['filter'],
normed=True)
for sql in sql_per_and_all_filters:
bundle = metricBundles.MetricBundle(metric,
slicer,
sql,
displayDict=displayDict)
bundleList.append(bundle)
# Number of observations, all and each filter
metric = metrics.CountMetric(col=colmap['mjd'],
metricName='Number of Exposures')
for sql in sql_per_and_all_filters:
bundle = metricBundles.MetricBundle(metric,
slicer,
sql,
displayDict=displayDict)
bundleList.append(bundle)
# The alt/az plots of all the pointings
slicer = slicers.HealpixSlicer(nside=nside,
latCol='zenithDistance',
lonCol=colmap['az'],
latLonDeg=colmap['raDecDeg'],
useCache=False)
stacker = stackers.ZenithDistStacker(altCol=colmap['alt'],
degrees=colmap['raDecDeg'])
metric = metrics.CountMetric(colmap['mjd'],
metricName='Nvisits as function of Alt/Az')
plotFuncs = [plots.LambertSkyMap()]
for sql in sql_per_and_all_filters:
bundle = metricBundles.MetricBundle(metric,
slicer,
sql,
plotFuncs=plotFuncs,
displayDict=displayDict,
stackerList=[stacker])
bundleList.append(bundle)
# Things to check per night
# Open Shutter per night
displayDict = {'group': 'Pointing Efficency', 'order': 2}
slicer = slicers.OneDSlicer(sliceColName=colmap['night'], binsize=1)
metric = metrics.OpenShutterFractionMetric(
slewTimeCol=colmap['slewtime'],
expTimeCol=colmap['exptime'],
visitTimeCol=colmap['visittime'])
sql = sqlConstraint
bundle = metricBundles.MetricBundle(metric,
slicer,
sql,
summaryMetrics=standardStats,
displayDict=displayDict)
bundleList.append(bundle)
# Number of filter changes per night
slicer = slicers.OneDSlicer(sliceColName=colmap['night'], binsize=1)
metric = metrics.NChangesMetric(col=colmap['filter'],
orderBy=colmap['mjd'],
metricName='Filter Changes')
bundle = metricBundles.MetricBundle(metric,
slicer,
sql,
summaryMetrics=standardStats,
displayDict=displayDict)
bundleList.append(bundle)
# A few basic maps
# Number of observations, coadded depths
displayDict = {'group': 'Basic Maps', 'order': 3}
slicer = slicers.HealpixSlicer(nside=nside,
latCol=colmap['dec'],
lonCol=colmap['ra'],
latLonDeg=colmap['raDecDeg'])
metric = metrics.CountMetric(col=colmap['mjd'])
plotDict = {'percentileClip': 95.}
for sql in sql_per_and_all_filters:
bundle = metricBundles.MetricBundle(metric,
slicer,
sql,
summaryMetrics=standardStats,
displayDict=displayDict,
plotDict=plotDict)
bundleList.append(bundle)
metric = metrics.Coaddm5Metric(m5Col=colmap['fiveSigmaDepth'])
for sql in sql_per_and_all_filters:
bundle = metricBundles.MetricBundle(metric,
slicer,
sql,
summaryMetrics=standardStats,
displayDict=displayDict)
bundleList.append(bundle)
# Checking a few basic science things
# Maybe check astrometry, observation pairs, SN
plotDict = {'percentileClip': 95.}
displayDict = {'group': 'Science', 'subgroup': 'Astrometry', 'order': 4}
stackerList = []
stacker = stackers.ParallaxFactorStacker(raCol=colmap['ra'],
decCol=colmap['dec'],
degrees=colmap['raDecDeg'],
dateCol=colmap['mjd'])
stackerList.append(stacker)
# Maybe parallax and proper motion, fraction of visits in a good pair for SS
displayDict['caption'] = r'Parallax precision of an $r=20$ flat SED star'
metric = metrics.ParallaxMetric(m5Col=colmap['fiveSigmaDepth'],
filterCol=colmap['filter'],
seeingCol=colmap['seeingGeom'])
sql = sqlConstraint
bundle = metricBundles.MetricBundle(metric,
slicer,
sql,
plotFuncs=subsetPlots,
displayDict=displayDict,
stackerList=stackerList,
plotDict=plotDict)
bundleList.append(bundle)
displayDict[
'caption'] = r'Proper motion precision of an $r=20$ flat SED star'
metric = metrics.ProperMotionMetric(m5Col=colmap['fiveSigmaDepth'],
mjdCol=colmap['mjd'],
filterCol=colmap['filter'],
seeingCol=colmap['seeingGeom'])
bundle = metricBundles.MetricBundle(metric,
slicer,
sql,
plotFuncs=subsetPlots,
displayDict=displayDict,
plotDict=plotDict)
bundleList.append(bundle)
# Solar system stuff
displayDict['caption'] = 'Fraction of observations that are in pairs'
displayDict['subgroup'] = 'Solar System'
sql = '%s (filter="g" or filter="r" or filter="i")' % sqlC
pairSlicer = slicers.HealpixSlicer(nside=pairnside,
latCol=colmap['dec'],
lonCol=colmap['ra'],
latLonDeg=colmap['raDecDeg'])
metric = metrics.PairFractionMetric(mjdCol=colmap['mjd'])
bundle = metricBundles.MetricBundle(metric,
pairSlicer,
sql,
plotFuncs=subsetPlots,
displayDict=displayDict)
bundleList.append(bundle)
# stats from the note column
if 'note' in colmap.keys():
displayDict = {'group': 'Basic Stats', 'subgroup': 'Percent stats'}
metric = metrics.StringCountMetric(col=colmap['note'],
percent=True,
metricName='Percents')
sql = ''
slicer = slicers.UniSlicer()
bundle = metricBundles.MetricBundle(metric,
slicer,
sql,
displayDict=displayDict)
bundleList.append(bundle)
displayDict['subgroup'] = 'Count Stats'
metric = metrics.StringCountMetric(col=colmap['note'],
metricName='Counts')
bundle = metricBundles.MetricBundle(metric,
slicer,
sql,
displayDict=displayDict)
bundleList.append(bundle)
for b in bundleList:
b.setRunName(runName)
# Add hourglass plots.
hrDict = hourglassBatch(colmap=colmap,
runName=runName,
nyears=nyears,
extraSql=sqlConstraint)
# Add basic slew stats.
try:
slewDict = slewBasics(colmap=colmap, runName=runName)
except KeyError as e:
warnings.warn(
'Could not add slew stats: missing required key %s from colmap' %
(e))
bd = metricBundles.makeBundlesDictFromList(bundleList)
bd.update(slewDict)
bd.update(hrDict)
return bd
def makeBundleList(dbFile,
night=1,
nside=64,
latCol='fieldDec',
lonCol='fieldRA',
notes=True,
colmap=None):
"""
Make a bundleList of things to run
"""
if colmap is None:
colmap = ColMapDict('opsimV4')
mjdCol = 'observationStartMJD'
altCol = 'altitude'
azCol = 'azimuth'
# Construct sql queries for each filter and all filters
filters = ['u', 'g', 'r', 'i', 'z', 'y']
sqls = ['night=%i and filter="%s"' % (night, f) for f in filters]
sqls.append('night=%i' % night)
bundleList = []
plotFuncs_lam = [plots.LambertSkyMap()]
# Hourglass
hourslicer = slicers.HourglassSlicer()
displayDict = {'group': 'Hourglass'}
md = ''
sql = 'night=%i' % night
metric = metrics.HourglassMetric(nightCol=colmap['night'],
mjdCol=colmap['mjd'],
metricName='Hourglass')
bundle = metricBundles.MetricBundle(metric,
hourslicer,
constraint=sql,
metadata=md,
displayDict=displayDict)
bundleList.append(bundle)
reg_slicer = slicers.HealpixSlicer(nside=nside,
lonCol=lonCol,
latCol=latCol,
latLonDeg=True)
altaz_slicer = slicers.HealpixSlicer(nside=nside,
latCol=altCol,
latLonDeg=True,
lonCol=azCol,
useCache=False)
unislicer = slicers.UniSlicer()
for sql in sqls:
# Number of exposures
metric = metrics.CountMetric(mjdCol, metricName='N visits')
bundle = metricBundles.MetricBundle(metric, reg_slicer, sql)
bundleList.append(bundle)
metric = metrics.CountMetric(mjdCol, metricName='N visits alt az')
bundle = metricBundles.MetricBundle(metric,
altaz_slicer,
sql,
plotFuncs=plotFuncs_lam)
bundleList.append(bundle)
metric = metrics.MeanMetric(mjdCol, metricName='Mean Visit Time')
bundle = metricBundles.MetricBundle(metric, reg_slicer, sql)
bundleList.append(bundle)
metric = metrics.MeanMetric(mjdCol,
metricName='Mean Visit Time alt az')
bundle = metricBundles.MetricBundle(metric,
altaz_slicer,
sql,
plotFuncs=plotFuncs_lam)
bundleList.append(bundle)
metric = metrics.CountMetric(mjdCol, metricName='N_visits')
bundle = metricBundles.MetricBundle(metric, unislicer, sql)
bundleList.append(bundle)
# Need pairs in window to get a map of how well it gathered SS pairs.
# Moon phase.
metric = metrics.NChangesMetric(col='filter', metricName='Filter Changes')
bundle = metricBundles.MetricBundle(metric, unislicer, 'night=%i' % night)
bundleList.append(bundle)
metric = metrics.BruteOSFMetric()
bundle = metricBundles.MetricBundle(metric, unislicer, 'night=%i' % night)
bundleList.append(bundle)
metric = metrics.MeanMetric('slewTime')
bundle = metricBundles.MetricBundle(metric, unislicer, 'night=%i' % night)
bundleList.append(bundle)
metric = metrics.MinMetric('slewTime')
bundle = metricBundles.MetricBundle(metric, unislicer, 'night=%i' % night)
bundleList.append(bundle)
metric = metrics.MaxMetric('slewTime')
bundle = metricBundles.MetricBundle(metric, unislicer, 'night=%i' % night)
bundleList.append(bundle)
# Make plots of the solar system pairs that were taken in the night
metric = metrics.PairMetric(mjdCol=mjdCol)
sql = 'night=%i and (filter ="r" or filter="g" or filter="i")' % night
bundle = metricBundles.MetricBundle(metric, reg_slicer, sql)
bundleList.append(bundle)
metric = metrics.PairMetric(mjdCol=mjdCol, metricName='z Pairs')
sql = 'night=%i and filter="z"' % night
bundle = metricBundles.MetricBundle(metric, reg_slicer, sql)
bundleList.append(bundle)
# Plot up each visit
metric = metrics.NightPointingMetric(mjdCol=mjdCol)
slicer = slicers.UniSlicer()
sql = 'night=%i' % night
plotFuncs = [plots.NightPointingPlotter()]
bundle = metricBundles.MetricBundle(metric,
slicer,
sql,
plotFuncs=plotFuncs)
bundleList.append(bundle)
# stats from the note column
if notes:
displayDict = {'group': 'Basic Stats', 'subgroup': 'Percent stats'}
metric = metrics.StringCountMetric(col='note',
percent=True,
metricName='Percents')
bundle = metricBundles.MetricBundle(metric,
unislicer,
sql,
displayDict=displayDict)
bundleList.append(bundle)
displayDict['subgroup'] = 'Count Stats'
metric = metrics.StringCountMetric(col='note', metricName='Counts')
bundle = metricBundles.MetricBundle(metric,
unislicer,
sql,
displayDict=displayDict)
bundleList.append(bundle)
return metricBundles.makeBundlesDictFromList(bundleList)
def fOBatch(colmap=None, runName='opsim', extraSql=None, extraMetadata=None, nside=64,
benchmarkArea=18000, benchmarkNvisits=825, ditherStacker=None, ditherkwargs=None):
"""Metrics for calculating fO.
Parameters
----------
colmap : dict or None, opt
A dictionary with a mapping of column names. Default will use OpsimV4 column names.
runName : str, opt
The name of the simulated survey. Default is "opsim".
nside : int, opt
Nside for the healpix slicer. Default 64.
extraSql : str or None, opt
Additional sql constraint to apply to all metrics.
extraMetadata : str or None, opt
Additional metadata to apply to all results.
ditherStacker: str or lsst.sims.maf.stackers.BaseDitherStacker
Optional dither stacker to use to define ra/dec columns.
ditherkwargs: dict, opt
Optional dictionary of kwargs for the dither stacker.
Returns
-------
metricBundleDict
"""
if colmap is None:
colmap = ColMapDict('opsimV4')
bundleList = []
sql = ''
metadata = 'All visits'
# Add additional sql constraint (such as wfdWhere) and metadata, if provided.
if (extraSql is not None) and (len(extraSql) > 0):
sql = extraSql
if extraMetadata is None:
metadata = extraSql.replace('filter =', '').replace('filter=', '')
metadata = metadata.replace('"', '').replace("'", '')
if extraMetadata is not None:
metadata = extraMetadata
subgroup = metadata
raCol, decCol, degrees, ditherStacker, ditherMeta = radecCols(ditherStacker, colmap, ditherkwargs)
# Don't want dither info in subgroup (too long), but do want it in bundle name.
metadata = combineMetadata(metadata, ditherMeta)
# Set up fO metric.
slicer = slicers.HealpixSlicer(nside=nside, lonCol=raCol, latCol=decCol, latLonDeg=degrees)
displayDict = {'group': 'FO metrics', 'subgroup': subgroup, 'order': 0}
# Configure the count metric which is what is used for f0 slicer.
metric = metrics.CountMetric(col=colmap['mjd'], metricName='fO')
plotDict = {'xlabel': 'Number of Visits', 'Asky': benchmarkArea,
'Nvisit': benchmarkNvisits, 'xMin': 0, 'xMax': 1500}
summaryMetrics = [metrics.fOArea(nside=nside, norm=False, metricName='fOArea',
Asky=benchmarkArea, Nvisit=benchmarkNvisits),
metrics.fOArea(nside=nside, norm=True, metricName='fOArea/benchmark',
Asky=benchmarkArea, Nvisit=benchmarkNvisits),
metrics.fONv(nside=nside, norm=False, metricName='fONv',
Asky=benchmarkArea, Nvisit=benchmarkNvisits),
metrics.fONv(nside=nside, norm=True, metricName='fONv/benchmark',
Asky=benchmarkArea, Nvisit=benchmarkNvisits)]
caption = 'The FO metric evaluates the overall efficiency of observing. '
caption += ('foNv: out of %.2f sq degrees, the area receives at least X and a median of Y visits '
'(out of %d, if compared to benchmark). ' % (benchmarkArea, benchmarkNvisits))
caption += ('fOArea: this many sq deg (out of %.2f sq deg if compared '
'to benchmark) receives at least %d visits. ' % (benchmarkArea, benchmarkNvisits))
displayDict['caption'] = caption
bundle = mb.MetricBundle(metric, slicer, sql, plotDict=plotDict,
stackerList = [ditherStacker],
displayDict=displayDict, summaryMetrics=summaryMetrics,
plotFuncs=[plots.FOPlot()], metadata=metadata)
bundleList.append(bundle)
# Set the runName for all bundles and return the bundleDict.
for b in bundleList:
b.setRunName(runName)
return mb.makeBundlesDictFromList(bundleList)
def glanceBatch(colmap=None, runName='opsim',
nside=64, filternames=('u', 'g', 'r', 'i', 'z', 'y'),
nyears=10, pairnside=32, sqlConstraint=None, slicer_camera='LSST'):
"""Generate a handy set of metrics that give a quick overview of how well a survey performed.
This is a meta-set of other batches, to some extent.
Parameters
----------
colmap : dict, opt
A dictionary with a mapping of column names. Default will use OpsimV4 column names.
run_name : str, opt
The name of the simulated survey. Default is "opsim".
nside : int, opt
The nside for the healpix slicers. Default 64.
filternames : list of str, opt
The list of individual filters to use when running metrics.
Default is ('u', 'g', 'r', 'i', 'z', 'y').
There is always an all-visits version of the metrics run as well.
nyears : int (10)
How many years to attempt to make hourglass plots for
pairnside : int (32)
nside to use for the pair fraction metric (it's slow, so nice to use lower resolution)
sqlConstraint : str or None, opt
Additional SQL constraint to apply to all metrics.
slicer_camera : str ('LSST')
Sets which spatial slicer to use. options are 'LSST' and 'ComCam'
Returns
-------
metricBundleDict
"""
if isinstance(colmap, str):
raise ValueError('colmap must be a dictionary, not a string')
if colmap is None:
colmap = ColMapDict('opsimV4')
bundleList = []
if sqlConstraint is None:
sqlC = ''
else:
sqlC = '(%s) and' % sqlConstraint
if slicer_camera == 'LSST':
spatial_slicer = slicers.HealpixSlicer
elif slicer_camera == 'ComCam':
spatial_slicer = slicers.HealpixComCamSlicer
else:
raise ValueError('Camera must be LSST or Comcam')
sql_per_filt = ['%s %s="%s"' % (sqlC, colmap['filter'], filtername) for filtername in filternames]
sql_per_and_all_filters = [sqlConstraint] + sql_per_filt
standardStats = standardSummary()
subsetPlots = [plots.HealpixSkyMap(), plots.HealpixHistogram()]
# Super basic things
displayDict = {'group': 'Basic Stats', 'order': 1}
sql = sqlConstraint
slicer = slicers.UniSlicer()
# Length of Survey
metric = metrics.FullRangeMetric(col=colmap['mjd'], metricName='Length of Survey (days)')
bundle = metricBundles.MetricBundle(metric, slicer, sql, displayDict=displayDict)
bundleList.append(bundle)
# Total number of filter changes
metric = metrics.NChangesMetric(col=colmap['filter'], orderBy=colmap['mjd'])
bundle = metricBundles.MetricBundle(metric, slicer, sql, displayDict=displayDict)
bundleList.append(bundle)
# Total open shutter fraction
metric = metrics.OpenShutterFractionMetric(slewTimeCol=colmap['slewtime'],
expTimeCol=colmap['exptime'],
visitTimeCol=colmap['visittime'])
bundle = metricBundles.MetricBundle(metric, slicer, sql, displayDict=displayDict)
bundleList.append(bundle)
# Total effective exposure time
metric = metrics.TeffMetric(m5Col=colmap['fiveSigmaDepth'],
filterCol=colmap['filter'], normed=True)
for sql in sql_per_and_all_filters:
bundle = metricBundles.MetricBundle(metric, slicer, sql, displayDict=displayDict)
bundleList.append(bundle)
# Number of observations, all and each filter
metric = metrics.CountMetric(col=colmap['mjd'], metricName='Number of Exposures')
for sql in sql_per_and_all_filters:
bundle = metricBundles.MetricBundle(metric, slicer, sql, displayDict=displayDict)
bundleList.append(bundle)
# The alt/az plots of all the pointings
slicer = spatial_slicer(nside=nside, latCol=colmap['alt'],
lonCol=colmap['az'], latLonDeg=colmap['raDecDeg'], useCache=False)
metric = metrics.CountMetric(colmap['mjd'], metricName='Nvisits as function of Alt/Az')
plotFuncs = [plots.LambertSkyMap()]
plotDict = {'norm': 'log'}
for sql in sql_per_and_all_filters:
bundle = metricBundles.MetricBundle(metric, slicer, sql, plotFuncs=plotFuncs,
displayDict=displayDict, plotDict=plotDict)
bundleList.append(bundle)
# Things to check per night
# Open Shutter per night
displayDict = {'group': 'Pointing Efficency', 'order': 2}
slicer = slicers.OneDSlicer(sliceColName=colmap['night'], binsize=1)
metric = metrics.OpenShutterFractionMetric(slewTimeCol=colmap['slewtime'],
expTimeCol=colmap['exptime'],
visitTimeCol=colmap['visittime'])
sql = sqlConstraint
bundle = metricBundles.MetricBundle(metric, slicer, sql,
summaryMetrics=standardStats, displayDict=displayDict)
bundleList.append(bundle)
# Number of filter changes per night
slicer = slicers.OneDSlicer(sliceColName=colmap['night'], binsize=1)
metric = metrics.NChangesMetric(col=colmap['filter'], orderBy=colmap['mjd'],
metricName='Filter Changes')
bundle = metricBundles.MetricBundle(metric, slicer, sql,
summaryMetrics=standardStats, displayDict=displayDict)
bundleList.append(bundle)
# A few basic maps
# Number of observations, coadded depths
extended_stats = standardStats.copy()
extended_stats.append(metrics.AreaSummaryMetric(decreasing=True, metricName='top18k'))
extended_stats.append(metrics.PercentileMetric(col='metricdata', percentile=10))
displayDict = {'group': 'Basic Maps', 'order': 3}
slicer = spatial_slicer(nside=nside, latCol=colmap['dec'], lonCol=colmap['ra'],
latLonDeg=colmap['raDecDeg'])
metric = metrics.CountMetric(col=colmap['mjd'])
plotDict = {'percentileClip': 95.}
for sql in sql_per_and_all_filters:
bundle = metricBundles.MetricBundle(metric, slicer, sql,
summaryMetrics=extended_stats,
displayDict=displayDict,
plotDict=plotDict)
bundleList.append(bundle)
metric = metrics.Coaddm5Metric(m5Col=colmap['fiveSigmaDepth'])
for sql in sql_per_and_all_filters:
bundle = metricBundles.MetricBundle(metric, slicer, sql,
summaryMetrics=extended_stats, displayDict=displayDict)
bundleList.append(bundle)
# Checking a few basic science things
# Maybe check astrometry, observation pairs, SN
plotDict = {'percentileClip': 95.}
displayDict = {'group': 'Science', 'subgroup': 'Astrometry', 'order': 4}
stackerList = []
stacker = stackers.ParallaxFactorStacker(raCol=colmap['ra'],
decCol=colmap['dec'],
degrees=colmap['raDecDeg'],
dateCol=colmap['mjd'])
stackerList.append(stacker)
astrom_stats = [metrics.AreaSummaryMetric(decreasing=False, metricName='best18k'),
metrics.PercentileMetric(col='metricdata', percentile=90)]
# Maybe parallax and proper motion, fraction of visits in a good pair for SS
displayDict['caption'] = r'Parallax precision of an $r=20$ flat SED star'
metric = metrics.ParallaxMetric(m5Col=colmap['fiveSigmaDepth'],
filterCol=colmap['filter'],
seeingCol=colmap['seeingGeom'])
sql = sqlConstraint
bundle = metricBundles.MetricBundle(metric, slicer, sql, plotFuncs=subsetPlots,
displayDict=displayDict, stackerList=stackerList,
plotDict=plotDict,
summaryMetrics=astrom_stats)
bundleList.append(bundle)
displayDict['caption'] = r'Proper motion precision of an $r=20$ flat SED star'
metric = metrics.ProperMotionMetric(m5Col=colmap['fiveSigmaDepth'],
mjdCol=colmap['mjd'],
filterCol=colmap['filter'],
seeingCol=colmap['seeingGeom'])
bundle = metricBundles.MetricBundle(metric, slicer, sql, plotFuncs=subsetPlots,
displayDict=displayDict, plotDict=plotDict,
summaryMetrics=astrom_stats)
bundleList.append(bundle)
# Solar system stuff
displayDict['caption'] = 'Fraction of observations that are in pairs'
displayDict['subgroup'] = 'Solar System'
sql = '%s (filter="g" or filter="r" or filter="i")' % sqlC
pairSlicer = slicers.HealpixSlicer(nside=pairnside, latCol=colmap['dec'], lonCol=colmap['ra'],
latLonDeg=colmap['raDecDeg'])
metric = metrics.PairFractionMetric(mjdCol=colmap['mjd'])
bundle = metricBundles.MetricBundle(metric, pairSlicer, sql, plotFuncs=subsetPlots,
displayDict=displayDict)
bundleList.append(bundle)
# stats from the note column
if 'note' in colmap.keys():
displayDict = {'group': 'Basic Stats', 'subgroup': 'Percent stats'}
metric = metrics.StringCountMetric(col=colmap['note'], percent=True, metricName='Percents')
sql = ''
slicer = slicers.UniSlicer()
bundle = metricBundles.MetricBundle(metric, slicer, sql, displayDict=displayDict)
bundleList.append(bundle)
displayDict['subgroup'] = 'Count Stats'
metric = metrics.StringCountMetric(col=colmap['note'], metricName='Counts')
bundle = metricBundles.MetricBundle(metric, slicer, sql, displayDict=displayDict)
bundleList.append(bundle)
for b in bundleList:
b.setRunName(runName)
bd = metricBundles.makeBundlesDictFromList(bundleList)
# Add hourglass plots.
hrDict = hourglassPlots(colmap=colmap, runName=runName, nyears=nyears, extraSql=sqlConstraint)
bd.update(hrDict)
# Add basic slew stats.
try:
slewDict = slewBasics(colmap=colmap, runName=runName)
bd.update(slewDict)
except KeyError as e:
warnings.warn('Could not add slew stats: missing required key %s from colmap' % (e))
return bd
def slewActivities(colmap=None,
runName='opsim',
totalSlewN=1,
sqlConstraint=None):
"""Generate a set of slew statistics focused on finding the contributions to the overall slew time.
These slew statistics must be run on the SlewActivities table in opsimv4 and opsimv3.
Note that the type of activities listed are different between v3 and v4.
Parameters
----------
colmap : dict or None, opt
A dictionary with a mapping of column names. Default will use OpsimV4 column names.
runName : str, opt
The name of the simulated survey. Default is "opsim".
totalSlewN : int, opt
The total number of slews in the simulated survey.
Used to calculate % of slew activities for each component.
Default is 1.
sqlConstraint : str or None, opt
SQL constraint to apply to metrics. Note this runs on Slew*State table, so constraints
should generally be based on slew_slewCount.
Returns
-------
metricBundleDict
"""
if totalSlewN == 1:
warnings.warn(
'TotalSlewN should be set (using 1). Percents from activities may be incorrect.'
)
if colmap is None:
colmap = ColMapDict('opsimV4')
bundleList = []
# All of these metrics run with a unislicer, on all the slew data.
slicer = slicers.UniSlicer()
print(colmap.keys())
if 'slewactivities' not in colmap:
raise ValueError(
"List of slewactivities not in colmap! Will not create slewActivities bundles."
)
slewTypes = colmap['slewactivities']
displayDict = {
'group': 'Slew',
'subgroup': 'Slew Activities',
'order': -1,
'caption': None
}
for slewType in slewTypes:
metadata = slewType
tableValue = colmap[slewType]
# Metrics for all activities of this type.
sql = 'activityDelay>0 and activity="%s"' % tableValue
if sqlConstraint is not None:
sqlconstraint = '(%s) and (%s)' % (sql, sqlConstraint)
metric = metrics.CountRatioMetric(col='activityDelay',
normVal=totalSlewN / 100.0,
metricName='ActivePerc')
displayDict[
'caption'] = 'Percent of total slews which include %s movement.' % slewType
displayDict['order'] += 1
bundle = mb.MetricBundle(metric,
slicer,
sqlconstraint,
displayDict=displayDict,
metadata=metadata)
bundleList.append(bundle)
metric = metrics.MeanMetric(col='activityDelay',
metricName='ActiveAve')
displayDict[
'caption'] = 'Mean amount of time (in seconds) for %s movements.' % (
slewType)
displayDict['order'] += 1
bundle = mb.MetricBundle(metric,
slicer,
sqlconstraint,
displayDict=displayDict,
metadata=metadata)
bundleList.append(bundle)
metric = metrics.MaxMetric(col='activityDelay', metricName='Max')
displayDict[
'caption'] = 'Max amount of time (in seconds) for %s movement.' % (
slewType)
displayDict['order'] += 1
bundle = mb.MetricBundle(metric,
slicer,
sqlconstraint,
displayDict=displayDict,
metadata=metadata)
bundleList.append(bundle)
# Metrics for activities of this type which are in the critical path.
sql = 'activityDelay>0 and inCriticalPath="True" and activity="%s"' % tableValue
if sqlConstraint is not None:
sqlconstraint = '(%s) and (%s)' % (sql, sqlConstraint)
metric = metrics.CountRatioMetric(col='activityDelay',
normVal=totalSlewN / 100.0,
metricName='ActivePerc in crit')
displayDict['caption'] = 'Percent of total slew which include %s movement, ' \
'and are in critical path.' % (slewType)
displayDict['order'] += 1
bundle = mb.MetricBundle(metric,
slicer,
sqlconstraint,
displayDict=displayDict,
metadata=metadata)
bundleList.append(bundle)
metric = metrics.MeanMetric(col='activityDelay',
metricName='ActiveAve in crit')
displayDict['caption'] = 'Mean time (in seconds) for %s movements, ' \
'when in critical path.' % (slewType)
displayDict['order'] += 1
bundle = mb.MetricBundle(metric,
slicer,
sqlconstraint,
displayDict=displayDict,
metadata=metadata)
bundleList.append(bundle)
for b in bundleList:
b.setRunName(runName)
return mb.makeBundlesDictFromList(bundleList)
def slewBasics(colmap=None, runName='opsim', sqlConstraint=None):
"""Generate a simple set of statistics about the slew times and distances.
These slew statistics can be run on the summary or default tables.
Parameters
----------
colmap : dict or None, opt
A dictionary with a mapping of column names. Default will use OpsimV4 column names.
runName : str, opt
The name of the simulated survey. Default is "opsim".
sqlConstraint : str or None, opt
SQL constraint to add to metrics. (note this runs on summary table).
Returns
-------
metricBundleDict
"""
if colmap is None:
colmap = ColMapDict('opsimV4')
bundleList = []
# Calculate basic stats on slew times. (mean/median/min/max + total).
slicer = slicers.UniSlicer()
metadata = 'All visits'
displayDict = {
'group': 'Slew',
'subgroup': 'Slew Basics',
'order': -1,
'caption': None
}
# Add total number of slews.
metric = metrics.CountMetric(colmap['slewtime'], metricName='Slew Count')
displayDict['caption'] = 'Total number of slews recorded in summary table.'
displayDict['order'] += 1
bundle = mb.MetricBundle(metric,
slicer,
sqlConstraint,
metadata=metadata,
displayDict=displayDict)
bundleList.append(bundle)
for metric in standardMetrics(colmap['slewtime']):
displayDict['caption'] = '%s in seconds.' % (metric.name)
displayDict['order'] += 1
bundle = mb.MetricBundle(metric,
slicer,
sqlConstraint,
metadata=metadata,
displayDict=displayDict)
bundleList.append(bundle)
# Slew Time histogram.
slicer = slicers.OneDSlicer(sliceColName=colmap['slewtime'], binsize=2)
metric = metrics.CountMetric(col=colmap['slewtime'],
metricName='Slew Time Histogram')
metadata = 'All visits'
plotDict = {'logScale': True, 'ylabel': 'Count'}
displayDict[
'caption'] = 'Histogram of slew times (seconds) for all visits.'
displayDict['order'] += 1
bundle = mb.MetricBundle(metric,
slicer,
sqlConstraint,
metadata=metadata,
plotDict=plotDict,
displayDict=displayDict)
bundleList.append(bundle)
# Slew distance histogram, if available.
if colmap['slewdist'] is not None:
slicer = slicers.OneDSlicer(sliceColName=colmap['slewdist'])
metric = metrics.CountMetric(col=colmap['slewdist'],
metricName='Slew Distance Histogram')
plotDict = {'logScale': True, 'ylabel': 'Count'}
displayDict[
'caption'] = 'Histogram of slew distances (angle) for all visits.'
displayDict['order'] += 1
bundle = mb.MetricBundle(metric,
slicer,
sqlConstraint,
metadata=metadata,
plotDict=plotDict,
displayDict=displayDict)
bundleList.append(bundle)
# Set the runName for all bundles and return the bundleDict.
for b in bundleList:
b.setRunName(runName)
return mb.makeBundlesDictFromList(bundleList)
def discoveryBatch(slicer, colmap=None, runName='opsim', detectionLosses='detection', metadata='',
albedo=None, Hmark=None, npReduce=np.mean, times=None, constraint=None):
if colmap is None:
colmap = ColMapDict('opsimV4')
bundleList = []
plotBundles = []
basicPlotDict = {'albedo': albedo, 'Hmark': Hmark, 'npReduce': npReduce,
'nxbins': 200, 'nybins': 200}
plotFuncs = [plots.MetricVsH()]
displayDict ={'group': 'Discovery'}
if detectionLosses not in ('detection', 'trailing'):
raise ValueError('Please choose detection or trailing as options for detectionLosses.')
if detectionLosses == 'trailing':
# These are the SNR-losses only.
magStacker = stackers.MoMagStacker(lossCol='dmagTrail')
detectionLosses = ' trailing loss'
else:
# This is SNR losses, plus additional loss due to detecting with stellar PSF.
magStacker = stackers.MoMagStacker(lossCol='dmagDetect')
detectionLosses = ' detection loss'
if times is None:
try:
timestep = 30
times = np.arange(slicer.obs[colmap['mjd']].min(), slicer.obs[colmap['mjd']].max() + timestep/2,
timestep)
except AttributeError:
raise warnings.warn('Cannot set times for completeness summary metrics. Will set up bundles, '
'but without summary metrics.')
if Hmark is None:
Hval = slicer.Hrange.mean()
else:
Hval = Hmark
# Set up the summary metrics.
if times is not None:
summaryTimeMetrics = summaryCompletenessAtTime(times, Hval=Hval, Hindex=0.33)
else:
summaryTimeMetrics = None
summaryHMetrics = summaryCompletenessOverH(requiredChances=1, Hindex=0.33)
# Set up a dictionary to pass to each metric for the column names.
colkwargs = {'mjdCol': colmap['mjd'], 'seeingCol': colmap['seeingGeom'],
'expTimeCol': colmap['exptime'], 'm5Col': colmap['fiveSigmaDepth'],
'nightCol': colmap['night'], 'filterCol': colmap['filter']}
def _setup_child_metrics(parentMetric):
childMetrics = {}
childMetrics['Time'] = metrics.Discovery_TimeMetric(parentMetric, **colkwargs)
childMetrics['N_Chances'] = metrics.Discovery_N_ChancesMetric(parentMetric, **colkwargs)
# Could expand to add N_chances per year, but not really necessary.
return childMetrics
def _configure_child_bundles(parentBundle):
dispDict = {'group': 'Discovery', 'subgroup': 'Time',
'caption': 'Time of discovery of objects', 'order': 0}
parentBundle.childBundles['Time'].setDisplayDict(dispDict)
parentBundle.childBundles['Time'].setSummaryMetrics(summaryTimeMetrics)
dispDict = {'group': 'Discovery', 'subgroup': 'NChances',
'caption': 'Number of chances for discovery of objects', 'order': 0}
parentBundle.childBundles['N_Chances'].setDisplayDict(dispDict)
parentBundle.childBundles['N_Chances'].setSummaryMetrics(summaryHMetrics)
return
# First standard SNR / probabilistic visibility (SNR~5)
# 3 pairs in 15
md = metadata + ' 3 pairs in 15 nights' + detectionLosses
# Set up plot dict.
plotDict = {'title': '%s: %s' % (runName, md)}
plotDict.update(basicPlotDict)
metric = metrics.DiscoveryMetric(nObsPerNight=2, tMin=0, tMax=90./60./24.,
nNightsPerWindow=3, tWindow=15, **colkwargs)
childMetrics = _setup_child_metrics(metric)
bundle = mb.MoMetricBundle(metric, slicer, constraint,
stackerList=[magStacker],
runName=runName, metadata=md,
childMetrics=childMetrics,
plotDict=plotDict, plotFuncs=plotFuncs,
displayDict=displayDict)
_configure_child_bundles(bundle)
bundleList.append(bundle)
# 3 pairs in 12
md = metadata + ' 3 pairs in 12 nights' + detectionLosses
plotDict = {'title': '%s: %s' % (runName, md)}
plotDict.update(basicPlotDict)
metric = metrics.DiscoveryMetric(nObsPerNight=2, tMin=0, tMax=90. / 60. / 24.,
nNightsPerWindow=3, tWindow=12, **colkwargs)
childMetrics = _setup_child_metrics(metric)
bundle = mb.MoMetricBundle(metric, slicer, constraint,
stackerList=[magStacker],
runName=runName, metadata=md,
childMetrics=childMetrics,
plotDict=plotDict, plotFuncs=plotFuncs,
displayDict=displayDict)
_configure_child_bundles(bundle)
bundleList.append(bundle)
# 3 pairs in 20
md = metadata + ' 3 pairs in 20 nights' + detectionLosses
plotDict = {'title': '%s: %s' % (runName, md)}
plotDict.update(basicPlotDict)
metric = metrics.DiscoveryMetric(nObsPerNight=2, tMin=0, tMax=90. / 60. / 24.,
nNightsPerWindow=3, tWindow=20, **colkwargs)
childMetrics = _setup_child_metrics(metric)
bundle = mb.MoMetricBundle(metric, slicer, constraint,
stackerList=[magStacker],
runName=runName, metadata=md,
childMetrics=childMetrics,
plotDict=plotDict, plotFuncs=plotFuncs,
displayDict=displayDict)
_configure_child_bundles(bundle)
bundleList.append(bundle)
# 3 pairs in 25
md = metadata + ' 3 pairs in 25 nights' + detectionLosses
plotDict = {'title': '%s: %s' % (runName, md)}
plotDict.update(basicPlotDict)
metric = metrics.DiscoveryMetric(nObsPerNight=2, tMin=0, tMax=90. / 60. / 24.,
nNightsPerWindow=3, tWindow=25, **colkwargs)
childMetrics = _setup_child_metrics(metric)
bundle = mb.MoMetricBundle(metric, slicer, constraint,
stackerList=[magStacker],
runName=runName, metadata=md,
childMetrics=childMetrics,
plotDict=plotDict, plotFuncs=plotFuncs,
displayDict=displayDict)
_configure_child_bundles(bundle)
bundleList.append(bundle)
# 3 pairs in 30
md = metadata + ' 3 pairs in 30 nights' + detectionLosses
plotDict = {'title': '%s: %s' % (runName, md)}
plotDict.update(basicPlotDict)
metric = metrics.DiscoveryMetric(nObsPerNight=2, tMin=0, tMax=90. / 60. / 24.,
nNightsPerWindow=3, tWindow=30, **colkwargs)
childMetrics = _setup_child_metrics(metric)
bundle = mb.MoMetricBundle(metric, slicer, constraint,
stackerList=[magStacker],
runName=runName, metadata=md,
childMetrics=childMetrics,
plotDict=plotDict, plotFuncs=plotFuncs,
displayDict=displayDict)
_configure_child_bundles(bundle)
bundleList.append(bundle)
# 4 pairs in 20
md = metadata + ' 4 pairs in 20 nights' + detectionLosses
plotDict = {'title': '%s: %s' % (runName, md)}
plotDict.update(basicPlotDict)
metric = metrics.DiscoveryMetric(nObsPerNight=2, tMin=0, tMax=90. / 60. / 24.,
nNightsPerWindow=4, tWindow=20, **colkwargs)
childMetrics = _setup_child_metrics(metric)
bundle = mb.MoMetricBundle(metric, slicer, constraint,
stackerList=[magStacker],
runName=runName, metadata=md,
childMetrics=childMetrics,
plotDict=plotDict, plotFuncs=plotFuncs,
displayDict=displayDict)
_configure_child_bundles(bundle)
bundleList.append(bundle)
# 3 triplets in 30
md = metadata + ' 3 triplets in 30 nights' + detectionLosses
plotDict = {'title': '%s: %s' % (runName, md)}
plotDict.update(basicPlotDict)
metric = metrics.DiscoveryMetric(nObsPerNight=3, tMin=0, tMax=120. / 60. / 24.,
nNightsPerWindow=3, tWindow=30, **colkwargs)
childMetrics = _setup_child_metrics(metric)
bundle = mb.MoMetricBundle(metric, slicer, constraint,
stackerList=[magStacker],
runName=runName, metadata=md,
childMetrics=childMetrics,
plotDict=plotDict, plotFuncs=plotFuncs,
displayDict=displayDict)
_configure_child_bundles(bundle)
bundleList.append(bundle)
# 3 quads in 30
md = metadata + ' 3 quads in 30 nights' + detectionLosses
plotDict = {'title': '%s: %s' % (runName, md)}
plotDict.update(basicPlotDict)
metric = metrics.DiscoveryMetric(nObsPerNight=4, tMin=0, tMax=150. / 60. / 24.,
nNightsPerWindow=3, tWindow=30, **colkwargs)
childMetrics = _setup_child_metrics(metric)
bundle = mb.MoMetricBundle(metric, slicer, constraint,
stackerList=[magStacker],
runName=runName, metadata=md,
childMetrics=childMetrics,
plotDict=plotDict, plotFuncs=plotFuncs,
displayDict=displayDict)
_configure_child_bundles(bundle)
bundleList.append(bundle)
# Play with SNR. SNR=4. Normal detection losses.
# 3 pairs in 15
md = metadata + ' 3 pairs in 15 nights SNR=4' + detectionLosses
plotDict = {'title': '%s: %s' % (runName, md)}
plotDict.update(basicPlotDict)
metric = metrics.DiscoveryMetric(nObsPerNight=2, tMin=0, tMax=90. / 60. / 24.,
nNightsPerWindow=3, tWindow=15, snrLimit=4, **colkwargs)
childMetrics = _setup_child_metrics(metric)
bundle = mb.MoMetricBundle(metric, slicer, constraint,
stackerList=[magStacker],
runName=runName, metadata=md,
childMetrics=childMetrics,
plotDict=plotDict, plotFuncs=plotFuncs,
displayDict=displayDict)
_configure_child_bundles(bundle)
bundleList.append(bundle)
# 3 pairs in 30, SNR=4
md = metadata + ' 3 pairs in 30 nights SNR=4' + detectionLosses
plotDict = {'title': '%s: %s' % (runName, md)}
plotDict.update(basicPlotDict)
metric = metrics.DiscoveryMetric(nObsPerNight=2, tMin=0, tMax=90. / 60. / 24.,
nNightsPerWindow=3, tWindow=30, snrLimit=4, **colkwargs)
childMetrics = _setup_child_metrics(metric)
bundle = mb.MoMetricBundle(metric, slicer, constraint,
stackerList=[magStacker],
runName=runName, metadata=md,
childMetrics=childMetrics,
plotDict=plotDict, plotFuncs=plotFuncs,
displayDict=displayDict)
_configure_child_bundles(bundle)
bundleList.append(bundle)
# Play with SNR. SNR=3
# 3 pairs in 15, SNR=3
md = metadata + ' 3 pairs in 15 nights SNR=3' + detectionLosses
plotDict = {'title': '%s: %s' % (runName, md)}
plotDict.update(basicPlotDict)
metric = metrics.DiscoveryMetric(nObsPerNight=2, tMin=0, tMax=90. / 60. / 24.,
nNightsPerWindow=3, tWindow=15, snrLimit=3, **colkwargs)
childMetrics = _setup_child_metrics(metric)
bundle = mb.MoMetricBundle(metric, slicer, constraint,
stackerList=[magStacker],
runName=runName, metadata=md,
childMetrics=childMetrics,
plotDict=plotDict, plotFuncs=plotFuncs,
displayDict=displayDict)
_configure_child_bundles(bundle)
bundleList.append(bundle)
# SNR = 0
# 3 pairs in 15, SNR=0
md = metadata + ' 3 pairs in 15 nights SNR=0' + detectionLosses
plotDict = {'title': '%s: %s' % (runName, md)}
plotDict.update(basicPlotDict)
metric = metrics.DiscoveryMetric(nObsPerNight=2, tMin=0, tMax=90. / 60. / 24.,
nNightsPerWindow=3, tWindow=15, snrLimit=0, **colkwargs)
childMetrics = _setup_child_metrics(metric)
bundle = mb.MoMetricBundle(metric, slicer, constraint,
stackerList=[magStacker],
runName=runName, metadata=md,
childMetrics=childMetrics,
plotDict=plotDict, plotFuncs=plotFuncs,
displayDict=displayDict)
_configure_child_bundles(bundle)
bundleList.append(bundle)
# Play with weird strategies.
# Single detection.
md = metadata + ' Single detection' + detectionLosses
plotDict = {'title': '%s: %s' % (runName, md)}
plotDict.update(basicPlotDict)
metric = metrics.DiscoveryMetric(nObsPerNight=1, tMin=0, tMax=90. / 60. / 24.,
nNightsPerWindow=1, tWindow=5, **colkwargs)
childMetrics = _setup_child_metrics(metric)
bundle = mb.MoMetricBundle(metric, slicer, constraint,
stackerList=[magStacker],
runName=runName, metadata=md,
childMetrics=childMetrics,
plotDict=plotDict, plotFuncs=plotFuncs,
displayDict=displayDict)
_configure_child_bundles(bundle)
bundleList.append(bundle)
# Single pair of detections.
md = metadata + ' Single pair' + detectionLosses
plotDict = {'title': '%s: %s' % (runName, md)}
plotDict.update(basicPlotDict)
metric = metrics.DiscoveryMetric(nObsPerNight=2, tMin=0, tMax=90. / 60. / 24.,
nNightsPerWindow=1, tWindow=5, **colkwargs)
childMetrics = _setup_child_metrics(metric)
bundle = mb.MoMetricBundle(metric, slicer, constraint,
stackerList=[magStacker],
runName=runName, metadata=md,
childMetrics=childMetrics,
plotDict=plotDict, plotFuncs=plotFuncs,
displayDict=displayDict)
_configure_child_bundles(bundle)
bundleList.append(bundle)
# High velocity discovery.
displayDict['subgroup'] = 'High Velocity'
# High velocity.
md = metadata + ' High velocity pair' + detectionLosses
plotDict = {'title': '%s: %s' % (runName, md)}
plotDict.update(basicPlotDict)
metric = metrics.HighVelocityNightsMetric(psfFactor=2., nObsPerNight=2, **colkwargs)
bundle = mb.MoMetricBundle(metric, slicer, constraint,
stackerList=[magStacker],
runName=runName, metadata=md,
plotDict=plotDict, plotFuncs=plotFuncs,
summaryMetrics=summaryTimeMetrics,
displayDict=displayDict)
bundleList.append(bundle)
# "magic" detection - 6 in 60 days (at SNR=5).
md = metadata + ' 6 detections in 60 nights' + detectionLosses
plotDict = {'title': '%s: %s' % (runName, md)}
plotDict.update(basicPlotDict)
metric = metrics.MagicDiscoveryMetric(nObs=6, tWindow=60, **colkwargs)
bundle = mb.MoMetricBundle(metric, slicer, constraint,
stackerList=[magStacker],
runName=runName, metadata=md,
plotDict=plotDict, plotFuncs=plotFuncs,
summaryMetrics=summaryHMetrics,
displayDict=displayDict)
bundleList.append(bundle)
# Set the runName for all bundles and return the bundleDict.
for b in bundleList:
b.setRunName(runName)
return mb.makeBundlesDictFromList(bundleList), plotBundles
def hourglassPlots(colmap=None,
runName='opsim',
nyears=10,
extraSql=None,
extraMetadata=None):
"""Run the hourglass metric, for each individual year.
Parameters
----------
colmap : dict, opt
A dictionary with a mapping of column names. Default will use OpsimV4 column names.
run_name : str, opt
The name of the simulated survey. Default is "opsim".
nyears : int (10), opt
How many years to attempt to make hourglass plots for. Default is 10.
extraSql : str, opt
Add an extra sql constraint before running metrics. Default None.
extraMetadata : str, opt
Add an extra piece of metadata before running metrics. Default None.
"""
if colmap is None:
colmap = ColMapDict('opsimV4')
bundleList = []
sql = ''
metadata = ''
# Add additional sql constraint (such as wfdWhere) and metadata, if provided.
if (extraSql is not None) and (len(extraSql) > 0):
sql = extraSql
if extraMetadata is None:
metadata = extraSql.replace('filter =', '').replace('filter=', '')
metadata = metadata.replace('"', '').replace("'", '')
if extraMetadata is not None:
metadata = extraMetadata
years = list(range(nyears + 1))
displayDict = {'group': 'Hourglass'}
for year in years[1:]:
displayDict['subgroup'] = 'Year %d' % year
displayDict['caption'] = 'Visualization of the filter usage of the telescope. ' \
'The black wavy line indicates lunar phase; the red and blue ' \
'solid lines indicate nautical and civil twilight.'
sqlconstraint = 'night > %i and night <= %i' % (365.25 * (year - 1),
365.25 * year)
if len(sql) > 0:
sqlconstraint = '(%s) and (%s)' % (sqlconstraint, sql)
md = metadata + ' year %i-%i' % (year - 1, year)
slicer = slicers.HourglassSlicer()
metric = metrics.HourglassMetric(nightCol=colmap['night'],
mjdCol=colmap['mjd'],
metricName='Hourglass')
bundle = mb.MetricBundle(metric,
slicer,
constraint=sqlconstraint,
metadata=md,
displayDict=displayDict)
bundleList.append(bundle)
# Set the runName for all bundles and return the bundleDict.
for b in bundleList:
b.setRunName(runName)
return mb.makeBundlesDictFromList(bundleList)
def nvisitsPerProp(opsdb, colmap=None, runName='opsim', binNights=1, extraSql=None):
"""Set up a group of all and per-proposal nvisits metrics.
Parameters
----------
opsdb : lsst.sims.maf.db.Database or lsst.sims.maf.db.OpsimDatabase* object
colmap : dict or None, opt
A dictionary with a mapping of column names. Default will use OpsimV4 column names.
runName : str, opt
The name of the simulated survey. Default is "opsim".
binNights : int, opt
Number of nights to count in each bin. Default = 1, count number of visits in each night.
sqlConstraint : str or None, opt
SQL constraint to add to all metrics.
Returns
-------
metricBundle
"""
if colmap is None:
colmap = getColMap(opsdb)
propids, proptags = opsdb.fetchPropInfo()
bdict = {}
bundleList = []
totvisits = opsdb.fetchNVisits()
metadata = 'All props'
if extraSql is not None and len(extraSql) > 0:
metadata += ' %s' % extraSql
# Nvisits per night, all proposals.
bdict.update(nvisitsPerNight(colmap=colmap, runName=runName, binNights=binNights,
extraSql=extraSql, extraMetadata=metadata, subgroup='All proposals'))
# Nvisits total, all proposals.
metric = metrics.CountMetric(colmap['mjd'], metricName='Nvisits')
slicer = slicers.UniSlicer()
summaryMetrics = [metrics.IdentityMetric(metricName='Count'),
metrics.NormalizeMetric(normVal=totvisits, metricName='Fraction of total')]
displayDict = {'group': 'Nvisit Summary', 'subgroup': 'Proposal distribution', 'order': -1}
displayDict['caption'] = 'Total number of visits for all proposals.'
if extraSql is not None and len(extraSql) > 0:
displayDict['caption'] += ' (with constraint %s.)' % extraSql
bundle = mb.MetricBundle(metric, slicer, extraSql, metadata=metadata,
displayDict=displayDict, summaryMetrics=summaryMetrics)
bundleList.append(bundle)
# Look for any multi-proposal groups that we should include.
for tag in proptags:
if len(proptags[tag]) > 1 or tag in ('WFD', 'DD'):
pids = proptags[tag]
sql = '('
for pid in pids[:-1]:
sql += '%s=%d or ' % (colmap['proposalId'], pid)
sql += ' %s=%d)' % (colmap['proposalId'], pids[-1])
metadata = '%s' % tag
if extraSql is not None:
sql = '(%s) and (%s)' % (sql, extraSql)
metadata += ' %s' % (extraSql)
bdict.update(nvisitsPerNight(colmap=colmap, runName=runName, binNights=binNights,
extraSql=sql, extraMetadata=metadata, subgroup=tag))
displayDict['order'] += 1
displayDict['caption'] = 'Number of visits and fraction of total visits, for %s.' % metadata
bundle = mb.MetricBundle(metric, slicer, sql, metadata=metadata,
summaryMetrics=summaryMetrics, displayDict=displayDict)
bundleList.append(bundle)
# And each proposal separately.
for propid in propids:
sql = '%s=%d' % (colmap['proposalId'], propid)
metadata = '%s' % (propids[propid])
if extraSql is not None:
sql += ' and (%s)' % (extraSql)
metadata += ' %s' % extraSql
bdict.update(nvisitsPerNight(colmap=colmap, runName=runName, binNights=binNights,
extraSql=sql, extraMetadata=metadata, subgroup='Per proposal'))
displayDict['order'] += 1
displayDict['caption'] = 'Number of visits and fraction of total visits, for %s.' % metadata
bundle = mb.MetricBundle(metric, slicer, constraint=sql, metadata=metadata,
summaryMetrics=summaryMetrics, displayDict=displayDict)
bundleList.append(bundle)
for b in bundleList:
b.setRunName(runName)
bdict.update(mb.makeBundlesDictFromList(bundleList))
return bdict
def runSlices(opsimName, metadata, simdata, fields, bins, args, opsDb, verbose=False):
# Set up the movie slicer.
movieslicer = setupMovieSlicer(simdata, bins)
# Set up formatting for output suffix.
sliceformat = '%s0%dd' %('%', int(np.log10(len(movieslicer)))+1)
# Get the telescope latitude info.
lat_tele = Site(name='LSST').latitude_rad
# Run through the movie slicer slicePoints and generate plots at each point.
for i, ms in enumerate(movieslicer):
t = time.time()
slicenumber = sliceformat %(i)
if verbose:
print(slicenumber)
# Set up metrics.
if args.movieStepsize != 0:
tstep = args.movieStepsize
else:
tstep = ms['slicePoint']['binRight'] - bins[i]
if tstep > 1:
tstep = 40./24./60./60.
# Add simple view of time to plot label.
times_from_start = ms['slicePoint']['binRight'] - (int(bins[0]) + 0.16 - 0.5)
# Opsim years are 365 days (not 365.25)
years = int(times_from_start/365)
days = times_from_start - years*365
plotlabel = 'Year %d Day %.4f' %(years, days)
# Set up metrics.
metricList, plotDictList = setupMetrics(opsimName, metadata, plotlabel=plotlabel,
t0=ms['slicePoint']['binRight'], tStep=tstep,
years=years, verbose=verbose)
# Identify the subset of simdata in the movieslicer 'data slice'
simdatasubset = simdata[ms['idxs']]
# Set up opsim slicer on subset of simdata provided by movieslicer
opslicer = slicers.OpsimFieldSlicer()
# Set up metricBundles to combine metrics, plotdicts and slicer.
bundles = []
sqlconstraint = ''
for metric, plotDict in zip(metricList, plotDictList):
bundles.append(metricBundles.MetricBundle(metric, opslicer, constraint=sqlconstraint,
metadata=metadata, runName=opsimName,
plotDict=plotDict))
# Remove (default) stackers from bundles, because we've already run them above on the original data.
for mb in bundles:
mb.stackerList = []
bundledict = metricBundles.makeBundlesDictFromList(bundles)
# Set up metricBundleGroup to handle metrics calculation + plotting
bg = metricBundles.MetricBundleGroup(bundledict, opsDb, outDir=args.outDir,
resultsDb=None, saveEarly=False)
# 'Hack' bundleGroup to just go ahead and run the metrics, without querying the database.
simData = simdatasubset
bg.fieldData = fields
bg.setCurrent(sqlconstraint)
bg.runCurrent(constraint = sqlconstraint, simData=simData)
# Plot data each metric, for this slice of the movie, adding slicenumber as a suffix for output plots.
# Plotting here, rather than automatically via sliceMetric method because we're going to rotate the sky,
# and add extra legend info and figure text (for FilterColors metric).
ph = plots.PlotHandler(outDir=args.outDir, figformat='png', dpi=72, thumbnail=False, savefig=False)
obsnow = np.where(simdatasubset['observationStartMJD'] ==
simdatasubset['observationStartMJD'].max())[0]
raCen = np.radians(np.mean(simdatasubset[obsnow]['observationStartLST']))
# Calculate horizon location.
horizonlon, horizonlat = addHorizon(lat_telescope=lat_tele)
# Create the plot for each metric and save it (after some additional manipulation).
for mb in bundles:
ph.setMetricBundles([mb])
fignum = ph.plot(plotFunc=plots.BaseSkyMap(), plotDicts={'raCen':raCen})
fig = plt.figure(fignum)
ax = plt.gca()
# Add horizon and zenith.
plt.plot(horizonlon, horizonlat, 'k.', alpha=0.3, markersize=1.8)
plt.plot(0, lat_tele, 'k+')
# For the FilterColors metric, add some extra items.
if mb.metric.name == 'FilterColors':
# Add the time stamp info (plotlabel) with a fancybox.
plt.figtext(0.75, 0.9, '%s' %(plotlabel), bbox=dict(boxstyle='Round, pad=0.7',
fc='w', ec='k', alpha=0.5))
# Add a legend for the filters.
filterstacker = stackers.FilterColorStacker()
for i, f in enumerate(['u', 'g', 'r', 'i', 'z', 'y']):
plt.figtext(0.92, 0.55 - i*0.035, f, color=filterstacker.filter_rgb_map[f])
# Add a moon.
moonRA = np.radians(np.mean(simdatasubset[obsnow]['moonRA']))
lon = -(moonRA - raCen - np.pi) % (np.pi*2) - np.pi
moonDec = np.radians(np.mean(simdatasubset[obsnow]['moonDec']))
# Note that moonphase is 0-100 (translate to 0-1). 0=new.
moonPhase = np.mean(simdatasubset[obsnow]['moonPhase'])/100.
alpha = np.max([moonPhase, 0.15])
circle = Circle((lon, moonDec), radius=0.05, color='k', alpha=alpha)
ax.add_patch(circle)
# Add some explanatory text.
ecliptic = Line2D([], [], color='r', label="Ecliptic plane")
galaxy = Line2D([], [], color='b', label="Galactic plane")
horizon = Line2D([], [], color='k', alpha=0.3, label="20 deg elevation limit")
moon = Line2D([], [], color='k', linestyle='', marker='o', markersize=8, alpha=alpha,
label="\nMoon (Dark=Full)\n (Light=New)")
zenith = Line2D([], [], color='k', linestyle='', marker='+', markersize=5, label="Zenith")
plt.legend(handles=[horizon, zenith, galaxy, ecliptic, moon], loc=[0.1, -0.35],
ncol=3, frameon=False,
title = 'Aitoff plot showing HA/Dec of simulated survey pointings',
numpoints=1, fontsize='small')
# Save figure.
plt.savefig(os.path.join(args.outDir, mb.metric.name + '_' + slicenumber + '_SkyMap.png'),
format='png', dpi=72)
plt.close('all')
dt, t = dtime(t)
if verbose:
print('Ran and plotted slice %s of movieslicer in %f s' %(slicenumber, dt))
def seasons(colmap=None, runName='opsim', nside=64, extraSql=None, extraMetadata=None,
ditherStacker=None, ditherkwargs=None):
"""Generate a set of statistics about the length and number of seasons.
Parameters
----------
colmap : dict or None, opt
A dictionary with a mapping of column names. Default will use OpsimV4 column names.
runName : str, opt
The name of the simulated survey. Default is "opsim".
nside : int, opt
Nside for the healpix slicer. Default 64.
extraSql : str or None, opt
Additional sql constraint to apply to all metrics.
extraMetadata : str or None, opt
Additional metadata to use for all outputs.
ditherStacker: str or lsst.sims.maf.stackers.BaseDitherStacker
Optional dither stacker to use to define ra/dec columns.
ditherkwargs: dict, opt
Optional dictionary of kwargs for the dither stacker.
Returns
-------
metricBundleDict
"""
if colmap is None:
colmap = ColMapDict('opsimV4')
bundleList = []
# Set up basic all and per filter sql constraints.
raCol, decCol, degrees, ditherStacker, ditherMeta = radecCols(ditherStacker, colmap, ditherkwargs)
metadata = combineMetadata(extraMetadata, ditherMeta)
filterlist, colors, orders, sqls, metadata = filterList(all=True,
extraSql=extraSql,
extraMetadata=metadata)
seasonStacker = stackers.SeasonStacker(mjdCol=colmap['mjd'], RACol=raCol,
degrees=degrees)
stackerList = [seasonStacker]
if ditherStacker is not None:
stackerList.append(ditherStacker)
slicer = slicers.HealpixSlicer(nside=nside, latCol=decCol, lonCol=raCol, latLonDeg=degrees)
displayDict = {'group': 'IntraSeason', 'subgroup': 'Season length', 'caption': None, 'order': 0}
# Histogram of the length of seasons.
"""
bins = np.arange(1, 20.5, 1)
metric = metrics.NightgapsMetric(bins=bins, nightCol=colmap['night'], metricName='DeltaNight Histogram')
plotDict = {'bins': bins, 'xlabel': 'dT (nights)'}
displayDict['caption'] = 'Histogram of the number of nights between consecutive visits to a ' \
'given point on the sky, considering separations between %d and %d' \
% (bins.min(), bins.max())
if metadata['all'] is None or len(metadata['all']) == 0:
displayDict['caption'] += ', all proposals.'
else:
displayDict['caption'] += ', %s.' % metadata['all']
plotFunc = plots.SummaryHistogram()
bundle = mb.MetricBundle(metric, slicer, sqls['all'], plotDict=plotDict,
displayDict=displayDict, metadata=metadata['all'], plotFuncs=[plotFunc])
bundleList.append(bundle)
"""
standardStats = standardSummary()
subsetPlots = [plots.HealpixSkyMap(), plots.HealpixHistogram()]
# Median inter-night gap (each and all filters)
metric = metrics.SeasonLengthMetric(metricName='Median Season Length', mjdCol=colmap['mjd'],
seasonCol='season', reduceFunc=np.median)
for f in filterlist:
displayDict['caption'] = 'Median season length, %s.' % metadata[f]
displayDict['order'] = orders[f]
maxS = 250
if f == 'all':
minS = 90
else:
minS = 30
plotDict = {'color': colors[f], 'colorMin': minS, 'colorMax': maxS, 'xMin': minS, 'xMax': maxS}
bundle = mb.MetricBundle(metric, slicer, sqls[f], metadata=metadata[f],
stackerList=stackerList, displayDict=displayDict,
plotFuncs=subsetPlots, plotDict=plotDict,
summaryMetrics=standardStats)
bundleList.append(bundle)
# Number of seasons?
# Set the runName for all bundles and return the bundleDict.
for b in bundleList:
b.setRunName(runName)
plotBundles = None
return mb.makeBundlesDictFromList(bundleList), plotBundles
def slewBasics(colmap=None, runName='opsim', sqlConstraint=None):
"""Generate a simple set of statistics about the slew times and distances.
These slew statistics can be run on the summary or default tables.
Parameters
----------
colmap : dict or None, opt
A dictionary with a mapping of column names. Default will use OpsimV4 column names.
runName : str, opt
The name of the simulated survey. Default is "opsim".
sqlConstraint : str or None, opt
SQL constraint to add to metrics. (note this runs on summary table).
Returns
-------
metricBundleDict
"""
if colmap is None:
colmap = ColMapDict('opsimV4')
bundleList = []
# Calculate basic stats on slew times. (mean/median/min/max + total).
slicer = slicers.UniSlicer()
metadata = 'All visits'
if sqlConstraint is not None and len(sqlConstraint) > 0:
metadata = '%s' % (sqlConstraint)
displayDict = {'group': 'Slew', 'subgroup': 'Slew Basics', 'order': -1, 'caption': None}
# Add total number of slews.
metric = metrics.CountMetric(colmap['slewtime'], metricName='Slew Count')
displayDict['caption'] = 'Total number of slews recorded in summary table.'
displayDict['order'] += 1
bundle = mb.MetricBundle(metric, slicer, sqlConstraint, metadata=metadata, displayDict=displayDict)
bundleList.append(bundle)
for metric in standardMetrics(colmap['slewtime']):
displayDict['caption'] = '%s in seconds.' % (metric.name)
displayDict['order'] += 1
bundle = mb.MetricBundle(metric, slicer, sqlConstraint, metadata=metadata, displayDict=displayDict)
bundleList.append(bundle)
# Slew Time histogram.
slicer = slicers.OneDSlicer(sliceColName=colmap['slewtime'], binsize=2)
metric = metrics.CountMetric(col=colmap['slewtime'], metricName='Slew Time Histogram')
metadata = 'All visits'
plotDict = {'logScale': True, 'ylabel': 'Count'}
displayDict['caption'] = 'Histogram of slew times (seconds) for all visits.'
displayDict['order'] += 1
bundle = mb.MetricBundle(metric, slicer, sqlConstraint, metadata=metadata,
plotDict=plotDict, displayDict=displayDict)
bundleList.append(bundle)
# Zoom in on slew time histogram near 0.
slicer = slicers.OneDSlicer(sliceColName=colmap['slewtime'], binsize=0.2, binMin=0, binMax=20)
metric = metrics.CountMetric(col=colmap['slewtime'], metricName='Zoom Slew Time Histogram')
metadata = 'All visits'
plotDict = {'logScale': True, 'ylabel': 'Count'}
displayDict['caption'] = 'Histogram of slew times (seconds) for all visits (zoom).'
displayDict['order'] += 1
bundle = mb.MetricBundle(metric, slicer, sqlConstraint, metadata=metadata,
plotDict=plotDict, displayDict=displayDict)
bundleList.append(bundle)
# Slew distance histogram, if available.
if colmap['slewdist'] is not None:
binsize = 2.0
if not colmap['raDecDeg']:
binsize = np.radians(binsize)
slicer = slicers.OneDSlicer(sliceColName=colmap['slewdist'], binsize=binsize)
metric = metrics.CountMetric(col=colmap['slewdist'], metricName='Slew Distance Histogram')
plotDict = {'logScale': True, 'ylabel': 'Count'}
displayDict['caption'] = 'Histogram of slew distances (angle) for all visits.'
displayDict['order'] += 1
bundle = mb.MetricBundle(metric, slicer, sqlConstraint, metadata=metadata,
plotDict=plotDict, displayDict=displayDict)
bundleList.append(bundle)
# Zoom on slew distance histogram.
binMax = 20.0
if not colmap['raDecDeg']:
binMax = np.radians(binMax)
slicer = slicers.OneDSlicer(sliceColName=colmap['slewdist'], binsize=binsize/10.,
binMin=0, binMax=binMax)
metric = metrics.CountMetric(col=colmap['slewdist'], metricName='Zoom Slew Distance Histogram')
plotDict = {'logScale': True, 'ylabel': 'Count'}
displayDict['caption'] = 'Histogram of slew distances (angle) for all visits.'
displayDict['order'] += 1
bundle = mb.MetricBundle(metric, slicer, sqlConstraint, metadata=metadata,
plotDict=plotDict, displayDict=displayDict)
bundleList.append(bundle)
# Set the runName for all bundles and return the bundleDict.
for b in bundleList:
b.setRunName(runName)
return mb.makeBundlesDictFromList(bundleList)
def nvisitsM5Maps(colmap=None, runName='opsim',
extraSql=None, extraMetadata=None,
nside=64, runLength=10.,
ditherStacker=None, ditherkwargs=None):
"""Generate number of visits and Coadded depth per RA/Dec point in all and per filters.
Parameters
----------
colmap : dict, opt
A dictionary with a mapping of column names. Default will use OpsimV4 column names.
runName : str, opt
The name of the simulated survey. Default is "opsim".
extraSql : str, opt
Additional constraint to add to any sql constraints (e.g. 'propId=1' or 'fieldID=522').
Default None, for no additional constraints.
extraMetadata : str, opt
Additional metadata to add before any below (i.e. "WFD"). Default is None.
nside : int, opt
Nside value for healpix slicer. Default 64.
If "None" is passed, the healpixslicer-based metrics will be skipped.
runLength : float, opt
Length of the simulated survey, for scaling values for the plot limits.
Default 10.
ditherStacker: str or lsst.sims.maf.stackers.BaseDitherStacker
Optional dither stacker to use to define ra/dec columns.
ditherkwargs: dict, opt
Optional dictionary of kwargs for the dither stacker.
Returns
-------
metricBundleDict
"""
if colmap is None:
colmap = ColMapDict('opsimV4')
bundleList = []
subgroup = extraMetadata
if subgroup is None:
subgroup = 'All visits'
raCol, decCol, degrees, ditherStacker, ditherMeta = radecCols(ditherStacker, colmap, ditherkwargs)
extraMetadata = combineMetadata(extraMetadata, ditherMeta)
# Set up basic all and per filter sql constraints.
filterlist, colors, orders, sqls, metadata = filterList(all=True,
extraSql=extraSql,
extraMetadata=extraMetadata)
# Set up some values to make nicer looking plots.
benchmarkVals = mafUtils.scaleBenchmarks(runLength, benchmark='design')
# Check that nvisits is not set to zero (for very short run length).
for f in benchmarkVals['nvisits']:
if benchmarkVals['nvisits'][f] == 0:
print('Updating benchmark nvisits value in %s to be nonzero' % (f))
benchmarkVals['nvisits'][f] = 1
benchmarkVals['coaddedDepth'] = mafUtils.calcCoaddedDepth(benchmarkVals['nvisits'],
benchmarkVals['singleVisitDepth'])
# Scale the nvisit ranges for the runLength.
nvisitsRange = {'u': [20, 80], 'g': [50, 150], 'r': [100, 250],
'i': [100, 250], 'z': [100, 300], 'y': [100, 300], 'all': [700, 1200]}
scale = runLength / 10.0
for f in nvisitsRange:
for i in [0, 1]:
nvisitsRange[f][i] = int(np.floor(nvisitsRange[f][i] * scale))
# Generate Nvisit maps in all and per filters
displayDict = {'group': 'Nvisits Maps', 'subgroup': subgroup}
metric = metrics.CountMetric(colmap['mjd'], metricName='NVisits', units='')
slicer = slicers.HealpixSlicer(nside=nside, latCol=decCol, lonCol=raCol,
latLonDeg=degrees)
for f in filterlist:
sql = sqls[f]
displayDict['caption'] = 'Number of visits per healpix in %s.' % metadata[f]
displayDict['order'] = orders[f]
binsize = 2
if f == 'all':
binsize = 5
plotDict = {'xMin': nvisitsRange[f][0], 'xMax': nvisitsRange[f][1],
'colorMin': nvisitsRange[f][0], 'colorMax': nvisitsRange[f][1],
'binsize': binsize, 'color': colors[f]}
bundle = mb.MetricBundle(metric, slicer, sql, metadata=metadata[f],
stackerList=ditherStacker,
displayDict=displayDict, plotDict=plotDict,
summaryMetrics=standardSummary())
bundleList.append(bundle)
# Generate Coadded depth maps per filter
displayDict = {'group': 'Coadded M5 Maps', 'subgroup': subgroup}
metric = metrics.Coaddm5Metric(m5Col=colmap['fiveSigmaDepth'], metricName='CoaddM5')
slicer = slicers.HealpixSlicer(nside=nside, latCol=decCol, lonCol=raCol,
latLonDeg=degrees)
for f in filterlist:
# Skip "all" for coadded depth.
if f == 'all':
continue
mag_zp = benchmarkVals['coaddedDepth'][f]
sql = sqls[f]
displayDict['caption'] = 'Coadded depth per healpix, with %s benchmark value subtracted (%.1f) ' \
'in %s.' % (f, mag_zp, metadata[f])
displayDict['caption'] += ' More positive numbers indicate fainter limiting magnitudes.'
displayDict['order'] = orders[f]
plotDict = {'zp': mag_zp, 'xMin': -0.6, 'xMax': 0.6,
'xlabel': 'coadded m5 - %.1f' % mag_zp,
'colorMin': -0.6, 'colorMax': 0.6, 'color': colors[f]}
bundle = mb.MetricBundle(metric, slicer, sql, metadata=metadata[f],
stackerList=ditherStacker,
displayDict=displayDict, plotDict=plotDict,
summaryMetrics=standardSummary())
bundleList.append(bundle)
# Set the runName for all bundles and return the bundleDict.
for b in bundleList:
b.setRunName(runName)
return mb.makeBundlesDictFromList(bundleList)
bundleList.append(bundle)
metric = metrics.CountMetric(col='expMJD', metricName='Vendor2')
slicer = slicers.HealpixSlicer(nside=nside, useCamera=True, chipNames=chips2,
lonCol='ditheredRA', latCol='ditheredDec')
bundle = metricBundles.MetricBundle(metric,slicer,sql)
bundleList.append(bundle)
# Now let's try it with the new stacker
metric = metrics.CountMetric(col='expMJD', metricName='Vendor1, rotAdded')
slicer = slicers.HealpixSlicer(nside=nside, rotSkyPosColName='rotatedRotSkyPos',
useCamera=True, chipNames=chips1,
lonCol='ditheredRA', latCol='ditheredDec')
bundle = metricBundles.MetricBundle(metric,slicer,sql)
bundleList.append(bundle)
metric = metrics.CountMetric(col='expMJD', metricName='Vendor2, rotAdded')
slicer = slicers.HealpixSlicer(nside=nside, rotSkyPosColName='rotatedRotSkyPos',
useCamera=True, chipNames=chips2)
bundle = metricBundles.MetricBundle(metric,slicer,sql)
bundleList.append(bundle)
bd = metricBundles.makeBundlesDictFromList(bundleList)
bg = metricBundles.MetricBundleGroup(bd, opsdb,
outDir=outDir, resultsDb=resultsDb)
bg.runAll()
bg.plotAll()
def findUncertainties(thisFilter='r', \
nside=64, tMax=730, \
dbFil='minion_1016_sqlite.db', \
crowdError=0.2, \
seeingCol='FWHMeff', \
cleanNpz=True, \
doPlots=False, \
wrapGalacs=True, \
selectStrip=True):
#, \
# hiRes=True):
"""Catalogs the uncertainties for a given database, returns the
file path"""
# doPlots switches on plotting. This makes things quite a bit
# slower for coarse healpix, and I haven't worked out how to
# specify the output plot directory yet. Recommend default to
# False.
# plot functions
plotFuncs = [plots.HealpixSkyMap(), plots.HealpixHistogram()]
opsdb = db.OpsimDatabase(dbFil)
outDir = 'crowding_test_2017-07-25'
resultsDb = db.ResultsDb(outDir=outDir)
# slicer, etc.
slicer = slicers.HealpixSlicer(nside=nside, useCache=False)
sql = 'filter="%s" and night < %i' % (thisFilter, tMax)
plotDict={'colorMax':27.}
# initialise the entire bundle list
bundleList = []
# set up for higher-resolution spatial maps DOESN'T WORK ON LAPTOP
#if hiRes:
# mafMap = maps.StellarDensityMap(nside=128)
#else:
# mafMap = maps.StellarDensityMap(nside=64)
# if passed a single number, turn the crowdErr into a list
if str(crowdError.__class__).find('list') < 0:
crowdVals = [np.copy(crowdError)]
else:
crowdVals = crowdError[:]
# build up the bundle list. Build up a list of crowding values and
# their column names. HARDCODED for the moment, can make the input list
# an argument if desired.
crowdStem = '%sCrowd' % (thisFilter)
lCrowdCols = []
# loop through the crowding values
#crowdVals = [0.2, 0.1, 0.05]
for errCrowd in crowdVals:
crowdName = '%s%.3f' % (crowdStem,errCrowd)
lCrowdCols.append(crowdName) # to pass later
metricThis = metrics.CrowdingMetric(crowding_error=errCrowd, \
seeingCol='FWHMeff')
bundleThis = metricBundles.MetricBundle(metricThis, slicer, sql, \
plotDict=plotDict, \
fileRoot=crowdName, \
runName=crowdName, \
plotFuncs=plotFuncs)
bundleList.append(bundleThis)
#metric = metrics.CrowdingMetric(crowding_error=crowdError, \
# seeingCol=seeingCol)
#bundle = metricBundles.MetricBundle(metric,\
# slicer,sql, plotDict=plotDict, \
# plotFuncs=plotFuncs)
#bundleList.append(bundle)
# ... then the m5col
metricCoadd = metrics.Coaddm5Metric()
bundleCoadd = metricBundles.MetricBundle(metricCoadd,\
slicer,sql,plotDict=plotDict, \
plotFuncs=plotFuncs)
bundleList.append(bundleCoadd)
# Let's also pass through some useful statistics
# per-HEALPIX. We'll want to bring across the output metric names
# as well so that we can conveniently access them later.
# some convenient plot functions
statsCols = ['FWHMeff', 'fiveSigmaDepth', 'airmass']
metricNames = [ 'MedianMetric', 'RobustRmsMetric', 'MinMetric', 'MaxMetric']
statsNames = {}
sKeyTail = '_%s_and_night_lt_%i_HEAL' % (thisFilter, tMax)
# may as well get good plot dicts too...
plotDicts = {}
plotDicts['FWHMeff_MedianMetric'] = {'colorMax':2.}
plotDicts['fiveSigmaDepth_MedianMetric'] = {'colorMax':26.}
plotDicts['airmass_MedianMetric'] = {'colorMax':2.5}
plotDicts['FWHMeff_RobustRmsMetric'] = {'colorMax':1.}
plotDicts['fiveSigmaDepth_RobustRmsMetric'] = {'colorMax':2.}
plotDicts['airmass_RobustRmsMetric'] = {'colorMax':1.}
# initialize the minmax values for the moment
plotDicts['FWHMeff_MinMetric'] = {'colorMax':3}
plotDicts['FWHMeff_MaxMetric'] = {'colorMax':3}
# ensure they all have xMax as well
for sKey in plotDicts.keys():
plotDicts[sKey]['xMax'] = plotDicts[sKey]['colorMax']
for colName in statsCols:
for metricName in metricNames:
# lift out the appropriate plotdict
plotDict = {}
sDict = '%s_%s' % (colName, metricName)
if sDict in plotDicts.keys():
plotDict = plotDicts[sDict]
thisMetric = getattr(metrics, metricName)
metricObj = thisMetric(col=colName)
bundleObj = metricBundles.MetricBundle(metricObj,slicer,sql, \
plotDict=plotDict, \
plotFuncs=plotFuncs)
bundleList.append(bundleObj)
# construct the output table column name and the key for
# the bundle object
tableCol = '%s%s_%s' % (thisFilter, colName, metricName)
statsNames[tableCol] = 'opsim_%s_%s%s' \
% (metricName.split('Metric')[0], colName, sKeyTail)
# as a debug, see if this is actually working...
# print tableCol, statsNames[tableCol], statsNames[tableCol]
# try the number of visits
col2Count = 'fiveSigmaDepth'
metricN = metrics.CountMetric(col=col2Count)
bundleN = metricBundles.MetricBundle(metricN, slicer, sql, \
plotFuncs=plotFuncs)
bundleList.append(bundleN)
countCol = '%sCount' % (thisFilter)
statsNames[countCol] = 'opsim_Count_%s%s' % (col2Count, sKeyTail)
# convert to the bundledict...
bundleDict = metricBundles.makeBundlesDictFromList(bundleList)
bgroup = metricBundles.MetricBundleGroup(bundleDict, \
opsdb, outDir=outDir, \
resultsDb=resultsDb)
# ... and run...
bgroup.runAll()
# ... also plot...
if doPlots:
bgroup.plotAll()
# now produce the table for this run
nameDepth = 'opsim_CoaddM5_%s_and_night_lt_%i_HEAL' \
% (thisFilter, tMax)
nameCrowd = 'opsim_Crowding_To_Precision_%s_and_night_lt_%i_HEAL' \
% (thisFilter, tMax)
npix = bgroup.bundleDict[nameDepth].metricValues.size
nsideFound = hp.npix2nside(npix)
ra, dec = healpyUtils.hpid2RaDec(nside, np.arange(npix))
cc = SkyCoord(ra=np.copy(ra), dec=np.copy(dec), frame='fk5', unit='deg')
# boolean mask for nonzero entries
bVal = ~bgroup.bundleDict[nameDepth].metricValues.mask
# Generate the table
tVals = Table()
tVals['HEALPIX'] = np.arange(npix)
tVals['RA'] = cc.ra.degree
tVals['DE'] = cc.dec.degree
tVals['l'] = cc.galactic.l.degree
tVals['b'] = cc.galactic.b.degree
# wrap Galactics?
if wrapGalacs:
bBig = tVals['l'] > 180.
tVals['l'][bBig] -= 360.
sCoadd = '%sCoadd' % (thisFilter)
sCrowd = '%sCrowd' % (thisFilter)
tVals[sCoadd] = Column(bgroup.bundleDict[nameDepth].metricValues, \
dtype='float')
# REPLACE the single-crowding with the set of columns, like so:
#tVals[sCrowd] = Column(bgroup.bundleDict[nameCrowd].metricValues, \
# dtype='float')
for colCrowd in lCrowdCols:
tVals[colCrowd] = Column(bgroup.bundleDict[colCrowd].metricValues, \
dtype='float', format='%.3f')
# enforce rounding. Three decimal places ought to be sufficient
# for most purposes. See if the Table constructor follows this
# through. (DOESN'T SEEM TO WORK when writing to fits anyway...)
tVals[sCoadd].format='%.3f'
#tVals[sCrowd].format='%.2f' # (may only get reported to 1 d.p. anyway)
#tVals['%sCrowdBri' % (thisFilter)] = \
# np.asarray(tVals[sCrowd] < tVals[sCoadd], 'int')
# add the mask as a boolean
tVals['%sGood' % (thisFilter)] = \
np.asarray(bVal, 'int')
# now add all the summary statistics for which we asked. Try
# specifying the datatype
for sStat in statsNames.keys():
tVals[sStat] = Column(\
bgroup.bundleDict[statsNames[sStat]].metricValues, \
dtype='float')
tVals[countCol] = Column(bgroup.bundleDict[statsNames[countCol]].metricValues, dtype='int')
# cut down by mask
#tVals = tVals[bVal]
# Set metadata and write to disk. Add comments later.
tVals.meta['nsideFound'] = nsideFound
tVals.meta['tMax'] = tMax
tVals.meta['crowdError'] = crowdVals
tVals.meta['countedCol'] = col2Count[:]
# Can select only within strip to cut down on space requirements
sSel=''
if selectStrip:
bMin = -30.
bMax = +25.
lMin = -150.
lMax = 80.
sSel = '_nrPlane'
bStrip = (tVals['b'] >= bMin) & \
(tVals['b'] <= bMax) & \
(tVals['l'] >= lMin) & \
(tVals['l'] <= lMax)
tVals = tVals[bStrip]
tVals.meta['sel_lMin'] = lMin
tVals.meta['sel_lMax'] = lMax
tVals.meta['sel_bMin'] = bMin
tVals.meta['sel_bMax'] = bMax
# metadata
tVals.meta['selectStrip'] = selectStrip
# generate output path
pathTab = '%s/table_uncty_%s_%s_nside%i_tmax%i%s.fits' % \
(outDir, dbFil.split('_sqlite')[0], thisFilter, nside, tMax, sSel)
# save the table
tVals.write(pathTab, overwrite=True)
# give this method the capability to remove the npz file (useful
# if we want to go to very high spatial resolution for some
# reason):
if cleanNpz:
for pathNp in glob.glob('%s/*.npz' % (outDir)):
os.remove(pathNp)
return pathTab
def scienceRadarBatch(colmap=None,
runName='opsim',
extraSql=None,
extraMetadata=None,
nside=64,
benchmarkArea=18000,
benchmarkNvisits=825,
DDF=True):
"""A batch of metrics for looking at survey performance relative to the SRD and the main
science drivers of LSST.
Parameters
----------
"""
# Hide dependencies
from mafContrib.LSSObsStrategy.galaxyCountsMetric_extended import GalaxyCountsMetric_extended
from mafContrib import Plasticc_metric, plasticc_slicer, load_plasticc_lc, TDEsAsciiMetric
if colmap is None:
colmap = ColMapDict('fbs')
if extraSql is None:
extraSql = ''
if extraSql == '':
joiner = ''
else:
joiner = ' and '
bundleList = []
# Get some standard per-filter coloring and sql constraints
filterlist, colors, filterorders, filtersqls, filtermetadata = filterList(
all=False, extraSql=extraSql, extraMetadata=extraMetadata)
standardStats = standardSummary(withCount=False)
healslicer = slicers.HealpixSlicer(nside=nside)
subsetPlots = [plots.HealpixSkyMap(), plots.HealpixHistogram()]
# Load up the plastic light curves
models = ['SNIa-normal', 'KN']
plasticc_models_dict = {}
for model in models:
plasticc_models_dict[model] = list(
load_plasticc_lc(model=model).values())
#########################
# SRD, DM, etc
#########################
fOb = fOBatch(runName=runName,
colmap=colmap,
extraSql=extraSql,
extraMetadata=extraMetadata,
benchmarkArea=benchmarkArea,
benchmarkNvisits=benchmarkNvisits)
astromb = astrometryBatch(runName=runName,
colmap=colmap,
extraSql=extraSql,
extraMetadata=extraMetadata)
rapidb = rapidRevisitBatch(runName=runName,
colmap=colmap,
extraSql=extraSql,
extraMetadata=extraMetadata)
# loop through and modify the display dicts - set SRD as group and their previous 'group' as the subgroup
temp_list = []
for key in fOb:
temp_list.append(fOb[key])
for key in astromb:
temp_list.append(astromb[key])
for key in rapidb:
temp_list.append(rapidb[key])
for metricb in temp_list:
metricb.displayDict['subgroup'] = metricb.displayDict['group'].replace(
'SRD', '').lstrip(' ')
metricb.displayDict['group'] = 'SRD'
bundleList.extend(temp_list)
displayDict = {
'group': 'SRD',
'subgroup': 'Year Coverage',
'order': 0,
'caption': 'Number of years with observations.'
}
slicer = slicers.HealpixSlicer(nside=nside)
metric = metrics.YearCoverageMetric()
for f in filterlist:
plotDict = {'colorMin': 7, 'colorMax': 10, 'color': colors[f]}
summary = [
metrics.AreaSummaryMetric(area=18000,
reduce_func=np.mean,
decreasing=True,
metricName='N Seasons (18k) %s' % f)
]
bundleList.append(
mb.MetricBundle(metric,
slicer,
filtersqls[f],
plotDict=plotDict,
metadata=filtermetadata[f],
displayDict=displayDict,
summaryMetrics=summary))
#########################
# Solar System
#########################
# Generally, we need to run Solar System metrics separately; they're a multi-step process.
#########################
# Cosmology
#########################
displayDict = {
'group': 'Cosmology',
'subgroup': 'Galaxy Counts',
'order': 0,
'caption': None
}
plotDict = {'percentileClip': 95., 'nTicks': 5}
sql = extraSql + joiner + 'filter="i"'
metadata = combineMetadata(extraMetadata, 'i band')
metric = GalaxyCountsMetric_extended(filterBand='i',
redshiftBin='all',
nside=nside)
summary = [
metrics.AreaSummaryMetric(area=18000,
reduce_func=np.sum,
decreasing=True,
metricName='N Galaxies (18k)')
]
summary.append(metrics.SumMetric(metricName='N Galaxies (all)'))
# make sure slicer has cache off
slicer = slicers.HealpixSlicer(nside=nside, useCache=False)
bundle = mb.MetricBundle(metric,
slicer,
sql,
plotDict=plotDict,
metadata=metadata,
displayDict=displayDict,
summaryMetrics=summary,
plotFuncs=subsetPlots)
bundleList.append(bundle)
displayDict['order'] += 1
# let's put Type Ia SN in here
displayDict['subgroup'] = 'SNe Ia'
# XXX-- use the light curves from PLASTICC here
displayDict['caption'] = 'Fraction of normal SNe Ia'
sql = extraSql
slicer = plasticc_slicer(plcs=plasticc_models_dict['SNIa-normal'],
seed=42,
badval=0)
metric = Plasticc_metric(metricName='SNIa')
# Set the maskval so that we count missing objects as zero.
summary_stats = [metrics.MeanMetric(maskVal=0)]
plotFuncs = [plots.HealpixSkyMap()]
bundle = mb.MetricBundle(metric,
slicer,
sql,
runName=runName,
summaryMetrics=summary_stats,
plotFuncs=plotFuncs,
metadata=extraMetadata,
displayDict=displayDict)
bundleList.append(bundle)
displayDict['order'] += 1
displayDict['subgroup'] = 'Camera Rotator'
displayDict[
'caption'] = 'Kuiper statistic (0 is uniform, 1 is delta function) of the '
slicer = slicers.HealpixSlicer(nside=nside)
metric1 = metrics.KuiperMetric('rotSkyPos')
metric2 = metrics.KuiperMetric('rotTelPos')
for f in filterlist:
for m in [metric1, metric2]:
plotDict = {'color': colors[f]}
displayDict['order'] = filterorders[f]
displayDict['caption'] += f"{m.colname} for visits in {f} band."
bundleList.append(
mb.MetricBundle(m,
slicer,
filtersqls[f],
plotDict=plotDict,
displayDict=displayDict,
summaryMetrics=standardStats,
plotFuncs=subsetPlots))
# XXX--need some sort of metric for weak lensing
#########################
# Variables and Transients
#########################
displayDict = {
'group': 'Variables/Transients',
'subgroup': 'Periodic Stars',
'order': 0,
'caption': None
}
for period in [
0.5,
1,
2,
]:
for magnitude in [21., 24.]:
amplitudes = [0.05, 0.1, 1.0]
periods = [period] * len(amplitudes)
starMags = [magnitude] * len(amplitudes)
plotDict = {
'nTicks': 3,
'colorMin': 0,
'colorMax': 3,
'xMin': 0,
'xMax': 3
}
metadata = combineMetadata(
'P_%.1f_Mag_%.0f_Amp_0.05-0.1-1' % (period, magnitude),
extraMetadata)
sql = None
displayDict['caption'] = 'Metric evaluates if a periodic signal of period %.1f days could ' \
'be detected for an r=%i star. A variety of amplitudes of periodicity ' \
'are tested: [1, 0.1, and 0.05] mag amplitudes, which correspond to ' \
'metric values of [1, 2, or 3]. ' % (period, magnitude)
metric = metrics.PeriodicDetectMetric(periods=periods,
starMags=starMags,
amplitudes=amplitudes,
metricName='PeriodDetection')
bundle = mb.MetricBundle(metric,
healslicer,
sql,
metadata=metadata,
displayDict=displayDict,
plotDict=plotDict,
plotFuncs=subsetPlots,
summaryMetrics=standardStats)
bundleList.append(bundle)
displayDict['order'] += 1
# XXX add some PLASTICC metrics for kilovnova and tidal disruption events.
displayDict['subgroup'] = 'KN'
displayDict['caption'] = 'Fraction of Kilonova (from PLASTICC)'
displayDict['order'] = 0
slicer = plasticc_slicer(plcs=plasticc_models_dict['KN'],
seed=43,
badval=0)
metric = Plasticc_metric(metricName='KN')
plotFuncs = [plots.HealpixSkyMap()]
summary_stats = [metrics.MeanMetric(maskVal=0)]
bundle = mb.MetricBundle(metric,
slicer,
extraSql,
runName=runName,
summaryMetrics=summary_stats,
plotFuncs=plotFuncs,
metadata=extraMetadata,
displayDict=displayDict)
bundleList.append(bundle)
# Tidal Disruption Events
displayDict['subgroup'] = 'TDE'
displayDict[
'caption'] = 'Fraction of TDE lightcurves that could be identified, outside of DD fields'
detectSNR = {'u': 5, 'g': 5, 'r': 5, 'i': 5, 'z': 5, 'y': 5}
# light curve parameters
epochStart = -22
peakEpoch = 0
nearPeakT = 10
postPeakT = 14 # two weeks
nPhaseCheck = 1
# condition parameters
nObsTotal = {'u': 0, 'g': 0, 'r': 0, 'i': 0, 'z': 0, 'y': 0}
nObsPrePeak = 1
nObsNearPeak = {'u': 0, 'g': 0, 'r': 0, 'i': 0, 'z': 0, 'y': 0}
nFiltersNearPeak = 3
nObsPostPeak = 0
nFiltersPostPeak = 2
metric = TDEsAsciiMetric(asciifile=None,
detectSNR=detectSNR,
epochStart=epochStart,
peakEpoch=peakEpoch,
nearPeakT=nearPeakT,
postPeakT=postPeakT,
nPhaseCheck=nPhaseCheck,
nObsTotal=nObsTotal,
nObsPrePeak=nObsPrePeak,
nObsNearPeak=nObsNearPeak,
nFiltersNearPeak=nFiltersNearPeak,
nObsPostPeak=nObsPostPeak,
nFiltersPostPeak=nFiltersPostPeak)
slicer = slicers.HealpixSlicer(nside=32)
sql = extraSql + joiner + "note not like '%DD%'"
md = extraMetadata
if md is None:
md = " NonDD"
else:
md += 'NonDD'
bundle = mb.MetricBundle(metric,
slicer,
sql,
runName=runName,
summaryMetrics=standardStats,
plotFuncs=plotFuncs,
metadata=md,
displayDict=displayDict)
bundleList.append(bundle)
# XXX -- would be good to add some microlensing events, for both MW and LMC/SMC.
#########################
# Milky Way
#########################
displayDict = {'group': 'Milky Way', 'subgroup': ''}
displayDict['subgroup'] = 'N stars'
slicer = slicers.HealpixSlicer(nside=nside, useCache=False)
sum_stats = [metrics.SumMetric(metricName='Total N Stars')]
for f in filterlist:
displayDict['order'] = filterorders[f]
displayDict['caption'] = 'Number of stars in %s band with an measurement error due to crowding ' \
'of less than 0.1 mag' % f
# Configure the NstarsMetric - note 'filtername' refers to the filter in which to evaluate crowding
metric = metrics.NstarsMetric(crowding_error=0.1,
filtername='r',
seeingCol=colmap['seeingGeom'],
m5Col=colmap['fiveSigmaDepth'])
plotDict = {'nTicks': 5, 'logScale': True, 'colorMin': 100}
bundle = mb.MetricBundle(metric,
slicer,
filtersqls[f],
runName=runName,
summaryMetrics=sum_stats,
plotFuncs=subsetPlots,
plotDict=plotDict,
displayDict=displayDict)
bundleList.append(bundle)
#########################
# DDF
#########################
if DDF:
# Hide this import to avoid adding a dependency.
from lsst.sims.featureScheduler.surveys import generate_dd_surveys, Deep_drilling_survey
ddf_surveys = generate_dd_surveys()
# Add on the Euclid fields
# XXX--to update. Should have a spot where all the DDF locations are stored.
ddf_surveys.append(
Deep_drilling_survey([], 58.97, -49.28, survey_name='DD:EDFSa'))
ddf_surveys.append(
Deep_drilling_survey([], 63.6, -47.60, survey_name='DD:EDFSb'))
# For doing a high-res sampling of the DDF for co-adds
ddf_radius = 1.8 # Degrees
ddf_nside = 512
ra, dec = hpid2RaDec(ddf_nside, np.arange(hp.nside2npix(ddf_nside)))
displayDict = {'group': 'DDF depths', 'subgroup': None}
for survey in ddf_surveys:
displayDict['subgroup'] = survey.survey_name
# Crop off the u-band only DDF
if survey.survey_name[0:4] != 'DD:u':
dist_to_ddf = angularSeparation(ra, dec, np.degrees(survey.ra),
np.degrees(survey.dec))
goodhp = np.where(dist_to_ddf <= ddf_radius)
slicer = slicers.UserPointsSlicer(ra=ra[goodhp],
dec=dec[goodhp],
useCamera=False)
for f in filterlist:
metric = metrics.Coaddm5Metric(
metricName=survey.survey_name + ', ' + f)
summary = [
metrics.MedianMetric(metricName='Median depth ' +
survey.survey_name + ', ' + f)
]
plotDict = {'color': colors[f]}
sql = filtersqls[f]
displayDict['order'] = filterorders[f]
displayDict['caption'] = 'Coadded m5 depth in %s band.' % (
f)
bundle = mb.MetricBundle(metric,
slicer,
sql,
metadata=filtermetadata[f],
displayDict=displayDict,
summaryMetrics=summary,
plotFuncs=[],
plotDict=plotDict)
bundleList.append(bundle)
displayDict = {'group': 'DDF Transients', 'subgroup': None}
for survey in ddf_surveys:
displayDict['subgroup'] = survey.survey_name
if survey.survey_name[0:4] != 'DD:u':
slicer = plasticc_slicer(
plcs=plasticc_models_dict['SNIa-normal'],
seed=42,
ra_cen=survey.ra,
dec_cen=survey.dec,
radius=np.radians(3.),
useCamera=False)
metric = Plasticc_metric(metricName=survey.survey_name +
' SNIa')
sql = extraSql
summary_stats = [metrics.MeanMetric(maskVal=0)]
plotFuncs = [plots.HealpixSkyMap()]
bundle = mb.MetricBundle(metric,
slicer,
sql,
runName=runName,
summaryMetrics=summary_stats,
plotFuncs=plotFuncs,
metadata=extraMetadata,
displayDict=displayDict)
bundleList.append(bundle)
displayDict['order'] = 10
# Set the runName for all bundles and return the bundleDict.
for b in bundleList:
b.setRunName(runName)
bundleDict = mb.makeBundlesDictFromList(bundleList)
return bundleDict
def quickDiscoveryBatch(slicer, colmap=None, runName='opsim', detectionLosses='detection', metadata='',
albedo=None, Hmark=None, npReduce=np.mean, times=None, constraint=None):
if colmap is None:
colmap = ColMapDict('opsimV4')
bundleList = []
plotBundles = []
basicPlotDict = {'albedo': albedo, 'Hmark': Hmark, 'npReduce': npReduce,
'nxbins': 200, 'nybins': 200}
plotFuncs = [plots.MetricVsH()]
displayDict ={'group': 'Discovery'}
if detectionLosses not in ('detection', 'trailing'):
raise ValueError('Please choose detection or trailing as options for detectionLosses.')
if detectionLosses == 'trailing':
magStacker = stackers.MoMagStacker(lossCol='dmagTrail')
detectionLosses = ' trailing loss'
else:
magStacker = stackers.MoMagStacker(lossCol='dmagDetect')
detectionLosses = ' detection loss'
if times is None:
try:
timestep = 30
times = np.arange(slicer.obs[colmap['mjd']].min(), slicer.obs[colmap['mjd']].max() + timestep/2,
timestep)
except AttributeError:
raise warnings.warn('Cannot set times for completeness summary metrics. Will set up bundles, '
'but without summary metrics.')
if Hmark is None:
Hval = slicer.Hrange.mean()
else:
Hval = Hmark
# Set up the summary metrics.
if times is not None:
summaryTimeMetrics = summaryCompletenessAtTime(times, Hval=Hval, Hindex=0.33)
else:
summaryTimeMetrics = None
summaryHMetrics = summaryCompletenessOverH(requiredChances=1, Hindex=0.33)
# Set up a dictionary to pass to each metric for the column names.
colkwargs = {'mjdCol': colmap['mjd'], 'seeingCol': colmap['seeingGeom'],
'expTimeCol': colmap['exptime'], 'm5Col': colmap['fiveSigmaDepth'],
'nightCol': colmap['night'], 'filterCol': colmap['filter']}
def _setup_child_metrics(parentMetric):
childMetrics = {}
childMetrics['Time'] = metrics.Discovery_TimeMetric(parentMetric, **colkwargs)
childMetrics['N_Chances'] = metrics.Discovery_N_ChancesMetric(parentMetric, **colkwargs)
# Could expand to add N_chances per year, but not really necessary.
return childMetrics
def _configure_child_bundles(parentBundle):
dispDict = {'group': 'Discovery', 'subgroup': 'Time',
'caption': 'Time of discovery of objects', 'order': 0}
parentBundle.childBundles['Time'].setDisplayDict(dispDict)
parentBundle.childBundles['Time'].setSummaryMetrics(summaryTimeMetrics)
dispDict = {'group': 'Discovery', 'subgroup': 'NChances',
'caption': 'Number of chances for discovery of objects', 'order': 0}
parentBundle.childBundles['N_Chances'].setDisplayDict(dispDict)
parentBundle.childBundles['N_Chances'].setSummaryMetrics(summaryHMetrics)
return
# 3 pairs in 15
md = metadata + ' 3 pairs in 15 nights SNR=5' + detectionLosses
# Set up plot dict.
plotDict = {'title': '%s: %s' % (runName, md)}
plotDict.update(basicPlotDict)
metric = metrics.DiscoveryMetric(nObsPerNight=2, tMin=0, tMax=90./60./24.,
nNightsPerWindow=3, tWindow=15, **colkwargs)
childMetrics = _setup_child_metrics(metric)
bundle = mb.MoMetricBundle(metric, slicer, constraint,
stackerList=[magStacker],
runName=runName, metadata=md,
childMetrics=childMetrics,
plotDict=plotDict, plotFuncs=plotFuncs,
displayDict=displayDict)
_configure_child_bundles(bundle)
bundleList.append(bundle)
# 3 pairs in 30
md = metadata + ' 3 pairs in 30 nights SNR=5' + detectionLosses
plotDict = {'title': '%s: %s' % (runName, md)}
plotDict.update(basicPlotDict)
metric = metrics.DiscoveryMetric(nObsPerNight=2, tMin=0, tMax=90. / 60. / 24.,
nNightsPerWindow=3, tWindow=30, snrLimit=5, **colkwargs)
childMetrics = _setup_child_metrics(metric)
bundle = mb.MoMetricBundle(metric, slicer, constraint,
stackerList=[magStacker],
runName=runName, metadata=md,
childMetrics=childMetrics,
plotDict=plotDict, plotFuncs=plotFuncs,
displayDict=displayDict)
_configure_child_bundles(bundle)
bundleList.append(bundle)
# Set the runName for all bundles and return the bundleDict.
for b in bundleList:
b.setRunName(runName)
return mb.makeBundlesDictFromList(bundleList), plotBundles
def tEffMetrics(colmap=None, runName='opsim',
extraSql=None, extraMetadata=None, nside=64,
ditherStacker=None, ditherkwargs=None):
"""Generate a series of Teff metrics. Teff total, per night, and sky maps (all and per filter).
Parameters
----------
colmap : dict, opt
A dictionary with a mapping of column names. Default will use OpsimV4 column names.
runName : str, opt
The name of the simulated survey. Default is "opsim".
extraSql : str, opt
Additional constraint to add to any sql constraints (e.g. 'propId=1' or 'fieldID=522').
Default None, for no additional constraints.
extraMetadata : str, opt
Additional metadata to add before any below (i.e. "WFD"). Default is None.
nside : int, opt
Nside value for healpix slicer. Default 64.
If "None" is passed, the healpixslicer-based metrics will be skipped.
ditherStacker: str or lsst.sims.maf.stackers.BaseDitherStacker
Optional dither stacker to use to define ra/dec columns.
ditherkwargs: dict, opt
Optional dictionary of kwargs for the dither stacker.
Returns
-------
metricBundleDict
"""
if colmap is None:
colmap = ColMapDict('opsimV4')
bundleList = []
subgroup = extraMetadata
if subgroup is None:
subgroup = 'All visits'
raCol, decCol, degrees, ditherStacker, ditherMeta = radecCols(ditherStacker, colmap, ditherkwargs)
extraMetadata = combineMetadata(extraMetadata, ditherMeta)
# Set up basic all and per filter sql constraints.
filterlist, colors, orders, sqls, metadata = filterList(all=True,
extraSql=extraSql,
extraMetadata=extraMetadata)
if metadata['all'] is None:
metadata['all'] = 'All visits'
subsetPlots = [plots.HealpixSkyMap(), plots.HealpixHistogram()]
# Total Teff and normalized Teff.
displayDict = {'group': 'T_eff Summary', 'subgroup': subgroup}
displayDict['caption'] = 'Total effective time of the survey (see Teff metric).'
displayDict['order'] = 0
metric = metrics.TeffMetric(m5Col=colmap['fiveSigmaDepth'], filterCol=colmap['filter'],
normed=False, metricName='Total Teff')
slicer = slicers.UniSlicer()
bundle = mb.MetricBundle(metric, slicer, constraint=sqls['all'], displayDict=displayDict,
metadata=metadata['all'])
bundleList.append(bundle)
displayDict['caption'] = 'Normalized total effective time of the survey (see Teff metric).'
displayDict['order'] = 1
metric = metrics.TeffMetric(m5Col=colmap['fiveSigmaDepth'], filterCol=colmap['filter'],
normed=True, metricName='Normalized Teff')
slicer = slicers.UniSlicer()
bundle = mb.MetricBundle(metric, slicer, constraint=sqls['all'], displayDict=displayDict,
metadata=metadata['all'])
bundleList.append(bundle)
# Generate Teff maps in all and per filters
displayDict = {'group': 'T_eff Maps', 'subgroup': subgroup}
if ditherMeta is not None:
for m in metadata:
metadata[m] = combineMetadata(metadata[m], ditherMeta)
metric = metrics.TeffMetric(m5Col=colmap['fiveSigmaDepth'], filterCol=colmap['filter'],
normed=True, metricName='Normalized Teff')
slicer = slicers.HealpixSlicer(nside=nside, latCol=decCol, lonCol=raCol,
latLonDeg=degrees)
for f in filterlist:
displayDict['caption'] = 'Normalized effective time of the survey, for %s' % metadata[f]
displayDict['order'] = orders[f]
plotDict = {'color': colors[f]}
bundle = mb.MetricBundle(metric, slicer, sqls[f], metadata=metadata[f],
stackerList=ditherStacker,
displayDict=displayDict, plotFuncs=subsetPlots, plotDict=plotDict,
summaryMetrics=standardSummary())
bundleList.append(bundle)
# Set the runName for all bundles and return the bundleDict.
for b in bundleList:
b.setRunName(runName)
return mb.makeBundlesDictFromList(bundleList)
def eastWestBias(colmap=None, runName='opsim', extraSql=None, extraMetadata=None):
"""Plot the number of observations to the east vs to the west, per night.
Parameters
----------
colmap : dict, opt
A dictionary with a mapping of column names. Default will use OpsimV4 column names.
runName : str, opt
The name of the simulated survey. Default is "opsim".
extraSql : str, opt
Additional constraint to add to any sql constraints (e.g. 'night<365')
Default None, for no additional constraints.
extraMetadata : str, opt
Additional metadata to add before any below (i.e. "WFD"). Default is None.
"""
if colmap is None:
colmap = ColMapDict('opsimV4')
bundleList = []
plotBundles = []
group = 'East vs West'
subgroup = 'All visits'
if extraMetadata is not None:
subgroup = extraMetadata
displayDict = {'group': group, 'subgroup': subgroup, 'order': 0}
eastvswest = 180
if not colmap['raDecDeg']:
eastvswest = np.radians(eastvswest)
displayDict['caption'] = 'Number of visits per night that occur with azimuth <= 180.'
if extraSql is not None:
displayDict['caption'] += ' With additional sql constraint %s.' % extraSql
metric = metrics.CountMetric(colmap['night'], metricName='Nvisits East')
slicer = slicers.OneDSlicer(sliceColName=colmap['night'], binsize=1)
sql = '%s <= %f' % (colmap['az'], eastvswest)
if extraSql is not None:
sql = '(%s) and (%s)' % (sql, extraSql)
plotDict = {'color': 'orange', 'label': 'East'}
bundle = mb.MetricBundle(metric, slicer, sql, metadata=extraMetadata,
displayDict=displayDict, plotDict=plotDict)
bundleList.append(bundle)
displayDict['caption'] = 'Number of visits per night that occur with azimuth > 180.'
if extraSql is not None:
displayDict['caption'] += ' With additional sql constraint %s.' % extraSql
metric = metrics.CountMetric(colmap['night'], metricName='Nvisits West')
slicer = slicers.OneDSlicer(sliceColName=colmap['night'], binsize=1)
sql = '%s > %f' % (colmap['az'], eastvswest)
if extraSql is not None:
sql = '(%s) and (%s)' % (sql, extraSql)
plotDict = {'color': 'blue', 'label': 'West'}
bundle = mb.MetricBundle(metric, slicer, sql, metadata=extraMetadata,
displayDict=displayDict, plotDict=plotDict)
bundleList.append(bundle)
# Set the runName for all bundles and return the bundleDict.
for b in bundleList:
b.setRunName(runName)
return mb.makeBundlesDictFromList(bundleList), plotBundles
def astrometryBatch(colmap=None, runName='opsim',
extraSql=None, extraMetadata=None,
nside=64, ditherStacker=None, ditherkwargs=None):
"""Metrics for evaluating proper motion and parallax.
Parameters
----------
colmap : dict or None, opt
A dictionary with a mapping of column names. Default will use OpsimV4 column names.
runName : str, opt
The name of the simulated survey. Default is "opsim".
nside : int, opt
Nside for the healpix slicer. Default 64.
extraSql : str or None, opt
Additional sql constraint to apply to all metrics.
extraMetadata : str or None, opt
Additional metadata to apply to all results.
ditherStacker: str or lsst.sims.maf.stackers.BaseDitherStacker
Optional dither stacker to use to define ra/dec columns.
ditherkwargs: dict, opt
Optional dictionary of kwargs for the dither stacker.
Returns
-------
metricBundleDict
"""
if colmap is None:
colmap = ColMapDict('opsimV4')
bundleList = []
sql = ''
metadata = 'All visits'
# Add additional sql constraint (such as wfdWhere) and metadata, if provided.
if (extraSql is not None) and (len(extraSql) > 0):
sql = extraSql
if extraMetadata is None:
metadata = extraSql.replace('filter =', '').replace('filter=', '')
metadata = metadata.replace('"', '').replace("'", '')
if extraMetadata is not None:
metadata = extraMetadata
subgroup = metadata
raCol, decCol, degrees, ditherStacker, ditherMeta = radecCols(ditherStacker, colmap, ditherkwargs)
# Don't want dither info in subgroup (too long), but do want it in bundle name.
metadata = combineMetadata(metadata, ditherMeta)
rmags_para = [22.4, 24.0]
rmags_pm = [20.5, 24.0]
# Set up parallax/dcr stackers.
parallaxStacker = stackers.ParallaxFactorStacker(raCol=raCol, decCol=decCol,
dateCol=colmap['mjd'], degrees=degrees)
dcrStacker = stackers.DcrStacker(filterCol=colmap['filter'], altCol=colmap['alt'], degrees=degrees,
raCol=raCol, decCol=decCol, lstCol=colmap['lst'],
site='LSST', mjdCol=colmap['mjd'])
# Set up parallax metrics.
slicer = slicers.HealpixSlicer(nside=nside, lonCol=raCol, latCol=decCol, latLonDeg=degrees)
subsetPlots = [plots.HealpixSkyMap(), plots.HealpixHistogram()]
displayDict = {'group': 'Parallax', 'subgroup': subgroup,
'order': 0, 'caption': None}
# Expected error on parallax at 10 AU.
plotmaxVals = (2.0, 15.0)
for rmag, plotmax in zip(rmags_para, plotmaxVals):
plotDict = {'xMin': 0, 'xMax': plotmax, 'colorMin': 0, 'colorMax': plotmax}
metric = metrics.ParallaxMetric(metricName='Parallax Error @ %.1f' % (rmag), rmag=rmag,
seeingCol=colmap['seeingGeom'], filterCol=colmap['filter'],
m5Col=colmap['fiveSigmaDepth'], normalize=False)
bundle = mb.MetricBundle(metric, slicer, sql, metadata=metadata,
stackerList=[parallaxStacker, ditherStacker],
displayDict=displayDict, plotDict=plotDict,
summaryMetrics=standardSummary(),
plotFuncs=subsetPlots)
bundleList.append(bundle)
displayDict['order'] += 1
# Parallax normalized to 'best possible' if all visits separated by 6 months.
# This separates the effect of cadence from depth.
for rmag in rmags_para:
metric = metrics.ParallaxMetric(metricName='Normalized Parallax @ %.1f' % (rmag), rmag=rmag,
seeingCol=colmap['seeingGeom'], filterCol=colmap['filter'],
m5Col=colmap['fiveSigmaDepth'], normalize=True)
bundle = mb.MetricBundle(metric, slicer, sql, metadata=metadata,
stackerList=[parallaxStacker, ditherStacker],
displayDict=displayDict,
summaryMetrics=standardSummary(),
plotFuncs=subsetPlots)
bundleList.append(bundle)
displayDict['order'] += 1
# Parallax factor coverage.
for rmag in rmags_para:
metric = metrics.ParallaxCoverageMetric(metricName='Parallax Coverage @ %.1f' % (rmag),
rmag=rmag, m5Col=colmap['fiveSigmaDepth'],
mjdCol=colmap['mjd'], filterCol=colmap['filter'],
seeingCol=colmap['seeingGeom'])
bundle = mb.MetricBundle(metric, slicer, sql, metadata=metadata,
stackerList=[parallaxStacker, ditherStacker],
displayDict=displayDict, summaryMetrics=standardSummary(),
plotFuncs=subsetPlots)
bundleList.append(bundle)
displayDict['order'] += 1
# Parallax problems can be caused by HA and DCR degeneracies. Check their correlation.
for rmag in rmags_para:
metric = metrics.ParallaxDcrDegenMetric(metricName='Parallax-DCR degeneracy @ %.1f' % (rmag),
rmag=rmag, seeingCol=colmap['seeingEff'],
filterCol=colmap['filter'], m5Col=colmap['fiveSigmaDepth'])
caption = 'Correlation between parallax offset magnitude and hour angle for a r=%.1f star.' % (rmag)
caption += ' (0 is good, near -1 or 1 is bad).'
bundle = mb.MetricBundle(metric, slicer, sql, metadata=metadata,
stackerList=[dcrStacker, parallaxStacker, ditherStacker],
displayDict=displayDict, summaryMetrics=standardSummary(),
plotFuncs=subsetPlots)
bundleList.append(bundle)
displayDict['order'] += 1
# Proper Motion metrics.
displayDict = {'group': 'Proper Motion', 'subgroup': subgroup, 'order': 0, 'caption': None}
# Proper motion errors.
plotmaxVals = (1.0, 5.0)
for rmag, plotmax in zip(rmags_pm, plotmaxVals):
plotDict = {'xMin': 0, 'xMax': plotmax, 'colorMin': 0, 'colorMax': plotmax}
metric = metrics.ProperMotionMetric(metricName='Proper Motion Error @ %.1f' % rmag,
rmag=rmag, m5Col=colmap['fiveSigmaDepth'],
mjdCol=colmap['mjd'], filterCol=colmap['filter'],
seeingCol=colmap['seeingGeom'], normalize=False)
bundle = mb.MetricBundle(metric, slicer, sql, metadata=metadata,
stackerList=[ditherStacker],
displayDict=displayDict, plotDict=plotDict,
summaryMetrics=standardSummary(),
plotFuncs=subsetPlots)
bundleList.append(bundle)
displayDict['order'] += 1
# Normalized proper motion.
for rmag in rmags_pm:
metric = metrics.ProperMotionMetric(metricName='Normalized Proper Motion @ %.1f' % rmag,
rmag=rmag, m5Col=colmap['fiveSigmaDepth'],
mjdCol=colmap['mjd'], filterCol=colmap['filter'],
seeingCol=colmap['seeingGeom'], normalize=True)
bundle = mb.MetricBundle(metric, slicer, sql, metadata=metadata,
stackerList=[ditherStacker],
displayDict=displayDict, summaryMetrics=standardSummary(),
plotFuncs=subsetPlots)
bundleList.append(bundle)
displayDict['order'] += 1
# Set the runName for all bundles and return the bundleDict.
for b in bundleList:
b.setRunName(runName)
return mb.makeBundlesDictFromList(bundleList)
def characterizationBatch(slicer, colmap=None, runName='opsim', metadata='',
albedo=None, Hmark=None, npReduce=np.mean, constraint=None,
windows=None, bins=None):
if colmap is None:
colmap = ColMapDict('opsimV4')
bundleList = []
plotBundles = []
# Set up a dictionary to pass to each metric for the column names.
colkwargs = {'mjdCol': colmap['mjd'], 'seeingCol': colmap['seeingGeom'],
'expTimeCol': colmap['exptime'], 'm5Col': colmap['fiveSigmaDepth'],
'nightCol': colmap['night'], 'filterCol': colmap['filter']}
basicPlotDict = {'albedo': albedo, 'Hmark': Hmark, 'npReduce': npReduce,
'nxbins': 200, 'nybins': 200}
plotFuncs = [plots.MetricVsH()]
displayDict ={'group': 'Characterization'}
if windows is None:
windows = np.arange(1, 200, 15.)
if bins is None:
bins = np.arange(5, 95, 10.)
# Number of observations.
md = metadata
plotDict = {'ylabel': 'Number of observations (#)',
'title': '%s: Number of observations %s' % (runName, md)}
plotDict.update(basicPlotDict)
metric = metrics.NObsMetric(**colkwargs)
bundle = mb.MoMetricBundle(metric, slicer, constraint,
runName=runName, metadata=md,
plotDict=plotDict, plotFuncs=plotFuncs,
summaryMetrics=None,
displayDict=displayDict)
bundleList.append(bundle)
# Observational arc.
md = metadata
plotDict = {'ylabel': 'Observational Arc (days)',
'title': '%s: Observational Arc Length %s' % (runName, md)}
plotDict.update(basicPlotDict)
metric = metrics.ObsArcMetric(**colkwargs)
bundle = mb.MoMetricBundle(metric, slicer, constraint,
runName=runName, metadata=md,
plotDict=plotDict, plotFuncs=plotFuncs,
summaryMetrics=None,
displayDict=displayDict)
bundleList.append(bundle)
# Activity detection.
for w in windows:
md = metadata + ' activity lasting %.0f days' % w
plotDict = {'title': '%s: Chances of detecting %s' % (runName, md),
'ylabel': 'Probability of detection per %.0f day window' % w}
metricName = 'Chances of detecting activity lasting %.0f days' % w
metric = metrics.ActivityOverTimeMetric(w, metricName=metricName, **colkwargs)
bundle = mb.MoMetricBundle(metric, slicer, constraint,
runName=runName, metadata=metadata,
plotDict=plotDict, plotFuncs=plotFuncs,
displayDict=displayDict)
bundleList.append(bundle)
for b in bins:
md = metadata + ' activity lasting %.2f of period' % (b/360.)
plotDict = {'title': '%s: Chances of detecting %s' % (runName, md),
'ylabel': 'Probability of detection per %.2f deg window' % b}
metricName = 'Chances of detecting activity lasting %.2f of the period' % (b/360.)
metric = metrics.ActivityOverPeriodMetric(b, metricName=metricName, **colkwargs)
bundle = mb.MoMetricBundle(metric, slicer, constraint,
runName=runName, metadata=metadata,
plotDict=plotDict, plotFuncs=plotFuncs,
displayDict=displayDict)
bundleList.append(bundle)
# Lightcurve inversion.
md = metadata
plotDict = {'yMin': 0, 'yMax': 1, 'ylabel': 'Fraction of objects',
'title': '%s: Fraction with potential lightcurve inversion %s' % (runName, md)}
plotDict.update(basicPlotDict)
metric = metrics.LightcurveInversionMetric(snrLimit=20, nObs=100, nDays=5*365, **colkwargs)
bundle = mb.MoMetricBundle(metric, slicer, constraint,
runName=runName, metadata=md,
plotDict=plotDict, plotFuncs=plotFuncs,
summaryMetrics=None,
displayDict=displayDict)
bundleList.append(bundle)
# Color determination.
snrLimit = 10
nHours = 2.0
nPairs = 1
md = metadata + ' u-g color'
plotDict = {'label': md,
'title': '%s: Fraction with potential u-g color measurement %s' % (runName, md)}
plotDict.update(basicPlotDict)
metric = metrics.ColorDeterminationMetric(nPairs=nPairs, snrLimit=snrLimit, nHours=nHours,
bOne='u', bTwo='g', **colkwargs)
bundle = mb.MoMetricBundle(metric, slicer, constraint,
runName=runName, metadata=md,
plotDict=plotDict, plotFuncs=plotFuncs,
summaryMetrics=None,
displayDict=displayDict)
bundleList.append(bundle)
md = metadata + ' g-r color'
plotDict = {'label': md,
'title': '%s: Fraction with potential g-r color measurement %s' % (runName, md)}
plotDict.update(basicPlotDict)
metric = metrics.ColorDeterminationMetric(nPairs=nPairs, snrLimit=snrLimit, nHours=nHours,
bOne='g', bTwo='r', **colkwargs)
bundle = mb.MoMetricBundle(metric, slicer, constraint,
runName=runName, metadata=md,
plotDict=plotDict, plotFuncs=plotFuncs,
summaryMetrics=None,
displayDict=displayDict)
bundleList.append(bundle)
md = metadata + ' r-i color'
plotDict = {'label': md,
'title': '%s: Fraction with potential r-i color measurement %s' % (runName, md)}
plotDict.update(basicPlotDict)
metric = metrics.ColorDeterminationMetric(nPairs=nPairs, snrLimit=snrLimit, nHours=nHours,
bOne='r', bTwo='i', **colkwargs)
bundle = mb.MoMetricBundle(metric, slicer, constraint,
runName=runName, metadata=md,
plotDict=plotDict, plotFuncs=plotFuncs,
summaryMetrics=None,
displayDict=displayDict)
bundleList.append(bundle)
md = metadata + ' i-z color'
plotDict = {'label': md,
'title': '%s: Fraction with potential i-z color measurement %s' % (runName, md)}
plotDict.update(basicPlotDict)
metric = metrics.ColorDeterminationMetric(nPairs=nPairs, snrLimit=snrLimit, nHours=nHours,
bOne='i', bTwo='z', **colkwargs)
bundle = mb.MoMetricBundle(metric, slicer, constraint,
runName=runName, metadata=md,
plotDict=plotDict, plotFuncs=plotFuncs,
summaryMetrics=None,
displayDict=displayDict)
bundleList.append(bundle)
md = metadata + ' z-y color'
plotDict = {'label': md,
'title': '%s: Fraction with potential z-y color measurement %s' % (runName, md)}
plotDict.update(basicPlotDict)
metric = metrics.ColorDeterminationMetric(nPairs=nPairs, snrLimit=snrLimit, nHours=nHours,
bOne='z', bTwo='y', **colkwargs)
bundle = mb.MoMetricBundle(metric, slicer, constraint,
runName=runName, metadata=md,
plotDict=plotDict, plotFuncs=plotFuncs,
summaryMetrics=None,
displayDict=displayDict)
bundleList.append(bundle)
# Set the runName for all bundles and return the bundleDict.
for b in bundleList:
b.setRunName(runName)
return mb.makeBundlesDictFromList(bundleList), plotBundles
def makeBundleList(dbFile, runName=None, nside=64, benchmark='design',
lonCol='fieldRA', latCol='fieldDec', seeingCol='FWHMgeom'):
"""
make a list of metricBundle objects to look at the scientific performance
of an opsim run.
"""
# List to hold everything we're going to make
bundleList = []
# List to hold metrics that shouldn't be saved
noSaveBundleList = []
# Connect to the databse
opsimdb = utils.connectOpsimDb(dbFile)
if runName is None:
runName = os.path.basename(dbFile).replace('_sqlite.db', '')
# Fetch the proposal ID values from the database
propids, propTags = opsimdb.fetchPropInfo()
# Fetch the telescope location from config
lat, lon, height = opsimdb.fetchLatLonHeight()
# Add metadata regarding dithering/non-dithered.
commonname = ''.join([a for a in lonCol if a in latCol])
if commonname == 'field':
slicermetadata = ' (non-dithered)'
else:
slicermetadata = ' (%s)' % (commonname)
# Construct a WFD SQL where clause so multiple propIDs can query by WFD:
wfdWhere = utils.createSQLWhere('WFD', propTags)
print '#FYI: WFD "where" clause: %s' % (wfdWhere)
ddWhere = utils.createSQLWhere('DD', propTags)
print '#FYI: DD "where" clause: %s' % (ddWhere)
# Set up benchmark values, scaled to length of opsim run.
runLength = opsimdb.fetchRunLength()
if benchmark == 'requested':
# Fetch design values for seeing/skybrightness/single visit depth.
benchmarkVals = utils.scaleBenchmarks(runLength, benchmark='design')
# Update nvisits with requested visits from config files.
benchmarkVals['nvisits'] = opsimdb.fetchRequestedNvisits(propId=propTags['WFD'])
# Calculate expected coadded depth.
benchmarkVals['coaddedDepth'] = utils.calcCoaddedDepth(benchmarkVals['nvisits'],
benchmarkVals['singleVisitDepth'])
elif (benchmark == 'stretch') or (benchmark == 'design'):
# Calculate benchmarks for stretch or design.
benchmarkVals = utils.scaleBenchmarks(runLength, benchmark=benchmark)
benchmarkVals['coaddedDepth'] = utils.calcCoaddedDepth(benchmarkVals['nvisits'],
benchmarkVals['singleVisitDepth'])
else:
raise ValueError('Could not recognize benchmark value %s, use design, stretch or requested.'
% (benchmark))
# Check that nvisits is not set to zero (for very short run length).
for f in benchmarkVals['nvisits']:
if benchmarkVals['nvisits'][f] == 0:
print 'Updating benchmark nvisits value in %s to be nonzero' % (f)
benchmarkVals['nvisits'][f] = 1
# Set values for min/max range of nvisits for All/WFD and DD plots. These are somewhat arbitrary.
nvisitsRange = {}
nvisitsRange['all'] = {'u': [20, 80], 'g': [50, 150], 'r': [100, 250],
'i': [100, 250], 'z': [100, 300], 'y': [100, 300]}
nvisitsRange['DD'] = {'u': [6000, 10000], 'g': [2500, 5000], 'r': [5000, 8000],
'i': [5000, 8000], 'z': [7000, 10000], 'y': [5000, 8000]}
# Scale these ranges for the runLength.
scale = runLength / 10.0
for prop in nvisitsRange:
for f in nvisitsRange[prop]:
for i in [0, 1]:
nvisitsRange[prop][f][i] = int(np.floor(nvisitsRange[prop][f][i] * scale))
# Filter list, and map of colors (for plots) to filters.
filters = ['u', 'g', 'r', 'i', 'z', 'y']
colors = {'u': 'cyan', 'g': 'g', 'r': 'y', 'i': 'r', 'z': 'm', 'y': 'k'}
filtorder = {'u': 1, 'g': 2, 'r': 3, 'i': 4, 'z': 5, 'y': 6}
# Easy way to run through all fi
# Set up a list of common summary stats
commonSummary = [metrics.MeanMetric(), metrics.RobustRmsMetric(), metrics.MedianMetric(),
metrics.PercentileMetric(metricName='25th%ile', percentile=25),
metrics.PercentileMetric(metricName='75th%ile', percentile=75),
metrics.MinMetric(), metrics.MaxMetric()]
allStats = commonSummary
# Set up some 'group' labels
reqgroup = 'A: Required SRD metrics'
depthgroup = 'B: Depth per filter'
uniformitygroup = 'C: Uniformity'
airmassgroup = 'D: Airmass distribution'
seeinggroup = 'E: Seeing distribution'
transgroup = 'F: Transients'
sngroup = 'G: SN Ia'
altAzGroup = 'H: Alt Az'
rangeGroup = 'I: Range of Dates'
intergroup = 'J: Inter-Night'
phaseGroup = 'K: Max Phase Gap'
NEOGroup = 'L: NEO Detection'
# Set up an object to track the metricBundles that we want to combine into merged plots.
mergedHistDict = {}
# Set the histogram merge function.
mergeFunc = plots.HealpixHistogram()
keys = ['NVisits', 'coaddm5', 'NormEffTime', 'Minseeing', 'seeingAboveLimit', 'minAirmass',
'fracAboveAirmass']
for key in keys:
mergedHistDict[key] = plots.PlotBundle(plotFunc=mergeFunc)
##
# Calculate the fO metrics for all proposals and WFD only.
order = 0
for prop in ('All prop', 'WFD only'):
if prop == 'All prop':
metadata = 'All Visits' + slicermetadata
sqlconstraint = ''
if prop == 'WFD only':
metadata = 'WFD only' + slicermetadata
sqlconstraint = '%s' % (wfdWhere)
# Configure the count metric which is what is used for f0 slicer.
m1 = metrics.CountMetric(col='expMJD', metricName='fO')
plotDict = {'xlabel': 'Number of Visits', 'Asky': benchmarkVals['Area'],
'Nvisit': benchmarkVals['nvisitsTotal'], 'xMin': 0, 'xMax': 1500}
summaryMetrics = [metrics.fOArea(nside=nside, norm=False, metricName='fOArea: Nvisits (#)',
Asky=benchmarkVals['Area'], Nvisit=benchmarkVals['nvisitsTotal']),
metrics.fOArea(nside=nside, norm=True, metricName='fOArea: Nvisits/benchmark',
Asky=benchmarkVals['Area'], Nvisit=benchmarkVals['nvisitsTotal']),
metrics.fONv(nside=nside, norm=False, metricName='fONv: Area (sqdeg)',
Asky=benchmarkVals['Area'], Nvisit=benchmarkVals['nvisitsTotal']),
metrics.fONv(nside=nside, norm=True, metricName='fONv: Area/benchmark',
Asky=benchmarkVals['Area'], Nvisit=benchmarkVals['nvisitsTotal'])]
caption = 'The FO metric evaluates the overall efficiency of observing. '
caption += ('fOArea: Nvisits = %.1f sq degrees receive at least this many visits out of %d. '
% (benchmarkVals['Area'], benchmarkVals['nvisitsTotal']))
caption += ('fONv: Area = this many square degrees out of %.1f receive at least %d visits.'
% (benchmarkVals['Area'], benchmarkVals['nvisitsTotal']))
displayDict = {'group': reqgroup, 'subgroup': 'F0', 'displayOrder': order, 'caption': caption}
order += 1
slicer = slicers.HealpixSlicer(nside=nside, lonCol=lonCol, latCol=latCol)
bundle = metricBundles.MetricBundle(m1, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, summaryMetrics=summaryMetrics,
plotFuncs=[plots.FOPlot()],
runName=runName, metadata=metadata)
bundleList.append(bundle)
###
# Calculate the Rapid Revisit Metrics.
order = 0
metadata = 'All Visits' + slicermetadata
sqlconstraint = ''
dTmin = 40.0 # seconds
dTmax = 30.0 # minutes
minNvisit = 100
pixArea = float(hp.nside2pixarea(nside, degrees=True))
scale = pixArea * hp.nside2npix(nside)
cutoff1 = 0.15
extraStats1 = [metrics.FracBelowMetric(cutoff=cutoff1, scale=scale, metricName='Area (sq deg)')]
extraStats1.extend(commonSummary)
slicer = slicers.HealpixSlicer(nside=nside, lonCol=lonCol, latCol=latCol)
m1 = metrics.RapidRevisitMetric(metricName='RapidRevisitUniformity',
dTmin=dTmin / 60.0 / 60.0 / 24.0, dTmax=dTmax / 60.0 / 24.0,
minNvisits=minNvisit)
plotDict = {'xMin': 0, 'xMax': 1}
summaryStats = extraStats1
caption = 'Deviation from uniformity for short revisit timescales, between %s and %s seconds, ' % (
dTmin, dTmax)
caption += 'for pointings with at least %d visits in this time range. ' % (minNvisit)
caption += 'Summary statistic "Area" below indicates the area on the sky which has a '
caption += 'deviation from uniformity of < %.2f.' % (cutoff1)
displayDict = {'group': reqgroup, 'subgroup': 'Rapid Revisit', 'displayOrder': order,
'caption': caption}
bundle = metricBundles.MetricBundle(m1, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, summaryMetrics=summaryStats,
runName=runName, metadata=metadata)
bundleList.append(bundle)
order += 1
m2 = metrics.NRevisitsMetric(dT=dTmax)
plotDict = {'xMin': 0, 'xMax': 1000, 'logScale': True}
cutoff2 = 800
extraStats2 = [metrics.FracAboveMetric(cutoff=cutoff2, scale=scale, metricName='Area (sq deg)')]
extraStats2.extend(commonSummary)
caption = 'Number of consecutive visits with return times faster than %.1f minutes, ' % dTmax
caption += 'in any filter, all proposals. '
caption += 'Summary statistic "Area" below indicates the area on the sky which has more than '
caption += '%d revisits within this time window.' % (cutoff2)
summaryStats = extraStats2
displayDict = {'group': reqgroup, 'subgroup': 'Rapid Revisit', 'displayOrder': order,
'caption': caption}
bundle = metricBundles.MetricBundle(m2, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, summaryMetrics=summaryStats,
runName=runName, metadata=metadata)
bundleList.append(bundle)
order += 1
m3 = metrics.NRevisitsMetric(dT=dTmax, normed=True)
plotDict = {'xMin': 0, 'xMax': 1, 'cbarFormat': '%.1f'}
cutoff3 = 0.6
extraStats3 = [metrics.FracAboveMetric(cutoff=cutoff3, scale=scale, metricName='Area (sq deg)')]
extraStats3.extend(commonSummary)
summaryStats = extraStats3
caption = 'Fraction of total visits where consecutive visits have return times faster '
caption += 'than %.1f minutes, in any filter, all proposals. ' % (dTmax)
caption += 'Summary statistic "Area" below indicates the area on the sky which has more '
caption += 'than %d revisits within this time window.' % (cutoff3)
displayDict = {'group': reqgroup, 'subgroup': 'Rapid Revisit', 'displayOrder': order,
'caption': caption}
bundle = metricBundles.MetricBundle(m3, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, summaryMetrics=summaryStats,
runName=runName, metadata=metadata)
bundleList.append(bundle)
order += 1
# And add a histogram of the time between quick revisits.
binMin = 0
binMax = 120.
binsize = 3.
bins_metric = np.arange(binMin / 60.0 / 24.0, (binMax + binsize) / 60. / 24., binsize / 60. / 24.)
bins_plot = bins_metric * 24.0 * 60.0
m1 = metrics.TgapsMetric(bins=bins_metric, metricName='dT visits')
plotDict = {'bins': bins_plot, 'xlabel': 'dT (minutes)'}
caption = ('Histogram of the time between consecutive revisits (<%.1f minutes), over entire sky.'
% (binMax))
displayDict = {'group': reqgroup, 'subgroup': 'Rapid Revisit', 'order': order,
'caption': caption}
slicer = slicers.HealpixSlicer(nside=nside, lonCol=lonCol, latCol=latCol)
plotFunc = plots.SummaryHistogram()
bundle = metricBundles.MetricBundle(m1, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, runName=runName,
metadata=metadata, plotFuncs=[plotFunc])
bundleList.append(bundle)
order += 1
##
# Trigonometric parallax and proper motion @ r=20 and r=24
slicer = slicers.HealpixSlicer(nside=nside, lonCol=lonCol, latCol=latCol)
sqlconstraint = ''
order = 0
metric = metrics.ParallaxMetric(metricName='Parallax 20', rmag=20, seeingCol=seeingCol)
summaryStats = allStats
plotDict = {'cbarFormat': '%.1f', 'xMin': 0, 'xMax': 3}
displayDict = {'group': reqgroup, 'subgroup': 'Parallax', 'order': order,
'caption': 'Parallax precision at r=20. (without refraction).'}
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, summaryMetrics=summaryStats,
runName=runName, metadata=metadata)
bundleList.append(bundle)
order += 1
metric = metrics.ParallaxMetric(metricName='Parallax 24', rmag=24, seeingCol=seeingCol)
plotDict = {'cbarFormat': '%.1f', 'xMin': 0, 'xMax': 10}
displayDict = {'group': reqgroup, 'subgroup': 'Parallax', 'order': order,
'caption': 'Parallax precision at r=24. (without refraction).'}
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, summaryMetrics=summaryStats,
runName=runName, metadata=metadata)
bundleList.append(bundle)
order += 1
metric = metrics.ParallaxMetric(metricName='Parallax Normed', rmag=24, normalize=True,
seeingCol=seeingCol)
plotDict = {'xMin': 0.5, 'xMax': 1.0}
displayDict = {'group': reqgroup, 'subgroup': 'Parallax', 'order': order,
'caption':
'Normalized parallax (normalized to optimum observation cadence, 1=optimal).'}
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, summaryMetrics=summaryStats,
runName=runName, metadata=metadata)
bundleList.append(bundle)
order += 1
metric = metrics.ParallaxCoverageMetric(metricName='Parallax Coverage 20', rmag=20, seeingCol=seeingCol)
plotDict = {}
caption = "Parallax factor coverage for an r=20 star (0 is bad, 0.5-1 is good). "
caption += "One expects the parallax factor coverage to vary because stars on the ecliptic "
caption += "can be observed when they have no parallax offset while stars at the pole are always "
caption += "offset by the full parallax offset."""
displayDict = {'group': reqgroup, 'subgroup': 'Parallax', 'order': order,
'caption': caption}
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, summaryMetrics=summaryStats,
runName=runName, metadata=metadata)
bundleList.append(bundle)
order += 1
metric = metrics.ParallaxCoverageMetric(metricName='Parallax Coverage 24', rmag=24, seeingCol=seeingCol)
plotDict = {}
caption = "Parallax factor coverage for an r=24 star (0 is bad, 0.5-1 is good). "
caption += "One expects the parallax factor coverage to vary because stars on the ecliptic "
caption += "can be observed when they have no parallax offset while stars at the pole are always "
caption += "offset by the full parallax offset."""
displayDict = {'group': reqgroup, 'subgroup': 'Parallax', 'order': order,
'caption': caption}
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, summaryMetrics=summaryStats,
runName=runName, metadata=metadata)
bundleList.append(bundle)
order += 1
metric = metrics.ParallaxHADegenMetric(metricName='Parallax-DCR degeneracy 20', rmag=20,
seeingCol=seeingCol)
plotDict = {}
caption = 'Correlation between parallax offset magnitude and hour angle an r=20 star.'
caption += ' (0 is good, near -1 or 1 is bad).'
displayDict = {'group': reqgroup, 'subgroup': 'Parallax', 'order': order,
'caption': caption}
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, summaryMetrics=summaryStats,
runName=runName, metadata=metadata)
bundleList.append(bundle)
order += 1
metric = metrics.ParallaxHADegenMetric(metricName='Parallax-DCR degeneracy 24', rmag=24,
seeingCol=seeingCol)
plotDict = {}
caption = 'Correlation between parallax offset magnitude and hour angle an r=24 star.'
caption += ' (0 is good, near -1 or 1 is bad).'
displayDict = {'group': reqgroup, 'subgroup': 'Parallax', 'order': order,
'caption': caption}
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, summaryMetrics=summaryStats,
runName=runName, metadata=metadata)
bundleList.append(bundle)
order += 1
metric = metrics.ProperMotionMetric(metricName='Proper Motion 20', rmag=20, seeingCol=seeingCol)
summaryStats = allStats
plotDict = {'xMin': 0, 'xMax': 3}
displayDict = {'group': reqgroup, 'subgroup': 'Proper Motion', 'order': order,
'caption': 'Proper Motion precision at r=20.'}
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, summaryMetrics=summaryStats,
runName=runName, metadata=metadata)
bundleList.append(bundle)
order += 1
metric = metrics.ProperMotionMetric(rmag=24, metricName='Proper Motion 24', seeingCol=seeingCol)
summaryStats = allStats
plotDict = {'xMin': 0, 'xMax': 10}
displayDict = {'group': reqgroup, 'subgroup': 'Proper Motion', 'order': order,
'caption': 'Proper Motion precision at r=24.'}
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, summaryMetrics=summaryStats,
runName=runName, metadata=metadata)
bundleList.append(bundle)
order += 1
metric = metrics.ProperMotionMetric(rmag=24, normalize=True, metricName='Proper Motion Normed',
seeingCol=seeingCol)
plotDict = {'xMin': 0.2, 'xMax': 0.7}
caption = 'Normalized proper motion at r=24. '
caption += '(normalized to optimum observation cadence - start/end. 1=optimal).'
displayDict = {'group': reqgroup, 'subgroup': 'Proper Motion', 'order': order,
'caption': caption}
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, summaryMetrics=summaryStats,
runName=runName, metadata=metadata)
bundleList.append(bundle)
order += 1
##
# Calculate the time uniformity in each filter, for each year.
order = 0
slicer = slicers.HealpixSlicer(nside=nside, lonCol=lonCol, latCol=latCol)
plotFuncs = [plots.TwoDMap()]
step = 0.5
bins = np.arange(0, 365.25 * 10 + 40, 40) - step
metric = metrics.AccumulateUniformityMetric(bins=bins)
plotDict = {'xlabel': 'Night (days)', 'xextent': [bins.min(
) + step, bins.max() + step], 'cbarTitle': 'Uniformity'}
for f in filters:
sqlconstraint = 'filter = "%s"' % (f)
caption = 'Deviation from uniformity in %s band. ' % f
caption += 'Northern Healpixels are at the top of the image.'
caption += '(0=perfectly uniform, 1=perfectly nonuniform).'
displayDict = {'group': uniformitygroup, 'subgroup': 'per night',
'order': filtorder[f], 'caption': caption}
metadata = '%s band' % (f) + slicermetadata
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, runName=runName, metadata=metadata,
plotFuncs=plotFuncs)
noSaveBundleList.append(bundle)
##
# Depth metrics.
slicer = slicers.HealpixSlicer(nside=nside, lonCol=lonCol, latCol=latCol)
for f in filters:
propCaption = '%s band, all proposals %s' % (f, slicermetadata)
sqlconstraint = 'filter = "%s"' % (f)
metadata = '%s band' % (f) + slicermetadata
# Number of visits.
metric = metrics.CountMetric(col='expMJD', metricName='NVisits')
plotDict = {'xlabel': 'Number of visits',
'xMin': nvisitsRange['all'][f][0],
'xMax': nvisitsRange['all'][f][1], 'binsize': 5,
'logScale': True, 'nTicks': 4, 'colorMin': 1}
summaryStats = allStats
displayDict = {'group': depthgroup, 'subgroup': 'Nvisits', 'order': filtorder[f],
'caption': 'Number of visits in filter %s, %s.' % (f, propCaption)}
histMerge = {'color': colors[f], 'label': '%s' % (f),
'binsize': 5,
'xMin': nvisitsRange['all'][f][0], 'xMax': nvisitsRange['all'][f][1],
'legendloc': 'upper right'}
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, runName=runName, metadata=metadata,
summaryMetrics=summaryStats)
mergedHistDict['NVisits'].addBundle(bundle, plotDict=histMerge)
bundleList.append(bundle)
# Coadded depth.
metric = metrics.Coaddm5Metric()
plotDict = {'zp': benchmarkVals['coaddedDepth'][f], 'xMin': -0.8, 'xMax': 0.8,
'xlabel': 'coadded m5 - %.1f' % benchmarkVals['coaddedDepth'][f]}
summaryStats = allStats
histMerge = {'legendloc': 'upper right', 'color': colors[f], 'label': '%s' % f, 'binsize': .02,
'xlabel': 'coadded m5 - benchmark value'}
caption = ('Coadded depth in filter %s, with %s value subtracted (%.1f), %s. '
% (f, benchmark, benchmarkVals['coaddedDepth'][f], propCaption))
caption += 'More positive numbers indicate fainter limiting magnitudes.'
displayDict = {'group': depthgroup, 'subgroup': 'Coadded Depth',
'order': filtorder[f], 'caption': caption}
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, runName=runName, metadata=metadata,
summaryMetrics=summaryStats)
mergedHistDict['coaddm5'].addBundle(bundle, plotDict=histMerge)
bundleList.append(bundle)
# Effective time.
metric = metrics.TeffMetric(metricName='Normalized Effective Time', normed=True,
fiducialDepth=benchmarkVals['singleVisitDepth'])
plotDict = {'xMin': 0.1, 'xMax': 1.1}
summaryStats = allStats
histMerge = {'legendLoc': 'upper right', 'color': colors[f], 'label': '%s' % f, 'binsize': 0.02}
caption = ('"Time Effective" in filter %s, calculated with fiducial single-visit depth of %s mag. '
% (f, benchmarkVals['singleVisitDepth'][f]))
caption += 'Normalized by the fiducial time effective, if every observation was at '
caption += 'the fiducial depth.'
displayDict = {'group': depthgroup, 'subgroup': 'Time Eff.',
'order': filtorder[f], 'caption': caption}
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, runName=runName, metadata=metadata,
summaryMetrics=summaryStats)
mergedHistDict['NormEffTime'].addBundle(bundle, plotDict=histMerge)
bundleList.append(bundle)
# Put in a z=0.5 Type Ia SN, based on Cambridge 2015 workshop notebook.
# Check for 1) detection in any band, 2) detection on the rise in any band,
# 3) good characterization
peaks = {'uPeak': 25.9, 'gPeak': 23.6, 'rPeak': 22.6, 'iPeak': 22.7, 'zPeak': 22.7, 'yPeak': 22.8}
peakTime = 15.
transDuration = peakTime + 30. # Days
metric = metrics.TransientMetric(riseSlope=-2. / peakTime, declineSlope=1.4 / 30.0,
transDuration=transDuration, peakTime=peakTime,
surveyDuration=runLength,
metricName='SNDetection', **peaks)
caption = 'Fraction of z=0.5 type Ia SN that are detected in any filter'
displayDict = {'group': transgroup, 'subgroup': 'Detected', 'caption': caption}
sqlconstraint = ''
metadata = '' + slicermetadata
plotDict = {}
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, runName=runName, metadata=metadata)
bundleList.append(bundle)
metric = metrics.TransientMetric(riseSlope=-2. / peakTime, declineSlope=1.4 / 30.0,
transDuration=transDuration, peakTime=peakTime,
surveyDuration=runLength,
nPrePeak=1, metricName='SNAlert', **peaks)
caption = 'Fraction of z=0.5 type Ia SN that are detected pre-peak in any filter'
displayDict = {'group': transgroup, 'subgroup': 'Detected on the rise', 'caption': caption}
plotDict = {}
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, runName=runName, metadata=metadata)
bundleList.append(bundle)
metric = metrics.TransientMetric(riseSlope=-2. / peakTime, declineSlope=1.4 / 30.,
transDuration=transDuration, peakTime=peakTime,
surveyDuration=runLength, metricName='SNLots',
nFilters=3, nPrePeak=3, nPerLC=2, **peaks)
caption = 'Fraction of z=0.5 type Ia SN that are observed 6 times, 3 pre-peak, '
caption += '3 post-peak, with observations in 3 filters'
displayDict = {'group': transgroup, 'subgroup': 'Well observed', 'caption': caption}
sqlconstraint = 'filter="r" or filter="g" or filter="i" or filter="z" '
plotDict = {}
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, runName=runName, metadata=metadata)
bundleList.append(bundle)
# Good seeing in r/i band metrics, including in first/second years.
order = 0
for tcolor, tlabel, timespan in zip(['k', 'g', 'r'], ['10 years', '1 year', '2 years'],
['', ' and night<=365', ' and night<=730']):
order += 1
for f in (['r', 'i']):
sqlconstraint = 'filter = "%s" %s' % (f, timespan)
propCaption = '%s band, all proposals %s, over %s.' % (f, slicermetadata, tlabel)
metadata = '%s band, %s' % (f, tlabel) + slicermetadata
seeing_limit = 0.7
airmass_limit = 1.2
metric = metrics.MinMetric(col=seeingCol)
summaryStats = allStats
plotDict = {'xMin': 0.35, 'xMax': 1.5, 'color': tcolor}
displayDict = {'group': seeinggroup, 'subgroup': 'Best Seeing',
'order': filtorder[f] * 100 + order,
'caption': 'Minimum FWHMgeom values in %s.' % (propCaption)}
histMerge = {'label': '%s %s' % (f, tlabel), 'color': tcolor,
'binsize': 0.03, 'xMin': 0.35, 'xMax': 1.5, 'legendloc': 'upper right'}
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, runName=runName, metadata=metadata,
summaryMetrics=summaryStats)
mergedHistDict['Minseeing'].addBundle(bundle, plotDict=histMerge)
bundleList.append(bundle)
metric = metrics.FracAboveMetric(col=seeingCol, cutoff=seeing_limit)
summaryStats = allStats
plotDict = {'xMin': 0, 'xMax': 1.1, 'color': tcolor}
displayDict = {'group': seeinggroup, 'subgroup': 'Good seeing fraction',
'order': filtorder[f] * 100 + order,
'caption': 'Fraction of total images with FWHMgeom worse than %.1f, in %s'
% (seeing_limit, propCaption)}
histMerge = {'color': tcolor, 'label': '%s %s' % (f, tlabel),
'binsize': 0.05, 'legendloc': 'upper right'}
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, runName=runName, metadata=metadata,
summaryMetrics=summaryStats)
mergedHistDict['seeingAboveLimit'].addBundle(bundle, plotDict=histMerge)
bundleList.append(bundle)
metric = metrics.MinMetric(col='airmass')
plotDict = {'xMin': 1, 'xMax': 1.5, 'color': tcolor}
summaryStats = allStats
displayDict = {'group': airmassgroup, 'subgroup': 'Best Airmass',
'order': filtorder[f] * 100 + order, 'caption':
'Minimum airmass in %s.' % (propCaption)}
histMerge = {'color': tcolor, 'label': '%s %s' % (f, tlabel),
'binsize': 0.03, 'legendloc': 'upper right'}
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, runName=runName, metadata=metadata,
summaryMetrics=summaryStats)
mergedHistDict['minAirmass'].addBundle(bundle, plotDict=histMerge)
bundleList.append(bundle)
metric = metrics.FracAboveMetric(col='airmass', cutoff=airmass_limit)
plotDict = {'xMin': 0, 'xMax': 1, 'color': tcolor}
summaryStats = allStats
displayDict = {'group': airmassgroup, 'subgroup': 'Low airmass fraction',
'order': filtorder[f] * 100 + order, 'caption':
'Fraction of total images with airmass higher than %.2f, in %s'
% (airmass_limit, propCaption)}
histMerge = {'color': tcolor, 'label': '%s %s' % (
f, tlabel), 'binsize': 0.05, 'legendloc': 'upper right'}
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, runName=runName, metadata=metadata,
summaryMetrics=summaryStats)
mergedHistDict['fracAboveAirmass'].addBundle(bundle, plotDict=histMerge)
bundleList.append(bundle)
# SNe metrics from UK workshop.
peaks = {'uPeak': 25.9, 'gPeak': 23.6, 'rPeak': 22.6, 'iPeak': 22.7, 'zPeak': 22.7, 'yPeak': 22.8}
peakTime = 15.
transDuration = peakTime + 30. # Days
metric = metrics.TransientMetric(riseSlope=-2. / peakTime, declineSlope=1.4 / 30.0,
transDuration=transDuration, peakTime=peakTime,
surveyDuration=runLength,
metricName='SNDetection', **peaks)
caption = 'Fraction of z=0.5 type Ia SN that are detected at any point in their light curve in any filter'
displayDict = {'group': sngroup, 'subgroup': 'Detected', 'caption': caption}
sqlconstraint = ''
plotDict = {}
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, runName=runName)
bundleList.append(bundle)
metric = metrics.TransientMetric(riseSlope=-2. / peakTime, declineSlope=1.4 / 30.0,
transDuration=transDuration, peakTime=peakTime,
surveyDuration=runLength,
nPrePeak=1, metricName='SNAlert', **peaks)
caption = 'Fraction of z=0.5 type Ia SN that are detected pre-peak in any filter'
displayDict = {'group': sngroup, 'subgroup': 'Detected on the rise', 'caption': caption}
plotDict = {}
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, runName=runName)
bundleList.append(bundle)
metric = metrics.TransientMetric(riseSlope=-2. / peakTime, declineSlope=1.4 / 30.,
transDuration=transDuration, peakTime=peakTime,
surveyDuration=runLength, metricName='SNLots',
nFilters=3, nPrePeak=3, nPerLC=2, **peaks)
caption = 'Fraction of z=0.5 type Ia SN that are observed 6 times, 3 pre-peak, '
caption += '3 post-peak, with observations in 3 filters'
displayDict = {'group': sngroup, 'subgroup': 'Well observed', 'caption': caption}
sqlconstraint = 'filter="r" or filter="g" or filter="i" or filter="z" '
plotDict = {}
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, runName=runName)
bundleList.append(bundle)
propIDOrderDict = {}
orderVal = 100
for propID in propids:
propIDOrderDict[propID] = orderVal
orderVal += 100
# Full range of dates:
metric = metrics.FullRangeMetric(col='expMJD')
plotFuncs = [plots.HealpixSkyMap(), plots.HealpixHistogram()]
caption = 'Time span of survey.'
sqlconstraint = ''
plotDict = {}
displayDict = {'group': rangeGroup, 'caption': caption}
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, plotDict=plotDict,
displayDict=displayDict, runName=runName)
bundleList.append(bundle)
for f in filters:
for propid in propids:
displayDict = {'group': rangeGroup, 'subgroup': propids[propid], 'caption': caption,
'order': filtorder[f]}
md = '%s, %s' % (f, propids[propid])
sql = 'filter="%s" and propID=%i' % (f, propid)
bundle = metricBundles.MetricBundle(metric, slicer, sql, plotDict=plotDict,
metadata=md, plotFuncs=plotFuncs,
displayDict=displayDict, runName=runName)
bundleList.append(bundle)
# Alt az plots
slicer = slicers.HealpixSlicer(nside=64, latCol='zenithDistance', lonCol='azimuth', useCache=False)
metric = metrics.CountMetric('expMJD', metricName='Nvisits as function of Alt/Az')
plotDict = {}
plotFuncs = [plots.LambertSkyMap()]
displayDict = {'group': altAzGroup, 'caption': 'Alt Az pointing distribution'}
for f in filters:
for propid in propids:
displayDict = {'group': altAzGroup, 'subgroup': propids[propid],
'caption': 'Alt Az pointing distribution',
'order': filtorder[f]}
md = '%s, %s' % (f, propids[propid])
sql = 'filter="%s" and propID=%i' % (f, propid)
bundle = metricBundles.MetricBundle(metric, slicer, sql, plotDict=plotDict,
plotFuncs=plotFuncs, metadata=md,
displayDict=displayDict, runName=runName)
bundleList.append(bundle)
sql = ''
md = 'all observations'
displayDict = {'group': altAzGroup, 'subgroup': 'All Observations',
'caption': 'Alt Az pointing distribution'}
bundle = metricBundles.MetricBundle(metric, slicer, sql, plotDict=plotDict,
plotFuncs=plotFuncs, metadata=md,
displayDict=displayDict, runName=runName)
bundleList.append(bundle)
# Median inter-night gap (each and all filters)
slicer = slicers.HealpixSlicer(nside=nside, lonCol=lonCol, latCol=latCol)
metric = metrics.InterNightGapsMetric(metricName='Median Inter-Night Gap')
sqls = ['filter = "%s"' % f for f in filters]
orders = [filtorder[f] for f in filters]
orders.append(0)
sqls.append('')
for sql, order in zip(sqls, orders):
displayDict = {'group': intergroup, 'subgroup': 'Median Gap', 'caption': 'Median gap between days',
'order': order}
bundle = metricBundles.MetricBundle(metric, slicer, sql, displayDict=displayDict, runName=runName)
bundleList.append(bundle)
# Max inter-night gap in r and all bands
dslicer = slicers.HealpixSlicer(nside=nside, lonCol='ditheredRA', latCol='ditheredDec')
metric = metrics.InterNightGapsMetric(metricName='Max Inter-Night Gap', reduceFunc=np.max)
plotDict = {'percentileClip': 95.}
for sql, order in zip(sqls, orders):
displayDict = {'group': intergroup, 'subgroup': 'Max Gap', 'caption': 'Max gap between nights',
'order': order}
bundle = metricBundles.MetricBundle(metric, dslicer, sql, displayDict=displayDict,
plotDict=plotDict, runName=runName)
bundleList.append(bundle)
# largest phase gap for periods
periods = [0.1, 1.0, 10., 100.]
sqls = {'u': 'filter = "u"', 'r': 'filter="r"',
'g,r,i,z': 'filter="g" or filter="r" or filter="i" or filter="z"',
'all': ''}
for sql in sqls.keys():
for period in periods:
displayDict = {'group': phaseGroup,
'subgroup': 'period=%.2f days, filter=%s' % (period, sql),
'caption': 'Maximum phase gaps'}
metric = metrics.PhaseGapMetric(nPeriods=1, periodMin=period, periodMax=period,
metricName='PhaseGap, %.1f' % period)
bundle = metricBundles.MetricBundle(metric, slicer, sqls[sql],
displayDict=displayDict, runName=runName)
bundleList.append(bundle)
# NEO XY plots
slicer = slicers.UniSlicer()
metric = metrics.PassMetric(metricName='NEODistances')
stacker = stackers.NEODistStacker()
stacker2 = stackers.EclipticStacker()
for f in filters:
plotFunc = plots.NeoDistancePlotter(eclipMax=10., eclipMin=-10.)
caption = 'Observations within 10 degrees of the ecliptic. Distance an H=22 NEO would be detected'
displayDict = {'group': NEOGroup, 'subgroup': 'xy', 'order': filtorder[f],
'caption': caption}
plotDict = {}
sqlconstraint = 'filter = "%s"' % (f)
bundle = metricBundles.MetricBundle(metric, slicer,
sqlconstraint, displayDict=displayDict,
stackerList=[stacker, stacker2],
plotDict=plotDict,
plotFuncs=[plotFunc])
noSaveBundleList.append(bundle)
# Solar elongation
sqls = ['filter = "%s"' % f for f in filters]
orders = [filtorder[f] for f in filters]
sqls.append('')
orders.append(0)
for sql, order in zip(sqls, orders):
plotFuncs = [plots.HealpixSkyMap(), plots.HealpixHistogram()]
displayDict = {'group': NEOGroup, 'subgroup': 'Solar Elongation',
'caption': 'Median solar elongation in degrees', 'order': order}
metric = metrics.MedianMetric('solarElong')
slicer = slicers.HealpixSlicer(nside=nside, lonCol=lonCol, latCol=latCol)
bundle = metricBundles.MetricBundle(metric, slicer, sql, displayDict=displayDict,
plotFuncs=plotFuncs)
bundleList.append(bundle)
plotFuncs = [plots.HealpixSkyMap(), plots.HealpixHistogram()]
displayDict = {'group': NEOGroup, 'subgroup': 'Solar Elongation',
'caption': 'Minimum solar elongation in degrees', 'order': order}
metric = metrics.MinMetric('solarElong')
slicer = slicers.HealpixSlicer(nside=nside, lonCol=lonCol, latCol=latCol)
bundle = metricBundles.MetricBundle(metric, slicer, sql, displayDict=displayDict,
plotFuncs=plotFuncs)
bundleList.append(bundle)
return (metricBundles.makeBundlesDictFromList(bundleList), mergedHistDict,
metricBundles.makeBundlesDictFromList(noSaveBundleList))
def astrometryBatch(colmap=None,
runName='opsim',
extraSql=None,
extraMetadata=None,
nside=64,
ditherStacker=None,
ditherkwargs=None):
"""Metrics for evaluating proper motion and parallax.
Parameters
----------
colmap : dict or None, opt
A dictionary with a mapping of column names. Default will use OpsimV4 column names.
runName : str, opt
The name of the simulated survey. Default is "opsim".
nside : int, opt
Nside for the healpix slicer. Default 64.
extraSql : str or None, opt
Additional sql constraint to apply to all metrics.
extraMetadata : str or None, opt
Additional metadata to apply to all results.
ditherStacker: str or lsst.sims.maf.stackers.BaseDitherStacker
Optional dither stacker to use to define ra/dec columns.
ditherkwargs: dict, opt
Optional dictionary of kwargs for the dither stacker.
Returns
-------
metricBundleDict
"""
if colmap is None:
colmap = ColMapDict('opsimV4')
bundleList = []
sql = ''
metadata = 'All visits'
# Add additional sql constraint (such as wfdWhere) and metadata, if provided.
if (extraSql is not None) and (len(extraSql) > 0):
sql = extraSql
if extraMetadata is None:
metadata = extraSql.replace('filter =', '').replace('filter=', '')
metadata = metadata.replace('"', '').replace("'", '')
if extraMetadata is not None:
metadata = extraMetadata
subgroup = metadata
raCol, decCol, degrees, ditherStacker, ditherMeta = radecCols(
ditherStacker, colmap, ditherkwargs)
# Don't want dither info in subgroup (too long), but do want it in bundle name.
metadata = combineMetadata(metadata, ditherMeta)
rmags_para = [22.4, 24.0]
rmags_pm = [20.5, 24.0]
# Set up parallax/dcr stackers.
parallaxStacker = stackers.ParallaxFactorStacker(raCol=raCol,
decCol=decCol,
dateCol=colmap['mjd'],
degrees=degrees)
dcrStacker = stackers.DcrStacker(filterCol=colmap['filter'],
altCol=colmap['alt'],
degrees=degrees,
raCol=raCol,
decCol=decCol,
lstCol=colmap['lst'],
site='LSST',
mjdCol=colmap['mjd'])
# Set up parallax metrics.
slicer = slicers.HealpixSlicer(nside=nside,
lonCol=raCol,
latCol=decCol,
latLonDeg=degrees)
subsetPlots = [plots.HealpixSkyMap(), plots.HealpixHistogram()]
displayDict = {
'group': 'Parallax',
'subgroup': subgroup,
'order': 0,
'caption': None
}
# Expected error on parallax at 10 AU.
plotmaxVals = (2.0, 15.0)
for rmag, plotmax in zip(rmags_para, plotmaxVals):
plotDict = {
'xMin': 0,
'xMax': plotmax,
'colorMin': 0,
'colorMax': plotmax
}
metric = metrics.ParallaxMetric(metricName='Parallax Error @ %.1f' %
(rmag),
rmag=rmag,
seeingCol=colmap['seeingGeom'],
filterCol=colmap['filter'],
m5Col=colmap['fiveSigmaDepth'],
normalize=False)
bundle = mb.MetricBundle(metric,
slicer,
sql,
metadata=metadata,
stackerList=[parallaxStacker, ditherStacker],
displayDict=displayDict,
plotDict=plotDict,
summaryMetrics=standardSummary(),
plotFuncs=subsetPlots)
bundleList.append(bundle)
displayDict['order'] += 1
# Parallax normalized to 'best possible' if all visits separated by 6 months.
# This separates the effect of cadence from depth.
for rmag in rmags_para:
metric = metrics.ParallaxMetric(
metricName='Normalized Parallax @ %.1f' % (rmag),
rmag=rmag,
seeingCol=colmap['seeingGeom'],
filterCol=colmap['filter'],
m5Col=colmap['fiveSigmaDepth'],
normalize=True)
bundle = mb.MetricBundle(metric,
slicer,
sql,
metadata=metadata,
stackerList=[parallaxStacker, ditherStacker],
displayDict=displayDict,
summaryMetrics=standardSummary(),
plotFuncs=subsetPlots)
bundleList.append(bundle)
displayDict['order'] += 1
# Parallax factor coverage.
for rmag in rmags_para:
metric = metrics.ParallaxCoverageMetric(
metricName='Parallax Coverage @ %.1f' % (rmag),
rmag=rmag,
m5Col=colmap['fiveSigmaDepth'],
mjdCol=colmap['mjd'],
filterCol=colmap['filter'],
seeingCol=colmap['seeingGeom'])
bundle = mb.MetricBundle(metric,
slicer,
sql,
metadata=metadata,
stackerList=[parallaxStacker, ditherStacker],
displayDict=displayDict,
summaryMetrics=standardSummary(),
plotFuncs=subsetPlots)
bundleList.append(bundle)
displayDict['order'] += 1
# Parallax problems can be caused by HA and DCR degeneracies. Check their correlation.
for rmag in rmags_para:
metric = metrics.ParallaxDcrDegenMetric(
metricName='Parallax-DCR degeneracy @ %.1f' % (rmag),
rmag=rmag,
seeingCol=colmap['seeingEff'],
filterCol=colmap['filter'],
m5Col=colmap['fiveSigmaDepth'])
caption = 'Correlation between parallax offset magnitude and hour angle for a r=%.1f star.' % (
rmag)
caption += ' (0 is good, near -1 or 1 is bad).'
bundle = mb.MetricBundle(
metric,
slicer,
sql,
metadata=metadata,
stackerList=[dcrStacker, parallaxStacker, ditherStacker],
displayDict=displayDict,
summaryMetrics=standardSummary(),
plotFuncs=subsetPlots)
bundleList.append(bundle)
displayDict['order'] += 1
# Proper Motion metrics.
displayDict = {
'group': 'Proper Motion',
'subgroup': subgroup,
'order': 0,
'caption': None
}
# Proper motion errors.
plotmaxVals = (1.0, 5.0)
for rmag, plotmax in zip(rmags_pm, plotmaxVals):
plotDict = {
'xMin': 0,
'xMax': plotmax,
'colorMin': 0,
'colorMax': plotmax
}
metric = metrics.ProperMotionMetric(
metricName='Proper Motion Error @ %.1f' % rmag,
rmag=rmag,
m5Col=colmap['fiveSigmaDepth'],
mjdCol=colmap['mjd'],
filterCol=colmap['filter'],
seeingCol=colmap['seeingGeom'],
normalize=False)
bundle = mb.MetricBundle(metric,
slicer,
sql,
metadata=metadata,
stackerList=[ditherStacker],
displayDict=displayDict,
plotDict=plotDict,
summaryMetrics=standardSummary(),
plotFuncs=subsetPlots)
bundleList.append(bundle)
displayDict['order'] += 1
# Normalized proper motion.
for rmag in rmags_pm:
metric = metrics.ProperMotionMetric(
metricName='Normalized Proper Motion @ %.1f' % rmag,
rmag=rmag,
m5Col=colmap['fiveSigmaDepth'],
mjdCol=colmap['mjd'],
filterCol=colmap['filter'],
seeingCol=colmap['seeingGeom'],
normalize=True)
bundle = mb.MetricBundle(metric,
slicer,
sql,
metadata=metadata,
stackerList=[ditherStacker],
displayDict=displayDict,
summaryMetrics=standardSummary(),
plotFuncs=subsetPlots)
bundleList.append(bundle)
displayDict['order'] += 1
# Set the runName for all bundles and return the bundleDict.
for b in bundleList:
b.setRunName(runName)
return mb.makeBundlesDictFromList(bundleList)
def fOBatch(colmap=None,
runName='opsim',
extraSql=None,
extraMetadata=None,
nside=64,
benchmarkArea=18000,
benchmarkNvisits=825,
ditherStacker=None,
ditherkwargs=None):
"""Metrics for calculating fO.
Parameters
----------
colmap : dict or None, opt
A dictionary with a mapping of column names. Default will use OpsimV4 column names.
runName : str, opt
The name of the simulated survey. Default is "opsim".
nside : int, opt
Nside for the healpix slicer. Default 64.
extraSql : str or None, opt
Additional sql constraint to apply to all metrics.
extraMetadata : str or None, opt
Additional metadata to apply to all results.
ditherStacker: str or lsst.sims.maf.stackers.BaseDitherStacker
Optional dither stacker to use to define ra/dec columns.
ditherkwargs: dict, opt
Optional dictionary of kwargs for the dither stacker.
Returns
-------
metricBundleDict
"""
if colmap is None:
colmap = ColMapDict('opsimV4')
bundleList = []
sql = ''
metadata = 'All visits'
# Add additional sql constraint (such as wfdWhere) and metadata, if provided.
if (extraSql is not None) and (len(extraSql) > 0):
sql = extraSql
if extraMetadata is None:
metadata = extraSql.replace('filter =', '').replace('filter=', '')
metadata = metadata.replace('"', '').replace("'", '')
if extraMetadata is not None:
metadata = extraMetadata
subgroup = metadata
raCol, decCol, degrees, ditherStacker, ditherMeta = radecCols(
ditherStacker, colmap, ditherkwargs)
# Don't want dither info in subgroup (too long), but do want it in bundle name.
metadata = combineMetadata(metadata, ditherMeta)
# Set up fO metric.
slicer = slicers.HealpixSlicer(nside=nside,
lonCol=raCol,
latCol=decCol,
latLonDeg=degrees)
displayDict = {'group': 'FO metrics', 'subgroup': subgroup, 'order': 0}
# Configure the count metric which is what is used for f0 slicer.
metric = metrics.CountMetric(col=colmap['mjd'], metricName='fO')
plotDict = {
'xlabel': 'Number of Visits',
'Asky': benchmarkArea,
'Nvisit': benchmarkNvisits,
'xMin': 0,
'xMax': 1500
}
summaryMetrics = [
metrics.fOArea(nside=nside,
norm=False,
metricName='fOArea',
Asky=benchmarkArea,
Nvisit=benchmarkNvisits),
metrics.fOArea(nside=nside,
norm=True,
metricName='fOArea/benchmark',
Asky=benchmarkArea,
Nvisit=benchmarkNvisits),
metrics.fONv(nside=nside,
norm=False,
metricName='fONv',
Asky=benchmarkArea,
Nvisit=benchmarkNvisits),
metrics.fONv(nside=nside,
norm=True,
metricName='fONv/benchmark',
Asky=benchmarkArea,
Nvisit=benchmarkNvisits)
]
caption = 'The FO metric evaluates the overall efficiency of observing. '
caption += (
'foNv: out of %.2f sq degrees, the area receives at least X and a median of Y visits '
'(out of %d, if compared to benchmark). ' %
(benchmarkArea, benchmarkNvisits))
caption += ('fOArea: this many sq deg (out of %.2f sq deg if compared '
'to benchmark) receives at least %d visits. ' %
(benchmarkArea, benchmarkNvisits))
displayDict['caption'] = caption
bundle = mb.MetricBundle(metric,
slicer,
sql,
plotDict=plotDict,
stackerList=[ditherStacker],
displayDict=displayDict,
summaryMetrics=summaryMetrics,
plotFuncs=[plots.FOPlot()],
metadata=metadata)
bundleList.append(bundle)
# Set the runName for all bundles and return the bundleDict.
for b in bundleList:
b.setRunName(runName)
return mb.makeBundlesDictFromList(bundleList)
def rapidRevisitBatch(colmap=None,
runName='opsim',
extraSql=None,
extraMetadata=None,
nside=64,
ditherStacker=None,
ditherkwargs=None):
"""Metrics for evaluating proper motion and parallax.
Parameters
----------
colmap : dict or None, opt
A dictionary with a mapping of column names. Default will use OpsimV4 column names.
runName : str, opt
The name of the simulated survey. Default is "opsim".
nside : int, opt
Nside for the healpix slicer. Default 64.
extraSql : str or None, opt
Additional sql constraint to apply to all metrics.
extraMetadata : str or None, opt
Additional metadata to apply to all results.
ditherStacker: str or lsst.sims.maf.stackers.BaseDitherStacker
Optional dither stacker to use to define ra/dec columns.
ditherkwargs: dict, opt
Optional dictionary of kwargs for the dither stacker.
Returns
-------
metricBundleDict
"""
if colmap is None:
colmap = ColMapDict('opsimV4')
bundleList = []
sql = ''
metadata = 'All visits'
# Add additional sql constraint (such as wfdWhere) and metadata, if provided.
if (extraSql is not None) and (len(extraSql) > 0):
sql = extraSql
if extraMetadata is None:
metadata = extraSql.replace('filter =', '').replace('filter=', '')
metadata = metadata.replace('"', '').replace("'", '')
if extraMetadata is not None:
metadata = extraMetadata
subgroup = metadata
raCol, decCol, degrees, ditherStacker, ditherMeta = radecCols(
ditherStacker, colmap, ditherkwargs)
# Don't want dither info in subgroup (too long), but do want it in bundle name.
metadata = combineMetadata(metadata, ditherMeta)
# Set up parallax metrics.
slicer = slicers.HealpixSlicer(nside=nside,
lonCol=raCol,
latCol=decCol,
latLonDeg=degrees)
subsetPlots = [plots.HealpixSkyMap(), plots.HealpixHistogram()]
displayDict = {
'group': 'Rapid Revisits',
'subgroup': subgroup,
'order': 0,
'caption': None
}
# Calculate the uniformity (KS test) of the quick revisits.
dTmin = 40.0 # seconds
dTmax = 30.0 # minutes
minNvisit = 100
pixArea = float(hp.nside2pixarea(nside, degrees=True))
scale = pixArea * hp.nside2npix(nside)
m1 = metrics.RapidRevisitUniformityMetric(
metricName='RapidRevisitUniformity',
mjdCol=colmap['mjd'],
dTmin=dTmin / 60.0 / 60.0 / 24.0,
dTmax=dTmax / 60.0 / 24.0,
minNvisits=minNvisit)
plotDict = {'xMin': 0, 'xMax': 1}
cutoff1 = 0.20
summaryStats = [
metrics.FracBelowMetric(cutoff=cutoff1,
scale=scale,
metricName='Area (sq deg)')
]
summaryStats.extend(standardSummary())
caption = 'Deviation from uniformity for short revisit timescales, between %s seconds and %s minutes, ' \
% (dTmin, dTmax)
caption += 'for pointings with at least %d visits in this time range. ' % (
minNvisit)
caption += 'Summary statistic "Area" indicates the area on the sky which has a '
caption += 'deviation from uniformity of < %.2f.' % (cutoff1)
displayDict['caption'] = caption
bundle = mb.MetricBundle(m1,
slicer,
sql,
plotDict=plotDict,
plotFuncs=subsetPlots,
stackerList=[ditherStacker],
metadata=metadata,
displayDict=displayDict,
summaryMetrics=summaryStats)
bundleList.append(bundle)
displayDict['order'] += 1
# Calculate the actual number of quick revisits.
dTmax = dTmax # time in minutes
m2 = metrics.NRevisitsMetric(dT=dTmax,
mjdCol=colmap['mjd'],
normed=False,
metricName='RapidRevisitN')
plotDict = {'xMin': 600, 'xMax': 1500, 'logScale': False}
cutoff2 = 800
summaryStats = [
metrics.FracAboveMetric(cutoff=cutoff2,
scale=scale,
metricName='Area (sq deg)')
]
summaryStats.extend(standardSummary())
caption = 'Number of consecutive visits with return times faster than %.1f minutes, ' % (
dTmax)
caption += 'in any filter, all proposals. '
caption += 'Summary statistic "Area" indicates the area on the sky which has more than '
caption += '%d revisits within this time window.' % (cutoff2)
displayDict['caption'] = caption
bundle = mb.MetricBundle(m2,
slicer,
sql,
plotDict=plotDict,
plotFuncs=subsetPlots,
stackerList=[ditherStacker],
metadata=metadata,
displayDict=displayDict,
summaryMetrics=summaryStats)
bundleList.append(bundle)
displayDict['order'] += 1
# Calculate whether a healpix gets enough rapid revisits in the right windows.
dTmin = 40.0 / 60.0 # (minutes) 40s minumum for rapid revisit range
dTpairs = 20.0 # minutes (time when pairs should start kicking in)
dTmax = 30.0 # 30 minute maximum for rapid revisit range
nOne = 82 # Number of revisits between 40s-30m required
nTwo = 28 # Number of revisits between 40s - tPairs required.
pixArea = float(hp.nside2pixarea(nside, degrees=True))
scale = pixArea * hp.nside2npix(nside)
m1 = metrics.RapidRevisitMetric(metricName='RapidRevisits',
mjdCol=colmap['mjd'],
dTmin=dTmin / 60.0 / 60.0 / 24.0,
dTpairs=dTpairs / 60.0 / 24.0,
dTmax=dTmax / 60.0 / 24.0,
minN1=nOne,
minN2=nTwo)
plotDict = {
'xMin': 0,
'xMax': 1,
'colorMin': 0,
'colorMax': 1,
'logScale': False
}
cutoff1 = 0.9
summaryStats = [
metrics.FracAboveMetric(cutoff=cutoff1,
scale=scale,
metricName='Area (sq deg)')
]
summaryStats.extend(standardSummary())
caption = 'Area that receives at least %d visits between %.3f and %.1f minutes, ' \
% (nOne, dTmin, dTmax)
caption += 'with at least %d of those visits falling between %.3f and %.1f minutes. ' \
% (nTwo, dTmin, dTpairs)
caption += 'Summary statistic "Area" indicates the area on the sky which meets this requirement.'
displayDict['caption'] = caption
bundle = mb.MetricBundle(m1,
slicer,
sql,
plotDict=plotDict,
plotFuncs=subsetPlots,
stackerList=[ditherStacker],
metadata=metadata,
displayDict=displayDict,
summaryMetrics=summaryStats)
bundleList.append(bundle)
displayDict['order'] += 1
# Set the runName for all bundles and return the bundleDict.
for b in bundleList:
b.setRunName(runName)
return mb.makeBundlesDictFromList(bundleList)
def metadataBasicsAngle(value,
colmap=None,
runName='opsim',
valueName=None,
groupName=None,
extraSql=None,
extraMetadata=None,
nside=64,
ditherStacker=None,
ditherkwargs=None):
"""Calculate basic metrics on visit metadata 'value', where value is a wrap-around angle.
Calculates extended standard metrics (with unislicer) on the quantity (all visits and per filter),
makes histogram of the value (all visits and per filter),
Parameters
----------
value : str
The column name for the quantity to evaluate. (column name in the database or created by a stacker).
colmap : dict or None, opt
A dictionary with a mapping of column names. Default will use OpsimV4 column names.
runName : str, opt
The name of the simulated survey. Default is "opsim".
valueName : str, opt
The name of the value to be reported in the resultsDb and added to the metric.
This is intended to help standardize metric comparison between sim versions.
value = name as it is in the database (seeingFwhmGeom, etc).
valueName = name to be recorded ('seeingGeom', etc.). Default is None, which will match 'value'.
groupName : str, opt
The group name for this quantity in the displayDict. Default is the same as 'valueName', capitalized.
extraSql : str, opt
Additional constraint to add to any sql constraints (e.g. 'propId=1' or 'fieldID=522').
Default None, for no additional constraints.
extraMetadata : str, opt
Additional metadata to add before any below (i.e. "WFD"). Default is None.
nside : int, opt
Nside value for healpix slicer. Default 64.
If "None" is passed, the healpixslicer-based metrics will be skipped.
ditherStacker: str or lsst.sims.maf.stackers.BaseDitherStacker
Optional dither stacker to use to define ra/dec columns.
ditherkwargs: dict, opt
Optional dictionary of kwargs for the dither stacker.
Returns
-------
metricBundleDict
"""
if colmap is None:
colmap = ColMapDict('opsimV4')
bundleList = []
if valueName is None:
valueName = value
if groupName is None:
groupName = valueName.capitalize()
subgroup = extraMetadata
else:
groupName = groupName.capitalize()
subgroup = valueName.capitalize()
if subgroup is None:
subgroup = 'All visits'
displayDict = {'group': groupName, 'subgroup': subgroup}
raCol, decCol, degrees, ditherStacker, ditherMeta = radecCols(
ditherStacker, colmap, ditherkwargs)
extraMetadata = combineMetadata(extraMetadata, ditherMeta)
# Set up basic all and per filter sql constraints.
filterlist, colors, orders, sqls, metadata = filterList(
all=True, extraSql=extraSql, extraMetadata=extraMetadata)
stackerList = [ditherStacker]
# Summarize values over all and per filter.
slicer = slicers.UniSlicer()
for f in filterlist:
for m in standardAngleMetrics(value, replace_colname=valueName):
displayDict['caption'] = '%s for %s.' % (m.name, metadata[f])
displayDict['order'] = orders[f]
bundle = mb.MetricBundle(m,
slicer,
sqls[f],
stackerList=stackerList,
metadata=metadata[f],
displayDict=displayDict)
bundleList.append(bundle)
# Histogram values over all and per filter.
for f in filterlist:
displayDict['caption'] = 'Histogram of %s' % (value)
if valueName != value:
displayDict['caption'] += ' (%s)' % (valueName)
displayDict['caption'] += ' for %s.' % (metadata[f])
displayDict['order'] = orders[f]
m = metrics.CountMetric(value, metricName='%s Histogram' % (valueName))
slicer = slicers.OneDSlicer(sliceColName=value)
bundle = mb.MetricBundle(m,
slicer,
sqls[f],
stackerList=stackerList,
metadata=metadata[f],
displayDict=displayDict)
bundleList.append(bundle)
# Make maps of min/median/max for all and per filter, per RA/Dec, with standard summary stats.
mList = []
mList.append(
metrics.MeanAngleMetric(value,
metricName='AngleMean %s' % (valueName)))
mList.append(
metrics.FullRangeAngleMetric(value,
metricName='AngleRange %s' % (valueName)))
mList.append(
metrics.RmsAngleMetric(value, metricName='AngleRms %s' % (valueName)))
slicer = slicers.HealpixSlicer(nside=nside,
latCol=decCol,
lonCol=raCol,
latLonDeg=degrees)
subsetPlots = [plots.HealpixSkyMap(), plots.HealpixHistogram()]
for f in filterlist:
for m in mList:
displayDict['caption'] = 'Map of %s' % m.name
if valueName != value:
displayDict['caption'] += ' (%s)' % value
displayDict['caption'] += ' for %s.' % metadata[f]
displayDict['order'] = orders[f]
bundle = mb.MetricBundle(m,
slicer,
sqls[f],
stackerList=stackerList,
metadata=metadata[f],
plotFuncs=subsetPlots,
displayDict=displayDict,
summaryMetrics=standardSummary())
bundleList.append(bundle)
# Set the runName for all bundles and return the bundleDict.
for b in bundleList:
b.setRunName(runName)
return mb.makeBundlesDictFromList(bundleList)
def rapidRevisitBatch(colmap=None, runName='opsim',
extraSql=None, extraMetadata=None, nside=64,
ditherStacker=None, ditherkwargs=None):
"""Metrics for evaluating proper motion and parallax.
Parameters
----------
colmap : dict or None, opt
A dictionary with a mapping of column names. Default will use OpsimV4 column names.
runName : str, opt
The name of the simulated survey. Default is "opsim".
nside : int, opt
Nside for the healpix slicer. Default 64.
extraSql : str or None, opt
Additional sql constraint to apply to all metrics.
extraMetadata : str or None, opt
Additional metadata to apply to all results.
ditherStacker: str or lsst.sims.maf.stackers.BaseDitherStacker
Optional dither stacker to use to define ra/dec columns.
ditherkwargs: dict, opt
Optional dictionary of kwargs for the dither stacker.
Returns
-------
metricBundleDict
"""
if colmap is None:
colmap = ColMapDict('opsimV4')
bundleList = []
sql = ''
metadata = 'All visits'
# Add additional sql constraint (such as wfdWhere) and metadata, if provided.
if (extraSql is not None) and (len(extraSql) > 0):
sql = extraSql
if extraMetadata is None:
metadata = extraSql.replace('filter =', '').replace('filter=', '')
metadata = metadata.replace('"', '').replace("'", '')
if extraMetadata is not None:
metadata = extraMetadata
subgroup = metadata
raCol, decCol, degrees, ditherStacker, ditherMeta = radecCols(ditherStacker, colmap, ditherkwargs)
# Don't want dither info in subgroup (too long), but do want it in bundle name.
metadata = combineMetadata(metadata, ditherMeta)
slicer = slicers.HealpixSlicer(nside=nside, lonCol=raCol, latCol=decCol, latLonDeg=degrees)
subsetPlots = [plots.HealpixSkyMap(), plots.HealpixHistogram()]
displayDict = {'group': 'Rapid Revisits', 'subgroup': subgroup,
'order': 0, 'caption': None}
"""
# Calculate the uniformity (KS test) of the quick revisits.
dTmin = 40.0 # seconds
dTmax = 30.0 # minutes
minNvisit = 100
pixArea = float(hp.nside2pixarea(nside, degrees=True))
scale = pixArea * hp.nside2npix(nside)
m1 = metrics.RapidRevisitUniformityMetric(metricName='RapidRevisitUniformity', mjdCol=colmap['mjd'],
dTmin=dTmin / 60.0 / 60.0 / 24.0, dTmax=dTmax / 60.0 / 24.0,
minNvisits=minNvisit)
plotDict = {'xMin': 0, 'xMax': 1}
cutoff1 = 0.20
summaryStats = [metrics.FracBelowMetric(cutoff=cutoff1, scale=scale, metricName='Area (sq deg)')]
summaryStats.extend(standardSummary())
caption = 'Deviation from uniformity for short revisit timescales, between %s seconds and %s minutes, ' \
% (dTmin, dTmax)
caption += 'for pointings with at least %d visits in this time range. ' % (minNvisit)
caption += 'Summary statistic "Area" indicates the area on the sky which has a '
caption += 'deviation from uniformity of < %.2f.' % (cutoff1)
displayDict['caption'] = caption
bundle = mb.MetricBundle(m1, slicer, sql, plotDict=plotDict, plotFuncs=subsetPlots,
stackerList=[ditherStacker],
metadata=metadata, displayDict=displayDict, summaryMetrics=summaryStats)
bundleList.append(bundle)
displayDict['order'] += 1
# Calculate the actual number of quick revisits.
dTmax = dTmax # time in minutes
m2 = metrics.NRevisitsMetric(dT=dTmax, mjdCol=colmap['mjd'], normed=False, metricName='RapidRevisitN')
plotDict = {'xMin': 600, 'xMax': 1500, 'logScale': False}
cutoff2 = 800
summaryStats = [metrics.FracAboveMetric(cutoff=cutoff2, scale=scale, metricName='Area (sq deg)')]
summaryStats.extend(standardSummary())
caption = 'Number of consecutive visits with return times faster than %.1f minutes, ' % (dTmax)
caption += 'in any filter, all proposals. '
caption += 'Summary statistic "Area" indicates the area on the sky which has more than '
caption += '%d revisits within this time window.' % (cutoff2)
displayDict['caption'] = caption
bundle = mb.MetricBundle(m2, slicer, sql, plotDict=plotDict, plotFuncs=subsetPlots,
stackerList=[ditherStacker],
metadata=metadata, displayDict=displayDict, summaryMetrics=summaryStats)
bundleList.append(bundle)
displayDict['order'] += 1
"""
# Calculate whether a healpix gets enough rapid revisits in the right windows.
dTmin = 40.0/60.0 # (minutes) 40s minumum for rapid revisit range
dTpairs = 20.0 # minutes (time when pairs should start kicking in)
dTmax = 30.0 # 30 minute maximum for rapid revisit range
nOne = 82 # Number of revisits between 40s-30m required
nTwo = 28 # Number of revisits between 40s - tPairs required.
pixArea = float(hp.nside2pixarea(nside, degrees=True))
scale = pixArea * hp.nside2npix(nside)
m1 = metrics.RapidRevisitMetric(metricName='RapidRevisits', mjdCol=colmap['mjd'],
dTmin=dTmin / 60.0 / 60.0 / 24.0, dTpairs = dTpairs / 60.0 / 24.0,
dTmax=dTmax / 60.0 / 24.0, minN1=nOne, minN2=nTwo)
plotDict = {'xMin': 0, 'xMax': 1, 'colorMin': 0, 'colorMax': 1, 'logScale': False}
cutoff1 = 0.9
summaryStats = [metrics.FracAboveMetric(cutoff=cutoff1, scale=scale, metricName='Area (sq deg)')]
summaryStats.extend(standardSummary())
caption = 'Area that receives at least %d visits between %.3f and %.1f minutes, ' \
% (nOne, dTmin, dTmax)
caption += 'with at least %d of those visits falling between %.3f and %.1f minutes. ' \
% (nTwo, dTmin, dTpairs)
caption += 'Summary statistic "Area" indicates the area on the sky which meets this requirement.'
displayDict['caption'] = caption
bundle = mb.MetricBundle(m1, slicer, sql, plotDict=plotDict, plotFuncs=subsetPlots,
stackerList=[ditherStacker],
metadata=metadata, displayDict=displayDict, summaryMetrics=summaryStats)
bundleList.append(bundle)
displayDict['order'] += 1
# Set the runName for all bundles and return the bundleDict.
for b in bundleList:
b.setRunName(runName)
return mb.makeBundlesDictFromList(bundleList)
def makeBundleList(dbFile,
night=1,
nside=64,
latCol='ditheredDec',
lonCol='ditheredRA'):
"""
Make a bundleList of things to run
"""
# Construct sql queries for each filter and all filters
filters = ['u', 'g', 'r', 'i', 'z', 'y']
sqls = ['night=%i and filter="%s"' % (night, f) for f in filters]
sqls.append('night=%i' % night)
bundleList = []
plotFuncs_lam = [plots.LambertSkyMap()]
reg_slicer = slicers.HealpixSlicer(nside=nside,
lonCol=lonCol,
latCol=latCol,
latLonDeg=False)
altaz_slicer = slicers.HealpixSlicer(nside=nside,
latCol='altitude',
latLonDeg=False,
lonCol='azimuth',
useCache=False)
unislicer = slicers.UniSlicer()
for sql in sqls:
# Number of exposures
metric = metrics.CountMetric('expMJD', metricName='N visits')
bundle = metricBundles.MetricBundle(metric, reg_slicer, sql)
bundleList.append(bundle)
metric = metrics.CountMetric('expMJD', metricName='N visits alt az')
bundle = metricBundles.MetricBundle(metric,
altaz_slicer,
sql,
plotFuncs=plotFuncs_lam)
bundleList.append(bundle)
metric = metrics.MeanMetric('expMJD', metricName='Mean Visit Time')
bundle = metricBundles.MetricBundle(metric, reg_slicer, sql)
bundleList.append(bundle)
metric = metrics.MeanMetric('expMJD',
metricName='Mean Visit Time alt az')
bundle = metricBundles.MetricBundle(metric,
altaz_slicer,
sql,
plotFuncs=plotFuncs_lam)
bundleList.append(bundle)
metric = metrics.CountMetric('expMJD', metricName='N_visits')
bundle = metricBundles.MetricBundle(metric, unislicer, sql)
bundleList.append(bundle)
# Need pairs in window to get a map of how well it gathered SS pairs.
# Moon phase.
metric = metrics.NChangesMetric(col='filter', metricName='Filter Changes')
bundle = metricBundles.MetricBundle(metric, unislicer, 'night=%i' % night)
bundleList.append(bundle)
metric = metrics.OpenShutterFractionMetric()
bundle = metricBundles.MetricBundle(metric, unislicer, 'night=%i' % night)
bundleList.append(bundle)
metric = metrics.MeanMetric('slewTime')
bundle = metricBundles.MetricBundle(metric, unislicer, 'night=%i' % night)
bundleList.append(bundle)
metric = metrics.MinMetric('slewTime')
bundle = metricBundles.MetricBundle(metric, unislicer, 'night=%i' % night)
bundleList.append(bundle)
metric = metrics.MaxMetric('slewTime')
bundle = metricBundles.MetricBundle(metric, unislicer, 'night=%i' % night)
bundleList.append(bundle)
# Make plots of the solar system pairs that were taken in the night
metric = metrics.PairMetric()
sql = 'night=%i and (filter ="r" or filter="g" or filter="i")' % night
bundle = metricBundles.MetricBundle(metric, reg_slicer, sql)
bundleList.append(bundle)
metric = metrics.PairMetric(metricName='z Pairs')
sql = 'night=%i and filter="z"' % night
bundle = metricBundles.MetricBundle(metric, reg_slicer, sql)
bundleList.append(bundle)
# Plot up each visit
metric = metrics.NightPointingMetric()
slicer = slicers.UniSlicer()
sql = sql = 'night=%i' % night
plotFuncs = [plots.NightPointingPlotter()]
bundle = metricBundles.MetricBundle(metric,
slicer,
sql,
plotFuncs=plotFuncs)
bundleList.append(bundle)
return metricBundles.makeBundlesDictFromList(bundleList)
def metadataMaps(value,
colmap=None,
runName='opsim',
valueName=None,
groupName=None,
extraSql=None,
extraMetadata=None,
nside=64):
"""Calculate 25/50/75 percentile values on maps across sky for a single metadata value.
TODO: handle stackers which need configuration (degrees, in particular) more automatically.
Currently have a hack for HA & normairmass.
Parameters
----------
value : str
The column name for the quantity to evaluate. (column name in the database or created by a stacker).
colmap : dict or None, opt
A dictionary with a mapping of column names. Default will use OpsimV4 column names.
runName : str, opt
The name of the simulated survey. Default is "opsim".
valueName : str, opt
The name of the value to be reported in the resultsDb and added to the metric.
This is intended to help standardize metric comparison between sim versions.
value = name as it is in the database (seeingFwhmGeom, etc).
valueName = name to be recorded ('seeingGeom', etc.). Default is None, which will match 'value'.
groupName : str, opt
The group name for this quantity in the displayDict. Default is the same as 'valueName', capitalized.
extraSql : str, opt
Additional constraint to add to any sql constraints (e.g. 'propId=1' or 'fieldID=522').
Default None, for no additional constraints.
extraMetadata : str, opt
Additional metadata to add before any below (i.e. "WFD"). Default is None.
nside : int, opt
Nside value for healpix slicer. Default 64.
If "None" is passed, the healpixslicer-based metrics will be skipped.
Returns
-------
metricBundleDict
"""
if colmap is None:
colmap = ColMapDict('opsimV4')
bundleList = []
if valueName is None:
valueName = value
if groupName is None:
groupName = valueName.capitalize()
subgroup = extraMetadata
else:
groupName = groupName.capitalize()
subgroup = valueName.capitalize()
if subgroup is None:
subgroup = 'All visits'
displayDict = {'group': groupName, 'subgroup': subgroup}
raCol, decCol, degrees, ditherStacker, ditherMeta = radecCols(
None, colmap, None)
extraMetadata = combineMetadata(extraMetadata, ditherMeta)
# Set up basic all and per filter sql constraints.
filterlist, colors, orders, sqls, metadata = filterList(
all=True, extraSql=extraSql, extraMetadata=extraMetadata)
# Hack to make HA work, but really I need to account for any stackers/colmaps.
if value == 'HA':
stackerList = [
stackers.HourAngleStacker(lstCol=colmap['lst'],
raCol=raCol,
degrees=degrees)
]
elif value == 'normairmass':
stackerList = [stackers.NormAirmassStacker(degrees=degrees)]
else:
stackerList = None
# Make maps of 25/median/75 for all and per filter, per RA/Dec, with standard summary stats.
mList = []
mList.append(
metrics.PercentileMetric(value,
percentile=25,
metricName='25thPercentile %s' % (valueName)))
mList.append(
metrics.MedianMetric(value, metricName='Median %s' % (valueName)))
mList.append(
metrics.PercentileMetric(value,
percentile=75,
metricName='75thPercentile %s' % (valueName)))
slicer = slicers.HealpixSlicer(nside=nside,
latCol=decCol,
lonCol=raCol,
latLonDeg=degrees)
subsetPlots = [plots.HealpixSkyMap(), plots.HealpixHistogram()]
for f in filterlist:
for m in mList:
displayDict['caption'] = 'Map of %s' % m.name
if valueName != value:
displayDict['caption'] += ' (%s)' % value
displayDict['caption'] += ' for %s.' % metadata[f]
displayDict['order'] = orders[f]
bundle = mb.MetricBundle(m,
slicer,
sqls[f],
stackerList=stackerList,
metadata=metadata[f],
plotFuncs=subsetPlots,
displayDict=displayDict,
summaryMetrics=standardSummary())
bundleList.append(bundle)
# Set the runName for all bundles and return the bundleDict.
for b in bundleList:
b.setRunName(runName)
plotBundles = []
return mb.makeBundlesDictFromList(bundleList), plotBundles
