def testPairFractionMetric(self):
metric = metrics.PairFractionMetric(timeCol='mjd')
times = np.arange(0, 200, 30) / 60. / 24.
data = np.core.records.fromarrays([times], names='mjd')
# These should all have pairs
result = metric.run(data)
self.assertEqual(result, 1.)
# These should have none
times = np.arange(0, 400, 100) / 60. / 24.
data = np.core.records.fromarrays([times], names='mjd')
result = metric.run(data)
self.assertEqual(result, 0.)
# If those are right, then this should be 50%
t1 = np.arange(0, 200, 30) / 60. / 24.
t2 = np.arange(0, 400, 100) / 60. / 24. + 1000
times = np.append(t1, t2)
data = np.core.records.fromarrays([times], names='mjd')
result = metric.run(data)
expected = np.size(t1)/float(np.size(t1)+np.size(t2))
self.assertEqual(result, expected)
def intraNight(colmap=None,
runName='opsim',
nside=64,
extraSql=None,
extraMetadata=None):
"""Generate a set of statistics about the pair/triplet/etc. rate within a night.
Parameters
----------
colmap : dict or None, opt
A dictionary with a mapping of column names. Default will use OpsimV4 column names.
runName : str, opt
The name of the simulated survey. Default is "opsim".
nside : int, opt
Nside for the healpix slicer. Default 64.
extraSql : str or None, opt
Additional sql constraint to apply to all metrics.
extraMetadata : str or None, opt
Additional metadata to apply to all results.
Returns
-------
metricBundleDict
"""
if colmap is None:
colmap = ColMapDict('opsimV4')
metadata = extraMetadata
if extraSql is None:
extraSql = ''
if extraSql is not None and len(extraSql) > 0:
if metadata is None:
metadata = extraSql
bundleList = []
standardStats = standardSummary()
subsetPlots = [plots.HealpixSkyMap(), plots.HealpixHistogram()]
# Look for the fraction of visits in gri where there are pairs within dtMin/dtMax.
displayDict = {
'group': 'IntraNight',
'subgroup': 'Pairs',
'caption': None,
'order': 0
}
sql = '(%s) and (filter="g" or filter="r" or filter="i")' % extraSql
md = 'gri'
if metadata is not None:
md += metadata
dtMin = 15.0
dtMax = 60.0
metric = metrics.PairFractionMetric(
mjdCol=colmap['mjd'],
minGap=dtMin,
maxGap=dtMax,
metricName='Fraction of visits in pairs (%.0f-%.0f min)' %
(dtMin, dtMax))
slicer = slicers.HealpixSlicer(nside=nside,
latCol=colmap['dec'],
lonCol=colmap['ra'],
latLonDeg=colmap['raDecDeg'])
displayDict['caption'] = 'Fraction of %s visits that have a paired visit' \
'between %.1f and %.1f minutes away. ' % (metadata, dtMin, dtMax)
displayDict[
'caption'] += 'If all visits were in pairs, this fraction would be 1.'
displayDict['order'] += 1
bundle = mb.MetricBundle(metric,
slicer,
sql,
metadata=md,
summaryMetrics=standardStats,
plotFuncs=subsetPlots,
displayDict=displayDict)
bundleList.append(bundle)
# Look at the fraction of visits which have another visit within dtMax, gri.
dtMax = 50.0
metric = metrics.NRevisitsMetric(
mjdCol=colmap['mjd'],
dT=dtMax,
normed=True,
metricName='Fraction of visits with a revisit < %.0f min' % dtMax)
displayDict['caption'] = 'Fraction of %s visits that have another visit ' \
'within %.1f min. ' % (metadata, dtMax)
displayDict[
'caption'] += 'If all visits were in pairs (only), this fraction would be 0.5.'
displayDict['order'] += 1
bundle = mb.MetricBundle(metric,
slicer,
sql,
metadata=md,
summaryMetrics=standardStats,
plotFuncs=subsetPlots,
displayDict=displayDict)
bundleList.append(bundle)
# Histogram of the time between revisits (all filters) within two hours.
binMin = 0
binMax = 120.
binsize = 5.
bins_metric = np.arange(binMin / 60.0 / 24.0,
(binMax + binsize) / 60. / 24.,
binsize / 60. / 24.)
bins_plot = bins_metric * 24.0 * 60.0
metric = metrics.TgapsMetric(bins=bins_metric,
timesCol=colmap['mjd'],
metricName='DeltaT Histogram')
slicer = slicers.HealpixSlicer(nside=nside,
latCol=colmap['dec'],
lonCol=colmap['ra'],
latLonDeg=colmap['raDecDeg'])
plotDict = {'bins': bins_plot, 'xlabel': 'dT (minutes)'}
displayDict['caption'] = 'Histogram of the time between consecutive visits to a given point ' \
'on the sky, considering visits between %.1f and %.1f minutes' % (binMin,
binMax)
displayDict['order'] += 1
plotFunc = plots.SummaryHistogram()
bundle = mb.MetricBundle(metric,
slicer,
extraSql,
plotDict=plotDict,
displayDict=displayDict,
metadata=metadata,
plotFuncs=[plotFunc])
bundleList.append(bundle)
# Set the runName for all bundles and return the bundleDict.
for b in bundleList:
b.setRunName(runName)
return mb.makeBundlesDictFromList(bundleList)
def glanceBatch(colmap=None,
runName='opsim',
nside=64,
filternames=('u', 'g', 'r', 'i', 'z', 'y'),
nyears=10,
pairnside=32,
sqlConstraint=None):
"""Generate a handy set of metrics that give a quick overview of how well a survey performed.
This is a meta-set of other batches, to some extent.
Parameters
----------
colmap : dict, opt
A dictionary with a mapping of column names. Default will use OpsimV4 column names.
run_name : str, opt
The name of the simulated survey. Default is "opsim".
nside : int, opt
The nside for the healpix slicers. Default 64.
filternames : list of str, opt
The list of individual filters to use when running metrics.
Default is ('u', 'g', 'r', 'i', 'z', 'y').
There is always an all-visits version of the metrics run as well.
nyears : int (10)
How many years to attempt to make hourglass plots for
pairnside : int (32)
nside to use for the pair fraction metric (it's slow, so nice to use lower resolution)
sqlConstraint : str or None, opt
Additional SQL constraint to apply to all metrics.
Returns
-------
metricBundleDict
"""
if isinstance(colmap, str):
raise ValueError('colmap must be a dictionary, not a string')
if colmap is None:
colmap = ColMapDict('opsimV4')
bundleList = []
if sqlConstraint is None:
sqlC = ''
else:
sqlC = '(%s) and' % sqlConstraint
sql_per_filt = [
'%s %s="%s"' % (sqlC, colmap['filter'], filtername)
for filtername in filternames
]
sql_per_and_all_filters = [sqlConstraint] + sql_per_filt
standardStats = standardSummary()
subsetPlots = [plots.HealpixSkyMap(), plots.HealpixHistogram()]
# Super basic things
displayDict = {'group': 'Basic Stats', 'order': 1}
sql = sqlConstraint
slicer = slicers.UniSlicer()
# Length of Survey
metric = metrics.FullRangeMetric(col=colmap['mjd'],
metricName='Length of Survey (days)')
bundle = metricBundles.MetricBundle(metric,
slicer,
sql,
displayDict=displayDict)
bundleList.append(bundle)
# Total number of filter changes
metric = metrics.NChangesMetric(col=colmap['filter'],
orderBy=colmap['mjd'])
bundle = metricBundles.MetricBundle(metric,
slicer,
sql,
displayDict=displayDict)
bundleList.append(bundle)
# Total open shutter fraction
metric = metrics.OpenShutterFractionMetric(
slewTimeCol=colmap['slewtime'],
expTimeCol=colmap['exptime'],
visitTimeCol=colmap['visittime'])
bundle = metricBundles.MetricBundle(metric,
slicer,
sql,
displayDict=displayDict)
bundleList.append(bundle)
# Total effective exposure time
metric = metrics.TeffMetric(m5Col=colmap['fiveSigmaDepth'],
filterCol=colmap['filter'],
normed=True)
for sql in sql_per_and_all_filters:
bundle = metricBundles.MetricBundle(metric,
slicer,
sql,
displayDict=displayDict)
bundleList.append(bundle)
# Number of observations, all and each filter
metric = metrics.CountMetric(col=colmap['mjd'],
metricName='Number of Exposures')
for sql in sql_per_and_all_filters:
bundle = metricBundles.MetricBundle(metric,
slicer,
sql,
displayDict=displayDict)
bundleList.append(bundle)
# The alt/az plots of all the pointings
slicer = slicers.HealpixSlicer(nside=nside,
latCol='zenithDistance',
lonCol=colmap['az'],
latLonDeg=colmap['raDecDeg'],
useCache=False)
stacker = stackers.ZenithDistStacker(altCol=colmap['alt'],
degrees=colmap['raDecDeg'])
metric = metrics.CountMetric(colmap['mjd'],
metricName='Nvisits as function of Alt/Az')
plotFuncs = [plots.LambertSkyMap()]
for sql in sql_per_and_all_filters:
bundle = metricBundles.MetricBundle(metric,
slicer,
sql,
plotFuncs=plotFuncs,
displayDict=displayDict,
stackerList=[stacker])
bundleList.append(bundle)
# Things to check per night
# Open Shutter per night
displayDict = {'group': 'Pointing Efficency', 'order': 2}
slicer = slicers.OneDSlicer(sliceColName=colmap['night'], binsize=1)
metric = metrics.OpenShutterFractionMetric(
slewTimeCol=colmap['slewtime'],
expTimeCol=colmap['exptime'],
visitTimeCol=colmap['visittime'])
sql = sqlConstraint
bundle = metricBundles.MetricBundle(metric,
slicer,
sql,
summaryMetrics=standardStats,
displayDict=displayDict)
bundleList.append(bundle)
# Number of filter changes per night
slicer = slicers.OneDSlicer(sliceColName=colmap['night'], binsize=1)
metric = metrics.NChangesMetric(col=colmap['filter'],
orderBy=colmap['mjd'],
metricName='Filter Changes')
bundle = metricBundles.MetricBundle(metric,
slicer,
sql,
summaryMetrics=standardStats,
displayDict=displayDict)
bundleList.append(bundle)
# A few basic maps
# Number of observations, coadded depths
displayDict = {'group': 'Basic Maps', 'order': 3}
slicer = slicers.HealpixSlicer(nside=nside,
latCol=colmap['dec'],
lonCol=colmap['ra'],
latLonDeg=colmap['raDecDeg'])
metric = metrics.CountMetric(col=colmap['mjd'])
plotDict = {'percentileClip': 95.}
for sql in sql_per_and_all_filters:
bundle = metricBundles.MetricBundle(metric,
slicer,
sql,
summaryMetrics=standardStats,
displayDict=displayDict,
plotDict=plotDict)
bundleList.append(bundle)
metric = metrics.Coaddm5Metric(m5Col=colmap['fiveSigmaDepth'])
for sql in sql_per_and_all_filters:
bundle = metricBundles.MetricBundle(metric,
slicer,
sql,
summaryMetrics=standardStats,
displayDict=displayDict)
bundleList.append(bundle)
# Checking a few basic science things
# Maybe check astrometry, observation pairs, SN
plotDict = {'percentileClip': 95.}
displayDict = {'group': 'Science', 'subgroup': 'Astrometry', 'order': 4}
stackerList = []
stacker = stackers.ParallaxFactorStacker(raCol=colmap['ra'],
decCol=colmap['dec'],
degrees=colmap['raDecDeg'],
dateCol=colmap['mjd'])
stackerList.append(stacker)
# Maybe parallax and proper motion, fraction of visits in a good pair for SS
displayDict['caption'] = r'Parallax precision of an $r=20$ flat SED star'
metric = metrics.ParallaxMetric(m5Col=colmap['fiveSigmaDepth'],
filterCol=colmap['filter'],
seeingCol=colmap['seeingGeom'])
sql = sqlConstraint
bundle = metricBundles.MetricBundle(metric,
slicer,
sql,
plotFuncs=subsetPlots,
displayDict=displayDict,
stackerList=stackerList,
plotDict=plotDict)
bundleList.append(bundle)
displayDict[
'caption'] = r'Proper motion precision of an $r=20$ flat SED star'
metric = metrics.ProperMotionMetric(m5Col=colmap['fiveSigmaDepth'],
mjdCol=colmap['mjd'],
filterCol=colmap['filter'],
seeingCol=colmap['seeingGeom'])
bundle = metricBundles.MetricBundle(metric,
slicer,
sql,
plotFuncs=subsetPlots,
displayDict=displayDict,
plotDict=plotDict)
bundleList.append(bundle)
# Solar system stuff
displayDict['caption'] = 'Fraction of observations that are in pairs'
displayDict['subgroup'] = 'Solar System'
sql = '%s (filter="g" or filter="r" or filter="i")' % sqlC
pairSlicer = slicers.HealpixSlicer(nside=pairnside,
latCol=colmap['dec'],
lonCol=colmap['ra'],
latLonDeg=colmap['raDecDeg'])
metric = metrics.PairFractionMetric(mjdCol=colmap['mjd'])
bundle = metricBundles.MetricBundle(metric,
pairSlicer,
sql,
plotFuncs=subsetPlots,
displayDict=displayDict)
bundleList.append(bundle)
# stats from the note column
if 'note' in colmap.keys():
displayDict = {'group': 'Basic Stats', 'subgroup': 'Percent stats'}
metric = metrics.StringCountMetric(col=colmap['note'],
percent=True,
metricName='Percents')
sql = ''
slicer = slicers.UniSlicer()
bundle = metricBundles.MetricBundle(metric,
slicer,
sql,
displayDict=displayDict)
bundleList.append(bundle)
displayDict['subgroup'] = 'Count Stats'
metric = metrics.StringCountMetric(col=colmap['note'],
metricName='Counts')
bundle = metricBundles.MetricBundle(metric,
slicer,
sql,
displayDict=displayDict)
bundleList.append(bundle)
for b in bundleList:
b.setRunName(runName)
# Add hourglass plots.
hrDict = hourglassBatch(colmap=colmap,
runName=runName,
nyears=nyears,
extraSql=sqlConstraint)
# Add basic slew stats.
try:
slewDict = slewBasics(colmap=colmap, runName=runName)
except KeyError as e:
warnings.warn(
'Could not add slew stats: missing required key %s from colmap' %
(e))
bd = metricBundles.makeBundlesDictFromList(bundleList)
bd.update(slewDict)
bd.update(hrDict)
return bd
def intraNight(colmap=None, runName='opsim', nside=64, extraSql=None, extraMetadata=None,
slicer=None, display_group='IntraNight', subgroup='Pairs'):
"""Generate a set of statistics about the pair/triplet/etc. rate within a night.
Parameters
----------
colmap : dict or None, opt
A dictionary with a mapping of column names. Default will use OpsimV4 column names.
runName : str, opt
The name of the simulated survey. Default is "opsim".
nside : int, opt
Nside for the healpix slicer. Default 64.
extraSql : str or None, opt
Additional sql constraint to apply to all metrics.
extraMetadata : str or None, opt
Additional metadata to apply to all results.
slicer : slicer object (None)
Optinally use something other than a HealpixSlicer
Returns
-------
metricBundleDict
"""
if colmap is None:
colmap = ColMapDict('opsimV4')
metadata = extraMetadata
if extraSql is not None and len(extraSql) > 0:
if metadata is None:
metadata = extraSql
raCol, decCol, degrees, ditherStacker, ditherMeta = radecCols(None, colmap, None)
metadata = combineMetadata(metadata, ditherMeta)
bundleList = []
standardStats = standardSummary()
subsetPlots = [plots.HealpixSkyMap(), plots.HealpixHistogram()]
if slicer is None:
slicer = slicers.HealpixSlicer(nside=nside, latCol=decCol, lonCol=raCol, latLonDeg=degrees)
# Look for the fraction of visits in gri where there are pairs within dtMin/dtMax.
displayDict = {'group': display_group, 'subgroup': subgroup, 'caption': None, 'order': 0}
if extraSql is not None and len(extraSql) > 0:
sql = '(%s) and (filter="g" or filter="r" or filter="i")' % extraSql
else:
sql = 'filter="g" or filter="r" or filter="i"'
md = 'gri'
if metadata is not None:
md += ' ' + metadata
dtMin = 10.0
dtMax = 60.0
metric = metrics.PairFractionMetric(mjdCol=colmap['mjd'], minGap=dtMin, maxGap=dtMax,
metricName='Fraction of visits in pairs (%.0f-%.0f min)' % (dtMin,
dtMax))
displayDict['caption'] = 'Fraction of %s visits that have a paired visit' \
'between %.1f and %.1f minutes away. ' % (md, dtMin, dtMax)
displayDict['caption'] += 'If all visits were in pairs, this fraction would be 1.'
displayDict['order'] += 1
bundle = mb.MetricBundle(metric, slicer, sql, metadata=md, summaryMetrics=standardStats,
plotFuncs=subsetPlots, displayDict=displayDict)
bundleList.append(bundle)
dtMin = 20.0
dtMax = 90.0
metric = metrics.PairFractionMetric(mjdCol=colmap['mjd'], minGap=dtMin, maxGap=dtMax,
metricName='Fraction of visits in pairs (%.0f-%.0f min)' % (dtMin,
dtMax))
displayDict['caption'] = 'Fraction of %s visits that have a paired visit' \
'between %.1f and %.1f minutes away. ' % (md, dtMin, dtMax)
displayDict['caption'] += 'If all visits were in pairs, this fraction would be 1.'
displayDict['order'] += 1
bundle = mb.MetricBundle(metric, slicer, sql, metadata=md, summaryMetrics=standardStats,
plotFuncs=subsetPlots, displayDict=displayDict)
bundleList.append(bundle)
# Look at the fraction of visits which have another visit within dtMax, gri.
dtMax = 60.0
metric = metrics.NRevisitsMetric(mjdCol=colmap['mjd'], dT=dtMax, normed=True,
metricName='Fraction of visits with a revisit < %.0f min' % dtMax)
displayDict['caption'] = 'Fraction of %s visits that have another visit ' \
'within %.1f min. ' % (md, dtMax)
displayDict['caption'] += 'If all visits were in pairs (only), this fraction would be 0.5.'
displayDict['order'] += 1
bundle = mb.MetricBundle(metric, slicer, sql, metadata=md, summaryMetrics=standardStats,
plotFuncs=subsetPlots, displayDict=displayDict)
bundleList.append(bundle)
# Intranight gap map, all filters. Returns value in hours.
metric = metrics.IntraNightGapsMetric(metricName='Median Intra-Night Gap', mjdCol=colmap['mjd'],
reduceFunc=np.median)
displayDict['caption'] = 'Median gap between consecutive visits within a night, all bands'
if metadata is None or len(metadata) == 0:
displayDict['caption'] += ', all proposals.'
else:
displayDict['caption'] += ', %s.' % metadata
displayDict['order'] += 1
plotDict = {'percentileClip': 95}
bundle = mb.MetricBundle(metric, slicer, extraSql, metadata=metadata, displayDict=displayDict,
plotFuncs=subsetPlots, plotDict=plotDict,
summaryMetrics=standardStats)
bundleList.append(bundle)
# Histogram the number of visits per night.
countbins = np.arange(0, 10, 1)
metric = metrics.NVisitsPerNightMetric(nightCol=colmap['night'], bins=countbins,
metricName="NVisitsPerNight")
plotDict = {'bins': countbins, 'xlabel': 'Number of visits each night'}
displayDict['caption'] = 'Histogram of the number of visits in each night, per point on the sky'
if metadata is None or len(metadata) == 0:
displayDict['caption'] += ', all proposals.'
else:
displayDict['caption'] += ', %s.' % metadata
displayDict['order'] = 0
plotFunc = plots.SummaryHistogram()
bundle = mb.MetricBundle(metric, slicer, extraSql, plotDict=plotDict,
displayDict=displayDict, metadata=metadata, plotFuncs=[plotFunc])
bundleList.append(bundle)
# Histogram of the time between revisits (all filters) within two hours.
binMin = 0
binMax = 120.
binsize = 5.
bins_metric = np.arange(binMin / 60.0 / 24.0, (binMax + binsize) / 60. / 24., binsize / 60. / 24.)
bins_plot = bins_metric * 24.0 * 60.0
metric = metrics.TgapsMetric(bins=bins_metric, timesCol=colmap['mjd'], metricName='DeltaT Histogram')
plotDict = {'bins': bins_plot, 'xlabel': 'dT (minutes)'}
displayDict['caption'] = 'Histogram of the time between consecutive visits to a given point ' \
'on the sky, considering visits between %.1f and %.1f minutes' % (binMin,
binMax)
if metadata is None or len(metadata) == 0:
displayDict['caption'] += ', all proposals.'
else:
displayDict['caption'] += ', %s.' % metadata
displayDict['order'] += 1
plotFunc = plots.SummaryHistogram()
bundle = mb.MetricBundle(metric, slicer, extraSql, plotDict=plotDict,
displayDict=displayDict, metadata=metadata, plotFuncs=[plotFunc])
bundleList.append(bundle)
# Set the runName for all bundles and return the bundleDict.
for b in bundleList:
b.setRunName(runName)
plotBundles = None
return mb.makeBundlesDictFromList(bundleList), plotBundles