def getData(opsDb, sqlconstraint):
# Define columns we want from opsim database (for metrics, slicers and stacker info).
colnames = [
'observationStartMJD', 'filter', 'fieldRA', 'fieldDec',
'observationStartLST', 'moonRA', 'moonDec', 'moonPhase', 'night'
]
# Get data from database.
simdata = opsDb.fetchMetricData(colnames, sqlconstraint)
if len(simdata) == 0:
raise Exception('No simdata found matching constraint %s' %
(sqlconstraint))
# Hack because observationStartLst in HOURS here. :(
simdata[
'observationStartLST'] = simdata['observationStartLST'] / 24.0 * 360.0
# Add stacker columns.
hourangleStacker = stackers.HourAngleStacker()
simdata = hourangleStacker.run(simdata)
filterStacker = stackers.FilterColorStacker()
simdata = filterStacker.run(simdata)
fieldIdStacker = stackers.OpSimFieldStacker()
simdata = fieldIdStacker.run(simdata)
# Fetch field data.
#fields = opsDb.fetchFieldsFromFieldTable(degreesToRadians=True)
fidx = np.unique(simdata['opsimFieldId'], return_index=True)[1]
fields = np.recarray(len(fidx),
dtype=[('opsimFieldId', np.float),
('fieldRA', np.float), ('fieldDec', np.float)])
fields['opsimFieldId'] = simdata['opsimFieldId'][fidx]
fields['fieldRA'] = np.radians(simdata['fieldRA'][fidx])
fields['fieldDec'] = np.radians(simdata['fieldDec'][fidx])
return simdata, fields
def testHAStacker(self):
"""Test the Hour Angle stacker"""
data = np.zeros(100,
dtype=list(zip(['lst', 'fieldRA'], [float, float])))
data['lst'] = np.arange(100) / 99. * np.pi * 2
stacker = stackers.HourAngleStacker()
data = stacker.run(data)
# Check that data is always wrapped
self.assertLess(np.max(data['HA']), 12.)
self.assertGreater(np.min(data['HA']), -12.)
# Check that HA is zero if lst == RA
data = np.zeros(1, dtype=list(zip(['lst', 'fieldRA'], [float, float])))
data = stacker.run(data)
self.assertEqual(data['HA'], 0.)
data = np.zeros(1, dtype=list(zip(['lst', 'fieldRA'], [float, float])))
data['lst'] = 2.
data['fieldRA'] = 2.
data = stacker.run(data)
self.assertEqual(data['HA'], 0.)
# Check a value
data = np.zeros(1, dtype=list(zip(['lst', 'fieldRA'], [float, float])))
data['lst'] = 0.
data['fieldRA'] = np.pi / 2.
data = stacker.run(data)
np.testing.assert_almost_equal(data['HA'], -6.)
def getData(opsDb, sqlconstraint):
# Define columns we want from opsim database (for metrics, slicers and
# stacker info).
colnames = [
'expMJD', 'filter', 'fieldID', 'fieldRA', 'fieldDec', 'lst', 'moonRA',
'moonDec', 'moonPhase', 'night'
]
# Get data from database.
simdata = opsDb.fetchMetricData(colnames, sqlconstraint)
if len(simdata) == 0:
raise Exception('No simdata found matching constraint %s' %
(sqlconstraint))
# Add stacker columns.
hourangleStacker = stackers.HourAngleStacker()
simdata = hourangleStacker.run(simdata)
# ZTF
filterStacker = stackers.FilterColorStacker(filterMap={
'g': 1,
'r': 2,
'i': 3
})
filterStacker.filter_rgb_map = {
'g': (0, 1, 0), # green
'r': (1, 0, 0), # red
'i': (1, 0, 1)
} # red
simdata = filterStacker.run(simdata)
# Fetch field data.
fields = opsDb.fetchFieldsFromFieldTable()
return simdata, fields
def getData(opsDb, sqlconstraint):
# Define columns we want from opsim database (for metrics, slicers and stacker info).
colnames = ['observationStartMJD', 'filter', 'fieldId', 'fieldRA', 'fieldDec', 'observationStartLST',
'moonRA', 'moonDec', 'moonPhase', 'night']
# Get data from database.
simdata = opsDb.fetchMetricData(colnames, sqlconstraint)
if len(simdata) == 0:
raise Exception('No simdata found matching constraint %s' %(sqlconstraint))
# Add stacker columns.
hourangleStacker = stackers.HourAngleStacker()
simdata = hourangleStacker.run(simdata)
filterStacker = stackers.FilterColorStacker()
simdata = filterStacker.run(simdata)
# Fetch field data.
fields = opsDb.fetchFieldsFromFieldTable(degreesToRadians=True)
return simdata, fields
def metadataMaps(value,
colmap=None,
runName='opsim',
valueName=None,
groupName=None,
extraSql=None,
extraMetadata=None,
nside=64):
"""Calculate 25/50/75 percentile values on maps across sky for a single metadata value.
TODO: handle stackers which need configuration (degrees, in particular) more automatically.
Currently have a hack for HA & normairmass.
Parameters
----------
value : str
The column name for the quantity to evaluate. (column name in the database or created by a stacker).
colmap : dict or None, opt
A dictionary with a mapping of column names. Default will use OpsimV4 column names.
runName : str, opt
The name of the simulated survey. Default is "opsim".
valueName : str, opt
The name of the value to be reported in the resultsDb and added to the metric.
This is intended to help standardize metric comparison between sim versions.
value = name as it is in the database (seeingFwhmGeom, etc).
valueName = name to be recorded ('seeingGeom', etc.). Default is None, which will match 'value'.
groupName : str, opt
The group name for this quantity in the displayDict. Default is the same as 'valueName', capitalized.
extraSql : str, opt
Additional constraint to add to any sql constraints (e.g. 'propId=1' or 'fieldID=522').
Default None, for no additional constraints.
extraMetadata : str, opt
Additional metadata to add before any below (i.e. "WFD"). Default is None.
nside : int, opt
Nside value for healpix slicer. Default 64.
If "None" is passed, the healpixslicer-based metrics will be skipped.
Returns
-------
metricBundleDict
"""
if colmap is None:
colmap = ColMapDict('opsimV4')
bundleList = []
if valueName is None:
valueName = value
if groupName is None:
groupName = valueName.capitalize()
subgroup = extraMetadata
else:
groupName = groupName.capitalize()
subgroup = valueName.capitalize()
if subgroup is None:
subgroup = 'All visits'
displayDict = {'group': groupName, 'subgroup': subgroup}
raCol, decCol, degrees, ditherStacker, ditherMeta = radecCols(
None, colmap, None)
extraMetadata = combineMetadata(extraMetadata, ditherMeta)
# Set up basic all and per filter sql constraints.
filterlist, colors, orders, sqls, metadata = filterList(
all=True, extraSql=extraSql, extraMetadata=extraMetadata)
# Hack to make HA work, but really I need to account for any stackers/colmaps.
if value == 'HA':
stackerList = [
stackers.HourAngleStacker(lstCol=colmap['lst'],
raCol=raCol,
degrees=degrees)
]
elif value == 'normairmass':
stackerList = [stackers.NormAirmassStacker(degrees=degrees)]
else:
stackerList = None
# Make maps of 25/median/75 for all and per filter, per RA/Dec, with standard summary stats.
mList = []
mList.append(
metrics.PercentileMetric(value,
percentile=25,
metricName='25thPercentile %s' % (valueName)))
mList.append(
metrics.MedianMetric(value, metricName='Median %s' % (valueName)))
mList.append(
metrics.PercentileMetric(value,
percentile=75,
metricName='75thPercentile %s' % (valueName)))
slicer = slicers.HealpixSlicer(nside=nside,
latCol=decCol,
lonCol=raCol,
latLonDeg=degrees)
subsetPlots = [plots.HealpixSkyMap(), plots.HealpixHistogram()]
for f in filterlist:
for m in mList:
displayDict['caption'] = 'Map of %s' % m.name
if valueName != value:
displayDict['caption'] += ' (%s)' % value
displayDict['caption'] += ' for %s.' % metadata[f]
displayDict['order'] = orders[f]
bundle = mb.MetricBundle(m,
slicer,
sqls[f],
stackerList=stackerList,
metadata=metadata[f],
plotFuncs=subsetPlots,
displayDict=displayDict,
summaryMetrics=standardSummary())
bundleList.append(bundle)
# Set the runName for all bundles and return the bundleDict.
for b in bundleList:
b.setRunName(runName)
plotBundles = []
return mb.makeBundlesDictFromList(bundleList), plotBundles
def metadataBasics(value,
colmap=None,
runName='opsim',
valueName=None,
groupName=None,
extraSql=None,
extraMetadata=None,
nside=64,
ditherStacker=None,
ditherkwargs=None):
"""Calculate basic metrics on visit metadata 'value' (e.g. airmass, normalized airmass, seeing..).
Calculates extended standard metrics (with unislicer) on the quantity (all visits and per filter),
makes histogram of the value (all visits and per filter),
TODO: handle stackers which need configuration (degrees, in particular) more automatically.
Currently have a hack for HA & normairmass.
Parameters
----------
value : str
The column name for the quantity to evaluate. (column name in the database or created by a stacker).
colmap : dict or None, opt
A dictionary with a mapping of column names. Default will use OpsimV4 column names.
runName : str, opt
The name of the simulated survey. Default is "opsim".
valueName : str, opt
The name of the value to be reported in the resultsDb and added to the metric.
This is intended to help standardize metric comparison between sim versions.
value = name as it is in the database (seeingFwhmGeom, etc).
valueName = name to be recorded ('seeingGeom', etc.). Default is None, which will match 'value'.
groupName : str, opt
The group name for this quantity in the displayDict. Default is the same as 'valueName', capitalized.
extraSql : str, opt
Additional constraint to add to any sql constraints (e.g. 'propId=1' or 'fieldID=522').
Default None, for no additional constraints.
extraMetadata : str, opt
Additional metadata to add before any below (i.e. "WFD"). Default is None.
nside : int, opt
Nside value for healpix slicer. Default 64.
If "None" is passed, the healpixslicer-based metrics will be skipped.
ditherStacker: str or lsst.sims.maf.stackers.BaseDitherStacker
Optional dither stacker to use to define ra/dec columns.
ditherkwargs: dict, opt
Optional dictionary of kwargs for the dither stacker.
Returns
-------
metricBundleDict
"""
if colmap is None:
colmap = ColMapDict('opsimV4')
bundleList = []
if valueName is None:
valueName = value
if groupName is None:
groupName = valueName.capitalize()
subgroup = extraMetadata
else:
groupName = groupName.capitalize()
subgroup = valueName.capitalize()
if subgroup is None:
subgroup = 'All visits'
displayDict = {'group': groupName, 'subgroup': subgroup}
raCol, decCol, degrees, ditherStacker, ditherMeta = radecCols(
ditherStacker, colmap, ditherkwargs)
extraMetadata = combineMetadata(extraMetadata, ditherMeta)
# Set up basic all and per filter sql constraints.
filterlist, colors, orders, sqls, metadata = filterList(
all=True, extraSql=extraSql, extraMetadata=extraMetadata)
# Hack to make HA work, but really I need to account for any stackers/colmaps.
if value == 'HA':
stackerList = [
stackers.HourAngleStacker(lstCol=colmap['lst'],
raCol=raCol,
degrees=degrees)
]
elif value == 'normairmass':
stackerList = [stackers.NormAirmassStacker(degrees=degrees)]
else:
stackerList = None
if ditherStacker is not None:
if stackerList is None:
stackerList = [ditherStacker]
else:
stackerList.append(ditherStacker)
# Summarize values over all and per filter (min/mean/median/max/percentiles/outliers/rms).
slicer = slicers.UniSlicer()
for f in filterlist:
for m in extendedMetrics(value, replace_colname=valueName):
displayDict['caption'] = '%s for %s.' % (m.name, metadata[f])
displayDict['order'] = orders[f]
bundle = mb.MetricBundle(m,
slicer,
sqls[f],
stackerList=stackerList,
metadata=metadata[f],
displayDict=displayDict)
bundleList.append(bundle)
# Histogram values over all and per filter.
for f in filterlist:
displayDict['caption'] = 'Histogram of %s' % (value)
if valueName != value:
displayDict['caption'] += ' (%s)' % (valueName)
displayDict['caption'] += ' for %s.' % (metadata[f])
displayDict['order'] = orders[f]
m = metrics.CountMetric(value, metricName='%s Histogram' % (valueName))
slicer = slicers.OneDSlicer(sliceColName=value)
bundle = mb.MetricBundle(m,
slicer,
sqls[f],
stackerList=stackerList,
metadata=metadata[f],
displayDict=displayDict)
bundleList.append(bundle)
# Make maps of min/median/max for all and per filter, per RA/Dec, with standard summary stats.
mList = []
mList.append(metrics.MinMetric(value, metricName='Min %s' % (valueName)))
mList.append(
metrics.MedianMetric(value, metricName='Median %s' % (valueName)))
mList.append(metrics.MaxMetric(value, metricName='Max %s' % (valueName)))
slicer = slicers.HealpixSlicer(nside=nside,
latCol=decCol,
lonCol=raCol,
latLonDeg=degrees)
subsetPlots = [plots.HealpixSkyMap(), plots.HealpixHistogram()]
for f in filterlist:
for m in mList:
displayDict['caption'] = 'Map of %s' % m.name
if valueName != value:
displayDict['caption'] += ' (%s)' % value
displayDict['caption'] += ' for %s.' % metadata[f]
displayDict['order'] = orders[f]
bundle = mb.MetricBundle(m,
slicer,
sqls[f],
stackerList=stackerList,
metadata=metadata[f],
plotFuncs=subsetPlots,
displayDict=displayDict,
summaryMetrics=standardSummary())
bundleList.append(bundle)
# Set the runName for all bundles and return the bundleDict.
for b in bundleList:
b.setRunName(runName)
return mb.makeBundlesDictFromList(bundleList)
def metadataBasics(value, colmap=None, runName='opsim',
valueName=None, groupName=None, extraSql=None, extraMetadata=None,
nside=64, filterlist=('u', 'g', 'r', 'i', 'z', 'y')):
"""Calculate basic metrics on visit metadata 'value' (e.g. airmass, normalized airmass, seeing..).
Calculates extended standard metrics (with unislicer) on the quantity (all visits and per filter),
makes histogram of the value (all visits and per filter),
TODO: handle stackers which need configuration (degrees, in particular) more automatically.
Currently have a hack for HA & normairmass.
Parameters
----------
value : str
The column name for the quantity to evaluate. (column name in the database or created by a stacker).
colmap : dict or None, opt
A dictionary with a mapping of column names. Default will use OpsimV4 column names.
runName : str, opt
The name of the simulated survey. Default is "opsim".
valueName : str, opt
The name of the value to be reported in the resultsDb and added to the metric.
This is intended to help standardize metric comparison between sim versions.
value = name as it is in the database (seeingFwhmGeom, etc).
valueName = name to be recorded ('seeingGeom', etc.). Default is None, which is set to match value.
groupName : str, opt
The group name for this quantity in the displayDict. Default is the same as 'value', capitalized.
extraSql : str, opt
Additional constraint to add to any sql constraints (e.g. 'propId=1' or 'fieldID=522').
Default None, for no additional constraints.
extraMetadata : str, opt
Additional metadata to add before any below (i.e. "WFD"). Default is None.
nside : int, opt
Nside value for healpix slicer. Default 64.
If "None" is passed, the healpixslicer-based metrics will be skipped.
filterlist : list of str, opt
List of the filternames to use for "per filter" evaluation. Default ('u', 'g', 'r', 'i', 'z', 'y').
If None is passed, the per-filter evaluations will be skipped.
Returns
-------
metricBundleDict
"""
if colmap is None:
colmap = ColMapDict('opsimV4')
bundleList = []
if valueName is None:
valueName = value
if groupName is None:
groupName = valueName.capitalize()
subgroup = extraMetadata
else:
groupName = groupName.capitalize()
subgroup = valueName.capitalize()
displayDict = {'group': groupName, 'subgroup': subgroup}
sqlconstraints = ['']
metadata = ['all bands']
if filterlist is not None:
sqlconstraints += ['%s = "%s"' % (colmap['filter'], f) for f in filterlist]
metadata += ['%s band' % f for f in filterlist]
if (extraSql is not None) and (len(extraSql) > 0):
tmp = []
for s in sqlconstraints:
if len(s) == 0:
tmp.append(extraSql)
else:
tmp.append('%s and (%s)' % (s, extraSql))
sqlconstraints = tmp
if extraMetadata is None:
metadata = ['%s %s' % (extraSql, m) for m in metadata]
if extraMetadata is not None:
metadata = ['%s %s' % (extraMetadata, m) for m in metadata]
# Hack to make HA work, but really I need to account for any stackers/colmaps.
if value == 'HA':
stackerList = [stackers.HourAngleStacker(lstCol=colmap['lst'], raCol=colmap['ra'],
degrees=colmap['raDecDeg'])]
elif value == 'normairmass':
stackerList = [stackers.NormAirmassStacker(degrees=colmap['raDecDeg'])]
else:
stackerList = None
# Summarize values over all and per filter (min/mean/median/max/percentiles/outliers/rms).
slicer = slicers.UniSlicer()
displayDict['caption'] = None
for sql, meta in zip(sqlconstraints, metadata):
displayDict['order'] = -1
for m in extendedMetrics(value, replace_colname=valueName):
displayDict['order'] += 1
bundle = mb.MetricBundle(m, slicer, sql, stackerList=stackerList,
metadata=meta, displayDict=displayDict)
bundleList.append(bundle)
# Histogram values over all and per filter.
for sql, meta in zip(sqlconstraints, metadata):
displayDict['caption'] = 'Histogram of %s' % (value)
if valueName != value:
displayDict['caption'] += ' (%s)' % (valueName)
displayDict['caption'] += ' for %s visits.' % (meta)
displayDict['order'] += 1
m = metrics.CountMetric(value, metricName='%s Histogram' % (valueName))
slicer = slicers.OneDSlicer(sliceColName=value)
bundle = mb.MetricBundle(m, slicer, sql, stackerList=stackerList,
metadata=meta, displayDict=displayDict)
bundleList.append(bundle)
# Make maps of min/median/max for all and per filter, per RA/Dec, with standard summary stats.
mList = []
mList.append(metrics.MinMetric(value, metricName='Min %s' % (valueName)))
mList.append(metrics.MedianMetric(value, metricName='Median %s' % (valueName)))
mList.append(metrics.MaxMetric(value, metricName='Max %s' % (valueName)))
slicer = slicers.HealpixSlicer(nside=nside, latCol=colmap['dec'], lonCol=colmap['ra'],
latLonDeg=colmap['raDecDeg'])
subsetPlots = [plots.HealpixSkyMap(), plots.HealpixHistogram()]
displayDict['caption'] = None
displayDict['order'] = -1
for sql, meta in zip(sqlconstraints, metadata):
for m in mList:
displayDict['order'] += 1
bundle = mb.MetricBundle(m, slicer, sql, stackerList=stackerList,
metadata=meta, plotFuncs=subsetPlots,
displayDict=displayDict,
summaryMetrics=standardSummary())
bundleList.append(bundle)
# Set the runName for all bundles and return the bundleDict.
for b in bundleList:
b.setRunName(runName)
return mb.makeBundlesDictFromList(bundleList)