def build_matched_dataset(repo,
dataIds,
matchRadius=None,
safeSnr=50.,
useJointCal=False,
skipTEx=False):
blob = Blob('MatchedMultiVisitDataset')
if not matchRadius:
matchRadius = afwGeom.Angle(1, afwGeom.arcseconds)
# Extract single filter
blob['filterName'] = Datum(quantity=set([dId['filter']
for dId in dataIds]).pop(),
description='Filter name')
# Record important configuration
blob['useJointCal'] = Datum(
quantity=useJointCal,
description='Whether jointcal/meas_mosaic calibrations were used')
# Match catalogs across visits
blob._catalog, blob._matchedCatalog = \
_loadAndMatchCatalogs(repo, dataIds, matchRadius,
useJointCal=useJointCal, skipTEx=False)
blob.magKey = blob._matchedCatalog.schema.find("base_PsfFlux_mag").key
# Reduce catalogs into summary statistics.
# These are the serialiable attributes of this class.
_reduceStars(blob, blob._matchedCatalog, safeSnr)
return blob
def setUp(self):
mag1 = Datum(quantity=5 * u.mag, label='mag1', description='Magnitude')
mag2 = Datum(quantity=10 * u.mag,
label='mag2',
description='Magnitude')
self.blob1 = Blob('blob1', mag1=mag1, mag2=mag2)
sep1 = Datum(quantity=5 * u.arcsec,
label='sep1',
description='Separation')
sep2 = Datum(quantity=10 * u.arcsec,
label='sep2',
description='Separation')
self.blob2 = Blob('blob2', sep1=sep1, sep2=sep2)
def build_photometric_error_model(matchedMultiVisitDataset,
selection,
medianRef=100,
matchRef=500):
r"""Returns a serializable analytic photometry error model for a single visit.
This model is originally presented in http://arxiv.org/abs/0805.2366v4
(Eq 4, 5):
.. math::
\sigma_1^2 &= \sigma_\mathrm{sys}^2 + \sigma_\mathrm{rand}^2 \\
x &= 10^{0.4(m-m_5)} \\
\sigma_\mathrm{rand}^2 &= (0.04 - \gamma) x + \gamma x^2~[\mathrm{mag}^2]
Parameters
----------
matchedMultiVisitDataset : `lsst.valididate.drp.matchreduce.MatchedMultiVisitDataset`
A dataset containing matched statistics for stars across multiple visits.
selection : `np.array` of `bool`
The selection of sources to use to build the model.
medianRef : `float` or `astropy.unit.Quantity`, optional
Median reference astrometric scatter (millimagnitudes by default).
matchRef : `int` or `astropy.unit.Quantity`, optional
Should match at least matchRef stars.
Returns
----------
blob : `lsst.verify.Blob`
Blob with datums:
- ``sigmaSys``: Systematic error floor.
- ``gamma``: Proxy for sky brightness and read noise.
- ``m5``: 5-sigma photometric depth (magnitudes).
- ``photRms``: RMS photometric scatter for 'good' stars (millimagnitudes).
Notes
-----
The scatter and match defaults are appropriate to SDSS are stored here.
For SDSS, stars with mag < 19.5 should be completely well measured.
This limit is a band-dependent statement most appropriate to r.
"""
blob = Blob('PhotometricErrorModel')
for field in ('brightSnrMin', 'brightSnrMax'):
blob[field] = matchedMultiVisitDataset[field]
# FIXME add a description field to blobs?
# _doc['doc'] \
# = "Photometric uncertainty model from " \
# "http://arxiv.org/abs/0805.2366v4 (Eq 4, 5): " \
# "sigma_1^2 = sigma_sys^2 + sigma_rand^2, " \
# "sigma_rand^2 = (0.04 - gamma) * x + gamma * x^2 [mag^2] " \
# "where x = 10**(0.4*(m-m_5))"
if not isinstance(medianRef, u.Quantity):
medianRef = medianRef * u.mmag
_compute(blob, selection, matchedMultiVisitDataset['mag'].quantity,
matchedMultiVisitDataset['magerr'].quantity,
matchedMultiVisitDataset['magrms'].quantity, medianRef, matchRef)
return blob
def setUp(self):
self.pa1 = Metric(
'validate_drp.PA1',
"The maximum rms of the unresolved source magnitude distribution "
"around the mean value (repeatability).",
'mmag',
tags=['photometric', 'LPM-17'],
reference_doc='LPM-17',
reference_url='http://ls.st/lpm-17',
reference_page=21)
self.blob1 = Blob('Blob1')
self.blob1['datum1'] = Datum(5 * u.arcsec, 'Datum 1')
self.blob1['datum2'] = Datum(28. * u.mag, 'Datum 2')
self.blob2 = Blob('Blob2')
self.blob2['datumN'] = Datum(11 * u.dimensionless_unscaled, 'Count')
def build_astrometric_error_model(matchedMultiVisitDataset,
brightSnr=100,
medianRef=100,
matchRef=500):
"""Serializable model of astrometry errors across multiple visits.
.. math::
\mathrm{astromRms} = C \theta / \mathrm{SNR} + \sigma_\mathrm{sys}
Parameters
----------
matchedMultiVisitDataset : `MatchedMultiVisitDataset`
A dataset containing matched statistics for stars across multiple
visits.
brightSnr : `float` or `astropy.unit.Quantity`, optional
Minimum SNR for a star to be considered "bright" (dimensionless).
medianRef : `float` or `astropy.unit.Quantity`, optional
Median reference astrometric scatter (default: milliarcsecond).
matchRef : int, optional
Should match at least matchRef number of stars (dimensionless).
Returns
-------
blob : `lsst.verify.Blob`
Blob with datums:
- ``brightSnr``: Threshold SNR for bright sources used in this model.
- ``C``: Model scaling factor.
- ``theta``: Seeing (milliarcsecond).
- ``sigmaSys``: Systematic error floor (milliarcsecond).
- ``astromRms``: Astrometric scatter (RMS) for good stars (milliarcsecond).
Notes
-----
The scatter and match defaults appropriate to SDSS are the defaults
for ``medianRef`` and ``matchRef``.
For SDSS, stars with mag < 19.5 should be completely well measured.
"""
blob = Blob('AnalyticAstrometryModel')
# FIXME add description field to blobs
# _doc['doc'] \
# = "Astrometric astrometry model: mas = C*theta/SNR + sigmaSys"
if not isinstance(brightSnr, u.Quantity):
brightSnr = brightSnr * u.Unit('')
if not isinstance(medianRef, u.Quantity):
medianRef = medianRef * u.marcsec
_compute(blob, matchedMultiVisitDataset['snr'].quantity,
matchedMultiVisitDataset['dist'].quantity,
len(matchedMultiVisitDataset.goodMatches), brightSnr, medianRef,
matchRef)
return blob
def setUp(self):
# Mock metrics
self.metric_photrms = Metric('test.PhotRms', 'Photometric RMS', 'mmag')
self.metric_photmed = Metric('test.PhotMedian', 'Median magntidue',
'mag')
self.metric_set = MetricSet([self.metric_photrms, self.metric_photmed])
# Mock specifications
self.spec_photrms_design = ThresholdSpecification(
'test.PhotRms.design', 20. * u.mmag, '<')
self.spec_set = SpecificationSet([self.spec_photrms_design])
# Mock measurements
self.meas_photrms = Measurement(self.metric_photrms,
15 * u.mmag,
notes={'note': 'value'})
self.meas_photrms.extras['n_stars'] = Datum(
250,
label='N stars',
description='Number of stars included in RMS estimate')
self.measurement_set = MeasurementSet([self.meas_photrms])
# Metrics for Job 2
self.metric_test_2 = Metric('test2.SourceCount', 'Source Count', '')
self.blob_test_2 = Blob('test2_blob',
sn=Datum(50 * u.dimensionless_unscaled,
label='S/N'))
self.metric_set_2 = MetricSet([self.metric_test_2])
# Specifications for Job 2
self.spec_test_2 = ThresholdSpecification(
'test2.SourceCount.design', 100 * u.dimensionless_unscaled, '>=')
self.spec_set_2 = SpecificationSet([self.spec_test_2])
# Measurements for Job 2
self.meas_test_2_SourceCount = Measurement(
self.metric_test_2, 200 * u.dimensionless_unscaled)
self.meas_test_2_SourceCount.link_blob(self.blob_test_2)
self.measurement_set_2 = MeasurementSet([self.meas_test_2_SourceCount])
def build_matched_dataset(repo,
dataIds,
matchRadius=None,
safeSnr=50.,
useJointCal=False,
skipTEx=False):
"""Construct a container for matched star catalogs from multple visits, with filtering,
summary statistics, and modelling.
`lsst.verify.Blob` instances are serializable to JSON.
Parameters
----------
repo : `str` or `Butler`
A Butler instance or a repository URL that can be used to construct
one.
dataIds : `list` of `dict`
List of `butler` data IDs of Image catalogs to compare to reference.
The `calexp` cpixel image is needed for the photometric calibration.
matchRadius : afwGeom.Angle(), optional
Radius for matching. Default is 1 arcsecond.
safeSnr : `float`, optional
Minimum median SNR for a match to be considered "safe".
useJointCal : `bool`, optional
Use jointcal/meas_mosaic outputs to calibrate positions and fluxes.
skipTEx : `bool`, optional
Skip TEx calculations (useful for older catalogs that don't have
PsfShape measurements).
Attributes of returned Blob
----------
filterName : `str`
Name of filter used for all observations.
mag : `astropy.units.Quantity`
Mean PSF magnitudes of stars over multiple visits (magnitudes).
magerr : `astropy.units.Quantity`
Median 1-sigma uncertainty of PSF magnitudes over multiple visits
(magnitudes).
magrms : `astropy.units.Quantity`
RMS of PSF magnitudes over multiple visits (magnitudes).
snr : `astropy.units.Quantity`
Median signal-to-noise ratio of PSF magnitudes over multiple visits
(dimensionless).
dist : `astropy.units.Quantity`
RMS of sky coordinates of stars over multiple visits (milliarcseconds).
*Not serialized.*
goodMatches
all good matches, as an afw.table.GroupView;
good matches contain only objects whose detections all have
1. a PSF Flux measurement with S/N > 1
2. a finite (non-nan) PSF magnitude. This separate check is largely
to reject failed zeropoints.
3. and do not have flags set for bad, cosmic ray, edge or saturated
*Not serialized.*
safeMatches
safe matches, as an afw.table.GroupView. Safe matches
are good matches that are sufficiently bright and sufficiently
compact.
*Not serialized.*
magKey
Key for `"base_PsfFlux_mag"` in the `goodMatches` and `safeMatches`
catalog tables.
*Not serialized.*
"""
blob = Blob('MatchedMultiVisitDataset')
if not matchRadius:
matchRadius = afwGeom.Angle(1, afwGeom.arcseconds)
# Extract single filter
blob['filterName'] = Datum(quantity=set([dId['filter']
for dId in dataIds]).pop(),
description='Filter name')
# Record important configuration
blob['useJointCal'] = Datum(
quantity=useJointCal,
description='Whether jointcal/meas_mosaic calibrations were used')
# Match catalogs across visits
blob._catalog, blob._matchedCatalog = \
_loadAndMatchCatalogs(repo, dataIds, matchRadius,
useJointCal=useJointCal, skipTEx=skipTEx)
blob.magKey = blob._matchedCatalog.schema.find("base_PsfFlux_mag").key
# Reduce catalogs into summary statistics.
# These are the serialiable attributes of this class.
_reduceStars(blob, blob._matchedCatalog, safeSnr)
return blob
def build_matched_dataset(repo,
dataIds,
matchRadius=None,
brightSnrMin=None,
brightSnrMax=None,
faintSnrMin=None,
faintSnrMax=None,
doApplyExternalPhotoCalib=False,
externalPhotoCalibName=None,
doApplyExternalSkyWcs=False,
externalSkyWcsName=None,
skipTEx=False,
skipNonSrd=False):
"""Construct a container for matched star catalogs from multple visits, with filtering,
summary statistics, and modelling.
`lsst.verify.Blob` instances are serializable to JSON.
Parameters
----------
repo : `str` or `lsst.daf.persistence.Butler`
A Butler instance or a repository URL that can be used to construct
one.
dataIds : `list` of `dict`
List of `butler` data IDs of Image catalogs to compare to reference.
The `calexp` cpixel image is needed for the photometric calibration.
matchRadius : `lsst.geom.Angle`, optional
Radius for matching. Default is 1 arcsecond.
brightSnrMin : `float`, optional
Minimum median SNR for a source to be considered bright; passed to `filterSources`.
brightSnrMax : `float`, optional
Maximum median SNR for a source to be considered bright; passed to `filterSources`.
faintSnrMin : `float`, optional
Minimum median SNR for a source to be considered faint; passed to `filterSources`.
faintSnrMax : `float`, optional
Maximum median SNR for a source to be considered faint; passed to `filterSources`.
doApplyExternalPhotoCalib : bool, optional
Apply external photoCalib to calibrate fluxes.
externalPhotoCalibName : str, optional
Type of external `PhotoCalib` to apply. Currently supported are jointcal,
fgcm, and fgcm_tract. Must be set if "doApplyExternalPhotoCalib" is True.
doApplyExternalSkyWcs : bool, optional
Apply external wcs to calibrate positions.
externalSkyWcsName : str, optional:
Type of external `wcs` to apply. Currently supported is jointcal.
Must be set if "doApplyExternalSkyWcs" is True.
skipTEx : `bool`, optional
Skip TEx calculations (useful for older catalogs that don't have
PsfShape measurements).
skipNonSrd : `bool`, optional
Skip any metrics not defined in the LSST SRD; default False.
Attributes of returned Blob
----------
filterName : `str`
Name of filter used for all observations.
mag : `astropy.units.Quantity`
Mean PSF magnitudes of stars over multiple visits (magnitudes).
magerr : `astropy.units.Quantity`
Median 1-sigma uncertainty of PSF magnitudes over multiple visits
(magnitudes).
magrms : `astropy.units.Quantity`
RMS of PSF magnitudes over multiple visits (magnitudes).
snr : `astropy.units.Quantity`
Median signal-to-noise ratio of PSF magnitudes over multiple visits
(dimensionless).
dist : `astropy.units.Quantity`
RMS of sky coordinates of stars over multiple visits (milliarcseconds).
*Not serialized.*
matchesFaint : `afw.table.GroupView`
Faint matches containing only objects that have:
1. A PSF Flux measurement with sufficient S/N.
2. A finite (non-nan) PSF magnitude. This separate check is largely
to reject failed zeropoints.
3. No flags set for bad, cosmic ray, edge or saturated.
4. Extendedness consistent with a point source.
*Not serialized.*
matchesBright : `afw.table.GroupView`
Bright matches matching a higher S/N threshold than matchesFaint.
*Not serialized.*
magKey
Key for `"base_PsfFlux_mag"` in the `matchesFaint` and `matchesBright`
catalog tables.
*Not serialized.*
Raises
------
RuntimeError:
Raised if "doApplyExternalPhotoCalib" is True and "externalPhotoCalibName"
is None, or if "doApplyExternalSkyWcs" is True and "externalSkyWcsName" is
None.
"""
if doApplyExternalPhotoCalib and externalPhotoCalibName is None:
raise RuntimeError(
"Must set externalPhotoCalibName if doApplyExternalPhotoCalib is True."
)
if doApplyExternalSkyWcs and externalSkyWcsName is None:
raise RuntimeError(
"Must set externalSkyWcsName if doApplyExternalSkyWcs is True.")
blob = Blob('MatchedMultiVisitDataset')
if not matchRadius:
matchRadius = geom.Angle(1, geom.arcseconds)
# Extract single filter
blob['filterName'] = Datum(quantity=set([dId['filter']
for dId in dataIds]).pop(),
description='Filter name')
# Record important configuration
blob['doApplyExternalPhotoCalib'] = Datum(
quantity=doApplyExternalPhotoCalib,
description=('Whether external photometric '
'calibrations were used.'))
blob['externalPhotoCalibName'] = Datum(
quantity=externalPhotoCalibName,
description='Name of external PhotoCalib dataset used.')
blob['doApplyExternalSkyWcs'] = Datum(
quantity=doApplyExternalSkyWcs,
description='Whether external wcs calibrations were used.')
blob['externalSkyWcsName'] = Datum(
quantity=externalSkyWcsName,
description='Name of external wcs dataset used.')
# Match catalogs across visits
blob._catalog, blob._matchedCatalog = \
_loadAndMatchCatalogs(repo, dataIds, matchRadius,
doApplyExternalPhotoCalib=doApplyExternalPhotoCalib,
externalPhotoCalibName=externalPhotoCalibName,
doApplyExternalSkyWcs=doApplyExternalSkyWcs,
externalSkyWcsName=externalSkyWcsName,
skipTEx=skipTEx, skipNonSrd=skipNonSrd)
blob.magKey = blob._matchedCatalog.schema.find("base_PsfFlux_mag").key
# Reduce catalogs into summary statistics.
# These are the serializable attributes of this class.
filterResult = filterSources(
blob._matchedCatalog,
brightSnrMin=brightSnrMin,
brightSnrMax=brightSnrMax,
faintSnrMin=faintSnrMin,
faintSnrMax=faintSnrMax,
)
blob['brightSnrMin'] = Datum(
quantity=filterResult.brightSnrMin * u.Unit(''),
label='Bright SNR Min',
description='Minimum median SNR for a source to be considered bright')
blob['brightSnrMax'] = Datum(
quantity=filterResult.brightSnrMax * u.Unit(''),
label='Bright SNR Max',
description='Maximum median SNR for a source to be considered bright')
summarizeSources(blob, filterResult)
return blob