def test_repr(self):
metric = 'test.cmodel_mag'
self.assertEqual(repr(Measurement(metric)),
"Measurement('test.cmodel_mag', None)")
value = 1235 * u.mag
self.assertEqual(
repr(Measurement(metric, value)),
"Measurement('test.cmodel_mag', <Quantity 1235. mag>)")
self.assertEqual(
repr(Measurement(metric, value, [self.blob1])),
"Measurement('test.cmodel_mag', <Quantity 1235. mag>, "
f"blobs=[{self.blob1!r}])")
notes = {metric + '.filter_name': 'r'}
extras = {'extra1': Datum(10. * u.arcmin, 'Extra 1')}
self.assertEqual(
repr(
Measurement(metric,
value,
notes=notes,
blobs=[self.blob1],
extras=extras)),
"Measurement('test.cmodel_mag', <Quantity 1235. mag>, "
f"blobs=[{self.blob1!r}], extras={extras!r}, notes={notes!r})")
def test_chain(self):
"""Test that running JobReporter on a chained collection retrieves the
metric value closest to the head of the chain.
"""
metricName = "pipe_tasks.CharacterizeImageTime"
id = {"instrument": "NotACam", "visit": 42, "detector": 101}
chainName = "test_chain"
# Note: relies on dict being ordered
chain = {"test_chain_run2": Measurement(metricName, 13.5 * u.s),
"test_chain_run1": Measurement(metricName, 5.0 * u.s),
}
for collection, result in chain.items():
self.repo.registry.registerCollection(
collection, dafButler.CollectionType.RUN)
self.repo.put(result,
"metricvalue_pipe_tasks_CharacterizeImageTime",
id,
run=collection)
self.repo.registry.registerCollection(
chainName, dafButler.CollectionType.CHAINED)
self.repo.registry.setCollectionChain(chainName, chain.keys())
reporter = JobReporter(repository=self.root,
collection=chainName,
metrics_package="pipe_tasks",
spec=None,
dataset_name="_tests")
jobs = reporter.run()
self.assertEqual(len(jobs), 1) # Only one data ID
values = list(jobs.values())[0].measurements
self.assertEqual(len(values), 1) # Only one metric
self.assertEqual(values[metricName], chain["test_chain_run2"])
def test_reload(self):
# Has Metric instance
pa1_meas = Measurement(
self.metric_set['testing.PA1'],
4. * u.mmag
)
# Don't have metric instances
am1_meas = Measurement(
'testing.AM1',
2. * u.marcsec
)
pa2_meas = Measurement(
'testing.PA2',
10. * u.mmag
)
measurements = MeasurementSet([pa1_meas, am1_meas, pa2_meas])
measurements.refresh_metrics(self.metric_set)
self.assertIs(
measurements['testing.PA1'].metric,
self.metric_set['testing.PA1']
)
self.assertIs(
measurements['testing.AM1'].metric,
self.metric_set['testing.AM1']
)
self.assertIs(
measurements['testing.PA2'].metric,
self.metric_set['testing.PA2']
)
def test_deferred_extras(self):
"""Test adding extras to an existing measurement."""
measurement = Measurement(self.pa1, 5. * u.mmag)
self.assertIn(str(self.pa1.name), measurement.blobs)
measurement.extras['extra1'] = Datum(10. * u.arcmin, 'Extra 1')
self.assertIn('extra1', measurement.extras)
def makeMeasurement(self, _dbHandle, outputDataId):
if outputDataId:
nChars = len(outputDataId["instrument"])
return Measurement(self.config.metricName,
nChars * u.dimensionless_unscaled)
else:
return Measurement(self.config.metricName,
0 * u.dimensionless_unscaled)
def run(self, matchedCatalog, metric_name):
self.log.info("Measuring PA1")
pa1 = photRepeat(matchedCatalog, snrMax=self.brightSnrMax, snrMin=self.brightSnrMin,
numRandomShuffles=self.numRandomShuffles, randomSeed=self.randomSeed)
if 'magDiff' in pa1.keys():
return Struct(measurement=Measurement("PA1", pa1['repeatability']))
else:
return Struct(measurement=Measurement("PA1", np.nan*u.mmag))
def test_PA1_measurement_with_nan(self):
"""Test (de)serialization of a measurement with value np.nan."""
measurement = Measurement('PA1', np.nan * u.mag)
json_doc = measurement.json
# a np.nan value is serialized to None
self.assertEqual(json_doc['value'], None)
# a None value is deserialized to np.nan
new_measurement = Measurement.deserialize(**json_doc)
self.assertEqual(str(new_measurement), 'PA1: nan mag')
def test_PA1_measurement_with_inf(self):
"""Test (de)serialization of a measurement with value np.inf."""
measurement = Measurement('PA1', np.inf * u.mag)
json_doc = measurement.json
# a np.inf value is also serialized to None
self.assertEqual(json_doc['value'], None)
# but once it is None, we can only deserialized it to np.nan
new_measurement = Measurement.deserialize(**json_doc)
self.assertEqual(str(new_measurement), 'PA1: nan mag')
def test_quantity_coercion(self):
# strings can't be changed into a Quantity
with self.assertRaises(TypeError):
Measurement('test_metric', quantity='hello')
# objects can't be a Quantity
with self.assertRaises(TypeError):
Measurement('test_metric', quantity=int)
m = Measurement('test_metric', quantity=5)
self.assertEqual(m.quantity, 5)
m = Measurement('test_metric', quantity=5.1)
self.assertEqual(m.quantity, 5.1)
def makeMeasurement(self, timings):
"""Compute a wall-clock measurement from metadata provided by
`lsst.pipe.base.timeMethod`.
Parameters
----------
timings : sequence [`dict` [`str`, any]]
A list where each element corresponds to a metadata object passed
to `run`. Each `dict` has the following keys:
``"StartTime"``
The time the target method started (`float` or `None`).
``"EndTime"``
The time the target method ended (`float` or `None`).
Returns
-------
measurement : `lsst.verify.Measurement` or `None`
The the total running time of the target method across all
elements of ``metadata``.
Raises
------
MetricComputationError
Raised if any of the timing metadata are invalid.
Notes
-----
This method does not return a measurement if no timing information was
provided by any of the metadata.
"""
# some timings indistinguishable from 0, so don't test totalTime > 0
timingFound = False
totalTime = 0.0
for singleRun in timings:
if singleRun["StartTime"] is not None \
or singleRun["EndTime"] is not None:
try:
totalTime += singleRun["EndTime"] - singleRun["StartTime"]
timingFound = True
except TypeError:
raise MetricComputationError("Invalid metadata")
# If both are None, assume the method was not run that time
if timingFound:
meas = Measurement(self.getOutputMetricName(self.config),
totalTime * u.second)
meas.notes['estimator'] = 'pipe.base.timeMethod'
return meas
else:
self.log.info("Nothing to do: no timing information for %s found.",
self.config.target)
return None
def setUp(self):
"""Use YAML in data/metrics for metric definitions."""
self.metrics_yaml_dirname = os.path.join(os.path.dirname(__file__),
'data')
self.metric_set = MetricSet.load_metrics_package(
self.metrics_yaml_dirname)
self.pa1_meas = Measurement(self.metric_set['testing.PA1'],
4. * u.mmag)
self.am1_meas = Measurement(self.metric_set['testing.AM1'],
2. * u.marcsec)
self.pa2_meas = Measurement(self.metric_set['testing.PA2'],
10. * u.mmag)
def run(self, matchedCatalog, metric_name):
self.log.info(f"Measuring {metric_name}")
D = self.config.annulus_r * u.arcmin
filteredCat = filterMatches(matchedCatalog)
nMinTEx = 50
if filteredCat.count <= nMinTEx:
return Struct(measurement=Measurement(metric_name, np.nan*u.Unit('')))
radius, xip, xip_err = correlation_function_ellipticity_from_matches(filteredCat)
operator = ThresholdSpecification.convert_operator_str(self.config.comparison_operator)
corr, corr_err = select_bin_from_corr(radius, xip, xip_err, radius=D, operator=operator)
return Struct(measurement=Measurement(metric_name, np.abs(corr)*u.Unit('')))
def test_PA1_deferred_quantity(self):
"""Test a measurement where the quantity is added later."""
measurement = Measurement(self.pa1)
json_doc = measurement.json
self.assertIsNone(json_doc['unit'])
self.assertIsNone(json_doc['value'])
with self.assertRaises(TypeError):
# wrong units
measurement.quantity = 5 * u.arcsec
measurement.quantity = 5 * u.mmag
quantity_allclose(measurement.quantity, 5 * u.mmag)
def test_PA1_measurement_without_metric(self):
"""Test a measurement without a Metric instance."""
measurement = Measurement('validate_drp.PA1', 0.002 * u.mag)
self.assertIsInstance(measurement.metric_name, Name)
self.assertEqual(measurement.metric_name, Name('validate_drp.PA1'))
self.assertIsNone(measurement.metric)
json_doc = measurement.json
# Units are not converted
self.assertEqual(json_doc['unit'], 'mag')
self.assertFloatsAlmostEqual(json_doc['value'], 0.002)
new_measurement = Measurement.deserialize(**json_doc)
self.assertEqual(measurement, new_measurement)
def makeMeasurement(self, values):
"""Compute the total number of SolarSystemObjects within a
detectorVisit.
Parameters
----------
values : `dict` [`str`, `int` or `None`]
A `dict` representation of the metadata. Each `dict` has the
following key:
``"numTotalSolarSystemObjects"``
The number of SolarSystemObjects within the observable detector
area (`int` or `None`). May be `None` if solar system
association was not attempted or the image was not
successfully associated.
Returns
-------
measurement : `lsst.verify.Measurement` or `None`
The total number of Solar System objects.
"""
if values["numTotalSolarSystemObjects"] is not None:
try:
nNew = int(values["numTotalSolarSystemObjects"])
except (ValueError, TypeError) as e:
raise MetricComputationError(
"Corrupted value of numTotalSolarSystemObjects") from e
else:
return Measurement(self.config.metricName, nNew * u.count)
else:
self.log.info("Nothing to do: no solar system results found.")
return None
def run(self, sources):
"""Count the number of science sources.
Parameters
----------
sources : iterable of `lsst.afw.table.SourceCatalog`
A collection of science source catalogs, one for each unit of
processing to be incorporated into this metric. Its elements may
be `None` to represent missing data.
Returns
-------
result : `lsst.pipe.base.Struct`
A `~lsst.pipe.base.Struct` containing the following component:
``measurement``
the total number of science sources (`lsst.verify.Measurement`
or `None`)
"""
nSciSources = 0
inputData = False
for catalog in sources:
if catalog is not None:
nSciSources += len(catalog)
inputData = True
if inputData:
meas = Measurement(self.getOutputMetricName(self.config), nSciSources * u.count)
else:
self.log.info("Nothing to do: no catalogs found.")
meas = None
return Struct(measurement=meas)
def mergeFractionsPartial(fractions, denominatorsMinusNumerators):
"""Weighted sum of some fractions.
Extras and notes will be dictionary-merged (i.e., if multiple extras or
notes are assigned to the same key, only one will be preserved) from
`fractions`.
.. warning::
This function does NOT perform input validation
Parameters
----------
fractions: list of `lsst.verify.Measurement`
The measurements to combine. Must be measurements of the same metric.
denominatorsMinusNumerators: list of `lsst.verify.Measurement`
The denominators of `fractions` minus the numerators. Must be
measurements of the same metric, and must have a one-to-one
correspondence with `fractions`.
Returns
-------
A Measurement containing the average of `fractions`, weighted
appropriately, and as much auxiliary data as could be reasonably saved.
"""
denominators = []
for fraction, partial in zip(fractions, denominatorsMinusNumerators):
denominator = partial.quantity / (1.0 - fraction.quantity)
denominators.append(
Measurement(partial.metric,
denominator,
extras=partial.extras,
notes=partial.notes))
return mergeFractions(fractions, denominators)
def test_PA1_deferred_metric(self):
"""Test a measurement when the Metric instance is added later."""
measurement = Measurement('PA1', 0.002 * u.mag)
self.assertIsNone(measurement.metric)
self.assertEqual(measurement.metric_name, Name(metric='PA1'))
# Try adding in a metric with the wrong units to existing quantity
other_metric = Metric('testing.other', 'Incompatible units', 'arcsec')
with self.assertRaises(TypeError):
measurement.metric = other_metric
# Add metric in; the name should also update
measurement.metric = self.pa1
self.assertEqual(measurement.metric, self.pa1)
self.assertEqual(measurement.metric_name, self.pa1.name)
def run(self, sciSources, diaSources):
"""Compute the ratio of DIASources to science sources.
Parameters
----------
sciSources : `lsst.afw.table.SourceCatalog` or `None`
A science source catalog, which may be empty or `None`.
diaSources : `lsst.afw.table.SourceCatalog` or `None`
A DIASource catalog for the same unit of processing
as ``sciSources``.
Returns
-------
result : `lsst.pipe.base.Struct`
A `~lsst.pipe.base.Struct` containing the following component:
``measurement``
the ratio (`lsst.verify.Measurement` or `None`)
"""
if diaSources is not None and sciSources is not None:
nSciSources = len(sciSources)
nDiaSources = len(diaSources)
metricName = self.config.metricName
if nSciSources <= 0.0:
raise MetricComputationError(
"No science sources found; ratio of DIASources to science sources ill-defined."
)
else:
meas = Measurement(
metricName,
nDiaSources / nSciSources * u.dimensionless_unscaled)
else:
self.log.info("Nothing to do: no catalogs found.")
meas = None
return Struct(measurement=meas)
def run(self, matchedFakes, band):
"""Compute the completeness of recovered fakes within a magnitude
range.
Parameters
----------
matchedFakes : `lsst.afw.table.SourceCatalog` or `None`
Catalog of fakes that were inserted into the ccdExposure matched
to their detected counterparts.
Returns
-------
result : `lsst.pipe.base.Struct`
A `~lsst.pipe.base.Struct` containing the following component:
``measurement``
the ratio (`lsst.verify.Measurement` or `None`)
"""
if matchedFakes is not None:
magnitudes = matchedFakes[f"{self.config.magVar}" % band]
magCutFakes = matchedFakes[np.logical_and(magnitudes > self.config.magMin,
magnitudes < self.config.magMax)]
if len(magCutFakes) <= 0.0:
raise MetricComputationError(
"No matched fakes catalog sources found; Completeness is "
"ill defined.")
else:
meas = Measurement(
self.config.metricName,
((magCutFakes["diaSourceId"] > 0).sum() / len(magCutFakes))
* u.dimensionless_unscaled)
else:
self.log.info("Nothing to do: no matched catalog found.")
meas = None
return Struct(measurement=meas)
def makeMeasurement(self, values):
"""Compute the number of new DIAObjects.
Parameters
----------
values : `dict` [`str`, `int` or `None`]
A `dict` representation of the metadata. Each `dict` has the
following key:
``"newObjects"``
The number of new objects created for this image (`int`
or `None`). May be `None` if the image was not successfully
associated.
Returns
-------
measurement : `lsst.verify.Measurement` or `None`
The total number of new objects.
"""
if values["newObjects"] is not None:
try:
nNew = int(values["newObjects"])
except (ValueError, TypeError) as e:
raise MetricComputationError(
"Corrupted value of numNewDiaObjects") from e
else:
return Measurement(self.config.metricName, nNew * u.count)
else:
self.log.info("Nothing to do: no association results found.")
return None
def makeMeasurement(self, timings):
"""Compute a wall-clock measurement from metadata provided by
`lsst.utils.timer.timeMethod`.
Parameters
----------
timings : `dict` [`str`, any]
A representation of the metadata passed to `run`. The `dict` has
the following keys:
``"StartTime"``
The time the target method started (`float` or `None`).
``"EndTime"``
The time the target method ended (`float` or `None`).
``"StartTimestamp"``, ``"EndTimestamp"``
The start and end timestamps, in an ISO 8601-compliant format
(`str` or `None`).
Returns
-------
measurement : `lsst.verify.Measurement` or `None`
The running time of the target method.
Raises
------
MetricComputationError
Raised if the timing metadata are invalid.
"""
if timings["StartTime"] is not None or timings["EndTime"] is not None:
try:
totalTime = timings["EndTime"] - timings["StartTime"]
except TypeError:
raise MetricComputationError("Invalid metadata")
else:
meas = Measurement(self.config.metricName,
totalTime * u.second)
meas.notes["estimator"] = "utils.timer.timeMethod"
if timings["StartTimestamp"]:
meas.extras["start"] = Datum(timings["StartTimestamp"])
if timings["EndTimestamp"]:
meas.extras["end"] = Datum(timings["EndTimestamp"])
return meas
else:
self.log.info("Nothing to do: no timing information for %s found.",
self.config.target)
return None
def run(self, matchedCatalog, metric_name):
self.log.info("Measuring PF1")
pa2_thresh = self.threshPA2 * u.mmag
pf1 = photRepeat(matchedCatalog,
numRandomShuffles=self.numRandomShuffles, randomSeed=self.randomSeed)
if 'magDiff' in pf1.keys():
# Previously, validate_drp used the first random sample from PA1 measurement
# Now, use all of them.
magDiffs = pf1['magDiff']
percentileAtPA2 = 100 * np.mean(np.abs(magDiffs.value) > pa2_thresh.value) * u.percent
return Struct(measurement=Measurement("PF1", percentileAtPA2))
else:
return Struct(measurement=Measurement("PF1", np.nan*u.percent))
def test_yamlpersist_complex(self):
measurement = Measurement(
self.pa1,
5. * u.mmag,
notes={'filter_name': 'r'},
blobs=[self.blob1],
extras={'extra1': Datum(10. * u.arcmin, 'Extra 1')})
self._check_yaml_round_trip(measurement)
def run(self, matchedCatalog, metric_name):
self.log.info("Measuring PA2")
pf1_thresh = self.threshPF1 * u.percent
pa2 = photRepeat(matchedCatalog,
numRandomShuffles=self.numRandomShuffles, randomSeed=self.randomSeed)
if 'magDiff' in pa2.keys():
# Previously, validate_drp used the first random sample from PA1 measurement
# Now, use all of them.
magDiffs = pa2['magDiff']
pf1Percentile = 100.*u.percent - pf1_thresh
return Struct(measurement=Measurement("PA2", np.percentile(np.abs(magDiffs.value),
pf1Percentile.value) * magDiffs.unit))
else:
return Struct(measurement=Measurement("PA2", np.nan*u.mmag))
def test_creation_with_extras(self):
"""Test creating a measurement with an extra."""
measurement = Measurement(
self.pa1,
5. * u.mmag,
extras={'extra1': Datum(10. * u.arcmin, 'Extra 1')})
self.assertIn(str(self.pa1.name), measurement.blobs)
self.assertIn('extra1', measurement.extras)
json_doc = measurement.json
self.assertIn(measurement.extras.identifier, json_doc['blob_refs'])
blobs = BlobSet([b for k, b in measurement.blobs.items()])
new_measurement = Measurement.deserialize(blobs=blobs, **json_doc)
self.assertIn('extra1', new_measurement.extras)
self.assertEqual(measurement, new_measurement)
def run(self, matchedCatalog, metric_name):
self.log.info(f"Measuring {metric_name}")
sepDistances = astromResiduals(matchedCatalog, self.config.bright_mag_cut,
self.config.faint_mag_cut, self.config.annulus_r,
self.config.width)
adxThresh = self.config.threshAD * u.marcsec
if len(sepDistances) <= 1:
return Struct(measurement=Measurement(metric_name, np.nan*u.percent))
else:
# absolute value of the difference between each astrometric rms
# and the median astrometric RMS
# absRmsDiffs = np.abs(rmsDistances - np.median(rmsDistances)).to(u.marcsec)
absDiffsMarcsec = (sepDistances-np.median(sepDistances)).to(u.marcsec)
percentileAtADx = 100 * np.mean(np.abs(absDiffsMarcsec.value) > adxThresh.value) * u.percent
return Struct(measurement=Measurement(metric_name, percentileAtADx))
def setUp(self):
self.job = Job()
self.job.metrics.insert(
Metric("foo.boringmetric", "", u.percent, tags=["redundant"]))
self.job.metrics.insert(
Metric("foo.fancymetric", "", u.meter, tags=["vital"]))
self.job.measurements.insert(
Measurement("foo.fancymetric", 2.0 * u.meter))
self.job.measurements.insert(
Measurement("foo.fanciermetric", 3.5 * u.second))
self.job.measurements["foo.fanciermetric"].notes["fanciness"] \
= "moderate"
self.job.measurements.insert(
Measurement("foo.fanciestmetric", 3.1415927 * u.kilogram))
self.job.meta["bar"] = "high"
self.job.meta["shape"] = "rotund"
self.job.specs.insert(
ThresholdSpecification("utterly_ridiculous", 1e10 * u.meter, ">"))
def run(self, sciSources, diaSources):
"""Compute the ratio of DIASources to science sources.
Parameters
----------
sciSources : iterable of `lsst.afw.table.SourceCatalog`
A collection of science source catalogs, one for each unit of
processing to be incorporated into this metric. Its elements may
be `None` to represent missing data.
diaSources : iterable of `lsst.afw.table.SourceCatalog`
A collection of difference imaging catalogs similar to
``sciSources``.
Returns
-------
result : `lsst.pipe.base.Struct`
A `~lsst.pipe.base.Struct` containing the following component:
``measurement``
the ratio (`lsst.verify.Measurement` or `None`)
"""
nSciSources = 0
nDiaSources = 0
inputData = False
for sciCatalog, diaCatalog in zip(sciSources, diaSources):
if diaCatalog is not None and sciCatalog is not None:
nSciSources += len(sciCatalog)
nDiaSources += len(diaCatalog)
inputData = True
if inputData:
metricName = self.getOutputMetricName(self.config)
if nSciSources <= 0.0:
raise MetricComputationError(
"No science sources found; ratio of DIASources to science sources ill-defined.")
meas = Measurement(metricName, 0.0 * u.dimensionless_unscaled)
else:
meas = Measurement(metricName, nDiaSources / nSciSources * u.dimensionless_unscaled)
else:
self.log.info("Nothing to do: no catalogs found.")
meas = None
return Struct(measurement=meas)
def run(self, catalog, metric_name, vIds):
self.log.info(f"Measuring {metric_name}")
if not catalog.isContiguous():
catalog = catalog.copy(deep=True)
extended = catalog.get('base_ClassificationExtendedness_value')
good_extended = extended[~catalog.get('base_ClassificationExtendedness_flag')]
n_gals = sum(good_extended)
frac = 100*(len(good_extended) - n_gals)/len(good_extended)
meas = Measurement("starFrac", frac * u.percent)
return Struct(measurement=meas)
