def test_repr(self): metric = 'test.cmodel_mag' self.assertEqual(repr(Measurement(metric)), "Measurement('test.cmodel_mag', None)") value = 1235 * u.mag self.assertEqual( repr(Measurement(metric, value)), "Measurement('test.cmodel_mag', <Quantity 1235. mag>)") self.assertEqual( repr(Measurement(metric, value, [self.blob1])), "Measurement('test.cmodel_mag', <Quantity 1235. mag>, " f"blobs=[{self.blob1!r}])") notes = {metric + '.filter_name': 'r'} extras = {'extra1': Datum(10. * u.arcmin, 'Extra 1')} self.assertEqual( repr( Measurement(metric, value, notes=notes, blobs=[self.blob1], extras=extras)), "Measurement('test.cmodel_mag', <Quantity 1235. mag>, " f"blobs=[{self.blob1!r}], extras={extras!r}, notes={notes!r})")
def test_chain(self): """Test that running JobReporter on a chained collection retrieves the metric value closest to the head of the chain. """ metricName = "pipe_tasks.CharacterizeImageTime" id = {"instrument": "NotACam", "visit": 42, "detector": 101} chainName = "test_chain" # Note: relies on dict being ordered chain = {"test_chain_run2": Measurement(metricName, 13.5 * u.s), "test_chain_run1": Measurement(metricName, 5.0 * u.s), } for collection, result in chain.items(): self.repo.registry.registerCollection( collection, dafButler.CollectionType.RUN) self.repo.put(result, "metricvalue_pipe_tasks_CharacterizeImageTime", id, run=collection) self.repo.registry.registerCollection( chainName, dafButler.CollectionType.CHAINED) self.repo.registry.setCollectionChain(chainName, chain.keys()) reporter = JobReporter(repository=self.root, collection=chainName, metrics_package="pipe_tasks", spec=None, dataset_name="_tests") jobs = reporter.run() self.assertEqual(len(jobs), 1) # Only one data ID values = list(jobs.values())[0].measurements self.assertEqual(len(values), 1) # Only one metric self.assertEqual(values[metricName], chain["test_chain_run2"])
def test_reload(self): # Has Metric instance pa1_meas = Measurement( self.metric_set['testing.PA1'], 4. * u.mmag ) # Don't have metric instances am1_meas = Measurement( 'testing.AM1', 2. * u.marcsec ) pa2_meas = Measurement( 'testing.PA2', 10. * u.mmag ) measurements = MeasurementSet([pa1_meas, am1_meas, pa2_meas]) measurements.refresh_metrics(self.metric_set) self.assertIs( measurements['testing.PA1'].metric, self.metric_set['testing.PA1'] ) self.assertIs( measurements['testing.AM1'].metric, self.metric_set['testing.AM1'] ) self.assertIs( measurements['testing.PA2'].metric, self.metric_set['testing.PA2'] )
def test_deferred_extras(self): """Test adding extras to an existing measurement.""" measurement = Measurement(self.pa1, 5. * u.mmag) self.assertIn(str(self.pa1.name), measurement.blobs) measurement.extras['extra1'] = Datum(10. * u.arcmin, 'Extra 1') self.assertIn('extra1', measurement.extras)
def makeMeasurement(self, _dbHandle, outputDataId): if outputDataId: nChars = len(outputDataId["instrument"]) return Measurement(self.config.metricName, nChars * u.dimensionless_unscaled) else: return Measurement(self.config.metricName, 0 * u.dimensionless_unscaled)
def run(self, matchedCatalog, metric_name): self.log.info("Measuring PA1") pa1 = photRepeat(matchedCatalog, snrMax=self.brightSnrMax, snrMin=self.brightSnrMin, numRandomShuffles=self.numRandomShuffles, randomSeed=self.randomSeed) if 'magDiff' in pa1.keys(): return Struct(measurement=Measurement("PA1", pa1['repeatability'])) else: return Struct(measurement=Measurement("PA1", np.nan*u.mmag))
def test_PA1_measurement_with_nan(self): """Test (de)serialization of a measurement with value np.nan.""" measurement = Measurement('PA1', np.nan * u.mag) json_doc = measurement.json # a np.nan value is serialized to None self.assertEqual(json_doc['value'], None) # a None value is deserialized to np.nan new_measurement = Measurement.deserialize(**json_doc) self.assertEqual(str(new_measurement), 'PA1: nan mag')
def test_PA1_measurement_with_inf(self): """Test (de)serialization of a measurement with value np.inf.""" measurement = Measurement('PA1', np.inf * u.mag) json_doc = measurement.json # a np.inf value is also serialized to None self.assertEqual(json_doc['value'], None) # but once it is None, we can only deserialized it to np.nan new_measurement = Measurement.deserialize(**json_doc) self.assertEqual(str(new_measurement), 'PA1: nan mag')
def test_quantity_coercion(self): # strings can't be changed into a Quantity with self.assertRaises(TypeError): Measurement('test_metric', quantity='hello') # objects can't be a Quantity with self.assertRaises(TypeError): Measurement('test_metric', quantity=int) m = Measurement('test_metric', quantity=5) self.assertEqual(m.quantity, 5) m = Measurement('test_metric', quantity=5.1) self.assertEqual(m.quantity, 5.1)
def makeMeasurement(self, timings): """Compute a wall-clock measurement from metadata provided by `lsst.pipe.base.timeMethod`. Parameters ---------- timings : sequence [`dict` [`str`, any]] A list where each element corresponds to a metadata object passed to `run`. Each `dict` has the following keys: ``"StartTime"`` The time the target method started (`float` or `None`). ``"EndTime"`` The time the target method ended (`float` or `None`). Returns ------- measurement : `lsst.verify.Measurement` or `None` The the total running time of the target method across all elements of ``metadata``. Raises ------ MetricComputationError Raised if any of the timing metadata are invalid. Notes ----- This method does not return a measurement if no timing information was provided by any of the metadata. """ # some timings indistinguishable from 0, so don't test totalTime > 0 timingFound = False totalTime = 0.0 for singleRun in timings: if singleRun["StartTime"] is not None \ or singleRun["EndTime"] is not None: try: totalTime += singleRun["EndTime"] - singleRun["StartTime"] timingFound = True except TypeError: raise MetricComputationError("Invalid metadata") # If both are None, assume the method was not run that time if timingFound: meas = Measurement(self.getOutputMetricName(self.config), totalTime * u.second) meas.notes['estimator'] = 'pipe.base.timeMethod' return meas else: self.log.info("Nothing to do: no timing information for %s found.", self.config.target) return None
def setUp(self): """Use YAML in data/metrics for metric definitions.""" self.metrics_yaml_dirname = os.path.join(os.path.dirname(__file__), 'data') self.metric_set = MetricSet.load_metrics_package( self.metrics_yaml_dirname) self.pa1_meas = Measurement(self.metric_set['testing.PA1'], 4. * u.mmag) self.am1_meas = Measurement(self.metric_set['testing.AM1'], 2. * u.marcsec) self.pa2_meas = Measurement(self.metric_set['testing.PA2'], 10. * u.mmag)
def run(self, matchedCatalog, metric_name): self.log.info(f"Measuring {metric_name}") D = self.config.annulus_r * u.arcmin filteredCat = filterMatches(matchedCatalog) nMinTEx = 50 if filteredCat.count <= nMinTEx: return Struct(measurement=Measurement(metric_name, np.nan*u.Unit(''))) radius, xip, xip_err = correlation_function_ellipticity_from_matches(filteredCat) operator = ThresholdSpecification.convert_operator_str(self.config.comparison_operator) corr, corr_err = select_bin_from_corr(radius, xip, xip_err, radius=D, operator=operator) return Struct(measurement=Measurement(metric_name, np.abs(corr)*u.Unit('')))
def test_PA1_deferred_quantity(self): """Test a measurement where the quantity is added later.""" measurement = Measurement(self.pa1) json_doc = measurement.json self.assertIsNone(json_doc['unit']) self.assertIsNone(json_doc['value']) with self.assertRaises(TypeError): # wrong units measurement.quantity = 5 * u.arcsec measurement.quantity = 5 * u.mmag quantity_allclose(measurement.quantity, 5 * u.mmag)
def test_PA1_measurement_without_metric(self): """Test a measurement without a Metric instance.""" measurement = Measurement('validate_drp.PA1', 0.002 * u.mag) self.assertIsInstance(measurement.metric_name, Name) self.assertEqual(measurement.metric_name, Name('validate_drp.PA1')) self.assertIsNone(measurement.metric) json_doc = measurement.json # Units are not converted self.assertEqual(json_doc['unit'], 'mag') self.assertFloatsAlmostEqual(json_doc['value'], 0.002) new_measurement = Measurement.deserialize(**json_doc) self.assertEqual(measurement, new_measurement)
def makeMeasurement(self, values): """Compute the total number of SolarSystemObjects within a detectorVisit. Parameters ---------- values : `dict` [`str`, `int` or `None`] A `dict` representation of the metadata. Each `dict` has the following key: ``"numTotalSolarSystemObjects"`` The number of SolarSystemObjects within the observable detector area (`int` or `None`). May be `None` if solar system association was not attempted or the image was not successfully associated. Returns ------- measurement : `lsst.verify.Measurement` or `None` The total number of Solar System objects. """ if values["numTotalSolarSystemObjects"] is not None: try: nNew = int(values["numTotalSolarSystemObjects"]) except (ValueError, TypeError) as e: raise MetricComputationError( "Corrupted value of numTotalSolarSystemObjects") from e else: return Measurement(self.config.metricName, nNew * u.count) else: self.log.info("Nothing to do: no solar system results found.") return None
def run(self, sources): """Count the number of science sources. Parameters ---------- sources : iterable of `lsst.afw.table.SourceCatalog` A collection of science source catalogs, one for each unit of processing to be incorporated into this metric. Its elements may be `None` to represent missing data. Returns ------- result : `lsst.pipe.base.Struct` A `~lsst.pipe.base.Struct` containing the following component: ``measurement`` the total number of science sources (`lsst.verify.Measurement` or `None`) """ nSciSources = 0 inputData = False for catalog in sources: if catalog is not None: nSciSources += len(catalog) inputData = True if inputData: meas = Measurement(self.getOutputMetricName(self.config), nSciSources * u.count) else: self.log.info("Nothing to do: no catalogs found.") meas = None return Struct(measurement=meas)
def mergeFractionsPartial(fractions, denominatorsMinusNumerators): """Weighted sum of some fractions. Extras and notes will be dictionary-merged (i.e., if multiple extras or notes are assigned to the same key, only one will be preserved) from `fractions`. .. warning:: This function does NOT perform input validation Parameters ---------- fractions: list of `lsst.verify.Measurement` The measurements to combine. Must be measurements of the same metric. denominatorsMinusNumerators: list of `lsst.verify.Measurement` The denominators of `fractions` minus the numerators. Must be measurements of the same metric, and must have a one-to-one correspondence with `fractions`. Returns ------- A Measurement containing the average of `fractions`, weighted appropriately, and as much auxiliary data as could be reasonably saved. """ denominators = [] for fraction, partial in zip(fractions, denominatorsMinusNumerators): denominator = partial.quantity / (1.0 - fraction.quantity) denominators.append( Measurement(partial.metric, denominator, extras=partial.extras, notes=partial.notes)) return mergeFractions(fractions, denominators)
def test_PA1_deferred_metric(self): """Test a measurement when the Metric instance is added later.""" measurement = Measurement('PA1', 0.002 * u.mag) self.assertIsNone(measurement.metric) self.assertEqual(measurement.metric_name, Name(metric='PA1')) # Try adding in a metric with the wrong units to existing quantity other_metric = Metric('testing.other', 'Incompatible units', 'arcsec') with self.assertRaises(TypeError): measurement.metric = other_metric # Add metric in; the name should also update measurement.metric = self.pa1 self.assertEqual(measurement.metric, self.pa1) self.assertEqual(measurement.metric_name, self.pa1.name)
def run(self, sciSources, diaSources): """Compute the ratio of DIASources to science sources. Parameters ---------- sciSources : `lsst.afw.table.SourceCatalog` or `None` A science source catalog, which may be empty or `None`. diaSources : `lsst.afw.table.SourceCatalog` or `None` A DIASource catalog for the same unit of processing as ``sciSources``. Returns ------- result : `lsst.pipe.base.Struct` A `~lsst.pipe.base.Struct` containing the following component: ``measurement`` the ratio (`lsst.verify.Measurement` or `None`) """ if diaSources is not None and sciSources is not None: nSciSources = len(sciSources) nDiaSources = len(diaSources) metricName = self.config.metricName if nSciSources <= 0.0: raise MetricComputationError( "No science sources found; ratio of DIASources to science sources ill-defined." ) else: meas = Measurement( metricName, nDiaSources / nSciSources * u.dimensionless_unscaled) else: self.log.info("Nothing to do: no catalogs found.") meas = None return Struct(measurement=meas)
def run(self, matchedFakes, band): """Compute the completeness of recovered fakes within a magnitude range. Parameters ---------- matchedFakes : `lsst.afw.table.SourceCatalog` or `None` Catalog of fakes that were inserted into the ccdExposure matched to their detected counterparts. Returns ------- result : `lsst.pipe.base.Struct` A `~lsst.pipe.base.Struct` containing the following component: ``measurement`` the ratio (`lsst.verify.Measurement` or `None`) """ if matchedFakes is not None: magnitudes = matchedFakes[f"{self.config.magVar}" % band] magCutFakes = matchedFakes[np.logical_and(magnitudes > self.config.magMin, magnitudes < self.config.magMax)] if len(magCutFakes) <= 0.0: raise MetricComputationError( "No matched fakes catalog sources found; Completeness is " "ill defined.") else: meas = Measurement( self.config.metricName, ((magCutFakes["diaSourceId"] > 0).sum() / len(magCutFakes)) * u.dimensionless_unscaled) else: self.log.info("Nothing to do: no matched catalog found.") meas = None return Struct(measurement=meas)
def makeMeasurement(self, values): """Compute the number of new DIAObjects. Parameters ---------- values : `dict` [`str`, `int` or `None`] A `dict` representation of the metadata. Each `dict` has the following key: ``"newObjects"`` The number of new objects created for this image (`int` or `None`). May be `None` if the image was not successfully associated. Returns ------- measurement : `lsst.verify.Measurement` or `None` The total number of new objects. """ if values["newObjects"] is not None: try: nNew = int(values["newObjects"]) except (ValueError, TypeError) as e: raise MetricComputationError( "Corrupted value of numNewDiaObjects") from e else: return Measurement(self.config.metricName, nNew * u.count) else: self.log.info("Nothing to do: no association results found.") return None
def makeMeasurement(self, timings): """Compute a wall-clock measurement from metadata provided by `lsst.utils.timer.timeMethod`. Parameters ---------- timings : `dict` [`str`, any] A representation of the metadata passed to `run`. The `dict` has the following keys: ``"StartTime"`` The time the target method started (`float` or `None`). ``"EndTime"`` The time the target method ended (`float` or `None`). ``"StartTimestamp"``, ``"EndTimestamp"`` The start and end timestamps, in an ISO 8601-compliant format (`str` or `None`). Returns ------- measurement : `lsst.verify.Measurement` or `None` The running time of the target method. Raises ------ MetricComputationError Raised if the timing metadata are invalid. """ if timings["StartTime"] is not None or timings["EndTime"] is not None: try: totalTime = timings["EndTime"] - timings["StartTime"] except TypeError: raise MetricComputationError("Invalid metadata") else: meas = Measurement(self.config.metricName, totalTime * u.second) meas.notes["estimator"] = "utils.timer.timeMethod" if timings["StartTimestamp"]: meas.extras["start"] = Datum(timings["StartTimestamp"]) if timings["EndTimestamp"]: meas.extras["end"] = Datum(timings["EndTimestamp"]) return meas else: self.log.info("Nothing to do: no timing information for %s found.", self.config.target) return None
def run(self, matchedCatalog, metric_name): self.log.info("Measuring PF1") pa2_thresh = self.threshPA2 * u.mmag pf1 = photRepeat(matchedCatalog, numRandomShuffles=self.numRandomShuffles, randomSeed=self.randomSeed) if 'magDiff' in pf1.keys(): # Previously, validate_drp used the first random sample from PA1 measurement # Now, use all of them. magDiffs = pf1['magDiff'] percentileAtPA2 = 100 * np.mean(np.abs(magDiffs.value) > pa2_thresh.value) * u.percent return Struct(measurement=Measurement("PF1", percentileAtPA2)) else: return Struct(measurement=Measurement("PF1", np.nan*u.percent))
def test_yamlpersist_complex(self): measurement = Measurement( self.pa1, 5. * u.mmag, notes={'filter_name': 'r'}, blobs=[self.blob1], extras={'extra1': Datum(10. * u.arcmin, 'Extra 1')}) self._check_yaml_round_trip(measurement)
def run(self, matchedCatalog, metric_name): self.log.info("Measuring PA2") pf1_thresh = self.threshPF1 * u.percent pa2 = photRepeat(matchedCatalog, numRandomShuffles=self.numRandomShuffles, randomSeed=self.randomSeed) if 'magDiff' in pa2.keys(): # Previously, validate_drp used the first random sample from PA1 measurement # Now, use all of them. magDiffs = pa2['magDiff'] pf1Percentile = 100.*u.percent - pf1_thresh return Struct(measurement=Measurement("PA2", np.percentile(np.abs(magDiffs.value), pf1Percentile.value) * magDiffs.unit)) else: return Struct(measurement=Measurement("PA2", np.nan*u.mmag))
def test_creation_with_extras(self): """Test creating a measurement with an extra.""" measurement = Measurement( self.pa1, 5. * u.mmag, extras={'extra1': Datum(10. * u.arcmin, 'Extra 1')}) self.assertIn(str(self.pa1.name), measurement.blobs) self.assertIn('extra1', measurement.extras) json_doc = measurement.json self.assertIn(measurement.extras.identifier, json_doc['blob_refs']) blobs = BlobSet([b for k, b in measurement.blobs.items()]) new_measurement = Measurement.deserialize(blobs=blobs, **json_doc) self.assertIn('extra1', new_measurement.extras) self.assertEqual(measurement, new_measurement)
def run(self, matchedCatalog, metric_name): self.log.info(f"Measuring {metric_name}") sepDistances = astromResiduals(matchedCatalog, self.config.bright_mag_cut, self.config.faint_mag_cut, self.config.annulus_r, self.config.width) adxThresh = self.config.threshAD * u.marcsec if len(sepDistances) <= 1: return Struct(measurement=Measurement(metric_name, np.nan*u.percent)) else: # absolute value of the difference between each astrometric rms # and the median astrometric RMS # absRmsDiffs = np.abs(rmsDistances - np.median(rmsDistances)).to(u.marcsec) absDiffsMarcsec = (sepDistances-np.median(sepDistances)).to(u.marcsec) percentileAtADx = 100 * np.mean(np.abs(absDiffsMarcsec.value) > adxThresh.value) * u.percent return Struct(measurement=Measurement(metric_name, percentileAtADx))
def setUp(self): self.job = Job() self.job.metrics.insert( Metric("foo.boringmetric", "", u.percent, tags=["redundant"])) self.job.metrics.insert( Metric("foo.fancymetric", "", u.meter, tags=["vital"])) self.job.measurements.insert( Measurement("foo.fancymetric", 2.0 * u.meter)) self.job.measurements.insert( Measurement("foo.fanciermetric", 3.5 * u.second)) self.job.measurements["foo.fanciermetric"].notes["fanciness"] \ = "moderate" self.job.measurements.insert( Measurement("foo.fanciestmetric", 3.1415927 * u.kilogram)) self.job.meta["bar"] = "high" self.job.meta["shape"] = "rotund" self.job.specs.insert( ThresholdSpecification("utterly_ridiculous", 1e10 * u.meter, ">"))
def run(self, sciSources, diaSources): """Compute the ratio of DIASources to science sources. Parameters ---------- sciSources : iterable of `lsst.afw.table.SourceCatalog` A collection of science source catalogs, one for each unit of processing to be incorporated into this metric. Its elements may be `None` to represent missing data. diaSources : iterable of `lsst.afw.table.SourceCatalog` A collection of difference imaging catalogs similar to ``sciSources``. Returns ------- result : `lsst.pipe.base.Struct` A `~lsst.pipe.base.Struct` containing the following component: ``measurement`` the ratio (`lsst.verify.Measurement` or `None`) """ nSciSources = 0 nDiaSources = 0 inputData = False for sciCatalog, diaCatalog in zip(sciSources, diaSources): if diaCatalog is not None and sciCatalog is not None: nSciSources += len(sciCatalog) nDiaSources += len(diaCatalog) inputData = True if inputData: metricName = self.getOutputMetricName(self.config) if nSciSources <= 0.0: raise MetricComputationError( "No science sources found; ratio of DIASources to science sources ill-defined.") meas = Measurement(metricName, 0.0 * u.dimensionless_unscaled) else: meas = Measurement(metricName, nDiaSources / nSciSources * u.dimensionless_unscaled) else: self.log.info("Nothing to do: no catalogs found.") meas = None return Struct(measurement=meas)
def run(self, catalog, metric_name, vIds): self.log.info(f"Measuring {metric_name}") if not catalog.isContiguous(): catalog = catalog.copy(deep=True) extended = catalog.get('base_ClassificationExtendedness_value') good_extended = extended[~catalog.get('base_ClassificationExtendedness_flag')] n_gals = sum(good_extended) frac = 100*(len(good_extended) - n_gals)/len(good_extended) meas = Measurement("starFrac", frac * u.percent) return Struct(measurement=meas)