def get_specs_metrics(job):
# Get specs for this filter
subset = job.specs.subset(required_meta={
'instrument': job.meta['instrument'],
'filter_name': job.meta['filter_name']
},
spec_tags=['chromatic'])
# Get specs that don't depend on filter
subset.update(
job.specs.subset(required_meta={'instrument': job.meta['instrument']},
spec_tags=['achromatic']))
metrics = {}
specs = {}
for spec in subset:
metric_name = spec.metric.split('_')[
0] # Take first part for linked metrics
if metric_name in metrics:
metrics[metric_name].append(
Name(package=spec.package, metric=spec.metric))
specs[metric_name].append(spec)
else:
metrics[metric_name] = [
Name(package=spec.package, metric=spec.metric),
]
specs[metric_name] = [
spec,
]
return specs, metrics
def test_setitem_delitem(self):
"""Test adding and deleting metrics."""
m1 = Metric('validate_drp.test',
'test',
'',
reference_url='example.com',
reference_doc='Doc',
reference_page=1)
metric_set = MetricSet()
self.assertEqual(len(metric_set), 0)
metric_set['validate_drp.test'] = m1
self.assertEqual(len(metric_set), 1)
self.assertEqual(metric_set['validate_drp.test'], m1)
with self.assertRaises(KeyError):
# inconsistent metric names
metric_set['validate_drp.new_test'] = m1
with self.assertRaises(TypeError):
# Not a metric name
n = Name('validate_drp')
m2 = Metric(n, 'test', '')
metric_set[n] = m2
del metric_set['validate_drp.test']
self.assertEqual(len(metric_set), 0)
def setUp(self):
super().setUp()
self.config = self._standardConfig()
self.metric = Name("verify.DummyMemory")
self.scienceTask = DummyTask()
self.scienceTask.run()
def testEmptyCatalog(self):
catalog = _makeDummyCatalog(0)
result = self.task.run([catalog])
meas = result.measurement
self.assertEqual(meas.metric_name, Name(metric="ip_diffim.numSciSources"))
self.assertEqual(meas.quantity, 0 * u.count)
def testAllAssociated(self):
result = self.task.run([{"test_value": 0}])
lsst.pipe.base.testUtils.assertValidOutput(self.task, result)
meas = result.measurement
self.assertEqual(meas.metric_name, Name(metric="ap_association.totalUnassociatedDiaObjects"))
self.assertEqual(meas.quantity, 0.0 * u.count)
def setUp(self):
super().setUp()
self.config = TimingMetricTestSuite._standardConfig()
self.metric = Name("verify.DummyTime")
self.scienceTask = DummyTask()
self.scienceTask.run()
def testSkySources(self):
catalog = _makeDummyCatalog(3, skyFlag=True)
result = self.task.run(catalog)
lsst.pipe.base.testUtils.assertValidOutput(self.task, result)
meas = result.measurement
self.assertEqual(meas.metric_name, Name(metric="ip_diffim.numSciSources"))
self.assertEqual(meas.quantity, (len(catalog) - 1) * u.count)
def testNoNew(self):
metadata = _makeAssociationMetadata(numNew=0)
result = self.task.run(metadata)
lsst.pipe.base.testUtils.assertValidOutput(self.task, result)
meas = result.measurement
self.assertEqual(meas.metric_name, Name(metric="ap_association.numNewDiaObjects"))
self.assertEqual(meas.quantity, 0.0 * u.count)
def testValid(self):
result = self.task.run([self.scienceTask.getFullMetadata()])
meas = result.measurement
self.assertIsInstance(meas, Measurement)
self.assertEqual(meas.metric_name, Name(metric=self.config.metric))
self.assertGreater(meas.quantity, 0.0 * u.second)
self.assertLess(meas.quantity, 2 * DummyTask.taskLength * u.second)
def testNoNew(self):
metadata = _makeAssociationMetadata(numNew=0)
result = self.task.run([metadata])
meas = result.measurement
self.assertEqual(meas.metric_name,
Name(metric="ap_association.numNewDiaObjects"))
self.assertEqual(meas.quantity, 0.0 * u.count)
def testEmptyDiaCatalog(self):
sciCatalog = _makeDummyCatalog(5)
diaCatalog = _makeDummyCatalog(0)
result = self.task.run([sciCatalog], [diaCatalog])
meas = result.measurement
self.assertEqual(meas.metric_name, Name(metric="ip_diffim.fracDiaSourcesToSciSources"))
self.assertEqual(meas.quantity, 0.0 * u.dimensionless_unscaled)
def testAllUpdated(self):
metadata = _makeAssociationMetadata(numUnassociated=0)
result = self.task.run([metadata])
meas = result.measurement
self.assertEqual(meas.metric_name,
Name(metric="ap_association.fracUpdatedDiaObjects"))
self.assertEqual(meas.quantity, 1.0 * u.dimensionless_unscaled)
def testAllUpdated(self):
metadata = _makeAssociationMetadata(numSso=0)
result = self.task.run(metadata)
lsst.pipe.base.testUtils.assertValidOutput(self.task, result)
meas = result.measurement
self.assertEqual(meas.metric_name, Name(metric="ap_association.numTotalSolarSystemObjects"))
self.assertEqual(meas.quantity, 0.0 * u.count)
def testAllUpdated(self):
metadata = _makeAssociationMetadata(numUnassociated=0)
result = self.task.run(metadata)
lsst.pipe.base.testUtils.assertValidOutput(self.task, result)
meas = result.measurement
self.assertEqual(meas.metric_name, Name(metric="ap_association.fracUpdatedDiaObjects"))
self.assertEqual(meas.quantity, 1.0 * u.dimensionless_unscaled)
def testValid(self):
catalog = _makeDummyCatalog(3)
result = self.task.run(catalog)
meas = result.measurement
self.assertEqual(meas.metric_name,
Name(metric="ip_diffim.numSciSources"))
self.assertEqual(meas.quantity, len(catalog) * u.count)
def testSkySources(self):
sciCatalog = _makeDummyCatalog(5, skyFlag=True)
diaCatalog = _makeDummyCatalog(3)
result = self.task.run(sciCatalog, diaCatalog)
lsst.pipe.base.testUtils.assertValidOutput(self.task, result)
meas = result.measurement
self.assertEqual(meas.metric_name, Name(metric="ip_diffim.fracDiaSourcesToSciSources"))
self.assertEqual(meas.quantity, len(diaCatalog) / (len(sciCatalog) - 1) * u.dimensionless_unscaled)
def testEmptyDiaCatalog(self):
sciCatalog = _makeDummyCatalog(5)
diaCatalog = _makeDummyCatalog(0)
result = self.task.run(sciCatalog, diaCatalog)
lsst.pipe.base.testUtils.assertValidOutput(self.task, result)
meas = result.measurement
self.assertEqual(meas.metric_name, Name(metric="ip_diffim.fracDiaSourcesToSciSources"))
self.assertEqual(meas.quantity, 0.0 * u.dimensionless_unscaled)
def testValid(self):
metadata = _makeAssociationMetadata()
result = self.task.run(metadata)
lsst.pipe.base.testUtils.assertValidOutput(self.task, result)
meas = result.measurement
self.assertEqual(meas.metric_name, Name(metric="ap_association.numAssociatedSsObjects"))
self.assertEqual(meas.quantity,
metadata.getAsDouble("association.numAssociatedSsObjects") * u.count)
def testEmptyCatalog(self):
catalog = _makeDummyCatalog(0)
result = self.task.run(catalog)
lsst.pipe.base.testUtils.assertValidOutput(self.task, result)
meas = result.measurement
self.assertEqual(meas.metric_name,
Name(metric="ip_diffim.numSciSources"))
assert_quantity_allclose(meas.quantity, 0 * u.count)
def testValid(self):
result = self.task.adaptArgsAndRun({"dbInfo": "DB source"},
{"dbInfo": {}}, {"measurement": {}})
meas = result.measurement
self.assertEqual(
meas.metric_name,
Name(metric="ap_association.totalUnassociatedDiaObjects"))
nObjects = self.mockCounter(self._makePpdb())
self.assertEqual(meas.quantity, nObjects * u.count)
def testValid(self):
metadata = _makeAssociationMetadata()
result = self.task.run(metadata)
lsst.pipe.base.testUtils.assertValidOutput(self.task, result)
meas = result.measurement
self.assertEqual(meas.metric_name, Name(metric="ap_association.fracUpdatedDiaObjects"))
nUpdated = metadata.getAsDouble("association.numUpdatedDiaObjects")
nTotal = metadata.getAsDouble("association.numUnassociatedDiaObjects") + nUpdated
self.assertEqual(meas.quantity, nUpdated / nTotal * u.dimensionless_unscaled)
def testValid(self):
metadata = _makeAssociationMetadata()
result = self.task.run([metadata])
meas = result.measurement
self.assertEqual(meas.metric_name,
Name(metric="ap_association.numNewDiaObjects"))
self.assertEqual(
meas.quantity,
metadata.getAsDouble("association.numNewDiaObjects") * u.count)
def testValid(self):
sciCatalog = _makeDummyCatalog(5)
diaCatalog = _makeDummyCatalog(3)
result = self.task.run(sciCatalog, diaCatalog)
meas = result.measurement
self.assertEqual(meas.metric_name,
Name(metric="ip_diffim.fracDiaSourcesToSciSources"))
self.assertEqual(
meas.quantity,
len(diaCatalog) / len(sciCatalog) * u.dimensionless_unscaled)
def testValidEmpty(self):
"""Test the run method with a valid but zero result.
"""
metricComplete = self.makeTask(self.magCut, self.magCut + 5)
result = metricComplete.run(self.fakeCat, self.band)
testUtils.assertValidOutput(metricComplete, result)
meas = result.measurement
self.assertEqual(meas.metric_name,
Name(metric="ap_pipe.apFakesCompleteness"))
self.assertEqual(meas.quantity, 0 * u.dimensionless_unscaled)
def test_query_metadata(self):
job = Job(meta={'filter_name': 'r', 'camera': 'MegaCam'})
s1 = ThresholdSpecification(Name('validate_drp.AM1.design_r'),
5. * u.marcsec,
'<',
metadata_query={'filter_name': 'r'})
s2 = ThresholdSpecification(Name('validate_drp.AM1.design_i'),
5. * u.marcsec,
'<',
metadata_query={'filter_name': 'i'})
s3 = ThresholdSpecification(Name('validate_drp.AM1.design_HSC_r'),
5. * u.marcsec,
'<',
metadata_query={
'filter_name': 'r',
'camera': 'HSC'
})
self.assertTrue(s1.query_metadata(job.meta))
self.assertFalse(s2.query_metadata(job.meta))
self.assertFalse(s3.query_metadata(job.meta))
def testAllAssociated(self):
with unittest.mock.patch(
"lsst.ap.association.metrics.countUnassociatedObjects",
return_value=0):
result = self.task.adaptArgsAndRun({"dbInfo": "DB source"},
{"dbInfo": {}},
{"measurement": {}})
meas = result.measurement
self.assertEqual(
meas.metric_name,
Name(metric="ap_association.totalUnassociatedDiaObjects"))
self.assertEqual(meas.quantity, 0.0 * u.count)
def testValid(self):
"""Test the run method.
"""
result = self.task.run(self.fakeCat, self.band)
testUtils.assertValidOutput(self.task, result)
meas = result.measurement
self.assertEqual(meas.metric_name,
Name(metric="ap_pipe.apFakesCompleteness"))
# Work around for Mac failing this test.
self.assertAlmostEqual(
meas.quantity.value,
((self.expectedAllMatched / self.targetSources) *
u.dimensionless_unscaled).value,
places=2)
def test_equality(self):
"""test __eq__."""
s1 = ThresholdSpecification(Name('validate_drp.AM1.design'),
5. * u.marcsec, '<')
# with compatible units
s3 = ThresholdSpecification(Name('validate_drp.AM1.design'),
5e-3 * u.arcsec, '<')
self.assertEqual(s1, s3)
# incompatible names
s4 = ThresholdSpecification(Name('validate_drp.AM1.stretch'),
5. * u.marcsec, '<')
self.assertNotEqual(s1, s4)
# incompatible threshold
s5 = ThresholdSpecification(Name('validate_drp.AM1.design'),
5. * u.arcsec, '<')
self.assertNotEqual(s1, s5)
# incompatible operator
s6 = ThresholdSpecification(Name('validate_drp.AM1.design'),
5. * u.marcsec, '>')
self.assertNotEqual(s1, s6)
def test_init(self):
"""Test initialization patterns."""
# with a fully-specified Name
s1 = ThresholdSpecification(Name('validate_drp.AM1.design'),
5. * u.marcsec, '<')
# with a fully-specified string-based name
s2 = ThresholdSpecification('validate_drp.AM1.design', 5. * u.marcsec,
'<')
self.assertEqual(s1, s2)
# bad operator
with self.assertRaises(TypeError):
ThresholdSpecification('validate_drp.AM1.design', 5. * u.marcsec,
'<<')
# bad quantity
with self.assertRaises(TypeError):
ThresholdSpecification('validate_drp.AM1.design', 5., '<')
# bad name
with self.assertRaises(TypeError):
ThresholdSpecification(Name(metric='validate_drp'), 5. * u.marcsec,
'<')
def test_PA1_measurement_without_metric(self):
"""Test a measurement without a Metric instance."""
measurement = Measurement('validate_drp.PA1', 0.002 * u.mag)
self.assertIsInstance(measurement.metric_name, Name)
self.assertEqual(measurement.metric_name, Name('validate_drp.PA1'))
self.assertIsNone(measurement.metric)
json_doc = measurement.json
# Units are not converted
self.assertEqual(json_doc['unit'], 'mag')
self.assertFloatsAlmostEqual(json_doc['value'], 0.002)
new_measurement = Measurement.deserialize(**json_doc)
self.assertEqual(measurement, new_measurement)