def testTransaction(self):
butler = Butler(self.tmpConfigFile, run="ingest")
datasetTypeName = "test_metric"
dimensions = butler.registry.dimensions.extract(
["instrument", "visit"])
dimensionEntries = (("instrument", {
"instrument": "DummyCam"
}), ("physical_filter", {
"instrument": "DummyCam",
"name": "d-r",
"abstract_filter": "R"
}), ("visit", {
"instrument": "DummyCam",
"id": 42,
"name": "fortytwo",
"physical_filter": "d-r"
}))
storageClass = self.storageClassFactory.getStorageClass(
"StructuredData")
metric = makeExampleMetrics()
dataId = {"instrument": "DummyCam", "visit": 42}
with self.assertRaises(TransactionTestError):
with butler.transaction():
# Create and register a DatasetType
datasetType = self.addDatasetType(datasetTypeName, dimensions,
storageClass,
butler.registry)
# Add needed Dimensions
for args in dimensionEntries:
butler.registry.insertDimensionData(*args)
# Store a dataset
ref = butler.put(metric, datasetTypeName, dataId)
self.assertIsInstance(ref, DatasetRef)
# Test getDirect
metricOut = butler.getDirect(ref)
self.assertEqual(metric, metricOut)
# Test get
metricOut = butler.get(datasetTypeName, dataId)
self.assertEqual(metric, metricOut)
# Check we can get components
self.assertGetComponents(butler, ref,
("summary", "data", "output"), metric)
raise TransactionTestError(
"This should roll back the entire transaction")
with self.assertRaises(KeyError):
butler.registry.getDatasetType(datasetTypeName)
with self.assertRaises(LookupError):
butler.registry.expandDataId(dataId)
# Should raise KeyError for missing DatasetType
with self.assertRaises(KeyError):
butler.get(datasetTypeName, dataId)
# Also check explicitly if Dataset entry is missing
self.assertIsNone(
butler.registry.find(butler.collection, datasetType, dataId))
# Direct retrieval should not find the file in the Datastore
with self.assertRaises(FileNotFoundError):
butler.getDirect(ref)
def testBasicPutGet(self):
butler = Butler(self.configFile)
# Create and register a DatasetType
datasetTypeName = "test_metric"
dataUnits = ("Camera", "Visit")
storageClass = self.storageClassFactory.getStorageClass(
"StructuredData")
self.registerDatasetTypes(datasetTypeName, dataUnits, storageClass,
butler.registry)
# Create and store a dataset
metric = makeExampleMetrics()
dataId = {"camera": "DummyCam", "visit": 42}
ref = butler.put(metric, datasetTypeName, dataId)
self.assertIsInstance(ref, DatasetRef)
# Test getDirect
metricOut = butler.getDirect(ref)
self.assertEqual(metric, metricOut)
# Test get
metricOut = butler.get(datasetTypeName, dataId)
self.assertEqual(metric, metricOut)
# Check we can get components
self.assertGetComponents(butler, datasetTypeName, dataId,
("summary", "data", "output"), metric)
class TestHipsOutputs(unittest.TestCase, MockCheckMixin):
"""Check that HIPS outputs are as expected."""
def setUp(self):
self.butler = Butler(os.path.join(getPackageDir("ci_hsc_gen3"),
"DATA"),
instrument="HSC",
skymap="discrete/ci_hsc",
writeable=False,
collections=["HSC/runs/ci_hsc_hips"])
self.skip_mock()
self._bands = ['r', 'i']
def test_hips_exist(self):
"""Test that the HIPS images exist and are readable."""
for band in self._bands:
datasets = set(
self.butler.registry.queryDatasets("deepCoadd_hpx", band=band))
# There are 64 HIPS images for each band.
self.assertEqual(len(datasets), 64)
for dataset in datasets:
self.assertTrue(self.butler.datastore.exists(dataset),
msg="File exists for deepCoadd_hpx")
exp = self.butler.getDirect(list(datasets)[0])
self.assertEqual(exp.wcs.getFitsMetadata()["CTYPE1"], "RA---HPX")
self.assertEqual(exp.wcs.getFitsMetadata()["CTYPE2"], "DEC--HPX")
def testStrayLightIngest(self):
"""Ingested stray light files."""
butler = Butler(self.root, run=self.outputRun)
straylightDir = os.path.join(testDataDirectory, "hsc", "straylight")
instrument = self.instrumentClass()
# This will warn about lots of missing files
with self.assertLogs(level="WARNING") as cm:
instrument.ingestStrayLightData(butler,
straylightDir,
transfer="auto")
collection = self.instrumentClass.makeCalibrationCollectionName()
datasets = list(
butler.registry.queryDatasetAssociations("yBackground",
collections=collection))
# Should have at least one dataset and dataset+warnings = 112
self.assertGreaterEqual(len(datasets), 1)
self.assertEqual(
len(datasets) + len(cm.output), len(instrument.getCamera()))
# Ensure that we can read the first stray light file
strayLight = butler.getDirect(datasets[0].ref)
self.assertIsInstance(strayLight, SubaruStrayLightData)
def runPutGetTest(self, storageClass, datasetTypeName):
butler = Butler(self.tmpConfigFile)
# There will not be a collection yet
collections = butler.registry.getAllCollections()
self.assertEqual(collections, set())
# Create and register a DatasetType
dimensions = butler.registry.dimensions.extract(
["instrument", "visit"])
datasetType = self.addDatasetType(datasetTypeName, dimensions,
storageClass, butler.registry)
# Add needed Dimensions
butler.registry.addDimensionEntry("instrument",
{"instrument": "DummyCamComp"})
butler.registry.addDimensionEntry("physical_filter", {
"instrument": "DummyCamComp",
"physical_filter": "d-r"
})
butler.registry.addDimensionEntry("visit", {
"instrument": "DummyCamComp",
"visit": 423,
"physical_filter": "d-r"
})
# Create and store a dataset
metric = makeExampleMetrics()
dataId = {"instrument": "DummyCamComp", "visit": 423}
# Create a DatasetRef for put
refIn = DatasetRef(datasetType, dataId, id=None)
# Put with a preexisting id should fail
with self.assertRaises(ValueError):
butler.put(metric, DatasetRef(datasetType, dataId, id=100))
# Put and remove the dataset once as a DatasetRef, once as a dataId,
# and once with a DatasetType
for args in ((refIn, ), (datasetTypeName, dataId), (datasetType,
dataId)):
with self.subTest(args=args):
ref = butler.put(metric, *args)
self.assertIsInstance(ref, DatasetRef)
# Test getDirect
metricOut = butler.getDirect(ref)
self.assertEqual(metric, metricOut)
# Test get
metricOut = butler.get(ref.datasetType.name, dataId)
self.assertEqual(metric, metricOut)
# Test get with a datasetRef
metricOut = butler.get(ref)
self.assertEqual(metric, metricOut)
# Check we can get components
if storageClass.isComposite():
self.assertGetComponents(butler, ref,
("summary", "data", "output"),
metric)
# Remove from collection only; after that we shouldn't be able
# to find it unless we use the dataset_id.
butler.remove(*args, delete=False)
with self.assertRaises(LookupError):
butler.datasetExists(*args)
# If we use the output ref with the dataset_id, we should
# still be able to load it with getDirect().
self.assertEqual(metric, butler.getDirect(ref))
# Reinsert into collection, then delete from Datastore *and*
# remove from collection.
butler.registry.associate(butler.collection, [ref])
butler.remove(*args)
# Lookup with original args should still fail.
with self.assertRaises(LookupError):
butler.datasetExists(*args)
# Now getDirect() should fail, too.
with self.assertRaises(FileNotFoundError):
butler.getDirect(ref)
# Registry still knows about it, if we use the dataset_id.
self.assertEqual(butler.registry.getDataset(ref.id), ref)
# Put again, then remove completely (this generates a new
# dataset record in registry, with a new ID - the old one
# still exists but it is not in any collection so we don't
# care).
ref = butler.put(metric, *args)
butler.remove(*args, remember=False)
# Lookup with original args should still fail.
with self.assertRaises(LookupError):
butler.datasetExists(*args)
# getDirect() should still fail.
with self.assertRaises(FileNotFoundError):
butler.getDirect(ref)
# Registry shouldn't be able to find it by dataset_id anymore.
self.assertIsNone(butler.registry.getDataset(ref.id))
# Put the dataset again, since the last thing we did was remove it.
ref = butler.put(metric, refIn)
# Get with parameters
stop = 4
sliced = butler.get(ref, parameters={"slice": slice(stop)})
self.assertNotEqual(metric, sliced)
self.assertEqual(metric.summary, sliced.summary)
self.assertEqual(metric.output, sliced.output)
self.assertEqual(metric.data[:stop], sliced.data)
# Combining a DatasetRef with a dataId should fail
with self.assertRaises(ValueError):
butler.get(ref, dataId)
# Getting with an explicit ref should fail if the id doesn't match
with self.assertRaises(ValueError):
butler.get(DatasetRef(ref.datasetType, ref.dataId, id=101))
# Getting a dataset with unknown parameters should fail
with self.assertRaises(KeyError):
butler.get(ref, parameters={"unsupported": True})
# Check we have a collection
collections = butler.registry.getAllCollections()
self.assertEqual(collections, {
"ingest",
})
def makeDiscreteSkyMap(repo,
config_file,
collections,
instrument,
skymap_id='discrete',
old_skymap_id=None):
"""Implements the command line interface `butler make-discrete-skymap` subcommand,
should only be called by command line tools and unit test code that tests
this function.
Constructs a skymap from calibrated exposure in the butler repository
Parameters
----------
repo : `str`
URI to the location to read the repo.
config_file : `str` or `None`
Path to a config file that contains overrides to the skymap config.
collections : `list` [`str`]
An expression specifying the collections to be searched (in order) when
reading datasets, and optionally dataset type restrictions on them.
At least one collection must be specified. This is the collection
with the calibrated exposures.
instrument : `str`
The name or fully-qualified class name of an instrument.
skymap_id : `str`, optional
The identifier of the skymap to save. Default is 'discrete'.
old_skymap_id : `str`, optional
The identifer of the skymap to append to. Must differ from
``skymap_id``. Ignored unless ``config.doAppend=True``.
"""
butler = Butler(repo, collections=collections, writeable=True)
instr = getInstrument(instrument, butler.registry)
config = MakeDiscreteSkyMapConfig()
instr.applyConfigOverrides(MakeDiscreteSkyMapTask._DefaultName, config)
if config_file is not None:
config.load(config_file)
# The coaddName for a SkyMap is only relevant in Gen2, and we completely
# ignore it here; once Gen2 is gone it can be removed.
oldSkyMap = None
if config.doAppend:
if old_skymap_id is None:
raise ValueError(
"old_skymap_id must be provided if config.doAppend is True.")
dataId = {'skymap': old_skymap_id}
try:
oldSkyMap = butler.get(BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
collections=collections,
dataId=dataId)
except LookupError as e:
msg = (
f"Could not find seed skymap with dataId {dataId} "
f"in collections {collections} but doAppend is {config.doAppend}. Aborting..."
)
raise LookupError(msg, *e.args[1:])
datasets = butler.registry.queryDatasets('calexp', collections=collections)
wcs_md_tuple_list = [(butler.getDirect('calexp.metadata', ref),
butler.getDirect('calexp.wcs', ref))
for ref in datasets]
task = MakeDiscreteSkyMapTask(config=config)
result = task.run(wcs_md_tuple_list, oldSkyMap)
result.skyMap.register(skymap_id, butler)
butler.put(result.skyMap,
BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
dataId={'skymap': skymap_id},
run=BaseSkyMap.SKYMAP_RUN_COLLECTION_NAME)
def checkRepo(self, files=None):
# Test amp parameter implementation for the LSST raw formatter. This
# is the same for all instruments, so repeating it in other test cases
# is wasteful.
butler = Butler(self.root, run=self.outputRun)
ref = butler.registry.findDataset("raw", self.dataIds[0])
full_assembled = butler.getDirect(ref)
unassembled_detector = self.instrumentClass().getCamera()[
ref.dataId["detector"]]
assembled_detector = full_assembled.getDetector()
for unassembled_amp, assembled_amp in zip(unassembled_detector,
assembled_detector):
# Check that we're testing what we think we're testing: these
# amps should differ in assembly state (offsets, flips), and they
# _may_ differ in fundamental geometry if we had to patch the
# overscan region sizes.
comparison = unassembled_amp.compareGeometry(assembled_amp)
self.assertTrue(comparison
& AmplifierGeometryComparison.ASSEMBLY_DIFFERS)
assembled_subimage = butler.getDirect(
ref, parameters={"amp": assembled_amp})
unassembled_subimage = butler.getDirect(
ref, parameters={"amp": unassembled_amp.getName()})
self.assertEqual(len(assembled_subimage.getDetector()), 1)
self.assertEqual(len(unassembled_subimage.getDetector()), 1)
self.assertEqual(len(assembled_subimage.getDetector()), 1)
self.assertEqual(len(unassembled_subimage.getDetector()), 1)
self.assertImagesEqual(
assembled_subimage.image,
full_assembled.image[assembled_amp.getRawBBox()])
self.assertImagesEqual(
unassembled_subimage.image,
flipImage(
full_assembled.image[assembled_amp.getRawBBox()],
flipLR=unassembled_amp.getRawFlipX(),
flipTB=unassembled_amp.getRawFlipY(),
),
)
self.assertAmplifiersEqual(assembled_subimage.getDetector()[0],
assembled_amp)
if comparison & comparison.REGIONS_DIFFER:
# We needed to patch overscans, but unassembled_amp (which
# comes straight from the camera) won't have those patches, so
# we can't compare it to the amp attached to
# unassembled_subimage (which does have those patches).
comparison2 = unassembled_subimage.getDetector(
)[0].compareGeometry(unassembled_amp)
self.assertTrue(comparison2
& AmplifierGeometryComparison.REGIONS_DIFFER)
# ...and that unassembled_subimage's amp has the same regions
# (after accounting for assembly/orientation) as assembled_amp.
comparison3 = unassembled_subimage.getDetector(
)[0].compareGeometry(assembled_amp)
self.assertTrue(comparison3
& AmplifierGeometryComparison.ASSEMBLY_DIFFERS)
self.assertFalse(comparison3
& AmplifierGeometryComparison.REGIONS_DIFFER)
else:
self.assertAmplifiersEqual(
unassembled_subimage.getDetector()[0], unassembled_amp)
def runPutGetTest(self, storageClass, datasetTypeName):
butler = Butler(self.tmpConfigFile, run="ingest")
# There will not be a collection yet
collections = butler.registry.getAllCollections()
self.assertEqual(collections, set())
# Create and register a DatasetType
dimensions = butler.registry.dimensions.extract(
["instrument", "visit"])
datasetType = self.addDatasetType(datasetTypeName, dimensions,
storageClass, butler.registry)
# Add needed Dimensions
butler.registry.insertDimensionData("instrument",
{"name": "DummyCamComp"})
butler.registry.insertDimensionData("physical_filter", {
"instrument": "DummyCamComp",
"name": "d-r",
"abstract_filter": "R"
})
butler.registry.insertDimensionData(
"visit", {
"instrument": "DummyCamComp",
"id": 423,
"name": "fourtwentythree",
"physical_filter": "d-r"
})
# Create and store a dataset
metric = makeExampleMetrics()
dataId = {"instrument": "DummyCamComp", "visit": 423}
# Create a DatasetRef for put
refIn = DatasetRef(datasetType, dataId, id=None)
# Put with a preexisting id should fail
with self.assertRaises(ValueError):
butler.put(metric, DatasetRef(datasetType, dataId, id=100))
# Put and remove the dataset once as a DatasetRef, once as a dataId,
# and once with a DatasetType
for args in ((refIn, ), (datasetTypeName, dataId), (datasetType,
dataId)):
with self.subTest(args=args):
ref = butler.put(metric, *args)
self.assertIsInstance(ref, DatasetRef)
# Test getDirect
metricOut = butler.getDirect(ref)
self.assertEqual(metric, metricOut)
# Test get
metricOut = butler.get(ref.datasetType.name, dataId)
self.assertEqual(metric, metricOut)
# Test get with a datasetRef
metricOut = butler.get(ref)
self.assertEqual(metric, metricOut)
# Test getDeferred with dataId
metricOut = butler.getDeferred(ref.datasetType.name,
dataId).get()
self.assertEqual(metric, metricOut)
# Test getDeferred with a datasetRef
metricOut = butler.getDeferred(ref).get()
self.assertEqual(metric, metricOut)
# Check we can get components
if storageClass.isComposite():
self.assertGetComponents(butler, ref,
("summary", "data", "output"),
metric)
# Remove from collection only; after that we shouldn't be able
# to find it unless we use the dataset_id.
butler.remove(*args, delete=False)
with self.assertRaises(LookupError):
butler.datasetExists(*args)
# If we use the output ref with the dataset_id, we should
# still be able to load it with getDirect().
self.assertEqual(metric, butler.getDirect(ref))
# Reinsert into collection, then delete from Datastore *and*
# remove from collection.
butler.registry.associate(butler.collection, [ref])
butler.remove(*args)
# Lookup with original args should still fail.
with self.assertRaises(LookupError):
butler.datasetExists(*args)
# Now getDirect() should fail, too.
with self.assertRaises(FileNotFoundError):
butler.getDirect(ref)
# Registry still knows about it, if we use the dataset_id.
self.assertEqual(butler.registry.getDataset(ref.id), ref)
# Put again, then remove completely (this generates a new
# dataset record in registry, with a new ID - the old one
# still exists but it is not in any collection so we don't
# care).
ref = butler.put(metric, *args)
butler.remove(*args, remember=False)
# Lookup with original args should still fail.
with self.assertRaises(LookupError):
butler.datasetExists(*args)
# getDirect() should still fail.
with self.assertRaises(FileNotFoundError):
butler.getDirect(ref)
# Registry shouldn't be able to find it by dataset_id anymore.
self.assertIsNone(butler.registry.getDataset(ref.id))
# Put the dataset again, since the last thing we did was remove it.
ref = butler.put(metric, refIn)
# Get with parameters
stop = 4
sliced = butler.get(ref, parameters={"slice": slice(stop)})
self.assertNotEqual(metric, sliced)
self.assertEqual(metric.summary, sliced.summary)
self.assertEqual(metric.output, sliced.output)
self.assertEqual(metric.data[:stop], sliced.data)
# getDeferred with parameters
sliced = butler.getDeferred(ref, parameters={
"slice": slice(stop)
}).get()
self.assertNotEqual(metric, sliced)
self.assertEqual(metric.summary, sliced.summary)
self.assertEqual(metric.output, sliced.output)
self.assertEqual(metric.data[:stop], sliced.data)
# getDeferred with deferred parameters
sliced = butler.getDeferred(ref).get(parameters={"slice": slice(stop)})
self.assertNotEqual(metric, sliced)
self.assertEqual(metric.summary, sliced.summary)
self.assertEqual(metric.output, sliced.output)
self.assertEqual(metric.data[:stop], sliced.data)
if storageClass.isComposite():
# Delete one component and check that the other components
# can still be retrieved
metricOut = butler.get(ref.datasetType.name, dataId)
compNameS = DatasetType.nameWithComponent(datasetTypeName,
"summary")
compNameD = DatasetType.nameWithComponent(datasetTypeName, "data")
summary = butler.get(compNameS, dataId)
self.assertEqual(summary, metric.summary)
self.assertTrue(butler.datastore.exists(ref.components["summary"]))
butler.remove(compNameS, dataId, remember=True)
with self.assertRaises(LookupError):
butler.datasetExists(compNameS, dataId)
self.assertFalse(butler.datastore.exists(
ref.components["summary"]))
self.assertTrue(butler.datastore.exists(ref.components["data"]))
data = butler.get(compNameD, dataId)
self.assertEqual(data, metric.data)
# Combining a DatasetRef with a dataId should fail
with self.assertRaises(ValueError):
butler.get(ref, dataId)
# Getting with an explicit ref should fail if the id doesn't match
with self.assertRaises(ValueError):
butler.get(DatasetRef(ref.datasetType, ref.dataId, id=101))
# Getting a dataset with unknown parameters should fail
with self.assertRaises(KeyError):
butler.get(ref, parameters={"unsupported": True})
# Check we have a collection
collections = butler.registry.getAllCollections()
self.assertEqual(collections, {
"ingest",
})
# Clean up to check that we can remove something that may have
# already had a component removed
butler.remove(ref.datasetType.name, dataId)
# Add a dataset back in since some downstream tests require
# something to be present
ref = butler.put(metric, refIn)
return butler
# Construct a butler with no run or collection, but make it writeable.
butler = Butler(self.tmpConfigFile, writeable=True)
# Create and register a DatasetType
dimensions = butler.registry.dimensions.extract(
["instrument", "visit"])
datasetType = self.addDatasetType(
"example", dimensions,
self.storageClassFactory.getStorageClass("StructuredData"),
butler.registry)
# Add needed Dimensions
butler.registry.insertDimensionData("instrument",
{"name": "DummyCamComp"})
butler.registry.insertDimensionData("physical_filter", {
"instrument": "DummyCamComp",
"name": "d-r",
"abstract_filter": "R"
})
butler.registry.insertDimensionData(
"visit", {
"instrument": "DummyCamComp",
"id": 423,
"name": "fourtwentythree",
"physical_filter": "d-r"
})
dataId = {"instrument": "DummyCamComp", "visit": 423}
# Create dataset.
metric = makeExampleMetrics()
# Register a new run and put dataset.
run = "deferred"
butler.registry.registerRun(run)
ref = butler.put(metric, datasetType, dataId, run=run)
# Putting with no run should fail with TypeError.
with self.assertRaises(TypeError):
butler.put(metric, datasetType, dataId)
# Dataset should exist.
self.assertTrue(
butler.datasetExists(datasetType, dataId, collection=run))
# We should be able to get the dataset back, but with and without
# a deferred dataset handle.
self.assertEqual(metric, butler.get(datasetType,
dataId,
collection=run))
self.assertEqual(
metric,
butler.getDeferred(datasetType, dataId, collection=run).get())
# Trying to find the dataset without any collection is a TypeError.
with self.assertRaises(TypeError):
butler.datasetExists(datasetType, dataId)
with self.assertRaises(TypeError):
butler.get(datasetType, dataId)
with self.assertRaises(TypeError):
butler.remove(datasetType, dataId)
# Associate the dataset with a different collection.
butler.registry.associate("tagged", [ref])
# Deleting the dataset from the new collection should make it findable
# in the original collection but without a Datastore entry.
butler.remove(datasetType, dataId, collection="tagged")
self.assertFalse(
butler.datasetExists(datasetType, dataId, collection=run))
class TestValidateOutputs(unittest.TestCase, MockCheckMixin):
"""Check that ci_hsc_gen3 outputs are as expected."""
def setUp(self):
self.butler = Butler(os.path.join(getPackageDir("ci_hsc_gen3"),
"DATA"),
instrument="HSC",
skymap="discrete/ci_hsc",
writeable=False,
collections=["HSC/runs/ci_hsc"])
self._num_exposures = len(DATA_IDS)
self._num_exposures_good_templates = 29
self._num_visits = len({data_id["visit"] for data_id in DATA_IDS})
self._num_tracts = 1
self._num_patches = 1
self._num_bands = len(
{data_id["physical_filter"]
for data_id in DATA_IDS})
self._min_sources = 100
def check_pipetasks(self, names, n_metadata, n_log):
"""Check general pipetask outputs (metadata, log, config).
Parameters
----------
names : `list` [`str`]
Task label names.
n_metadata : `int`
Number of expected metadata quanta.
n_log : `int`
Number of expected log quanta
"""
for name in names:
self.check_datasets([f"{name}_config"], 1)
self.check_datasets([f"{name}_metadata"], n_metadata)
self.check_datasets([f"{name}_log"], n_log)
def check_datasets(self,
dataset_types,
n_expected,
additional_checks=[],
**kwargs):
"""Check dataset existence, and run additional checks.
Parameters
----------
dataset_types : `list` [`str`]
List of dataset types to check.
n_expected : `int`
Number of each dataset_type expected in repo.
additional_checks : `list` [`func`], optional
List of additional check functions to run on each dataset.
**kwargs : `dict`, optional
Additional keywords to send to ``additional_checks``.
"""
for dataset_type in dataset_types:
self.skip_mock(dataset_type)
datasets = set(self.butler.registry.queryDatasets(dataset_type))
self.assertEqual(len(datasets),
n_expected,
msg=f"Number of {dataset_type}")
for dataset in datasets:
self.assertTrue(self.butler.datastore.exists(dataset),
msg=f"File exists for {dataset}")
if additional_checks:
data = self.butler.getDirect(dataset)
for additional_check in additional_checks:
additional_check(data, **kwargs)
def check_sources(self,
source_dataset_types,
n_expected,
min_src,
additional_checks=[],
**kwargs):
"""Check that the source catalogs have enough sources and
run additional checks.
Parameters
----------
dataset_types : `list` [`str`]
List of dataset types to check.
n_expected : `int`
Number of each dataset_type expected in repo.
min_src : `int`
Minimum number of sources for each dataset.
additional_checks : `list` [`func`], optional
List of additional check functions to run on each dataset.
**kwargs : `dict`, optional
Additional keywords to send to ``additional_checks``.
"""
for source_dataset_type in source_dataset_types:
self.skip_mock(source_dataset_type)
datasets = set(
self.butler.registry.queryDatasets(source_dataset_type))
self.assertEqual(len(datasets),
n_expected,
msg=f"Number of {source_dataset_type}")
for dataset in datasets:
catalog = self.butler.getDirect(dataset)
self.assertGreater(len(catalog),
min_src,
msg=f"Number of sources in {dataset}")
for additional_check in additional_checks:
additional_check(catalog, **kwargs)
def test_raw(self):
"Test existence of raw exposures." ""
self.check_datasets(["raw"], self._num_exposures)
def test_isr_characterize_calibrate(self):
"""Test existence of isr/calibration related files."""
self.check_pipetasks(["isr", "characterizeImage", "calibrate"],
self._num_exposures, self._num_exposures)
self.check_datasets([
"postISRCCD", "icExp", "icExpBackground", "icSrc", "calexp",
"calexpBackground"
], self._num_exposures)
self.check_datasets(["icSrc_schema", "src_schema"], 1)
self.check_sources(["src"],
self._num_exposures,
self._min_sources,
additional_checks=[
self.check_aperture_corrections,
self.check_psf_stars_and_flags
])
def test_source_tables(self):
"""Test existence of source tables."""
self.check_pipetasks(
["writeRecalibratedSourceTable", "transformSourceTable"],
self._num_exposures, self._num_exposures)
self.check_pipetasks(["consolidateSourceTable"], self._num_visits,
self._num_visits)
self.check_sources(["sourceTable"], self._num_exposures,
self._min_sources)
self.check_sources(["sourceTable_visit"], self._num_visits,
self._min_sources)
def test_visit_summary(self):
"""Test existence of visit summaries."""
self.check_pipetasks(["consolidateVisitSummary"], self._num_visits,
self._num_visits)
self.check_datasets(["visitSummary"], self._num_visits)
def test_isolated_star_association(self):
"""Test existence of isolated star tables."""
self.check_pipetasks(["isolatedStarAssociation"], self._num_tracts,
self._num_tracts)
self.check_datasets(["isolated_star_cat", "isolated_star_sources"],
self._num_tracts)
def test_finalize_characterization(self):
"""Test existence of finalized characterization outputs."""
self.check_pipetasks(["finalizeCharacterization"], self._num_visits,
self._num_visits)
self.check_datasets(
["finalized_psf_ap_corr_catalog", "finalized_src_table"],
self._num_visits)
def test_make_tables(self):
"""Test existence of ccd and visit tables."""
self.check_pipetasks(["makeCcdVisitTable", "makeVisitTable"], 1, 1)
self.check_datasets(["ccdVisitTable", "visitTable"], 1)
def test_make_warp(self):
"""Test existence of warps."""
self.check_pipetasks(["makeWarp"], self._num_visits, self._num_visits)
self.check_datasets(["deepCoadd_directWarp"], self._num_visits)
def test_assemble_coadd(self):
"""Test existence of coadds."""
def check_bright_star_mask(coadd):
mask = coadd.getMaskedImage().getMask()
mask_val = mask.getPlaneBitMask("BRIGHT_OBJECT")
num_bright = (mask.getArray() & mask_val).sum()
self.assertGreater(num_bright,
0,
msg="Some pixels are masked as BRIGHT_OBJECT")
def check_transmission_curves(coadd):
self.assertTrue(coadd.getInfo().getTransmissionCurve() is not None,
msg="TransmissionCurves are attached to coadds")
n_output = self._num_patches * self._num_bands
self.check_pipetasks(["assembleCoadd"], n_output, n_output)
self.check_datasets(["deepCoadd"],
n_output,
additional_checks=[
check_bright_star_mask,
check_transmission_curves
])
def test_healsparse_property_maps(self):
"""Test existence of healsparse property maps."""
self.check_pipetasks(["healSparsePropertyMaps"],
self._num_tracts * self._num_bands,
self._num_tracts * self._num_bands)
self.check_pipetasks(["consolidateHealSparsePropertyMaps"],
self._num_bands, self._num_bands)
self.check_datasets([
"deepCoadd_dcr_ddec_map_weighted_mean",
"deepCoadd_dcr_dra_map_weighted_mean",
"deepCoadd_dcr_e1_map_weighted_mean",
"deepCoadd_dcr_e2_map_weighted_mean",
"deepCoadd_exposure_time_map_sum",
"deepCoadd_psf_e1_map_weighted_mean",
"deepCoadd_psf_e2_map_weighted_mean",
"deepCoadd_psf_maglim_map_weighted_mean",
"deepCoadd_psf_size_map_weighted_mean",
"deepCoadd_sky_background_map_weighted_mean",
"deepCoadd_sky_noise_map_weighted_mean"
], self._num_tracts * self._num_bands)
self.check_datasets([
"deepCoadd_dcr_ddec_consolidated_map_weighted_mean",
"deepCoadd_dcr_dra_consolidated_map_weighted_mean",
"deepCoadd_dcr_e1_consolidated_map_weighted_mean",
"deepCoadd_dcr_e2_consolidated_map_weighted_mean",
"deepCoadd_exposure_time_consolidated_map_sum",
"deepCoadd_psf_e1_consolidated_map_weighted_mean",
"deepCoadd_psf_e2_consolidated_map_weighted_mean",
"deepCoadd_psf_maglim_consolidated_map_weighted_mean",
"deepCoadd_psf_size_consolidated_map_weighted_mean",
"deepCoadd_sky_background_consolidated_map_weighted_mean",
"deepCoadd_sky_noise_consolidated_map_weighted_mean"
], self._num_bands)
def test_coadd_detection(self):
"""Test existence of coadd detection catalogs."""
n_output = self._num_patches * self._num_bands
self.check_pipetasks(["detection", "measure"], n_output, n_output)
self.check_pipetasks(
["mergeDetections", "deblend", "mergeMeasurements"], 1, 1)
self.check_datasets(
["deepCoadd_calexp", "deepCoadd_calexp_background"], n_output)
self.check_datasets(["deepCoadd_calexp"],
self._num_patches * self._num_bands)
self.check_sources(["deepCoadd_det", "deepCoadd_meas"], n_output,
self._min_sources)
self.check_sources(["deepCoadd_deblendedCatalog"], self._num_patches,
self._min_sources)
self.check_datasets(
["deepCoadd_scarletModelData"],
self._num_patches,
)
def check_propagated_flags(catalog, **kwargs):
self.assertTrue(
"calib_psf_candidate" in catalog.schema,
msg="calib_psf_candidate field exists in deepCoadd_meas catalog"
)
self.assertTrue(
"calib_psf_used" in catalog.schema,
msg="calib_psf_used field exists in deepCoadd_meas catalog")
self.assertTrue(
"calib_astrometry_used" in catalog.schema,
msg=
"calib_astrometry_used field exists in deepCoadd_meas catalog")
self.assertTrue(
"calib_photometry_used" in catalog.schema,
msg=
"calib_photometry_used field exists in deepCoadd_meas catalog")
self.assertTrue(
"calib_psf_reserved" in catalog.schema,
msg="calib_psf_reserved field exists in deepCoadd_meas catalog"
)
def check_failed_children(catalog, **kwargs):
children_failed = []
for column in catalog.schema:
if column.field.getName().startswith("merge_footprint"):
for parent in catalog.getChildren(0):
for child in catalog.getChildren(parent.getId()):
if child[column.key] != parent[column.key]:
children_failed.append(child.getId())
self.assertEqual(
len(children_failed),
0,
msg=
f"merge_footprint from parent propagated to children {children_failed}"
)
def check_deepcoadd_stellar_fraction(catalog):
# Check that at least 90% of the stars we used to model the PSF end
# up classified as stars on the coadd. We certainly need much more
# purity than that to build good PSF models, but this should verify
# that flag propagation, aperture correction, and extendendess are
# all running and configured reasonably (but it may not be
# sensitive enough to detect subtle bugs).
# 2020-1-13: There is an issue with the PSF that was
# identified in DM-28294 and will be fixed in DM-12058,
# which affects scarlet i-band models. So we set the
# minStellarFraction based on the deblender and band used.
# TODO: Once DM-12058 is merged this band-aid can be removed.
min_stellar_fraction = 0.9
if "deblend_scarletFlux" in catalog.schema.getNames():
min_stellar_fraction = 0.7
self.check_psf_stars_and_flags(
catalog,
min_stellar_fraction=min_stellar_fraction,
do_check_flags=False)
self.check_sources(["deepCoadd_meas"],
n_output,
self._min_sources,
additional_checks=[
self.check_aperture_corrections,
check_propagated_flags, check_failed_children,
check_deepcoadd_stellar_fraction
])
self.check_sources(["deepCoadd_ref", "deepCoadd_mergeDet"],
self._num_patches, self._min_sources)
self.check_datasets([
"deepCoadd_det_schema", "deepCoadd_meas_schema",
"deepCoadd_mergeDet_schema", "deepCoadd_peak_schema",
"deepCoadd_ref_schema"
], 1)
def test_object_tables(self):
"""Test existence of object tables."""
self.check_pipetasks(["writeObjectTable", "transformObjectTable"],
self._num_patches, self._num_patches)
self.check_pipetasks(["consolidateObjectTable"], self._num_tracts,
self._num_tracts)
self.check_datasets(["deepCoadd_obj", "objectTable"],
self._num_patches)
self.check_datasets(["objectTable_tract"], self._num_tracts)
def test_forced_phot_ccd(self):
"""Test existence of forced photometry tables (sources)."""
self.check_pipetasks(["forcedPhotCcd"], self._num_exposures,
self._num_exposures)
self.check_sources(["forced_src"],
self._num_exposures,
self._min_sources,
additional_checks=[self.check_aperture_corrections])
self.check_datasets(["forced_src_schema"], 1)
def test_forced_phot_coadd(self):
"""Test existence of forced photometry tables (objects)."""
n_output = self._num_patches * self._num_bands
self.check_pipetasks(["forcedPhotCoadd"], n_output, n_output)
self.check_sources(["deepCoadd_forced_src"],
n_output,
self._min_sources,
additional_checks=[self.check_aperture_corrections])
def test_forced_phot_diffim(self):
"""Test existence of forced photometry tables (diffim)."""
self.check_pipetasks([
"forcedPhotDiffim", "forcedPhotCcdOnDiaObjects",
"forcedPhotDiffOnDiaObjects"
], self._num_exposures, self._num_exposures)
self.check_sources(["forced_diff", "forced_diff_diaObject"],
self._num_exposures_good_templates,
self._min_sources)
self.check_datasets(
["forced_diff_schema", "forced_diff_diaObject_schema"], 1)
def test_templates(self):
"""Test existence of templates."""
self.check_pipetasks(["getTemplate"], self._num_exposures,
self._num_exposures)
self.check_pipetasks(["templateGen", "selectGoodSeeingVisits"],
self._num_patches * self._num_bands,
self._num_patches * self._num_bands)
self.check_datasets(["goodSeeingDiff_templateExp"],
self._num_exposures)
self.check_datasets(["goodSeeingDiff_diaSrc_schema"], 1)
def test_image_difference(self):
"""Test existence of image differences."""
self.check_pipetasks(["imageDifference"], self._num_exposures,
self._num_exposures)
self.check_datasets(["goodSeeingDiff_differenceExp"],
self._num_exposures_good_templates)
self.check_datasets(["goodSeeingDiff_diaSrc_schema"], 1)
def test_dia_source_tables(self):
"""Test existence of dia source tables."""
self.check_pipetasks(["consolidateAssocDiaSourceTable"], 1, 1)
self.check_pipetasks(["consolidateDiaSourceTable"], self._num_visits,
self._num_visits)
self.check_sources(["diaSourceTable"], self._num_visits,
self._min_sources // 2)
self.check_sources(["diaSourceTable_tract"], self._num_tracts,
self._min_sources // 2)
def test_forced_source_tables(self):
"""Test existence of forces source tables."""
self.check_pipetasks(
["writeForcedSourceTable", "writeForcedSourceOnDiaObjectTable"],
self._num_exposures, self._num_exposures)
self.check_pipetasks([
"transformForcedSourceTable",
"transformForcedSourceOnDiaObjectTable",
"consolidateForcedSourceTable",
"consolidateForcedSourceOnDiaObjectTable"
], 1, 1)
self.check_sources(["forced_diff_diaObject"],
self._num_exposures_good_templates,
self._min_sources)
# There are fewer forced sources
self.check_sources(["forced_src_diaObject"], self._num_exposures,
self._min_sources // 4)
self.check_datasets(
["forced_diff_diaObject_schema", "forced_src_diaObject_schema"], 1)
def test_skymap(self):
"""Test existence of skymap."""
self.check_datasets(["skyMap"], 1)
def test_skycorr(self):
"""Test existence of skycorr."""
self.check_pipetasks(["skyCorr"], self._num_visits, self._num_visits)
self.check_datasets(["skyCorr"], self._num_exposures)
def check_aperture_corrections(self, catalog, **kwargs):
for alg in ("base_PsfFlux", "base_GaussianFlux"):
self.assertTrue(f"{alg}_apCorr" in catalog.schema,
msg=f"{alg}_apCorr in schema")
self.assertTrue(f"{alg}_apCorrErr" in catalog.schema,
msg=f"{alg}_apCorrErr in schema")
self.assertTrue(f"{alg}_flag_apCorr" in catalog.schema,
msg=f"{alg}_flag_apCorr in schema")
def check_psf_stars_and_flags(self,
catalog,
min_stellar_fraction=0.95,
do_check_flags=True,
**kwargs):
primary = catalog["detect_isPrimary"]
psf_stars_used = catalog["calib_psf_used"] & primary
ext_stars = catalog["base_ClassificationExtendedness_value"] < 0.5
self.assertGreater(
(ext_stars & psf_stars_used).sum(),
min_stellar_fraction * psf_stars_used.sum(),
msg=
f"At least {min_stellar_fraction} of PSF sources are classified as stars"
)
if do_check_flags:
psf_stars_reserved = catalog["calib_psf_reserved"]
psf_stars_candidate = catalog["calib_psf_candidate"]
self.assertGreaterEqual(
psf_stars_candidate.sum(),
psf_stars_used.sum() + psf_stars_reserved.sum(),
msg=
"Number of candidate PSF stars >= sum of used and reserved stars"
)
class JobReporter:
"""A class for extracting metric values from a Gen 3 repository and
repackaging them as Job objects.
Parameters
----------
repository : `str`
Path to a Butler configuration YAML file or a directory containing one.
collection : `str`
Name of the collection to search for metric values.
metrics_package : `str` or `None`
If provided, the namespace by which to filter selected metrics.
spec : `str`
The level of specification to filter metrics by.
dataset_name : `str`
The name of the dataset to report to SQuaSH through the
``ci_dataset`` tag.
"""
def __init__(self, repository, collection, metrics_package, spec,
dataset_name):
# Hard coding verify_metrics as the packager for now.
# It would be easy to pass this in as an argument, if necessary.
self.metrics = MetricSet.load_metrics_package(
package_name_or_path='verify_metrics', subset=metrics_package)
self.butler = Butler(repository)
self.registry = self.butler.registry
self.spec = spec
self.collection = collection
self.dataset_name = dataset_name
def run(self):
"""Collate job information.
Returns
-------
jobs : `dict` [`str`, `lsst.verify.Job`]
A mapping of `~lsst.verify.Job` objects, indexed by a string
representation of their data ID.
"""
jobs = {}
for metric in self.metrics:
dataset = f'metricvalue_{metric.package}_{metric.metric}'
datasetRefs = set(
self.registry.queryDatasets(dataset,
collections=self.collection,
findFirst=True))
for ref in datasetRefs:
# getDirect skips dataset resolution; ref is guaranteed to
# be valid.
m = self.butler.getDirect(ref)
# make the name the same as what SQuaSH Expects
m.metric_name = metric
# queryDatasets guarantees ref.dataId.hasFull()
dataId = ref.dataId.full.byName()
key = make_key(ref)
# For backward-compatibility with Gen 2 SQuaSH uploads
pfilt = dataId.get('physical_filter')
if not pfilt:
# Grab the physical filter associated with the abstract
# filter. In general there may be more than one. Take the
# shortest assuming it is the most generic.
pfilts = [
el.name for el in self.registry.queryDimensionRecords(
'physical_filter', dataId=ref.dataId)
]
pfilt = min(pfilts, key=len)
if key not in jobs.keys():
job_metadata = {
'filter': pfilt,
'butler_generation': 'Gen3',
'ci_dataset': self.dataset_name,
}
job_metadata.update(dataId)
# Get dataset_repo_url from repository somehow?
jobs[key] = Job(meta=job_metadata, metrics=self.metrics)
jobs[key].measurements.insert(m)
return jobs
class TestCoaddOutputs(unittest.TestCase, MockCheckMixin):
"""Check that coadd outputs are as expected.
Many tests here are ported from
https://github.com/lsst/pipe_tasks/blob/
fd7d5e23d3c71e5d440153bc4faae7de9d5918c5/tests/nopytest_test_coadds.py
"""
def setUp(self):
self.butler = Butler(os.path.join(getPackageDir("ci_hsc_gen3"), "DATA"),
instrument="HSC", skymap="discrete/ci_hsc",
writeable=False, collections=["HSC/runs/ci_hsc"])
self.skip_mock()
self._tract = 0
self._patch = 69
self._bands = ['r', 'i']
def test_forced_id_names(self):
"""Test that forced photometry ID fields are named as expected
(DM-8210).
Specifically, coadd forced photometry should have only "id" and
"parent" fields, while CCD forced photometry should have those,
"objectId", and "parentObjectId".
"""
coadd_schema = self.butler.get("deepCoadd_forced_src_schema").schema
self.assertIn("id", coadd_schema)
self.assertIn("parent", coadd_schema)
self.assertNotIn("objectId", coadd_schema)
self.assertNotIn("parentObjectId", coadd_schema)
ccd_schema = self.butler.get("forced_src_schema").schema
self.assertIn("id", ccd_schema)
self.assertIn("parent", ccd_schema)
self.assertIn("objectId", ccd_schema)
self.assertIn("parentObjectId", ccd_schema)
def test_alg_metadata_output(self):
"""Test that the algorithm metadata is persisted correctly
from MeasureMergedCoaddSourcesTask.
"""
for band in self._bands:
cat = self.butler.get(
"deepCoadd_meas",
band=band,
tract=self._tract,
patch=self._patch
)
meta = cat.getMetadata()
for circ_aperture_flux_radius in meta.getArray('BASE_CIRCULARAPERTUREFLUX_RADII'):
self.assertIsInstance(circ_aperture_flux_radius, numbers.Number)
# Each time the run method of a measurement task is executed,
# algorithm metadata is appended to the algorithm metadata object.
# Depending on how many times a measurement task is run,
# a metadata entry may be a single value or multiple values.
for n_offset in meta.getArray('NOISE_OFFSET'):
self.assertIsInstance(n_offset, numbers.Number)
for noise_src in meta.getArray('NOISE_SOURCE'):
self.assertEqual(noise_src, 'measure')
for noise_exp_id in meta.getArray('NOISE_EXPOSURE_ID'):
self.assertIsInstance(noise_exp_id, numbers.Number)
for noise_seed_mul in meta.getArray('NOISE_SEED_MULTIPLIER'):
self.assertIsInstance(noise_seed_mul, numbers.Number)
def test_schema_consistency(self):
"""Test that _schema catalogs are consistent with the data catalogs."""
det_schema = self.butler.get("deepCoadd_det_schema").schema
meas_schema = self.butler.get("deepCoadd_meas_schema").schema
mergeDet_schema = self.butler.get("deepCoadd_mergeDet_schema").schema
ref_schema = self.butler.get("deepCoadd_ref_schema").schema
coadd_forced_schema = self.butler.get("deepCoadd_forced_src_schema").schema
ccd_forced_schema = self.butler.get("forced_src_schema").schema
for band in self._bands:
det = self.butler.get("deepCoadd_det", band=band, tract=self._tract, patch=self._patch)
self.assertEqual(det.schema, det_schema)
mergeDet = self.butler.get("deepCoadd_mergeDet", band=band, tract=self._tract, patch=self._patch)
self.assertEqual(mergeDet.schema, mergeDet_schema)
meas = self.butler.get("deepCoadd_meas", band=band, tract=self._tract, patch=self._patch)
self.assertEqual(meas.schema, meas_schema)
ref = self.butler.get("deepCoadd_ref", band=band, tract=self._tract, patch=self._patch)
self.assertEqual(ref.schema, ref_schema)
coadd_forced_src = self.butler.get(
"deepCoadd_forced_src",
band=band,
tract=self._tract,
patch=self._patch
)
self.assertEqual(coadd_forced_src.schema, coadd_forced_schema)
ccd_forced_src = self.butler.get(
"forced_src",
tract=self._tract,
visit=DATA_IDS[0]["visit"],
detector=DATA_IDS[0]["detector"]
)
self.assertEqual(ccd_forced_src.schema, ccd_forced_schema)
def test_coadd_transmission_curves(self):
"""Test that coadded TransmissionCurves agree with the inputs."""
wavelengths = np.linspace(4000, 7000, 10)
n_object_test = 10
ctx = np.random.RandomState(12345)
for band in self._bands:
n_tested = 0
exp = self.butler.get("deepCoadd_calexp", band=band, tract=self._tract, patch=self._patch)
cat = self.butler.get("objectTable", band=band, tract=self._tract, patch=self._patch)
transmission_curve = exp.getInfo().getTransmissionCurve()
coadd_inputs = exp.getInfo().getCoaddInputs().ccds
wcs = exp.getWcs()
to_check = ctx.choice(len(cat), size=n_object_test, replace=False)
for index in to_check:
coadd_coord = geom.SpherePoint(cat["coord_ra"].values[index]*geom.degrees,
cat["coord_dec"].values[index]*geom.degrees)
summed_throughput = np.zeros(wavelengths.shape, dtype=np.float64)
weight_sum = 0.0
for rec in coadd_inputs.subsetContaining(coadd_coord, includeValidPolygon=True):
det_pos = rec.getWcs().skyToPixel(coadd_coord)
det_trans = rec.getTransmissionCurve()
weight = rec.get("weight")
summed_throughput += det_trans.sampleAt(det_pos, wavelengths)*weight
weight_sum += weight
if weight_sum == 0.0:
continue
summed_throughput /= weight_sum
coadd_pos = wcs.skyToPixel(coadd_coord)
coadd_throughput = transmission_curve.sampleAt(coadd_pos, wavelengths)
np.testing.assert_array_almost_equal(coadd_throughput, summed_throughput)
n_tested += 1
self.assertGreater(n_tested, 5)
def test_mask_planes_exist(self):
"""Test that the input mask planes have been added."""
for data_id in DATA_IDS:
mask = self.butler.get("calexp.mask", data_id)
self.assertIn("CROSSTALK", mask.getMaskPlaneDict())
self.assertIn("NOT_DEBLENDED", mask.getMaskPlaneDict())
# Expected to fail until DM-5174 is fixed.
@unittest.expectedFailure
def test_masks_removed(self):
"""Test that certain mask planes have been removed from the coadds.
This is expected to fail until DM-5174 is fixed.
"""
for band in self._bands:
mask = self.butler.get("deepCoadd_calexp.mask", band=band, tract=self._tract, patch=self._patch)
self.assertNotIn("CROSSTALK", mask.getMaskPlaneDict())
self.assertNotIn("NOT_DEBLENDED", mask.getMaskPlaneDict())
def test_warp_inputs(self):
"""Test that the warps have the correct inputs."""
skymap = self.butler.get("skyMap")
tract_info = skymap[self._tract]
for warp_type in ["directWarp", "psfMatchedWarp"]:
datasets = set(self.butler.registry.queryDatasets(f"deepCoadd_{warp_type}"))
# We only need to test one dataset
dataset = list(datasets)[0]
warp = self.butler.getDirect(dataset)
self.assertEqual(warp.wcs, tract_info.wcs)
coadd_inputs = warp.getInfo().getCoaddInputs()
self.assertEqual(len(coadd_inputs.visits), 1)
visit_record = coadd_inputs.visits[0]
self.assertEqual(visit_record.getWcs(), warp.wcs)
self.assertEqual(visit_record.getBBox(), warp.getBBox())
self.assertGreater(len(coadd_inputs.ccds), 0)
wcs_cat = self.butler.get(
"jointcalSkyWcsCatalog",
visit=visit_record.getId(),
tract=self._tract
)
photocalib_cat = self.butler.get(
"jointcalPhotoCalibCatalog",
visit=visit_record.getId(),
tract=self._tract
)
final_psf_cat = self.butler.get(
"finalized_psf_ap_corr_catalog",
visit=visit_record.getId()
)
# We only need to test one input ccd
det_record = coadd_inputs.ccds[0]
exp_bbox = self.butler.get(
"calexp.bbox",
visit=det_record["visit"],
detector=det_record["ccd"]
)
self.assertEqual(det_record.getWcs(), wcs_cat.find(det_record["ccd"]).getWcs())
self.assertEqual(
det_record.getPhotoCalib(),
photocalib_cat.find(det_record["ccd"]).getPhotoCalib()
)
self.assertEqual(det_record.getBBox(), exp_bbox)
self.assertIsNotNone(det_record.getTransmissionCurve())
center = det_record.getBBox().getCenter()
np.testing.assert_array_almost_equal(
det_record.getPsf().computeKernelImage(center).array,
final_psf_cat.find(det_record["ccd"]).getPsf().computeKernelImage(center).array
)
input_map = det_record.getApCorrMap()
final_map = final_psf_cat.find(det_record["ccd"]).getApCorrMap()
self.assertEqual(len(input_map), len(final_map))
for key in input_map.keys():
self.assertEqual(input_map[key], final_map[key])
self.assertIsNotNone(coadd_inputs.visits.find(det_record["visit"]))
def test_coadd_inputs(self):
"""Test that the coadds have the correct inputs."""
skymap = self.butler.get("skyMap")
tract_info = skymap[self._tract]
for band in self._bands:
wcs = self.butler.get("deepCoadd_calexp.wcs", band=band, tract=self._tract, patch=self._patch)
self.assertEqual(wcs, tract_info.wcs)
coadd_inputs = self.butler.get(
"deepCoadd_calexp.coaddInputs",
band=band,
tract=self._tract,
patch=self._patch
)
# We only need to test one input ccd
det_record = coadd_inputs.ccds[0]
wcs_cat = self.butler.get(
"jointcalSkyWcsCatalog",
visit=det_record["visit"],
tract=self._tract
)
photocalib_cat = self.butler.get(
"jointcalPhotoCalibCatalog",
visit=det_record["visit"],
tract=self._tract
)
final_psf_cat = self.butler.get(
"finalized_psf_ap_corr_catalog",
visit=det_record["visit"]
)
exp_bbox = self.butler.get(
"calexp.bbox",
visit=det_record["visit"],
detector=det_record["ccd"]
)
self.assertEqual(det_record.getWcs(), wcs_cat.find(det_record["ccd"]).getWcs())
self.assertEqual(
det_record.getPhotoCalib(),
photocalib_cat.find(det_record["ccd"]).getPhotoCalib()
)
self.assertEqual(det_record.getBBox(), exp_bbox)
self.assertIsNotNone(det_record.getTransmissionCurve())
center = det_record.getBBox().getCenter()
np.testing.assert_array_almost_equal(
det_record.getPsf().computeKernelImage(center).array,
final_psf_cat.find(det_record["ccd"]).getPsf().computeKernelImage(center).array
)
input_map = det_record.getApCorrMap()
final_map = final_psf_cat.find(det_record["ccd"]).getApCorrMap()
self.assertEqual(len(input_map), len(final_map))
for key in input_map.keys():
self.assertEqual(input_map[key], final_map[key])
self.assertIsNotNone(coadd_inputs.visits.find(det_record["visit"]))
def test_psf_installation(self):
"""Test that the coadd psf is installed."""
for band in self._bands:
wcs = self.butler.get("deepCoadd_calexp.wcs", band=band, tract=self._tract, patch=self._patch)
coadd_inputs = self.butler.get(
"deepCoadd_calexp.coaddInputs",
band=band,
tract=self._tract,
patch=self._patch
)
coadd_psf = self.butler.get(
"deepCoadd_calexp.psf",
band=band,
tract=self._tract,
patch=self._patch
)
new_psf = lsst.meas.algorithms.CoaddPsf(coadd_inputs.ccds, wcs)
self.assertEqual(coadd_psf.getComponentCount(), len(coadd_inputs.ccds))
self.assertEqual(new_psf.getComponentCount(), len(coadd_inputs.ccds))
for n, record in enumerate(coadd_inputs.ccds):
center = record.getBBox().getCenter()
np.testing.assert_array_almost_equal(
coadd_psf.getPsf(n).computeKernelImage(center).array,
record.getPsf().computeKernelImage(center).array
)
np.testing.assert_array_almost_equal(
new_psf.getPsf(n).computeKernelImage(center).array,
record.getPsf().computeKernelImage(center).array
)
self.assertEqual(coadd_psf.getWcs(n), record.getWcs())
self.assertEqual(new_psf.getWcs(n), record.getWcs())
self.assertEqual(coadd_psf.getBBox(n), record.getBBox())
self.assertEqual(new_psf.getBBox(n), record.getBBox())
def test_coadd_psf(self):
"""Test that the stars on the coadd are well represented by
the attached PSF.
"""
n_object_test = 10
n_good_test = 5
ctx = np.random.RandomState(12345)
for band in self._bands:
exp = self.butler.get("deepCoadd_calexp", band=band, tract=self._tract, patch=self._patch)
coadd_psf = exp.getPsf()
cat = self.butler.get("objectTable", band=band, tract=self._tract, patch=self._patch)
star_cat = cat[(cat["i_extendedness"] < 0.5)
& (cat["detect_isPrimary"])
& (cat[f"{band}_psfFlux"] > 0.0)
& (cat[f"{band}_psfFlux"]/cat[f"{band}_psfFluxErr"] > 50.0)
& (cat[f"{band}_psfFlux"]/cat[f"{band}_psfFluxErr"] < 200.0)]
to_check = ctx.choice(len(star_cat), size=n_object_test, replace=False)
n_good = 0
for index in to_check:
position = geom.Point2D(star_cat["x"].values[index], star_cat["y"].values[index])
psf_image = coadd_psf.computeImage(position)
psf_image_bbox = psf_image.getBBox()
star_image = lsst.afw.image.ImageF(
exp.maskedImage.image,
psf_image_bbox
).convertD()
star_image /= star_image.array.sum()
psf_image /= psf_image.array.sum()
residuals = lsst.afw.image.ImageD(star_image, True)
residuals -= psf_image
# This is just a quick check that the coadd psf model works
# reasonably well for the stars. It is not meant as a detailed
# test of the psf modeling capability.
if np.max(np.abs(residuals.array)) < 0.01:
n_good += 1
self.assertGreater(n_good, n_good_test)
class HscIngestTestCase(lsst.utils.tests.TestCase):
def setUp(self):
# Use a temporary working directory
self.root = tempfile.mkdtemp(dir=TESTDIR)
Butler.makeRepo(self.root)
self.butler = Butler(self.root, run="raw")
# Register the instrument and its static metadata
HyperSuprimeCam().register(self.butler.registry)
# Make a default config for test methods to play with
self.config = RawIngestTask.ConfigClass()
self.config.onError = "break"
self.file = os.path.join(testDataDirectory, "hsc", "raw",
"HSCA90402512.fits.gz")
self.dataId = dict(instrument="HSC", exposure=904024, detector=50)
def tearDown(self):
if os.path.exists(self.root):
shutil.rmtree(self.root, ignore_errors=True)
def runIngest(self, files=None):
if files is None:
files = [self.file]
task = RawIngestTask(config=self.config, butler=self.butler)
task.log.setLevel(
task.log.FATAL) # silence logs, since we expect a lot of warnings
task.run(files)
def runIngestTest(self, files=None):
self.runIngest(files)
exposure = self.butler.get("raw", self.dataId)
metadata = self.butler.get("raw.metadata", self.dataId)
image = self.butler.get("raw.image", self.dataId)
self.assertImagesEqual(exposure.image, image)
self.assertEqual(metadata.toDict(), exposure.getMetadata().toDict())
def testSymLink(self):
self.config.transfer = "symlink"
self.runIngestTest()
def testCopy(self):
self.config.transfer = "copy"
self.runIngestTest()
def testHardLink(self):
self.config.transfer = "hardlink"
self.runIngestTest()
def testInPlace(self):
# hardlink into repo root manually
newPath = os.path.join(self.butler.datastore.root,
os.path.basename(self.file))
os.link(self.file, newPath)
self.config.transfer = None
self.runIngestTest([newPath])
def testOnConflictFail(self):
self.config.transfer = "symlink"
self.config.conflict = "fail"
self.runIngest() # this one shd
with self.assertRaises(Exception):
self.runIngest() # this ont
def testOnConflictIgnore(self):
self.config.transfer = "symlink"
self.config.conflict = "ignore"
self.runIngest() # this one should succeed
n1, = self.butler.registry.query("SELECT COUNT(*) FROM Dataset")
self.runIngest() # this ong should silently fail
n2, = self.butler.registry.query("SELECT COUNT(*) FROM Dataset")
self.assertEqual(n1, n2)
def testOnConflictStash(self):
self.config.transfer = "symlink"
self.config.conflict = "ignore"
self.config.stash = "stash"
self.runIngest() # this one should write to 'rawn
self.runIngest() # this one should write to 'stashn
dt = self.butler.registry.getDatasetType("raw.metadata")
ref1 = self.butler.registry.find(self.butler.collection, dt,
self.dataId)
ref2 = self.butler.registry.find("stash", dt, self.dataId)
self.assertNotEqual(ref1.id, ref2.id)
self.assertEqual(
self.butler.get(ref1).toDict(),
self.butler.getDirect(ref2).toDict())
def testOnErrorBreak(self):
self.config.transfer = "symlink"
self.config.onError = "break"
# Failing to ingest this nonexistent file after ingesting the valid one should
# leave the valid one in the registry, despite raising an exception.
with self.assertRaises(Exception):
self.runIngest(files=[self.file, "nonexistent.fits"])
dt = self.butler.registry.getDatasetType("raw.metadata")
self.assertIsNotNone(
self.butler.registry.find(self.butler.collection, dt, self.dataId))
def testOnErrorContinue(self):
self.config.transfer = "symlink"
self.config.onError = "continue"
# Failing to ingest nonexistent files before and after ingesting the
# valid one should leave the valid one in the registry and not raise
# an exception.
self.runIngest(
files=["nonexistent.fits", self.file, "still-not-here.fits"])
dt = self.butler.registry.getDatasetType("raw.metadata")
self.assertIsNotNone(
self.butler.registry.find(self.butler.collection, dt, self.dataId))
def testOnErrorRollback(self):
self.config.transfer = "symlink"
self.config.onError = "rollback"
# Failing to ingest nonexistent files after ingesting the
# valid one should leave the registry empty.
with self.assertRaises(Exception):
self.runIngest(file=[self.file, "nonexistent.fits"])
try:
dt = self.butler.registry.getDatasetType("raw.metadata")
except KeyError:
# If we also rollback registering the DatasetType, that's fine,
# but not required.
pass
else:
self.assertIsNotNone(
self.butler.registry.find(self.butler.collection, dt,
self.dataId))