def testCuts(self):
nSrc = 5
refCat = self.makeRefCatalog()
matches = self.makeMatches(refCat, self.srcCat, nSrc)
sources = self.sourceSelector.run(self.srcCat, matches=matches, exposure=self.exposure).sourceCat
self.assertEqual(len(sources), nSrc)
# Set one of the source flags to be bad
matches[0].second.set(self.sourceSelector.config.badFlags[0], True)
sources = self.sourceSelector.run(self.srcCat, matches=matches, exposure=self.exposure).sourceCat
self.assertEqual(len(sources), nSrc-1)
# Set one of the ref flags to be bad
matches[1].first.set("photometric", False)
sources = self.sourceSelector.run(self.srcCat, matches=matches, exposure=self.exposure).sourceCat
self.assertEqual(len(sources), nSrc-2)
# Set one of the colors to be bad
grMin = self.sourceSelector.config.grMin
rFluxField = getRefFluxField(refCat.schema, "r")
gFluxField = getRefFluxField(refCat.schema, "g")
gFlux = 10**(-0.4 * (grMin - 0.1)) * matches[2].first.get(rFluxField)
matches[2].first.set(gFluxField, gFlux)
sources = self.sourceSelector.run(self.srcCat, matches=matches, exposure=self.exposure).sourceCat
self.assertEqual(len(sources), nSrc-3)
# Set one of the types to be bad
if self.sourceSelector.config.selectStar and not self.sourceSelector.config.selectGalaxy:
matches[3].first.set("resolved", True)
sources = self.sourceSelector.run(self.srcCat, matches=matches, exposure=self.exposure).sourceCat
self.assertEqual(len(sources), nSrc-4)
def testCuts(self):
nSrc = 5
refCat = self.makeRefCatalog()
matches = self.makeMatches(refCat, self.srcCat, nSrc)
sources = self.sourceSelector.run(self.srcCat, matches=matches, exposure=self.exposure).sourceCat
self.assertEqual(len(sources), nSrc)
# Set one of the source flags to be bad
matches[0].second.set(self.sourceSelector.config.badFlags[0], True)
sources = self.sourceSelector.run(self.srcCat, matches=matches, exposure=self.exposure).sourceCat
self.assertEqual(len(sources), nSrc-1)
# Set one of the ref flags to be bad
matches[1].first.set("photometric", False)
sources = self.sourceSelector.run(self.srcCat, matches=matches, exposure=self.exposure).sourceCat
self.assertEqual(len(sources), nSrc-2)
# Set one of the colors to be bad
grMin = self.sourceSelector.config.grMin
rFluxField = getRefFluxField(refCat.schema, "r")
gFluxField = getRefFluxField(refCat.schema, "g")
gFlux = 10**(-0.4 * (grMin - 0.1)) * matches[2].first.get(rFluxField)
matches[2].first.set(gFluxField, gFlux)
sources = self.sourceSelector.run(self.srcCat, matches=matches, exposure=self.exposure).sourceCat
self.assertEqual(len(sources), nSrc-3)
# Set one of the types to be bad
if self.sourceSelector.config.selectStar and not self.sourceSelector.config.selectGalaxy:
matches[3].first.set("resolved", True)
sources = self.sourceSelector.run(self.srcCat, matches=matches, exposure=self.exposure).sourceCat
self.assertEqual(len(sources), nSrc-4)
def testDefaultFilterAndFilterMap(self):
"""Test defaultFilter and filterMap parameters of LoadIndexedReferenceObjectsConfig."""
config = LoadIndexedReferenceObjectsConfig()
config.defaultFilter = "b"
config.filterMap = {"aprime": "a"}
loader = LoadIndexedReferenceObjectsTask(butler=self.testButler, config=config)
for tupl, idList in self.compCats.items():
cent = make_coord(*tupl)
lcat = loader.loadSkyCircle(cent, self.searchRadius)
self.assertEqual(lcat.fluxField, "camFlux")
if len(idList) > 0:
defFluxFieldName = getRefFluxField(lcat.refCat.schema, None)
self.assertTrue(defFluxFieldName in lcat.refCat.schema)
aprimeFluxFieldName = getRefFluxField(lcat.refCat.schema, "aprime")
self.assertTrue(aprimeFluxFieldName in lcat.refCat.schema)
break # just need one test
def testDefaultFilterAndFilterMap(self):
"""Test defaultFilter and filterMap parameters of LoadIndexedReferenceObjectsConfig."""
config = LoadIndexedReferenceObjectsConfig()
config.defaultFilter = "b"
config.filterMap = {"aprime": "a"}
loader = LoadIndexedReferenceObjectsTask(butler=self.testButler, config=config)
for tupl, idList in self.compCats.items():
cent = make_coord(*tupl)
lcat = loader.loadSkyCircle(cent, self.searchRadius)
self.assertEqual(lcat.fluxField, "camFlux")
if len(idList) > 0:
defFluxFieldName = getRefFluxField(lcat.refCat.schema, None)
self.assertTrue(defFluxFieldName in lcat.refCat.schema)
aprimeFluxFieldName = getRefFluxField(lcat.refCat.schema, "aprime")
self.assertTrue(aprimeFluxFieldName in lcat.refCat.schema)
break # just need one test
def loadSkyCircle(self, ctrCoord, radius, filterName, **kwargs):
refCat = self.make_synthetic_refcat(_synthCenter, _synthFlux)
fluxField = getRefFluxField(schema=refCat.schema,
filterName=filterName)
return pipeBase.Struct(refCat=self.make_synthetic_refcat(
_synthCenter, _synthFlux),
fluxField=fluxField)
def loadSkyCircle(self, ctrCoord, radius, filterName=None):
"""!Load reference objects that overlap a circular sky region
@param[in] ctrCoord center of search region (an lsst.afw.geom.Coord)
@param[in] radius radius of search region (an lsst.afw.geom.Angle)
@param[in] filterName name of filter, or None for the default filter;
used for flux values in case we have flux limits (which are not yet implemented)
@return an lsst.pipe.base.Struct containing:
- refCat a catalog of reference objects with the
\link meas_algorithms_loadReferenceObjects_Schema standard schema \endlink
as documented in LoadReferenceObjects, including photometric, resolved and variable;
hasCentroid is False for all objects. None if no ref objects available.
- fluxField = name of flux field for specified filterName. None if refCat is None.
"""
id_list, boundary_mask = self.indexer.get_pixel_ids(ctrCoord, radius)
shards = self.get_shards(id_list)
refCat = self.butler.get(self.ref_dataset_name,
dataId=self.make_data_id('master_schema'),
immediate=True)
fluxField = getRefFluxField(schema=refCat.schema,
filterName=filterName)
for shard, is_on_boundary in zip(shards, boundary_mask):
if shard is None:
continue
if is_on_boundary:
refCat.extend(self._trim_to_circle(shard, ctrCoord, radius))
else:
refCat.extend(shard)
# return reference catalog
return pipeBase.Struct(
refCat=refCat,
fluxField=fluxField,
)
def _determine_flux_fields(self, center, filterList):
"""
Determine the flux field names for a reference catalog.
Will set self._fluxFields, self._referenceFilter.
Parameters
----------
center: `lsst.geom.SpherePoint`
The center around which to load test sources.
filterList: `list`
list of `str` of camera filter names.
"""
# Record self._fluxFilters for checks on subsequent calls
self._fluxFilters = filterList
# Search for a good filter to use to load the reference catalog
# via the refObjLoader task which requires a valid filterName
foundReferenceFilter = False
for filterName in filterList:
refFilterName = self.config.filterMap.get(filterName)
if refFilterName is None:
continue
try:
results = self.refObjLoader.loadSkyCircle(
center, 0.05 * lsst.geom.degrees, refFilterName)
foundReferenceFilter = True
self._referenceFilter = refFilterName
break
except RuntimeError:
# This just means that the filterName wasn't listed
# in the reference catalog. This is okay.
pass
if not foundReferenceFilter:
raise RuntimeError("Could not find any valid flux field(s) %s" %
(", ".join(filterList)))
# Retrieve all the fluxField names
self._fluxFields = []
for filterName in filterList:
fluxField = None
refFilterName = self.config.filterMap.get(filterName)
if refFilterName is not None:
try:
fluxField = getRefFluxField(results.refCat.schema,
filterName=refFilterName)
except RuntimeError:
# This flux field isn't available. Set to None
fluxField = None
if fluxField is None:
self.log.warning(
f'No reference flux field for camera filter {filterName}')
self._fluxFields.append(fluxField)
def loadSkyCircle(self, ctrCoord, radius, filterName=None):
"""!Load reference objects that overlap a circular sky region
@param[in] ctrCoord center of search region (an lsst.afw.geom.Coord)
@param[in] radius radius of search region (an lsst.afw.geom.Angle)
@param[in] filterName name of filter, or None for the default filter;
used for flux values in case we have flux limits (which are not yet implemented)
@return an lsst.pipe.base.Struct containing:
- refCat a catalog of reference objects with the
\link meas_algorithms_loadReferenceObjects_Schema standard schema \endlink
as documented in LoadReferenceObjects, including photometric, resolved and variable;
hasCentroid is False for all objects. None if no ref objects available.
- fluxField = name of flux field for specified filterName. None if refCat is None.
"""
id_list, boundary_mask = self.indexer.get_pixel_ids(ctrCoord, radius)
shards = self.get_shards(id_list)
refCat = self.butler.get(self.ref_dataset_name, dataId=self.make_data_id('master_schema'),
immediate=True)
fluxField = getRefFluxField(schema=refCat.schema, filterName=filterName)
for shard, is_on_boundary in zip(shards, boundary_mask):
if shard is None:
continue
if is_on_boundary:
refCat.extend(self._trim_to_circle(shard, ctrCoord, radius))
else:
refCat.extend(shard)
# return reference catalog
return pipeBase.Struct(
refCat = refCat,
fluxField = fluxField,
)
def testFilterAliasMap(self):
"""Make a schema with filter aliases."""
for filterMap in ({}, {"camr": "r"}):
config = TrivialLoader.ConfigClass()
config.filterMap = filterMap
loader = TrivialLoader(config=config)
refSchema = TrivialLoader.makeMinimalSchema(filterNameList="r")
loader._addFluxAliases(
refSchema,
anyFilterMapsToThis=config.anyFilterMapsToThis,
filterMap=config.filterMap)
self.assertIn("r_flux", refSchema)
self.assertIn("r_fluxErr", refSchema)
# camera filters aliases are named <filter>_camFlux
if "camr" in filterMap:
self.assertEqual(getRefFluxField(refSchema, "camr"),
"camr_camFlux")
else:
with self.assertRaisesRegex(
RuntimeError,
r"Could not find flux field\(s\) camr_camFlux, camr_flux"
):
getRefFluxField(refSchema, "camr")
refCat = afwTable.SimpleCatalog(refSchema)
refObj = refCat.addNew()
refObj["r_flux"] = 1.23
self.assertAlmostEqual(
refCat[0].get(getRefFluxField(refSchema, "r")), 1.23)
if "camr" in filterMap:
self.assertAlmostEqual(
refCat[0].get(getRefFluxField(refSchema, "camr")), 1.23)
refObj["r_fluxErr"] = 0.111
if "camr" in filterMap:
self.assertEqual(refCat[0].get("camr_camFluxErr"), 0.111)
fluxKey, fluxErrKey = getRefFluxKeys(refSchema, "r")
self.assertEqual(refCat[0].get(fluxKey), 1.23)
self.assertEqual(refCat[0].get(fluxErrKey), 0.111)
if "camr" in filterMap:
fluxKey, fluxErrKey = getRefFluxKeys(refSchema, "camr")
self.assertEqual(refCat[0].get(fluxErrKey), 0.111)
else:
with self.assertRaises(RuntimeError):
getRefFluxKeys(refSchema, "camr")
def testFilterAliasMap(self):
"""Make a schema with filter aliases."""
for defaultFilter in ("", "r", "camr"):
for filterMap in ({}, {"camr": "r"}):
config = TrivialLoader.ConfigClass()
config.defaultFilter = defaultFilter
config.filterMap = filterMap
loader = TrivialLoader(config=config)
refSchema = TrivialLoader.makeMinimalSchema(filterNameList="r")
try:
loader._addFluxAliases(refSchema)
self.assertNotEqual(defaultFilter, "camr")
except Exception:
# only reference filters are allowed as default filters
self.assertEqual(defaultFilter, "camr")
continue
self.assertIn("r_flux", refSchema)
self.assertIn("r_fluxErr", refSchema)
# camera filters aliases are named <filter>_camFlux
if "camr" in filterMap:
self.assertEqual(getRefFluxField(refSchema, "camr"),
"camr_camFlux")
else:
with self.assertRaises(RuntimeError):
getRefFluxField(refSchema, "camr")
# if a non-empty default filter is specified then camFlux
# and camFluxErr should be present
hasDefault = bool(defaultFilter)
self.assertEqual("camFlux" in refSchema, hasDefault)
self.assertEqual("camFluxErr" in refSchema, hasDefault)
refCat = afwTable.SimpleCatalog(refSchema)
refObj = refCat.addNew()
refObj["r_flux"] = 1.23
self.assertAlmostEqual(
refCat[0].get(getRefFluxField(refSchema, "r")), 1.23)
if "camr" in filterMap:
self.assertAlmostEqual(
refCat[0].get(getRefFluxField(refSchema, "camr")),
1.23)
if hasDefault:
self.assertEqual(getRefFluxField(refSchema, ""), "camFlux")
self.assertAlmostEqual(
refCat[0].get(getRefFluxField(refSchema, "")), 1.23)
refObj["r_fluxErr"] = 0.111
if "camr" in filterMap:
self.assertEqual(refCat[0].get("camr_camFluxErr"), 0.111)
fluxKey, fluxErrKey = getRefFluxKeys(refSchema, "r")
self.assertEqual(refCat[0].get(fluxKey), 1.23)
self.assertEqual(refCat[0].get(fluxErrKey), 0.111)
if "camr" in filterMap:
fluxKey, fluxErrKey = getRefFluxKeys(refSchema, "camr")
self.assertEqual(refCat[0].get(fluxErrKey), 0.111)
else:
with self.assertRaises(RuntimeError):
getRefFluxKeys(refSchema, "camr")
def testMakeMinimalSchema(self):
"""Make a schema and check it."""
for filterNameList in (["r"], ["foo", "_bar"]):
for (addIsPhotometric, addIsResolved, addIsVariable,
coordErrDim, addProperMotion, properMotionErrDim,
addParallax, addParallaxErr) in itertools.product(
(False, True), (False, True), (False, True),
(-1, 0, 1, 2, 3, 4), (False, True), (-1, 0, 1, 2, 3, 4),
(False, True), (False, True)):
argDict = dict(
filterNameList=filterNameList,
addIsPhotometric=addIsPhotometric,
addIsResolved=addIsResolved,
addIsVariable=addIsVariable,
coordErrDim=coordErrDim,
addProperMotion=addProperMotion,
properMotionErrDim=properMotionErrDim,
addParallax=addParallax,
addParallaxErr=addParallaxErr,
)
if coordErrDim not in (0, 2, 3) or \
(addProperMotion and properMotionErrDim not in (0, 2, 3)):
with self.assertRaises(ValueError):
LoadReferenceObjectsTask.makeMinimalSchema(**argDict)
else:
refSchema = LoadReferenceObjectsTask.makeMinimalSchema(**argDict)
self.assertTrue("coord_ra" in refSchema)
self.assertTrue("coord_dec" in refSchema)
self.assertTrue("centroid_x" in refSchema)
self.assertTrue("centroid_y" in refSchema)
self.assertTrue("hasCentroid" in refSchema)
for filterName in filterNameList:
fluxField = filterName + "_flux"
self.assertIn(fluxField, refSchema)
self.assertNotIn("x" + fluxField, refSchema)
fluxErrField = fluxField + "Err"
self.assertIn(fluxErrField, refSchema)
self.assertEqual(getRefFluxField(refSchema, filterName), filterName + "_flux")
self.assertEqual("resolved" in refSchema, addIsResolved)
self.assertEqual("variable" in refSchema, addIsVariable)
self.assertEqual("photometric" in refSchema, addIsPhotometric)
self.assertEqual("photometric" in refSchema, addIsPhotometric)
self.assertEqual("epoch" in refSchema, addProperMotion or addParallax)
self.assertEqual("coord_raErr" in refSchema, coordErrDim > 0)
self.assertEqual("coord_decErr" in refSchema, coordErrDim > 0)
self.assertEqual("coord_ra_dec_Cov" in refSchema, coordErrDim == 3)
self.assertEqual("pm_ra" in refSchema, addProperMotion)
self.assertEqual("pm_dec" in refSchema, addProperMotion)
self.assertEqual("pm_raErr" in refSchema, addProperMotion and properMotionErrDim > 0)
self.assertEqual("pm_decErr" in refSchema, addProperMotion and properMotionErrDim > 0)
self.assertEqual("pm_flag" in refSchema, addProperMotion)
self.assertEqual("pm_ra_dec_Cov" in refSchema,
addProperMotion and properMotionErrDim == 3)
self.assertEqual("parallax" in refSchema, addParallax)
self.assertEqual("parallaxErr" in refSchema, addParallax and addParallaxErr)
self.assertEqual("parallax_flag" in refSchema, addParallax)
def loadSkyCircle(self, ctrCoord, radius, filterName=None):
"""!Load reference objects that overlap a circular sky region
@param[in] ctrCoord center of search region (an lsst.afw.geom.Coord)
@param[in] radius radius of search region (an lsst.afw.geom.Angle)
@param[in] filterName name of filter, or None for the default filter;
used for flux values in case we have flux limits (which are not yet implemented)
@return an lsst.pipe.base.Struct containing:
- refCat a catalog of reference objects with the
\link meas_algorithms_loadReferenceObjects_Schema standard schema \endlink
as documented in LoadReferenceObjects, including photometric, resolved and variable;
hasCentroid is False for all objects.
- fluxField = name of flux field for specified filterName. None if refCat is None.
"""
id_list, boundary_mask = self.indexer.get_pixel_ids(ctrCoord, radius)
shards = self.get_shards(id_list)
refCat = self.butler.get('ref_cat',
dataId=self.indexer.make_data_id(
'master_schema', self.ref_dataset_name),
immediate=True)
self._addFluxAliases(refCat.schema)
fluxField = getRefFluxField(schema=refCat.schema,
filterName=filterName)
for shard, is_on_boundary in zip(shards, boundary_mask):
if shard is None:
continue
if is_on_boundary:
refCat.extend(self._trim_to_circle(shard, ctrCoord, radius))
else:
refCat.extend(shard)
# make sure catalog is contiguous
if not refCat.isContiguous():
refCat = refCat.copy()
# add and initialize centroid and hasCentroid fields (these are added
# after loading to avoid wasting space in the saved catalogs)
# the new fields are automatically initialized to (nan, nan) and False
# so no need to set them explicitly
mapper = afwTable.SchemaMapper(refCat.schema, True)
mapper.addMinimalSchema(refCat.schema, True)
mapper.editOutputSchema().addField("centroid_x", type=float)
mapper.editOutputSchema().addField("centroid_y", type=float)
mapper.editOutputSchema().addField("hasCentroid", type="Flag")
expandedCat = afwTable.SimpleCatalog(mapper.getOutputSchema())
expandedCat.extend(refCat, mapper=mapper)
del refCat # avoid accidentally returning the unexpanded reference catalog
# return reference catalog
return pipeBase.Struct(
refCat=expandedCat,
fluxField=fluxField,
)
def testAnyFilterMapsToThisAlias(self):
# test anyFilterMapsToThis
config = TrivialLoader.ConfigClass()
config.anyFilterMapsToThis = "gg"
loader = TrivialLoader(config=config)
refSchema = TrivialLoader.makeMinimalSchema(filterNameList=["gg"])
loader._addFluxAliases(refSchema)
self.assertEqual(getRefFluxField(refSchema, "r"), "gg_flux")
# raise if "gg" is not in the refcat filter list
with self.assertRaises(RuntimeError):
refSchema = TrivialLoader.makeMinimalSchema(filterNameList=["rr"])
refSchema = loader._addFluxAliases(refSchema)
def testMakeMinimalSchema(self):
"""Make a schema and check it."""
for filterNameList in (["r"], ["foo", "_bar"]):
for (addIsPhotometric, addIsResolved, addIsVariable,
coordErrDim, addProperMotion, properMotionErrDim,
addParallax) in itertools.product(
(False, True), (False, True), (False, True),
(-1, 0, 1, 2, 3, 4), (False, True), (-1, 0, 1, 2, 3, 4),
(False, True)):
argDict = dict(
filterNameList=filterNameList,
addIsPhotometric=addIsPhotometric,
addIsResolved=addIsResolved,
addIsVariable=addIsVariable,
coordErrDim=coordErrDim,
addProperMotion=addProperMotion,
properMotionErrDim=properMotionErrDim,
addParallax=addParallax,
)
if coordErrDim not in (0, 2, 3) or \
(addProperMotion and properMotionErrDim not in (0, 2, 3)):
with self.assertRaises(ValueError):
LoadReferenceObjectsTask.makeMinimalSchema(**argDict)
else:
refSchema = LoadReferenceObjectsTask.makeMinimalSchema(**argDict)
self.assertTrue("coord_ra" in refSchema)
self.assertTrue("coord_dec" in refSchema)
for filterName in filterNameList:
fluxField = filterName + "_flux"
self.assertIn(fluxField, refSchema)
self.assertNotIn("x" + fluxField, refSchema)
fluxErrField = fluxField + "Err"
self.assertIn(fluxErrField, refSchema)
self.assertEqual(getRefFluxField(refSchema, filterName), filterName + "_flux")
self.assertEqual("resolved" in refSchema, addIsResolved)
self.assertEqual("variable" in refSchema, addIsVariable)
self.assertEqual("photometric" in refSchema, addIsPhotometric)
self.assertEqual("photometric" in refSchema, addIsPhotometric)
self.assertEqual("epoch" in refSchema, addProperMotion or addParallax)
self.assertEqual("coord_raErr" in refSchema, coordErrDim > 0)
self.assertEqual("coord_decErr" in refSchema, coordErrDim > 0)
self.assertEqual("coord_ra_dec_Cov" in refSchema, coordErrDim == 3)
self.assertEqual("pm_ra" in refSchema, addProperMotion)
self.assertEqual("pm_dec" in refSchema, addProperMotion)
self.assertEqual("pm_raErr" in refSchema, addProperMotion and properMotionErrDim > 0)
self.assertEqual("pm_decErr" in refSchema, addProperMotion and properMotionErrDim > 0)
self.assertEqual("pm_flag" in refSchema, addProperMotion)
self.assertEqual("pm_ra_dec_Cov" in refSchema,
addProperMotion and properMotionErrDim == 3)
self.assertEqual("parallax" in refSchema, addParallax)
self.assertEqual("parallaxErr" in refSchema, addParallax)
self.assertEqual("parallax_flag" in refSchema, addParallax)
def testFilterAliasMap(self):
"""Make a schema with filter aliases."""
for defaultFilter in ("", "r", "camr"):
for filterMap in ({}, {"camr": "r"}):
config = TrivialLoader.ConfigClass()
config.defaultFilter = defaultFilter
config.filterMap = filterMap
loader = TrivialLoader(config=config)
refSchema = TrivialLoader.makeMinimalSchema(filterNameList="r")
try:
loader._addFluxAliases(refSchema)
self.assertNotEqual(defaultFilter, "camr")
except Exception:
# only reference filters are allowed as default filters
self.assertEqual(defaultFilter, "camr")
continue
self.assertIn("r_flux", refSchema)
self.assertIn("r_fluxErr", refSchema)
# camera filters aliases are named <filter>_camFlux
if "camr" in filterMap:
self.assertEqual(getRefFluxField(refSchema, "camr"), "camr_camFlux")
else:
with self.assertRaises(RuntimeError):
getRefFluxField(refSchema, "camr")
# if a non-empty default filter is specified then camFlux
# and camFluxErr should be present
hasDefault = bool(defaultFilter)
self.assertEqual("camFlux" in refSchema, hasDefault)
self.assertEqual("camFluxErr" in refSchema, hasDefault)
refCat = afwTable.SimpleCatalog(refSchema)
refObj = refCat.addNew()
refObj["r_flux"] = 1.23
self.assertAlmostEqual(refCat[0].get(getRefFluxField(refSchema, "r")), 1.23)
if "camr" in filterMap:
self.assertAlmostEqual(refCat[0].get(getRefFluxField(refSchema, "camr")), 1.23)
if hasDefault:
self.assertEqual(getRefFluxField(refSchema, ""), "camFlux")
self.assertAlmostEqual(refCat[0].get(getRefFluxField(refSchema, "")), 1.23)
refObj["r_fluxErr"] = 0.111
if "camr" in filterMap:
self.assertEqual(refCat[0].get("camr_camFluxErr"), 0.111)
fluxKey, fluxErrKey = getRefFluxKeys(refSchema, "r")
self.assertEqual(refCat[0].get(fluxKey), 1.23)
self.assertEqual(refCat[0].get(fluxErrKey), 0.111)
if "camr" in filterMap:
fluxKey, fluxErrKey = getRefFluxKeys(refSchema, "camr")
self.assertEqual(refCat[0].get(fluxErrKey), 0.111)
else:
with self.assertRaises(RuntimeError):
getRefFluxKeys(refSchema, "camr")
def loadSkyCircle(self, ctrCoord, radius, filterName=None, epoch=None):
shardIdList, isOnBoundaryList = self.indexer.getShardIds(
ctrCoord, radius)
shards = self.getShards(shardIdList)
refCat = self.butler.get('ref_cat',
dataId=self.indexer.makeDataId(
'master_schema', self.ref_dataset_name),
immediate=True)
self._addFluxAliases(refCat.schema)
fluxField = getRefFluxField(schema=refCat.schema,
filterName=filterName)
for shard, isOnBoundary in zip(shards, isOnBoundaryList):
if shard is None:
continue
if isOnBoundary:
refCat.extend(self._trimToCircle(shard, ctrCoord, radius))
else:
refCat.extend(shard)
if epoch is not None and "pm_ra" in refCat.schema:
# check for a catalog in a non-standard format
if isinstance(refCat.schema["pm_ra"].asKey(),
lsst.afw.table.KeyAngle):
self.applyProperMotions(refCat, epoch)
else:
self.log.warn(
"Catalog pm_ra field is not an Angle; not applying proper motion"
)
# add and initialize centroid and hasCentroid fields (these are
# added after loading to avoid wasting space in the saved catalogs)
# the new fields are automatically initialized to (nan, nan) and
# False so no need to set them explicitly
mapper = afwTable.SchemaMapper(refCat.schema, True)
mapper.addMinimalSchema(refCat.schema, True)
mapper.editOutputSchema().addField("centroid_x", type=float)
mapper.editOutputSchema().addField("centroid_y", type=float)
mapper.editOutputSchema().addField("hasCentroid", type="Flag")
expandedCat = afwTable.SimpleCatalog(mapper.getOutputSchema())
expandedCat.extend(refCat, mapper=mapper)
del refCat # avoid accidentally returning the unexpanded ref cat
# make sure catalog is contiguous
if not expandedCat.isContiguous():
expandedCat = expandedCat.copy(True)
# return reference catalog
return pipeBase.Struct(
refCat=expandedCat,
fluxField=fluxField,
)
def _make_fake_refcat(self):
"""Make a fake reference catalog and the bits necessary to use it."""
center = lsst.geom.SpherePoint(30, -30, lsst.geom.degrees)
flux = 10
radius = 1 * lsst.geom.degrees
filterName = 'fake'
fakeRefCat = make_fake_refcat(center, flux, filterName)
fluxField = getRefFluxField(fakeRefCat.schema, filterName)
returnStruct = lsst.pipe.base.Struct(refCat=fakeRefCat, fluxField=fluxField)
refObjLoader = mock.Mock(spec=LoadIndexedReferenceObjectsTask)
refObjLoader.loadSkyCircle.return_value = returnStruct
return refObjLoader, center, radius, filterName, fakeRefCat
def _make_fake_refcat(self):
"""Make a fake reference catalog and the bits necessary to use it."""
center = lsst.geom.SpherePoint(30, -30, lsst.geom.degrees)
flux = 10
radius = 1 * lsst.geom.degrees
filterName = 'fake'
fakeRefCat = make_fake_refcat(center, flux, filterName)
fluxField = getRefFluxField(fakeRefCat.schema, filterName)
returnStruct = lsst.pipe.base.Struct(refCat=fakeRefCat,
fluxField=fluxField)
refObjLoader = mock.Mock(spec=LoadIndexedReferenceObjectsTask)
refObjLoader.loadSkyCircle.return_value = returnStruct
return refObjLoader, center, radius, filterName, fakeRefCat
def testMakeMinimalSchema(self):
"""Make a schema and check it."""
for filterNameList in (["r"], ["foo", "_bar"]):
for addFluxSigma, addIsPhotometric, addIsResolved, addIsVariable in itertools.product(
(False, True), (False, True), (False, True), (False, True)):
refSchema = LoadReferenceObjectsTask.makeMinimalSchema(
filterNameList=filterNameList,
addFluxSigma=addFluxSigma,
addIsPhotometric=addIsPhotometric,
addIsResolved=addIsResolved,
addIsVariable=addIsVariable,
)
self.assertEqual("resolved" in refSchema, addIsResolved)
self.assertEqual("variable" in refSchema, addIsVariable)
self.assertEqual("photometric" in refSchema, addIsPhotometric)
for filterName in filterNameList:
fluxField = filterName + "_flux"
self.assertIn(fluxField, refSchema)
self.assertNotIn("x" + fluxField, refSchema)
fluxSigmaField = fluxField + "Sigma"
self.assertEqual(fluxSigmaField in refSchema, addFluxSigma)
self.assertEqual(getRefFluxField(refSchema, filterName),
filterName + "_flux")
def testMakeMinimalSchema(self):
"""Make a schema and check it
"""
for filterNameList in (["r"], ["foo", "_bar"]):
for addFluxSigma, addIsPhotometric, addIsResolved, addIsVariable in itertools.product(
(False, True), (False, True), (False, True), (False, True)):
refSchema = LoadReferenceObjectsTask.makeMinimalSchema(
filterNameList = filterNameList,
addFluxSigma = addFluxSigma,
addIsPhotometric = addIsPhotometric,
addIsResolved = addIsResolved,
addIsVariable = addIsVariable,
)
self.assertEqual("resolved" in refSchema, addIsResolved)
self.assertEqual("variable" in refSchema, addIsVariable)
self.assertEqual("photometric" in refSchema, addIsPhotometric)
for filterName in filterNameList:
fluxField = filterName + "_flux"
self.assertIn(fluxField, refSchema)
self.assertNotIn("x" + fluxField, refSchema)
fluxSigmaField = fluxField + "Sigma"
self.assertEqual(fluxSigmaField in refSchema, addFluxSigma)
self.assertEqual(getRefFluxField(refSchema, filterName), filterName + "_flux")
def selectSources(self, sourceCat, matches=None, exposure=None):
"""Return a selection of sources for Kernel candidates.
Parameters
----------
sourceCat : `lsst.afw.table.SourceCatalog`
Catalog of sources to select from.
This catalog must be contiguous in memory.
matches : `list` of `lsst.afw.table.ReferenceMatch`
A match vector as produced by meas_astrom.
exposure : `lsst.afw.image.Exposure` or None
The exposure the catalog was built from; used for debug display.
Returns
-------
struct : `lsst.pipe.base.Struct`
The struct contains the following data:
- selected : `array` of `bool``
Boolean array of sources that were selected, same length as
sourceCat.
"""
import lsstDebug
display = lsstDebug.Info(__name__).display
displayExposure = lsstDebug.Info(__name__).displayExposure
pauseAtEnd = lsstDebug.Info(__name__).pauseAtEnd
if matches is None:
raise RuntimeError("DiaCatalogSourceSelector requires matches")
mi = exposure.getMaskedImage()
if display and displayExposure:
disp = afwDisplay.Display(frame=lsstDebug.frame)
disp.mtv(mi, title="Kernel candidates")
#
# Look for flags in each Source
#
isGoodSource = CheckSource(sourceCat, self.config.fluxLim, self.config.fluxMax, self.config.badFlags)
# Go through and find all the acceptable candidates in the catalogue
selected = np.zeros(len(sourceCat), dtype=bool)
if display and displayExposure:
symbs = []
ctypes = []
doColorCut = True
refSchema = matches[0][0].schema
rRefFluxField = measAlg.getRefFluxField(refSchema, "r")
gRefFluxField = measAlg.getRefFluxField(refSchema, "g")
for i, (ref, source, d) in enumerate(matches):
if not isGoodSource(source):
if display and displayExposure:
symbs.append("+")
ctypes.append(afwDisplay.RED)
else:
isStar = not ref.get("resolved")
isVar = not ref.get("photometric")
gMag = None
rMag = None
if doColorCut:
try:
gMag = -2.5 * np.log10(ref.get(gRefFluxField))
rMag = -2.5 * np.log10(ref.get(rRefFluxField))
except KeyError:
self.log.warn("Cannot cut on color info; fields 'g' and 'r' do not exist")
doColorCut = False
isRightColor = True
else:
isRightColor = (gMag-rMag) >= self.config.grMin and (gMag-rMag) <= self.config.grMax
isRightType = (self.config.selectStar and isStar) or (self.config.selectGalaxy and not isStar)
isRightVar = (self.config.includeVariable) or (self.config.includeVariable is isVar)
if isRightType and isRightVar and isRightColor:
selected[i] = True
if display and displayExposure:
symbs.append("+")
ctypes.append(afwDisplay.GREEN)
elif display and displayExposure:
symbs.append("o")
ctypes.append(afwDisplay.BLUE)
if display and displayExposure:
disp = afwDisplay.Display(frame=lsstDebug.frame)
with disp.Buffering():
for (ref, source, d), symb, ctype in zip(matches, symbs, ctypes):
disp.dot(symb, source.getX() - mi.getX0(), source.getY() - mi.getY0(),
size=4, ctype=ctype)
if display:
lsstDebug.frame += 1
if pauseAtEnd:
input("Continue? y[es] p[db] ")
return Struct(selected=selected)
def selectStars(self, exposure, sourceCat, matches=None):
"""Select sources for Kernel candidates
@param[in] exposure the exposure containing the sources
@param[in] sourceCat catalog of sources that may be stars (an lsst.afw.table.SourceCatalog)
@param[in] matches a match vector as produced by meas_astrom; required
(defaults to None to match the StarSelector API and improve error handling)
@return an lsst.pipe.base.Struct containing:
- starCat a list of sources to be used as kernel candidates
"""
import lsstDebug
display = lsstDebug.Info(__name__).display
displayExposure = lsstDebug.Info(__name__).displayExposure
pauseAtEnd = lsstDebug.Info(__name__).pauseAtEnd
if matches is None:
raise RuntimeError("DiaCatalogSourceSelector requires matches")
mi = exposure.getMaskedImage()
if display:
if displayExposure:
ds9.mtv(mi, title="Kernel candidates", frame=lsstDebug.frame)
#
# Look for flags in each Source
#
isGoodSource = CheckSource(sourceCat, self.config.fluxLim, self.config.fluxMax, self.config.badFlags)
#
# Go through and find all the acceptable candidates in the catalogue
#
starCat = SourceCatalog(sourceCat.schema)
if display and displayExposure:
symbs = []
ctypes = []
doColorCut = True
refSchema = matches[0][0].schema
rRefFluxField = measAlg.getRefFluxField(refSchema, "r")
gRefFluxField = measAlg.getRefFluxField(refSchema, "g")
for ref, source, d in matches:
if not isGoodSource(source):
if display and displayExposure:
symbs.append("+")
ctypes.append(ds9.RED)
else:
isStar = not ref.get("resolved")
isVar = not ref.get("photometric")
gMag = None
rMag = None
if doColorCut:
try:
gMag = -2.5 * np.log10(ref.get(gRefFluxField))
rMag = -2.5 * np.log10(ref.get(rRefFluxField))
except KeyError:
self.log.warn("Cannot cut on color info; fields 'g' and 'r' do not exist")
doColorCut = False
isRightColor = True
else:
isRightColor = (gMag-rMag) >= self.config.grMin and (gMag-rMag) <= self.config.grMax
isRightType = (self.config.selectStar and isStar) or (self.config.selectGalaxy and not isStar)
isRightVar = (self.config.includeVariable) or (self.config.includeVariable is isVar)
if isRightType and isRightVar and isRightColor:
starCat.append(source)
if display and displayExposure:
symbs.append("+")
ctypes.append(ds9.GREEN)
elif display and displayExposure:
symbs.append("o")
ctypes.append(ds9.BLUE)
if display and displayExposure:
with ds9.Buffering():
for (ref, source, d), symb, ctype in zip(matches, symbs, ctypes):
if display and displayExposure:
ds9.dot(symb, source.getX() - mi.getX0(), source.getY() - mi.getY0(),
size=4, ctype=ctype, frame=lsstDebug.frame)
if display:
lsstDebug.frame += 1
if pauseAtEnd:
input("Continue? y[es] p[db] ")
return Struct(
starCat=starCat,
)
def loadSkyCircle(self,
ctrCoord,
radius,
filterName=None,
epoch=None,
centroids=True):
"""!Load reference objects that overlap a circular sky region
@param[in] ctrCoord center of search region (an afwGeom.Coord)
@param[in] radius radius of search region (an geom.Angle)
@param[in] filterName name of filter, or None for the default filter;
used for flux values in case we have flux limits (which are not yet implemented)
@param[in] epoch Epoch for proper motion and parallax correction
(an astropy.time.Time), or None
centroids : `bool` (optional)
Ignored: a.net refcats always have centroid fields.
No proper motion correction is made, since our astrometry.net catalogs
typically don't support that, and even if they do they format is uncertain.
Users interested in proper motion corrections should use the
lsst.meas.algorithms.LoadIndexedReferenceObjectsTask or they will need to
subclass and define how the proper motion correction is to be done.
@return an lsst.pipe.base.Struct containing:
- refCat a catalog of reference objects with the
\link meas_algorithms_loadReferenceObjects_Schema standard schema \endlink
as documented in LoadReferenceObjects, including photometric, resolved and variable;
hasCentroid is False for all objects.
- fluxField = name of flux field for specified filterName
"""
self._readIndexFiles()
names = []
mcols = []
ecols = []
for col, mcol in self.andConfig.magColumnMap.items():
names.append(col)
mcols.append(mcol)
ecols.append(self.andConfig.magErrorColumnMap.get(col, ''))
margs = (names, mcols, ecols)
solver = self._getSolver()
# Find multi-index files within range
multiInds = self._getMIndexesWithinRange(ctrCoord, radius)
# compute solver.getCatalog arguments that follow the list of star kd-trees:
# - center equatorial angle (e.g. RA) in deg
# - center polar angle (e.g. Dec) in deg
# - radius, in deg
# - idColumn
# - (margs)
# - star-galaxy column
# - variability column
fixedArgTuple = (
ctrCoord,
radius,
self.andConfig.idColumn,
) + margs + (
self.andConfig.starGalaxyColumn,
self.andConfig.variableColumn,
True, # eliminate duplicate IDs
)
self.log.debug("search for objects at %s with radius %s deg", ctrCoord,
radius.asDegrees())
with LoadMultiIndexes(multiInds):
# We just want to pass the star kd-trees, so just pass the
# first element of each multi-index.
inds = tuple(mi[0] for mi in multiInds)
refCat = solver.getCatalog(inds, *fixedArgTuple)
self._addFluxAliases(schema=refCat.schema)
fluxField = getRefFluxField(schema=refCat.schema,
filterName=filterName)
# NOTE: sourceSelectors require contiguous catalogs, so ensure
# contiguity now, so views are preserved from here on.
if not refCat.isContiguous():
refCat = refCat.copy(deep=True)
# Update flux fields to be nJy. a.net catalogs do not have a conversion script.
self.log.warn(
"Loading A.net reference catalog with old style units in schema.")
self.log.warn(
"A.net reference catalogs will not be supported in the future.")
self.log.warn("See RFC-562 and RFC-575 for more details.")
refCat = convertToNanojansky(refCat, self.log)
self.log.debug("found %d objects", len(refCat))
return pipeBase.Struct(
refCat=refCat,
fluxField=fluxField,
)
def extractMagArrays(self, matches, filterName, sourceKeys):
"""!Extract magnitude and magnitude error arrays from the given matches.
@param[in] matches Reference/source matches, a @link lsst::afw::table::ReferenceMatchVector@endlink
@param[in] filterName Name of filter being calibrated
@param[in] sourceKeys Struct of source catalog keys, as returned by getSourceKeys()
@return Struct containing srcMag, refMag, srcMagErr, refMagErr, and magErr numpy arrays
where magErr is an error in the magnitude; the error in srcMag - refMag
If nonzero, config.magErrFloor will be added to magErr *only* (not srcMagErr or refMagErr), as
magErr is what is later used to determine the zero point.
Struct also contains refFluxFieldList: a list of field names of the reference catalog used for fluxes
(1 or 2 strings)
@note These magnitude arrays are the @em inputs to the photometric calibration, some may have been
discarded by clipping while estimating the calibration (https://jira.lsstcorp.org/browse/DM-813)
"""
srcFluxArr = np.array([m.second.get(sourceKeys.flux) for m in matches])
srcFluxErrArr = np.array(
[m.second.get(sourceKeys.fluxErr) for m in matches])
if not np.all(np.isfinite(srcFluxErrArr)):
# this is an unpleasant hack; see DM-2308 requesting a better solution
self.log.warn(
"Source catalog does not have flux uncertainties; using sqrt(flux)."
)
srcFluxErrArr = np.sqrt(srcFluxArr)
# convert source flux from DN to an estimate of Jy
JanskysPerABFlux = 3631.0
srcFluxArr = srcFluxArr * JanskysPerABFlux
srcFluxErrArr = srcFluxErrArr * JanskysPerABFlux
if not matches:
raise RuntimeError("No reference stars are available")
refSchema = matches[0].first.schema
applyColorTerms = self.config.applyColorTerms
applyCTReason = "config.applyColorTerms is %s" % (
self.config.applyColorTerms, )
if self.config.applyColorTerms is None:
# apply color terms if color term data is available and photoCatName specified
ctDataAvail = len(self.config.colorterms.data) > 0
photoCatSpecified = self.config.photoCatName is not None
applyCTReason += " and data %s available" % ("is" if ctDataAvail
else "is not")
applyCTReason += " and photoRefCat %s provided" % (
"is" if photoCatSpecified else "is not")
applyColorTerms = ctDataAvail and photoCatSpecified
if applyColorTerms:
self.log.info(
"Applying color terms for filterName=%r, config.photoCatName=%s because %s",
filterName, self.config.photoCatName, applyCTReason)
ct = self.config.colorterms.getColorterm(
filterName=filterName,
photoCatName=self.config.photoCatName,
doRaise=True)
else:
self.log.info("Not applying color terms because %s", applyCTReason)
ct = None
if ct: # we have a color term to worry about
fluxFieldList = [
getRefFluxField(refSchema, filt)
for filt in (ct.primary, ct.secondary)
]
missingFluxFieldList = []
for fluxField in fluxFieldList:
try:
refSchema.find(fluxField).key
except KeyError:
missingFluxFieldList.append(fluxField)
if missingFluxFieldList:
self.log.warn(
"Source catalog does not have fluxes for %s; ignoring color terms",
" ".join(missingFluxFieldList))
ct = None
if not ct:
fluxFieldList = [getRefFluxField(refSchema, filterName)]
refFluxArrList = [] # list of ref arrays, one per flux field
refFluxErrArrList = [] # list of ref flux arrays, one per flux field
for fluxField in fluxFieldList:
fluxKey = refSchema.find(fluxField).key
refFluxArr = np.array([m.first.get(fluxKey) for m in matches])
try:
fluxErrKey = refSchema.find(fluxField + "Sigma").key
refFluxErrArr = np.array(
[m.first.get(fluxErrKey) for m in matches])
except KeyError:
# Reference catalogue may not have flux uncertainties; HACK
self.log.warn(
"Reference catalog does not have flux uncertainties for %s; using sqrt(flux).",
fluxField)
refFluxErrArr = np.sqrt(refFluxArr)
refFluxArrList.append(refFluxArr)
refFluxErrArrList.append(refFluxErrArr)
if ct: # we have a color term to worry about
refMagArr1 = np.array(
[abMagFromFlux(rf1) for rf1 in refFluxArrList[0]]) # primary
refMagArr2 = np.array(
[abMagFromFlux(rf2) for rf2 in refFluxArrList[1]]) # secondary
refMagArr = ct.transformMags(refMagArr1, refMagArr2)
refFluxErrArr = ct.propagateFluxErrors(refFluxErrArrList[0],
refFluxErrArrList[1])
else:
refMagArr = np.array(
[abMagFromFlux(rf) for rf in refFluxArrList[0]])
srcMagArr = np.array([abMagFromFlux(sf) for sf in srcFluxArr])
# Fitting with error bars in both axes is hard
# for now ignore reference flux error, but ticket DM-2308 is a request for a better solution
magErrArr = np.array([
abMagErrFromFluxErr(fe, sf)
for fe, sf in zip(srcFluxErrArr, srcFluxArr)
])
if self.config.magErrFloor != 0.0:
magErrArr = (magErrArr**2 + self.config.magErrFloor**2)**0.5
srcMagErrArr = np.array([
abMagErrFromFluxErr(sfe, sf)
for sfe, sf in zip(srcFluxErrArr, srcFluxArr)
])
refMagErrArr = np.array([
abMagErrFromFluxErr(rfe, rf)
for rfe, rf in zip(refFluxErrArr, refFluxArr)
])
good = np.isfinite(srcMagArr) & np.isfinite(refMagArr)
return pipeBase.Struct(
srcMag=srcMagArr[good],
refMag=refMagArr[good],
magErr=magErrArr[good],
srcMagErr=srcMagErrArr[good],
refMagErr=refMagErrArr[good],
refFluxFieldList=fluxFieldList,
)
def loadSkyCircle(self, ctrCoord, radius, filterName=None, epoch=None):
shardIdList, isOnBoundaryList = self.indexer.getShardIds(ctrCoord, radius)
shards = self.getShards(shardIdList)
refCat = self.butler.get('ref_cat',
dataId=self.indexer.makeDataId('master_schema', self.ref_dataset_name),
immediate=True)
# load the catalog, one shard at a time
for shard, isOnBoundary in zip(shards, isOnBoundaryList):
if shard is None:
continue
if isOnBoundary:
refCat.extend(self._trimToCircle(shard, ctrCoord, radius))
else:
refCat.extend(shard)
# apply proper motion corrections
if epoch is not None and "pm_ra" in refCat.schema:
# check for a catalog in a non-standard format
if isinstance(refCat.schema["pm_ra"].asKey(), lsst.afw.table.KeyAngle):
self.applyProperMotions(refCat, epoch)
else:
self.log.warn("Catalog pm_ra field is not an Angle; not applying proper motion")
# update version=0 style refcats to have nJy fluxes
if self.dataset_config.format_version == 0 or not hasNanojanskyFluxUnits(refCat.schema):
self.log.warn("Found version 0 reference catalog with old style units in schema.")
self.log.warn("run `meas_algorithms/bin/convert_refcat_to_nJy.py` to convert fluxes to nJy.")
self.log.warn("See RFC-575 for more details.")
refCat = convertToNanojansky(refCat, self.log)
else:
# For version >= 1, the version should be in the catalog header,
# too, and should be consistent with the version in the config.
catVersion = getFormatVersionFromRefCat(refCat)
if catVersion != self.dataset_config.format_version:
raise RuntimeError(f"Format version in reference catalog ({catVersion}) does not match"
f" format_version field in config ({self.dataset_config.format_version})")
self._addFluxAliases(refCat.schema)
fluxField = getRefFluxField(schema=refCat.schema, filterName=filterName)
# add and initialize centroid and hasCentroid fields (these are
# added after loading to avoid wasting space in the saved catalogs)
# the new fields are automatically initialized to (nan, nan) and
# False so no need to set them explicitly
mapper = afwTable.SchemaMapper(refCat.schema, True)
mapper.addMinimalSchema(refCat.schema, True)
mapper.editOutputSchema().addField("centroid_x", type=float)
mapper.editOutputSchema().addField("centroid_y", type=float)
mapper.editOutputSchema().addField("hasCentroid", type="Flag")
expandedCat = afwTable.SimpleCatalog(mapper.getOutputSchema())
expandedCat.extend(refCat, mapper=mapper)
del refCat # avoid accidentally returning the unexpanded ref cat
# make sure catalog is contiguous
if not expandedCat.isContiguous():
expandedCat = expandedCat.copy(True)
# return reference catalog
return pipeBase.Struct(
refCat=expandedCat,
fluxField=fluxField,
)
def selectMatches(self, matches, sourceKeys, filterName, frame=None):
"""!Select reference/source matches according the criteria specified in the config.
\param[in] matches ReferenceMatchVector (not modified)
\param[in] sourceKeys Struct of source catalog keys, as returned by getSourceKeys()
\param[in] filterName name of camera filter; used to obtain the reference flux field
\param[in] frame ds9 frame number to use for debugging display
if frame is non-None, display information about trimmed objects on that ds9 frame:
- Bad: red x
- Unsuitable objects: blue + (and a cyan o if a galaxy)
- Failed flux cut: magenta *
\return a \link lsst.afw.table.ReferenceMatchVector\endlink that contains only the selected matches.
If a schema was passed during task construction, a flag field will be set on sources
in the selected matches.
\throws ValueError There are no valid matches.
"""
self.log.debug("Number of input matches: %d", len(matches))
if self.config.doSelectUnresolved:
# Select only resolved sources if asked to do so.
matches = [
m for m in matches
if self.isUnresolved(m.second, sourceKeys.starGal)
]
self.log.debug(
"Number of matches after culling resolved sources: %d",
len(matches))
if len(matches) == 0:
raise ValueError("No input matches")
for m in matches:
if self.candidateKey is not None:
m.second.set(self.candidateKey, True)
# Only use stars for which the flags indicate the photometry is good.
afterFlagCutInd = [
i for i, m in enumerate(matches)
if checkSourceFlags(m.second, sourceKeys)
]
afterFlagCut = [matches[i] for i in afterFlagCutInd]
self.log.debug("Number of matches after source flag cuts: %d",
len(afterFlagCut))
if len(afterFlagCut) != len(matches):
if frame is not None:
with ds9.Buffering():
for i, m in enumerate(matches):
if i not in afterFlagCutInd:
x, y = m.second.getCentroid()
ds9.dot("x",
x,
y,
size=4,
frame=frame,
ctype=ds9.RED)
matches = afterFlagCut
if len(matches) == 0:
raise ValueError("All matches eliminated by source flags")
refSchema = matches[0].first.schema
try:
photometricKey = refSchema.find("photometric").key
try:
resolvedKey = refSchema.find("resolved").key
except:
resolvedKey = None
try:
variableKey = refSchema.find("variable").key
except:
variableKey = None
except:
self.log.warn(
"No 'photometric' flag key found in reference schema.")
photometricKey = None
if photometricKey is not None:
afterRefCutInd = [
i for i, m in enumerate(matches) if m.first.get(photometricKey)
]
afterRefCut = [matches[i] for i in afterRefCutInd]
if len(afterRefCut) != len(matches):
if frame is not None:
with ds9.Buffering():
for i, m in enumerate(matches):
if i not in afterRefCutInd:
x, y = m.second.getCentroid()
ds9.dot("+",
x,
y,
size=4,
frame=frame,
ctype=ds9.BLUE)
if resolvedKey and m.first.get(resolvedKey):
ds9.dot("o",
x,
y,
size=6,
frame=frame,
ctype=ds9.CYAN)
if variableKey and m.first.get(variableKey):
ds9.dot("o",
x,
y,
size=6,
frame=frame,
ctype=ds9.MAGENTA)
matches = afterRefCut
self.log.debug("Number of matches after reference catalog cuts: %d",
len(matches))
if len(matches) == 0:
raise RuntimeError(
"No sources remain in match list after reference catalog cuts."
)
fluxName = getRefFluxField(refSchema, filterName)
fluxKey = refSchema.find(fluxName).key
if self.config.magLimit is not None:
fluxLimit = fluxFromABMag(self.config.magLimit)
afterMagCutInd = [
i for i, m in enumerate(matches)
if (m.first.get(fluxKey) > fluxLimit
and m.second.getPsfFlux() > 0.0)
]
else:
afterMagCutInd = [
i for i, m in enumerate(matches) if m.second.getPsfFlux() > 0.0
]
afterMagCut = [matches[i] for i in afterMagCutInd]
if len(afterMagCut) != len(matches):
if frame is not None:
with ds9.Buffering():
for i, m in enumerate(matches):
if i not in afterMagCutInd:
x, y = m.second.getCentroid()
ds9.dot("*",
x,
y,
size=4,
frame=frame,
ctype=ds9.MAGENTA)
matches = afterMagCut
self.log.debug("Number of matches after magnitude limit cuts: %d",
len(matches))
if len(matches) == 0:
raise RuntimeError(
"No sources remaining in match list after magnitude limit cuts."
)
if frame is not None:
with ds9.Buffering():
for m in matches:
x, y = m.second.getCentroid()
ds9.dot("o", x, y, size=4, frame=frame, ctype=ds9.GREEN)
result = []
for m in matches:
if self.usedKey is not None:
m.second.set(self.usedKey, True)
result.append(m)
return result
def run(self, dataRefList, ct=None, debug=False, verbose=False):
ccdSet = self.readCcd(dataRefList)
self.removeNonExistCcd(dataRefList, ccdSet)
sourceSet = []
matchList = []
astrom = measAstrom.ANetBasicAstrometryTask(self.config.astrom)
ssVisit = dict()
mlVisit = dict()
dataRefListUsed = list()
wcsDic = dict()
calibDic = dict()
ffpDic = dict()
for dataRef in dataRefList:
if dataRef.dataId['visit'] not in ssVisit:
ssVisit[dataRef.dataId['visit']] = list()
mlVisit[dataRef.dataId['visit']] = list()
wcsDic[dataRef.dataId['visit']] = dict()
calibDic[dataRef.dataId['visit']] = dict()
ffpDic[dataRef.dataId['visit']] = dict()
try:
if not dataRef.datasetExists('src'):
raise RuntimeError("no data for src %s" % (dataRef.dataId))
if not dataRef.datasetExists('jointcal_wcs'):
raise RuntimeError("no data for wcs %s" % (dataRef.dataId))
if not dataRef.datasetExists('fcr'):
raise RuntimeError("no data for fcr %s" % (dataRef.dataId))
wcs = dataRef.get('jointcal_wcs')
md = dataRef.get('calexp_md')
filterName = afwImage.Filter(md).getName()
md = dataRef.get('fcr_md')
ffp = measMosaic.FluxFitParams(md)
photoCalib = afwImage.makePhotoCalibFromMetadata(md)
sources = dataRef.get('src',
flags=afwTable.SOURCE_IO_NO_FOOTPRINTS,
immediate=True)
icSrces = dataRef.get('icSrc',
flags=afwTable.SOURCE_IO_NO_FOOTPRINTS,
immediate=True)
packedMatches = dataRef.get('icMatch')
matches = astrom.joinMatchListWithCatalog(packedMatches, icSrces)
matches = [m for m in matches if m[0] is not None]
if matches:
refSchema = matches[0][0].schema
if ct:
# Add a "flux" field to the match records which contains the
# colorterm-corrected reference flux. The field name is hard-coded in
# lsst::meas::mosaic::Source.
mapper = afwTable.SchemaMapper(refSchema)
for key, field in refSchema:
mapper.addMapping(key)
key_f = mapper.editOutputSchema().addField("flux", type=float, doc="Reference flux")
table = afwTable.SimpleTable.make(mapper.getOutputSchema())
table.preallocate(len(matches))
for match in matches:
newMatch = table.makeRecord()
newMatch.assign(match[0], mapper)
match[0] = newMatch
key_p = refSchema.find(refSchema.join(ct.primary, "flux")).key
key_s = refSchema.find(refSchema.join(ct.secondary, "flux")).key
refFlux1 = numpy.array([m[0].get(key_p) for m in matches])
refFlux2 = numpy.array([m[0].get(key_s) for m in matches])
refMag1 = -2.5*numpy.log10(refFlux1)
refMag2 = -2.5*numpy.log10(refFlux2)
refMag = ct.transformMags(refMag1, refMag2)
refFlux = numpy.power(10.0, -0.4*refMag)
matches = [setCatFlux(m, f, key_f) for m, f in zip(matches, refFlux) if f == f]
else:
# No colorterm; we can get away with aliasing the reference flux.
refFluxField = measAlg.getRefFluxField(refSchema, filterName)
refSchema.getAliasMap().set("flux", refFluxField)
sources = self.selectStars(sources)
matches = self.selectStars(matches, True)
except Exception as e:
print("Failed to read: %s" % (e))
sources = None
continue
if sources is not None:
for s in sources:
if numpy.isfinite(s.getRa().asDegrees()): # get rid of NaN
src = measMosaic.Source(s)
src.setExp(dataRef.dataId['visit'])
src.setChip(dataRef.dataId['ccd'])
ssVisit[dataRef.dataId['visit']].append(src)
for m in matches:
if m[0] is not None and m[1] is not None:
match = (measMosaic.Source(m[0], wcs), measMosaic.Source(m[1]))
match[1].setExp(dataRef.dataId['visit'])
match[1].setChip(dataRef.dataId['ccd'])
mlVisit[dataRef.dataId['visit']].append(match)
wcsDic[dataRef.dataId['visit']][dataRef.dataId['ccd']] = wcs
calibDic[dataRef.dataId['visit']][dataRef.dataId['ccd']] = photoCalib
ffpDic[dataRef.dataId['visit']][dataRef.dataId['ccd']] = ffp
dataRefListUsed.append(dataRef)
for visit in ssVisit:
sourceSet.append(ssVisit[visit])
matchList.append(mlVisit[visit])
d_lim = afwGeom.Angle(self.config.radXMatch, afwGeom.arcseconds)
nbrightest = self.config.nBrightest
allMat, allSource = self.mergeCatalog(sourceSet, matchList, ccdSet, d_lim)
dx_m, dy_m, dx_s, dy_s, m0_m, dm_m, m0_s, dm_s = self.makeDiffPosFlux(allMat, allSource, wcsDic, calibDic, ffpDic)
self.plotFlux(m0_m, dm_m, m0_s, dm_s)
self.makeFluxStat(allMat, allSource, calibDic, ffpDic, wcsDic)
self.plotPos(dx_m, dy_m, dx_s, dy_s)
self.plotPosAsMag(m0_s, dx_s, dy_s)
self.writeCatalog(allSource, wcsDic, calibDic, ffpDic)
def loadSkyCircle(self,
ctrCoord,
radius,
filterName=None,
epoch=None,
centroids=False):
shardIdList, isOnBoundaryList = self.indexer.getShardIds(
ctrCoord, radius)
shards = self.getShards(shardIdList)
refCat = self.butler.get('ref_cat',
dataId=self.indexer.makeDataId(
'master_schema', self.ref_dataset_name),
immediate=True)
# load the catalog, one shard at a time
for shard, isOnBoundary in zip(shards, isOnBoundaryList):
if shard is None:
continue
if isOnBoundary:
refCat.extend(self._trimToCircle(shard, ctrCoord, radius))
else:
refCat.extend(shard)
# make sure catalog is contiguous: must do this before PM calculations
if not refCat.isContiguous():
refCat = refCat.copy(True)
# apply proper motion corrections
if epoch is not None and "pm_ra" in refCat.schema:
# check for a catalog in a non-standard format
if isinstance(refCat.schema["pm_ra"].asKey(),
lsst.afw.table.KeyAngle):
self.applyProperMotions(refCat, epoch)
else:
self.log.warn(
"Catalog pm_ra field is not an Angle; not applying proper motion"
)
# update version=0 style refcats to have nJy fluxes
if self.dataset_config.format_version == 0 or not hasNanojanskyFluxUnits(
refCat.schema):
self.log.warn(
"Found version 0 reference catalog with old style units in schema."
)
self.log.warn(
"run `meas_algorithms/bin/convert_refcat_to_nJy.py` to convert fluxes to nJy."
)
self.log.warn("See RFC-575 for more details.")
refCat = convertToNanojansky(refCat, self.log)
else:
# For version >= 1, the version should be in the catalog header,
# too, and should be consistent with the version in the config.
catVersion = getFormatVersionFromRefCat(refCat)
if catVersion != self.dataset_config.format_version:
raise RuntimeError(
f"Format version in reference catalog ({catVersion}) does not match"
f" format_version field in config ({self.dataset_config.format_version})"
)
self._addFluxAliases(refCat.schema)
fluxField = getRefFluxField(schema=refCat.schema,
filterName=filterName)
if centroids:
# add and initialize centroid and hasCentroid fields (these are
# added after loading to avoid wasting space in the saved catalogs)
# the new fields are automatically initialized to (nan, nan) and
# False so no need to set them explicitly
mapper = afwTable.SchemaMapper(refCat.schema, True)
mapper.addMinimalSchema(refCat.schema, True)
mapper.editOutputSchema().addField("centroid_x", type=float)
mapper.editOutputSchema().addField("centroid_y", type=float)
mapper.editOutputSchema().addField("hasCentroid", type="Flag")
expandedCat = afwTable.SimpleCatalog(mapper.getOutputSchema())
expandedCat.extend(refCat, mapper=mapper)
refCat = expandedCat
# return reference catalog
return pipeBase.Struct(
refCat=refCat,
fluxField=fluxField,
)
def extractMagArrays(self, matches, filterLabel, sourceKeys):
"""!Extract magnitude and magnitude error arrays from the given matches.
@param[in] matches Reference/source matches, a @link lsst::afw::table::ReferenceMatchVector@endlink
@param[in] filterLabel Label of filter being calibrated
@param[in] sourceKeys Struct of source catalog keys, as returned by getSourceKeys()
@return Struct containing srcMag, refMag, srcMagErr, refMagErr, and magErr numpy arrays
where magErr is an error in the magnitude; the error in srcMag - refMag
If nonzero, config.magErrFloor will be added to magErr *only* (not srcMagErr or refMagErr), as
magErr is what is later used to determine the zero point.
Struct also contains refFluxFieldList: a list of field names of the reference catalog used for fluxes
(1 or 2 strings)
@note These magnitude arrays are the @em inputs to the photometric calibration, some may have been
discarded by clipping while estimating the calibration (https://jira.lsstcorp.org/browse/DM-813)
"""
srcInstFluxArr = np.array(
[m.second.get(sourceKeys.instFlux) for m in matches])
srcInstFluxErrArr = np.array(
[m.second.get(sourceKeys.instFluxErr) for m in matches])
if not np.all(np.isfinite(srcInstFluxErrArr)):
# this is an unpleasant hack; see DM-2308 requesting a better solution
self.log.warn(
"Source catalog does not have flux uncertainties; using sqrt(flux)."
)
srcInstFluxErrArr = np.sqrt(srcInstFluxArr)
# convert source instFlux from DN to an estimate of nJy
referenceFlux = (0 * u.ABmag).to_value(u.nJy)
srcInstFluxArr = srcInstFluxArr * referenceFlux
srcInstFluxErrArr = srcInstFluxErrArr * referenceFlux
if not matches:
raise RuntimeError("No reference stars are available")
refSchema = matches[0].first.schema
applyColorTerms = self.config.applyColorTerms
applyCTReason = "config.applyColorTerms is %s" % (
self.config.applyColorTerms, )
if self.config.applyColorTerms is None:
# apply color terms if color term data is available and photoCatName specified
ctDataAvail = len(self.config.colorterms.data) > 0
photoCatSpecified = self.config.photoCatName is not None
applyCTReason += " and data %s available" % ("is" if ctDataAvail
else "is not")
applyCTReason += " and photoRefCat %s provided" % (
"is" if photoCatSpecified else "is not")
applyColorTerms = ctDataAvail and photoCatSpecified
if applyColorTerms:
self.log.info(
"Applying color terms for filter=%r, config.photoCatName=%s because %s",
filterLabel.physicalLabel, self.config.photoCatName,
applyCTReason)
colorterm = self.config.colorterms.getColorterm(
filterLabel.physicalLabel,
self.config.photoCatName,
doRaise=True)
refCat = afwTable.SimpleCatalog(matches[0].first.schema)
# extract the matched refCat as a Catalog for the colorterm code
refCat.reserve(len(matches))
for x in matches:
record = refCat.addNew()
record.assign(x.first)
refMagArr, refMagErrArr = colorterm.getCorrectedMagnitudes(refCat)
fluxFieldList = [
getRefFluxField(refSchema, filt)
for filt in (colorterm.primary, colorterm.secondary)
]
else:
# no colorterms to apply
self.log.info("Not applying color terms because %s", applyCTReason)
colorterm = None
fluxFieldList = [getRefFluxField(refSchema, filterLabel.bandLabel)]
fluxField = getRefFluxField(refSchema, filterLabel.bandLabel)
fluxKey = refSchema.find(fluxField).key
refFluxArr = np.array([m.first.get(fluxKey) for m in matches])
try:
fluxErrKey = refSchema.find(fluxField + "Err").key
refFluxErrArr = np.array(
[m.first.get(fluxErrKey) for m in matches])
except KeyError:
# Reference catalogue may not have flux uncertainties; HACK DM-2308
self.log.warn(
"Reference catalog does not have flux uncertainties for %s; using sqrt(flux).",
fluxField)
refFluxErrArr = np.sqrt(refFluxArr)
refMagArr = u.Quantity(refFluxArr, u.nJy).to_value(u.ABmag)
# HACK convert to Jy until we have a replacement for this (DM-16903)
refMagErrArr = abMagErrFromFluxErr(refFluxErrArr * 1e-9,
refFluxArr * 1e-9)
# compute the source catalog magnitudes and errors
srcMagArr = u.Quantity(srcInstFluxArr, u.nJy).to_value(u.ABmag)
# Fitting with error bars in both axes is hard
# for now ignore reference flux error, but ticket DM-2308 is a request for a better solution
# HACK convert to Jy until we have a replacement for this (DM-16903)
magErrArr = abMagErrFromFluxErr(srcInstFluxErrArr * 1e-9,
srcInstFluxArr * 1e-9)
if self.config.magErrFloor != 0.0:
magErrArr = (magErrArr**2 + self.config.magErrFloor**2)**0.5
srcMagErrArr = abMagErrFromFluxErr(srcInstFluxErrArr * 1e-9,
srcInstFluxArr * 1e-9)
good = np.isfinite(srcMagArr) & np.isfinite(refMagArr)
return pipeBase.Struct(
srcMag=srcMagArr[good],
refMag=refMagArr[good],
magErr=magErrArr[good],
srcMagErr=srcMagErrArr[good],
refMagErr=refMagErrArr[good],
refFluxFieldList=fluxFieldList,
)
def readSrc(self, dataRef):
"""Read source catalog etc for input dataRef
The following are returned:
Source catalog, matched list, and wcs will be read from 'src', 'srcMatch', and 'calexp_md',
respectively.
NOTE: If the detector has nQuarter%4 != 0 (i.e. it is rotated w.r.t the focal plane
coordinate system), the (x, y) pixel values of the centroid slot for the source
catalogs are rotated such that pixel (0, 0) is the LLC (i.e. the coordinate system
expected by meas_mosaic).
If color transformation information is given, it will be applied to the reference flux
of the matched list. The source catalog and matched list will be converted to measMosaic's
Source and SourceMatch and returned.
The number of 'Source's in each cell defined by config.cellSize will be limited to brightest
config.nStarPerCell.
"""
self.log = Log.getDefaultLogger()
dataId = dataRef.dataId
try:
if not dataRef.datasetExists("src"):
raise RuntimeError("no data for src %s" % (dataId))
if not dataRef.datasetExists("calexp_md"):
raise RuntimeError("no data for calexp_md %s" % (dataId))
calexp_md = dataRef.get("calexp_md", immediate=True)
detector = dataRef.get("camera")[dataRef.dataId["ccd"]] # OK for HSC; maybe not for other cameras
wcs = afwGeom.makeSkyWcs(calexp_md)
nQuarter = detector.getOrientation().getNQuarter()
sources = dataRef.get("src", immediate=True, flags=afwTable.SOURCE_IO_NO_FOOTPRINTS)
# Check if we are looking at HSC stack outputs: if so, no pixel rotation of sources is
# required, but alias mapping must be set to associate HSC's schema with that of LSST.
hscRun = mosaicUtils.checkHscStack(calexp_md)
if hscRun is None:
if nQuarter%4 != 0:
dims = afwImage.bboxFromMetadata(calexp_md).getDimensions()
sources = mosaicUtils.rotatePixelCoords(sources, dims.getX(), dims.getY(),
nQuarter)
# Set some alias maps for the source catalog where needed for
# backwards compatibility
if self.config.srcSchemaMap and hscRun:
aliasMap = sources.schema.getAliasMap()
for lsstName, otherName in self.config.srcSchemaMap.items():
aliasMap.set(lsstName, otherName)
if self.config.flagsToAlias and "calib_psfUsed" in sources.schema:
aliasMap = sources.schema.getAliasMap()
for lsstName, otherName in self.config.flagsToAlias.items():
aliasMap.set(lsstName, otherName)
refObjLoader = self.config.loadAstrom.apply(butler=dataRef.getButler())
srcMatch = dataRef.get("srcMatch", immediate=True)
if hscRun is not None:
# The reference object loader grows the bbox by the config parameter pixelMargin. This
# is set to 50 by default but is not reflected by the radius parameter set in the
# metadata, so some matches may reside outside the circle searched within this radius
# Thus, increase the radius set in the metadata fed into joinMatchListWithCatalog() to
# accommodate.
matchmeta = srcMatch.table.getMetadata()
rad = matchmeta.getDouble("RADIUS")
matchmeta.setDouble("RADIUS", rad*1.05, "field radius in degrees, approximate, padded")
matches = refObjLoader.joinMatchListWithCatalog(srcMatch, sources)
# Set the aliap map for the matched sources (i.e. the [1] attribute schema for each match)
if self.config.srcSchemaMap is not None and hscRun is not None:
for mm in matches:
aliasMap = mm[1].schema.getAliasMap()
for lsstName, otherName in self.config.srcSchemaMap.items():
aliasMap.set(lsstName, otherName)
if hscRun is not None:
for slot in ("PsfFlux", "ModelFlux", "ApFlux", "GaussianFlux", "Centroid", "Shape"):
getattr(matches[0][1].getTable(), "define" + slot)(
getattr(sources, "get" + slot + "Definition")())
# For some reason, the CalibFlux slot in sources is coming up as centroid_sdss, so
# set it to flux_naive explicitly
for slot in ("CalibFlux", ):
getattr(matches[0][1].getTable(), "define" + slot)("flux_naive")
matches = [m for m in matches if m[0] is not None]
refSchema = matches[0][0].schema if matches else None
if self.cterm is not None and len(matches) != 0:
# Add a "flux" field to the input schema of the first element
# of the match and populate it with a colorterm correct flux.
mapper = afwTable.SchemaMapper(refSchema)
for key, field in refSchema:
mapper.addMapping(key)
fluxKey = mapper.editOutputSchema().addField("flux", type=float, doc="Reference flux")
fluxErrKey = mapper.editOutputSchema().addField("fluxErr", type=float,
doc="Reference flux uncertainty")
table = afwTable.SimpleTable.make(mapper.getOutputSchema())
table.preallocate(len(matches))
for match in matches:
newMatch = table.makeRecord()
newMatch.assign(match[0], mapper)
match[0] = newMatch
# extract the matched refCat as a Catalog for the colorterm code
refCat = afwTable.SimpleCatalog(matches[0].first.schema)
refCat.reserve(len(matches))
for x in matches:
record = refCat.addNew()
record.assign(x.first)
refMag, refMagErr = self.cterm.getCorrectedMagnitudes(refCat,
afwImage.Filter(calexp_md).getName())
# NOTE: mosaic assumes fluxes are in Jy
refFlux = (refMag*astropy.units.ABmag).to_value(astropy.units.Jy)
refFluxErr = afwImage.fluxErrFromABMagErr(refMagErr, refMag)
matches = [self.setCatFlux(m, flux, fluxKey, fluxErr, fluxErrKey) for
m, flux, fluxErr in zip(matches, refFlux, refFluxErr) if flux == flux]
else:
filterName = afwImage.Filter(calexp_md).getName()
refFluxField = measAlg.getRefFluxField(refSchema, filterName)
refSchema.getAliasMap().set("flux", refFluxField)
# LSST reads in a_net catalogs with flux in "janskys", so must convert back to DN.
matches = mosaicUtils.matchJanskyToDn(matches)
selSources = self.selectStars(sources, self.config.includeSaturated)
selMatches = self.selectStars(matches, self.config.includeSaturated)
retSrc = list()
retMatch = list()
if len(selMatches) > self.config.minNumMatch:
naxis1, naxis2 = afwImage.bboxFromMetadata(calexp_md).getDimensions()
if hscRun is None:
if nQuarter%2 != 0:
naxis1, naxis2 = naxis2, naxis1
bbox = afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(naxis1, naxis2))
cellSet = afwMath.SpatialCellSet(bbox, self.config.cellSize, self.config.cellSize)
for s in selSources:
if numpy.isfinite(s.getRa().asDegrees()): # get rid of NaN
src = measMosaic.Source(s)
src.setExp(dataId["visit"])
src.setChip(dataId["ccd"])
try:
tmp = measMosaic.SpatialCellSource(src)
cellSet.insertCandidate(tmp)
except:
self.log.info("FAILED TO INSERT CANDIDATE: visit=%d ccd=%d x=%f y=%f" %
(dataRef.dataId["visit"], dataRef.dataId["ccd"],
src.getX(), src.getY()) + " bbox=" + str(bbox))
for cell in cellSet.getCellList():
cell.sortCandidates()
for i, cand in enumerate(cell):
src = cand.getSource()
retSrc.append(src)
if i == self.config.nStarPerCell - 1:
break
for m in selMatches:
if m[0] is not None and m[1] is not None:
match = (measMosaic.Source(m[0], wcs), measMosaic.Source(m[1]))
match[1].setExp(dataId["visit"])
match[1].setChip(dataId["ccd"])
retMatch.append(match)
else:
self.log.info("%8d %3d : %d/%d matches Suspicious to wrong match. Ignore this CCD" %
(dataRef.dataId["visit"], dataRef.dataId["ccd"], len(selMatches), len(matches)))
except Exception as e:
self.log.warn("Failed to read %s: %s" % (dataId, e))
return dataId, [None, None, None]
return dataId, [retSrc, retMatch, wcs]
def selectSources(self, sourceCat, matches=None, exposure=None):
"""Return a selection of sources for Kernel candidates.
Parameters:
-----------
sourceCat : `lsst.afw.table.SourceCatalog`
Catalog of sources to select from.
This catalog must be contiguous in memory.
matches : `list` of `lsst.afw.table.ReferenceMatch`
A match vector as produced by meas_astrom.
exposure : `lsst.afw.image.Exposure` or None
The exposure the catalog was built from; used for debug display.
Return
------
struct : `lsst.pipe.base.Struct`
The struct contains the following data:
- selected : `array` of `bool``
Boolean array of sources that were selected, same length as
sourceCat.
"""
import lsstDebug
display = lsstDebug.Info(__name__).display
displayExposure = lsstDebug.Info(__name__).displayExposure
pauseAtEnd = lsstDebug.Info(__name__).pauseAtEnd
if matches is None:
raise RuntimeError("DiaCatalogSourceSelector requires matches")
mi = exposure.getMaskedImage()
if display:
if displayExposure:
ds9.mtv(mi, title="Kernel candidates", frame=lsstDebug.frame)
#
# Look for flags in each Source
#
isGoodSource = CheckSource(sourceCat, self.config.fluxLim,
self.config.fluxMax, self.config.badFlags)
# Go through and find all the acceptable candidates in the catalogue
selected = np.zeros(len(sourceCat), dtype=bool)
if display and displayExposure:
symbs = []
ctypes = []
doColorCut = True
refSchema = matches[0][0].schema
rRefFluxField = measAlg.getRefFluxField(refSchema, "r")
gRefFluxField = measAlg.getRefFluxField(refSchema, "g")
for i, (ref, source, d) in enumerate(matches):
if not isGoodSource(source):
if display and displayExposure:
symbs.append("+")
ctypes.append(ds9.RED)
else:
isStar = not ref.get("resolved")
isVar = not ref.get("photometric")
gMag = None
rMag = None
if doColorCut:
try:
gMag = -2.5 * np.log10(ref.get(gRefFluxField))
rMag = -2.5 * np.log10(ref.get(rRefFluxField))
except KeyError:
self.log.warn(
"Cannot cut on color info; fields 'g' and 'r' do not exist"
)
doColorCut = False
isRightColor = True
else:
isRightColor = (gMag - rMag) >= self.config.grMin and (
gMag - rMag) <= self.config.grMax
isRightType = (self.config.selectStar
and isStar) or (self.config.selectGalaxy
and not isStar)
isRightVar = (self.config.includeVariable) or (
self.config.includeVariable is isVar)
if isRightType and isRightVar and isRightColor:
selected[i] = True
if display and displayExposure:
symbs.append("+")
ctypes.append(ds9.GREEN)
elif display and displayExposure:
symbs.append("o")
ctypes.append(ds9.BLUE)
if display and displayExposure:
with ds9.Buffering():
for (ref, source,
d), symb, ctype in zip(matches, symbs, ctypes):
if display and displayExposure:
ds9.dot(symb,
source.getX() - mi.getX0(),
source.getY() - mi.getY0(),
size=4,
ctype=ctype,
frame=lsstDebug.frame)
if display:
lsstDebug.frame += 1
if pauseAtEnd:
input("Continue? y[es] p[db] ")
return Struct(selected=selected)
def extractMagArrays(self, matches, filterName, sourceKeys):
"""!Extract magnitude and magnitude error arrays from the given matches.
@param[in] matches Reference/source matches, a @link lsst::afw::table::ReferenceMatchVector@endlink
@param[in] filterName Name of filter being calibrated
@param[in] sourceKeys Struct of source catalog keys, as returned by getSourceKeys()
@return Struct containing srcMag, refMag, srcMagErr, refMagErr, and magErr numpy arrays
where magErr is an error in the magnitude; the error in srcMag - refMag
If nonzero, config.magErrFloor will be added to magErr *only* (not srcMagErr or refMagErr), as
magErr is what is later used to determine the zero point.
Struct also contains refFluxFieldList: a list of field names of the reference catalog used for fluxes
(1 or 2 strings)
@note These magnitude arrays are the @em inputs to the photometric calibration, some may have been
discarded by clipping while estimating the calibration (https://jira.lsstcorp.org/browse/DM-813)
"""
srcInstFluxArr = np.array([m.second.get(sourceKeys.instFlux) for m in matches])
srcInstFluxErrArr = np.array([m.second.get(sourceKeys.instFluxErr) for m in matches])
if not np.all(np.isfinite(srcInstFluxErrArr)):
# this is an unpleasant hack; see DM-2308 requesting a better solution
self.log.warn("Source catalog does not have flux uncertainties; using sqrt(flux).")
srcInstFluxErrArr = np.sqrt(srcInstFluxArr)
# convert source instFlux from DN to an estimate of nJy
referenceFlux = (0*u.ABmag).to_value(u.nJy)
srcInstFluxArr = srcInstFluxArr * referenceFlux
srcInstFluxErrArr = srcInstFluxErrArr * referenceFlux
if not matches:
raise RuntimeError("No reference stars are available")
refSchema = matches[0].first.schema
applyColorTerms = self.config.applyColorTerms
applyCTReason = "config.applyColorTerms is %s" % (self.config.applyColorTerms,)
if self.config.applyColorTerms is None:
# apply color terms if color term data is available and photoCatName specified
ctDataAvail = len(self.config.colorterms.data) > 0
photoCatSpecified = self.config.photoCatName is not None
applyCTReason += " and data %s available" % ("is" if ctDataAvail else "is not")
applyCTReason += " and photoRefCat %s provided" % ("is" if photoCatSpecified else "is not")
applyColorTerms = ctDataAvail and photoCatSpecified
if applyColorTerms:
self.log.info("Applying color terms for filterName=%r, config.photoCatName=%s because %s",
filterName, self.config.photoCatName, applyCTReason)
colorterm = self.config.colorterms.getColorterm(
filterName=filterName, photoCatName=self.config.photoCatName, doRaise=True)
refCat = afwTable.SimpleCatalog(matches[0].first.schema)
# extract the matched refCat as a Catalog for the colorterm code
refCat.reserve(len(matches))
for x in matches:
record = refCat.addNew()
record.assign(x.first)
refMagArr, refMagErrArr = colorterm.getCorrectedMagnitudes(refCat, filterName)
fluxFieldList = [getRefFluxField(refSchema, filt) for filt in (colorterm.primary,
colorterm.secondary)]
else:
# no colorterms to apply
self.log.info("Not applying color terms because %s", applyCTReason)
colorterm = None
fluxFieldList = [getRefFluxField(refSchema, filterName)]
fluxField = getRefFluxField(refSchema, filterName)
fluxKey = refSchema.find(fluxField).key
refFluxArr = np.array([m.first.get(fluxKey) for m in matches])
try:
fluxErrKey = refSchema.find(fluxField + "Err").key
refFluxErrArr = np.array([m.first.get(fluxErrKey) for m in matches])
except KeyError:
# Reference catalogue may not have flux uncertainties; HACK DM-2308
self.log.warn("Reference catalog does not have flux uncertainties for %s; using sqrt(flux).",
fluxField)
refFluxErrArr = np.sqrt(refFluxArr)
refMagArr = u.Quantity(refFluxArr, u.nJy).to_value(u.ABmag)
# HACK convert to Jy until we have a replacement for this (DM-16903)
refMagErrArr = abMagErrFromFluxErr(refFluxErrArr*1e-9, refFluxArr*1e-9)
# compute the source catalog magnitudes and errors
srcMagArr = u.Quantity(srcInstFluxArr, u.nJy).to_value(u.ABmag)
# Fitting with error bars in both axes is hard
# for now ignore reference flux error, but ticket DM-2308 is a request for a better solution
# HACK convert to Jy until we have a replacement for this (DM-16903)
magErrArr = abMagErrFromFluxErr(srcInstFluxErrArr*1e-9, srcInstFluxArr*1e-9)
if self.config.magErrFloor != 0.0:
magErrArr = (magErrArr**2 + self.config.magErrFloor**2)**0.5
srcMagErrArr = abMagErrFromFluxErr(srcInstFluxErrArr*1e-9, srcInstFluxArr*1e-9)
good = np.isfinite(srcMagArr) & np.isfinite(refMagArr)
return pipeBase.Struct(
srcMag=srcMagArr[good],
refMag=refMagArr[good],
magErr=magErrArr[good],
srcMagErr=srcMagErrArr[good],
refMagErr=refMagErrArr[good],
refFluxFieldList=fluxFieldList,
)
def loadSkyCircle(self, ctrCoord, radius, filterName=None):
"""!Load reference objects that overlap a circular sky region
@param[in] ctrCoord center of search region (an afwGeom.Coord)
@param[in] radius radius of search region (an afwGeom.Angle)
@param[in] filterName name of filter, or None for the default filter;
used for flux values in case we have flux limits (which are not yet implemented)
@return an lsst.pipe.base.Struct containing:
- refCat a catalog of reference objects with the
\link meas_algorithms_loadReferenceObjects_Schema standard schema \endlink
as documented in LoadReferenceObjects, including photometric, resolved and variable;
hasCentroid is False for all objects.
- fluxField = name of flux field for specified filterName
"""
self._readIndexFiles()
names = []
mcols = []
ecols = []
for col, mcol in self.andConfig.magColumnMap.items():
names.append(col)
mcols.append(mcol)
ecols.append(self.andConfig.magErrorColumnMap.get(col, ''))
margs = (names, mcols, ecols)
solver = self._getSolver()
# Find multi-index files within range
multiInds = self._getMIndexesWithinRange(ctrCoord, radius)
# compute solver.getCatalog arguments that follow the list of star kd-trees:
# - center equatorial angle (e.g. RA) in deg
# - center polar angle (e.g. Dec) in deg
# - radius, in deg
# - idColumn
# - (margs)
# - star-galaxy column
# - variability column
fixedArgTuple = (
ctrCoord,
radius,
self.andConfig.idColumn,
) + margs + (
self.andConfig.starGalaxyColumn,
self.andConfig.variableColumn,
True, # eliminate duplicate IDs
)
self.log.logdebug("search for objects at %s with radius %s deg" % (ctrCoord, radius.asDegrees()))
with LoadMultiIndexes(multiInds):
# We just want to pass the star kd-trees, so just pass the
# first element of each multi-index.
inds = tuple(mi[0] for mi in multiInds)
refCat = solver.getCatalog(inds, *fixedArgTuple)
self._addFluxAliases(schema=refCat.schema)
fluxField = getRefFluxField(schema=refCat.schema, filterName=filterName)
self.log.logdebug("found %d objects" % (len(refCat),))
return pipeBase.Struct(
refCat = refCat,
fluxField = fluxField,
)
def readSrc(self, dataRef):
"""Read source catalog etc for input dataRef
The following are returned:
Source catalog, matched list, and wcs will be read from 'src', 'srcMatch', and 'calexp_md',
respectively.
NOTE: If the detector has nQuarter%4 != 0 (i.e. it is rotated w.r.t the focal plane
coordinate system), the (x, y) pixel values of the centroid slot for the source
catalogs are rotated such that pixel (0, 0) is the LLC (i.e. the coordinate system
expected by meas_mosaic).
If color transformation information is given, it will be applied to the reference flux
of the matched list. The source catalog and matched list will be converted to measMosaic's
Source and SourceMatch and returned.
The number of 'Source's in each cell defined by config.cellSize will be limited to brightest
config.nStarPerCell.
"""
self.log = Log.getDefaultLogger()
dataId = dataRef.dataId
try:
if not dataRef.datasetExists("src"):
raise RuntimeError("no data for src %s" % (dataId))
if not dataRef.datasetExists("calexp_md"):
raise RuntimeError("no data for calexp_md %s" % (dataId))
calexp_md = dataRef.get("calexp_md", immediate=True)
detector = dataRef.get("camera")[dataRef.dataId[
"ccd"]] # OK for HSC; maybe not for other cameras
wcs = afwGeom.makeSkyWcs(calexp_md)
nQuarter = detector.getOrientation().getNQuarter()
sources = dataRef.get("src",
immediate=True,
flags=afwTable.SOURCE_IO_NO_FOOTPRINTS)
# Check if we are looking at HSC stack outputs: if so, no pixel rotation of sources is
# required, but alias mapping must be set to associate HSC's schema with that of LSST.
hscRun = mosaicUtils.checkHscStack(calexp_md)
if hscRun is None:
if nQuarter % 4 != 0:
dims = afwImage.bboxFromMetadata(calexp_md).getDimensions()
sources = mosaicUtils.rotatePixelCoords(
sources, dims.getX(), dims.getY(), nQuarter)
# Set the aliap map for the source catalog
if self.config.srcSchemaMap is not None and hscRun is not None:
aliasMap = sources.schema.getAliasMap()
for lsstName, otherName in self.config.srcSchemaMap.items():
aliasMap.set(lsstName, otherName)
refObjLoader = self.config.loadAstrom.apply(
butler=dataRef.getButler())
srcMatch = dataRef.get("srcMatch", immediate=True)
if hscRun is not None:
# The reference object loader grows the bbox by the config parameter pixelMargin. This
# is set to 50 by default but is not reflected by the radius parameter set in the
# metadata, so some matches may reside outside the circle searched within this radius
# Thus, increase the radius set in the metadata fed into joinMatchListWithCatalog() to
# accommodate.
matchmeta = srcMatch.table.getMetadata()
rad = matchmeta.getDouble("RADIUS")
matchmeta.setDouble(
"RADIUS", rad * 1.05,
"field radius in degrees, approximate, padded")
matches = refObjLoader.joinMatchListWithCatalog(srcMatch, sources)
# Set the aliap map for the matched sources (i.e. the [1] attribute schema for each match)
if self.config.srcSchemaMap is not None and hscRun is not None:
for mm in matches:
aliasMap = mm[1].schema.getAliasMap()
for lsstName, otherName in self.config.srcSchemaMap.items(
):
aliasMap.set(lsstName, otherName)
if hscRun is not None:
for slot in ("PsfFlux", "ModelFlux", "ApFlux", "InstFlux",
"Centroid", "Shape"):
getattr(matches[0][1].getTable(), "define" + slot)(getattr(
sources, "get" + slot + "Definition")())
# For some reason, the CalibFlux slot in sources is coming up as centroid_sdss, so
# set it to flux_naive explicitly
for slot in ("CalibFlux", ):
getattr(matches[0][1].getTable(),
"define" + slot)("flux_naive")
matches = [m for m in matches if m[0] is not None]
refSchema = matches[0][0].schema if matches else None
if self.cterm is not None and len(matches) != 0:
# Add a "flux" field to the input schema of the first element
# of the match and populate it with a colorterm correct flux.
mapper = afwTable.SchemaMapper(refSchema)
for key, field in refSchema:
mapper.addMapping(key)
fluxKey = mapper.editOutputSchema().addField(
"flux", type=float, doc="Reference flux")
fluxSigmaKey = mapper.editOutputSchema().addField(
"fluxSigma", type=float, doc="Reference flux uncertainty")
table = afwTable.SimpleTable.make(mapper.getOutputSchema())
table.preallocate(len(matches))
for match in matches:
newMatch = table.makeRecord()
newMatch.assign(match[0], mapper)
match[0] = newMatch
primaryFluxKey = refSchema.find(
refSchema.join(self.cterm.primary, "flux")).key
secondaryFluxKey = refSchema.find(
refSchema.join(self.cterm.secondary, "flux")).key
primaryFluxSigmaKey = refSchema.find(
refSchema.join(self.cterm.primary, "fluxSigma")).key
secondaryFluxSigmaKey = refSchema.find(
refSchema.join(self.cterm.secondary, "fluxSigma")).key
refFlux1 = numpy.array(
[m[0].get(primaryFluxKey) for m in matches])
refFlux2 = numpy.array(
[m[0].get(secondaryFluxKey) for m in matches])
refFluxSigma1 = numpy.array(
[m[0].get(primaryFluxSigmaKey) for m in matches])
refFluxSigma2 = numpy.array(
[m[0].get(secondaryFluxSigmaKey) for m in matches])
refMag1 = -2.5 * numpy.log10(refFlux1)
refMag2 = -2.5 * numpy.log10(refFlux2)
refMag = self.cterm.transformMags(refMag1, refMag2)
refFlux = numpy.power(10.0, -0.4 * refMag)
refFluxSigma = self.cterm.propagateFluxErrors(
refFluxSigma1, refFluxSigma2)
matches = [
self.setCatFlux(m, flux, fluxKey, fluxSigma, fluxSigmaKey)
for m, flux, fluxSigma in zip(matches, refFlux,
refFluxSigma) if flux == flux
]
else:
filterName = afwImage.Filter(calexp_md).getName()
refFluxField = measAlg.getRefFluxField(refSchema, filterName)
refSchema.getAliasMap().set("flux", refFluxField)
# LSST reads in a_net catalogs with flux in "janskys", so must convert back to DN.
matches = mosaicUtils.matchJanskyToDn(matches)
selSources = self.selectStars(sources,
self.config.includeSaturated)
selMatches = self.selectStars(matches,
self.config.includeSaturated)
retSrc = list()
retMatch = list()
if len(selMatches) > self.config.minNumMatch:
naxis1, naxis2 = afwImage.bboxFromMetadata(
calexp_md).getDimensions()
if hscRun is None:
if nQuarter % 2 != 0:
naxis1, naxis2 = naxis2, naxis1
bbox = afwGeom.Box2I(afwGeom.Point2I(0, 0),
afwGeom.Extent2I(naxis1, naxis2))
cellSet = afwMath.SpatialCellSet(bbox, self.config.cellSize,
self.config.cellSize)
for s in selSources:
if numpy.isfinite(s.getRa().asDegrees()): # get rid of NaN
src = measMosaic.Source(s)
src.setExp(dataId["visit"])
src.setChip(dataId["ccd"])
try:
tmp = measMosaic.SpatialCellSource(src)
cellSet.insertCandidate(tmp)
except:
self.log.info(
"FAILED TO INSERT CANDIDATE: visit=%d ccd=%d x=%f y=%f"
% (dataRef.dataId["visit"], dataRef.
dataId["ccd"], src.getX(), src.getY()) +
" bbox=" + str(bbox))
for cell in cellSet.getCellList():
cell.sortCandidates()
for i, cand in enumerate(cell):
src = cand.getSource()
retSrc.append(src)
if i == self.config.nStarPerCell - 1:
break
for m in selMatches:
if m[0] is not None and m[1] is not None:
match = (measMosaic.Source(m[0], wcs),
measMosaic.Source(m[1]))
match[1].setExp(dataId["visit"])
match[1].setChip(dataId["ccd"])
retMatch.append(match)
else:
self.log.info(
"%8d %3d : %d/%d matches Suspicious to wrong match. Ignore this CCD"
% (dataRef.dataId["visit"], dataRef.dataId["ccd"],
len(selMatches), len(matches)))
except Exception as e:
self.log.warn("Failed to read %s: %s" % (dataId, e))
return dataId, [None, None, None]
return dataId, [retSrc, retMatch, wcs]
def loadSkyCircle(self, ctrCoord, radius, filterName=None):
"""!Load reference objects that overlap a circular sky region
@param[in] ctrCoord center of search region (an afwGeom.Coord)
@param[in] radius radius of search region (an afwGeom.Angle)
@param[in] filterName name of filter, or None for the default filter;
used for flux values in case we have flux limits (which are not yet implemented)
@return an lsst.pipe.base.Struct containing:
- refCat a catalog of reference objects with the
\link meas_algorithms_loadReferenceObjects_Schema standard schema \endlink
as documented in LoadReferenceObjects, including photometric, resolved and variable;
hasCentroid is False for all objects.
- fluxField = name of flux field for specified filterName
"""
self._readIndexFiles()
names = []
mcols = []
ecols = []
for col, mcol in self.andConfig.magColumnMap.items():
names.append(col)
mcols.append(mcol)
ecols.append(self.andConfig.magErrorColumnMap.get(col, ''))
margs = (names, mcols, ecols)
solver = self._getSolver()
# Find multi-index files within range
multiInds = self._getMIndexesWithinRange(ctrCoord, radius)
# compute solver.getCatalog arguments that follow the list of star kd-trees:
# - center equatorial angle (e.g. RA) in deg
# - center polar angle (e.g. Dec) in deg
# - radius, in deg
# - idColumn
# - (margs)
# - star-galaxy column
# - variability column
fixedArgTuple = (
ctrCoord,
radius,
self.andConfig.idColumn,
) + margs + (
self.andConfig.starGalaxyColumn,
self.andConfig.variableColumn,
True, # eliminate duplicate IDs
)
self.log.debug("search for objects at %s with radius %s deg", ctrCoord,
radius.asDegrees())
with LoadMultiIndexes(multiInds):
# We just want to pass the star kd-trees, so just pass the
# first element of each multi-index.
inds = tuple(mi[0] for mi in multiInds)
refCat = solver.getCatalog(inds, *fixedArgTuple)
self._addFluxAliases(schema=refCat.schema)
fluxField = getRefFluxField(schema=refCat.schema,
filterName=filterName)
# NOTE: sourceSelectors require contiguous catalogs, so ensure
# contiguity now, so views are preserved from here on.
if not refCat.isContiguous():
refCat = refCat.copy(deep=True)
self.log.debug("found %d objects", len(refCat))
return pipeBase.Struct(
refCat=refCat,
fluxField=fluxField,
)
def selectMatches(self, matches, sourceKeys, filterName, frame=None):
"""!Select reference/source matches according the criteria specified in the config.
\param[in] matches ReferenceMatchVector (not modified)
\param[in] sourceKeys Struct of source catalog keys, as returned by getSourceKeys()
\param[in] filterName name of camera filter; used to obtain the reference flux field
\param[in] frame ds9 frame number to use for debugging display
if frame is non-None, display information about trimmed objects on that ds9 frame:
- Bad: red x
- Unsuitable objects: blue + (and a cyan o if a galaxy)
- Failed flux cut: magenta *
\return a \link lsst.afw.table.ReferenceMatchVector\endlink that contains only the selected matches.
If a schema was passed during task construction, a flag field will be set on sources
in the selected matches.
\throws ValueError There are no valid matches.
"""
self.log.logdebug("Number of input matches: %d" % (len(matches)))
if len(matches) == 0:
raise ValueError("No input matches")
# Only use stars for which the flags indicate the photometry is good.
afterFlagCutInd = [i for i, m in enumerate(matches) if checkSourceFlags(m.second, sourceKeys)]
afterFlagCut = [matches[i] for i in afterFlagCutInd]
self.log.logdebug("Number of matches after source flag cuts: %d" % (len(afterFlagCut)))
if len(afterFlagCut) != len(matches):
if frame is not None:
with ds9.Buffering():
for i, m in enumerate(matches):
if i not in afterFlagCutInd:
x, y = m.second.getCentroid()
ds9.dot("x", x, y, size=4, frame=frame, ctype=ds9.RED)
matches = afterFlagCut
if len(matches) == 0:
raise ValueError("All matches eliminated by source flags")
refSchema = matches[0].first.schema
try:
photometricKey = refSchema.find("photometric").key
try:
resolvedKey = refSchema.find("resolved").key
except:
resolvedKey = None
try:
variableKey = refSchema.find("variable").key
except:
variableKey = None
except:
self.log.warn("No 'photometric' flag key found in reference schema.")
photometricKey = None
if photometricKey is not None:
afterRefCutInd = [i for i, m in enumerate(matches) if m.first.get(photometricKey)]
afterRefCut = [matches[i] for i in afterRefCutInd]
if len(afterRefCut) != len(matches):
if frame is not None:
with ds9.Buffering():
for i, m in enumerate(matches):
if i not in afterRefCutInd:
x, y = m.second.getCentroid()
ds9.dot("+", x, y, size=4, frame=frame, ctype=ds9.BLUE)
if resolvedKey and m.first.get(resolvedKey):
ds9.dot("o", x, y, size=6, frame=frame, ctype=ds9.CYAN)
if variableKey and m.first.get(variableKey):
ds9.dot("o", x, y, size=6, frame=frame, ctype=ds9.MAGENTA)
matches = afterRefCut
self.log.logdebug("Number of matches after reference catalog cuts: %d" % (len(matches)))
if len(matches) == 0:
raise RuntimeError("No sources remain in match list after reference catalog cuts.")
fluxName = getRefFluxField(refSchema, filterName)
fluxKey = refSchema.find(fluxName).key
if self.config.magLimit is not None:
fluxLimit = fluxFromABMag(self.config.magLimit)
afterMagCutInd = [i for i, m in enumerate(matches) if (m.first.get(fluxKey) > fluxLimit
and m.second.getPsfFlux() > 0.0)]
else:
afterMagCutInd = [i for i, m in enumerate(matches) if m.second.getPsfFlux() > 0.0]
afterMagCut = [matches[i] for i in afterMagCutInd]
if len(afterMagCut) != len(matches):
if frame is not None:
with ds9.Buffering():
for i, m in enumerate(matches):
if i not in afterMagCutInd:
x, y = m.second.getCentroid()
ds9.dot("*", x, y, size=4, frame=frame, ctype=ds9.MAGENTA)
matches = afterMagCut
self.log.logdebug("Number of matches after magnitude limit cuts: %d" % (len(matches)))
if len(matches) == 0:
raise RuntimeError("No sources remaining in match list after magnitude limit cuts.")
if frame is not None:
with ds9.Buffering():
for m in matches:
x, y = m.second.getCentroid()
ds9.dot("o", x, y, size=4, frame=frame, ctype=ds9.GREEN)
result = afwTable.ReferenceMatchVector()
for m in matches:
if self.outputField is not None:
m.second.set(self.outputField, True)
result.append(m)
return result
def extractMagArrays(self, matches, filterName, sourceKeys):
"""!Extract magnitude and magnitude error arrays from the given matches.
\param[in] matches Reference/source matches, a \link lsst::afw::table::ReferenceMatchVector\endlink
\param[in] filterName Name of filter being calibrated
\param[in] sourceKeys Struct of source catalog keys, as returned by getSourceKeys()
\return Struct containing srcMag, refMag, srcMagErr, refMagErr, and magErr numpy arrays
where magErr is an error in the magnitude; the error in srcMag - refMag
If nonzero, config.magErrFloor will be added to magErr *only* (not srcMagErr or refMagErr), as
magErr is what is later used to determine the zero point.
Struct also contains refFluxFieldList: a list of field names of the reference catalog used for fluxes
(1 or 2 strings)
\note These magnitude arrays are the \em inputs to the photometric calibration, some may have been
discarded by clipping while estimating the calibration (https://jira.lsstcorp.org/browse/DM-813)
"""
srcFluxArr = np.array([m.second.get(sourceKeys.flux) for m in matches])
srcFluxErrArr = np.array([m.second.get(sourceKeys.fluxErr) for m in matches])
if not np.all(np.isfinite(srcFluxErrArr)):
# this is an unpleasant hack; see DM-2308 requesting a better solution
self.log.warn("Source catalog does not have flux uncertainties; using sqrt(flux).")
srcFluxErrArr = np.sqrt(srcFluxArr)
# convert source flux from DN to an estimate of Jy
JanskysPerABFlux = 3631.0
srcFluxArr = srcFluxArr * JanskysPerABFlux
srcFluxErrArr = srcFluxErrArr * JanskysPerABFlux
if not matches:
raise RuntimeError("No reference stars are available")
refSchema = matches[0].first.schema
applyColorTerms = self.config.applyColorTerms
applyCTReason = "config.applyColorTerms is %s" % (self.config.applyColorTerms,)
if self.config.applyColorTerms is None:
# apply color terms if color term data is available and photoCatName specified
ctDataAvail = len(self.config.colorterms.data) > 0
photoCatSpecified = self.config.photoCatName is not None
applyCTReason += " and data %s available" % ("is" if ctDataAvail else "is not")
applyCTReason += " and photoRefCat %s None" % ("is not" if photoCatSpecified else "is")
applyColorTerms = ctDataAvail and photoCatSpecified
if applyColorTerms:
self.log.info("Applying color terms for filterName=%r, config.photoCatName=%s because %s" %
(filterName, self.config.photoCatName, applyCTReason))
ct = self.config.colorterms.getColorterm(
filterName=filterName, photoCatName=self.config.photoCatName, doRaise=True)
else:
self.log.info("Not applying color terms because %s" % (applyCTReason,))
ct = None
if ct: # we have a color term to worry about
fluxFieldList = [getRefFluxField(refSchema, filt) for filt in (ct.primary, ct.secondary)]
missingFluxFieldList = []
for fluxField in fluxFieldList:
try:
refSchema.find(fluxField).key
except KeyError:
missingFluxFieldList.append(fluxField)
if missingFluxFieldList:
self.log.warn("Source catalog does not have fluxes for %s; ignoring color terms" %
" ".join(missingFluxFieldList))
ct = None
if not ct:
fluxFieldList = [getRefFluxField(refSchema, filterName)]
refFluxArrList = [] # list of ref arrays, one per flux field
refFluxErrArrList = [] # list of ref flux arrays, one per flux field
for fluxField in fluxFieldList:
fluxKey = refSchema.find(fluxField).key
refFluxArr = np.array([m.first.get(fluxKey) for m in matches])
try:
fluxErrKey = refSchema.find(fluxField + "Sigma").key
refFluxErrArr = np.array([m.first.get(fluxErrKey) for m in matches])
except KeyError:
# Reference catalogue may not have flux uncertainties; HACK
self.log.warn("Reference catalog does not have flux uncertainties for %s; using sqrt(flux)."
% fluxField)
refFluxErrArr = np.sqrt(refFluxArr)
refFluxArrList.append(refFluxArr)
refFluxErrArrList.append(refFluxErrArr)
if ct: # we have a color term to worry about
refMagArr1 = np.array([abMagFromFlux(rf1) for rf1 in refFluxArrList[0]]) # primary
refMagArr2 = np.array([abMagFromFlux(rf2) for rf2 in refFluxArrList[1]]) # secondary
refMagArr = ct.transformMags(refMagArr1, refMagArr2)
refFluxErrArr = ct.propagateFluxErrors(refFluxErrArrList[0], refFluxErrArrList[1])
else:
refMagArr = np.array([abMagFromFlux(rf) for rf in refFluxArrList[0]])
srcMagArr = np.array([abMagFromFlux(sf) for sf in srcFluxArr])
# Fitting with error bars in both axes is hard
# for now ignore reference flux error, but ticket DM-2308 is a request for a better solution
magErrArr = np.array([abMagErrFromFluxErr(fe, sf) for fe, sf in izip(srcFluxErrArr, srcFluxArr)])
if self.config.magErrFloor != 0.0:
magErrArr = (magErrArr**2 + self.config.magErrFloor**2)**0.5
srcMagErrArr = np.array([abMagErrFromFluxErr(sfe, sf) for sfe, sf in izip(srcFluxErrArr, srcFluxArr)])
refMagErrArr = np.array([abMagErrFromFluxErr(rfe, rf) for rfe, rf in izip(refFluxErrArr, refFluxArr)])
return pipeBase.Struct(
srcMag = srcMagArr,
refMag = refMagArr,
magErr = magErrArr,
srcMagErr = srcMagErrArr,
refMagErr = refMagErrArr,
refFluxFieldList = fluxFieldList,
)
def selectStars(self, exposure, sourceCat, matches=None):
"""Select sources for Kernel candidates
@param[in] exposure the exposure containing the sources
@param[in] sourceCat catalog of sources that may be stars (an lsst.afw.table.SourceCatalog)
@param[in] matches a match vector as produced by meas_astrom; required
(defaults to None to match the StarSelector API and improve error handling)
@return an lsst.pipe.base.Struct containing:
- starCat a list of sources to be used as kernel candidates
"""
import lsstDebug
display = lsstDebug.Info(__name__).display
displayExposure = lsstDebug.Info(__name__).displayExposure
pauseAtEnd = lsstDebug.Info(__name__).pauseAtEnd
if matches is None:
raise RuntimeError("DiaCatalogSourceSelector requires matches")
mi = exposure.getMaskedImage()
if display:
if displayExposure:
ds9.mtv(mi, title="Kernel candidates", frame=lsstDebug.frame)
#
# Look for flags in each Source
#
isGoodSource = CheckSource(sourceCat, self.config.fluxLim, self.config.fluxMax, self.config.badFlags)
#
# Go through and find all the acceptable candidates in the catalogue
#
starCat = SourceCatalog(sourceCat.schema)
doColorCut = True
with ds9.Buffering():
refSchema = matches[0][0].schema
print "-----------------"
print refSchema
print "-----------------"
rRefFluxField = measAlg.getRefFluxField(refSchema, "r")
gRefFluxField = measAlg.getRefFluxField(refSchema, "g")
for ref, source, d in matches:
if not isGoodSource(source):
symb, ctype = "+", ds9.RED
else:
isStar = not ref.get("resolved")
isVar = not ref.get("photometric")
gMag = None
rMag = None
if doColorCut:
try:
gMag = -2.5 * np.log10(ref.get(gRefFluxField))
rMag = -2.5 * np.log10(ref.get(rRefFluxField))
except KeyError:
self.log.warn("Cannot cut on color info; fields 'g' and 'r' do not exist")
doColorCut = False
isRightColor = True
else:
isRightColor = (gMag-rMag) >= self.config.grMin and (gMag-rMag) <= self.config.grMax
isRightType = (self.config.selectStar and isStar) or (self.config.selectGalaxy and not isStar)
isRightVar = (self.config.includeVariable) or (self.config.includeVariable is isVar)
if isRightType and isRightVar and isRightColor:
starCat.append(source)
symb, ctype = "+", ds9.GREEN
else:
symb, ctype = "o", ds9.BLUE
if display and displayExposure:
ds9.dot(symb, source.getX() - mi.getX0(), source.getY() - mi.getY0(),
size=4, ctype=ctype, frame=lsstDebug.frame)
if display:
lsstDebug.frame += 1
if pauseAtEnd:
raw_input("Continue? y[es] p[db] ")
return Struct(
starCat = starCat,
)
def _determineFluxFields(self, center, filterList):
"""Determine the flux field names for a reference catalog.
This method sets self._fluxFields, self._referenceFilter.
Parameters
----------
center : `lsst.geom.SpherePoint`
The center around which to load test sources.
filterList : `list` [`str`]
List of camera physicalFilter names.
"""
# Search for a good filter to use to load the reference catalog
# via the refObjLoader task which requires a valid filterName
foundReferenceFilter = False
# Store the original config
_config = self.refObjLoader.config
configTemp = LoadReferenceObjectsConfig()
configIntersection = {k: getattr(self.refObjLoader.config, k)
for k, v in self.refObjLoader.config.toDict().items()
if (k in configTemp.keys() and k != "connections")}
# We must turn off the proper motion checking to find the refFilter.
configIntersection['requireProperMotion'] = False
configTemp.update(**configIntersection)
self.refObjLoader.config = configTemp
for filterName in filterList:
if self.config.refObjLoader.anyFilterMapsToThis is not None:
refFilterName = self.config.refObjLoader.anyFilterMapsToThis
else:
refFilterName = self.config.refObjLoader.filterMap.get(filterName)
if refFilterName is None:
continue
try:
results = self.refObjLoader.loadSkyCircle(center,
0.05*lsst.geom.degrees,
refFilterName)
foundReferenceFilter = True
self._referenceFilter = refFilterName
break
except RuntimeError as err:
# This just means that the filterName wasn't listed
# in the reference catalog. This is okay.
if 'not find flux' in err.args[0]:
# The filterName wasn't listed in the reference catalog.
# This is not a fatal failure (yet)
pass
else:
raise err
self.refObjLoader.config = _config
if not foundReferenceFilter:
raise RuntimeError("Could not find any valid flux field(s) %s" %
(", ".join(filterList)))
# Record self._fluxFilters for checks on subsequent calls
self._fluxFilters = filterList
# Retrieve all the fluxField names
self._fluxFields = []
for filterName in filterList:
fluxField = None
if self.config.refObjLoader.anyFilterMapsToThis is not None:
refFilterName = self.config.refObjLoader.anyFilterMapsToThis
else:
refFilterName = self.config.refObjLoader.filterMap.get(filterName)
if refFilterName is not None:
try:
fluxField = getRefFluxField(results.refCat.schema, filterName=refFilterName)
except RuntimeError:
# This flux field isn't available. Set to None
fluxField = None
if fluxField is None:
self.log.warning('No reference flux field for camera filter %s', filterName)
self._fluxFields.append(fluxField)
