def run(self,
exposure,
exposureIdInfo=None,
background=None,
icSourceCat=None):
"""Produce calibration outputs with no processing.
Parameters
----------
exposure : `lsst.afw.image.Exposure`
Exposure to calibrate.
exposureIdInfo : `lsst.obs.base.ExposureIdInfo`
ID info for exposure.
background : `lsst.afw.math.BackgroundList`
Background model already subtracted from exposure.
icSourceCat : `lsst.afw.table.SourceCatalog`
A SourceCatalog from CharacterizeImageTask from which we can copy some fields.
Returns
-------
result : `lsst.pipe.base.Struct`
Struct containing these fields:
``outputExposure``
Calibrated science exposure with refined WCS and PhotoCalib
(`lsst.afw.image.Exposure`).
``outputBackground``
Model of background subtracted from exposure
(`lsst.afw.math.BackgroundList`).
``outputCat``
Catalog of measured sources (`lsst.afw.table.SourceCatalog`).
``astromMatches``
List of source/refObj matches from the astrometry solver
(`list` [`lsst.afw.table.ReferenceMatch`]).
"""
# Can't persist empty BackgroundList; DM-33714
bg = afwMath.BackgroundMI(
geom.Box2I(geom.Point2I(0, 0), geom.Point2I(16, 16)),
afwImage.MaskedImageF(16, 16))
return Struct(
outputExposure=exposure,
outputBackground=afwMath.BackgroundList(bg),
outputCat=afwTable.SourceCatalog(),
astromMatches=[],
)
def find(self, objId, matchRadius=10):
"""Return the object's family (you may specify either the ID for the parent or a child)"""
x, y = None, None
try:
x, y = objId
except TypeError:
pass
if x is not None:
oneObjCatalog = afwTable.SourceCatalog(self.cat.getSchema())
centroidName = self.cat.table.getSchema().getAliasMap().get("slot_Centroid")
oneObjCatalog.table.defineCentroid(centroidName)
s = oneObjCatalog.addNew()
s.set("%s_x" % centroidName, x)
s.set("%s_y" % centroidName, y)
matched = afwTable.matchXy(self.cat, oneObjCatalog, matchRadius)
if len(matched) == 0:
print("Unable to find object at (%.2f, %.2f)" % (x, y), file=sys.stderr)
return None
if False:
objId = self.mapperInfo.splitId(matched[0][0].getId(), asDict=True)["objId"]
else:
objId = matched[0][0].getId()
if False:
family = [f for f in self if self.mapperInfo.getId(f[0]) == objId]
else:
family = [f for f in self if f[0].getId() == objId]
if family:
return family[0]
for family in self:
for child in family[1]:
if False:
if self.mapperInfo.getId(child) == objId:
return family
else:
if child.getId() == objId:
return family
return None
def __init__(self, butler=None, refObjLoader=None, schema=None, **kwargs):
"""!Construct a CharacterizeImageTask
@param[in] butler A butler object is passed to the refObjLoader constructor in case
it is needed to load catalogs. May be None if a catalog-based star selector is
not used, if the reference object loader constructor does not require a butler,
or if a reference object loader is passed directly via the refObjLoader argument.
@param[in] refObjLoader An instance of LoadReferenceObjectsTasks that supplies an
external reference catalog to a catalog-based star selector. May be None if a
catalog star selector is not used or the loader can be constructed from the
butler argument.
@param[in,out] schema initial schema (an lsst.afw.table.SourceTable), or None
@param[in,out] kwargs other keyword arguments for lsst.pipe.base.CmdLineTask
"""
super().__init__(**kwargs)
if schema is None:
schema = SourceTable.makeMinimalSchema()
self.schema = schema
self.makeSubtask("background")
self.makeSubtask("installSimplePsf")
self.makeSubtask("repair")
self.makeSubtask("measurePsf", schema=self.schema)
if self.config.doMeasurePsf and self.measurePsf.usesMatches:
if not refObjLoader:
self.makeSubtask('refObjLoader', butler=butler)
refObjLoader = self.refObjLoader
self.makeSubtask("ref_match", refObjLoader=refObjLoader)
self.algMetadata = dafBase.PropertyList()
self.makeSubtask('detection', schema=self.schema)
if self.config.doDeblend:
self.makeSubtask("deblend", schema=self.schema)
self.makeSubtask('measurement',
schema=self.schema,
algMetadata=self.algMetadata)
if self.config.doApCorr:
self.makeSubtask('measureApCorr', schema=self.schema)
self.makeSubtask('applyApCorr', schema=self.schema)
self.makeSubtask('catalogCalculation', schema=self.schema)
if self.config.doComputeSummaryStats:
self.makeSubtask('computeSummaryStats')
self._initialFrame = getDebugFrame(self._display, "frame") or 1
self._frame = self._initialFrame
self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
self.outputSchema = afwTable.SourceCatalog(self.schema)
def makePluginAndCat(alg, name, control=None, metadata=False, centroid=None):
print("Making plugin ", alg, name)
if control == None:
control = alg.ConfigClass()
schema = afwTable.SourceTable.makeMinimalSchema()
if centroid:
schema.addField(centroid + "_x", type=float)
schema.addField(centroid + "_y", type=float)
schema.addField(centroid + "_flag", type='Flag')
schema.getAliasMap().set("slot_Centroid", centroid)
if metadata:
plugin = alg(control, name, schema, dafBase.PropertySet())
else:
plugin = alg(control, name, schema)
cat = afwTable.SourceCatalog(schema)
if centroid:
cat.defineCentroid(centroid)
return plugin, cat
def getReferences(self, dataRef, exposure):
"""Return an iterable of reference sources which overlap the exposure
Copied from forcedPhotCoaddTask, modief to call fetchInPactches from this class
@param dataRef Data reference from butler corresponding to the image to be measured;
should have tract, patch, and filter keys.
@param exposure lsst.afw.image.Exposure to be measured (not used by this implementation)
All work is delegated to the references subtask; see CoaddSrcReferencesTask for information
about the default behavior.
"""
skyMap = dataRef.get(self.dataPrefix + "skyMap", immediate=True)
tractInfo = skyMap[dataRef.dataId["tract"]]
patch = tuple(int(v) for v in dataRef.dataId["patch"].split(","))
patchInfo = tractInfo.getPatchInfo(patch)
references = afwTable.SourceCatalog(self.references.schema)
references.extend(self.fetchInPatches(dataRef, patchList=[patchInfo]))
return references
def run(args):
exposure = loadData(args.image)
if args.debug:
afwDisplay.Display(frame=1).mtv(exposure)
schema = afwTable.SourceTable.makeMinimalSchema()
# Create the detection task
config = SourceDetectionTask.ConfigClass()
config.thresholdPolarity = "both"
config.background.isNanSafe = True
config.thresholdValue = 3
detectionTask = SourceDetectionTask(config=config, schema=schema)
# And the measurement Task
config = DipoleMeasurementTask.ConfigClass()
config.plugins.names.remove('base_SkyCoord')
algMetadata = dafBase.PropertyList()
measureTask = DipoleMeasurementTask(schema, algMetadata, config=config)
# Create the output table
tab = afwTable.SourceTable.make(schema)
# Process the data
results = detectionTask.run(tab, exposure)
# Merge the positve and negative sources
fpSet = results.fpSets.positive
growFootprint = 2
fpSet.merge(results.fpSets.negative, growFootprint, growFootprint, False)
diaSources = afwTable.SourceCatalog(tab)
fpSet.makeSources(diaSources)
print("Merged %s Sources into %d diaSources (from %d +ve, %d -ve)" %
(len(results.sources), len(diaSources), results.fpSets.numPos,
results.fpSets.numNeg))
measureTask.run(diaSources, exposure)
# Display dipoles if debug enabled
if args.debug:
dpa = DipoleAnalysis()
dpa.displayDipoles(exposure, diaSources)
def testUsedFlag(self):
"""Test that the solver will record number of sources used to table
if it is passed a schema on initialization.
"""
self.exposure.setWcs(self.tanWcs)
loadRes = self.refObjLoader.loadPixelBox(bbox=self.bbox,
wcs=self.tanWcs,
filterName="r")
refCat = loadRes.refCat
refCentroidKey = afwTable.Point2DKey(refCat.schema["centroid"])
refFluxRKey = refCat.schema["r_flux"].asKey()
sourceSchema = afwTable.SourceTable.makeMinimalSchema()
measBase.SingleFrameMeasurementTask(
schema=sourceSchema) # expand the schema
config = AstrometryTask.ConfigClass()
config.wcsFitter.order = 2
config.wcsFitter.numRejIter = 0
# schema must be passed to the solver task constructor
solver = AstrometryTask(config=config,
refObjLoader=self.refObjLoader,
schema=sourceSchema)
sourceCat = afwTable.SourceCatalog(sourceSchema)
sourceCat.reserve(len(refCat))
sourceCentroidKey = afwTable.Point2DKey(sourceSchema["slot_Centroid"])
sourceInstFluxKey = sourceSchema["slot_ApFlux_instFlux"].asKey()
sourceInstFluxErrKey = sourceSchema["slot_ApFlux_instFluxErr"].asKey()
for refObj in refCat:
src = sourceCat.addNew()
src.set(sourceCentroidKey, refObj.get(refCentroidKey))
src.set(sourceInstFluxKey, refObj.get(refFluxRKey))
src.set(sourceInstFluxErrKey, refObj.get(refFluxRKey) / 100)
results = solver.run(
sourceCat=sourceCat,
exposure=self.exposure,
)
# check that the used flag is set the right number of times
count = 0
for source in sourceCat:
if source.get('calib_astrometry_used'):
count += 1
self.assertEqual(count, len(results.matches))
def makeSourceCatalog(self,
table,
exposure,
doSmooth=True,
sigma=None,
clearMask=True):
if self.negativeFlagKey is not None and self.negativeFlagKey not in table.getSchema(
):
raise ValueError("Table has incorrect Schema")
# detect the footprints as usual
fpSets = self.detectFootprints(exposure=exposure,
doSmooth=doSmooth,
sigma=sigma,
clearMask=clearMask)
#ignore objects whose footprints do NOT overlap with the 'FAKE' mask
mask = exposure.getMaskedImage().getMask()
fakebit = mask.getPlaneBitMask('FAKE')
fpPos = fpSets.positive.getFootprints()
removes = []
for i_foot, foot in enumerate(fpPos):
footTmp = afwDetect.Footprint(foot)
footTmp.intersectMask(mask, fakebit)
if footTmp.getArea() == foot.getArea():
removes.append(i_foot)
removes = sorted(removes, reverse=True)
for r in removes:
del fpPos[r]
self.log.info("Found %d sources near fake footprints" % len(fpPos))
fpSets.numPos = len(fpPos)
if fpSets.negative:
del fpSets.negative.getFootprints()[0:]
fpSets.negative = None
# make sources
sources = afwTable.SourceCatalog(table)
table.preallocate(fpSets.numPos)
if fpSets.positive:
fpSets.positive.makeSources(sources)
return pipeBase.Struct(sources=sources, fpSets=fpSets)
def testPsfFlux(self):
"""Test that fluxes are measured correctly."""
#
# Total flux in image
#
flux = afwMath.makeStatistics(self.exp.getMaskedImage(),
afwMath.SUM).getValue()
self.assertAlmostEqual(flux / self.instFlux, 1.0)
#
# Various algorithms
#
rad = 10.0
schema = afwTable.SourceTable.makeMinimalSchema()
schema.addField("centroid_x", type=float)
schema.addField("centroid_y", type=float)
schema.addField("centroid_flag", type='Flag')
sfm_config = measBase.SingleFrameMeasurementConfig()
sfm_config.doReplaceWithNoise = False
sfm_config.plugins = ["base_CircularApertureFlux", "base_PsfFlux"]
sfm_config.slots.centroid = "centroid"
sfm_config.slots.shape = None
sfm_config.slots.psfFlux = None
sfm_config.slots.gaussianFlux = None
sfm_config.slots.apFlux = None
sfm_config.slots.modelFlux = None
sfm_config.slots.calibFlux = None
sfm_config.plugins["base_SdssShape"].maxShift = 10.0
sfm_config.plugins["base_CircularApertureFlux"].radii = [rad]
task = measBase.SingleFrameMeasurementTask(schema, config=sfm_config)
measCat = afwTable.SourceCatalog(schema)
source = measCat.addNew()
source.set("centroid_x", self.xc)
source.set("centroid_y", self.yc)
task.run(measCat, self.exp)
for algName in ["base_CircularApertureFlux_10_0", "base_PsfFlux"]:
instFlux = source.get(algName + "_instFlux")
flag = source.get(algName + "_flag")
self.assertEqual(flag, False)
self.assertAlmostEqual(
instFlux / self.instFlux, 1.0, 4,
"Measuring with %s: %g v. %g" %
(algName, instFlux, self.instFlux))
def detectDipoleSources(self, doMerge=True, diffim=None, detectSigma=5.5, grow=3):
"""!Utility function for detecting dipoles.
Detect pos/neg sources in the diffim, then merge them. A
bigger "grow" parameter leads to a larger footprint which
helps with dipole measurement for faint dipoles.
"""
if diffim is None:
diffim = self.diffim
# Start with a minimal schema - only the fields all SourceCatalogs need
schema = afwTable.SourceTable.makeMinimalSchema()
# Customize the detection task a bit (optional)
detectConfig = measAlg.SourceDetectionConfig()
detectConfig.returnOriginalFootprints = False # should be the default
psfSigma = diffim.getPsf().computeShape().getDeterminantRadius()
# code from imageDifference.py:
detectConfig.thresholdPolarity = "both"
detectConfig.thresholdValue = detectSigma
# detectConfig.nSigmaToGrow = psfSigma
detectConfig.reEstimateBackground = True # if False, will fail often for faint sources on gradients?
detectConfig.thresholdType = "pixel_stdev"
# Create the detection task. We pass the schema so the task can declare a few flag fields
detectTask = measAlg.SourceDetectionTask(schema, config=detectConfig)
table = afwTable.SourceTable.make(schema)
catalog = detectTask.makeSourceCatalog(table, diffim, sigma=psfSigma)
# Now do the merge.
if doMerge:
fpSet = catalog.fpSets.positive
fpSet.merge(catalog.fpSets.negative, grow, grow, False)
sources = afwTable.SourceCatalog(table)
fpSet.makeSources(sources)
return sources
else:
return detectTask, schema
def testMeasureCentroid(self):
"""Test that we can use our silly centroid through the usual Tasks.
"""
testLib.SillyCentroidControl()
x, y = 10, 20
im = afwImage.MaskedImageF(lsst.geom.ExtentI(512, 512))
im.set(0)
arr = im.getImage().getArray()
arr[y, x] = 1000
exp = afwImage.makeExposure(im)
schema = afwTable.SourceTable.makeMinimalSchema()
schema.addField(
"flags_negative",
type="Flag",
doc="set if source was detected as significantly negative")
sfm_config = lsst.meas.base.sfm.SingleFrameMeasurementConfig()
sfm_config.plugins = ["testLib_SillyCentroid"]
sfm_config.plugins["testLib_SillyCentroid"].param = 5
sfm_config.slots.centroid = "testLib_SillyCentroid"
sfm_config.slots.shape = None
sfm_config.slots.psfShape = None
sfm_config.slots.psfFlux = None
sfm_config.slots.gaussianFlux = None
sfm_config.slots.calibFlux = None
sfm_config.slots.apFlux = None
sfm_config.slots.modelFlux = None
sfm_config.doReplaceWithNoise = False
task = lsst.meas.base.SingleFrameMeasurementTask(schema,
config=sfm_config)
measCat = afwTable.SourceCatalog(schema)
measCat.defineCentroid("testLib_SillyCentroid")
source = measCat.addNew()
source.set("testLib_SillyCentroid_x", x)
source.set("testLib_SillyCentroid_y", y)
source.set("parent", 0)
source.set("flags_negative", False)
# now run the SFM task with the test plugin
task.run(measCat, exp)
self.assertEqual(len(measCat), 1)
self.assertEqual(measCat[0].getY(), y + 5)
self.assertEqual(measCat[0].getX(), x + 5)
def _convertResult(self, res, table_name, catalog=None):
"""Convert result set into output catalog.
Parameters
----------
res : `sqlalchemy.ResultProxy`
SQLAlchemy result set returned by query.
table_name : `str`
Name of the table.
catalog : `lsst.afw.table.BaseCatalog`
If not None then extend existing catalog
Returns
-------
catalog : `lsst.afw.table.SourceCatalog`
If ``catalog`` is None then new instance is returned, otherwise
``catalog`` is updated and returned.
"""
# make catalog schema
columns = res.keys()
schema, col_map = self._schema.getAfwSchema(table_name, columns)
if catalog is None:
_LOG.debug("_convertResult: schema: %s", schema)
_LOG.debug("_convertResult: col_map: %s", col_map)
catalog = afwTable.SourceCatalog(schema)
# fill catalog
for row in res:
record = catalog.addNew()
for col, value in row.items():
# some columns may exist in database but not included in afw schema
col = col_map.get(col)
if col is not None:
if isinstance(value, datetime):
# convert datetime to number of seconds
value = int(
(value -
datetime.utcfromtimestamp(0)).total_seconds())
elif col.getTypeString() == 'Angle' and value is not None:
value = value * geom.degrees
if value is not None:
record.set(col, value)
return catalog
def _runDetection(self, params):
"""!Run 'diaSource' detection on the diffim, including merging of
positive and negative sources.
Then run DipoleFitTask on the image and return the resulting catalog.
"""
# Create the various tasks and schema -- avoid code reuse.
testImage = params.testImage
detectTask, schema = testImage.detectDipoleSources(doMerge=False,
minBinSize=32)
measureConfig = measBase.SingleFrameMeasurementConfig()
measureConfig.slots.calibFlux = None
measureConfig.slots.modelFlux = None
measureConfig.slots.gaussianFlux = None
measureConfig.slots.shape = None
measureConfig.slots.centroid = "ip_diffim_NaiveDipoleCentroid"
measureConfig.doReplaceWithNoise = False
measureConfig.plugins.names = [
"base_CircularApertureFlux", "base_PixelFlags", "base_SkyCoord",
"base_PsfFlux", "ip_diffim_NaiveDipoleCentroid",
"ip_diffim_NaiveDipoleFlux", "ip_diffim_PsfDipoleFlux"
]
# Here is where we make the dipole fitting task. It can run the other measurements as well.
# This is an example of how to pass it a custom config.
measureTask = DipoleFitTask(config=measureConfig, schema=schema)
table = afwTable.SourceTable.make(schema)
detectResult = detectTask.run(table, testImage.diffim)
# catalog = detectResult.sources
# deblendTask.run(self.dipole, catalog, psf=self.dipole.getPsf())
fpSet = detectResult.fpSets.positive
fpSet.merge(detectResult.fpSets.negative, 2, 2, False)
sources = afwTable.SourceCatalog(table)
fpSet.makeSources(sources)
measureTask.run(sources, testImage.diffim, testImage.posImage,
testImage.negImage)
return sources
def testClusterEdgeCases(self):
cfg = cluster.ClusteringConfig()
st = table.SourceTable.make(table.SourceTable.makeMinimalSchema())
cat = table.SourceCatalog(st)
cfg.epsilonArcsec = -1.0
cfg.minNeighbors = 0
cfg.pointsPerLeaf = 4
cfg.leafExtentThresholdArcsec = 7200.0
self.assertRaises(ex.Exception, cluster.cluster, cat,
cfg.makeControl())
cfg.epsilonArcsec = 0.0
cfg.minNeighbors = -1
self.assertRaises(ex.Exception, cluster.cluster, cat,
cfg.makeControl())
cfg.minNeighbors = 0
cfg.pointsPerLeaf = 0
self.assertRaises(ex.Exception, cluster.cluster, cat,
cfg.makeControl())
cfg.pointsPerLeaf = 4
s = cat.addNew()
s.setRa(0.0 * afwGeom.radians)
s.setDec(0.0 * afwGeom.radians)
c = cluster.cluster(cat, cfg.makeControl())
self.assertEqual(len(c), 1)
cfg.minNeighbors = 2
c = cluster.cluster(cat, cfg.makeControl())
self.assertEqual(countClusters(c), 0)
cfg.epsilonArcsec = 3600.0 # 1-degree clustering distance
s = cat.addNew()
s.setRa(0.5 * afwGeom.degrees)
s.setDec(0. * afwGeom.degrees)
s = cat.addNew()
s.setRa(0. * afwGeom.degrees)
s.setDec(0.5 * afwGeom.degrees)
c = cluster.cluster(cat, cfg.makeControl())
self.assertEqual(countClusters(c), 1)
cfg.minNeighbors = 3
c = cluster.cluster(cat, cfg.makeControl())
self.assertEqual(countClusters(c), 0)
cfg.minNeighbors = 0
cfg.epsilonArcsec = 1.0 # 1 arcsec clustering distance
c = cluster.cluster(cat, cfg.makeControl())
self.assertEqual(len(c), 3)
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.schema = afwTable.SourceTable.makeMinimalSchema()
self.algMetadata = dafBase.PropertyList()
self.makeSubtask("detection", schema=self.schema)
self.makeSubtask("measurement", schema=self.schema,
algMetadata=self.algMetadata)
if self.config.doApCorr:
self.makeSubtask("applyApCorr", schema=self.measurement.schema)
if self.config.doForcedMeasurement:
self.schema.addField(
"ip_diffim_forced_PsfFlux_instFlux", "D",
"Forced PSF flux measured on the direct image.",
units="count")
self.schema.addField(
"ip_diffim_forced_PsfFlux_instFluxErr", "D",
"Forced PSF flux error measured on the direct image.",
units="count")
self.schema.addField(
"ip_diffim_forced_PsfFlux_area", "F",
"Forced PSF flux effective area of PSF.",
units="pixel")
self.schema.addField(
"ip_diffim_forced_PsfFlux_flag", "Flag",
"Forced PSF flux general failure flag.")
self.schema.addField(
"ip_diffim_forced_PsfFlux_flag_noGoodPixels", "Flag",
"Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
self.schema.addField(
"ip_diffim_forced_PsfFlux_flag_edge", "Flag",
"Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
self.makeSubtask("forcedMeasurement", refSchema=self.schema)
self.schema.addField("refMatchId", "L", "unique id of reference catalog match")
self.schema.addField("srcMatchId", "L", "unique id of source match")
if self.config.doSkySources:
self.makeSubtask("skySources")
self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.")
# initialize InitOutputs
self.outputSchema = afwTable.SourceCatalog(self.schema)
self.outputSchema.getTable().setMetadata(self.algMetadata)
def testMinusOne(self):
schema = afwTable.SourceTable.makeMinimalSchema()
table = afwTable.SourceTable.make(schema)
catalog = afwTable.SourceCatalog(table)
catalog.addNew()
self.assertEqual(len(catalog), 1)
catalog[-1]
catalog.addNew()
catalog.addNew()
catalog[1] = catalog[2]
del catalog[2]
print catalog
for src in catalog:
print src.getId()
self.assertEqual(len(catalog), 2)
self.assertEqual(catalog[0].getId(), 1)
self.assertEqual(catalog[1].getId(), 3)
self.assertEqual(catalog[-1].getId(), 3)
self.assertEqual(catalog[-2].getId(), 1)
def testArgumentErrors(self):
"""Test argument sanity checking in matchOptimisticB
"""
matchControl = matchOptimisticB.MatchOptimisticBControl()
sourceCat = self.loadSourceCatalog(self.filename)
emptySourceCat = afwTable.SourceCatalog(sourceCat.schema)
refCat = self.computePosRefCatalog(sourceCat)
emptyRefCat = afwTable.SimpleCatalog(refCat.schema)
with self.assertRaises(pexExcept.InvalidParameterError):
matchOptimisticB.matchOptimisticB(
emptyRefCat,
sourceCat,
matchControl,
self.wcs,
0,
)
with self.assertRaises(pexExcept.InvalidParameterError):
matchOptimisticB.matchOptimisticB(
refCat,
emptySourceCat,
matchControl,
self.wcs,
0,
)
with self.assertRaises(pexExcept.InvalidParameterError):
matchOptimisticB.matchOptimisticB(
refCat,
sourceCat,
matchControl,
self.wcs,
len(refCat),
)
with self.assertRaises(pexExcept.InvalidParameterError):
matchOptimisticB.matchOptimisticB(
refCat,
sourceCat,
matchControl,
self.wcs,
-1,
)
def testFlags(self):
"""test that all the calib_photometry flags are set to reasonable values"""
schema = self.srcCat.schema
task = PhotoCalTask(self.refObjLoader, config=self.config, schema=schema)
mapper = afwTable.SchemaMapper(self.srcCat.schema, schema)
cat = afwTable.SourceCatalog(schema)
for name in self.srcCat.schema.getNames():
mapper.addMapping(self.srcCat.schema.find(name).key)
cat.extend(self.srcCat, mapper=mapper)
# test that by default, no stars are reserved and all used are candidates
task.run(exposure=self.exposure, sourceCat=cat)
used = 0
for source in cat:
if source.get("calib_photometry_used"):
used += 1
self.assertFalse(source.get("calib_photometry_reserved"))
# test that some are actually used
self.assertGreater(used, 0)
def testFlags(self):
"""test that all the calib_photometry flags are set to reasonable values"""
schema = self.srcCat.schema
task = PhotoCalTask(self.refObjLoader,
config=self.config,
schema=schema)
mapper = afwTable.SchemaMapper(self.srcCat.schema, schema)
cat = afwTable.SourceCatalog(schema)
for name in self.srcCat.schema.getNames():
mapper.addMapping(self.srcCat.schema.find(name).key)
cat.extend(self.srcCat, mapper=mapper)
# test that by default, no stars are reserved and used < candidates
task.run(exposure=self.exposure, sourceCat=cat)
candidates = 0
used = 0
reserved = 0
for source in cat:
if source.get("calib_photometryCandidate"):
candidates += 1
if source.get("calib_photometryUsed"):
used += 1
if source.get("calib_photometryReserved"):
reserved += 1
self.assertLessEqual(used, candidates)
self.assertEqual(reserved, 0)
# set the reserve fraction, and see if the right proportion are reserved.
self.config.reserveFraction = .3
task.run(exposure=self.exposure, sourceCat=cat)
candidates = 0
reserved = 0
used = 0
for source in cat:
if source.get("calib_photometryCandidate"):
candidates += 1
if source.get("calib_photometryUsed"):
used += 1
if source.get("calib_photometryReserved"):
reserved += 1
self.assertEqual(reserved, int(.3 * candidates))
self.assertLessEqual(used, (candidates - reserved))
def setUp(self):
# Create a schema object, and populate it with a field to simulate results from measurements on an
# image
schema = afwTable.SourceTable.makeMinimalSchema()
startKey = schema.addField("start", type="D")
# Instantiate a config object adding each of the above plugins, and use it to create a task
abConfig = afterBurner.AfterburnerConfig()
abConfig.plugins.names = [
"afterburnerFail", "singleRecordAfterburner",
"multiRecordAfterburner", "dependentAfterburner"
]
abTask = afterBurner.AfterburnerTask(schema=schema, config=abConfig)
# Create a catalog with five sources as input to the task
self.catalog = afwTable.SourceCatalog(schema)
self.numObjects = 5
for i in range(self.numObjects):
rec = self.catalog.addNew()
rec.set("start", float(i + 1))
# Run the afterburner task, outputs will be checked in test methods
abTask.run(self.catalog)
def runMeasure(task, schema, exposure):
"""
Run a measurement task which has previously been initialized on a single source
"""
cat = afwTable.SourceCatalog(schema)
source = cat.addNew()
dettask = measAlg.SourceDetectionTask()
# Suppress non-essential task output.
dettask.log.setLevel(dettask.log.WARN)
# We are expecting this task to log an error. Suppress it, so that it
# doesn't appear on the console or in logs, and incorrectly cause the user
# to assume a failure.
task.log.setLevel(task.log.FATAL)
footprints = dettask.detectFootprints(exposure,
sigma=4.0).positive.getFootprints()
source.setFootprint(footprints[0])
task.run(cat, exposure)
return source
def getMergedSourceCatalog(self, catalogs, filters, peakDist, schema,
idFactory, samePeakDist):
"""Add multiple catalogs and get the SourceCatalog with merged Footprints"""
import lsst.afw.table as afwTable
table = afwTable.SourceTable.make(schema, idFactory)
mergedList = afwTable.SourceCatalog(table)
# if peak is not an array, create an array the size of catalogs
try:
len(samePeakDist)
except TypeError:
samePeakDist = [samePeakDist] * len(catalogs)
try:
len(peakDist)
except TypeError:
peakDist = [peakDist] * len(catalogs)
if len(peakDist) != len(catalogs):
raise ValueError(
"Number of catalogs (%d) does not match length of peakDist (%d)"
% (len(catalogs), len(peakDist)))
if len(samePeakDist) != len(catalogs):
raise ValueError(
"Number of catalogs (%d) does not match length of samePeakDist (%d)"
% (len(catalogs), len(samePeakDist)))
if len(filters) != len(catalogs):
raise ValueError(
"Number of catalogs (%d) does not match number of filters (%d)"
% (len(catalogs), len(filters)))
self.clearCatalog()
for cat, filter, dist, sameDist in zip(catalogs, filters, peakDist,
samePeakDist):
self.addCatalog(table, cat, filter, dist, True, sameDist)
self.getFinalSources(mergedList)
return mergedList
def make_input_source_catalog(n_objects, add_flags=False):
"""Create tests objects to map into apData products.
Parameters
----------
n_objects: `int`
Number of objects to create.
"""
schema = afwTable.SourceTable.makeMinimalSchema()
schema.addField("base_NaiveCentroid_x", type="D")
schema.addField("base_NaiveCentroid_y", type="D")
schema.addField("base_PsfFlux_instFlux", type="D")
schema.addField("base_PsfFlux_instFluxErr", type="D")
schema.addField("ip_diffim_DipoleFit_separation", type="D")
schema.addField("ip_diffim_DipoleFit_orientation", type="D")
schema.addField("ip_diffim_DipoleFit_neg_instFlux", type="D")
schema.addField("ip_diffim_DipoleFit_neg_instFluxErr", type="D")
schema.addField("ip_diffim_DipoleFit_pos_instFlux", type="D")
schema.addField("ip_diffim_DipoleFit_pos_instFluxErr", type="D")
schema.addField("ip_diffim_forced_PsfFlux_instFlux", type="D")
schema.addField("ip_diffim_forced_PsfFlux_instFluxErr", type="D")
if add_flags:
schema.addField("base_PixelFlags_flag", type="Flag")
schema.addField("base_PixelFlags_flag_offimage", type="Flag")
objects = afwTable.SourceCatalog(schema)
objects.preallocate(n_objects)
objects.definePsfFlux("base_PsfFlux")
objects.defineCentroid("base_NaiveCentroid")
for obj_idx in range(n_objects):
obj = objects.addNew()
for subSchema in schema:
if isinstance(obj.get(subSchema.getKey()), geom.Angle):
obj.set(subSchema.getKey(), 1. * geom.degrees)
elif subSchema.getField().getName(
) == "ip_diffim_DipoleFit_neg_instFlux":
obj.set(subSchema.getKey(), -1)
else:
obj.set(subSchema.getKey(), 1)
return objects
def makeSourceCatalog(self,
table,
exposure,
doSmooth=True,
sigma=None,
clearMask=True):
"""Run source detection and create a SourceCatalog.
To avoid dealing with sources and tables, use detectFootprints() to just get the FootprintSets.
@param table lsst.afw.table.SourceTable object that will be used to created the SourceCatalog.
@param exposure Exposure to process; DETECTED mask plane will be set in-place.
@param doSmooth if True, smooth the image before detection using a Gaussian of width sigma
@param sigma sigma of PSF (pixels); used for smoothing and to grow detections;
if None then measure the sigma of the PSF of the exposure
@param clearMask Clear DETECTED{,_NEGATIVE} planes before running detection
@return a Struct with:
sources -- an lsst.afw.table.SourceCatalog object
fpSets --- Struct returned by detectFootprints
@raise pipe_base TaskError if sigma=None, doSmooth=True and the exposure has no PSF
"""
if self.negativeFlagKey is not None and self.negativeFlagKey not in table.getSchema(
):
raise ValueError("Table has incorrect Schema")
fpSets = self.detectFootprints(exposure=exposure,
doSmooth=doSmooth,
sigma=sigma,
clearMask=clearMask)
sources = afwTable.SourceCatalog(table)
table.preallocate(fpSets.numPos +
fpSets.numNeg) # not required, but nice
if fpSets.negative:
fpSets.negative.makeSources(sources)
if self.negativeFlagKey:
for record in sources:
record.set(self.negativeFlagKey, True)
if fpSets.positive:
fpSets.positive.makeSources(sources)
return pipeBase.Struct(sources=sources, fpSets=fpSets)
def __init__(self):
self.schema = afwTab.SourceTable.makeMinimalSchema()
setMethods = [x for x in qaDataUtils.getSourceSetAccessors()]
#self.schema.addField('Id', type="L")
self.setKeys = []
self.setNames = []
self.keyDict = {}
for sm in setMethods:
if sm == 'Id':
continue
key = self.schema.addField(sm, type="D")
self.setKeys.append(key)
self.setNames.append(sm)
self.keyDict[sm] = key
setattr(self, sm+"Key", key)
self.table = afwTab.SourceTable.make(self.schema)
self.catalog = afwTab.SourceCatalog(self.table)
def mySetup(self):
msConfig = measBase.SingleFrameMeasurementConfig()
msConfig.algorithms.names = ["base_SdssCentroid"]
msConfig.slots.centroid = "base_SdssCentroid"
msConfig.slots.shape = None
msConfig.slots.apFlux = None
msConfig.slots.modelFlux = None
msConfig.slots.psfFlux = None
msConfig.slots.instFlux = None
msConfig.slots.calibFlux = None
schema = afwTable.SourceTable.makeMinimalSchema()
task = measBase.SingleFrameMeasurementTask(schema, config=msConfig)
measCat = afwTable.SourceCatalog(schema)
source = measCat.addNew()
fp = afwDetection.Footprint(self.exp.getBBox(afwImage.LOCAL))
fp.addPeak(50, 50, 1000.0)
source.setFootprint(fp)
# Then run the default SFM task. Results not checked
task.run(measCat, self.exp)
return source
def run(self, inputCatalog, exposure=None):
"""Copy data from the inputCatalog into an output catalog with
requested columns.
Parameters
----------
inputCatalog: `lsst.afw.table.SourceCatalog`
Input catalog with data to be copied into new output catalog.
Returns
-------
outputCatalog: `lsst.afw.table.SourceCatalog`
Output catalog with data copied from input and new column names.
"""
outputCatalog = afwTable.SourceCatalog(self.outputSchema)
outputCatalog.extend(inputCatalog, self.mapper)
if not outputCatalog.isContiguous():
raise RuntimeError("Output catalogs must be contiguous.")
return outputCatalog
def setUp(self):
# Create a schema object, and populate it with a field to simulate results from measurements on an
# image
schema = afwTable.SourceTable.makeMinimalSchema()
schema.addField("start", type="D")
# Instantiate a config object adding each of the above plugins, and use it to create a task
catCalcConfig = catCalc.CatalogCalculationConfig()
catCalcConfig.plugins.names = [
"FailcatalogCalculation", "singleRecordCatalogCalculation",
"multiRecordCatalogCalculation", "dependentCatalogCalulation"
]
catCalcTask = catCalc.CatalogCalculationTask(schema=schema,
config=catCalcConfig)
# Create a catalog with five sources as input to the task
self.catalog = afwTable.SourceCatalog(schema)
self.numObjects = 5
for i in range(self.numObjects):
rec = self.catalog.addNew()
rec.set("start", float(i + 1))
# Run the catalogCalculation task, outputs will be checked in test methods
catCalcTask.run(self.catalog)
def makeSourceCat(self, distortedWcs):
"""Make a source catalog by reading the position reference stars and distorting the positions
"""
loadRes = self.refObjLoader.loadPixelBox(bbox=self.bbox, wcs=distortedWcs, filterName="r")
refCat = loadRes.refCat
refCentroidKey = afwTable.Point2DKey(refCat.schema["centroid"])
refFluxRKey = refCat.schema["r_flux"].asKey()
sourceSchema = self.makeSourceSchema()
sourceCat = afwTable.SourceCatalog(sourceSchema)
sourceCentroidKey = afwTable.Point2DKey(sourceSchema["slot_Centroid"])
sourceFluxKey = sourceSchema["slot_PsfFlux_instFlux"].asKey()
sourceFluxErrKey = sourceSchema["slot_PsfFlux_instFluxErr"].asKey()
sourceCat.reserve(len(refCat))
for refObj in refCat:
src = sourceCat.addNew()
src.set(sourceCentroidKey, refObj.get(refCentroidKey))
src.set(sourceFluxKey, refObj.get(refFluxRKey))
src.set(sourceFluxErrKey, refObj.get(refFluxRKey)/100)
return sourceCat
def run(self, table, exposure, doSmooth=True, sigma=None, clearMask=True):
"""!Run source detection and create a SourceCatalog.
\param table lsst.afw.table.SourceTable object that will be used to create the SourceCatalog.
\param exposure Exposure to process; DETECTED mask plane will be set in-place.
\param doSmooth if True, smooth the image before detection using a Gaussian of width sigma
(default: True)
\param sigma sigma of PSF (pixels); used for smoothing and to grow detections;
if None then measure the sigma of the PSF of the exposure (default: None)
\param clearMask Clear DETECTED{,_NEGATIVE} planes before running detection (default: True)
\return a lsst.pipe.base.Struct with:
- sources -- an lsst.afw.table.SourceCatalog object
- fpSets --- lsst.pipe.base.Struct returned by \link detectFootprints \endlink
\throws ValueError if flags.negative is needed, but isn't in table's schema
\throws lsst.pipe.base.TaskError if sigma=None, doSmooth=True and the exposure has no PSF
\note
If you want to avoid dealing with Sources and Tables, you can use detectFootprints()
to just get the afw::detection::FootprintSet%s.
"""
if self.negativeFlagKey is not None and self.negativeFlagKey not in table.getSchema(
):
raise ValueError("Table has incorrect Schema")
fpSets = self.detectFootprints(exposure=exposure,
doSmooth=doSmooth,
sigma=sigma,
clearMask=clearMask)
sources = afwTable.SourceCatalog(table)
table.preallocate(fpSets.numPos +
fpSets.numNeg) # not required, but nice
if fpSets.negative:
fpSets.negative.makeSources(sources)
if self.negativeFlagKey:
for record in sources:
record.set(self.negativeFlagKey, True)
if fpSets.positive:
fpSets.positive.makeSources(sources)
return pipeBase.Struct(sources=sources, fpSets=fpSets)
