def __init__(self, schema, peakSchema=None, **kwargs):
"""!
Create the task, adding necessary fields to the given schema.
@param[in,out] schema Schema object for measurement fields; will be modified in-place.
@param[in] peakSchema Schema of Footprint Peaks that will be passed to the deblender.
Any fields beyond the PeakTable minimal schema will be transferred
to the main source Schema. If None, no fields will be transferred
from the Peaks.
@param[in] **kwargs Passed to Task.__init__.
"""
pipeBase.Task.__init__(self, **kwargs)
peakMinimalSchema = afwDet.PeakTable.makeMinimalSchema()
if peakSchema is None:
# In this case, the peakSchemaMapper will transfer nothing, but we'll still have one
# to simplify downstream code
self.peakSchemaMapper = afwTable.SchemaMapper(
peakMinimalSchema, schema)
else:
self.peakSchemaMapper = afwTable.SchemaMapper(peakSchema, schema)
for item in peakSchema:
if item.key not in peakMinimalSchema:
self.peakSchemaMapper.addMapping(item.key, item.field)
# Because SchemaMapper makes a copy of the output schema you give its ctor, it isn't
# updating this Schema in place. That's probably a design flaw, but in the meantime,
# we'll keep that schema in sync with the peakSchemaMapper.getOutputSchema() manually,
# by adding the same fields to both.
schema.addField(item.field)
assert schema == self.peakSchemaMapper.getOutputSchema(
), "Logic bug mapping schemas"
self.addSchemaKeys(schema)
def addCalibColumns(self, catalog, dataRef):
"""Add columns with local calibration evaluated at each centroid
for backwards compatibility with old repos.
This exists for the purpose of converting old src catalogs
(which don't have the expected local calib columns) to Source Tables.
Parameters
----------
catalog: `afwTable.SourceCatalog`
catalog to which calib columns will be added
dataRef: `lsst.daf.persistence.ButlerDataRef
for fetching the calibs from disk.
Returns
-------
newCat: `afwTable.SourceCatalog`
Source Catalog with requested local calib columns
"""
mapper = afwTable.SchemaMapper(catalog.schema)
measureConfig = SingleFrameMeasurementTask.ConfigClass()
measureConfig.doReplaceWithNoise = False
# Just need the WCS or the PhotoCalib attached to an exposue
exposure = dataRef.get('calexp_sub',
bbox=lsst.geom.Box2I(lsst.geom.Point2I(0, 0),
lsst.geom.Point2I(0, 0)))
mapper = afwTable.SchemaMapper(catalog.schema)
mapper.addMinimalSchema(catalog.schema, True)
schema = mapper.getOutputSchema()
exposureIdInfo = dataRef.get("expIdInfo")
measureConfig.plugins.names = []
if self.config.doApplyExternalSkyWcs:
plugin = 'base_LocalWcs'
if plugin in schema:
raise RuntimeError(
f"{plugin} already in src catalog. Set doApplyExternalSkyWcs=False"
)
else:
measureConfig.plugins.names.add(plugin)
if self.config.doApplyExternalPhotoCalib:
plugin = 'base_LocalPhotoCalib'
if plugin in schema:
raise RuntimeError(
f"{plugin} already in src catalog. Set doApplyExternalPhotoCalib=False"
)
else:
measureConfig.plugins.names.add(plugin)
measurement = SingleFrameMeasurementTask(config=measureConfig,
schema=schema)
newCat = afwTable.SourceCatalog(schema)
newCat.extend(catalog, mapper=mapper)
measurement.run(measCat=newCat,
exposure=exposure,
exposureId=exposureIdInfo.expId)
return newCat
def measureForcedSources(self, diaSources, science, wcs):
"""Perform forced measurement of the diaSources on the science image.
Parameters
----------
diaSources : `lsst.afw.table.SourceCatalog`
The catalog of detected sources.
science : `lsst.afw.image.ExposureF`
Science exposure that the template was subtracted from.
wcs : `lsst.afw.geom.SkyWcs`
Coordinate system definition (wcs) for the exposure.
"""
# Run forced psf photometry on the PVI at the diaSource locations.
# Copy the measured flux and error into the diaSource.
forcedSources = self.forcedMeasurement.generateMeasCat(
science, diaSources, wcs)
self.forcedMeasurement.run(forcedSources, science, diaSources, wcs)
mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFlux")[0],
"ip_diffim_forced_PsfFlux_instFlux", True)
mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFluxErr")[0],
"ip_diffim_forced_PsfFlux_instFluxErr", True)
mapper.addMapping(forcedSources.schema.find("base_PsfFlux_area")[0],
"ip_diffim_forced_PsfFlux_area", True)
mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag")[0],
"ip_diffim_forced_PsfFlux_flag", True)
mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_noGoodPixels")[0],
"ip_diffim_forced_PsfFlux_flag_noGoodPixels", True)
mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_edge")[0],
"ip_diffim_forced_PsfFlux_flag_edge", True)
for diaSource, forcedSource in zip(diaSources, forcedSources):
diaSource.assign(forcedSource, mapper)
def make_refcat(ra, dec):
"""
Make a reference catalog for forced photometry
Parameters:
-----------
ra : sequence or array
Right Ascension in decimal degrees
dec : sequence or array
Declination in decimal degrees
Returns:
--------
src_cat : lsst.afw.table.tableLib.SourceCatalog
Source catalog for the forced photometry task
"""
schema = out_butler.get('src_schema', immediate=True).schema
mapper = afwTable.SchemaMapper(schema)
mapper.addMinimalSchema(schema)
newSchema = mapper.getOutputSchema()
src_cat = afwTable.SourceCatalog(newSchema)
for row in zip(ra, dec):
record = src_cat.addNew()
record.set('coord_ra', Angle(row[0] * degrees))
record.set('coord_dec', Angle(row[1] * degrees))
return (src_cat)
def run(self, dataRef):
calexp = dataRef.get("calexp")
psf = calexp.getPsf()
sources = dataRef.get("src")
mapper = afwTable.SchemaMapper(sources.getSchema())
# map all the existing fields
mapper.addMinimalSchema(sources.getSchema(), True)
schema = mapper.getOutputSchema()
# It would be better for the schema-populating code to not be in
# the SourceDeblendTask constructor!
self.makeSubtask("deblend", schema=schema)
outsources = afwTable.SourceCatalog(schema)
outsources.reserve(2 * len(sources))
outsources.extend(sources, mapper=mapper)
sources = outsources
print(len(sources), 'sources before deblending')
self.deblend.run(calexp, sources)
print(len(sources), 'sources after deblending')
fn = 'deblended.fits'
print('Writing sources...')
sources.writeFits(fn)
print('Wrote sources to', fn)
fn = 'calexp.fits'
calexp.writeFits(fn)
print('Wrote calexp to', fn)
fn = 'psf.fits'
psf.writeFits(fn)
print('Wrote PSF to', fn)
def __init__(self, initInputs, **kwargs):
super().__init__(initInputs=initInputs, **kwargs)
schema = initInputs["inputSchema"].schema
self.peakSchema = initInputs["peakSchema"].schema
self.schemaMapper = afwTable.SchemaMapper(schema)
self.schemaMapper.addMinimalSchema(schema)
self.schema = self.schemaMapper.getOutputSchema()
def __init__(self, butler=None, schema=None, initInputs=None, **kwargs):
"""!
Initialize the task.
@param[in] schema: the schema of the detection catalogs used as input to this one
@param[in] butler: a butler used to read the input schema from disk, if schema is None
The task will set its own self.schema attribute to the schema of the output merged catalog.
"""
super().__init__(**kwargs)
if initInputs is not None:
inputSchema = initInputs['inputSchema'].schema
else:
inputSchema = self.getInputSchema(butler=butler, schema=schema)
self.schemaMapper = afwTable.SchemaMapper(inputSchema, True)
self.schemaMapper.addMinimalSchema(inputSchema, True)
self.instFluxKey = inputSchema.find(self.config.snName +
"_instFlux").getKey()
self.instFluxErrKey = inputSchema.find(self.config.snName +
"_instFluxErr").getKey()
self.fluxFlagKey = inputSchema.find(self.config.snName +
"_flag").getKey()
self.flagKeys = {}
for band in self.config.priorityList:
short = getShortFilterName(band)
outputKey = self.schemaMapper.editOutputSchema().addField(
"merge_measurement_%s" % short,
type="Flag",
doc=
"Flag field set if the measurements here are from the %s filter"
% band)
peakKey = inputSchema.find("merge_peak_%s" % short).key
footprintKey = inputSchema.find("merge_footprint_%s" % short).key
self.flagKeys[band] = pipeBase.Struct(peak=peakKey,
footprint=footprintKey,
output=outputKey)
self.schema = self.schemaMapper.getOutputSchema()
self.pseudoFilterKeys = []
for filt in self.config.pseudoFilterList:
try:
self.pseudoFilterKeys.append(
self.schema.find("merge_peak_%s" % filt).getKey())
except Exception as e:
self.log.warn(
"merge_peak is not set for pseudo-filter %s: %s" %
(filt, e))
self.badFlags = {}
for flag in self.config.flags:
try:
self.badFlags[flag] = self.schema.find(flag).getKey()
except KeyError as exc:
self.log.warn("Can't find flag %s in schema: %s" % (
flag,
exc,
))
self.outputSchema = afwTable.SourceCatalog(self.schema)
def __init__(self, measConfig, inputSchema, outputDataset, *args,
**kwargs):
"""!Initialize TransformTask.
@param[in] measConfig Configuration for the measurement task which
produced the measurments being transformed.
@param[in] inputSchema The schema of the input catalog.
@param[in] outputDataset The butler dataset type of the output catalog.
@param[in] *args Passed through to pipeBase.Task.__init__()
@param[in] *kwargs Passed through to pipeBase.Task.__init__()
"""
pipeBase.Task.__init__(self, *args, **kwargs)
# This task can be used to generate multiple different output dataset types. We
# need to be able to specify the output type together with its schema.
self.outputDataset = outputDataset
# Define a mapper and add the basic fields to be copied.
self.mapper = afwTable.SchemaMapper(inputSchema)
for field in self.config.copyFields:
self.mapper.addMapping(inputSchema.find(field).key)
# Build a list of all transforms that will be applied to the input. We
# will iterate over this in run().
self.transforms = []
for name in measConfig.plugins.names:
config = measConfig.plugins.get(name)
transformClass = measConfig.plugins.registry.get(
name).PluginClass.getTransformClass()
self.transforms.append(transformClass(config, name, self.mapper))
def __init__(self, schema, peakSchema=None, **kwargs):
"""Create the task, adding necessary fields to the given schema.
Parameters
----------
schema : `lsst.afw.table.schema.schema.Schema`
Schema object for measurement fields; will be modified in-place.
peakSchema : `lsst.afw.table.schema.schema.Schema`
Schema of Footprint Peaks that will be passed to the deblender.
Any fields beyond the PeakTable minimal schema will be transferred
to the main source Schema. If None, no fields will be transferred
from the Peaks.
filters : list of str
Names of the filters used for the eposures. This is needed to store
the SED as a field
**kwargs
Passed to Task.__init__.
"""
pipeBase.Task.__init__(self, **kwargs)
peakMinimalSchema = afwDet.PeakTable.makeMinimalSchema()
if peakSchema is None:
# In this case, the peakSchemaMapper will transfer nothing, but
# we'll still have one
# to simplify downstream code
self.peakSchemaMapper = afwTable.SchemaMapper(
peakMinimalSchema, schema)
else:
self.peakSchemaMapper = afwTable.SchemaMapper(peakSchema, schema)
for item in peakSchema:
if item.key not in peakMinimalSchema:
self.peakSchemaMapper.addMapping(item.key, item.field)
# Because SchemaMapper makes a copy of the output schema
# you give its ctor, it isn't updating this Schema in
# place. That's probably a design flaw, but in the
# meantime, we'll keep that schema in sync with the
# peakSchemaMapper.getOutputSchema() manually, by adding
# the same fields to both.
schema.addField(item.field)
assert schema == self.peakSchemaMapper.getOutputSchema(
), "Logic bug mapping schemas"
self._addSchemaKeys(schema)
self.schema = schema
self.toCopyFromParent = [
item.key for item in self.schema
if item.field.getName().startswith("merge_footprint")
]
def _formatCatalog(self, fgcmStarCat, offsets, bands):
"""
Turn an FGCM-formatted star catalog, applying zeropoint offsets.
Parameters
----------
fgcmStarCat : `lsst.afw.Table.SimpleCatalog`
SimpleCatalog as output by fgcmcal
offsets : `list` with len(self.bands) entries
Zeropoint offsets to apply
bands : `list` [`str`]
List of band names from FGCM output
Returns
-------
formattedCat: `lsst.afw.table.SimpleCatalog`
SimpleCatalog suitable for using as a reference catalog
"""
sourceMapper = afwTable.SchemaMapper(fgcmStarCat.schema)
minSchema = LoadIndexedReferenceObjectsTask.makeMinimalSchema(
bands, addCentroid=False, addIsResolved=True, coordErrDim=0)
sourceMapper.addMinimalSchema(minSchema)
for band in bands:
sourceMapper.editOutputSchema().addField('%s_nGood' % (band),
type=np.int32)
sourceMapper.editOutputSchema().addField('%s_nTotal' % (band),
type=np.int32)
sourceMapper.editOutputSchema().addField('%s_nPsfCandidate' %
(band),
type=np.int32)
formattedCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema())
formattedCat.reserve(len(fgcmStarCat))
formattedCat.extend(fgcmStarCat, mapper=sourceMapper)
# Note that we don't have to set `resolved` because the default is False
for b, band in enumerate(bands):
mag = fgcmStarCat['mag_std_noabs'][:, b].astype(
np.float64) + offsets[b]
# We want fluxes in nJy from calibrated AB magnitudes
# (after applying offset). Updated after RFC-549 and RFC-575.
flux = (mag * units.ABmag).to_value(units.nJy)
fluxErr = (np.log(10.) /
2.5) * flux * fgcmStarCat['magErr_std'][:, b].astype(
np.float64)
formattedCat['%s_flux' % (band)][:] = flux
formattedCat['%s_fluxErr' % (band)][:] = fluxErr
formattedCat['%s_nGood' % (band)][:] = fgcmStarCat['ngood'][:, b]
formattedCat['%s_nTotal' % (band)][:] = fgcmStarCat['ntotal'][:, b]
formattedCat['%s_nPsfCandidate' %
(band)][:] = fgcmStarCat['npsfcand'][:, b]
addRefCatMetadata(formattedCat)
return formattedCat
def __init__(self, inputSchema, outputSchema, **kwargs):
pipeBase.Task.__init__(self, **kwargs)
self.inputSchema = inputSchema
self.outputSchema = outputSchema
self.mapper = afwTable.SchemaMapper(inputSchema, outputSchema)
for inputName, outputName in self.config.copyColumns.items():
self.mapper.addMapping(
self.inputSchema.find(inputName).key, outputName, True)
def _performTransform(self, transformClass, inCat, doExtend=True):
"""Operate on inCat with a transform of class transformClass"""
mapper = afwTable.SchemaMapper(inCat.schema)
config = SillyCentroidConfig()
transform = transformClass(config, self.pluginName, mapper)
outCat = afwTable.BaseCatalog(mapper.getOutputSchema())
if doExtend:
outCat.extend(inCat, mapper=mapper)
transform(inCat, outCat, makeWcs(), afwImage.Calib())
return outCat
def run():
exposure, srcCat = loadData()
schema = srcCat.getSchema()
#
# Create the astrometry task
#
config = AstrometryTask.ConfigClass()
config.refObjLoader.filterMap = {"_unknown_": "r"}
config.matcher.sourceFluxType = "Psf" # sample catalog does not contain aperture flux
aTask = AstrometryTask(config=config)
#
# And the photometry Task
#
config = PhotoCalTask.ConfigClass()
config.applyColorTerms = False # we don't have any available, so this suppresses a warning
pTask = PhotoCalTask(config=config, schema=schema)
#
# The tasks may have added extra elements to the schema (e.g. AstrometryTask's centroidKey to
# handle distortion; photometryTask's config.outputField). If this is so, we need to add
# these columns to the Source table.
#
# We wouldn't need to do this if we created the schema prior to measuring the exposure,
# but in this case we read the sources from disk
#
if schema != srcCat.getSchema(): # the tasks added fields
print("Adding columns to the source catalogue")
cat = afwTable.SourceCatalog(schema)
cat.table.defineCentroid(srcCat.table.getCentroidDefinition())
cat.table.definePsfFlux(srcCat.table.getPsfFluxDefinition())
scm = afwTable.SchemaMapper(srcCat.getSchema(), schema)
for schEl in srcCat.getSchema():
scm.addMapping(schEl.getKey(), True)
cat.extend(srcCat, True, scm) # copy srcCat to cat, adding new columns
srcCat = cat
del cat
#
# Process the data
#
matches = aTask.run(exposure, srcCat).matches
result = pTask.run(exposure, matches)
calib = result.calib
fm0, fm0Err = calib.getFluxMag0()
print("Used %d calibration sources out of %d matches" %
(len(result.matches), len(matches)))
delta = result.arrays.refMag - result.arrays.srcMag
q25, q75 = np.percentile(delta, [25, 75])
print("RMS error is %.3fmmsg (robust %.3f, Calib says %.3f)" %
(np.std(delta), 0.741 *
(q75 - q25), 2.5 / np.log(10) * fm0Err / fm0))
def __init__(self, icSourceSchema=None, **kwargs):
"""!Construct a CalibrateTask
@param[in] icSourceSchema schema for icSource catalog, or None.
If measuring aperture correction and the task detectAndMeasure cannot determine
its own suitable candidates, then this argument must be specified.
See also config field `icSourceFieldsToCopy`.
@param[in,out] kwargs other keyword arguments for lsst.pipe.base.CmdLineTask
"""
print " \n ******* In calib, init function " #mg
pipeBase.Task.__init__(self, **kwargs)
if icSourceSchema is not None:
# use a schema mapper to avoid copying each field separately
self.schemaMapper = afwTable.SchemaMapper(icSourceSchema)
self.schemaMapper.addMinimalSchema(
afwTable.SourceTable.makeMinimalSchema(), False)
# Add fields to copy from an icSource catalog
# and a field to indicate that the source matched a source in that catalog
# If any fields are missing then raise an exception, but first find all missing fields
# in order to make the error message more useful.
self.calibSourceKey = self.schemaMapper.addOutputField(
afwTable.Field["Flag"]("calib_detected",
"Source was detected as an icSource"))
missingFieldNames = []
for fieldName in self.config.icSourceFieldsToCopy:
try:
schemaItem = icSourceSchema.find(fieldName)
except Exception:
missingFieldNames.append(fieldName)
else:
# field found; if addMapping fails then raise an exception
self.schemaMapper.addMapping(schemaItem.getKey())
if missingFieldNames:
raise RuntimeError(
"isSourceCat is missing fields %s specified in icSourceFieldsToCopy"
% (missingFieldNames, ))
# produce a temporary schema to pass to the subtasks; finalize it later
self.schema = self.schemaMapper.editOutputSchema()
else:
self.schemaMapper = None
self.schema = afwTable.SourceTable.makeMinimalSchema()
self.makeSubtask("detectAndMeasure", schema=self.schema)
if self.config.doAstrometry or self.config.doPhotoCal:
self.makeSubtask("astrometry", schema=self.schema)
if self.config.doPhotoCal:
self.makeSubtask("photoCal", schema=self.schema)
if self.schemaMapper is not None:
# finalize the schema
self.schema = self.schemaMapper.getOutputSchema()
def applyCalib(catalog, calib, hscRun=None):
"""Convert all fluxes in a catalog to magnitudes
The fluxes are converted in-place, so that the "_flux*" are now really
magnitudes.
"""
fluxKeys, errKeys = getFluxKeys(catalog.schema, hscRun=hscRun)
mapper = afwTable.SchemaMapper(catalog.schema, True)
for item in catalog.schema:
name = item.field.getName()
if name in fluxKeys:
continue
mapper.addMapping(item.key)
aliasMap = catalog.schema.getAliasMap()
newFluxKeys = {}
newErrKeys = {}
for name in fluxKeys:
fluxField = catalog.schema.find(name).field
newName = name.replace("flux", "mag")
newField = fluxField.__class__(newName, "Calibrated magnitude from %s (%s)" %
(fluxField.getName(), fluxField.getDoc()), "mag")
newFluxKeys[newName] = mapper.addMapping(fluxKeys[name], newField)
sigmaName = "Sigma"
if hscRun is not None:
sigmaName = "_err"
if name + sigmaName in errKeys:
errField = catalog.schema.find(name + sigmaName).field
newErrField = errField.__class__(newName + sigmaName,
"Calibrated magnitude error from %s (%s)" %
(errField.getName(), errField.getDoc()), "mag")
newErrKeys[newName] = mapper.addMapping(errKeys[name + sigmaName], newErrField)
aliasMap.set(name, newName)
aliasMap.set(name + sigmaName, newName + sigmaName)
calib.setThrowOnNegativeFlux(False)
newCatalog = afwTable.SourceCatalog(mapper.getOutputSchema())
newCatalog.extend(catalog, mapper=mapper)
for name, key in newFluxKeys.items():
flux = newCatalog[key]
if name in newErrKeys:
fluxErr = newCatalog[newErrKeys[name]]
magArray = numpy.array([calib.getMagnitude(f, e) for f, e in zip(flux, fluxErr)])
mag = magArray[:,0]
fluxErr[:] = magArray[:,1]
else:
mag = numpy.array([calib.getMagnitude(f) for f in flux])
flux[:] = mag
return newCatalog
def loadSkyCircle(self, ctrCoord, radius, filterName=None):
"""!Load reference objects that overlap a circular sky region
@param[in] ctrCoord center of search region (an lsst.afw.geom.Coord)
@param[in] radius radius of search region (an lsst.afw.geom.Angle)
@param[in] filterName name of filter, or None for the default filter;
used for flux values in case we have flux limits (which are not yet implemented)
@return an lsst.pipe.base.Struct containing:
- refCat a catalog of reference objects with the
\link meas_algorithms_loadReferenceObjects_Schema standard schema \endlink
as documented in LoadReferenceObjects, including photometric, resolved and variable;
hasCentroid is False for all objects.
- fluxField = name of flux field for specified filterName. None if refCat is None.
"""
id_list, boundary_mask = self.indexer.get_pixel_ids(ctrCoord, radius)
shards = self.get_shards(id_list)
refCat = self.butler.get('ref_cat',
dataId=self.indexer.make_data_id(
'master_schema', self.ref_dataset_name),
immediate=True)
self._addFluxAliases(refCat.schema)
fluxField = getRefFluxField(schema=refCat.schema,
filterName=filterName)
for shard, is_on_boundary in zip(shards, boundary_mask):
if shard is None:
continue
if is_on_boundary:
refCat.extend(self._trim_to_circle(shard, ctrCoord, radius))
else:
refCat.extend(shard)
# make sure catalog is contiguous
if not refCat.isContiguous():
refCat = refCat.copy()
# add and initialize centroid and hasCentroid fields (these are added
# after loading to avoid wasting space in the saved catalogs)
# the new fields are automatically initialized to (nan, nan) and False
# so no need to set them explicitly
mapper = afwTable.SchemaMapper(refCat.schema, True)
mapper.addMinimalSchema(refCat.schema, True)
mapper.editOutputSchema().addField("centroid_x", type=float)
mapper.editOutputSchema().addField("centroid_y", type=float)
mapper.editOutputSchema().addField("hasCentroid", type="Flag")
expandedCat = afwTable.SimpleCatalog(mapper.getOutputSchema())
expandedCat.extend(refCat, mapper=mapper)
del refCat # avoid accidentally returning the unexpanded reference catalog
# return reference catalog
return pipeBase.Struct(
refCat=expandedCat,
fluxField=fluxField,
)
def setPeakSignificance(self, exposure, footprints, threshold, negative=False):
"""Set the significance of each detected peak to the pixel value divided
by the appropriate standard-deviation for ``config.thresholdType``.
Only sets significance for "stdev" and "pixel_stdev" thresholdTypes;
we leave it undefined for "value" and "variance" as it does not have a
well-defined meaning in those cases.
Parameters
----------
exposure : `lsst.afw.image.Exposure`
Exposure that footprints were detected on, likely the convolved,
local background-subtracted image.
footprints : `lsst.afw.detection.FootprintSet`
Footprints detected on the image.
threshold : `lsst.afw.detection.Threshold`
Threshold used to find footprints.
negative : `bool`, optional
Are we calculating for negative sources?
"""
if footprints is None or footprints.getFootprints() == []:
return footprints
polarity = -1 if negative else 1
# All incoming footprints have the same schema.
mapper = afwTable.SchemaMapper(footprints.getFootprints()[0].peaks.schema)
mapper.addMinimalSchema(footprints.getFootprints()[0].peaks.schema)
mapper.addOutputField("significance", type=float,
doc="Ratio of peak value to configured standard deviation.")
# Copy the old peaks to the new ones with a significance field.
# Do this independent of the threshold type, so we always have a
# significance field.
newFootprints = afwDet.FootprintSet(footprints)
for old, new in zip(footprints.getFootprints(), newFootprints.getFootprints()):
newPeaks = afwDet.PeakCatalog(mapper.getOutputSchema())
newPeaks.extend(old.peaks, mapper=mapper)
new.getPeaks().clear()
new.setPeakCatalog(newPeaks)
# Compute the significance values.
if self.config.thresholdType == "pixel_stdev":
for footprint in newFootprints.getFootprints():
footprint.updatePeakSignificance(exposure.variance, polarity)
elif self.config.thresholdType == "stdev":
sigma = threshold.getValue() / self.config.thresholdValue
for footprint in newFootprints.getFootprints():
footprint.updatePeakSignificance(polarity*sigma)
else:
for footprint in newFootprints.getFootprints():
for peak in footprint.peaks:
peak["significance"] = 0
return newFootprints
def testApplyCppTransform(self):
"""Test that we can apply a simple C++ transform"""
inCat = self._generateCatalog()
sillyControl = testLib.SillyCentroidControl()
mapper = afwTable.SchemaMapper(inCat.schema)
sillyTransform = testLib.SillyTransform(sillyControl, self.pluginName, mapper)
outCat = afwTable.BaseCatalog(mapper.getOutputSchema())
outCat.extend(inCat, mapper=mapper)
self.assertEqual(len(inCat), len(outCat))
sillyTransform(inCat, outCat, makeWcs(), afwImage.Calib())
self._checkSillyOutputs(inCat, outCat)
def applyCalib(catalog, photoCalib, hscRun=None):
"""Convert all fluxes in a catalog to magnitudes
The fluxes are converted in-place, so that the "_flux*" are now really
magnitudes.
"""
fluxKeys, errKeys = getFluxKeys(catalog.schema, hscRun=hscRun)
mapper = afwTable.SchemaMapper(catalog.schema, True)
for item in catalog.schema:
name = item.field.getName()
if name in fluxKeys:
continue
mapper.addMapping(item.key)
aliasMap = catalog.schema.getAliasMap()
newFluxKeys = {}
newErrKeys = {}
for name in fluxKeys:
fluxField = catalog.schema.find(name).field
newName = name.replace("instFlux", "mag")
newField = fluxField.__class__(
newName, "Calibrated magnitude from %s (%s)" %
(fluxField.getName(), fluxField.getDoc()), "mag")
newFluxKeys[newName] = mapper.addMapping(fluxKeys[name], newField)
errName = "Err"
if hscRun is not None:
errName = "_err"
if name + errName in errKeys:
errField = catalog.schema.find(name + errName).field
newErrField = errField.__class__(
newName + errName, "Calibrated magnitude error from %s (%s)" %
(errField.getName(), errField.getDoc()), "mag")
newErrKeys[newName] = mapper.addMapping(errKeys[name + errName],
newErrField)
aliasMap.set(name, newName)
aliasMap.set(name + errName, newName + errName)
newCatalog = afwTable.SourceCatalog(mapper.getOutputSchema())
newCatalog.extend(catalog, mapper=mapper)
for name, key in newFluxKeys.items():
flux = newCatalog[key]
if name in newErrKeys:
result = photoCalib.instFluxToMagnitude(newCatalog,
name.strip('_mag'))
flux[:] = result[:, 0]
newCatalog[newErrKeys[name]] = result[:, 1]
else:
flux[:] = numpy.array(
[photoCalib.instFluxToMagnitude(f) for f in flux])
return newCatalog
def applyMosaicResultsCatalog(dataRef, catalog, addCorrection=True):
"""!Apply the results of meas_mosaic to a source catalog
The coordinates and all fluxes are updated in-place with the meas_mosaic solution.
This assumes that the mosaic solution exists; an exception will be raised
in the event that it does not.
"""
ffp = getFluxFitParams(dataRef)
calexp_md = dataRef.get("calexp_md", immediate=True)
hscRun = mosaicUtils.checkHscStack(calexp_md)
if hscRun is None:
detector = dataRef.get("camera")[dataRef.dataId["ccd"]]
nQuarter = detector.getOrientation().getNQuarter()
if nQuarter % 4 != 0:
dimensions = dataRef.get("calexp_bbox").getDimensions()
catalog = mosaicUtils.rotatePixelCoords(catalog, dimensions.getX(),
dimensions.getY(),
nQuarter)
xx, yy = catalog.getX(), catalog.getY()
corr = numpy.power(10.0, -0.4 * ffp.ffp.eval(xx, yy)) * calculateJacobian(
ffp.wcs, xx, yy)
if addCorrection:
mapper = afwTable.SchemaMapper(catalog.schema, True)
for s in catalog.schema:
mapper.addMapping(s.key)
corrField = afwTable.Field[float](
"mosaic_corr", "Magnitude correction from meas_mosaic")
corrKey = mapper.addOutputField(corrField)
outCatalog = type(catalog)(mapper.getOutputSchema())
outCatalog.extend(catalog, mapper=mapper)
outCatalog[corrKey][:] = corr
catalog = outCatalog
fluxKeys, errKeys = getFluxKeys(catalog.schema, hscRun=hscRun)
for name, key in list(fluxKeys.items()) + list(errKeys.items()):
# Note this skips correcting the aperture fluxes in HSC processed data, but that's ok because
# we are using the flux_sinc as our comparison to base_CircularApertureFlux_12_0_flux
if key.subfields is None:
catalog[key][:] *= corr
# Now rotate them back to the LSST coord system
if hscRun is None:
if nQuarter % 4 != 0:
catalog = mosaicUtils.rotatePixelCoordsBack(
catalog, dimensions.getX(), dimensions.getY(), nQuarter)
wcs = getWcs(dataRef)
for rec in catalog:
rec.updateCoord(wcs)
return Struct(catalog=catalog, wcs=wcs, ffp=ffp)
def loadSkyCircle(self, ctrCoord, radius, filterName=None, epoch=None):
shardIdList, isOnBoundaryList = self.indexer.getShardIds(
ctrCoord, radius)
shards = self.getShards(shardIdList)
refCat = self.butler.get('ref_cat',
dataId=self.indexer.makeDataId(
'master_schema', self.ref_dataset_name),
immediate=True)
self._addFluxAliases(refCat.schema)
fluxField = getRefFluxField(schema=refCat.schema,
filterName=filterName)
for shard, isOnBoundary in zip(shards, isOnBoundaryList):
if shard is None:
continue
if isOnBoundary:
refCat.extend(self._trimToCircle(shard, ctrCoord, radius))
else:
refCat.extend(shard)
if epoch is not None and "pm_ra" in refCat.schema:
# check for a catalog in a non-standard format
if isinstance(refCat.schema["pm_ra"].asKey(),
lsst.afw.table.KeyAngle):
self.applyProperMotions(refCat, epoch)
else:
self.log.warn(
"Catalog pm_ra field is not an Angle; not applying proper motion"
)
# add and initialize centroid and hasCentroid fields (these are
# added after loading to avoid wasting space in the saved catalogs)
# the new fields are automatically initialized to (nan, nan) and
# False so no need to set them explicitly
mapper = afwTable.SchemaMapper(refCat.schema, True)
mapper.addMinimalSchema(refCat.schema, True)
mapper.editOutputSchema().addField("centroid_x", type=float)
mapper.editOutputSchema().addField("centroid_y", type=float)
mapper.editOutputSchema().addField("hasCentroid", type="Flag")
expandedCat = afwTable.SimpleCatalog(mapper.getOutputSchema())
expandedCat.extend(refCat, mapper=mapper)
del refCat # avoid accidentally returning the unexpanded ref cat
# make sure catalog is contiguous
if not expandedCat.isContiguous():
expandedCat = expandedCat.copy(True)
# return reference catalog
return pipeBase.Struct(
refCat=expandedCat,
fluxField=fluxField,
)
def run(self, dataRef):
self.log.info("Processing %s" % (dataRef.dataId))
calexp = dataRef.get('calexp')
srcs = dataRef.get('src')
print('Calexp:', calexp)
print('srcs:', srcs)
## FIXME -- this whole mapping business is very fragile -- it
## seems to fail, eg, if you don't set "-c
## doMeasurement=False" when creating the input 'srcs' list.
mapper = afwTable.SchemaMapper(srcs.getSchema())
# map all the existing fields
mapper.addMinimalSchema(srcs.getSchema(), True)
schema = mapper.getOutputSchema()
self.algMetadata = dafBase.PropertyList()
if self.config.doDeblend:
self.makeSubtask("deblend", schema=schema)
if self.config.doMeasurement:
self.makeSubtask("measurement",
schema=schema,
algMetadata=self.algMetadata)
self.schema = schema
parents = []
for src in srcs:
if src.getParent() == 0:
parents.append(src)
outsources = afwTable.SourceCatalog(schema)
outsources.reserve(len(parents))
outsources.extend(parents, mapper=mapper)
srcs = outsources
print(len(srcs), 'sources before deblending')
if self.config.doDeblend:
self.deblend.run(calexp, srcs)
if self.config.doMeasurement:
self.measurement.run(calexp, srcs)
if srcs is not None and self.config.doWriteSources:
sourceWriteFlags = (0
if self.config.doWriteHeavyFootprintsInSources
else afwTable.SOURCE_IO_NO_HEAVY_FOOTPRINTS)
print('Writing "src" outputs')
if self.config.sourceOutputFile:
srcs.writeFits(self.config.sourceOutputFile,
flags=sourceWriteFlags)
else:
dataRef.put(srcs, 'src', flags=sourceWriteFlags)
def rotateIcSrc(icSrc, theta):
"""!Return a new icSrc catalog with columns added to hold rotated focal
plane coordinates as if measured under a rotation of the focal plane by
theta.
@param icSrc Input ic SourceCatalog
@oaram theta afwGeom.Angle specifying the rotation to apply
@returns A new sourceCatalog
"""
schema = icSrc.schema
schemaMapper = afwTable.SchemaMapper(schema, schema)
for key, field in schema:
schemaMapper.addMapping(key, field.getName())
newSchema = schemaMapper.editOutputSchema()
# Make keys for old columns and new columns (both in the new schema)
fpKey = afwTable.ArrayDKey([
schema.find("base_FPPosition_x").key,
schema.find("base_FPPosition_y").key
])
newFpKeyList = [
newSchema.addField("base_FPPosition_rot_x", type=np.float64),
newSchema.addField("base_FPPosition_rot_y", type=np.float64)
]
newFpKey = afwTable.ArrayDKey(newFpKeyList)
# Copy unrotated columns
newIcSrc = afwTable.SourceCatalog(newSchema)
newIcSrc.reserve(len(icSrc))
for record in icSrc:
newIcSrc.addNew().assign(record, schemaMapper)
# Collect items to be transformed
fps = np.zeros((len(icSrc), 2), dtype=np.float64)
for i, r in enumerate(icSrc):
fps[i] = r.get(fpKey)
# Assemble transformation matrix
rotZ = zernikeRotMatrix(3, theta)
rot2 = rotZ[1:3, 1:3]
# Do the transformation
newFps = np.dot(rot2, fps.T).T
# Write into new columns
for i, r in enumerate(newIcSrc):
r.set(newFpKey, newFps[i].astype(np.float64))
return newIcSrc
def _formatCatalog(self, fgcmStarCat, offsets):
"""
Turn an FGCM-formatted star catalog, applying zeropoint offsets.
Parameters
----------
fgcmStarCat: `afwTable.SimpleCatalog`
SimpleCatalog as output by fgcmcal
offsets: `list` with len(self.bands) entries
Zeropoint offsets to apply
Returns
-------
formattedCat: `afwTable.SimpleCatalog`
SimpleCatalog suitable for using as a reference catalog
"""
sourceMapper = afwTable.SchemaMapper(fgcmStarCat.schema)
minSchema = LoadIndexedReferenceObjectsTask.makeMinimalSchema(
self.bands, addCentroid=False, addIsResolved=True, coordErrDim=0)
sourceMapper.addMinimalSchema(minSchema)
for band in self.bands:
sourceMapper.editOutputSchema().addField('%s_nGood' % (band),
type=np.int32)
formattedCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema())
formattedCat.reserve(len(fgcmStarCat))
formattedCat.extend(fgcmStarCat, mapper=sourceMapper)
# Note that we don't have to set `resolved` because the default is False
for b, band in enumerate(self.bands):
mag = fgcmStarCat['mag_std_noabs'][:, b] + offsets[b]
# We want fluxes in Jy from calibrated AB magnitudes
# (after applying offset)
# TODO: Full implementation of RFC-549 will have all reference
# catalogs in nJy instead of Jy.
flux = afwImage.fluxFromABMag(mag)
fluxErr = afwImage.fluxErrFromABMagErr(
fgcmStarCat['magErr_std'][:, b], mag)
formattedCat['%s_flux' % (band)][:] = flux
formattedCat['%s_fluxErr' % (band)][:] = fluxErr
formattedCat['%s_nGood' % (band)][:] = fgcmStarCat['ngood'][:, b]
return formattedCat
def testFlags(self):
"""test that all the calib_photometry flags are set to reasonable values"""
schema = self.srcCat.schema
task = PhotoCalTask(self.refObjLoader, config=self.config, schema=schema)
mapper = afwTable.SchemaMapper(self.srcCat.schema, schema)
cat = afwTable.SourceCatalog(schema)
for name in self.srcCat.schema.getNames():
mapper.addMapping(self.srcCat.schema.find(name).key)
cat.extend(self.srcCat, mapper=mapper)
# test that by default, no stars are reserved and all used are candidates
task.run(exposure=self.exposure, sourceCat=cat)
used = 0
for source in cat:
if source.get("calib_photometry_used"):
used += 1
self.assertFalse(source.get("calib_photometry_reserved"))
# test that some are actually used
self.assertGreater(used, 0)
def testFlags(self):
"""test that all the calib_photometry flags are set to reasonable values"""
schema = self.srcCat.schema
task = PhotoCalTask(self.refObjLoader,
config=self.config,
schema=schema)
mapper = afwTable.SchemaMapper(self.srcCat.schema, schema)
cat = afwTable.SourceCatalog(schema)
for name in self.srcCat.schema.getNames():
mapper.addMapping(self.srcCat.schema.find(name).key)
cat.extend(self.srcCat, mapper=mapper)
# test that by default, no stars are reserved and used < candidates
task.run(exposure=self.exposure, sourceCat=cat)
candidates = 0
used = 0
reserved = 0
for source in cat:
if source.get("calib_photometryCandidate"):
candidates += 1
if source.get("calib_photometryUsed"):
used += 1
if source.get("calib_photometryReserved"):
reserved += 1
self.assertLessEqual(used, candidates)
self.assertEqual(reserved, 0)
# set the reserve fraction, and see if the right proportion are reserved.
self.config.reserveFraction = .3
task.run(exposure=self.exposure, sourceCat=cat)
candidates = 0
reserved = 0
used = 0
for source in cat:
if source.get("calib_photometryCandidate"):
candidates += 1
if source.get("calib_photometryUsed"):
used += 1
if source.get("calib_photometryReserved"):
reserved += 1
self.assertEqual(reserved, int(.3 * candidates))
self.assertLessEqual(used, (candidates - reserved))
def _formatCatalog(self, fgcmStarCat, offsets):
"""
Turn an FGCM-formatted star catalog, applying zp offsets.
parameters
----------
fgcmStarCat: SimpleCatalog
SimpleCatalog as output by fgcmcal
offsets: list with len(self.bands) entries
Zeropoint offsets to apply
returns
-------
formattedCat: SimpleCatalog
SimpleCatalog suitable for ref_cat
"""
sourceMapper = afwTable.SchemaMapper(fgcmStarCat.schema)
sourceMapper.addMinimalSchema(afwTable.SimpleTable.makeMinimalSchema())
for band in self.bands:
sourceMapper.editOutputSchema().addField('%s_flux' % (band),
type=np.float64)
sourceMapper.editOutputSchema().addField('%s_fluxErr' % (band),
type=np.float64)
sourceMapper.editOutputSchema().addField('%s_nGood' % (band),
type=np.float64)
formattedCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema())
formattedCat.reserve(len(fgcmStarCat))
formattedCat.extend(fgcmStarCat, mapper=sourceMapper)
for b, band in enumerate(self.bands):
mag = fgcmStarCat['mag_std_noabs'][:, b] + offsets[b]
flux = afwImage.fluxFromABMag(mag)
fluxErr = afwImage.fluxErrFromABMagErr(
fgcmStarCat['magerr_std'][:, b], mag)
formattedCat['%s_flux' % (band)][:] = flux
formattedCat['%s_fluxErr' % (band)][:] = fluxErr
formattedCat['%s_nGood' % (band)][:] = fgcmStarCat['ngood'][:, b]
return formattedCat
def _makeSourceMapper(self, sourceSchema):
"""
Make a schema mapper for fgcm sources
Parameters
----------
sourceSchema: `afwTable.Schema`
Default source schema from the butler
Returns
-------
sourceMapper: `afwTable.schemaMapper`
Mapper to the FGCM source schema
"""
# create a mapper to the preferred output
sourceMapper = afwTable.SchemaMapper(sourceSchema)
# map to ra/dec
sourceMapper.addMapping(sourceSchema['coord_ra'].asKey(), 'ra')
sourceMapper.addMapping(sourceSchema['coord_dec'].asKey(), 'dec')
sourceMapper.addMapping(sourceSchema[self.config.jacobianName].asKey(),
'jacobian')
sourceMapper.addMapping(sourceSchema['slot_Centroid_x'].asKey(), 'x')
sourceMapper.addMapping(sourceSchema['slot_Centroid_y'].asKey(), 'y')
# and add the fields we want
sourceMapper.editOutputSchema().addField("visit",
type=np.int32,
doc="Visit number")
sourceMapper.editOutputSchema().addField("ccd",
type=np.int32,
doc="CCD number")
sourceMapper.editOutputSchema().addField("instMag",
type=np.float32,
doc="Instrumental magnitude")
sourceMapper.editOutputSchema().addField(
"instMagErr", type=np.float32, doc="Instrumental magnitude error")
return sourceMapper
def create_source_catalog_from_external_catalog(self,
dataRef,
coord_file,
dataset='src',
debug=False):
butler = dataRef.getButler()
schema = butler.get(dataset + "_schema", immediate=True).schema
mapper = afwTable.SchemaMapper(schema)
mapper.addMinimalSchema(schema)
newSchema = mapper.getOutputSchema()
info = load_external_catalog_info(coord_file)
src_cat = afwTable.SourceCatalog(newSchema)
for row in info:
record = src_cat.addNew()
record.set('coord_ra', Angle(row['RA'] * degrees))
record.set('coord_dec', Angle(row['Dec'] * degrees))
if debug:
print(src_cat['coord_ra'], src_cat['coord_dec'])
return (src_cat)
def __init__(self, config: pexConfig.Config, initInput: Mapping, *args,
**kwargs):
super().__init__(config=config, *args, **kwargs)
self.apRad = self.config.apRad
inputSchema = initInput["inputSchema"].schema
# Create a camera mapper to create a copy of the input schema
self.mapper = afwTable.SchemaMapper(inputSchema)
self.mapper.addMinimalSchema(inputSchema, True)
# Add the new field
self.apKey = self.mapper.editOutputSchema().addField(
"apFlux", type=np.float64, doc="Ap flux measured")
# Get the output schema
self.schema = self.mapper.getOutputSchema()
# create the catalog in which new measurements will be stored
self.outputCatalog = afwTable.SourceCatalog(self.schema)
# Put the outputSchema into a SourceCatalog container. This var name
# matches an initOut so will be persisted
self.outputSchema = afwTable.SourceCatalog(self.schema)