def testMatchXyMatchControl(self):
"""Test using MatchControl to return all matches, and add a test for closest==False at the same time"""
for closest in (True, False):
for includeMismatches in (True, False):
mc = afwTable.MatchControl()
mc.findOnlyClosest = closest
mc.includeMismatches = includeMismatches
matches = afwTable.matchXy(self.cat1, self.cat2, 0.01, mc)
if False:
for m in matches:
print closest, m.first.getId(), m.second.getId(), m.distance
if includeMismatches:
catMatches = afwTable.SourceCatalog(self.table)
catMismatches = afwTable.SourceCatalog(self.table)
for m in matches:
if m[1] != None:
if not any(x == m[0] for x in catMatches):
catMatches.append(m[0])
else:
catMismatches.append(m[0])
matches = afwTable.matchXy(catMatches, self.cat2, 0.01, mc)
mc.includeMismatches = False
noMatches = afwTable.matchXy(catMismatches, self.cat2, 0.01, mc)
self.assertEquals(len(noMatches), 0)
self.assertEquals(len(matches), self.nUniqueMatch if closest else self.nUniqueMatch + 1)
for m in matches:
if closest:
self.assertEquals(m.first.getId() + self.nobj, m.second.getId())
self.assertEquals(m.distance, 0.0)
def testMatchXyMatchControl(self):
"""Test using MatchControl to return all matches
Also tests closest==False at the same time
"""
for closest in (True, False):
for includeMismatches in (True, False):
mc = afwTable.MatchControl()
mc.findOnlyClosest = closest
mc.includeMismatches = includeMismatches
matches = afwTable.matchXy(
self.cat1, self.cat2, self.matchRadius, mc)
if False:
for m in matches:
print(closest, m.first.getId(),
m.second.getId(), m.distance)
if includeMismatches:
catMatches = afwTable.SourceCatalog(self.table)
catMismatches = afwTable.SourceCatalog(self.table)
for m in matches:
if m[1] is not None:
if not any(x == m[0] for x in catMatches):
catMatches.append(m[0])
else:
catMismatches.append(m[0])
matches = afwTable.matchXy(
catMatches, self.cat2, self.matchRadius, mc)
mc.includeMismatches = False
noMatches = afwTable.matchXy(
catMismatches, self.cat2, self.matchRadius, mc)
self.assertEqual(len(noMatches), 0)
# If we're not getting only the closest match, then we get an extra match due to the
# source we offset by 2 pixels and a bit. Everything else
# should match exactly.
self.assertEqual(
len(matches),
self.nUniqueMatch if closest else self.nUniqueMatch + 1)
self.assertEqual(
sum(1 for m in matches if m.distance == 0.0),
self.nUniqueMatch)
for m in matches:
if closest:
self.assertEqual(m.first.getId() +
self.nobj, m.second.getId())
else:
self.assertLessEqual(m.distance, self.matchRadius)
def testMatchXyMatchControl(self):
"""Test using MatchControl to return all matches
Also tests closest==False at the same time
"""
for closest in (True, False):
for includeMismatches in (True, False):
mc = afwTable.MatchControl()
mc.findOnlyClosest = closest
mc.includeMismatches = includeMismatches
matches = afwTable.matchXy(self.cat1, self.cat2,
self.matchRadius, mc)
if False:
for m in matches:
print(closest, m.first.getId(), m.second.getId(),
m.distance)
if includeMismatches:
catMatches = afwTable.SourceCatalog(self.table)
catMismatches = afwTable.SourceCatalog(self.table)
for m in matches:
if m[1] is not None:
if not any(x == m[0] for x in catMatches):
catMatches.append(m[0])
else:
catMismatches.append(m[0])
matches = afwTable.matchXy(catMatches, self.cat2,
self.matchRadius, mc)
mc.includeMismatches = False
noMatches = afwTable.matchXy(catMismatches, self.cat2,
self.matchRadius, mc)
self.assertEqual(len(noMatches), 0)
# If we're not getting only the closest match, then we get an extra match due to the
# source we offset by 2 pixels and a bit. Everything else
# should match exactly.
self.assertEqual(
len(matches),
self.nUniqueMatch if closest else self.nUniqueMatch + 1)
self.assertEqual(sum(1 for m in matches if m.distance == 0.0),
self.nUniqueMatch)
for m in matches:
if closest:
self.assertEqual(m.first.getId() + self.nobj,
m.second.getId())
else:
self.assertLessEqual(m.distance, self.matchRadius)
def propagatePsfFlags(keysToCopy, calibSources, sources, matchRadius=1):
"""Match the calibSources and sources, and propagate Interesting Flags (e.g. PSF star) to the sources
"""
if calibSources is None or sources is None:
return
closest = False # return all matched objects
matched = afwTable.matchXy(calibSources, sources, matchRadius, closest)
#
# Because we had to allow multiple matches to handle parents, we now need to
# prune to the best matches
#
bestMatches = {}
for m0, m1, d in matched:
id0 = m0.getId()
if bestMatches.has_key(id0):
if d > bestMatches[id0][2]:
continue
bestMatches[id0] = (m0, m1, d)
matched = bestMatches.values()
#
# Check that we got it right
#
if len(set(m[0].getId() for m in matched)) != len(matched):
print("At least one calibSource is matched to more than one Source")
#
# Copy over the desired flags
#
for cs, s, d in matched:
for skey, ckey in keysToCopy:
s.set(skey, cs.get(ckey))
def testMatchXy(self):
matches = afwTable.matchXy(self.cat1, self.cat2, self.matchRadius)
self.assertEquals(len(matches), self.nUniqueMatch)
for m in matches:
self.assertEquals(m.first.getId() + self.nobj, m.second.getId())
self.assertEquals(m.distance, 0.0)
def testMatchXy(self):
matches = afwTable.matchXy(self.cat1, self.cat2, self.matchRadius)
self.assertEqual(len(matches), self.nUniqueMatch)
for m in matches:
self.assertEqual(m.first.getId() + self.nobj, m.second.getId())
self.assertEqual(m.distance, 0.0)
def propagatePsfFlags(keysToCopy, calibSources, sources, matchRadius=1):
"""Match the calibSources and sources, and propagate Interesting Flags (e.g. PSF star) to the sources
"""
if calibSources is None or sources is None:
return
closest = False # return all matched objects
matched = afwTable.matchXy(calibSources, sources, matchRadius, closest)
#
# Because we had to allow multiple matches to handle parents, we now need to
# prune to the best matches
#
bestMatches = {}
for m0, m1, d in matched:
id0 = m0.getId()
if bestMatches.has_key(id0):
if d > bestMatches[id0][2]:
continue
bestMatches[id0] = (m0, m1, d)
matched = bestMatches.values()
#
# Check that we got it right
#
if len(set(m[0].getId() for m in matched)) != len(matched):
print("At least one calibSource is matched to more than one Source")
#
# Copy over the desired flags
#
for cs, s, d in matched:
for skey, ckey in keysToCopy:
s.set(skey, cs.get(ckey))
def copyIcSourceFields(self, icSourceCat, sourceCat):
"""!Match sources in icSourceCat and sourceCat and copy the specified fields
@param[in] icSourceCat catalog from which to copy fields
@param[in,out] sourceCat catalog to which to copy fields
The fields copied are those specified by `config.icSourceFieldsToCopy`
that actually exist in the schema. This was set up by the constructor
using self.schemaMapper.
"""
if self.schemaMapper is None:
raise RuntimeError("To copy icSource fields you must specify icSourceSchema "
"and icSourceKeys when constructing this task")
if icSourceCat is None or sourceCat is None:
raise RuntimeError("icSourceCat and sourceCat must both be specified")
if len(self.config.icSourceFieldsToCopy) == 0:
self.log.warn("copyIcSourceFields doing nothing because icSourceFieldsToCopy is empty")
return
closest = False # return all matched objects
matches = afwTable.matchXy(icSourceCat, sourceCat, self.config.matchRadiusPix, closest)
if self.config.detectAndMeasure.doDeblend:
deblendKey = sourceCat.schema["deblend_nChild"].asKey()
matches = [m for m in matches if m[1].get(deblendKey) == 0] # if deblended, keep children
# Because we had to allow multiple matches to handle parents, we now need to
# prune to the best matches
bestMatches = {} # closest matches as a dict of icSourceCat source ID:
# (icSourceCat source, sourceCat source, distance in pixels)
for m0, m1, d in matches:
id0 = m0.getId()
match = bestMatches.get(id0)
if match is None or d <= match[2]:
bestMatches[id0] = (m0, m1, d)
matches = bestMatches.values()
# Check that no sourceCat sources are listed twice (we already know that each match has a unique
# icSourceCat source ID, due to using that ID as the key in bestMatches)
numMatches = len(matches)
numUniqueSources = len(set(m[1].getId() for m in matches))
if numUniqueSources != numMatches:
self.log.warn("%d icSourceCat sources matched only %d sourceCat sources" %
(numMatches, numUniqueSources))
self.log.info("Copying flags from icSourceCat to sourceCat for %s sources" % (numMatches,))
# For each match: set the calibSourceKey flag and copy the desired fields
for icSrc, src, d in matches:
src.setFlag(self.calibSourceKey, True)
# src.assign copies the footprint from icSrc, which we don't want (DM-407)
# so set icSrc's footprint to src's footprint before src.assign, then restore it
icSrcFootprint = icSrc.getFootprint()
try:
icSrc.setFootprint(src.getFootprint())
src.assign(icSrc, self.schemaMapper)
finally:
icSrc.setFootprint(icSrcFootprint)
def testSelfMatchXy(self):
"""Test doing a self-matches"""
for symmetric in (True, False):
mc = afwTable.MatchControl()
mc.symmetricMatch = symmetric
matches = afwTable.matchXy(self.cat2, 0.01, mc)
if False:
for m in matches:
print m.first.getId(), m.second.getId(), m.distance
self.assertEquals(len(matches), 2 if symmetric else 1)
def testSelfMatchXy(self):
"""Test doing a self-matches"""
for symmetric in (True, False):
mc = afwTable.MatchControl()
mc.symmetricMatch = symmetric
matches = afwTable.matchXy(self.cat2, self.matchRadius, mc)
if False:
for m in matches:
print(m.first.getId(), m.second.getId(), m.distance)
# There is only one source that matches another source when we do a self-match: the one
# offset by 2 pixels and a bit.
# If we're getting symmetric matches, that multiplies the expected number by 2 because it
# produces s1,s2 and s2,s1.
self.assertEqual(len(matches), 2 if symmetric else 1)
def testSelfMatchXy(self):
"""Test doing a self-matches"""
for symmetric in (True, False):
mc = afwTable.MatchControl()
mc.symmetricMatch = symmetric
matches = afwTable.matchXy(self.cat2, self.matchRadius, mc)
if False:
for m in matches:
print m.first.getId(), m.second.getId(), m.distance
# There is only one source that matches another source when we do a self-match: the one
# offset by 2 pixels and a bit.
# If we're getting symmetric matches, that multiplies the expected number by 2 because it
# produces s1,s2 and s2,s1.
self.assertEquals(len(matches), 2 if symmetric else 1)
def find(self, objId, matchRadius=10):
"""Return the object's family (you may specify either the ID for the parent or a child)"""
x, y = None, None
try:
x, y = objId
except TypeError:
pass
if x is not None:
oneObjCatalog = afwTable.SourceCatalog(self.cat.getSchema())
centroidName = self.cat.table.getCentroidDefinition()
oneObjCatalog.table.defineCentroid(centroidName)
s = oneObjCatalog.addNew()
s.set("%s_x" % centroidName, x)
s.set("%s_y" % centroidName, y)
matched = afwTable.matchXy(self.cat, oneObjCatalog, matchRadius)
if len(matched) == 0:
print >> sys.stderr, "Unable to find object at (%.2f, %.2f)" % (
x, y)
return None
if False:
objId = self.mapperInfo.splitId(matched[0][0].getId(),
asDict=True)["objId"]
else:
objId = matched[0][0].getId()
if False:
family = [f for f in self if self.mapperInfo.getId(f[0]) == objId]
else:
family = [f for f in self if f[0].getId() == objId]
if family:
return family[0]
for family in self:
for child in family[1]:
if False:
if self.mapperInfo.getId(child) == objId:
return family
else:
if child.getId() == objId:
return family
return None
def find(self, objId, matchRadius=10):
"""Return the object's family (you may specify either the ID for the parent or a child)"""
x, y = None, None
try:
x, y = objId
except TypeError:
pass
if x is not None:
oneObjCatalog = afwTable.SourceCatalog(self.cat.getSchema())
centroidName = self.cat.table.getCentroidDefinition()
oneObjCatalog.table.defineCentroid(centroidName)
s = oneObjCatalog.addNew()
s.set("%s_x" % centroidName, x)
s.set("%s_y" % centroidName, y)
matched = afwTable.matchXy(self.cat, oneObjCatalog, matchRadius)
if len(matched) == 0:
print("Unable to find object at (%.2f, %.2f)" % (x, y), file=sys.stderr)
return None
if False:
objId = self.mapperInfo.splitId(matched[0][0].getId(), asDict=True)["objId"]
else:
objId = matched[0][0].getId()
if False:
family = [f for f in self if self.mapperInfo.getId(f[0]) == objId]
else:
family = [f for f in self if f[0].getId() == objId]
if family:
return family[0]
for family in self:
for child in family[1]:
if False:
if self.mapperInfo.getId(child) == objId:
return family
else:
if child.getId() == objId:
return family
return None
def testMeasureCentroid(self):
"""Test that we can instantiate and play with a measureCentroid"""
exposure = afwImage.makeExposure(self.mi)
self.ds.makeSources(self.ssMeasured)
ID = 1
for s in self.ssMeasured:
s.setId(ID)
ID += 1
foot = s.getFootprint()
bbox = foot.getBBox()
xc = (bbox.getMinX() + bbox.getMaxX()) // 2
yc = (bbox.getMinY() + bbox.getMaxY()) // 2
self.centroider.apply(s, exposure, afwGeom.Point2D(xc, yc))
if display:
ds9.dot("x", c.getX(), c.getY(), ctype=ds9.GREEN)
#
# OK, we've measured all the sources. Compare positions with Dave Monet's values
#
# FIXME: this test will fail until source matching in afw is updated to use afw/table
mat = afwTable.matchXy(self.ssTruth, self.ssMeasured, 1.0)
#self.assertEqual(ID, len(mat)) # we matched all the input sources
eps = 6e-6 # offset in pixels between measured centroid and the Truth
for match in mat:
dx = match[0].getX() - match[1].getX()
dy = match[0].getY() - match[1].getY()
good = True if math.hypot(dx, dy) < eps else False
if not good:
msg = "Star at (%.1f, %.1f): (dx, dy) = %g, %g)" % \
(match[0].getXAstrom(), match[0].getYAstrom(), dx, dy)
if True:
print msg
else:
self.assertTrue(good, msg)
def testMeasureCentroid(self):
"""Test that we can instantiate and play with a measureCentroid"""
exposure = afwImage.makeExposure(self.mi)
self.ds.makeSources(self.ssMeasured)
ID = 1
self.task.run(self.ssMeasured, exposure)
for s in self.ssMeasured:
s.setId(ID)
ID += 1
foot = s.getFootprint()
bbox = foot.getBBox()
xc = (bbox.getMinX() + bbox.getMaxX()) // 2
yc = (bbox.getMinY() + bbox.getMaxY()) // 2
if display:
ds9.dot("x", xc, yc, ctype=ds9.GREEN)
#
# OK, we've measured all the sources. Compare positions with Dave Monet's values
#
mat = afwTable.matchXy(self.ssTruth, self.ssMeasured, 1.0)
self.assertEqual(ID, len(mat)) # we matched all the input sources
# offset in pixels between measured centroid and the Truth
eps = 6e-6
for match in mat:
dx = match[0].getX() - match[1].getX()
dy = match[0].getY() - match[1].getY()
good = True if math.hypot(dx, dy) < eps else False
if not good:
msg = "Star at (%.1f, %.1f): (dx, dy) = %g, %g)" % \
(match[0].getXAstrom(), match[0].getYAstrom(), dx, dy)
if True:
print(msg)
else:
self.assertTrue(good, msg)
def testMeasureCentroid(self):
"""Test that we can instantiate and play with a measureCentroid"""
exposure = afwImage.makeExposure(self.mi)
self.ds.makeSources(self.ssMeasured)
ID = 1
for s in self.ssMeasured:
s.setId(ID); ID += 1
foot = s.getFootprint()
bbox = foot.getBBox()
xc = (bbox.getMinX() + bbox.getMaxX())//2
yc = (bbox.getMinY() + bbox.getMaxY())//2
self.centroider.apply(s, exposure, afwGeom.Point2D(xc, yc))
if display:
ds9.dot("x", c.getX(), c.getY(), ctype=ds9.GREEN)
#
# OK, we've measured all the sources. Compare positions with Dave Monet's values
#
# FIXME: this test will fail until source matching in afw is updated to use afw/table
mat = afwTable.matchXy(self.ssTruth, self.ssMeasured, 1.0)
#self.assertEqual(ID, len(mat)) # we matched all the input sources
eps = 6e-6 # offset in pixels between measured centroid and the Truth
for match in mat:
dx = match[0].getX() - match[1].getX()
dy = match[0].getY() - match[1].getY()
good = True if math.hypot(dx, dy) < eps else False
if not good:
msg = "Star at (%.1f, %.1f): (dx, dy) = %g, %g)" % \
(match[0].getXAstrom(), match[0].getYAstrom(), dx, dy)
if True:
print msg
else:
self.assertTrue(good, msg)
def copyCalibrationFields(self, calibCat, sourceCat, fieldsToCopy):
"""Match sources in calibCat and sourceCat and copy the specified fields
Parameters
----------
calibCat : `lsst.afw.table.SourceCatalog`
Catalog from which to copy fields.
sourceCat : `lsst.afw.table.SourceCatalog`
Catalog to which to copy fields.
fieldsToCopy : `lsst.pex.config.listField.List`
Fields to copy from calibCat to SoourceCat.
Returns
-------
newCat : `lsst.afw.table.SourceCatalog`
Catalog which includes the copied fields.
The fields copied are those specified by `fieldsToCopy` that actually exist
in the schema of `calibCat`.
This version was based on and adapted from the one in calibrateTask.
"""
# Make a new SourceCatalog with the data from sourceCat so that we can add the new columns to it
sourceSchemaMapper = afwTable.SchemaMapper(sourceCat.schema)
sourceSchemaMapper.addMinimalSchema(sourceCat.schema, True)
calibSchemaMapper = afwTable.SchemaMapper(calibCat.schema,
sourceCat.schema)
# Add the desired columns from the option fieldsToCopy
missingFieldNames = []
for fieldName in fieldsToCopy:
if fieldName in calibCat.schema:
schemaItem = calibCat.schema.find(fieldName)
calibSchemaMapper.editOutputSchema().addField(
schemaItem.getField())
schema = calibSchemaMapper.editOutputSchema()
calibSchemaMapper.addMapping(schemaItem.getKey(),
schema.find(fieldName).getField())
else:
missingFieldNames.append(fieldName)
if missingFieldNames:
raise RuntimeError(
f"calibCat is missing fields {missingFieldNames} specified in "
"fieldsToCopy")
if "calib_detected" not in calibSchemaMapper.getOutputSchema():
self.calibSourceKey = calibSchemaMapper.addOutputField(
afwTable.Field["Flag"]("calib_detected",
"Source was detected as an icSource"))
else:
self.calibSourceKey = None
schema = calibSchemaMapper.getOutputSchema()
newCat = afwTable.SourceCatalog(schema)
newCat.reserve(len(sourceCat))
newCat.extend(sourceCat, sourceSchemaMapper)
# Set the aliases so it doesn't complain.
for k, v in sourceCat.schema.getAliasMap().items():
newCat.schema.getAliasMap().set(k, v)
select = newCat["deblend_nChild"] == 0
matches = afwTable.matchXy(newCat[select], calibCat,
self.config.matchRadiusPix)
# Check that no sourceCat sources are listed twice (we already know
# that each match has a unique calibCat source ID, due to using
# that ID as the key in bestMatches)
numMatches = len(matches)
numUniqueSources = len(set(m[1].getId() for m in matches))
if numUniqueSources != numMatches:
self.log.warn(
"%d calibCat sources matched only %d sourceCat sources",
numMatches, numUniqueSources)
self.log.info(
"Copying flags from calibCat to sourceCat for %s sources",
numMatches)
# For each match: set the calibSourceKey flag and copy the desired
# fields
for src, calibSrc, d in matches:
if self.calibSourceKey:
src.setFlag(self.calibSourceKey, True)
# src.assign copies the footprint from calibSrc, which we don't want
# (DM-407)
# so set calibSrc's footprint to src's footprint before src.assign,
# then restore it
calibSrcFootprint = calibSrc.getFootprint()
try:
calibSrc.setFootprint(src.getFootprint())
src.assign(calibSrc, calibSchemaMapper)
finally:
calibSrc.setFootprint(calibSrcFootprint)
return newCat
def run(self, sensorRef, templateIdList=None):
"""Subtract an image from a template coadd and measure the result
Steps include:
- warp template coadd to match WCS of image
- PSF match image to warped template
- subtract image from PSF-matched, warped template
- persist difference image
- detect sources
- measure sources
@param sensorRef: sensor-level butler data reference, used for the following data products:
Input only:
- calexp
- psf
- ccdExposureId
- ccdExposureId_bits
- self.config.coaddName + "Coadd_skyMap"
- self.config.coaddName + "Coadd"
Input or output, depending on config:
- self.config.coaddName + "Diff_subtractedExp"
Output, depending on config:
- self.config.coaddName + "Diff_matchedExp"
- self.config.coaddName + "Diff_src"
@return pipe_base Struct containing these fields:
- subtractedExposure: exposure after subtracting template;
the unpersisted version if subtraction not run but detection run
None if neither subtraction nor detection run (i.e. nothing useful done)
- subtractRes: results of subtraction task; None if subtraction not run
- sources: detected and possibly measured sources; None if detection not run
"""
self.log.info("Processing %s" % (sensorRef.dataId))
# initialize outputs and some intermediate products
subtractedExposure = None
subtractRes = None
selectSources = None
kernelSources = None
controlSources = None
diaSources = None
# We make one IdFactory that will be used by both icSrc and src datasets;
# I don't know if this is the way we ultimately want to do things, but at least
# this ensures the source IDs are fully unique.
expBits = sensorRef.get("ccdExposureId_bits")
expId = int(sensorRef.get("ccdExposureId"))
idFactory = afwTable.IdFactory.makeSource(expId, 64 - expBits)
# Retrieve the science image we wish to analyze
exposure = sensorRef.get("calexp", immediate=True)
if self.config.doAddCalexpBackground:
mi = exposure.getMaskedImage()
mi += sensorRef.get("calexpBackground").getImage()
if not exposure.hasPsf():
raise pipeBase.TaskError("Exposure has no psf")
sciencePsf = exposure.getPsf()
subtractedExposureName = self.config.coaddName + "Diff_differenceExp"
templateExposure = None # Stitched coadd exposure
templateSources = None # Sources on the template image
if self.config.doSubtract:
template = self.getTemplate.run(exposure,
sensorRef,
templateIdList=templateIdList)
templateExposure = template.exposure
templateSources = template.sources
# compute scienceSigmaOrig: sigma of PSF of science image before pre-convolution
scienceSigmaOrig = sciencePsf.computeShape().getDeterminantRadius()
# sigma of PSF of template image before warping
templateSigma = templateExposure.getPsf().computeShape(
).getDeterminantRadius()
# if requested, convolve the science exposure with its PSF
# (properly, this should be a cross-correlation, but our code does not yet support that)
# compute scienceSigmaPost: sigma of science exposure with pre-convolution, if done,
# else sigma of original science exposure
if self.config.doPreConvolve:
convControl = afwMath.ConvolutionControl()
# cannot convolve in place, so make a new MI to receive convolved image
srcMI = exposure.getMaskedImage()
destMI = srcMI.Factory(srcMI.getDimensions())
srcPsf = sciencePsf
if self.config.useGaussianForPreConvolution:
# convolve with a simplified PSF model: a double Gaussian
kWidth, kHeight = sciencePsf.getLocalKernel(
).getDimensions()
preConvPsf = SingleGaussianPsf(kWidth, kHeight,
scienceSigmaOrig)
else:
# convolve with science exposure's PSF model
preConvPsf = srcPsf
afwMath.convolve(destMI, srcMI, preConvPsf.getLocalKernel(),
convControl)
exposure.setMaskedImage(destMI)
scienceSigmaPost = scienceSigmaOrig * math.sqrt(2)
else:
scienceSigmaPost = scienceSigmaOrig
# If requested, find sources in the image
if self.config.doSelectSources:
if not sensorRef.datasetExists("src"):
self.log.warn(
"Src product does not exist; running detection, measurement, selection"
)
# Run own detection and measurement; necessary in nightly processing
selectSources = self.subtract.getSelectSources(
exposure,
sigma=scienceSigmaPost,
doSmooth=not self.doPreConvolve,
idFactory=idFactory,
)
else:
self.log.info("Source selection via src product")
# Sources already exist; for data release processing
selectSources = sensorRef.get("src")
# Number of basis functions
nparam = len(
makeKernelBasisList(
self.subtract.config.kernel.active,
referenceFwhmPix=scienceSigmaPost * FwhmPerSigma,
targetFwhmPix=templateSigma * FwhmPerSigma))
if self.config.doAddMetrics:
# Modify the schema of all Sources
kcQa = KernelCandidateQa(nparam)
selectSources = kcQa.addToSchema(selectSources)
if self.config.kernelSourcesFromRef:
# match exposure sources to reference catalog
astromRet = self.astrometer.loadAndMatch(
exposure=exposure, sourceCat=selectSources)
matches = astromRet.matches
elif templateSources:
# match exposure sources to template sources
mc = afwTable.MatchControl()
mc.findOnlyClosest = False
matches = afwTable.matchRaDec(templateSources,
selectSources,
1.0 * afwGeom.arcseconds, mc)
else:
raise RuntimeError(
"doSelectSources=True and kernelSourcesFromRef=False,"
+
"but template sources not available. Cannot match science "
+
"sources with template sources. Run process* on data from "
+ "which templates are built.")
kernelSources = self.sourceSelector.selectStars(
exposure, selectSources, matches=matches).starCat
random.shuffle(kernelSources, random.random)
controlSources = kernelSources[::self.config.controlStepSize]
kernelSources = [
k for i, k in enumerate(kernelSources)
if i % self.config.controlStepSize
]
if self.config.doSelectDcrCatalog:
redSelector = DiaCatalogSourceSelectorTask(
DiaCatalogSourceSelectorConfig(
grMin=self.sourceSelector.config.grMax,
grMax=99.999))
redSources = redSelector.selectStars(
exposure, selectSources, matches=matches).starCat
controlSources.extend(redSources)
blueSelector = DiaCatalogSourceSelectorTask(
DiaCatalogSourceSelectorConfig(
grMin=-99.999,
grMax=self.sourceSelector.config.grMin))
blueSources = blueSelector.selectStars(
exposure, selectSources, matches=matches).starCat
controlSources.extend(blueSources)
if self.config.doSelectVariableCatalog:
varSelector = DiaCatalogSourceSelectorTask(
DiaCatalogSourceSelectorConfig(includeVariable=True))
varSources = varSelector.selectStars(
exposure, selectSources, matches=matches).starCat
controlSources.extend(varSources)
self.log.info(
"Selected %d / %d sources for Psf matching (%d for control sample)"
% (len(kernelSources), len(selectSources),
len(controlSources)))
allresids = {}
if self.config.doUseRegister:
self.log.info("Registering images")
if templateSources is None:
# Run detection on the template, which is
# temporarily background-subtracted
templateSources = self.subtract.getSelectSources(
templateExposure,
sigma=templateSigma,
doSmooth=True,
idFactory=idFactory)
# Third step: we need to fit the relative astrometry.
#
wcsResults = self.fitAstrometry(templateSources,
templateExposure,
selectSources)
warpedExp = self.register.warpExposure(templateExposure,
wcsResults.wcs,
exposure.getWcs(),
exposure.getBBox())
templateExposure = warpedExp
# Create debugging outputs on the astrometric
# residuals as a function of position. Persistence
# not yet implemented; expected on (I believe) #2636.
if self.config.doDebugRegister:
# Grab matches to reference catalog
srcToMatch = {x.second.getId(): x.first for x in matches}
refCoordKey = wcsResults.matches[0].first.getTable(
).getCoordKey()
inCentroidKey = wcsResults.matches[0].second.getTable(
).getCentroidKey()
sids = [m.first.getId() for m in wcsResults.matches]
positions = [
m.first.get(refCoordKey) for m in wcsResults.matches
]
residuals = [
m.first.get(refCoordKey).getOffsetFrom(
wcsResults.wcs.pixelToSky(
m.second.get(inCentroidKey)))
for m in wcsResults.matches
]
allresids = dict(zip(sids, zip(positions, residuals)))
cresiduals = [
m.first.get(refCoordKey).getTangentPlaneOffset(
wcsResults.wcs.pixelToSky(
m.second.get(inCentroidKey)))
for m in wcsResults.matches
]
colors = numpy.array([
-2.5 * numpy.log10(srcToMatch[x].get("g")) +
2.5 * numpy.log10(srcToMatch[x].get("r")) for x in sids
if x in srcToMatch.keys()
])
dlong = numpy.array([
r[0].asArcseconds() for s, r in zip(sids, cresiduals)
if s in srcToMatch.keys()
])
dlat = numpy.array([
r[1].asArcseconds() for s, r in zip(sids, cresiduals)
if s in srcToMatch.keys()
])
idx1 = numpy.where(
colors < self.sourceSelector.config.grMin)
idx2 = numpy.where(
(colors >= self.sourceSelector.config.grMin)
& (colors <= self.sourceSelector.config.grMax))
idx3 = numpy.where(
colors > self.sourceSelector.config.grMax)
rms1Long = IqrToSigma * \
(numpy.percentile(dlong[idx1], 75)-numpy.percentile(dlong[idx1], 25))
rms1Lat = IqrToSigma * (numpy.percentile(dlat[idx1], 75) -
numpy.percentile(dlat[idx1], 25))
rms2Long = IqrToSigma * \
(numpy.percentile(dlong[idx2], 75)-numpy.percentile(dlong[idx2], 25))
rms2Lat = IqrToSigma * (numpy.percentile(dlat[idx2], 75) -
numpy.percentile(dlat[idx2], 25))
rms3Long = IqrToSigma * \
(numpy.percentile(dlong[idx3], 75)-numpy.percentile(dlong[idx3], 25))
rms3Lat = IqrToSigma * (numpy.percentile(dlat[idx3], 75) -
numpy.percentile(dlat[idx3], 25))
self.log.info("Blue star offsets'': %.3f %.3f, %.3f %.3f" %
(numpy.median(dlong[idx1]), rms1Long,
numpy.median(dlat[idx1]), rms1Lat))
self.log.info(
"Green star offsets'': %.3f %.3f, %.3f %.3f" %
(numpy.median(dlong[idx2]), rms2Long,
numpy.median(dlat[idx2]), rms2Lat))
self.log.info("Red star offsets'': %.3f %.3f, %.3f %.3f" %
(numpy.median(dlong[idx3]), rms3Long,
numpy.median(dlat[idx3]), rms3Lat))
self.metadata.add("RegisterBlueLongOffsetMedian",
numpy.median(dlong[idx1]))
self.metadata.add("RegisterGreenLongOffsetMedian",
numpy.median(dlong[idx2]))
self.metadata.add("RegisterRedLongOffsetMedian",
numpy.median(dlong[idx3]))
self.metadata.add("RegisterBlueLongOffsetStd", rms1Long)
self.metadata.add("RegisterGreenLongOffsetStd", rms2Long)
self.metadata.add("RegisterRedLongOffsetStd", rms3Long)
self.metadata.add("RegisterBlueLatOffsetMedian",
numpy.median(dlat[idx1]))
self.metadata.add("RegisterGreenLatOffsetMedian",
numpy.median(dlat[idx2]))
self.metadata.add("RegisterRedLatOffsetMedian",
numpy.median(dlat[idx3]))
self.metadata.add("RegisterBlueLatOffsetStd", rms1Lat)
self.metadata.add("RegisterGreenLatOffsetStd", rms2Lat)
self.metadata.add("RegisterRedLatOffsetStd", rms3Lat)
# warp template exposure to match exposure,
# PSF match template exposure to exposure,
# then return the difference
# Return warped template... Construct sourceKernelCand list after subtract
self.log.info("Subtracting images")
subtractRes = self.subtract.subtractExposures(
templateExposure=templateExposure,
scienceExposure=exposure,
candidateList=kernelSources,
convolveTemplate=self.config.convolveTemplate,
doWarping=not self.config.doUseRegister)
subtractedExposure = subtractRes.subtractedExposure
if self.config.doWriteMatchedExp:
sensorRef.put(subtractRes.matchedExposure,
self.config.coaddName + "Diff_matchedExp")
if self.config.doDetection:
self.log.info("Computing diffim PSF")
if subtractedExposure is None:
subtractedExposure = sensorRef.get(subtractedExposureName)
# Get Psf from the appropriate input image if it doesn't exist
if not subtractedExposure.hasPsf():
if self.config.convolveTemplate:
subtractedExposure.setPsf(exposure.getPsf())
else:
if templateExposure is None:
template = self.getTemplate.run(
exposure, sensorRef, templateIdList=templateIdList)
subtractedExposure.setPsf(template.exposure.getPsf())
# If doSubtract is False, then subtractedExposure was fetched from disk (above), thus it may have
# already been decorrelated. Thus, we do not do decorrelation if doSubtract is False.
if self.config.doDecorrelation and self.config.doSubtract:
decorrResult = self.decorrelate.run(exposure, templateExposure,
subtractedExposure,
subtractRes.psfMatchingKernel)
subtractedExposure = decorrResult.correctedExposure
if self.config.doDetection:
self.log.info("Running diaSource detection")
# Erase existing detection mask planes
mask = subtractedExposure.getMaskedImage().getMask()
mask &= ~(mask.getPlaneBitMask("DETECTED")
| mask.getPlaneBitMask("DETECTED_NEGATIVE"))
table = afwTable.SourceTable.make(self.schema, idFactory)
table.setMetadata(self.algMetadata)
results = self.detection.makeSourceCatalog(
table=table,
exposure=subtractedExposure,
doSmooth=not self.config.doPreConvolve)
if self.config.doMerge:
fpSet = results.fpSets.positive
fpSet.merge(results.fpSets.negative, self.config.growFootprint,
self.config.growFootprint, False)
diaSources = afwTable.SourceCatalog(table)
fpSet.makeSources(diaSources)
self.log.info("Merging detections into %d sources" %
(len(diaSources)))
else:
diaSources = results.sources
if self.config.doMeasurement:
self.log.info("Running diaSource measurement")
if not self.config.doDipoleFitting:
self.measurement.run(diaSources, subtractedExposure)
else:
if self.config.doSubtract:
self.measurement.run(diaSources, subtractedExposure,
exposure,
subtractRes.matchedExposure)
else:
self.measurement.run(diaSources, subtractedExposure,
exposure)
# Match with the calexp sources if possible
if self.config.doMatchSources:
if sensorRef.datasetExists("src"):
# Create key,val pair where key=diaSourceId and val=sourceId
matchRadAsec = self.config.diaSourceMatchRadius
matchRadPixel = matchRadAsec / exposure.getWcs(
).pixelScale().asArcseconds()
srcMatches = afwTable.matchXy(sensorRef.get("src"),
diaSources, matchRadPixel)
srcMatchDict = dict([(srcMatch.second.getId(),
srcMatch.first.getId())
for srcMatch in srcMatches])
self.log.info("Matched %d / %d diaSources to sources" %
(len(srcMatchDict), len(diaSources)))
else:
self.log.warn(
"Src product does not exist; cannot match with diaSources"
)
srcMatchDict = {}
# Create key,val pair where key=diaSourceId and val=refId
refAstromConfig = AstrometryConfig()
refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec
refAstrometer = AstrometryTask(refAstromConfig)
astromRet = refAstrometer.run(exposure=exposure,
sourceCat=diaSources)
refMatches = astromRet.matches
if refMatches is None:
self.log.warn(
"No diaSource matches with reference catalog")
refMatchDict = {}
else:
self.log.info(
"Matched %d / %d diaSources to reference catalog" %
(len(refMatches), len(diaSources)))
refMatchDict = dict([(refMatch.second.getId(),
refMatch.first.getId())
for refMatch in refMatches])
# Assign source Ids
for diaSource in diaSources:
sid = diaSource.getId()
if sid in srcMatchDict:
diaSource.set("srcMatchId", srcMatchDict[sid])
if sid in refMatchDict:
diaSource.set("refMatchId", refMatchDict[sid])
if diaSources is not None and self.config.doWriteSources:
sensorRef.put(diaSources,
self.config.coaddName + "Diff_diaSrc")
if self.config.doAddMetrics and self.config.doSelectSources:
self.log.info("Evaluating metrics and control sample")
kernelCandList = []
for cell in subtractRes.kernelCellSet.getCellList():
for cand in cell.begin(False): # include bad candidates
kernelCandList.append(cand)
# Get basis list to build control sample kernels
basisList = kernelCandList[0].getKernel(
KernelCandidateF.ORIG).getKernelList()
controlCandList = \
diffimTools.sourceTableToCandidateList(controlSources,
subtractRes.warpedExposure, exposure,
self.config.subtract.kernel.active,
self.config.subtract.kernel.active.detectionConfig,
self.log, doBuild=True, basisList=basisList)
kcQa.apply(kernelCandList,
subtractRes.psfMatchingKernel,
subtractRes.backgroundModel,
dof=nparam)
kcQa.apply(controlCandList, subtractRes.psfMatchingKernel,
subtractRes.backgroundModel)
if self.config.doDetection:
kcQa.aggregate(selectSources, self.metadata, allresids,
diaSources)
else:
kcQa.aggregate(selectSources, self.metadata, allresids)
sensorRef.put(selectSources,
self.config.coaddName + "Diff_kernelSrc")
if self.config.doWriteSubtractedExp:
sensorRef.put(subtractedExposure, subtractedExposureName)
self.runDebug(exposure, subtractRes, selectSources, kernelSources,
diaSources)
return pipeBase.Struct(
subtractedExposure=subtractedExposure,
subtractRes=subtractRes,
sources=diaSources,
)
def copyIcSourceFields(self, icSourceCat, sourceCat):
"""!Match sources in icSourceCat and sourceCat and copy the specified fields
@param[in] icSourceCat catalog from which to copy fields
@param[in,out] sourceCat catalog to which to copy fields
The fields copied are those specified by `config.icSourceFieldsToCopy`
that actually exist in the schema. This was set up by the constructor
using self.schemaMapper.
"""
if self.schemaMapper is None:
raise RuntimeError("To copy icSource fields you must specify "
"icSourceSchema nd icSourceKeys when "
"constructing this task")
if icSourceCat is None or sourceCat is None:
raise RuntimeError("icSourceCat and sourceCat must both be "
"specified")
if len(self.config.icSourceFieldsToCopy) == 0:
self.log.warn("copyIcSourceFields doing nothing because "
"icSourceFieldsToCopy is empty")
return
mc = afwTable.MatchControl()
mc.findOnlyClosest = False # return all matched objects
matches = afwTable.matchXy(icSourceCat, sourceCat,
self.config.matchRadiusPix, mc)
if self.config.doDeblend:
deblendKey = sourceCat.schema["deblend_nChild"].asKey()
# if deblended, keep children
matches = [m for m in matches if m[1].get(deblendKey) == 0]
# Because we had to allow multiple matches to handle parents, we now
# need to prune to the best matches
# closest matches as a dict of icSourceCat source ID:
# (icSourceCat source, sourceCat source, distance in pixels)
bestMatches = {}
for m0, m1, d in matches:
id0 = m0.getId()
match = bestMatches.get(id0)
if match is None or d <= match[2]:
bestMatches[id0] = (m0, m1, d)
matches = list(bestMatches.values())
# Check that no sourceCat sources are listed twice (we already know
# that each match has a unique icSourceCat source ID, due to using
# that ID as the key in bestMatches)
numMatches = len(matches)
numUniqueSources = len(set(m[1].getId() for m in matches))
if numUniqueSources != numMatches:
self.log.warn("{} icSourceCat sources matched only {} sourceCat "
"sources".format(numMatches, numUniqueSources))
self.log.info("Copying flags from icSourceCat to sourceCat for "
"%s sources" % (numMatches,))
# For each match: set the calibSourceKey flag and copy the desired
# fields
for icSrc, src, d in matches:
src.setFlag(self.calibSourceKey, True)
# src.assign copies the footprint from icSrc, which we don't want
# (DM-407)
# so set icSrc's footprint to src's footprint before src.assign,
# then restore it
icSrcFootprint = icSrc.getFootprint()
try:
icSrc.setFootprint(src.getFootprint())
src.assign(icSrc, self.schemaMapper)
finally:
icSrc.setFootprint(icSrcFootprint)
def testMatchXy(self):
matches = afwTable.matchXy(self.cat1, self.cat2, 0.01)
self.assertEquals(len(matches), 6)
for m in matches:
self.assertEquals(m.first.getId() + 10, m.second.getId())
self.assertEquals(m.distance, 0.0)
def doPlotWithDetectionsHighlighted(self, runTestResult=None, transientsOnly=True, addPresub=False,
xaxisIsScienceForcedPhot=False, alpha=0.5,
divideByInput=False, actuallyPlot=True, skyLimited=False,
matchDist=np.sqrt(1.5), **kwargs):
import lsst.afw.table as afwTable
import lsst.daf.base as dafBase
import lsst.afw.table.catalogMatches as catMatch
if actuallyPlot:
import matplotlib.pyplot as plt
import matplotlib
matplotlib.style.use('ggplot')
#fp_DIFFIM=fp_Zogy, label='Zogy', color='b', alpha=1.0,
res = runTestResult
if runTestResult is None or (runTestResult is not None and 'sources' not in runTestResult):
res = self.runTest(returnSources=True, matchDist=matchDist)
src = res['sources']
#del res['sources']
#print res
cats = self.doForcedPhot(transientsOnly=transientsOnly)
sources, fp1, fp2, fp_Zogy, fp_AL, fp_ALd = cats
# if xaxisIsScienceForcedPhot is True, then don't use sources['inputFlux_science'] --
# use fp2['base_PsfFlux_flux'] instead.
if not xaxisIsScienceForcedPhot:
srces = sources['inputFlux_science']
else:
srces = fp2['base_PsfFlux_flux']
df = pd.DataFrame()
df['inputFlux'] = sources['inputFlux_science']
df['templateFlux'] = fp1['base_PsfFlux_flux']
df['scienceFlux'] = fp2['base_PsfFlux_flux']
df['inputId'] = sources['id']
df['inputCentroid_x'] = sources['centroid_x']
df['inputCentroid_y'] = sources['centroid_y']
snrCalced = self.im2.calcSNR(sources['inputFlux_science'], skyLimited=skyLimited)
df['inputSNR'] = snrCalced
fp_DIFFIM = [fp_Zogy, fp_AL, fp_ALd]
label = ['Zogy', 'ALstack', 'ALstack_decorr']
color = ['b', 'r', 'g']
for i, fp_d in enumerate(fp_DIFFIM):
df[label[i] + '_SNR'] = fp_d['base_PsfFlux_flux']/fp_d['base_PsfFlux_fluxSigma']
df[label[i] + '_flux'] = fp_d['base_PsfFlux_flux']
df[label[i] + '_fluxSigma'] = fp_d['base_PsfFlux_fluxSigma']
if actuallyPlot:
# Plot all sources
yvals = fp_d['base_PsfFlux_flux']/fp_d['base_PsfFlux_fluxSigma']
if divideByInput:
yvals /= df['inputSNR']
plt.scatter(srces, yvals,
color=color[i], alpha=alpha, label=label[i])
#plt.scatter(srces,
# fp_d['base_PsfFlux_flux']/fp_d['base_PsfFlux_fluxSigma'],
# color='k', marker='x', alpha=alpha, label=None)
if not xaxisIsScienceForcedPhot:
matches = afwTable.matchXy(sources, src[label[i]], matchDist)
metadata = dafBase.PropertyList()
matchCat = catMatch.matchesToCatalog(matches, metadata)
sources_detected = catalogToDF(sources)
detected = np.in1d(sources_detected['id'], matchCat['ref_id'])
sources_detected = sources_detected[detected]
sources_detected = sources_detected['inputFlux_science']
snrCalced_detected = snrCalced[detected]
fp_Zogy_detected = catalogToDF(fp_d)
detected = np.in1d(fp_Zogy_detected['id'], matchCat['ref_id'])
fp_Zogy_detected = fp_Zogy_detected[detected]
else:
matches = afwTable.matchXy(fp2, src[label[i]], matchDist)
metadata = dafBase.PropertyList()
matchCat = catMatch.matchesToCatalog(matches, metadata)
sources_detected = catalogToDF(fp2)
detected = np.in1d(sources_detected['id'], matchCat['ref_id'])
sources_detected = sources_detected[detected]
sources_detected = sources_detected['base_PsfFlux_flux']
snrCalced_detected = snrCalced[detected]
fp_Zogy_detected = catalogToDF(fp_d)
detected = np.in1d(fp_Zogy_detected['id'], matchCat['ref_id'])
fp_Zogy_detected = fp_Zogy_detected[detected]
df[label[i] + '_detected'] = detected
if actuallyPlot and len(detected) > 0:
mStyle = matplotlib.markers.MarkerStyle('o', 'none')
yvals = fp_Zogy_detected['base_PsfFlux_flux']/fp_Zogy_detected['base_PsfFlux_fluxSigma']
if divideByInput:
yvals /= snrCalced_detected
plt.scatter(sources_detected, yvals,
#label=label[i], s=20, color=color[i], alpha=alpha) #, edgecolors='r')
label=None, s=30, edgecolors='r', facecolors='none', marker='o', alpha=1.0) # edgecolors=color[i],
if addPresub: # Add measurements in original science and template images
df['templateSNR'] = fp1['base_PsfFlux_flux']/fp1['base_PsfFlux_fluxSigma']
df['scienceSNR'] = fp2['base_PsfFlux_flux']/fp2['base_PsfFlux_fluxSigma']
if actuallyPlot:
yvals = fp1['base_PsfFlux_flux']/fp1['base_PsfFlux_fluxSigma']
if divideByInput:
yvals /= df['inputSNR']
plt.scatter(srces, yvals,
label='template', color='y', alpha=alpha)
yvals = fp2['base_PsfFlux_flux']/fp2['base_PsfFlux_fluxSigma']
if divideByInput:
yvals /= df['inputSNR']
plt.scatter(srces, yvals,
label='science', color='orange', alpha=alpha-0.2)
if actuallyPlot:
if not divideByInput:
if xaxisIsScienceForcedPhot:
plt.scatter(srces, snrCalced, color='k', alpha=alpha-0.2, s=7, label='Input SNR')
else:
plt.plot(srces, snrCalced, color='k', alpha=alpha-0.2, label='Input SNR')
plt.ylabel('measured SNR')
else:
plt.ylabel('measured SNR / input SNR')
if len(detected) > 0:
plt.scatter([10000], [0], s=30, edgecolors='r', facecolors='none', marker='o', label='Detected')
legend = plt.legend(loc='upper left', scatterpoints=3)
for label in legend.get_texts():
label.set_fontsize('x-small')
if not xaxisIsScienceForcedPhot:
plt.xlabel('input flux')
else:
plt.xlabel('science flux (measured)')
return df, res
def testMatchXy(self):
matches = afwTable.matchXy(self.cat1, self.cat2, 0.01)
self.assertEquals(len(matches), 6)
for m in matches:
self.assertEquals(m.first.getId() + 10, m.second.getId())
self.assertEquals(m.distance, 0.0)
def run(self, sensorRef):
"""Subtract an image from a template coadd and measure the result
Steps include:
- warp template coadd to match WCS of image
- PSF match image to warped template
- subtract image from PSF-matched, warped template
- persist difference image
- detect sources
- measure sources
@param sensorRef: sensor-level butler data reference, used for the following data products:
Input only:
- calexp
- psf
- ccdExposureId
- ccdExposureId_bits
- self.config.coaddName + "Coadd_skyMap"
- self.config.coaddName + "Coadd"
Input or output, depending on config:
- self.config.coaddName + "Diff_subtractedExp"
Output, depending on config:
- self.config.coaddName + "Diff_matchedExp"
- self.config.coaddName + "Diff_src"
@return pipe_base Struct containing these fields:
- subtractedExposure: exposure after subtracting template;
the unpersisted version if subtraction not run but detection run
None if neither subtraction nor detection run (i.e. nothing useful done)
- subtractRes: results of subtraction task; None if subtraction not run
- sources: detected and possibly measured sources; None if detection not run
"""
self.log.info("Processing %s" % (sensorRef.dataId))
# initialize outputs and some intermediate products
subtractedExposure = None
subtractRes = None
selectSources = None
kernelSources = None
controlSources = None
diaSources = None
# We make one IdFactory that will be used by both icSrc and src datasets;
# I don't know if this is the way we ultimately want to do things, but at least
# this ensures the source IDs are fully unique.
expBits = sensorRef.get("ccdExposureId_bits")
expId = long(sensorRef.get("ccdExposureId"))
idFactory = afwTable.IdFactory.makeSource(expId, 64 - expBits)
# Retrieve the science image we wish to analyze
exposure = sensorRef.get("calexp", immediate=True)
if self.config.doAddCalexpBackground:
mi = exposure.getMaskedImage()
mi += sensorRef.get("calexpBackground").getImage()
if not exposure.hasPsf():
raise pipeBase.TaskError("Exposure has no psf")
sciencePsf = exposure.getPsf()
if self.config.useWinter2013Hacks and self.config.winter2013borderMask > 0:
self.log.warn("USING WINTER2013 HACK: MASKING BORDER PIXELS")
bbox = exposure.getBBox(afwImage.PARENT)
bbox.grow(-self.config.winter2013borderMask)
self.setEdgeBits(exposure.getMaskedImage(), bbox,
exposure.getMaskedImage().getMask().getPlaneBitMask("NO_DATA"))
# compute scienceSigmaOrig: sigma of PSF of science image before pre-convolution
ctr = afwGeom.Box2D(exposure.getBBox(afwImage.PARENT)).getCenter()
psfAttr = PsfAttributes(sciencePsf, afwGeom.Point2I(ctr))
scienceSigmaOrig = psfAttr.computeGaussianWidth(psfAttr.ADAPTIVE_MOMENT)
subtractedExposureName = self.config.coaddName + "Diff_differenceExp"
templateExposure = None # Stitched coadd exposure
templateSources = None # Sources on the template image
if self.config.doSubtract:
templateExposure, templateSources = self.getTemplate(exposure, sensorRef)
# sigma of PSF of template image before warping
ctr = afwGeom.Box2D(templateExposure.getBBox(afwImage.PARENT)).getCenter()
psfAttr = PsfAttributes(templateExposure.getPsf(), afwGeom.Point2I(ctr))
templateSigma = psfAttr.computeGaussianWidth(psfAttr.ADAPTIVE_MOMENT)
# if requested, convolve the science exposure with its PSF
# (properly, this should be a cross-correlation, but our code does not yet support that)
# compute scienceSigmaPost: sigma of science exposure with pre-convolution, if done,
# else sigma of original science exposure
if self.config.doPreConvolve:
convControl = afwMath.ConvolutionControl()
# cannot convolve in place, so make a new MI to receive convolved image
srcMI = exposure.getMaskedImage()
destMI = srcMI.Factory(srcMI.getDimensions())
srcPsf = sciencePsf
if self.config.useGaussianForPreConvolution:
# convolve with a simplified PSF model: a double Gaussian
kWidth, kHeight = sciencePsf.getKernel().getDimensions()
preConvPsf = SingleGaussianPsf(kWidth, kHeight, scienceSigmaOrig)
else:
# convolve with science exposure's PSF model
preConvPsf = psf
afwMath.convolve(destMI, srcMI, preConvPsf.getKernel(), convControl)
exposure.setMaskedImage(destMI)
scienceSigmaPost = scienceSigmaOrig * math.sqrt(2)
else:
scienceSigmaPost = scienceSigmaOrig
# If requested, find sources in the image
if self.config.doSelectSources:
if not sensorRef.datasetExists("src"):
self.log.warn("Src product does not exist; running detection, measurement, selection")
# Run own detection and measurement; necessary in nightly processing
selectSources = self.subtract.getSelectSources(
exposure,
sigma = scienceSigmaPost,
doSmooth = not self.doPreConvolve,
idFactory = idFactory,
)
else:
self.log.info("Source selection via src product")
# Sources already exist; for data release processing
selectSources = sensorRef.get("src")
# Number of basis functions
nparam = len(makeKernelBasisList(self.subtract.config.kernel.active,
referenceFwhmPix = scienceSigmaPost * FwhmPerSigma,
targetFwhmPix = templateSigma * FwhmPerSigma))
if self.config.doAddMetrics:
# Modify the schema of all Sources
self.kcQa = diUtils.KernelCandidateQa(nparam, self.log)
selectSources = self.kcQa.addToSchema(selectSources)
astrometer = measAstrom.Astrometry(measAstrom.MeasAstromConfig())
astromRet = astrometer.useKnownWcs(selectSources, exposure=exposure)
matches = astromRet.matches
kernelSources = self.sourceSelector.selectSources(exposure, selectSources, matches=matches)
random.shuffle(kernelSources, random.random)
controlSources = kernelSources[::self.config.controlStepSize]
[kernelSources.remove(x) for x in controlSources]
self.log.info("Selected %d / %d sources for Psf matching (%d for control sample)" \
% (len(kernelSources), len(selectSources), len(controlSources)))
allresids = {}
if self.config.doUseRegister:
self.log.info("Registering images")
if templateSources is None:
# First step: we need to subtract the background out
# for detection and measurement. Use large binsize
# for the background estimation.
binsize = self.config.templateBgBinSize
# Second step: we need to run detection on the
# background-subtracted template
#
# Estimate FWHM for detection
templateSources = self.subtract.getSelectSources(
templateExposure,
sigma = templateSigma,
doSmooth = True,
idFactory = idFactory,
binsize = binsize,
)
# Third step: we need to fit the relative astrometry.
#
# One problem is that the SIP fits are w.r.t. CRPIX,
# and these coadd patches have the CRPIX of the entire
# tract, i.e. off the image. This causes
# register.fitWcs to fail. A workaround for now is to
# re-fit the Wcs which returns with a CRPIX that is on
# the image, and *then* to fit for the relative Wcs.
#
# Requires low Sip order to avoid overfitting
# useWinter2013Hacks includes us using the deep calexp
# as the template. In this case we don't need to
# refit the Wcs.
if not self.config.useWinter2013Hacks:
sipOrder = self.config.templateSipOrder
astrometer = measAstrom.Astrometry(measAstrom.MeasAstromConfig(sipOrder=sipOrder))
newWcs = astrometer.determineWcs(templateSources, templateExposure).getWcs()
results = self.register.run(templateSources, newWcs,
templateExposure.getBBox(afwImage.PARENT), selectSources)
else:
if self.config.winter2013WcsShift > 0.0:
offset = afwGeom.Extent2D(self.config.winter2013WcsShift,
self.config.winter2013WcsShift)
cKey = templateSources[0].getTable().getCentroidKey()
for source in templateSources:
centroid = source.get(cKey)
source.set(cKey, centroid+offset)
elif self.config.winter2013WcsRms > 0.0:
cKey = templateSources[0].getTable().getCentroidKey()
for source in templateSources:
offset = afwGeom.Extent2D(self.config.winter2013WcsRms*numpy.random.normal(),
self.config.winter2013WcsRms*numpy.random.normal())
centroid = source.get(cKey)
source.set(cKey, centroid+offset)
results = self.register.run(templateSources, templateExposure.getWcs(),
templateExposure.getBBox(afwImage.PARENT), selectSources)
warpedExp = self.register.warpExposure(templateExposure, results.wcs,
exposure.getWcs(), exposure.getBBox(afwImage.PARENT))
templateExposure = warpedExp
# Create debugging outputs on the astrometric
# residuals as a function of position. Persistence
# not yet implemented; expected on (I believe) #2636.
if self.config.doDebugRegister:
refCoordKey = results.matches[0].first.getTable().getCoordKey()
inCentroidKey = results.matches[0].second.getTable().getCentroidKey()
sids = [m.first.getId() for m in results.matches]
positions = [m.first.get(refCoordKey) for m in results.matches]
residuals = [m.first.get(refCoordKey).getOffsetFrom(
results.wcs.pixelToSky(m.second.get(inCentroidKey))) for
m in results.matches]
allresids = dict(zip(sids, zip(positions, residuals)))
# warp template exposure to match exposure,
# PSF match template exposure to exposure,
# then return the difference
#Return warped template... Construct sourceKernelCand list after subtract
self.log.info("Subtracting images")
subtractRes = self.subtract.subtractExposures(
templateExposure = templateExposure,
scienceExposure = exposure,
scienceFwhmPix = scienceSigmaPost * FwhmPerSigma,
templateFwhmPix = templateSigma * FwhmPerSigma,
candidateList = kernelSources,
convolveTemplate = self.config.convolveTemplate,
doWarping = not self.config.doUseRegister
)
subtractedExposure = subtractRes.subtractedExposure
if self.config.doWriteMatchedExp:
sensorRef.put(subtractRes.matchedExposure, self.config.coaddName + "Diff_matchedExp")
if self.config.doDetection:
self.log.info("Running diaSource detection")
if subtractedExposure is None:
subtractedExposure = sensorRef.get(subtractedExposureName)
# Get Psf from the appropriate input image if it doesn't exist
if not subtractedExposure.hasPsf():
if self.config.convolveTemplate:
subtractedExposure.setPsf(exposure.getPsf())
else:
if templateExposure is None:
templateExposure, templateSources = self.getTemplate(exposure, sensorRef)
subtractedExposure.setPsf(templateExposure.getPsf())
# Erase existing detection mask planes
mask = subtractedExposure.getMaskedImage().getMask()
mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
table = afwTable.SourceTable.make(self.schema, idFactory)
table.setMetadata(self.algMetadata)
results = self.detection.makeSourceCatalog(
table = table,
exposure = subtractedExposure,
doSmooth = not self.config.doPreConvolve
)
if self.config.doMerge:
fpSet = results.fpSets.positive
fpSet.merge(results.fpSets.negative, self.config.growFootprint,
self.config.growFootprint, False)
diaSources = afwTable.SourceCatalog(table)
fpSet.makeSources(diaSources)
self.log.info("Merging detections into %d sources" % (len(diaSources)))
else:
diaSources = results.sources
if self.config.doMeasurement:
self.log.info("Running diaSource measurement")
if len(diaSources) < self.config.maxDiaSourcesToMeasure:
self.dipolemeasurement.run(subtractedExposure, diaSources)
else:
self.measurement.run(subtractedExposure, diaSources)
# Match with the calexp sources if possible
if self.config.doMatchSources:
if sensorRef.datasetExists("src"):
# Create key,val pair where key=diaSourceId and val=sourceId
matchRadAsec = self.config.diaSourceMatchRadius
matchRadPixel = matchRadAsec / exposure.getWcs().pixelScale().asArcseconds()
# This does not do what I expect so I cobbled together a brute force method in python
srcMatches = afwTable.matchXy(sensorRef.get("src"), diaSources, matchRadPixel, True)
srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId()) for \
srcMatch in srcMatches])
self.log.info("Matched %d / %d diaSources to sources" % (len(srcMatchDict),
len(diaSources)))
else:
self.log.warn("Src product does not exist; cannot match with diaSources")
srcMatchDict = {}
# Create key,val pair where key=diaSourceId and val=refId
astrometer = measAstrom.Astrometry(measAstrom.MeasAstromConfig(catalogMatchDist=matchRadAsec))
astromRet = astrometer.useKnownWcs(diaSources, exposure=exposure)
refMatches = astromRet.matches
if refMatches is None:
self.log.warn("No diaSource matches with reference catalog")
refMatchDict = {}
else:
self.log.info("Matched %d / %d diaSources to reference catalog" % (len(refMatches),
len(diaSources)))
refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId()) for \
refMatch in refMatches])
# Assign source Ids
for diaSource in diaSources:
sid = diaSource.getId()
if srcMatchDict.has_key(sid):
diaSource.set("srcMatchId", srcMatchDict[sid])
if refMatchDict.has_key(sid):
diaSource.set("refMatchId", refMatchDict[sid])
if diaSources is not None and self.config.doWriteSources:
sourceWriteFlags = (0 if self.config.doWriteHeavyFootprintsInSources
else afwTable.SOURCE_IO_NO_HEAVY_FOOTPRINTS)
sensorRef.put(diaSources, self.config.coaddName + "Diff_diaSrc", flags=sourceWriteFlags)
if self.config.doAddMetrics and self.config.doSelectSources:
self.log.info("Evaluating metrics and control sample")
kernelCandList = []
for cell in subtractRes.kernelCellSet.getCellList():
for cand in cell.begin(False): # include bad candidates
kernelCandList.append(cast_KernelCandidateF(cand))
# Get basis list to build control sample kernels
basisList = afwMath.cast_LinearCombinationKernel(
kernelCandList[0].getKernel(KernelCandidateF.ORIG)).getKernelList()
controlCandList = \
diffimTools.sourceTableToCandList(controlSources, subtractRes.warpedExposure, exposure,
self.config.subtract.kernel.active,
self.config.subtract.kernel.active.detectionConfig,
self.log, dobuild=True, basisList=basisList)
self.kcQa.apply(kernelCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel,
dof=nparam)
self.kcQa.apply(controlCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel)
if self.config.doDetection:
self.kcQa.aggregate(selectSources, self.metadata, allresids, diaSources)
else:
self.kcQa.aggregate(selectSources, self.metadata, allresids)
#Persist using butler
sensorRef.put(selectSources, self.config.coaddName + "Diff_kernelSrc")
if self.config.doWriteSubtractedExp:
sensorRef.put(subtractedExposure, subtractedExposureName)
self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources)
return pipeBase.Struct(
subtractedExposure = subtractedExposure,
subtractRes = subtractRes,
sources = diaSources,
)
def catalogStatistics(self, exposure, catalog, uncorrectedCatalog,
statControl):
"""Measure the catalog statistics.
Parameters
----------
exposure : `lsst.afw.image.Exposure`
The exposure to measure.
catalog : `lsst.afw.table.Table`
The catalog to measure.
uncorrectedCatalog : `lsst.afw.table.Table`
The uncorrected catalog to measure.
statControl : `lsst.afw.math.StatisticsControl`
Statistics control object with parameters defined by
the config.
Returns
-------
outputStatistics : `dict` [`str`, `dict` [`str`, scalar]]
A dictionary indexed by the amplifier name, containing
dictionaries of the statistics measured and their values.
"""
outputStatistics = {}
matches = afwTable.matchXy(catalog, uncorrectedCatalog,
self.config.matchRadiusPix)
outputStatistics['NUM_MATCHES'] = len(matches)
magnitude = []
sizeDiff = []
if not matches:
outputStatistics['MAGNITUDES'] = magnitude
outputStatistics['SIZE_DIFF'] = sizeDiff
return outputStatistics
for source, uncorrectedSource, d in matches:
# This uses the simple difference in source moments.
sourceMagnitude = -2.5 * np.log10(source.getPsfInstFlux())
sourceSize = source['base_SdssShape_xx'] + source[
'base_SdssShape_yy']
uncorrectedSize = uncorrectedSource[
'base_SdssShape_xx'] + uncorrectedSource['base_SdssShape_yy']
magnitude.append(float(sourceMagnitude))
sizeDiff.append(float(uncorrectedSize - sourceSize))
mask = np.isfinite(magnitude) * np.isfinite(sizeDiff)
if 'BRIGHT_SLOPE' in self.config.catalogStatKeywords:
exponentialFit = least_squares(modelResidual, [0.0, -0.01, 0.0],
args=(np.array(magnitude)[mask],
np.array(sizeDiff)[mask]),
loss='cauchy')
outputStatistics['BRIGHT_SLOPE'] = float(exponentialFit.x[1])
outputStatistics['FIT_SUCCESS'] = exponentialFit.success
self.debugFit('brightSlope', magnitude, sizeDiff, exponentialFit.x)
outputStatistics['MAGNITUDES'] = magnitude
outputStatistics['SIZE_DIFF'] = sizeDiff
return outputStatistics
def run(self, sensorRef, templateIdList=None):
"""Subtract an image from a template coadd and measure the result
Steps include:
- warp template coadd to match WCS of image
- PSF match image to warped template
- subtract image from PSF-matched, warped template
- persist difference image
- detect sources
- measure sources
@param sensorRef: sensor-level butler data reference, used for the following data products:
Input only:
- calexp
- psf
- ccdExposureId
- ccdExposureId_bits
- self.config.coaddName + "Coadd_skyMap"
- self.config.coaddName + "Coadd"
Input or output, depending on config:
- self.config.coaddName + "Diff_subtractedExp"
Output, depending on config:
- self.config.coaddName + "Diff_matchedExp"
- self.config.coaddName + "Diff_src"
@return pipe_base Struct containing these fields:
- subtractedExposure: exposure after subtracting template;
the unpersisted version if subtraction not run but detection run
None if neither subtraction nor detection run (i.e. nothing useful done)
- subtractRes: results of subtraction task; None if subtraction not run
- sources: detected and possibly measured sources; None if detection not run
"""
self.log.info("Processing %s" % (sensorRef.dataId))
# initialize outputs and some intermediate products
subtractedExposure = None
subtractRes = None
selectSources = None
kernelSources = None
controlSources = None
diaSources = None
# We make one IdFactory that will be used by both icSrc and src datasets;
# I don't know if this is the way we ultimately want to do things, but at least
# this ensures the source IDs are fully unique.
expBits = sensorRef.get("ccdExposureId_bits")
expId = long(sensorRef.get("ccdExposureId"))
idFactory = afwTable.IdFactory.makeSource(expId, 64 - expBits)
# Retrieve the science image we wish to analyze
exposure = sensorRef.get("calexp", immediate=True)
if self.config.doAddCalexpBackground:
mi = exposure.getMaskedImage()
mi += sensorRef.get("calexpBackground").getImage()
if not exposure.hasPsf():
raise pipeBase.TaskError("Exposure has no psf")
sciencePsf = exposure.getPsf()
subtractedExposureName = self.config.coaddName + "Diff_differenceExp"
templateExposure = None # Stitched coadd exposure
templateSources = None # Sources on the template image
if self.config.doSubtract:
print templateIdList
template = self.getTemplate.run(exposure, sensorRef, templateIdList=templateIdList)
templateExposure = template.exposure
templateSources = template.sources
# compute scienceSigmaOrig: sigma of PSF of science image before pre-convolution
ctr = afwGeom.Box2D(exposure.getBBox()).getCenter()
psfAttr = PsfAttributes(sciencePsf, afwGeom.Point2I(ctr))
scienceSigmaOrig = psfAttr.computeGaussianWidth(psfAttr.ADAPTIVE_MOMENT)
# sigma of PSF of template image before warping
ctr = afwGeom.Box2D(templateExposure.getBBox()).getCenter()
psfAttr = PsfAttributes(templateExposure.getPsf(), afwGeom.Point2I(ctr))
templateSigma = psfAttr.computeGaussianWidth(psfAttr.ADAPTIVE_MOMENT)
# if requested, convolve the science exposure with its PSF
# (properly, this should be a cross-correlation, but our code does not yet support that)
# compute scienceSigmaPost: sigma of science exposure with pre-convolution, if done,
# else sigma of original science exposure
if self.config.doPreConvolve:
convControl = afwMath.ConvolutionControl()
# cannot convolve in place, so make a new MI to receive convolved image
srcMI = exposure.getMaskedImage()
destMI = srcMI.Factory(srcMI.getDimensions())
srcPsf = sciencePsf
if self.config.useGaussianForPreConvolution:
# convolve with a simplified PSF model: a double Gaussian
kWidth, kHeight = sciencePsf.getLocalKernel().getDimensions()
preConvPsf = SingleGaussianPsf(kWidth, kHeight, scienceSigmaOrig)
else:
# convolve with science exposure's PSF model
preConvPsf = srcPsf
afwMath.convolve(destMI, srcMI, preConvPsf.getLocalKernel(), convControl)
exposure.setMaskedImage(destMI)
scienceSigmaPost = scienceSigmaOrig * math.sqrt(2)
else:
scienceSigmaPost = scienceSigmaOrig
# If requested, find sources in the image
if self.config.doSelectSources:
if not sensorRef.datasetExists("src"):
self.log.warn("Src product does not exist; running detection, measurement, selection")
# Run own detection and measurement; necessary in nightly processing
selectSources = self.subtract.getSelectSources(
exposure,
sigma = scienceSigmaPost,
doSmooth = not self.doPreConvolve,
idFactory = idFactory,
)
else:
self.log.info("Source selection via src product")
# Sources already exist; for data release processing
selectSources = sensorRef.get("src")
# Number of basis functions
nparam = len(makeKernelBasisList(self.subtract.config.kernel.active,
referenceFwhmPix=scienceSigmaPost * FwhmPerSigma,
targetFwhmPix=templateSigma * FwhmPerSigma))
if self.config.doAddMetrics:
# Modify the schema of all Sources
kcQa = KernelCandidateQa(nparam)
selectSources = kcQa.addToSchema(selectSources)
if self.config.kernelSourcesFromRef:
# match exposure sources to reference catalog
astromRet = self.astrometer.loadAndMatch(exposure=exposure, sourceCat=selectSources)
matches = astromRet.matches
elif templateSources:
# match exposure sources to template sources
matches = afwTable.matchRaDec(templateSources, selectSources, 1.0*afwGeom.arcseconds,
False)
else:
raise RuntimeError("doSelectSources=True and kernelSourcesFromRef=False," +
"but template sources not available. Cannot match science " +
"sources with template sources. Run process* on data from " +
"which templates are built.")
kernelSources = self.sourceSelector.selectStars(exposure, selectSources,
matches=matches).starCat
random.shuffle(kernelSources, random.random)
controlSources = kernelSources[::self.config.controlStepSize]
kernelSources = [k for i,k in enumerate(kernelSources) if i % self.config.controlStepSize]
if self.config.doSelectDcrCatalog:
redSelector = DiaCatalogSourceSelectorTask(
DiaCatalogSourceSelectorConfig(grMin=self.sourceSelector.config.grMax, grMax=99.999))
redSources = redSelector.selectStars(exposure, selectSources, matches=matches).starCat
controlSources.extend(redSources)
blueSelector = DiaCatalogSourceSelectorTask(
DiaCatalogSourceSelectorConfig(grMin=-99.999, grMax=self.sourceSelector.config.grMin))
blueSources = blueSelector.selectStars(exposure, selectSources, matches=matches).starCat
controlSources.extend(blueSources)
if self.config.doSelectVariableCatalog:
varSelector = DiaCatalogSourceSelectorTask(
DiaCatalogSourceSelectorConfig(includeVariable=True))
varSources = varSelector.selectStars(exposure, selectSources, matches=matches).starCat
controlSources.extend(varSources)
self.log.info("Selected %d / %d sources for Psf matching (%d for control sample)"
% (len(kernelSources), len(selectSources), len(controlSources)))
allresids = {}
if self.config.doUseRegister:
self.log.info("Registering images")
if templateSources is None:
# Run detection on the template, which is
# temporarily background-subtracted
templateSources = self.subtract.getSelectSources(
templateExposure,
sigma=templateSigma,
doSmooth=True,
idFactory=idFactory
)
# Third step: we need to fit the relative astrometry.
#
wcsResults = self.fitAstrometry(templateSources, templateExposure, selectSources)
warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs,
exposure.getWcs(), exposure.getBBox())
templateExposure = warpedExp
# Create debugging outputs on the astrometric
# residuals as a function of position. Persistence
# not yet implemented; expected on (I believe) #2636.
if self.config.doDebugRegister:
# Grab matches to reference catalog
srcToMatch = {x.second.getId() : x.first for x in matches}
refCoordKey = wcsResults.matches[0].first.getTable().getCoordKey()
inCentroidKey = wcsResults.matches[0].second.getTable().getCentroidKey()
sids = [m.first.getId() for m in wcsResults.matches]
positions = [m.first.get(refCoordKey) for m in wcsResults.matches]
residuals = [m.first.get(refCoordKey).getOffsetFrom(wcsResults.wcs.pixelToSky(
m.second.get(inCentroidKey))) for m in wcsResults.matches]
allresids = dict(zip(sids, zip(positions, residuals)))
cresiduals = [m.first.get(refCoordKey).getTangentPlaneOffset(
wcsResults.wcs.pixelToSky(
m.second.get(inCentroidKey))) for m in wcsResults.matches]
colors = numpy.array([-2.5*numpy.log10(srcToMatch[x].get("g"))
+ 2.5*numpy.log10(srcToMatch[x].get("r"))
for x in sids if x in srcToMatch.keys()])
dlong = numpy.array([r[0].asArcseconds() for s,r in zip(sids, cresiduals)
if s in srcToMatch.keys()])
dlat = numpy.array([r[1].asArcseconds() for s,r in zip(sids, cresiduals)
if s in srcToMatch.keys()])
idx1 = numpy.where(colors<self.sourceSelector.config.grMin)
idx2 = numpy.where((colors>=self.sourceSelector.config.grMin)&
(colors<=self.sourceSelector.config.grMax))
idx3 = numpy.where(colors>self.sourceSelector.config.grMax)
rms1Long = IqrToSigma*(numpy.percentile(dlong[idx1],75)-numpy.percentile(dlong[idx1],25))
rms1Lat = IqrToSigma*(numpy.percentile(dlat[idx1],75)-numpy.percentile(dlat[idx1],25))
rms2Long = IqrToSigma*(numpy.percentile(dlong[idx2],75)-numpy.percentile(dlong[idx2],25))
rms2Lat = IqrToSigma*(numpy.percentile(dlat[idx2],75)-numpy.percentile(dlat[idx2],25))
rms3Long = IqrToSigma*(numpy.percentile(dlong[idx3],75)-numpy.percentile(dlong[idx3],25))
rms3Lat = IqrToSigma*(numpy.percentile(dlat[idx3],75)-numpy.percentile(dlat[idx3],25))
self.log.info("Blue star offsets'': %.3f %.3f, %.3f %.3f" % (numpy.median(dlong[idx1]),
rms1Long,
numpy.median(dlat[idx1]),
rms1Lat))
self.log.info("Green star offsets'': %.3f %.3f, %.3f %.3f" % (numpy.median(dlong[idx2]),
rms2Long,
numpy.median(dlat[idx2]),
rms2Lat))
self.log.info("Red star offsets'': %.3f %.3f, %.3f %.3f" % (numpy.median(dlong[idx3]),
rms3Long,
numpy.median(dlat[idx3]),
rms3Lat))
self.metadata.add("RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1]))
self.metadata.add("RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2]))
self.metadata.add("RegisterRedLongOffsetMedian", numpy.median(dlong[idx3]))
self.metadata.add("RegisterBlueLongOffsetStd", rms1Long)
self.metadata.add("RegisterGreenLongOffsetStd", rms2Long)
self.metadata.add("RegisterRedLongOffsetStd", rms3Long)
self.metadata.add("RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1]))
self.metadata.add("RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2]))
self.metadata.add("RegisterRedLatOffsetMedian", numpy.median(dlat[idx3]))
self.metadata.add("RegisterBlueLatOffsetStd", rms1Lat)
self.metadata.add("RegisterGreenLatOffsetStd", rms2Lat)
self.metadata.add("RegisterRedLatOffsetStd", rms3Lat)
# warp template exposure to match exposure,
# PSF match template exposure to exposure,
# then return the difference
#Return warped template... Construct sourceKernelCand list after subtract
self.log.info("Subtracting images")
subtractRes = self.subtract.subtractExposures(
templateExposure=templateExposure,
scienceExposure=exposure,
candidateList=kernelSources,
convolveTemplate=self.config.convolveTemplate,
doWarping=not self.config.doUseRegister
)
subtractedExposure = subtractRes.subtractedExposure
if self.config.doWriteMatchedExp:
sensorRef.put(subtractRes.matchedExposure, self.config.coaddName + "Diff_matchedExp")
if self.config.doDetection:
self.log.info("Running diaSource detection")
if subtractedExposure is None:
subtractedExposure = sensorRef.get(subtractedExposureName)
# Get Psf from the appropriate input image if it doesn't exist
if not subtractedExposure.hasPsf():
if self.config.convolveTemplate:
subtractedExposure.setPsf(exposure.getPsf())
else:
if templateExposure is None:
template = self.getTemplate.run(exposure, sensorRef, templateIdList=templateIdList)
subtractedExposure.setPsf(template.exposure.getPsf())
# Erase existing detection mask planes
mask = subtractedExposure.getMaskedImage().getMask()
mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
table = afwTable.SourceTable.make(self.schema, idFactory)
table.setMetadata(self.algMetadata)
results = self.detection.makeSourceCatalog(
table=table,
exposure=subtractedExposure,
doSmooth=not self.config.doPreConvolve
)
if self.config.doMerge:
fpSet = results.fpSets.positive
fpSet.merge(results.fpSets.negative, self.config.growFootprint,
self.config.growFootprint, False)
diaSources = afwTable.SourceCatalog(table)
fpSet.makeSources(diaSources)
self.log.info("Merging detections into %d sources" % (len(diaSources)))
else:
diaSources = results.sources
if self.config.doMeasurement:
self.log.info("Running diaSource measurement")
self.measurement.run(diaSources, subtractedExposure)
# Match with the calexp sources if possible
if self.config.doMatchSources:
if sensorRef.datasetExists("src"):
# Create key,val pair where key=diaSourceId and val=sourceId
matchRadAsec = self.config.diaSourceMatchRadius
matchRadPixel = matchRadAsec / exposure.getWcs().pixelScale().asArcseconds()
srcMatches = afwTable.matchXy(sensorRef.get("src"), diaSources, matchRadPixel, True)
srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId()) for
srcMatch in srcMatches])
self.log.info("Matched %d / %d diaSources to sources" % (len(srcMatchDict),
len(diaSources)))
else:
self.log.warn("Src product does not exist; cannot match with diaSources")
srcMatchDict = {}
# Create key,val pair where key=diaSourceId and val=refId
refAstromConfig = measAstrom.AstrometryConfig()
refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec
refAstrometer = measAstrom.AstrometryTask(refAstromConfig)
astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources)
refMatches = astromRet.matches
if refMatches is None:
self.log.warn("No diaSource matches with reference catalog")
refMatchDict = {}
else:
self.log.info("Matched %d / %d diaSources to reference catalog" % (len(refMatches),
len(diaSources)))
refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId()) for \
refMatch in refMatches])
# Assign source Ids
for diaSource in diaSources:
sid = diaSource.getId()
if srcMatchDict.has_key(sid):
diaSource.set("srcMatchId", srcMatchDict[sid])
if refMatchDict.has_key(sid):
diaSource.set("refMatchId", refMatchDict[sid])
if diaSources is not None and self.config.doWriteSources:
sensorRef.put(diaSources, self.config.coaddName + "Diff_diaSrc")
if self.config.doAddMetrics and self.config.doSelectSources:
self.log.info("Evaluating metrics and control sample")
kernelCandList = []
for cell in subtractRes.kernelCellSet.getCellList():
for cand in cell.begin(False): # include bad candidates
kernelCandList.append(cast_KernelCandidateF(cand))
# Get basis list to build control sample kernels
basisList = afwMath.cast_LinearCombinationKernel(
kernelCandList[0].getKernel(KernelCandidateF.ORIG)).getKernelList()
controlCandList = \
diffimTools.sourceTableToCandidateList(controlSources,
subtractRes.warpedExposure, exposure,
self.config.subtract.kernel.active,
self.config.subtract.kernel.active.detectionConfig,
self.log, doBuild=True, basisList=basisList)
kcQa.apply(kernelCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel,
dof=nparam)
kcQa.apply(controlCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel)
if self.config.doDetection:
kcQa.aggregate(selectSources, self.metadata, allresids, diaSources)
else:
kcQa.aggregate(selectSources, self.metadata, allresids)
sensorRef.put(selectSources, self.config.coaddName + "Diff_kernelSrc")
if self.config.doWriteSubtractedExp:
sensorRef.put(subtractedExposure, subtractedExposureName)
self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources)
return pipeBase.Struct(
subtractedExposure=subtractedExposure,
subtractRes=subtractRes,
sources=diaSources,
)
def runTest(self, subtractMethods=['ALstack', 'Zogy_S', 'Zogy', 'ALstack_decorr'],
zogyImageSpace=False, matchDist=np.sqrt(1.5), returnSources=False, **kwargs):
D_Zogy = S_Zogy = res = D_AL = None
src = {}
# Run diffim first
for subMethod in subtractMethods:
if subMethod is 'ALstack' or subMethod is 'ALstack_decorr':
if self.ALres is None:
self.ALres = self.doAlInStack(doPreConv=False, doDecorr=True, **kwargs)
if subMethod is 'Zogy_S':
if self.S_Zogy is None:
self.doZogy(computeScorr=True, inImageSpace=zogyImageSpace)
S_Zogy = self.S_Zogy
D_Zogy = self.D_Zogy
if subMethod is 'Zogy':
if self.D_Zogy is None:
self.doZogy(computeScorr=False, inImageSpace=zogyImageSpace)
D_Zogy = self.D_Zogy
if subMethod is 'AL': # my clean-room (pure python, slow) version of A&L
try:
if self.D_AL is None:
self.doAL(spatialKernelOrder=0, spatialBackgroundOrder=1)
D_AL = self.D_AL
except Exception as e:
print(e)
D_AL = None
# Run detection next
#try:
if subMethod is 'ALstack': # Note we DONT set it to 5.5 -- best for noise-free template.
src_AL = doDetection(self.ALres.subtractedExposure)
src['ALstack'] = src_AL
elif subMethod is 'ALstack_decorr':
src_AL2 = doDetection(self.ALres.decorrelatedDiffim)
src['ALstack_decorr'] = src_AL2
elif subMethod is 'Zogy':
D_Zogy = afwExp(D_Zogy)
src_Zogy = doDetection(D_Zogy)
src['Zogy'] = src_Zogy
elif subMethod is 'Zogy_S': # 'pixel_stdev' doesn't work, so just divide the image by
S_Zogy = afwExp(S_Zogy)
tmp_S = S_Zogy.clone() # the variance and use a 'value' threshold.
#tmp_S.im /= tmp_S.var
#tmp_S.var /= tmp_S.var
tmp_S_i = tmp_S.getMaskedImage().getImage().getArray()
tmp_S_i[:, :] /= tmp_S.getMaskedImage().getVariance().getArray()[:, :]
#src_SZogy = doDetection(S_ZOGY,
# thresholdType='pixel_stdev', doSmooth=False)
src_SZogy = doDetection(tmp_S, thresholdType='value', doSmooth=False)
src['SZogy'] = src_SZogy
elif subMethod is 'AL' and D_AL is not None:
D_AL = afwExp(D_AL)
src_AL = doDetection(D_AL)
src['AL'] = src_AL
#except Exception as e:
# print(e)
# pass
# Compare detections to input sources and get true positives and false negatives
changedCentroid = self.getCentroidsCatalog(transientsOnly=True)
import lsst.afw.table as afwTable
detections = matchCat = {}
for key in src:
srces = src[key]
srces = srces[~srces['base_PsfFlux_flag']] # this works!
matches = afwTable.matchXy(changedCentroid, srces, matchDist) # these should not need uniquifying
true_pos = len(matches)
false_neg = len(changedCentroid) - len(matches)
false_pos = len(srces) - len(matches)
detections[key] = {'TP': true_pos, 'FN': false_neg, 'FP': false_pos}
# sources, fp1, fp2, fp_Zogy, fp_AL, fp_ALd = self.doForcedPhot(transientsOnly=True)
# if mc_Zogy is not None:
# matches = afwTable.matchXy(pp_Zogy, sources, 1.0)
# matchedCat = catMatch.matchesToCatalog(matches, metadata)
if returnSources:
detections['sources'] = src
return detections
