def cosmicRay(self, exposure, keepCRs=None):
"""Mask cosmic rays
@param[in,out] exposure Exposure to process
@param keepCRs Don't interpolate over the CR pixels (defer to pex_config if None)
"""
import lsstDebug
display = lsstDebug.Info(__name__).display
displayCR = lsstDebug.Info(__name__).displayCR
assert exposure, "No exposure provided"
psf = exposure.getPsf()
assert psf, "No psf provided"
# Blow away old mask
try:
mask = exposure.getMaskedImage().getMask()
crBit = mask.getMaskPlane("CR")
mask.clearMaskPlane(crBit)
except Exception:
pass
exposure0 = exposure # initial value of exposure
binSize = self.config.cosmicray.background.binSize
nx, ny = exposure.getWidth()/binSize, exposure.getHeight()/binSize
if nx*ny <= 1:
bg = afwMath.makeStatistics(exposure.getMaskedImage(), afwMath.MEDIAN).getValue()
bkgd = None
else:
exposure = exposure.Factory(exposure, True)
bkgd, exposure = measAlg.estimateBackground(exposure, self.config.cosmicray.background,
subtract=True)
bg = 0.0
if keepCRs is None:
keepCRs = self.config.cosmicray.keepCRs
try:
crs = measAlg.findCosmicRays(exposure.getMaskedImage(),
psf, bg, pexConfig.makePolicy(self.config.cosmicray), keepCRs)
if bkgd:
# Add back background image
img = exposure.getMaskedImage().getImage()
img += bkgd.getImageF()
del img
# Replace original image with CR subtracted image
mimg = exposure0.getMaskedImage()
mimg <<= exposure.getMaskedImage()
del mimg
except Exception, e:
if display:
import lsst.afw.display.ds9 as ds9
ds9.mtv(exposure0, title="Failed CR")
raise
def process(self, dataRef):
"""Process focus CCD in preparation for focus measurement
@param dataRef: Data reference for CCD
@return Struct(sources: source measurements,
ccdId: CCD number,
filterName: name of filter,
dims: exposure dimensions
)
"""
import lsstDebug
display = lsstDebug.Info(__name__).display
exp = self.isr.runDataRef(dataRef).exposure
if display:
import lsst.afw.display.ds9 as ds9
ds9.mtv(exp, title="Post-ISR", frame=1)
self.installPsf.run(exposure=exp)
bg, exp = measAlg.estimateBackground(exp, self.config.background, subtract=True)
if display:
ds9.mtv(exp, title="Post-background", frame=2)
dmResults = self.detectAndMeasure.run(exp, dataRef.get("expIdInfo"))
sources = dmResults.sourceCat
self.starSelector.run(exp, sources, isStarField="hscPipeline_focus_candidate")
if display:
ds9.mtv(exp, title="Post-measurement", frame=3)
with ds9.Buffering():
for s in sources:
ds9.dot("o", s.getX(), s.getY(), frame=3,
ctype=ds9.GREEN if s.get("calib.psf.candidate") else ds9.RED)
import pdb;pdb.set_trace() # pause to allow inspection
filterName = exp.getFilter().getName()
if self.config.doWrite:
dataRef.put(sources, "src")
dataRef.put(exp, "visitim")
return Struct(sources=sources, ccdId=dataRef.dataId["ccd"], filterName=filterName,
dims=exp.getDimensions())
def run(display=False):
"""Subtract background, mask cosmic rays, then detect and measure
"""
# Create the tasks; note that background estimation is performed by a function,
# not a task, though it has a config
repairConfig = RepairTask.ConfigClass()
repairTask = RepairTask(config=repairConfig)
backgroundConfig = estimateBackground.ConfigClass()
damConfig = DetectAndMeasureTask.ConfigClass()
damConfig.detection.thresholdValue = 5.0
damConfig.detection.includeThresholdMultiplier = 1.0
damConfig.measurement.doApplyApCorr = "yes"
detectAndMeasureTask = DetectAndMeasureTask(config=damConfig)
# load the data
# Exposure ID and the number of bits required for exposure IDs are usually obtained from a data repo,
# but here we pick reasonable values (there are 64 bits to share between exposure IDs and source IDs).
exposure = loadData()
exposureIdInfo = ExposureIdInfo(expId=1, expBits=5)
# repair cosmic rays
repairTask.run(exposure=exposure)
# subtract an initial estimate of background level
estBg, exposure = estimateBackground(
exposure=exposure,
backgroundConfig=backgroundConfig,
subtract=True,
)
# detect and measure
damRes = detectAndMeasureTask.run(exposure=exposure,
exposureIdInfo=exposureIdInfo)
if display:
displayAstrometry(frame=2,
exposure=damRes.exposure,
sourceCat=damRes.sourceCat,
pause=False)
def run(display=False):
"""Subtract background, mask cosmic rays, then detect and measure
"""
# Create the tasks; note that background estimation is performed by a function,
# not a task, though it has a config
repairConfig = RepairTask.ConfigClass()
repairTask = RepairTask(config=repairConfig)
backgroundConfig = estimateBackground.ConfigClass()
damConfig = DetectAndMeasureTask.ConfigClass()
damConfig.detection.thresholdValue = 5.0
damConfig.detection.includeThresholdMultiplier = 1.0
damConfig.measurement.doApplyApCorr = "yes"
detectAndMeasureTask = DetectAndMeasureTask(config=damConfig)
# load the data
# Exposure ID and the number of bits required for exposure IDs are usually obtained from a data repo,
# but here we pick reasonable values (there are 64 bits to share between exposure IDs and source IDs).
exposure = loadData()
exposureIdInfo = ExposureIdInfo(expId=1, expBits=5)
# repair cosmic rays
repairTask.run(exposure=exposure)
# subtract an initial estimate of background level
estBg, exposure = estimateBackground(
exposure = exposure,
backgroundConfig = backgroundConfig,
subtract = True,
)
# detect and measure
damRes = detectAndMeasureTask.run(exposure=exposure, exposureIdInfo=exposureIdInfo)
if display:
displayAstrometry(frame=2, exposure=damRes.exposure, sourceCat=damRes.sourceCat, pause=False)
def characterize(self, exposure, exposureIdInfo, background=None):
"""!Characterize a science image
Peforms the following operations:
- Iterate the following config.psfIterations times, or once if config.doMeasurePsf false:
- detect and measure sources and estimate PSF (see detectMeasureAndEstimatePsf for details)
- interpolate over cosmic rays
- perform final measurement
@param[in,out] exposure exposure to characterize (an lsst.afw.image.ExposureF or similar).
The following changes are made:
- update or set psf
- set apCorrMap
- update detection and cosmic ray mask planes
- subtract background and interpolate over cosmic rays
@param[in] exposureIdInfo ID info for exposure (an lsst.daf.butlerUtils.ExposureIdInfo)
@param[in] background model of background model already subtracted from exposure
(an lsst.afw.math.BackgroundList). May be None if no background has been subtracted,
which is typical for image characterization.
@return pipe_base Struct containing these fields, all from the final iteration
of detectMeasureAndEstimatePsf:
- exposure: characterized exposure; image is repaired by interpolating over cosmic rays,
mask is updated accordingly, and the PSF model is set
- sourceCat: detected sources (an lsst.afw.table.SourceCatalog)
- background: model of background subtracted from exposure (an lsst.afw.math.BackgroundList)
- psfCellSet: spatial cells of PSF candidates (an lsst.afw.math.SpatialCellSet)
"""
self._frame = self._initialFrame # reset debug display frame
if not self.config.doMeasurePsf and not exposure.hasPsf():
raise RuntimeError("exposure has no PSF model and config.doMeasurePsf false")
if background is None:
background = BackgroundList()
# make a deep copy of the mask
originalMask = exposure.getMaskedImage().getMask().clone()
# subtract an initial estimate of background level
estBg = estimateBackground(
exposure = exposure,
backgroundConfig = self.config.background,
subtract = False, # this makes a deep copy, which we don't want
)[0]
image = exposure.getMaskedImage().getImage()
image -= estBg.getImageF()
background.append(estBg)
psfIterations = self.config.psfIterations if self.config.doMeasurePsf else 1
for i in range(psfIterations):
if i > 1:
# restore original mask so that detections and cosmic rays
# are only marked by the final iteration
exposure.getMaskedImage().getMask()[:] = originalMask
dmeRes = self.detectMeasureAndEstimatePsf(
exposure = exposure,
exposureIdInfo = exposureIdInfo,
background = background,
)
psf = dmeRes.exposure.getPsf()
psfSigma = psf.computeShape().getDeterminantRadius()
psfDimensions = psf.computeImage().getDimensions()
medBackground = np.median(dmeRes.background.getImage().getArray())
self.log.info("iter %s; PSF sigma=%0.2f, dimensions=%s; median background=%0.2f" % \
(i + 1, psfSigma, psfDimensions, medBackground))
self.display("psf", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat)
# perform final repair with final PSF
self.repair.run(exposure=dmeRes.exposure)
self.display("repair", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat)
# perform final measurement with final PSF, including measuring and applying aperture correction,
# if wanted
self.detectAndMeasure.measure(
exposure = dmeRes.exposure,
exposureIdInfo = exposureIdInfo,
sourceCat = dmeRes.sourceCat,
allowApCorr = True, # the default; listed for emphasis
)
self.display("measure", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat)
return dmeRes
def run(self, exposure, defects=None, idFactory=None):
"""!Run the calibration task on an exposure
\param[in,out] exposure Exposure to calibrate; measured PSF will be installed there as well
\param[in] defects List of defects on exposure
\param[in] idFactory afw.table.IdFactory to use for source catalog.
\return a pipeBase.Struct with fields:
- exposure: Repaired exposure
- backgrounds: A list of background models applied in the calibration phase
- psf: Point spread function
- sources: Sources used in calibration
- matches: A list of reference object/source matches (an lsst.afw.table.ReferenceMatchVector)
- matchMeta: Metadata about the field (an lsst.daf.base.PropertyList)
- photocal: Output of photocal subtask
It is moderately important to provide a decent initial guess for the seeing if you want to
deal with cosmic rays. If there's a PSF in the exposure it'll be used; failing that the
CalibrateConfig.initialPsf is consulted (although the pixel scale will be taken from the
WCS if available).
If the exposure contains an lsst.afw.image.Calib object with the exposure time set, MAGZERO
will be set in the task metadata.
"""
assert exposure is not None, "No exposure provided"
if not exposure.hasPsf():
self.installInitialPsf(exposure)
if idFactory is None:
idFactory = afwTable.IdFactory.makeSimple()
backgrounds = afwMath.BackgroundList()
keepCRs = True # At least until we know the PSF
self.repair.run(exposure, defects=defects, keepCRs=keepCRs)
self.display('repair', exposure=exposure)
if self.config.doBackground:
with self.timer("background"):
bg, exposure = measAlg.estimateBackground(exposure, self.config.background, subtract=True)
backgrounds.append(bg)
self.display('background', exposure=exposure)
# Make both tables from the same detRet, since detection can only be run once
table1 = afwTable.SourceTable.make(self.schema1, idFactory)
table1.setMetadata(self.algMetadata)
detRet = self.detection.makeSourceCatalog(table1, exposure)
sources1 = detRet.sources
if detRet.fpSets.background:
backgrounds.append(detRet.fpSets.background)
# do the initial measurement. This is normally done for star selection, but do it
# even if the psf is not going to be calculated for consistency
self.initialMeasurement.run(exposure, sources1, allowApCorr=False)
if self.config.doPsf:
matches = None
if self.config.doAstrometry:
# If doAstrometry is False, we force the Star Selector to either make them itself
# or hope it doesn't need them.
origWcs = exposure.getWcs()
try:
astromRet = self.astrometry.run(exposure, sources1)
matches = astromRet.matches
except RuntimeError as e:
if self.config.requireAstrometry:
raise
self.log.warn("Unable to perform astrometry (%s): attempting to proceed" % e)
finally:
# Restore original Wcs: we're going to repeat the astrometry later, and if it succeeded
# this time, running it again with the same basic setup means it should succeed again.
exposure.setWcs(origWcs)
psfRet = self.measurePsf.run(exposure, sources1, matches=matches)
psf = psfRet.psf
elif exposure.hasPsf():
psf = exposure.getPsf()
else:
psf = None
# Wash, rinse, repeat with proper PSF
if self.config.doPsf:
self.repair.run(exposure, defects=defects, keepCRs=None)
self.display('PSF_repair', exposure=exposure)
if self.config.doBackground:
# Background estimation ignores (by default) pixels with the
# DETECTED bit set, so now we re-estimate the background,
# ignoring sources. (see BackgroundConfig.ignoredPixelMask)
with self.timer("background"):
# Subtract background
bg, exposure = measAlg.estimateBackground(
exposure, self.config.background, subtract=True,
statsKeys=('BGMEAN2', 'BGVAR2'))
self.log.info("Fit and subtracted background")
backgrounds.append(bg)
self.display('PSF_background', exposure=exposure)
# make a second table with which to do the second measurement
# the schemaMapper will copy the footprints and ids, which is all we need.
table2 = afwTable.SourceTable.make(self.schema, idFactory)
table2.setMetadata(self.algMetadata)
sources = afwTable.SourceCatalog(table2)
# transfer to a second table -- note that the slots do not have to be reset here
# as long as measurement.run follows immediately
sources.extend(sources1, self.schemaMapper)
if self.config.doMeasureApCorr:
# Run measurement through all flux measurements (all have the same execution order),
# then apply aperture corrections, then run the rest of the measurements
self.measurement.run(exposure, sources, endOrder=BasePlugin.APCORR_ORDER)
apCorrMap = self.measureApCorr.run(bbox=exposure.getBBox(), catalog=sources).apCorrMap
exposure.getInfo().setApCorrMap(apCorrMap)
self.measurement.run(exposure, sources, beginOrder=BasePlugin.APCORR_ORDER)
else:
self.measurement.run(exposure, sources)
matches, matchMeta = None, None
if self.config.doAstrometry:
try:
astromRet = self.astrometry.run(exposure, sources)
matches = astromRet.matches
matchMeta = astromRet.matchMeta
except RuntimeError as e:
if self.config.requireAstrometry:
raise
self.log.warn("Unable to perform astrometry (%s): attempting to proceed" % e)
if self.config.doPhotoCal:
try:
if not matches:
raise RuntimeError("No matches available")
photocalRet = self.photocal.run(exposure, matches)
except Exception, e:
if self.config.requirePhotoCal:
raise
self.log.warn("Failed to determine photometric zero-point: %s" % e)
photocalRet = None
self.metadata.set('MAGZERO', float("NaN"))
if photocalRet:
self.log.info("Photometric zero-point: %f" % photocalRet.calib.getMagnitude(1.0))
exposure.getCalib().setFluxMag0(photocalRet.calib.getFluxMag0())
metadata = exposure.getMetadata()
# convert to (mag/sec/adu) for metadata
try:
magZero = photocalRet.zp - 2.5 * math.log10(exposure.getCalib().getExptime() )
metadata.set('MAGZERO', magZero)
except:
self.log.warn("Could not set normalized MAGZERO in header: no exposure time")
metadata.set('MAGZERO_RMS', photocalRet.sigma)
metadata.set('MAGZERO_NOBJ', photocalRet.ngood)
metadata.set('COLORTERM1', 0.0)
metadata.set('COLORTERM2', 0.0)
metadata.set('COLORTERM3', 0.0)
def doit(args):
dataId = args
print "# Running", dataId
# I need to create a separate instance in each thread
mapper = Mapper(root = "/lsst7/stripe82/dr7/runs/", calibRoot = None, outputRoot = None)
butler = dafPersist.ButlerFactory(mapper = mapper).create()
# Grab science pixels
im = butler.get(datasetType="fpC", dataId = dataId).convertF()
# Remove the 128 pixel duplicate overlap between fields
# See python/lsst/obs/sdss/processCcdSdss.py for guidance
bbox = im.getBBox()
begin = bbox.getBegin()
extent = bbox.getDimensions()
extent -= afwGeom.Extent2I(0, 128)
tbbox = afwGeom.BoxI(begin, extent)
im = afwImage.ImageF(im, tbbox, True)
# Remove 1000 count pedestal
im -= 1000.0
# Create image variance from gain
calib, gain = butler.get(datasetType="tsField", dataId = dataId)
var = afwImage.ImageF(im, True)
var /= gain
# Note I need to do a bit extra for the mask; I actually need to call
# convertfpM with allPlanes = True to get all the SDSS info
#
# mask = butler.get(datasetType="fpM", dataId = dataId)
fpMFile = butler.mapper.map_fpM(dataId = dataId).getLocations()[0]
mask = convertfpM(fpMFile, True)
# Remove the 128 pixel duplicate overlap...
mask = afwImage.MaskU(mask, tbbox, True)
# We need this for the background estimation
exp = afwImage.ExposureF(afwImage.MaskedImageF(im, mask, var))
# Subtract off background, and scale by stdev
# This will turn the image into "sigma"
if False:
ctrl = afwMath.StatisticsControl(5.0, 5)
bitPlanes = mask.getMaskPlaneDict().values()
bitMask = 2**bitPlanes[0]
for plane in bitPlanes[1:]:
bitMask |= 2**bitPlanes[plane]
ctrl.setAndMask(bitMask)
stat = afwMath.makeStatistics(im, mask, afwMath.STDEVCLIP | afwMath.MEDIAN | afwMath.NPOINT, ctrl)
stdev = stat.getValue(afwMath.STDEVCLIP)
bg = stat.getValue(afwMath.MEDIAN)
elif False:
# It should be that afwMath.NPOINT = len(idx[0])
# Not the case, exactly, so go with what you know
idx = np.where(mask.getArray() == 0)
gdata = im.getArray()[idx]
bg = np.median(gdata)
stdev = 0.741 * (np.percentile(gdata, 75) - np.percentile(gdata, 25))
else:
# Do full-on background subtraction
bgctrl = measAlg.BackgroundConfig(binSize=512, statisticsProperty="MEANCLIP", ignoredPixelMask=mask.getMaskPlaneDict().keys())
bg, bgsubexp = measAlg.estimateBackground(exp, bgctrl, subtract=True)
im = bgsubexp.getMaskedImage().getImage()
sctrl = afwMath.StatisticsControl()
sctrl.setAndMask(reduce(lambda x, y, mask=mask: x | mask.getPlaneBitMask(y), bgctrl.ignoredPixelMask, 0x0))
stdev = afwMath.makeStatistics(im, mask, afwMath.STDEVCLIP, sctrl).getValue(afwMath.STDEVCLIP)
im /= stdev
# Decision point: do I send the convolution a MaskedImage, in which
# case the mask is also spread, or just an Image, and not spread
# the mask...
#
# I think for now I will not spread the mask so that it represents the
# condition of the underlying pixels, not the Psf-filtered ones
psf = butler.get(datasetType="psField", dataId = dataId)
wcs = butler.get(datasetType="asTrans", dataId = dataId)
# Image convolved with the Psf, i.e. maximum point source likelihood image
cim = afwImage.ImageF(im, True)
afwMath.convolve(cim, im, psf.getKernel(), True)
# The pixels that are "good" in the image, i.e. ignore borders
cBBox = psf.getKernel().shrinkBBox(cim.getBBox())
cim = afwImage.ImageF(cim, cBBox)
mask = afwImage.MaskU(mask, cBBox)
# Create an ra,decl map for the good pixels
raIm = afwImage.ImageF(cim.getDimensions())
decIm = afwImage.ImageF(cim.getDimensions())
nx, ny = cim.getDimensions()
# But note that the Wcs expects their coordinates in the non-shrunk image
x0 = cBBox.getBeginX()
y0 = cBBox.getBeginY()
x1 = cBBox.getEndX()
y1 = cBBox.getEndY()
for y in range(ny):
for x in range(nx):
ra, decl = getRaDecl(wcs, x+x0, y+y0)
raIm.set(x, y, ra)
decIm.set(x, y, decl)
run = dataId["run"]
camcol = dataId["camcol"]
field = dataId["field"]
filterName = dataId["filter"]
if doWriteSql:
# Make the table inputs
xll, yll = getRaDecl(wcs, 0 +x0, 0+ y0)
xlr, ylr = getRaDecl(wcs, x1+x0, 0+ y0)
xur, yur = getRaDecl(wcs, x1+x0, y1+y0)
xul, yul = getRaDecl(wcs, 0 +x0, y1+y0)
tc = calib.getMidTime()
t0 = dafBase.DateTime(tc.nsecs() - int(0.5 * calib.getExptime() * 1e9), dafBase.DateTime.TAI)
t1 = dafBase.DateTime(tc.nsecs() + int(0.5 * calib.getExptime() * 1e9), dafBase.DateTime.TAI)
# Magic for the day; 2**63 because BIGINT is signed
fieldId = int(md5.new(" ".join(map(str, [run, filterName, camcol, field]))).hexdigest(), 16) % 2**63
pfile = "pixel-%06d-%s%s-%04d.csv" % (run, filterName, camcol, field)
ffile = "field-%06d-%s%s-%04d.pgsql" % (run, filterName, camcol, field)
pbuff = open(pfile, "w")
fbuff = open(ffile, "w")
fbuff.write("INSERT INTO fields (fieldId, run, camcol, field, filter, bbox, tmid, trange) VALUES\n")
fbuff.write(" (%d, %d, %d, %d, '%s', ST_GeomFromText('POLYGON((\n" % (fieldId, run, camcol, field, filterName))
fbuff.write(" %.9f %.9f, %.9f %.9f,\n" % (xll, yll, xlr, ylr))
fbuff.write(" %.9f %.9f, %.9f %.9f,\n" % (xur, yur, xul, yul))
fbuff.write(" %.9f %.9f))',3786),\n" % (xll, yll))
fbuff.write(" '%s',\n" % (re.sub("T", " ", tc.toString())))
fbuff.write(" '[%s, %s]');\n" % (re.sub("T", " ", t0.toString()), re.sub("T", " ", t1.toString())))
for y in range(ny):
for x in range(nx):
# Note the different orders of raIm,decIm and cim,mask
pbuff.write("%d, %.9f, %.9f, %f, %d\n" % (fieldId, raIm.get(x,y), decIm.get(x,y), cim.get(x,y), mask.get(x,y)))
pbuff.close()
fbuff.close()
if False:
cim.writeFits("image-%06d-%s%s-%04d.fits" % (run, filterName, camcol, field))
mask.writeFits("mask-%06d-%s%s-%04d.fits" % (run, filterName, camcol, field))
raIm.writeFits("ra-%06d-%s%s-%04d.fits" % (run, filterName, camcol, field))
decIm.writeFits("dec-%06d-%s%s-%04d.fits" % (run, filterName, camcol, field))
def characterize(self, exposure, exposureIdInfo, background=None):
"""!Characterize a science image
Peforms the following operations:
- Iterate the following config.psfIterations times, or once if config.doMeasurePsf false:
- detect and measure sources and estimate PSF (see detectMeasureAndEstimatePsf for details)
- interpolate over cosmic rays
- perform final measurement
@param[in,out] exposure exposure to characterize (an lsst.afw.image.ExposureF or similar).
The following changes are made:
- update or set psf
- set apCorrMap
- update detection and cosmic ray mask planes
- subtract background and interpolate over cosmic rays
@param[in] exposureIdInfo ID info for exposure (an lsst.daf.butlerUtils.ExposureIdInfo)
@param[in] background model of background model already subtracted from exposure
(an lsst.afw.math.BackgroundList). May be None if no background has been subtracted,
which is typical for image characterization.
@return pipe_base Struct containing these fields, all from the final iteration
of detectMeasureAndEstimatePsf:
- exposure: characterized exposure; image is repaired by interpolating over cosmic rays,
mask is updated accordingly, and the PSF model is set
- sourceCat: detected sources (an lsst.afw.table.SourceCatalog)
- background: model of background subtracted from exposure (an lsst.afw.math.BackgroundList)
- psfCellSet: spatial cells of PSF candidates (an lsst.afw.math.SpatialCellSet)
"""
self._frame = self._initialFrame # reset debug display frame
if not self.config.doMeasurePsf and not exposure.hasPsf():
raise RuntimeError(
"exposure has no PSF model and config.doMeasurePsf false")
if background is None:
background = BackgroundList()
# make a deep copy of the mask
originalMask = exposure.getMaskedImage().getMask().clone()
# subtract an initial estimate of background level
estBg = estimateBackground(
exposure=exposure,
backgroundConfig=self.config.background,
subtract=False, # this makes a deep copy, which we don't want
)[0]
image = exposure.getMaskedImage().getImage()
image -= estBg.getImageF()
background.append(estBg)
psfIterations = self.config.psfIterations if self.config.doMeasurePsf else 1
for i in range(psfIterations):
if i > 1:
# restore original mask so that detections and cosmic rays
# are only marked by the final iteration
exposure.getMaskedImage().getMask()[:] = originalMask
dmeRes = self.detectMeasureAndEstimatePsf(
exposure=exposure,
exposureIdInfo=exposureIdInfo,
background=background,
)
psf = dmeRes.exposure.getPsf()
psfSigma = psf.computeShape().getDeterminantRadius()
psfDimensions = psf.computeImage().getDimensions()
medBackground = np.median(dmeRes.background.getImage().getArray())
self.log.info("iter %s; PSF sigma=%0.2f, dimensions=%s; median background=%0.2f" % \
(i + 1, psfSigma, psfDimensions, medBackground))
self.display("psf",
exposure=dmeRes.exposure,
sourceCat=dmeRes.sourceCat)
# perform final repair with final PSF
self.repair.run(exposure=dmeRes.exposure)
self.display("repair",
exposure=dmeRes.exposure,
sourceCat=dmeRes.sourceCat)
# perform final measurement with final PSF, including measuring and applying aperture correction,
# if wanted
self.detectAndMeasure.measure(
exposure=dmeRes.exposure,
exposureIdInfo=exposureIdInfo,
sourceCat=dmeRes.sourceCat,
allowApCorr=True, # the default; listed for emphasis
)
self.display("measure",
exposure=dmeRes.exposure,
sourceCat=dmeRes.sourceCat)
return dmeRes
def run(self, exposure, defects=None, idFactory=None, expId=0):
"""Calibrate an exposure: measure PSF, subtract background, measure astrometry and photometry
@param[in,out] exposure Exposure to calibrate; measured Psf, Wcs, ApCorr, Calib, etc. will
be installed there as well
@param[in] defects List of defects on exposure
@param[in] idFactory afw.table.IdFactory to use for source catalog.
@param[in] expId Exposure id used for random number generation.
@return a pipeBase.Struct with fields:
- backgrounds: A list of background models applied in the calibration phase
- psf: Point spread function
- sources: Sources used in calibration
- matches: Astrometric matches
- matchMeta: Metadata for astrometric matches
- apCorrMap: Map of aperture corrections
- photocal: Output of photocal subtask
"""
assert exposure is not None, "No exposure provided"
if not exposure.hasPsf():
self.installInitialPsf(exposure)
if idFactory is None:
idFactory = afwTable.IdFactory.makeSimple()
backgrounds = afwMath.BackgroundList()
keepCRs = True # At least until we know the PSF
self.repair.run(exposure, defects=defects, keepCRs=keepCRs)
self.display('repair', exposure=exposure)
if self.config.doBackground:
with self.timer("background"):
bg, exposure = measAlg.estimateBackground(exposure, self.config.background, subtract=True)
backgrounds.append(bg)
self.display('background', exposure=exposure)
table = afwTable.SourceTable.make(self.schema, idFactory)
table.setMetadata(self.algMetadata)
detRet = self.detection.makeSourceCatalog(table, exposure)
sources = detRet.sources
if detRet.fpSets.background:
backgrounds.append(detRet.fpSets.background)
if self.config.doPsf:
self.initialMeasurement.measure(exposure, sources)
matches = None
if self.config.doAstrometry:
# If doAstrometry is False, we force the Star Selector to either make them itself
# or hope it doesn't need them.
origWcs = exposure.getWcs()
try:
astromRet = self.astrometry.run(exposure, sources)
matches = astromRet.matches
except RuntimeError as e:
if self.config.requireAstrometry:
raise
self.log.warn("Unable to perform astrometry (%s): attempting to proceed" % e)
finally:
# Restore original Wcs: we're going to repeat the astrometry later, and if it succeeded
# this time, running it again with the same basic setup means it should succeed again.
exposure.setWcs(origWcs)
# This is an initial, throw-away run of photocal, since we need a valid Calib to run CModel,
# and we need to run CModel to compute aperture corrections from it.
if self.config.doPhotoCal:
try:
if not matches:
raise RuntimeError("No matches available")
photocalRet = self.photocal.run(exposure, matches,
prefix=self.config.initialMeasurement.prefix,
doSelectUnresolved=False)# don't trust s/g without good PSF
self.log.info("Initial photometric zero-point: %f" % photocalRet.calib.getMagnitude(1.0))
exposure.getCalib().setFluxMag0(photocalRet.calib.getFluxMag0())
except Exception, e:
self.log.warn("Failed to determine initial photometric zero-point: %s" % e)
psfRet = self.measurePsf.run(exposure, sources, expId=expId, matches=matches)
psf = psfRet.psf
psf = None
# Wash, rinse, repeat with proper PSF
if self.config.doPsf:
self.repair.run(exposure, defects=defects, keepCRs=None)
self.display('repair', exposure=exposure)
if self.config.doBackground:
# Background estimation ignores (by default) pixels with the
# DETECTED bit set, so now we re-estimate the background,
# ignoring sources. (see BackgroundConfig.ignoredPixelMask)
with self.timer("background"):
# Subtract background
bg, exposure = measAlg.estimateBackground(
exposure, self.config.background, subtract=True,
statsKeys=('BGMEAN2', 'BGVAR2'))
self.log.info("Fit and subtracted background")
backgrounds.append(bg)
self.display('background', exposure=exposure)
if self.config.doMeasureApCorr:
# Because we want to both compute the aperture corrections and apply them - and we do the latter
# as a source measurement plugin ("CorrectFluxes"), we have to sandwich the aperture correction
# measurement in between two source measurement passes, using the priority range arguments added
# just for this purpose.
apCorrApplyPriority = self.config.measurement.algorithms["correctfluxes"].priority
self.measurement.run(exposure, sources, endPriority=apCorrApplyPriority)
if self.config.doCurveOfGrowth:
curveOfGrowth = self.applyCurveOfGrowth(sources)
def run(self, exposure, defects=None, idFactory=None):
"""Calibrate an exposure: measure PSF, subtract background, measure astrometry and photometry
@param[in,out] exposure Exposure to calibrate; measured PSF will be installed there as well
@param[in] defects List of defects on exposure
@param[in] idFactory afw.table.IdFactory to use for source catalog.
@return a pipeBase.Struct with fields:
- backgrounds: A list of background models applied in the calibration phase
- psf: Point spread function
- apCorr: Aperture correction
- sources: Sources used in calibration
- matches: Astrometric matches
- matchMeta: Metadata for astrometric matches
- photocal: Output of photocal subtask
"""
assert exposure is not None, "No exposure provided"
self.installInitialPsf(exposure)
if idFactory is None:
idFactory = afwTable.IdFactory.makeSimple()
backgrounds = []
keepCRs = True # At least until we know the PSF
self.repair.run(exposure, defects=defects, keepCRs=keepCRs)
self.display('repair', exposure=exposure)
if self.config.doBackground:
with self.timer("background"):
bg, exposure = measAlg.estimateBackground(exposure, self.config.background, subtract=True)
backgrounds.append(bg)
self.display('background', exposure=exposure)
table = afwTable.SourceTable.make(self.schema, idFactory)
table.setMetadata(self.algMetadata)
detRet = self.detection.makeSourceCatalog(table, exposure)
sources = detRet.sources
if detRet.fpSets.background:
backgrounds.append(detRet.fpSets.background)
if self.config.doPsf:
self.initialMeasurement.measure(exposure, sources)
if self.config.doAstrometry:
astromRet = self.astrometry.run(exposure, sources)
matches = astromRet.matches
else:
# If doAstrometry is False, we force the Star Selector to either make them itself
# or hope it doesn't need them.
matches = None
psfRet = self.measurePsf.run(exposure, sources, matches=matches)
cellSet = psfRet.cellSet
psf = psfRet.psf
else:
psf, cellSet = None, None
# Wash, rinse, repeat with proper PSF
if self.config.doPsf:
self.repair.run(exposure, defects=defects, keepCRs=None)
self.display('repair', exposure=exposure)
if self.config.doBackground:
# Background estimation ignores (by default) pixels with the
# DETECTED bit set, so now we re-estimate the background,
# ignoring sources. (see BackgroundConfig.ignoredPixelMask)
with self.timer("background"):
# Subtract background
bg, exposure = measAlg.estimateBackground(
exposure, self.config.background, subtract=True,
statsKeys=('BGMEAN2', 'BGVAR2'))
self.log.info("Fit and subtracted background")
backgrounds.append(bg)
self.display('background', exposure=exposure)
if self.config.doComputeApCorr or self.config.doAstrometry or self.config.doPhotoCal:
self.measurement.measure(exposure, sources) # don't use run, because we don't have apCorr yet
if self.config.doComputeApCorr:
assert(self.config.doPsf)
apCorr = self.computeApCorr(exposure, cellSet)
else:
apCorr = None
if self.measurement.config.doApplyApCorr:
assert(apCorr is not None)
self.measurement.applyApCorr(sources, apCorr)
if self.config.doAstrometry:
astromRet = self.astrometry.run(exposure, sources)
matches = astromRet.matches
matchMeta = astromRet.matchMeta
else:
matches, matchMeta = None, None
if self.config.doPhotoCal:
assert(matches is not None)
try:
photocalRet = self.photocal.run(exposure, matches)
except Exception, e:
self.log.warn("Failed to determine photometric zero-point: %s" % e)
photocalRet = None
if photocalRet:
self.log.info("Photometric zero-point: %f" % photocalRet.calib.getMagnitude(1.0))
exposure.getCalib().setFluxMag0(photocalRet.calib.getFluxMag0())
metadata = exposure.getMetadata()
# convert to (mag/sec/adu) for metadata
try:
magZero = photocalRet.zp - 2.5 * math.log10(exposure.getCalib().getExptime() )
metadata.set('MAGZERO', magZero)
except:
self.log.warn("Could not set normalized MAGZERO in header: no exposure time")
metadata.set('MAGZERO_RMS', photocalRet.sigma)
metadata.set('MAGZERO_NOBJ', photocalRet.ngood)
metadata.set('COLORTERM1', 0.0)
metadata.set('COLORTERM2', 0.0)
metadata.set('COLORTERM3', 0.0)
def cosmicRay(self, exposure, keepCRs=None):
"""Mask cosmic rays
\param[in,out] exposure Exposure to process
\param[in] keepCRs Don't interpolate over the CR pixels (defer to pex_config if None)
"""
import lsstDebug
display = lsstDebug.Info(__name__).display
displayCR = lsstDebug.Info(__name__).displayCR
assert exposure, "No exposure provided"
psf = exposure.getPsf()
assert psf, "No psf provided"
# Blow away old mask
try:
mask = exposure.getMaskedImage().getMask()
crBit = mask.getMaskPlane("CR")
mask.clearMaskPlane(crBit)
except Exception:
pass
exposure0 = exposure # initial value of exposure
binSize = self.config.cosmicray.background.binSize
nx, ny = exposure.getWidth()/binSize, exposure.getHeight()/binSize
# Treat constant background as a special case to avoid the extra complexity in calling
# measAlg.estimateBackground().
if nx*ny <= 1:
medianBg = afwMath.makeStatistics(exposure.getMaskedImage(), afwMath.MEDIAN).getValue()
modelBg = None
else:
exposure = exposure.Factory(exposure, True)
modelBg, exposure = measAlg.estimateBackground(exposure, self.config.cosmicray.background,
subtract=True)
medianBg = 0.0
if keepCRs is None:
keepCRs = self.config.cosmicray.keepCRs
try:
crs = measAlg.findCosmicRays(exposure.getMaskedImage(), psf, medianBg,
pexConfig.makePolicy(self.config.cosmicray), keepCRs)
if modelBg:
# Add back background image
img = exposure.getMaskedImage()
img += modelBg.getImageF()
del img
# Replace original image with CR subtracted image
exposure0.setMaskedImage(exposure.getMaskedImage())
except Exception:
if display:
import lsst.afw.display.ds9 as ds9
ds9.mtv(exposure0, title="Failed CR")
raise
num = 0
if crs is not None:
mask = exposure0.getMaskedImage().getMask()
crBit = mask.getPlaneBitMask("CR")
afwDet.setMaskFromFootprintList(mask, crs, crBit)
num = len(crs)
if display and displayCR:
import lsst.afw.display.ds9 as ds9
import lsst.afw.display.utils as displayUtils
ds9.incrDefaultFrame()
ds9.mtv(exposure0, title="Post-CR")
with ds9.Buffering():
for cr in crs:
displayUtils.drawBBox(cr.getBBox(), borderWidth=0.55)
self.log.info("Identified %s cosmic rays." % (num,))
def createLikeImage(self, data):
print "# Converting", data
# Convert from Pandas to Dictionary
dataId = {}
for key in self.ukeys:
dataId[key] = data[key]
# I need to create a separate butler instance in case of multi threading
mapper = Mapper(root = "/lsst7/stripe82/dr7/runs/", calibRoot = None, outputRoot = None)
butler = dafPersist.ButlerFactory(mapper = mapper).create()
# Grab science pixels
im = butler.get(datasetType="fpC", dataId = dataId).convertF()
# Remove the 128 pixel duplicate overlap between fields
# See python/lsst/obs/sdss/processCcdSdss.py for guidance
bbox = im.getBBox()
begin = bbox.getBegin()
extent = bbox.getDimensions()
extent -= afwGeom.Extent2I(0, 128)
tbbox = afwGeom.BoxI(begin, extent)
im = afwImage.ImageF(im, tbbox, True)
nx0, ny0 = extent
# Remove 1000 count pedestal
im -= 1000.0
# Create image variance from gain
calib, gain = butler.get(datasetType="tsField", dataId = dataId)
var = afwImage.ImageF(im, True)
var /= gain
# Note I need to do a bit extra for the mask; I actually need to call
# convertfpM with allPlanes = True to get all the SDSS info
fpMFile = butler.mapper.map_fpM(dataId = dataId).getLocations()[0]
mask = convertfpM(fpMFile, True)
# Remove the 128 pixel duplicate overlap...
mask = afwImage.MaskU(mask, tbbox, True)
# Convert to Exposure for the background estimation
exp = afwImage.ExposureF(afwImage.MaskedImageF(im, mask, var))
# Subtract off background, and scale by stdev.
# This will turn the image into a "sigma" image
bgctrl = measAlg.BackgroundConfig(binSize=512, statisticsProperty="MEANCLIP", ignoredPixelMask=mask.getMaskPlaneDict().keys())
bg, bgsubexp = measAlg.estimateBackground(exp, bgctrl, subtract=True)
im = bgsubexp.getMaskedImage().getImage()
sctrl = afwMath.StatisticsControl()
sctrl.setAndMask(reduce(lambda x, y, mask=mask: x | mask.getPlaneBitMask(y), bgctrl.ignoredPixelMask, 0x0))
stdev = afwMath.makeStatistics(im, mask, afwMath.STDEVCLIP, sctrl).getValue(afwMath.STDEVCLIP)
im /= stdev
# Grab the Psf for filtering, and the Wcs for inclusion in final Exposure
psf = butler.get(datasetType="psField", dataId = dataId)
wcs = butler.get(datasetType="asTrans", dataId = dataId)
# Decision point: do I send the convolution a MaskedImage, in which
# case the mask is also spread, or just an Image, and not spread
# the mask...
#
# I think for now I will not spread the mask so that it represents the
# condition of the underlying pixels, not the Psf-filtered ones
#
# Create sigma image convolved with the Psf, i.e. maximum point source likelihood image
cim = afwImage.ImageF(im, True)
afwMath.convolve(cim, im, psf.getKernel(), True)
# Note that the convolution will create border regions of "bad" pixels.
# If we tried to trim these, we would have to tweak the Wcs CRPIX etc. So just leave as-is.
cexp = afwImage.ExposureF(afwImage.MaskedImageF(cim, mask, var))
cexp.setWcs(wcs)
return cexp
def run(self, exposure, defects=None, idFactory=None):
"""!Run the calibration task on an exposure
\param[in,out] exposure Exposure to calibrate; measured PSF will be installed there as well
\param[in] defects List of defects on exposure
\param[in] idFactory afw.table.IdFactory to use for source catalog.
\return a pipeBase.Struct with fields:
- exposure: Repaired exposure
- backgrounds: A list of background models applied in the calibration phase
- psf: Point spread function
- sources: Sources used in calibration
- matches: Astrometric matches
- matchMeta: Metadata for astrometric matches
- photocal: Output of photocal subtask
It is moderately important to provide a decent initial guess for the seeing if you want to
deal with cosmic rays. If there's a PSF in the exposure it'll be used; failing that the
CalibrateConfig.initialPsf is consulted (although the pixel scale will be taken from the
WCS if available).
If the exposure contains an lsst.afw.image.Calib object with the exposure time set, MAGZERO
will be set in the task metadata.
"""
assert exposure is not None, "No exposure provided"
if not exposure.hasPsf():
self.installInitialPsf(exposure)
if idFactory is None:
idFactory = afwTable.IdFactory.makeSimple()
backgrounds = afwMath.BackgroundList()
keepCRs = True # At least until we know the PSF
self.repair.run(exposure, defects=defects, keepCRs=keepCRs)
self.display('repair', exposure=exposure)
if self.config.doBackground:
with self.timer("background"):
bg, exposure = measAlg.estimateBackground(exposure, self.config.background, subtract=True)
backgrounds.append(bg)
self.display('background', exposure=exposure)
# Make both tables from the same detRet, since detRet can only be run once
table1 = afwTable.SourceTable.make(self.schema1, idFactory)
table1.setMetadata(self.algMetadata)
detRet = self.detection.makeSourceCatalog(table1, exposure)
sources1 = detRet.sources
if detRet.fpSets.background:
backgrounds.append(detRet.fpSets.background)
if self.config.doPsf:
self.initialMeasurement.measure(exposure, sources1)
# ### Do not compute astrometry before PSF determination. Astrometry will be computed afterwards
# ###
# if self.config.doAstrometry:
# astromRet = self.astrometry.run(exposure, sources1)
# matches = astromRet.matches
# else:
# If doAstrometry is False, we force the Star Selector to either make them itself
# or hope it doesn't need them.
# matches = None
matches = None
psfRet = self.measurePsf.run(exposure, sources1, matches=matches)
cellSet = psfRet.cellSet
psf = psfRet.psf
elif exposure.hasPsf():
psf = exposure.getPsf()
cellSet = None
else:
psf, cellSet = None, None
# Wash, rinse, repeat with proper PSF
if self.config.doPsf:
self.repair.run(exposure, defects=defects, keepCRs=None)
self.display('PSF_repair', exposure=exposure)
if self.config.doBackground:
# Background estimation ignores (by default) pixels with the
# DETECTED bit set, so now we re-estimate the background,
# ignoring sources. (see BackgroundConfig.ignoredPixelMask)
with self.timer("background"):
# Subtract background
bg, exposure = measAlg.estimateBackground(
exposure, self.config.background, subtract=True,
statsKeys=('BGMEAN2', 'BGVAR2'))
self.log.info("Fit and subtracted background")
backgrounds.append(bg)
self.display('PSF_background', exposure=exposure)
if self.config.doAstrometry or self.config.doPhotoCal:
# make a second table with which to do the second measurement
# the schemaMapper will copy the footprints and ids, which is all we need.
table2 = afwTable.SourceTable.make(self.schema, idFactory)
table2.setMetadata(self.algMetadata)
sources = afwTable.SourceCatalog(table2)
# transfer to a second table
sources.extend(sources1, self.schemaMapper)
self.measurement.run(exposure, sources)
else:
sources = sources1
if self.config.doAstrometry:
astromRet = self.astrometry.run(exposure, sources)
matches = astromRet.matches
matchMeta = astromRet.matchMeta
else:
matches, matchMeta = None, None
if self.config.doPhotoCal:
assert(matches is not None)
try:
photocalRet = self.photocal.run(exposure, matches)
except Exception, e:
raise
self.log.warn("Failed to determine photometric zero-point: %s" % e)
photocalRet = None
self.metadata.set('MAGZERO', float("NaN"))
if photocalRet:
self.log.info("Photometric zero-point: %f" % photocalRet.calib.getMagnitude(1.0))
exposure.getCalib().setFluxMag0(photocalRet.calib.getFluxMag0())
metadata = exposure.getMetadata()
# convert to (mag/sec/adu) for metadata
try:
magZero = photocalRet.zp - 2.5 * math.log10(exposure.getCalib().getExptime() )
metadata.set('MAGZERO', magZero)
except:
self.log.warn("Could not set normalized MAGZERO in header: no exposure time")
metadata.set('MAGZERO_RMS', photocalRet.sigma)
metadata.set('MAGZERO_NOBJ', photocalRet.ngood)
metadata.set('COLORTERM1', 0.0)
metadata.set('COLORTERM2', 0.0)
metadata.set('COLORTERM3', 0.0)
def convert(dataId):
print "# Converting", dataId
# I need to create a separate instance in each thread
mapper = Mapper(root = "/lsst7/stripe82/dr7/runs/", calibRoot = None, outputRoot = None)
butler = dafPersist.ButlerFactory(mapper = mapper).create()
# Grab science pixels
im = butler.get(datasetType="fpC", dataId = dataId).convertF()
# Remove the 128 pixel duplicate overlap between fields
# See python/lsst/obs/sdss/processCcdSdss.py for guidance
bbox = im.getBBox()
begin = bbox.getBegin()
extent = bbox.getDimensions()
extent -= afwGeom.Extent2I(0, 128)
tbbox = afwGeom.BoxI(begin, extent)
im = afwImage.ImageF(im, tbbox, True)
nx0, ny0 = extent
# Remove 1000 count pedestal
im -= 1000.0
# Create image variance from gain
calib, gain = butler.get(datasetType="tsField", dataId = dataId)
var = afwImage.ImageF(im, True)
var /= gain
# Note I need to do a bit extra for the mask; I actually need to call
# convertfpM with allPlanes = True to get all the SDSS info
#
# mask = butler.get(datasetType="fpM", dataId = dataId)
fpMFile = butler.mapper.map_fpM(dataId = dataId).getLocations()[0]
mask = convertfpM(fpMFile, True)
# Remove the 128 pixel duplicate overlap...
mask = afwImage.MaskU(mask, tbbox, True)
# We need this for the background estimation
exp = afwImage.ExposureF(afwImage.MaskedImageF(im, mask, var))
# Subtract off background, and scale by stdev
# This will turn the image into "sigma"
bgctrl = measAlg.BackgroundConfig(binSize=512, statisticsProperty="MEANCLIP", ignoredPixelMask=mask.getMaskPlaneDict().keys())
bg, bgsubexp = measAlg.estimateBackground(exp, bgctrl, subtract=True)
im = bgsubexp.getMaskedImage().getImage()
sctrl = afwMath.StatisticsControl()
sctrl.setAndMask(reduce(lambda x, y, mask=mask: x | mask.getPlaneBitMask(y), bgctrl.ignoredPixelMask, 0x0))
stdev = afwMath.makeStatistics(im, mask, afwMath.STDEVCLIP, sctrl).getValue(afwMath.STDEVCLIP)
im /= stdev
# Additional info
psf = butler.get(datasetType="psField", dataId = dataId)
wcs = butler.get(datasetType="asTrans", dataId = dataId)
# Decision point: do I send the convolution a MaskedImage, in which
# case the mask is also spread, or just an Image, and not spread
# the mask...
#
# I think for now I will not spread the mask so that it represents the
# condition of the underlying pixels, not the Psf-filtered ones
# Image convolved with the Psf, i.e. maximum point source likelihood image
cim = afwImage.ImageF(im, True)
afwMath.convolve(cim, im, psf.getKernel(), True)
# NOTE: DO WE SHRINK THE IMAGES HERE? IF SO, WE NEED TO TWEAK WCS
# For now, I will not do so
# The pixels that are "good" in the image, i.e. ignore borders
#cBBox = psf.getKernel().shrinkBBox(cim.getBBox())
#cim = afwImage.ImageF(cim, cBBox)
#mask = afwImage.MaskU(mask, cBBox)
cexp = afwImage.ExposureF(afwImage.MaskedImageF(cim, mask, var))
cexp.setWcs(wcs)
return cexp
