def coadd(self, cache, data):
"""!Construct coadd for a patch and measure
Only slave nodes execute this method.
Because only one argument may be passed, it is expected to
contain multiple elements, which are:
@param patchRef: data reference for patch
@param selectDataList: List of SelectStruct for inputs
"""
patchRef = getDataRef(cache.butler, data.patchId, cache.coaddType)
selectDataList = data.selectDataList
coadd = None
with self.logOperation("coadding %s" % (patchRef.dataId,), catch=True):
if self.config.doOverwriteCoadd or not patchRef.datasetExists(cache.coaddType):
coaddResults = self.assembleCoadd.run(patchRef, selectDataList)
if coaddResults is not None:
coadd = coaddResults.coaddExposure
elif patchRef.datasetExists(cache.coaddType):
self.log.info("%s: Reading coadd %s" % (NODE, patchRef.dataId))
coadd = patchRef.get(cache.coaddType, immediate=True)
if coadd is None:
return
with self.logOperation("detection on %s" % (patchRef.dataId,), catch=True):
idFactory = self.detectCoaddSources.makeIdFactory(patchRef)
# This includes background subtraction, so do it before writing the coadd
detResults = self.detectCoaddSources.runDetection(coadd, idFactory)
self.detectCoaddSources.write(coadd, detResults, patchRef)
def run(self, tractPatchRefList, butler, selectIdList=[]):
"""!Determine which tracts are non-empty before processing
@param tractPatchRefList: List of tracts and patches to include in the coaddition
@param butler: butler reference object
@param selectIdList: List of data Ids (i.e. visit, ccd) to consider when making the coadd
@return list of references to sel.runTract function evaluation for each tractPatchRefList member
"""
pool = Pool("tracts")
pool.storeSet(butler=butler, skymap=butler.get(
self.config.coaddName + "Coadd_skyMap"))
tractIdList = []
for patchRefList in tractPatchRefList:
tractSet = set([patchRef.dataId["tract"]
for patchRef in patchRefList])
assert len(tractSet) == 1
tractIdList.append(tractSet.pop())
selectDataList = [data for data in pool.mapNoBalance(self.readSelection, selectIdList) if
data is not None]
nonEmptyList = pool.mapNoBalance(
self.checkTract, tractIdList, selectDataList)
tractPatchRefList = [patchRefList for patchRefList, nonEmpty in
zip(tractPatchRefList, nonEmptyList) if nonEmpty]
self.log.info("Non-empty tracts (%d): %s" % (len(tractPatchRefList),
[patchRefList[0].dataId["tract"] for patchRefList in
tractPatchRefList]))
# Install the dataRef in the selectDataList
for data in selectDataList:
data.dataRef = getDataRef(butler, data.dataId, "calexp")
# Process the non-empty tracts
return [self.runTract(patchRefList, butler, selectDataList) for patchRefList in tractPatchRefList]
def runMeasureMerged(self, cache, dataId):
"""Run measurement on a patch for a single filter
Only slave nodes execute this method.
@param cache: Pool cache, with butler
@param dataId: Data identifier for patch
@return whether the patch requires reprocessing.
"""
with self.logOperation("measurement on %s" % (dataId,)):
dataRef = getDataRef(cache.butler, dataId, self.config.coaddName + "Coadd_calexp")
reprocessing = False # Does this patch require reprocessing?
if (not self.config.clobberMeasurements and
dataRef.datasetExists(self.config.coaddName + "Coadd_meas")):
if not self.config.reprocessing:
return False
catalog = dataRef.get(self.config.coaddName + "Coadd_meas")
bigFlag = catalog["deblend.parent-too-big"]
numOldBig = bigFlag.sum()
if numOldBig == 0:
self.log.info("No large footprints in %s" % (dataRef.dataId,))
return False
numNewBig = sum((self.measureCoaddSources.deblend.isLargeFootprint(src.getFootprint()) for
src in catalog[bigFlag]))
if numNewBig == numOldBig:
self.log.info("All %d formerly large footprints continue to be large in %s" %
(numOldBig, dataRef.dataId,))
return False
self.log.info("Found %d large footprints to be reprocessed in %s" %
(numOldBig - numNewBig, dataRef.dataId))
reprocessing = True
self.measureCoaddSources.run(dataRef)
return reprocessing
def run(self, tractPatchRefList, butler, selectIdList=[]):
"""!Determine which tracts are non-empty before processing
@param tractPatchRefList: List of tracts and patches to include in the coaddition
@param butler: butler reference object
@param selectIdList: List of data Ids (i.e. visit, ccd) to consider when making the coadd
@return list of references to sel.runTract function evaluation for each tractPatchRefList member
"""
pool = Pool("tracts")
pool.storeSet(butler=butler, skymap=butler.get(self.config.coaddName + "Coadd_skyMap"))
tractIdList = []
for patchRefList in tractPatchRefList:
tractSet = set([patchRef.dataId["tract"] for patchRef in patchRefList])
assert len(tractSet) == 1
tractIdList.append(tractSet.pop())
selectDataList = [data for data in pool.mapNoBalance(self.readSelection, selectIdList) if
data is not None]
nonEmptyList = pool.mapNoBalance(self.checkTract, tractIdList, selectDataList)
tractPatchRefList = [patchRefList for patchRefList, nonEmpty in
zip(tractPatchRefList, nonEmptyList) if nonEmpty]
self.log.info("Non-empty tracts (%d): %s" % (len(tractPatchRefList),
[patchRefList[0].dataId["tract"] for patchRefList in
tractPatchRefList]))
# Install the dataRef in the selectDataList
for data in selectDataList:
data.dataRef = getDataRef(butler, data.dataId, "calexp")
# Process the non-empty tracts
return [self.runTract(patchRefList, butler, selectDataList) for patchRefList in tractPatchRefList]
def coadd(self, cache, data):
"""!Construct coadd for a patch and measure
Only slave nodes execute this method.
Because only one argument may be passed, it is expected to
contain multiple elements, which are:
@param patchRef: data reference for patch
@param selectDataList: List of SelectStruct for inputs
"""
patchRef = getDataRef(cache.butler, data.patchId, cache.coaddType)
selectDataList = data.selectDataList
coadd = None
with self.logOperation("coadding %s" % (patchRef.dataId, ),
catch=True):
if self.config.doOverwriteCoadd or not patchRef.datasetExists(
cache.coaddType):
coaddResults = self.assembleCoadd.run(patchRef, selectDataList)
if coaddResults is not None:
coadd = coaddResults.coaddExposure
elif patchRef.datasetExists(cache.coaddType):
self.log.info("%s: Reading coadd %s" % (NODE, patchRef.dataId))
coadd = patchRef.get(cache.coaddType, immediate=True)
if coadd is None:
return
with self.logOperation("detection on %s" % (patchRef.dataId, ),
catch=True):
idFactory = self.detectCoaddSources.makeIdFactory(patchRef)
# This includes background subtraction, so do it before writing the
# coadd
detResults = self.detectCoaddSources.runDetection(coadd, idFactory)
self.detectCoaddSources.write(coadd, detResults, patchRef)
def runMergeDetections(self, cache, dataIdList):
"""Run detection merging on a patch
Only slave nodes execute this method.
@param cache: Pool cache, containing butler
@param dataIdList: List of data identifiers for the patch in different filters
"""
with self.logOperation("merge detections from %s" % (dataIdList,)):
dataRefList = [getDataRef(cache.butler, dataId, self.config.coaddName + "Coadd") for
dataId in dataIdList]
if (not self.config.clobberMergedDetections and
dataRefList[0].datasetExists(self.config.coaddName + "Coadd_mergeDet")):
return
self.mergeCoaddDetections.run(dataRefList)
def runForcedPhot(self, cache, dataId):
"""Run forced photometry on a patch for a single filter
Only slave nodes execute this method.
@cache: Pool cache, with butler
@dataId: Data identifier for patch
"""
with self.logOperation("forced photometry on %s" % (dataId,)):
dataRef = getDataRef(cache.butler, dataId, self.config.coaddName + "Coadd")
if (not self.config.clobberForcedPhotometry and
not self.config.reprocessing and
dataRef.datasetExists(self.config.coaddName + "Coadd_forced_src")):
return
self.forcedPhotCoadd.run(dataRef)
def warp(self, cache, patchId, selectDataList):
"""Warp all images for a patch
Only slave nodes execute this method.
Because only one argument may be passed, it is expected to
contain multiple elements, which are:
@param patchRef: data reference for patch
@param selectDataList: List of SelectStruct for inputs
@return selectDataList with non-overlapping elements removed
"""
patchRef = getDataRef(cache.butler, patchId, cache.coaddType)
selectDataList = self.selectExposures(patchRef, selectDataList)
with self.logOperation("warping %s" % (patchRef.dataId,), catch=True):
self.makeCoaddTempExp.run(patchRef, selectDataList)
return selectDataList
def runMergeMeasurements(self, cache, dataIdList):
"""Run measurement merging on a patch
Only slave nodes execute this method.
@cache: Pool cache, containing butler
@dataIdList: List of data identifiers for the patch in different filters
"""
with self.logOperation("merge measurements from %s" % (dataIdList,)):
dataRefList = [getDataRef(cache.butler, dataId, self.config.coaddName + "Coadd") for
dataId in dataIdList]
if (not self.config.clobberMergedMeasurements and
not self.config.reprocessing and
dataRefList[0].datasetExists(self.config.coaddName + "Coadd_ref")):
return
self.mergeCoaddMeasurements.run(dataRefList)
def runForcedPhot(self, cache, dataId):
"""!Run forced photometry on a patch for a single filter
Only slave nodes execute this method.
@param cache: Pool cache, with butler
@param dataId: Data identifier for patch
"""
with self.logOperation("forced photometry on %s" % (dataId, )):
dataRef = getDataRef(cache.butler, dataId,
self.config.coaddName + "Coadd_calexp")
if (not self.config.clobberForcedPhotometry
and not self.config.reprocessing
and dataRef.datasetExists(self.config.coaddName +
"Coadd_forced_src")):
return
self.forcedPhotCoadd.run(dataRef)
def runMeasureMerged(self, cache, dataId):
"""!Run measurement on a patch for a single filter
Only slave nodes execute this method.
@param cache: Pool cache, with butler
@param dataId: Data identifier for patch
@return whether the patch requires reprocessing.
"""
with self.logOperation("measurement on %s" % (dataId, )):
dataRef = getDataRef(cache.butler, dataId,
self.config.coaddName + "Coadd_calexp")
reprocessing = False # Does this patch require reprocessing?
if ("measureCoaddSources" in self.reuse and dataRef.datasetExists(
self.config.coaddName + "Coadd_meas", write=True)):
if not self.config.reprocessing:
self.log.info(
"Skipping measureCoaddSources for %s; output already exists"
% dataId)
return False
catalog = dataRef.get(self.config.coaddName + "Coadd_meas")
bigFlag = catalog["deblend.parent-too-big"]
numOldBig = bigFlag.sum()
if numOldBig == 0:
self.log.info("No large footprints in %s" %
(dataRef.dataId, ))
return False
numNewBig = sum(
(self.measureCoaddSources.deblend.isLargeFootprint(
src.getFootprint()) for src in catalog[bigFlag]))
if numNewBig == numOldBig:
self.log.info(
"All %d formerly large footprints continue to be large in %s"
% (
numOldBig,
dataRef.dataId,
))
return False
self.log.info(
"Found %d large footprints to be reprocessed in %s" %
(numOldBig - numNewBig, dataRef.dataId))
reprocessing = True
self.measureCoaddSources.run(dataRef)
return reprocessing
def warp(self, cache, patchId, selectDataList):
"""!Warp all images for a patch
Only slave nodes execute this method.
Because only one argument may be passed, it is expected to
contain multiple elements, which are:
@param patchRef: data reference for patch
@param selectDataList: List of SelectStruct for inputs
@return selectDataList with non-overlapping elements removed
"""
patchRef = getDataRef(cache.butler, patchId, cache.coaddType)
selectDataList = self.selectExposures(patchRef, selectDataList)
with self.logOperation("warping %s" % (patchRef.dataId, ), catch=True):
self.makeCoaddTempExp.runDataRef(patchRef, selectDataList)
return selectDataList
def processPool(self, cache, dataId):
"""Process focus CCD under pool
This is a mediator for the 'process' method when running
under the Pool.
Only slave nodes run this method.
@param cache: Pool cache
@param dataId: Data identifier for CCD
@return Processing results (from 'process' method)
"""
try:
return self.process(getDataRef(cache.butler, dataId))
except Exception as e:
self.log.warn("Failed to process %s (%s: %s):\n%s" %
(dataId, e.__class__.__name__, e, traceback.format_exc()))
return None
def runDataRef(self, patchRefList):
"""!Run multiband processing on coadds
Only the master node runs this method.
No real MPI communication (scatter/gather) takes place: all I/O goes
through the disk. We want the intermediate stages on disk, and the
component Tasks are implemented around this, so we just follow suit.
@param patchRefList: Data references to run measurement
"""
print(len(patchRefList))
for patchRef in patchRefList:
if patchRef:
butler = patchRef.getButler()
break
else:
raise RuntimeError("No valid patches")
pool = Pool("all")
pool.cacheClear()
pool.storeSet(butler=butler)
# Group by patch
patches = {}
tract = None
for patchRef in patchRefList:
dataId = patchRef.dataId
if tract is None:
tract = dataId["tract"]
else:
assert tract == dataId["tract"]
patch = dataId["patch"]
if patch not in patches:
patches[patch] = []
patches[patch].append(dataId)
print(patches.values())
dataRefList = [
getDataRef(cache.butler, dataId,
self.config.coaddName + "Coadd_calexp")
for dataId in patches.values()
]
pool.map(self.runAssociation, dataRefList)
def runMergeDetections(self, cache, dataIdList):
"""!Run detection merging on a patch
Only slave nodes execute this method.
@param cache: Pool cache, containing butler
@param dataIdList: List of data identifiers for the patch in different filters
"""
with self.logOperation("merge detections from %s" % (dataIdList, )):
dataRefList = [
getDataRef(cache.butler, dataId,
self.config.coaddName + "Coadd_calexp")
for dataId in dataIdList
]
if (not self.config.clobberMergedDetections
and dataRefList[0].datasetExists(self.config.coaddName +
"Coadd_mergeDet")):
return
self.mergeCoaddDetections.run(dataRefList)
def coadd(self, cache, data):
"""!Construct coadd for a patch and measure
Only slave nodes execute this method.
Because only one argument may be passed, it is expected to
contain multiple elements, which are:
@param patchRef: data reference for patch
@param selectDataList: List of SelectStruct for inputs
"""
patchRef = getDataRef(cache.butler, data.patchId, cache.coaddType)
selectDataList = data.selectDataList
coadd = None
with self.logOperation("coadding %s" % (patchRef.dataId, ),
catch=True):
if self.config.doOverwriteCoadd or not patchRef.datasetExists(
cache.coaddType):
coaddResults = self.assembleCoadd.run(patchRef, selectDataList)
if coaddResults is not None:
coadd = coaddResults.coaddExposure
elif patchRef.datasetExists(cache.coaddType):
self.log.info("%s: Reading coadd %s" % (NODE, patchRef.dataId))
coadd = patchRef.get(cache.coaddType, immediate=True)
if coadd is None:
return
# The section of code below determines if the detection task should be
# run. If detection is run, then the products are written out as
# deepCoadd_calexp. If detection is not run, then the outputs of the
# assemble task are written out as deepCoadd.
if self.config.doDetection:
with self.logOperation("detection on {}".format(patchRef.dataId),
catch=True):
idFactory = self.detectCoaddSources.makeIdFactory(patchRef)
# This includes background subtraction, so do it before writing
# the coadd
detResults = self.detectCoaddSources.runDetection(
coadd, idFactory)
self.detectCoaddSources.write(coadd, detResults, patchRef)
else:
patchRef.put(coadd, self.assembleCoadd.config.coaddName + "Coadd")
def readSelection(self, cache, selectId):
"""Read Wcs of selected inputs
This method only runs on slave nodes.
This method is similar to SelectDataIdContainer.makeDataRefList,
creating a Struct like a SelectStruct, except with a dataId instead
of a dataRef (to ease MPI).
@param cache: Pool cache
@param selectId: Data identifier for selected input
@return a SelectStruct with a dataId instead of dataRef
"""
try:
ref = getDataRef(cache.butler, selectId, "calexp")
self.log.info("Reading Wcs from %s" % (selectId,))
md = ref.get("calexp_md", immediate=True)
wcs = afwImage.makeWcs(md)
data = Struct(dataId=selectId, wcs=wcs, dims=(md.get("NAXIS1"), md.get("NAXIS2")))
except FitsError as e:
self.log.warn("Unable to construct Wcs from %s" % (selectId,))
return None
return data
def runForcedPhot(self, cache, dataId):
"""!Run forced photometry on a patch for a single filter
Only slave nodes execute this method.
@param cache: Pool cache, with butler
@param dataId: Data identifier for patch
"""
with self.logOperation("forced photometry on %s" % (dataId, )):
dataRef = getDataRef(cache.butler, dataId,
self.coaddType + "Coadd_calexp")
if ("forcedPhotCoadd" in self.reuse
and not self.config.reprocessing and dataRef.datasetExists(
self.config.coaddName + "Coadd_forced_src",
write=True)):
self.log.info(
"Skipping forcedPhotCoadd for %s; output already exists" %
dataId)
return
self.forcedPhotCoadd.runDataRef(dataRef)
def readSelection(self, cache, selectId):
"""!Read Wcs of selected inputs
This method only runs on slave nodes.
This method is similar to SelectDataIdContainer.makeDataRefList,
creating a Struct like a SelectStruct, except with a dataId instead
of a dataRef (to ease MPI).
@param cache: Pool cache
@param selectId: Data identifier for selected input
@return a SelectStruct with a dataId instead of dataRef
"""
try:
ref = getDataRef(cache.butler, selectId, "calexp")
self.log.info("Reading Wcs from %s" % (selectId,))
md = ref.get("calexp_md", immediate=True)
wcs = afwGeom.makeSkyWcs(md)
data = Struct(dataId=selectId, wcs=wcs, bbox=afwImage.bboxFromMetadata(md))
except FitsError:
self.log.warn("Unable to construct Wcs from %s" % (selectId,))
return None
return data
def runMergeDetections(self, cache, dataIdList):
"""!Run detection merging on a patch
Only slave nodes execute this method.
@param cache: Pool cache, containing butler
@param dataIdList: List of data identifiers for the patch in different filters
"""
with self.logOperation("merge detections from %s" % (dataIdList, )):
dataRefList = [
getDataRef(cache.butler, dataId,
self.coaddType + "Coadd_calexp")
for dataId in dataIdList
]
if ("mergeCoaddDetections" in self.reuse
and dataRefList[0].datasetExists(
self.config.coaddName + "Coadd_mergeDet", write=True)):
self.log.info(
"Skipping mergeCoaddDetections for %s; output already exists."
% dataRefList[0].dataId)
return
self.mergeCoaddDetections.runDataRef(dataRefList)
def combine(self, cache, struct, outputId):
"""!Combine multiple exposures of a particular CCD and write the output
Only the slave nodes execute this method.
This is a helper routine, containing just the start of what's needed to
actually combine the inputs as an aid to subclasses. Note that
thismethod does not return what the ``scatterCombine`` method expects;
the user should call this method, and then return the combined exposure.
Parameters
----------
cache : `lsst.pipe.base.Struct`
Process pool cache. Contains a data butler (``butler``).
struct : `lsst.pipe.base.Struct`
Parameters for the combination, which has the following components:
- ``ccdName`` (`tuple`): Name tuple for CCD.
- ``ccdIdList`` (`list`): List of data identifiers for combination.
- ``scales``: Unused by this implementation.
Returns
-------
dataRefList : `dict`
Data identifier for combined image (exposure part only).
outputId : `dict`
Fully-qualified data identifier for the output.
"""
# Check if we need to look up any keys that aren't in the output dataId
fullOutputId = {k: struct.ccdName[i] for i, k in enumerate(self.config.ccdKeys)}
self.addMissingKeys(fullOutputId, cache.butler)
fullOutputId.update(outputId) # must be after the call to queryMetadata
dataRefList = [getDataRef(cache.butler, dataId) if dataId is not None else None for
dataId in struct.ccdIdList]
self.log.info("Combining %d inputs %s on %s" % (len(dataRefList), fullOutputId, NODE))
return Struct(dataRefList=dataRefList, outputId=fullOutputId)
def runMeasurements(self, cache, dataId):
"""Run measurement on a patch for a single filter
Only slave nodes execute this method.
Parameters
----------
cache: Pool cache
Pool cache, with butler
dataId: dataRef
Data identifier for patch
"""
with self.logOperation("measurements on %s" % (dataId, )):
dataRef = getDataRef(cache.butler, dataId,
self.coaddType + "Coadd_calexp")
if ("measureCoaddSources" in self.reuse
and not self.config.reprocessing and dataRef.datasetExists(
self.config.coaddName + "Coadd_meas", write=True)):
self.log.info(
"Skipping measuretCoaddSources for %s; output already exists"
% dataId)
return
self.measureCoaddSources.runDataRef(dataRef)
def runCombine(self, cache, dataIdList):
"""! Run combination on a patch
For all of the visits that overlap this patch in the band create a catalog with
mjd, id, filter and values in the keepFields config parameter.
It will write a file as a pandas DataFrame to the appropriate deepDiff directory,
but the products are not currently declared in the mapper file.
"""
dataRefList = [
getDataRef(cache.butler, dataId,
self.config.coaddName + "Coadd_calexp")
for dataId in dataIdList
]
try:
diaObject = dataRefList[0].get(
f"{self.config.coaddName}Diff_diaObject")
uri = dataRefList[0].getUri(
f"{self.config.coaddName}Diff_diaObject")
except Exception:
self.log.info('Cannot read diaObject for %s' %
(dataRefList[0].dataId))
return
# Use a dictionary of arrays to store data. Probably something more efficient
data = {}
data['id'] = []
data['mjd'] = []
data['filter'] = []
for key, value in self.config.keepFields.items():
data[key] = []
for dataRef in dataRefList:
try:
calexp = dataRef.get(f"{self.config.coaddName}Coadd_calexp")
except Exception:
self.log.info('Cannot read data for %s' % (dataRef.dataId))
continue
visitCatalog = calexp.getInfo().getCoaddInputs().ccds
for visitRec in visitCatalog:
visit = int(visitRec.get('visit'))
ccd = int(visitRec.get('ccd'))
dataId = {"visit": visit, self.config.ccdKey: ccd}
try:
src = cache.butler.get(
f"{self.config.coaddName}Diff_forced_dia_src", dataId)
diff = cache.butler.get(
f"{self.config.coaddName}Diff_differenceExp", dataId)
except Exception as e:
self.log.debug('Cannot read data for %d %d. skipping %s',
visit, ccd, e)
continue
mjd = diff.getInfo().getVisitInfo().getDate().get(
system=DateTime.MJD)
band = diff.getInfo().getFilter().getName()
self.log.info('Reading diff forced src with %d sources %s',
len(src), dataId)
matches = np.in1d(src['dia_object_id'],
diaObject['id'],
assume_unique=True)
data['id'].extend(src['dia_object_id'][matches])
data['mjd'].extend([mjd] * np.sum(matches))
data['filter'].extend([band] * np.sum(matches))
for key, value in self.config.keepFields.items():
data[key].extend(src[value][matches])
tract = dataRefList[0].dataId['tract']
patch = dataRefList[0].dataId['patch']
path = os.path.dirname(uri)
df = pd.DataFrame(data)
getattr(df, "to_" + self.config.storage)(
f'{path}/diaCombined_{tract}_{patch}.{self.config.storage}')
def runAssociation(self, cache, dataIdList, selectDataList):
"""! Run association on a patch
For all of the visits that overlap this patch in the band create a DIAObject
catalog. Only the objects in the non-overlaping area of the tract and patch
are included.
"""
dataRefList = [
getDataRef(cache.butler, dataId,
self.config.coaddName + "Coadd_calexp")
for dataId in dataIdList
]
# We need the WCS for the patch, so we can use the first entry in the dataIdList
dataRef = dataRefList[0]
tract = dataRef.dataId['tract']
skyInfo = getSkyInfo(coaddName=self.config.coaddName, patchRef=dataRef)
skyMap = skyInfo.skyMap
try:
calexp = dataRef.get(f"{self.config.coaddName}Coadd_calexp")
except Exception:
self.log.info('Cannot read coadd data for %s' % (dataRef.dataId))
return
coaddWcs = calexp.getWcs()
innerPatchBox = geom.Box2D(skyInfo.patchInfo.getInnerBBox())
expBits = dataRef.get("deepMergedCoaddId_bits")
expId = int(dataRef.get("deepMergedCoaddId"))
idFactory = afwTable.IdFactory.makeSource(expId, 64 - expBits)
if len(selectDataList) == 0:
differenceImages = self.catalogGenerator
else:
differenceImages = self.idListGenerator
initializeSelector = False
for diffIm in differenceImages(cache, dataRefList, selectDataList):
if initializeSelector is False:
self.associator.initialize(diffIm.src.schema, idFactory)
initializeSelector = True
if len(diffIm.src) == 0:
continue
srcWcs = diffIm.exp.getWcs()
isInside = np.array([
innerPatchBox.contains(
coaddWcs.skyToPixel(srcWcs.pixelToSky(a.getCentroid())))
for a in diffIm.src
],
dtype=bool)
isGood = np.array([
rec.getFootprint().contains(geom.Point2I(rec.getCentroid()))
for rec in diffIm.src
], )
isInnerTract = np.array([
skyMap.findTract(srcWcs.pixelToSky(
a.getCentroid())).getId() == tract for a in diffIm.src
])
mask = (isInside) & (isGood) & (isInnerTract)
src = diffIm.src[mask]
if len(src) == 0:
continue
self.log.info(
'Reading difference image %d %d, %s with %d possible sources' %
(diffIm.visit, diffIm.ccd, diffIm.filter, len(src)))
footprints = []
region = calexp.getBBox(afwImage.PARENT)
for ii, rec in enumerate(src):
# transformations on large footprints can take a long time
# We truncate the footprint since we will rarely be interested
# in such large footprints
if rec.getFootprint().getArea() > self.config.maxFootprintArea:
spans = afwGeom.SpanSet.fromShape(
self.config.defaultFootprintRadius,
afwGeom.Stencil.CIRCLE,
geom.Point2I(rec.getCentroid()))
foot = afwDet.Footprint(spans)
foot.addPeak(int(rec.getX()), int(rec.getY()), 1)
else:
foot = rec.getFootprint()
footprints.append(foot.transform(srcWcs, coaddWcs, region))
self.associator.addCatalog(src, diffIm.filter, diffIm.visit,
diffIm.ccd, diffIm.calib, footprints)
result = self.associator.finalize(idFactory)
if len(dataRefList) > 0 and result is not None:
dataRefList[0].put(result,
self.config.coaddName + 'Diff_diaObject')
self.log.info('Total objects found %d' % len(result))
idCatalog = self.associator.getObjectIds()
dataRefList[0].put(idCatalog,
self.config.coaddName + 'Diff_diaObjectId')
def runDataRef(self, expRef, butler):
"""Process a single exposure, with scatter-gather-scatter using MPI.
"""
dataIdList = dict([(ccdRef.get("ccdExposureId"), ccdRef.dataId)
for ccdRef in expRef.subItems("ccd")
if ccdRef.datasetExists("donutSrc")])
dataIdList = collections.OrderedDict(sorted(dataIdList.items()))
visit = expRef.dataId['visit']
self.log.info("Running on visit {}".format(visit))
donutConfig = getDonutConfig(expRef)
x = []
y = []
vals = collections.OrderedDict()
jmax = donutConfig.jmaxs[-1]
for k in ['r0'] + ['z{}'.format(z) for z in range(4, jmax + 1)]:
vals[k] = []
for dataId in dataIdList.values():
self.log.info("Loading ccd {}".format(dataId['ccd']))
sensorRef = getDataRef(butler, dataId)
donutSrc = sensorRef.get("donutSrc")
icSrc = sensorRef.get("icSrc")
icExp = sensorRef.get("icExp")
x.extend([
icSrc.find(donut.getId())['base_FPPosition_x']
for donut in donutSrc
])
y.extend([
icSrc.find(donut.getId())['base_FPPosition_y']
for donut in donutSrc
])
for k, v in iteritems(vals):
v.extend(donut['zfit_jmax{}_{}'.format(jmax, k)]
for donut in donutSrc)
plotDir = getPlotDir(sensorRef.getButler(), "donutSrc",
first(dataIdList.values()))
outfn = os.path.join(plotDir, "donutFitParam-{:07d}.pdf".format(visit))
with PdfPages(outfn) as pdf:
for k, v in iteritems(vals):
self.log.info("Plotting {}".format(k))
if k.startswith('z') and k != 'z4':
cmap = 'Spectral_r'
vmin = -max(np.abs(v))
vmax = max(np.abs(v))
else:
cmap = 'viridis'
vmin = min(v)
vmax = max(v)
fig, axes = subplots(1, 1, figsize=(8, 6.2))
scatPlot = axes.scatter(x,
y,
c=v,
s=15,
linewidths=0.5,
cmap=cmap,
vmin=vmin,
vmax=vmax)
axes.set_title(k)
plotCameraOutline(axes, expRef.get("camera"))
fig.tight_layout()
fig.colorbar(scatPlot)
pdf.savefig(fig, dpi=100)
def runDataRef(self, expRef, butler):
"""Make a stamp analysis image for a single exposure, binned by CCD.
"""
dataIdList = dict([(ccdRef.get("ccdExposureId"), ccdRef.dataId)
for ccdRef in expRef.subItems("ccd")
if ccdRef.datasetExists("donutSrc")])
dataIdList = collections.OrderedDict(sorted(dataIdList.items()))
visit = expRef.dataId['visit']
self.log.info("Running on visit {}".format(visit))
camera = expRef.get("camera")
donutConfig = getDonutConfig(expRef)
plotDir = getPlotDir(expRef.getButler(), "donutSrc",
first(dataIdList.values()))
# Collect images
images = {}
models = {}
resids = {}
for dataId in dataIdList.values():
ccd = dataId['ccd']
self.log.info("Loading ccd {}".format(ccd))
sensorRef = getDataRef(butler, dataId)
icExp = sensorRef.get("icExp")
donutSrc = sensorRef.get("donutSrc")
icSrc = sensorRef.get("icSrc")
if len(donutSrc) == 0:
continue
s2n = []
for donut in donutSrc:
icRec = icSrc.find(donut.getId())
s2n.append(icRec['base_CircularApertureFlux_25_0_flux'] /
icRec['base_CircularApertureFlux_25_0_fluxSigma'])
idx = int(np.argsort(s2n)[-1])
donutRecord = donutSrc[idx]
icRecord = icSrc.find(donutRecord.getId())
data = getDonut(icRecord, icExp, donutConfig)
model = getModel(donutRecord, icRecord, icExp, donutConfig, camera)
resid = data - model
images[ccd] = data
models[ccd] = model
resids[ccd] = resid
# Make plots
self.makePlot(images,
os.path.join(
plotDir,
"donutStampCcdData-{:07d}.pdf".format(visit),
),
camera,
dataIdList,
cmap='viridis')
self.makePlot(models,
os.path.join(
plotDir,
"donutStampCcdModel-{:07d}.pdf".format(visit),
),
camera,
dataIdList,
cmap='viridis')
self.makePlot(resids,
os.path.join(
plotDir,
"donutStampCcdResid-{:07d}.pdf".format(visit),
),
camera,
dataIdList,
cmap='Spectral_r')
def runDataRef(self, expRef, butler):
"""Make a stamp analysis image for a single exposure
"""
dataIdList = dict([(ccdRef.get("ccdExposureId"), ccdRef.dataId)
for ccdRef in expRef.subItems("ccd")
if ccdRef.datasetExists("donutSrc")])
dataIdList = collections.OrderedDict(sorted(dataIdList.items()))
visit = expRef.dataId['visit']
self.log.info("Running on visit {}".format(visit))
camera = expRef.get("camera")
donutConfig = getDonutConfig(expRef)
plotDir = getPlotDir(expRef.getButler(), "donutSrc",
first(dataIdList.values()))
xs = []
ys = []
s2ns = []
sensorRefs = []
ids = []
for dataId in dataIdList.values():
ccd = dataId['ccd']
self.log.info("Loading ccd {}".format(ccd))
sensorRef = getDataRef(butler, dataId)
donutSrc = sensorRef.get("donutSrc")
icSrc = sensorRef.get("icSrc")
if len(donutSrc) == 0:
continue
for donut in donutSrc:
donutId = donut.getId()
icRec = icSrc.find(donutId)
s2ns.append(icRec['base_CircularApertureFlux_25_0_flux'] /
icRec['base_CircularApertureFlux_25_0_fluxSigma'])
xs.append(icRec['base_FPPosition_x'])
ys.append(icRec['base_FPPosition_y'])
sensorRefs.append(sensorRef)
ids.append(donutId)
bds = camera.getFpBBox()
xbds = np.linspace(bds.getMinX(), bds.getMaxX(),
self.config.nStamp + 1)
ybds = np.linspace(bds.getMinY(), bds.getMaxY(),
self.config.nStamp + 1)
# Find brightest donut in each grid cell and retrieve that one.
s2ns = np.array(s2ns)
xs = np.array(xs)
ys = np.array(ys)
imageDict = {}
modelDict = {}
residDict = {}
for ix, (xmin, xmax) in enumerate(zip(xbds[:-1], xbds[1:])):
for iy, (ymin, ymax) in enumerate(zip(ybds[:-1], ybds[1:])):
w = (xs > xmin) & (xs <= xmax) & (ys > ymin) & (ys <= ymax)
if not np.any(w):
continue
widx = int(np.argsort(s2ns[w])[-1])
idx = w.nonzero()[0][widx]
sensorRef = sensorRefs[idx]
donutId = ids[idx]
icExp = sensorRef.get("icExp")
icSrc = sensorRef.get("icSrc")
donutSrc = sensorRef.get("donutSrc")
icRec = icSrc.find(donutId)
donut = donutSrc.find(donutId)
data = getDonut(icRec, icExp, donutConfig)
model = getModel(donut, icRec, icExp, donutConfig, camera)
imageDict[(ix, iy)] = data
modelDict[(ix, iy)] = model
residDict[(ix, iy)] = data - model
# Make plots
self.makePlot(imageDict,
os.path.join(plotDir,
"donutStampData-{:07d}.pdf".format(visit)),
xbds,
ybds,
camera,
cmap='viridis')
self.makePlot(modelDict,
os.path.join(plotDir,
"donutStampModel-{:07d}.pdf".format(visit)),
xbds,
ybds,
camera,
cmap='viridis')
self.makePlot(residDict,
os.path.join(plotDir,
"donutStampResid-{:07d}.pdf".format(visit)),
xbds,
ybds,
camera,
cmap='Spectral_r')
def coadd(self, cache, data):
"""!Construct coadd for a patch and measure
Only slave nodes execute this method.
Because only one argument may be passed, it is expected to
contain multiple elements, which are:
@param patchRef: data reference for patch
@param selectDataList: List of SelectStruct for inputs
"""
patchRef = getDataRef(cache.butler, data.patchId, cache.coaddType)
selectDataList = data.selectDataList
coadd = None
# We skip the assembleCoadd step if either the *Coadd dataset exists
# or we aren't configured to write it, we're supposed to reuse
# detectCoaddSources outputs too, and those outputs already exist.
canSkipDetection = (
"detectCoaddSources" in self.reuse and patchRef.datasetExists(
self.detectCoaddSources.config.coaddName + "Coadd_det",
write=True))
if "assembleCoadd" in self.reuse:
if patchRef.datasetExists(cache.coaddType, write=True):
self.log.info(
"%s: Skipping assembleCoadd for %s; outputs already exist."
% (NODE, patchRef.dataId))
coadd = patchRef.get(cache.coaddType, immediate=True)
elif not self.config.assembleCoadd.doWrite and self.config.doDetection and canSkipDetection:
self.log.info(
"%s: Skipping assembleCoadd and detectCoaddSources for %s; outputs already exist."
% (NODE, patchRef.dataId))
return
if coadd is None:
with self.logOperation("coadding %s" % (patchRef.dataId, ),
catch=True):
coaddResults = self.assembleCoadd.runDataRef(
patchRef, selectDataList)
if coaddResults is not None:
coadd = coaddResults.coaddExposure
canSkipDetection = False # can't skip it because coadd may have changed
if coadd is None:
return
# The section of code below determines if the detection task should be
# run. If detection is run, then the products are written out as
# deepCoadd_calexp. If detection is not run, then the outputs of the
# assemble task are written out as deepCoadd.
if self.config.doDetection:
if canSkipDetection:
self.log.info(
"%s: Skipping detectCoaddSources for %s; outputs already exist."
% (NODE, patchRef.dataId))
return
with self.logOperation("detection on {}".format(patchRef.dataId),
catch=True):
idFactory = self.detectCoaddSources.makeIdFactory(patchRef)
expId = int(patchRef.get(self.config.coaddName + "CoaddId"))
# This includes background subtraction, so do it before writing
# the coadd
detResults = self.detectCoaddSources.run(coadd,
idFactory,
expId=expId)
self.detectCoaddSources.write(detResults, patchRef)
else:
if self.config.hasFakes:
patchRef.put(
coadd,
"fakes_" + self.assembleCoadd.config.coaddName + "Coadd")
else:
patchRef.put(coadd,
self.assembleCoadd.config.coaddName + "Coadd")
def runDeblendMerged(self, cache, dataIdList):
"""Run the deblender on a list of dataId's
Only slave nodes execute this method.
Parameters
----------
cache: Pool cache
Pool cache with butler.
dataIdList: list
Data identifier for patch in each band.
Returns
-------
result: bool
whether the patch requires reprocessing.
"""
with self.logOperation("deblending %s" % (dataIdList, )):
dataRefList = [
getDataRef(cache.butler, dataId,
self.coaddType + "Coadd_calexp")
for dataId in dataIdList
]
reprocessing = False # Does this patch require reprocessing?
if ("deblendCoaddSources" in self.reuse and all([
dataRef.datasetExists(self.config.coaddName + "Coadd_" +
self.measurementInput,
write=True)
for dataRef in dataRefList
])):
if not self.config.reprocessing:
self.log.info(
"Skipping deblendCoaddSources for %s; output already exists"
% dataIdList)
return False
# Footprints are the same every band, therefore we can check just one
catalog = dataRefList[0].get(self.config.coaddName + "Coadd_" +
self.measurementInput)
bigFlag = catalog["deblend_parentTooBig"]
# Footprints marked too large by the previous deblender run
numOldBig = bigFlag.sum()
if numOldBig == 0:
self.log.info("No large footprints in %s" %
(dataRefList[0].dataId))
return False
# This if-statement can be removed after DM-15662
if self.config.deblendCoaddSources.simultaneous:
deblender = self.deblendCoaddSources.multiBandDeblend
else:
deblender = self.deblendCoaddSources.singleBandDeblend
# isLargeFootprint() can potentially return False for a source that is marked
# too big in the catalog, because of "new"/different deblender configs.
# numNewBig is the number of footprints that *will* be too big if reprocessed
numNewBig = sum((deblender.isLargeFootprint(src.getFootprint())
for src in catalog[bigFlag]))
if numNewBig == numOldBig:
self.log.info(
"All %d formerly large footprints continue to be large in %s"
% (
numOldBig,
dataRefList[0].dataId,
))
return False
self.log.info(
"Found %d large footprints to be reprocessed in %s" %
(numOldBig - numNewBig,
[dataRef.dataId for dataRef in dataRefList]))
reprocessing = True
self.deblendCoaddSources.runDataRef(dataRefList)
return reprocessing
