def run(self, catalog, filterName=None):
"""!Load reference objects and match to them
@param[in] catalog Catalog to match to (lsst.afw.table.SourceCatalog)
@param[in] filterName Name of filter, for loading fluxes (str)
@return Struct with matches (lsst.afw.table.SourceMatchVector) and
matchMeta (lsst.meas.astrom.MatchMetadata)
"""
circle = self.calculateCircle(catalog)
matchMeta = self.refObjLoader.getMetadataCircle(circle.center, circle.radius, filterName)
emptyResult = Struct(matches=[], matchMeta=matchMeta)
sourceSelection = self.sourceSelection.run(catalog)
if len(sourceSelection.sourceCat) == 0:
self.log.warn("No objects selected from %d objects in source catalog", len(catalog))
return emptyResult
refData = self.refObjLoader.loadSkyCircle(circle.center, circle.radius, filterName)
refCat = refData.refCat
refSelection = self.referenceSelection.run(refCat)
if len(refSelection.sourceCat) == 0:
self.log.warn("No objects selected from %d objects in reference catalog", len(refCat))
return emptyResult
matches = afwTable.matchRaDec(refSelection.sourceCat, sourceSelection.sourceCat,
self.config.matchRadius*arcseconds)
self.log.info("Matched %d from %d/%d input and %d/%d reference sources" %
(len(matches), len(sourceSelection.sourceCat), len(catalog),
len(refSelection.sourceCat), len(refCat)))
return Struct(matches=matches, matchMeta=matchMeta, refCat=refCat, sourceSelection=sourceSelection,
refSelection=refSelection)
def run(self, coadds, filters, mergedDetections, idFactory):
sources = self._makeSourceCatalog(mergedDetections, idFactory)
multiExposure = afwImage.MultibandExposure.fromExposures(filters, coadds)
fluxCatalogs, templateCatalogs = self.multibandDeblend.run(multiExposure, sources)
retStruct = Struct(templateCatalogs)
if self.config.multibandDeblend.conserveFlux:
retStruct.fluxCatalogs = fluxCatalogs
return retStruct
def run(self, coadds, filters, mergedDetections, idFactory):
sources = self._makeSourceCatalog(mergedDetections, idFactory)
multiExposure = afwImage.MultibandExposure.fromExposures(filters, coadds)
fluxCatalogs, templateCatalogs = self.multibandDeblend.run(multiExposure, sources)
retStruct = Struct(templateCatalogs)
if self.config.multibandDeblend.conserveFlux:
retStruct.fluxCatalogs = fluxCatalogs
return retStruct
def run(self, matchedCatalog, metric_name):
self.log.info("Measuring PA1")
pa1 = photRepeat(matchedCatalog, snrMax=self.brightSnrMax, snrMin=self.brightSnrMin,
numRandomShuffles=self.numRandomShuffles, randomSeed=self.randomSeed)
if 'magDiff' in pa1.keys():
return Struct(measurement=Measurement("PA1", pa1['repeatability']))
else:
return Struct(measurement=Measurement("PA1", np.nan*u.mmag))
def makeDistortionData(self):
"""Make distortion data
The data format is a dict of detector name: ccdData, where
ccdData is a Struct containing these fields:
- serial: detector.getSerial
- cornerDict: a dict of pixPosKey, cornerData, where:
- pixPosKey: self.asKey(pixPos) where pixPos is pixel position
- cornerData is Struct contains these fields (all of
type lsst.geom.Point2D):
- pixPos: pixel position
- focalPlane: focal plane position computed from pixPos
- fieldAngle: fieldAngle position computed from focalPlane
- focalPlaneRoundTrip: focal plane position computed from
fieldAngle
- pixPosRoundTrip: pixel position computed from focalPlane
"""
hsc = HyperSuprimeCam()
camera = hsc.getCamera()
focalPlaneToFieldAngle = camera.getTransformMap().getTransform(
FOCAL_PLANE, FIELD_ANGLE)
data = {} # dict of detector name: CcdData
for detector in camera:
# for each corner of each CCD:
# - get pixel position
# - convert to focal plane coordinates using the detector and
# record it
# - convert to field angle (this is the conversion that uses
# HscDistortion) and record it
# - convert back to focal plane (testing inverse direction of
# HscDistortion) and record it
# - convert back to pixel position and record it; pixel <-> focal
# plane is affine so there is no reason to doubt the inverse
# transform, but there is no harm
pixelsToFocalPlane = detector.getTransform(PIXELS, FOCAL_PLANE)
cornerDict = {}
for pixPos in detector.getCorners(PIXELS):
pixPos = pixPos
focalPlane = pixelsToFocalPlane.applyForward(pixPos)
fieldAngle = focalPlaneToFieldAngle.applyForward(focalPlane)
focalPlaneRoundTrip = focalPlaneToFieldAngle.applyInverse(
fieldAngle)
pixPosRoundTrip = pixelsToFocalPlane.applyInverse(focalPlane)
cornerDict[self.toKey(pixPos)] = Struct(
pixPos=pixPos,
focalPlane=focalPlane,
fieldAngle=fieldAngle,
focalPlaneRoundTrip=focalPlaneRoundTrip,
pixPosRoundTrip=pixPosRoundTrip,
)
data[detector.getName()] = Struct(serial=detector.getSerial(),
cornerDict=cornerDict)
return data
def run(self, matchedCatalog, metric_name):
self.log.info(f"Measuring {metric_name}")
D = self.config.annulus_r * u.arcmin
filteredCat = filterMatches(matchedCatalog)
nMinTEx = 50
if filteredCat.count <= nMinTEx:
return Struct(measurement=Measurement(metric_name, np.nan*u.Unit('')))
radius, xip, xip_err = correlation_function_ellipticity_from_matches(filteredCat)
operator = ThresholdSpecification.convert_operator_str(self.config.comparison_operator)
corr, corr_err = select_bin_from_corr(radius, xip, xip_err, radius=D, operator=operator)
return Struct(measurement=Measurement(metric_name, np.abs(corr)*u.Unit('')))
def __init__(self, name, schema, seeing, config, metadata):
deconvKey = schema.addField(name + "_deconv", type="Flag",
doc="deconvolution required for seeing %f; no measurement made" %
(seeing,))
aperture = lsst.meas.base.CircularApertureFluxAlgorithm(config.aperture.makeControl(), name,
schema, metadata)
kronKeys = Struct(
result=lsst.meas.base.FluxResultKey.addFields(schema, name + "_kron",
doc="convolved Kron flux: seeing %f" % (seeing,)),
flag=schema.addField(name + "_kron_flag", type="Flag",
doc="convolved Kron flux failed: seeing %f" % (seeing,)),
)
Struct.__init__(self, deconvKey=deconvKey, aperture=aperture, kronKeys=kronKeys)
def runQuantum(self, butlerQC, inputRefs, outputRefs):
inputs = butlerQC.get(inputRefs)
packedId, maxBits = butlerQC.quantum.dataId.pack("tract_patch",
returnMaxBits=True)
inputs["skySeed"] = packedId
inputs["idFactory"] = afwTable.IdFactory.makeSource(
packedId, 64 - maxBits)
catalogDict = {
ref.dataId['abstract_filter']: cat
for ref, cat in zip(inputRefs.catalogs, inputs['catalogs'])
}
inputs['catalogs'] = catalogDict
skyMap = inputs.pop('skyMap')
# Can use the first dataId to find the tract and patch being worked on
tractNumber = inputRefs.catalogs[0].dataId['tract']
tractInfo = skyMap[tractNumber]
patchInfo = tractInfo.getPatchInfo(
inputRefs.catalogs[0].dataId['patch'])
skyInfo = Struct(skyMap=skyMap,
tractInfo=tractInfo,
patchInfo=patchInfo,
wcs=tractInfo.getWcs(),
bbox=patchInfo.getOuterBBox())
inputs['skyInfo'] = skyInfo
outputs = self.run(**inputs)
butlerQC.put(outputs, outputRefs)
def catalogGenerator(self, cache, dataRefList, selectDataList=[]):
"""! Get a generator of difference images from the visit catalof for each datRef
"""
for dataRef in dataRefList:
try:
calexp = dataRef.get(f"{self.config.coaddName}Coadd_calexp")
except Exception:
self.log.info('Cannot coadd read data for %s' %
(dataRef.dataId))
continue
visitCatalog = calexp.getInfo().getCoaddInputs().ccds
for visitRec in visitCatalog:
visit = visitRec.get('visit')
ccd = visitRec.get('ccd')
dataId = {"visit": visit, self.config.ccdKey: ccd}
try:
exp = cache.butler.get(
f"{self.config.coaddName}Diff_differenceExp", dataId)
src = cache.butler.get(
f"{self.config.coaddName}Diff_diaSrc", dataId)
except Exception as e:
self.log.debug(
'Cannot read difference data for %d %d. skipping %s' %
(visit, ccd, e))
continue
data = Struct(visit=visit,
ccd=ccd,
exp=exp,
src=src,
filter=dataRef.dataId['filter'],
calib=exp.getPhotoCalib())
yield data
def run(self, args):
"""Run ingest
We read and ingest the files in parallel, and then
stuff the registry database in serial.
"""
# Parallel
pool = Pool(None)
filenameList = self.expandFiles(args.files)
dataList = [
Struct(filename=filename, position=ii)
for ii, filename in enumerate(filenameList)
]
infoList = pool.map(self.runFileWrapper, dataList, args)
# Serial
root = args.input
context = self.register.openRegistry(root,
create=args.create,
dryrun=args.dryrun)
with context as registry:
for hduInfoList in infoList:
if hduInfoList is None:
continue
for info in hduInfoList:
self.register.addRow(registry,
info,
dryrun=args.dryrun,
create=args.create)
def selectSources(self, sourceCat, matches=None, exposure=None):
"""Return a selection of sources that are useful for astrometry.
Parameters:
-----------
sourceCat : `lsst.afw.table.SourceCatalog`
Catalog of sources to select from.
This catalog must be contiguous in memory.
matches : `list` of `lsst.afw.table.ReferenceMatch` or None
Ignored in this SourceSelector.
exposure : `lsst.afw.image.Exposure` or None
The exposure the catalog was built from; used for debug display.
Return
------
struct : `lsst.pipe.base.Struct`
The struct contains the following data:
- selected : `array` of `bool``
Boolean array of sources that were selected, same length as
sourceCat.
"""
self._getSchemaKeys(sourceCat.schema)
bad = reduce(lambda x, y: np.logical_or(x, sourceCat.get(y)), self.config.badFlags, False)
good = self._isGood(sourceCat)
return Struct(selected=good & ~bad)
def run(self, input: Union[TaskMetadata,
Dict[str, int]]) -> Struct: # type: ignore
"""Run the task, adding the configured key-value pair to the input
argument and returning it as the output.
Parameters
----------
input : `dict`
Dictionary to update and return.
Returns
-------
result : `lsst.pipe.base.Struct`
Struct with a single ``output`` attribute.
"""
self.log.info("Run method given data of type: %s",
get_full_type_name(input))
output = input.copy()
output[self.config.key] = self.config.value
# Can change the return type via configuration.
if "TaskMetadata" in self.config.outputSC:
output = TaskMetadata.from_dict(output) # type: ignore
elif type(output) == TaskMetadata:
# Want the output to be a dict
output = output.to_dict()
self.log.info("Run method returns data of type: %s",
get_full_type_name(output))
return Struct(output=output)
def validateIsrResults(self):
"""results should be a struct with components that are
not None if included in the configuration file.
Returns
-------
results : `pipeBase.Struct`
Results struct generated from the current ISR configuration.
"""
self.task = IsrTask(config=self.config)
results = self.task.run(
self.inputExp,
camera=self.camera,
bias=self.dataRef.get("bias"),
dark=self.dataRef.get("dark"),
flat=self.dataRef.get("flat"),
bfKernel=self.dataRef.get("bfKernel"),
defects=self.dataRef.get("defects"),
fringes=Struct(fringes=self.dataRef.get("fringe"), seed=1234),
opticsTransmission=self.dataRef.get("transmission_"),
filterTransmission=self.dataRef.get("transmission_"),
sensorTransmission=self.dataRef.get("transmission_"),
atmosphereTransmission=self.dataRef.get("transmission_"))
self.assertIsInstance(results, Struct)
self.assertIsInstance(results.exposure, afwImage.Exposure)
return results
def construct(self, name, fraction):
"""Construct the test environment
This isn't called 'setUp' because we want to vary the `fraction`.
Parameters
----------
name : `str`
Name of column for flagging reserved sources (without "_reserved").
fraction : `float`
Fraction of sources to reserve.
Return struct elements
----------------------
catalog : `lsst.afw.table.SourceCatalog`
Catalog of sources.
task : `lsst.meas.algorithms.ReserveSourcesTask`
Task to do the reservations.
key : `lsst.afw.table.Key`
Key to the flag column.
"""
schema = lsst.afw.table.SourceTable.makeMinimalSchema()
config = lsst.meas.algorithms.ReserveSourcesConfig()
config.fraction = fraction
task = lsst.meas.algorithms.ReserveSourcesTask(
columnName=name,
schema=schema,
doc="Documentation is good",
config=config)
key = schema[name + "_reserved"].asKey()
catalog = lsst.afw.table.SourceCatalog(schema)
catalog.reserve(self.num)
for _ in range(self.num):
catalog.addNew()
return Struct(catalog=catalog, task=task, key=key)
def run(self, diaSourceCat, diffIm, band, ccdVisitId, funcs=None):
"""Produce transformation outputs with no processing.
Parameters
----------
diaSourceCat : `lsst.afw.table.SourceCatalog`
The catalog to transform.
diffIm : `lsst.afw.image.Exposure`
An image, to provide supplementary information.
band : `str`
The band in which the sources were observed.
ccdVisitId : `int`
The exposure ID in which the sources were observed.
funcs
Unused.
Returns
-------
results : `lsst.pipe.base.Struct`
Results struct with components:
``diaSourceTable``
Catalog of DiaSources (`pandas.DataFrame`).
"""
return Struct(diaSourceTable=pandas.DataFrame(), )
def run(self,
science,
matchedTemplate,
difference,
selectSources,
idFactory=None):
"""Detect and measure sources on a difference image.
Parameters
----------
science : `lsst.afw.image.ExposureF`
Science exposure that the template was subtracted from.
matchedTemplate : `lsst.afw.image.ExposureF`
Warped and PSF-matched template that was used produce the
difference image.
difference : `lsst.afw.image.ExposureF`
Result of subtracting template from the science image.
selectSources : `lsst.afw.table.SourceCatalog`
Identified sources on the science exposure.
idFactory : `lsst.afw.table.IdFactory`, optional
Generator object to assign ids to detected sources in the difference image.
Returns
-------
results : `lsst.pipe.base.Struct`
``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
Subtracted exposure with detection mask applied.
``diaSources`` : `lsst.afw.table.SourceCatalog`
The catalog of detected sources.
"""
return Struct(
subtractedMeasuredExposure=difference,
diaSources=afwTable.SourceCatalog(),
)
def run(self, coaddExposures, bbox, wcs, dataIds, **kwargs):
"""Warp coadds from multiple tracts to form a template for image diff.
Where the tracts overlap, the resulting template image is averaged.
The PSF on the template is created by combining the CoaddPsf on each
template image into a meta-CoaddPsf.
Parameters
----------
coaddExposures : `list` of `lsst.afw.image.Exposure`
Coadds to be mosaicked
bbox : `lsst.geom.Box2I`
Template Bounding box of the detector geometry onto which to
resample the coaddExposures
wcs : `lsst.afw.geom.SkyWcs`
Template WCS onto which to resample the coaddExposures
dataIds : `list` of `lsst.daf.butler.DataCoordinate`
Record of the tract and patch of each coaddExposure.
**kwargs
Any additional keyword parameters.
Returns
-------
result : `lsst.pipe.base.Struct` containing
- ``template`` : a template coadd exposure assembled out of patches
"""
return Struct(template=afwImage.ExposureF(), )
def selectSources(self, sourceCat, matches=None):
"""
!Return a catalog of sources: a subset of sourceCat.
If sourceCat is cotiguous in memory, will use vectorized tests for ~100x
execution speed advantage over non-contiguous catalogs. This would be
even faster if we didn't have to check footprints for multiple peaks.
@param[in] sourceCat catalog of sources that may be sources
(an lsst.afw.table.SourceCatalog)
@return a pipeBase.Struct containing:
- sourceCat a catalog of sources
"""
self._getSchemaKeys(sourceCat.schema)
if sourceCat.isContiguous():
good = self._isUsable_vector(sourceCat)
result = sourceCat[good]
else:
result = table.SourceCatalog(sourceCat.table)
for i, source in enumerate(sourceCat):
if self._isUsable(source):
result.append(source)
return Struct(sourceCat=result)
def readFringes(self, dataRef, assembler=None):
"""Read the fringe frame(s)
The current implementation assumes only a single fringe frame and
will have to be updated to support multi-mode fringe subtraction.
This implementation could be optimised by persisting the fringe
positions and fluxes.
@param dataRef Data reference for the science exposure
@param assembler An instance of AssembleCcdTask (for assembling fringe frames)
@return Struct(fringes: fringe exposure or list of fringe exposures;
seed: 32-bit uint derived from ccdExposureId for random number generator
"""
try:
fringe = dataRef.get("fringe", immediate=True)
except Exception as e:
raise RuntimeError("Unable to retrieve fringe for %s: %s" % (dataRef.dataId, e))
if assembler is not None:
fringe = assembler.assembleCcd(fringe)
seed = self.config.stats.rngSeedOffset + dataRef.get("ccdExposureId", immediate=True)
# Seed for numpy.random.RandomState must be convertable to a 32 bit unsigned integer
seed %= 2**32
return Struct(fringes=fringe,
seed=seed)
def runDataRefs(self, datarefs, customMetadata=None):
"""Call all registered metric tasks on each dataref.
This method loads all datasets required to compute a particular
metric, and persists the metrics as one or more `lsst.verify.Job`
objects. Only metrics that successfully produce a
`~lsst.verify.Measurement` will be included in a job.
Parameters
----------
datarefs : `list` of `lsst.daf.persistence.ButlerDataRef`
The data to measure. Datarefs may be complete or partial; each
generates a measurement at the same granularity (e.g., a
dataref with only ``"visit"`` specified generates visit-level
measurements).
customMetadata : `dict`, optional
Any metadata that are needed for a specific pipeline, but that are
not needed by the ``lsst.verify`` framework or by general-purpose
measurement analysis code (these cases are handled by the
`~MetricsControllerConfig.metadataAdder` subtask). If omitted,
only generic metadata are added. Both keys and values must be valid
inputs to `~lsst.verify.Metadata`.
Returns
-------
struct : `lsst.pipe.base.Struct`
A `~lsst.pipe.base.Struct` containing the following component:
- ``jobs`` : a list of collections of measurements (`list` of
`lsst.verify.Job`). Each job in the list contains the
measurement(s) for the corresponding dataref, and each job has
at most one measurement for each element in `self.measurers`. A
particular measurement is omitted if it could not be created.
Notes
-----
Some objects may be persisted, or incorrectly persisted, in the event
of an exception.
"""
jobs = []
index = 0
for dataref in datarefs:
job = Job.load_metrics_package()
try:
self.metadataAdder.run(job, dataref=dataref)
if customMetadata:
job.meta.update(customMetadata)
for task in self.measurers:
self._computeSingleMeasurement(job, task, dataref)
finally:
jobFile = self._getJobFilePath(index, dataref.dataId)
self.log.info("Persisting metrics to %s...", jobFile)
# This call order maximizes the chance that job gets
# written, and to a unique file
index += 1
job.write(jobFile)
jobs.append(job)
return Struct(jobs=jobs)
def run(self, sciSources, diaSources):
"""Compute the ratio of DIASources to science sources.
Parameters
----------
sciSources : `lsst.afw.table.SourceCatalog` or `None`
A science source catalog, which may be empty or `None`.
diaSources : `lsst.afw.table.SourceCatalog` or `None`
A DIASource catalog for the same unit of processing
as ``sciSources``.
Returns
-------
result : `lsst.pipe.base.Struct`
A `~lsst.pipe.base.Struct` containing the following component:
``measurement``
the ratio (`lsst.verify.Measurement` or `None`)
"""
if diaSources is not None and sciSources is not None:
nSciSources = len(sciSources)
nDiaSources = len(diaSources)
metricName = self.config.metricName
if nSciSources <= 0.0:
raise MetricComputationError(
"No science sources found; ratio of DIASources to science sources ill-defined."
)
else:
meas = Measurement(
metricName,
nDiaSources / nSciSources * u.dimensionless_unscaled)
else:
self.log.info("Nothing to do: no catalogs found.")
meas = None
return Struct(measurement=meas)
def run(self, matchedFakes, band):
"""Compute the completeness of recovered fakes within a magnitude
range.
Parameters
----------
matchedFakes : `lsst.afw.table.SourceCatalog` or `None`
Catalog of fakes that were inserted into the ccdExposure matched
to their detected counterparts.
Returns
-------
result : `lsst.pipe.base.Struct`
A `~lsst.pipe.base.Struct` containing the following component:
``measurement``
the ratio (`lsst.verify.Measurement` or `None`)
"""
if matchedFakes is not None:
magnitudes = matchedFakes[f"{self.config.magVar}" % band]
magCutFakes = matchedFakes[np.logical_and(magnitudes > self.config.magMin,
magnitudes < self.config.magMax)]
if len(magCutFakes) <= 0.0:
raise MetricComputationError(
"No matched fakes catalog sources found; Completeness is "
"ill defined.")
else:
meas = Measurement(
self.config.metricName,
((magCutFakes["diaSourceId"] > 0).sum() / len(magCutFakes))
* u.dimensionless_unscaled)
else:
self.log.info("Nothing to do: no matched catalog found.")
meas = None
return Struct(measurement=meas)
def getFluxFitParams(dataRef):
"""Retrieve the flux correction parameters determined by meas_mosaic
If the flux correction parameters do not exist, an exception will
be raised.
"""
calexp_md = dataRef.get("calexp_md", immediate=True)
hscRun = mosaicUtils.checkHscStack(calexp_md)
if hscRun is not None:
ffpHeader = dataRef.get("fcr_hsc_md", immediate=True)
else:
ffpHeader = dataRef.get("fcr_md", immediate=True)
photoCalib = dataRef.get("fcr_photoCalib")
ffp = FluxFitParams(ffpHeader)
wcs = getWcs(dataRef)
if hscRun is None:
detector = dataRef.get("camera")[dataRef.dataId["ccd"]]
nQuarter = detector.getOrientation().getNQuarter()
if nQuarter % 4 != 0:
# Have to put this import here due to circular dependence in forcedPhotCcd.py in meas_base
import lsst.meas.astrom as measAstrom
dimensions = dataRef.get("calexp_bbox").getDimensions()
wcs = measAstrom.rotateWcsPixelsBy90(wcs, nQuarter, dimensions)
return Struct(ffp=ffp, calib=photoCalib, wcs=wcs)
def testTaskAPI(self):
"""Test that the Tasks work
Checks both MeasureCrosstalkTask and the CrosstalkTask.
"""
# make exposure available to NullIsrTask
# without NullIsrTask's `self` hiding this test class's `self`
exposure = self.exposure
class NullIsrTask(IsrTask):
def runDataRef(self, dataRef):
return Struct(exposure=exposure)
config = MeasureCrosstalkTask.ConfigClass()
config.isr.retarget(NullIsrTask)
config.threshold = self.value - 1
measure = MeasureCrosstalkTask(config=config)
fakeDataRef = Struct(dataId={'fake': 1})
coeff, coeffErr, coeffNum = measure.reduce(
[measure.runDataRef(fakeDataRef)])
self.checkCoefficients(coeff, coeffErr, coeffNum)
config = IsrTask.ConfigClass()
config.crosstalk.minPixelToMask = self.value - 1
config.crosstalk.crosstalkMaskPlane = self.crosstalkStr
isr = IsrTask(config=config)
isr.crosstalk.run(self.exposure)
self.checkSubtracted(self.exposure)
def run(self, exposure):
for maskPlane in self.config.maskPlanesToInterpolate:
interpolateFromMask(exposure.maskedImage,
fwhm=self.config.fwhm,
maskName=maskPlane)
result = Struct(exposure=exposure)
return result
def run(self, coadd, inputCatalog):
"""Operate on in-memory data.
With default implementation of `runQuantum()` keyword arguments
correspond to field names in a config.
Parameters
----------
coadd : object
Input data object (input dataset type is configured as scalar)
inputCatalog : object
Input data object (input dataset type is configured as scalar)
Returns
-------
`Struct` instance with produced result.
"""
_LOG.info("executing %s: coadd=%s inputCatalog=%s", self.getName(),
coadd, type(inputCatalog))
# output data, scalar in this case, just return input catalog without change
data = inputCatalog
# attribute name of struct is the same as a config field name
return Struct(outputCatalog=data)
def runQuantum(self, butlerQC, inputRefs, outputRefs):
inputs = butlerQC.get(inputRefs)
exposureIdInfo = ExposureIdInfo.fromDataId(butlerQC.quantum.dataId,
"tract_patch")
inputs["skySeed"] = exposureIdInfo.expId
inputs["idFactory"] = exposureIdInfo.makeSourceIdFactory()
catalogDict = {
ref.dataId['band']: cat
for ref, cat in zip(inputRefs.catalogs, inputs['catalogs'])
}
inputs['catalogs'] = catalogDict
skyMap = inputs.pop('skyMap')
# Can use the first dataId to find the tract and patch being worked on
tractNumber = inputRefs.catalogs[0].dataId['tract']
tractInfo = skyMap[tractNumber]
patchInfo = tractInfo.getPatchInfo(
inputRefs.catalogs[0].dataId['patch'])
skyInfo = Struct(skyMap=skyMap,
tractInfo=tractInfo,
patchInfo=patchInfo,
wcs=tractInfo.getWcs(),
bbox=patchInfo.getOuterBBox())
inputs['skyInfo'] = skyInfo
outputs = self.run(**inputs)
butlerQC.put(outputs, outputRefs)
def __call__(self, image, detector, log=None):
"""Correct for non-linearity
@param[in] image image to be corrected (an lsst.afw.image.Image)
@param[in] detector detector info about image (an lsst.afw.cameraGeom.Detector)
@param[in] log logger (an lsst.log.Log), or None to disable logging;
a warning is logged if any amplifiers are skipped because the square coefficient is 0
@return an lsst.pipe.base.Struct containing at least the following fields:
- nAmps number of amplifiers found
- nLinearized number of amplifiers linearized
@throw RuntimeError if the linearity type is wrong
"""
self.checkLinearityType(detector)
ampInfoCat = detector.getAmplifiers()
numLinearized = 0
for ampInfo in ampInfoCat:
sqCoeff = ampInfo.getLinearityCoeffs()[0]
if sqCoeff != 0:
bbox = ampInfo.getBBox()
ampArr = image.Factory(image, bbox).getArray()
ampArr *= (1 + sqCoeff*ampArr)
numLinearized += 1
numAmps = len(ampInfoCat)
if numAmps > numLinearized and log is not None:
log.warn("%s of %s amps in detector \"%s\" were not linearized (coefficient = 0)",
numAmps - numLinearized, numAmps, detector.getName())
return Struct(
numAmps=numAmps,
numLinearized=numLinearized,
)
def testInvalidMetricSegregation(self, _mockWriter, _mockButler,
_mockMetricsLoader):
self.config.measurers = ["demoMetric"]
self.task = MetricsControllerTask(self.config)
with unittest.mock.patch.object(_DemoMetricTask,
"adaptArgsAndRun") as mockCall:
# Run _DemoMetricTask twice, with one failure and one result
mockCall.side_effect = (MetricComputationError,
unittest.mock.DEFAULT)
expectedValue = 1.0 * u.second
mockCall.return_value = Struct(measurement=lsst.verify.Measurement(
_metricName(), expectedValue))
dataIds = [{"visit": 42, "ccd": 101, "filter": "k"},
{"visit": 42, "ccd": 102, "filter": "k"}]
datarefs = [_makeMockDataref(dataId) for dataId in dataIds]
jobs = self.task.runDataRefs(datarefs).jobs
self.assertEqual(len(jobs), len(datarefs))
# Failed job
self.assertEqual(len(jobs[0].measurements), 0)
# Successful job
self.assertTrue(jobs[1].meta["tested"])
self.assertEqual(len(jobs[1].measurements), 1)
assert_quantity_allclose(
jobs[1].measurements[_metricName()].quantity,
expectedValue)
def run(self, patchRefList, butler, selectDataList=[]):
"""!Run stacking on a tract
This method only runs on the master node.
@param patchRefList: List of patch data references for tract
@param butler: Data butler
@param selectDataList: List of SelectStruct for inputs
"""
pool = Pool("stacker")
pool.cacheClear()
pool.storeSet(butler=butler,
warpType=self.config.coaddName + "Coadd_directWarp",
coaddType=self.config.coaddName + "Coadd")
patchIdList = [patchRef.dataId for patchRef in patchRefList]
selectedData = pool.map(self.warp, patchIdList, selectDataList)
if self.config.doBackgroundReference:
self.backgroundReference.runDataRef(patchRefList, selectDataList)
def refNamer(patchRef):
return tuple(map(int, patchRef.dataId["patch"].split(",")))
lookup = dict(zip(map(refNamer, patchRefList), selectedData))
coaddData = [
Struct(patchId=patchRef.dataId,
selectDataList=lookup[refNamer(patchRef)])
for patchRef in patchRefList
]
pool.map(self.coadd, coaddData)
def groupPatchExposures(patchDataRef,
calexpDataRefList,
coaddDatasetType="deepCoadd",
tempExpDatasetType="deepCoadd_tempExp"):
"""Group calibrated exposures overlapping a patch by the warped
(temporary) exposure they contribute to.
For example, if the instrument has a mosaic camera, each group would
consist of the subset of CCD exposures from a single camera exposure
that potentially overlap the patch.
@return Struct with:
- groups: Dict of <group tuple>: <list of data references for group>
- keys: List of keys for group tuple
"""
butler = patchDataRef.getButler()
tempExpKeys = butler.getKeys(datasetType=tempExpDatasetType)
coaddKeys = sorted(butler.getKeys(datasetType=coaddDatasetType))
keys = sorted(set(tempExpKeys) -
set(coaddKeys)) # Keys that will specify an exposure
patchId = patchDataRef.dataId
groups = groupDataRefs(keys, calexpDataRefList)
# Supplement the groups with the coadd-specific information (e.g., tract, patch; these are constant)
coaddValues = tuple(patchId[k] for k in coaddKeys)
groups = dict((k + coaddValues, v) for k, v in groups.iteritems())
keys += tuple(coaddKeys)
return Struct(groups=groups, keys=keys)
def run(self, sensorRefList):
"""Process all arms of the same kind within an exposure
The sequence of operations is:
- remove instrument signature
- measure PSF
- subtract sky from the image
- extract the spectra from the fiber traces
- write the outputs
Parameters
----------
sensorRef : `lsst.daf.persistence.ButlerDataRef`
Data reference for sensors to process.
Returns
-------
exposureList : `list` of `lsst.afw.image.Exposure`
Exposure data for sensors.
pfsList : `list` of PSFs
Point-spread functions; if ``doMeasurePsf`` is set.
lsfList : `list` of LSFs
Line-spread functions; if ``doMeasurePsf`` is set.
sky2d : `pfs.drp.stella.FocalPlaneFunction`
2D sky subtraction solution.
spectraList : `list` of `pfs.drp.stella.SpectrumSet`
Sets of extracted spectra.
originalList : `list` of `pfs.drp.stella.SpectrumSet`
Sets of extracted spectra before continuum subtraction.
Will be identical to ``spectra`` if continuum subtraction
was not performed.
fiberTraceList : `list` of `pfs.drp.stella.FiberTraceSet`
Fiber traces.
detectorMapList : `list` of `pfs.drp.stella.DetectorMap`
Mappings of wl,fiber to detector position.
"""
self.log.info("Processing %s" % ([sensorRef.dataId for sensorRef in sensorRefList]))
exposureList = []
psfList = []
lsfList = []
for sensorRef in sensorRefList:
exposure = self.isr.runDataRef(sensorRef).exposure
if self.config.doRepair:
self.repairExposure(exposure)
if self.config.doSkySwindle:
self.skySwindle(sensorRef, exposure.image)
exposureList.append(exposure)
if self.config.doMeasurePsf:
psfList = self.measurePsf.run(sensorRefList, exposureList)
lsfList = [self.calculateLsf(psf) for psf in psfList]
else:
psfList = [None]*len(sensorRefList)
lsfList = [None]*len(sensorRefList)
results = Struct(exposureList=exposureList, psfList=psfList, lsfList=lsfList)
fiberTraceList = [sensorRef.get("fibertrace") for sensorRef in sensorRefList]
detectorMapList = [sensorRef.get("detectormap") for sensorRef in sensorRefList]
pfsConfig = sensorRefList[0].get("pfsConfig")
results.fiberTraceList = fiberTraceList
results.detectorMapList = detectorMapList
results.pfsConfig = pfsConfig
if self.config.doSubtractSky2d:
results.sky2d = self.subtractSky2d.run(exposureList, pfsConfig, psfList,
fiberTraceList, detectorMapList)
if self.config.doExtractSpectra:
originalList = []
spectraList = []
for exposure, fiberTraces, detectorMap in zip(exposureList, fiberTraceList, detectorMapList):
spectra = self.extractSpectra.run(exposure.maskedImage, fiberTraces, detectorMap).spectra
originalList.append(spectra)
if self.config.doSubtractContinuum:
continua = self.fitContinuum.run(spectra)
exposure.maskedImage -= continua.makeImage(exposure.getBBox(), fiberTraces)
spectra = self.extractSpectra.run(exposure.maskedImage, fiberTraces,
detectorMap).spectra
spectraList.append(spectra)
results.originalList = originalList
results.spectraList = spectraList
self.write(sensorRefList, results)
return results
