def run(self, catalog, filterName=None): """!Load reference objects and match to them @param[in] catalog Catalog to match to (lsst.afw.table.SourceCatalog) @param[in] filterName Name of filter, for loading fluxes (str) @return Struct with matches (lsst.afw.table.SourceMatchVector) and matchMeta (lsst.meas.astrom.MatchMetadata) """ circle = self.calculateCircle(catalog) matchMeta = self.refObjLoader.getMetadataCircle(circle.center, circle.radius, filterName) emptyResult = Struct(matches=[], matchMeta=matchMeta) sourceSelection = self.sourceSelection.run(catalog) if len(sourceSelection.sourceCat) == 0: self.log.warn("No objects selected from %d objects in source catalog", len(catalog)) return emptyResult refData = self.refObjLoader.loadSkyCircle(circle.center, circle.radius, filterName) refCat = refData.refCat refSelection = self.referenceSelection.run(refCat) if len(refSelection.sourceCat) == 0: self.log.warn("No objects selected from %d objects in reference catalog", len(refCat)) return emptyResult matches = afwTable.matchRaDec(refSelection.sourceCat, sourceSelection.sourceCat, self.config.matchRadius*arcseconds) self.log.info("Matched %d from %d/%d input and %d/%d reference sources" % (len(matches), len(sourceSelection.sourceCat), len(catalog), len(refSelection.sourceCat), len(refCat))) return Struct(matches=matches, matchMeta=matchMeta, refCat=refCat, sourceSelection=sourceSelection, refSelection=refSelection)
def run(self, coadds, filters, mergedDetections, idFactory): sources = self._makeSourceCatalog(mergedDetections, idFactory) multiExposure = afwImage.MultibandExposure.fromExposures(filters, coadds) fluxCatalogs, templateCatalogs = self.multibandDeblend.run(multiExposure, sources) retStruct = Struct(templateCatalogs) if self.config.multibandDeblend.conserveFlux: retStruct.fluxCatalogs = fluxCatalogs return retStruct
def run(self, matchedCatalog, metric_name): self.log.info("Measuring PA1") pa1 = photRepeat(matchedCatalog, snrMax=self.brightSnrMax, snrMin=self.brightSnrMin, numRandomShuffles=self.numRandomShuffles, randomSeed=self.randomSeed) if 'magDiff' in pa1.keys(): return Struct(measurement=Measurement("PA1", pa1['repeatability'])) else: return Struct(measurement=Measurement("PA1", np.nan*u.mmag))
def makeDistortionData(self): """Make distortion data The data format is a dict of detector name: ccdData, where ccdData is a Struct containing these fields: - serial: detector.getSerial - cornerDict: a dict of pixPosKey, cornerData, where: - pixPosKey: self.asKey(pixPos) where pixPos is pixel position - cornerData is Struct contains these fields (all of type lsst.geom.Point2D): - pixPos: pixel position - focalPlane: focal plane position computed from pixPos - fieldAngle: fieldAngle position computed from focalPlane - focalPlaneRoundTrip: focal plane position computed from fieldAngle - pixPosRoundTrip: pixel position computed from focalPlane """ hsc = HyperSuprimeCam() camera = hsc.getCamera() focalPlaneToFieldAngle = camera.getTransformMap().getTransform( FOCAL_PLANE, FIELD_ANGLE) data = {} # dict of detector name: CcdData for detector in camera: # for each corner of each CCD: # - get pixel position # - convert to focal plane coordinates using the detector and # record it # - convert to field angle (this is the conversion that uses # HscDistortion) and record it # - convert back to focal plane (testing inverse direction of # HscDistortion) and record it # - convert back to pixel position and record it; pixel <-> focal # plane is affine so there is no reason to doubt the inverse # transform, but there is no harm pixelsToFocalPlane = detector.getTransform(PIXELS, FOCAL_PLANE) cornerDict = {} for pixPos in detector.getCorners(PIXELS): pixPos = pixPos focalPlane = pixelsToFocalPlane.applyForward(pixPos) fieldAngle = focalPlaneToFieldAngle.applyForward(focalPlane) focalPlaneRoundTrip = focalPlaneToFieldAngle.applyInverse( fieldAngle) pixPosRoundTrip = pixelsToFocalPlane.applyInverse(focalPlane) cornerDict[self.toKey(pixPos)] = Struct( pixPos=pixPos, focalPlane=focalPlane, fieldAngle=fieldAngle, focalPlaneRoundTrip=focalPlaneRoundTrip, pixPosRoundTrip=pixPosRoundTrip, ) data[detector.getName()] = Struct(serial=detector.getSerial(), cornerDict=cornerDict) return data
def run(self, matchedCatalog, metric_name): self.log.info(f"Measuring {metric_name}") D = self.config.annulus_r * u.arcmin filteredCat = filterMatches(matchedCatalog) nMinTEx = 50 if filteredCat.count <= nMinTEx: return Struct(measurement=Measurement(metric_name, np.nan*u.Unit(''))) radius, xip, xip_err = correlation_function_ellipticity_from_matches(filteredCat) operator = ThresholdSpecification.convert_operator_str(self.config.comparison_operator) corr, corr_err = select_bin_from_corr(radius, xip, xip_err, radius=D, operator=operator) return Struct(measurement=Measurement(metric_name, np.abs(corr)*u.Unit('')))
def __init__(self, name, schema, seeing, config, metadata): deconvKey = schema.addField(name + "_deconv", type="Flag", doc="deconvolution required for seeing %f; no measurement made" % (seeing,)) aperture = lsst.meas.base.CircularApertureFluxAlgorithm(config.aperture.makeControl(), name, schema, metadata) kronKeys = Struct( result=lsst.meas.base.FluxResultKey.addFields(schema, name + "_kron", doc="convolved Kron flux: seeing %f" % (seeing,)), flag=schema.addField(name + "_kron_flag", type="Flag", doc="convolved Kron flux failed: seeing %f" % (seeing,)), ) Struct.__init__(self, deconvKey=deconvKey, aperture=aperture, kronKeys=kronKeys)
def runQuantum(self, butlerQC, inputRefs, outputRefs): inputs = butlerQC.get(inputRefs) packedId, maxBits = butlerQC.quantum.dataId.pack("tract_patch", returnMaxBits=True) inputs["skySeed"] = packedId inputs["idFactory"] = afwTable.IdFactory.makeSource( packedId, 64 - maxBits) catalogDict = { ref.dataId['abstract_filter']: cat for ref, cat in zip(inputRefs.catalogs, inputs['catalogs']) } inputs['catalogs'] = catalogDict skyMap = inputs.pop('skyMap') # Can use the first dataId to find the tract and patch being worked on tractNumber = inputRefs.catalogs[0].dataId['tract'] tractInfo = skyMap[tractNumber] patchInfo = tractInfo.getPatchInfo( inputRefs.catalogs[0].dataId['patch']) skyInfo = Struct(skyMap=skyMap, tractInfo=tractInfo, patchInfo=patchInfo, wcs=tractInfo.getWcs(), bbox=patchInfo.getOuterBBox()) inputs['skyInfo'] = skyInfo outputs = self.run(**inputs) butlerQC.put(outputs, outputRefs)
def catalogGenerator(self, cache, dataRefList, selectDataList=[]): """! Get a generator of difference images from the visit catalof for each datRef """ for dataRef in dataRefList: try: calexp = dataRef.get(f"{self.config.coaddName}Coadd_calexp") except Exception: self.log.info('Cannot coadd read data for %s' % (dataRef.dataId)) continue visitCatalog = calexp.getInfo().getCoaddInputs().ccds for visitRec in visitCatalog: visit = visitRec.get('visit') ccd = visitRec.get('ccd') dataId = {"visit": visit, self.config.ccdKey: ccd} try: exp = cache.butler.get( f"{self.config.coaddName}Diff_differenceExp", dataId) src = cache.butler.get( f"{self.config.coaddName}Diff_diaSrc", dataId) except Exception as e: self.log.debug( 'Cannot read difference data for %d %d. skipping %s' % (visit, ccd, e)) continue data = Struct(visit=visit, ccd=ccd, exp=exp, src=src, filter=dataRef.dataId['filter'], calib=exp.getPhotoCalib()) yield data
def run(self, args): """Run ingest We read and ingest the files in parallel, and then stuff the registry database in serial. """ # Parallel pool = Pool(None) filenameList = self.expandFiles(args.files) dataList = [ Struct(filename=filename, position=ii) for ii, filename in enumerate(filenameList) ] infoList = pool.map(self.runFileWrapper, dataList, args) # Serial root = args.input context = self.register.openRegistry(root, create=args.create, dryrun=args.dryrun) with context as registry: for hduInfoList in infoList: if hduInfoList is None: continue for info in hduInfoList: self.register.addRow(registry, info, dryrun=args.dryrun, create=args.create)
def selectSources(self, sourceCat, matches=None, exposure=None): """Return a selection of sources that are useful for astrometry. Parameters: ----------- sourceCat : `lsst.afw.table.SourceCatalog` Catalog of sources to select from. This catalog must be contiguous in memory. matches : `list` of `lsst.afw.table.ReferenceMatch` or None Ignored in this SourceSelector. exposure : `lsst.afw.image.Exposure` or None The exposure the catalog was built from; used for debug display. Return ------ struct : `lsst.pipe.base.Struct` The struct contains the following data: - selected : `array` of `bool`` Boolean array of sources that were selected, same length as sourceCat. """ self._getSchemaKeys(sourceCat.schema) bad = reduce(lambda x, y: np.logical_or(x, sourceCat.get(y)), self.config.badFlags, False) good = self._isGood(sourceCat) return Struct(selected=good & ~bad)
def run(self, input: Union[TaskMetadata, Dict[str, int]]) -> Struct: # type: ignore """Run the task, adding the configured key-value pair to the input argument and returning it as the output. Parameters ---------- input : `dict` Dictionary to update and return. Returns ------- result : `lsst.pipe.base.Struct` Struct with a single ``output`` attribute. """ self.log.info("Run method given data of type: %s", get_full_type_name(input)) output = input.copy() output[self.config.key] = self.config.value # Can change the return type via configuration. if "TaskMetadata" in self.config.outputSC: output = TaskMetadata.from_dict(output) # type: ignore elif type(output) == TaskMetadata: # Want the output to be a dict output = output.to_dict() self.log.info("Run method returns data of type: %s", get_full_type_name(output)) return Struct(output=output)
def validateIsrResults(self): """results should be a struct with components that are not None if included in the configuration file. Returns ------- results : `pipeBase.Struct` Results struct generated from the current ISR configuration. """ self.task = IsrTask(config=self.config) results = self.task.run( self.inputExp, camera=self.camera, bias=self.dataRef.get("bias"), dark=self.dataRef.get("dark"), flat=self.dataRef.get("flat"), bfKernel=self.dataRef.get("bfKernel"), defects=self.dataRef.get("defects"), fringes=Struct(fringes=self.dataRef.get("fringe"), seed=1234), opticsTransmission=self.dataRef.get("transmission_"), filterTransmission=self.dataRef.get("transmission_"), sensorTransmission=self.dataRef.get("transmission_"), atmosphereTransmission=self.dataRef.get("transmission_")) self.assertIsInstance(results, Struct) self.assertIsInstance(results.exposure, afwImage.Exposure) return results
def construct(self, name, fraction): """Construct the test environment This isn't called 'setUp' because we want to vary the `fraction`. Parameters ---------- name : `str` Name of column for flagging reserved sources (without "_reserved"). fraction : `float` Fraction of sources to reserve. Return struct elements ---------------------- catalog : `lsst.afw.table.SourceCatalog` Catalog of sources. task : `lsst.meas.algorithms.ReserveSourcesTask` Task to do the reservations. key : `lsst.afw.table.Key` Key to the flag column. """ schema = lsst.afw.table.SourceTable.makeMinimalSchema() config = lsst.meas.algorithms.ReserveSourcesConfig() config.fraction = fraction task = lsst.meas.algorithms.ReserveSourcesTask( columnName=name, schema=schema, doc="Documentation is good", config=config) key = schema[name + "_reserved"].asKey() catalog = lsst.afw.table.SourceCatalog(schema) catalog.reserve(self.num) for _ in range(self.num): catalog.addNew() return Struct(catalog=catalog, task=task, key=key)
def run(self, diaSourceCat, diffIm, band, ccdVisitId, funcs=None): """Produce transformation outputs with no processing. Parameters ---------- diaSourceCat : `lsst.afw.table.SourceCatalog` The catalog to transform. diffIm : `lsst.afw.image.Exposure` An image, to provide supplementary information. band : `str` The band in which the sources were observed. ccdVisitId : `int` The exposure ID in which the sources were observed. funcs Unused. Returns ------- results : `lsst.pipe.base.Struct` Results struct with components: ``diaSourceTable`` Catalog of DiaSources (`pandas.DataFrame`). """ return Struct(diaSourceTable=pandas.DataFrame(), )
def run(self, science, matchedTemplate, difference, selectSources, idFactory=None): """Detect and measure sources on a difference image. Parameters ---------- science : `lsst.afw.image.ExposureF` Science exposure that the template was subtracted from. matchedTemplate : `lsst.afw.image.ExposureF` Warped and PSF-matched template that was used produce the difference image. difference : `lsst.afw.image.ExposureF` Result of subtracting template from the science image. selectSources : `lsst.afw.table.SourceCatalog` Identified sources on the science exposure. idFactory : `lsst.afw.table.IdFactory`, optional Generator object to assign ids to detected sources in the difference image. Returns ------- results : `lsst.pipe.base.Struct` ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF` Subtracted exposure with detection mask applied. ``diaSources`` : `lsst.afw.table.SourceCatalog` The catalog of detected sources. """ return Struct( subtractedMeasuredExposure=difference, diaSources=afwTable.SourceCatalog(), )
def run(self, coaddExposures, bbox, wcs, dataIds, **kwargs): """Warp coadds from multiple tracts to form a template for image diff. Where the tracts overlap, the resulting template image is averaged. The PSF on the template is created by combining the CoaddPsf on each template image into a meta-CoaddPsf. Parameters ---------- coaddExposures : `list` of `lsst.afw.image.Exposure` Coadds to be mosaicked bbox : `lsst.geom.Box2I` Template Bounding box of the detector geometry onto which to resample the coaddExposures wcs : `lsst.afw.geom.SkyWcs` Template WCS onto which to resample the coaddExposures dataIds : `list` of `lsst.daf.butler.DataCoordinate` Record of the tract and patch of each coaddExposure. **kwargs Any additional keyword parameters. Returns ------- result : `lsst.pipe.base.Struct` containing - ``template`` : a template coadd exposure assembled out of patches """ return Struct(template=afwImage.ExposureF(), )
def selectSources(self, sourceCat, matches=None): """ !Return a catalog of sources: a subset of sourceCat. If sourceCat is cotiguous in memory, will use vectorized tests for ~100x execution speed advantage over non-contiguous catalogs. This would be even faster if we didn't have to check footprints for multiple peaks. @param[in] sourceCat catalog of sources that may be sources (an lsst.afw.table.SourceCatalog) @return a pipeBase.Struct containing: - sourceCat a catalog of sources """ self._getSchemaKeys(sourceCat.schema) if sourceCat.isContiguous(): good = self._isUsable_vector(sourceCat) result = sourceCat[good] else: result = table.SourceCatalog(sourceCat.table) for i, source in enumerate(sourceCat): if self._isUsable(source): result.append(source) return Struct(sourceCat=result)
def readFringes(self, dataRef, assembler=None): """Read the fringe frame(s) The current implementation assumes only a single fringe frame and will have to be updated to support multi-mode fringe subtraction. This implementation could be optimised by persisting the fringe positions and fluxes. @param dataRef Data reference for the science exposure @param assembler An instance of AssembleCcdTask (for assembling fringe frames) @return Struct(fringes: fringe exposure or list of fringe exposures; seed: 32-bit uint derived from ccdExposureId for random number generator """ try: fringe = dataRef.get("fringe", immediate=True) except Exception as e: raise RuntimeError("Unable to retrieve fringe for %s: %s" % (dataRef.dataId, e)) if assembler is not None: fringe = assembler.assembleCcd(fringe) seed = self.config.stats.rngSeedOffset + dataRef.get("ccdExposureId", immediate=True) # Seed for numpy.random.RandomState must be convertable to a 32 bit unsigned integer seed %= 2**32 return Struct(fringes=fringe, seed=seed)
def runDataRefs(self, datarefs, customMetadata=None): """Call all registered metric tasks on each dataref. This method loads all datasets required to compute a particular metric, and persists the metrics as one or more `lsst.verify.Job` objects. Only metrics that successfully produce a `~lsst.verify.Measurement` will be included in a job. Parameters ---------- datarefs : `list` of `lsst.daf.persistence.ButlerDataRef` The data to measure. Datarefs may be complete or partial; each generates a measurement at the same granularity (e.g., a dataref with only ``"visit"`` specified generates visit-level measurements). customMetadata : `dict`, optional Any metadata that are needed for a specific pipeline, but that are not needed by the ``lsst.verify`` framework or by general-purpose measurement analysis code (these cases are handled by the `~MetricsControllerConfig.metadataAdder` subtask). If omitted, only generic metadata are added. Both keys and values must be valid inputs to `~lsst.verify.Metadata`. Returns ------- struct : `lsst.pipe.base.Struct` A `~lsst.pipe.base.Struct` containing the following component: - ``jobs`` : a list of collections of measurements (`list` of `lsst.verify.Job`). Each job in the list contains the measurement(s) for the corresponding dataref, and each job has at most one measurement for each element in `self.measurers`. A particular measurement is omitted if it could not be created. Notes ----- Some objects may be persisted, or incorrectly persisted, in the event of an exception. """ jobs = [] index = 0 for dataref in datarefs: job = Job.load_metrics_package() try: self.metadataAdder.run(job, dataref=dataref) if customMetadata: job.meta.update(customMetadata) for task in self.measurers: self._computeSingleMeasurement(job, task, dataref) finally: jobFile = self._getJobFilePath(index, dataref.dataId) self.log.info("Persisting metrics to %s...", jobFile) # This call order maximizes the chance that job gets # written, and to a unique file index += 1 job.write(jobFile) jobs.append(job) return Struct(jobs=jobs)
def run(self, sciSources, diaSources): """Compute the ratio of DIASources to science sources. Parameters ---------- sciSources : `lsst.afw.table.SourceCatalog` or `None` A science source catalog, which may be empty or `None`. diaSources : `lsst.afw.table.SourceCatalog` or `None` A DIASource catalog for the same unit of processing as ``sciSources``. Returns ------- result : `lsst.pipe.base.Struct` A `~lsst.pipe.base.Struct` containing the following component: ``measurement`` the ratio (`lsst.verify.Measurement` or `None`) """ if diaSources is not None and sciSources is not None: nSciSources = len(sciSources) nDiaSources = len(diaSources) metricName = self.config.metricName if nSciSources <= 0.0: raise MetricComputationError( "No science sources found; ratio of DIASources to science sources ill-defined." ) else: meas = Measurement( metricName, nDiaSources / nSciSources * u.dimensionless_unscaled) else: self.log.info("Nothing to do: no catalogs found.") meas = None return Struct(measurement=meas)
def run(self, matchedFakes, band): """Compute the completeness of recovered fakes within a magnitude range. Parameters ---------- matchedFakes : `lsst.afw.table.SourceCatalog` or `None` Catalog of fakes that were inserted into the ccdExposure matched to their detected counterparts. Returns ------- result : `lsst.pipe.base.Struct` A `~lsst.pipe.base.Struct` containing the following component: ``measurement`` the ratio (`lsst.verify.Measurement` or `None`) """ if matchedFakes is not None: magnitudes = matchedFakes[f"{self.config.magVar}" % band] magCutFakes = matchedFakes[np.logical_and(magnitudes > self.config.magMin, magnitudes < self.config.magMax)] if len(magCutFakes) <= 0.0: raise MetricComputationError( "No matched fakes catalog sources found; Completeness is " "ill defined.") else: meas = Measurement( self.config.metricName, ((magCutFakes["diaSourceId"] > 0).sum() / len(magCutFakes)) * u.dimensionless_unscaled) else: self.log.info("Nothing to do: no matched catalog found.") meas = None return Struct(measurement=meas)
def getFluxFitParams(dataRef): """Retrieve the flux correction parameters determined by meas_mosaic If the flux correction parameters do not exist, an exception will be raised. """ calexp_md = dataRef.get("calexp_md", immediate=True) hscRun = mosaicUtils.checkHscStack(calexp_md) if hscRun is not None: ffpHeader = dataRef.get("fcr_hsc_md", immediate=True) else: ffpHeader = dataRef.get("fcr_md", immediate=True) photoCalib = dataRef.get("fcr_photoCalib") ffp = FluxFitParams(ffpHeader) wcs = getWcs(dataRef) if hscRun is None: detector = dataRef.get("camera")[dataRef.dataId["ccd"]] nQuarter = detector.getOrientation().getNQuarter() if nQuarter % 4 != 0: # Have to put this import here due to circular dependence in forcedPhotCcd.py in meas_base import lsst.meas.astrom as measAstrom dimensions = dataRef.get("calexp_bbox").getDimensions() wcs = measAstrom.rotateWcsPixelsBy90(wcs, nQuarter, dimensions) return Struct(ffp=ffp, calib=photoCalib, wcs=wcs)
def testTaskAPI(self): """Test that the Tasks work Checks both MeasureCrosstalkTask and the CrosstalkTask. """ # make exposure available to NullIsrTask # without NullIsrTask's `self` hiding this test class's `self` exposure = self.exposure class NullIsrTask(IsrTask): def runDataRef(self, dataRef): return Struct(exposure=exposure) config = MeasureCrosstalkTask.ConfigClass() config.isr.retarget(NullIsrTask) config.threshold = self.value - 1 measure = MeasureCrosstalkTask(config=config) fakeDataRef = Struct(dataId={'fake': 1}) coeff, coeffErr, coeffNum = measure.reduce( [measure.runDataRef(fakeDataRef)]) self.checkCoefficients(coeff, coeffErr, coeffNum) config = IsrTask.ConfigClass() config.crosstalk.minPixelToMask = self.value - 1 config.crosstalk.crosstalkMaskPlane = self.crosstalkStr isr = IsrTask(config=config) isr.crosstalk.run(self.exposure) self.checkSubtracted(self.exposure)
def run(self, exposure): for maskPlane in self.config.maskPlanesToInterpolate: interpolateFromMask(exposure.maskedImage, fwhm=self.config.fwhm, maskName=maskPlane) result = Struct(exposure=exposure) return result
def run(self, coadd, inputCatalog): """Operate on in-memory data. With default implementation of `runQuantum()` keyword arguments correspond to field names in a config. Parameters ---------- coadd : object Input data object (input dataset type is configured as scalar) inputCatalog : object Input data object (input dataset type is configured as scalar) Returns ------- `Struct` instance with produced result. """ _LOG.info("executing %s: coadd=%s inputCatalog=%s", self.getName(), coadd, type(inputCatalog)) # output data, scalar in this case, just return input catalog without change data = inputCatalog # attribute name of struct is the same as a config field name return Struct(outputCatalog=data)
def runQuantum(self, butlerQC, inputRefs, outputRefs): inputs = butlerQC.get(inputRefs) exposureIdInfo = ExposureIdInfo.fromDataId(butlerQC.quantum.dataId, "tract_patch") inputs["skySeed"] = exposureIdInfo.expId inputs["idFactory"] = exposureIdInfo.makeSourceIdFactory() catalogDict = { ref.dataId['band']: cat for ref, cat in zip(inputRefs.catalogs, inputs['catalogs']) } inputs['catalogs'] = catalogDict skyMap = inputs.pop('skyMap') # Can use the first dataId to find the tract and patch being worked on tractNumber = inputRefs.catalogs[0].dataId['tract'] tractInfo = skyMap[tractNumber] patchInfo = tractInfo.getPatchInfo( inputRefs.catalogs[0].dataId['patch']) skyInfo = Struct(skyMap=skyMap, tractInfo=tractInfo, patchInfo=patchInfo, wcs=tractInfo.getWcs(), bbox=patchInfo.getOuterBBox()) inputs['skyInfo'] = skyInfo outputs = self.run(**inputs) butlerQC.put(outputs, outputRefs)
def __call__(self, image, detector, log=None): """Correct for non-linearity @param[in] image image to be corrected (an lsst.afw.image.Image) @param[in] detector detector info about image (an lsst.afw.cameraGeom.Detector) @param[in] log logger (an lsst.log.Log), or None to disable logging; a warning is logged if any amplifiers are skipped because the square coefficient is 0 @return an lsst.pipe.base.Struct containing at least the following fields: - nAmps number of amplifiers found - nLinearized number of amplifiers linearized @throw RuntimeError if the linearity type is wrong """ self.checkLinearityType(detector) ampInfoCat = detector.getAmplifiers() numLinearized = 0 for ampInfo in ampInfoCat: sqCoeff = ampInfo.getLinearityCoeffs()[0] if sqCoeff != 0: bbox = ampInfo.getBBox() ampArr = image.Factory(image, bbox).getArray() ampArr *= (1 + sqCoeff*ampArr) numLinearized += 1 numAmps = len(ampInfoCat) if numAmps > numLinearized and log is not None: log.warn("%s of %s amps in detector \"%s\" were not linearized (coefficient = 0)", numAmps - numLinearized, numAmps, detector.getName()) return Struct( numAmps=numAmps, numLinearized=numLinearized, )
def testInvalidMetricSegregation(self, _mockWriter, _mockButler, _mockMetricsLoader): self.config.measurers = ["demoMetric"] self.task = MetricsControllerTask(self.config) with unittest.mock.patch.object(_DemoMetricTask, "adaptArgsAndRun") as mockCall: # Run _DemoMetricTask twice, with one failure and one result mockCall.side_effect = (MetricComputationError, unittest.mock.DEFAULT) expectedValue = 1.0 * u.second mockCall.return_value = Struct(measurement=lsst.verify.Measurement( _metricName(), expectedValue)) dataIds = [{"visit": 42, "ccd": 101, "filter": "k"}, {"visit": 42, "ccd": 102, "filter": "k"}] datarefs = [_makeMockDataref(dataId) for dataId in dataIds] jobs = self.task.runDataRefs(datarefs).jobs self.assertEqual(len(jobs), len(datarefs)) # Failed job self.assertEqual(len(jobs[0].measurements), 0) # Successful job self.assertTrue(jobs[1].meta["tested"]) self.assertEqual(len(jobs[1].measurements), 1) assert_quantity_allclose( jobs[1].measurements[_metricName()].quantity, expectedValue)
def run(self, patchRefList, butler, selectDataList=[]): """!Run stacking on a tract This method only runs on the master node. @param patchRefList: List of patch data references for tract @param butler: Data butler @param selectDataList: List of SelectStruct for inputs """ pool = Pool("stacker") pool.cacheClear() pool.storeSet(butler=butler, warpType=self.config.coaddName + "Coadd_directWarp", coaddType=self.config.coaddName + "Coadd") patchIdList = [patchRef.dataId for patchRef in patchRefList] selectedData = pool.map(self.warp, patchIdList, selectDataList) if self.config.doBackgroundReference: self.backgroundReference.runDataRef(patchRefList, selectDataList) def refNamer(patchRef): return tuple(map(int, patchRef.dataId["patch"].split(","))) lookup = dict(zip(map(refNamer, patchRefList), selectedData)) coaddData = [ Struct(patchId=patchRef.dataId, selectDataList=lookup[refNamer(patchRef)]) for patchRef in patchRefList ] pool.map(self.coadd, coaddData)
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_tempExp"): """Group calibrated exposures overlapping a patch by the warped (temporary) exposure they contribute to. For example, if the instrument has a mosaic camera, each group would consist of the subset of CCD exposures from a single camera exposure that potentially overlap the patch. @return Struct with: - groups: Dict of <group tuple>: <list of data references for group> - keys: List of keys for group tuple """ butler = patchDataRef.getButler() tempExpKeys = butler.getKeys(datasetType=tempExpDatasetType) coaddKeys = sorted(butler.getKeys(datasetType=coaddDatasetType)) keys = sorted(set(tempExpKeys) - set(coaddKeys)) # Keys that will specify an exposure patchId = patchDataRef.dataId groups = groupDataRefs(keys, calexpDataRefList) # Supplement the groups with the coadd-specific information (e.g., tract, patch; these are constant) coaddValues = tuple(patchId[k] for k in coaddKeys) groups = dict((k + coaddValues, v) for k, v in groups.iteritems()) keys += tuple(coaddKeys) return Struct(groups=groups, keys=keys)
def run(self, sensorRefList): """Process all arms of the same kind within an exposure The sequence of operations is: - remove instrument signature - measure PSF - subtract sky from the image - extract the spectra from the fiber traces - write the outputs Parameters ---------- sensorRef : `lsst.daf.persistence.ButlerDataRef` Data reference for sensors to process. Returns ------- exposureList : `list` of `lsst.afw.image.Exposure` Exposure data for sensors. pfsList : `list` of PSFs Point-spread functions; if ``doMeasurePsf`` is set. lsfList : `list` of LSFs Line-spread functions; if ``doMeasurePsf`` is set. sky2d : `pfs.drp.stella.FocalPlaneFunction` 2D sky subtraction solution. spectraList : `list` of `pfs.drp.stella.SpectrumSet` Sets of extracted spectra. originalList : `list` of `pfs.drp.stella.SpectrumSet` Sets of extracted spectra before continuum subtraction. Will be identical to ``spectra`` if continuum subtraction was not performed. fiberTraceList : `list` of `pfs.drp.stella.FiberTraceSet` Fiber traces. detectorMapList : `list` of `pfs.drp.stella.DetectorMap` Mappings of wl,fiber to detector position. """ self.log.info("Processing %s" % ([sensorRef.dataId for sensorRef in sensorRefList])) exposureList = [] psfList = [] lsfList = [] for sensorRef in sensorRefList: exposure = self.isr.runDataRef(sensorRef).exposure if self.config.doRepair: self.repairExposure(exposure) if self.config.doSkySwindle: self.skySwindle(sensorRef, exposure.image) exposureList.append(exposure) if self.config.doMeasurePsf: psfList = self.measurePsf.run(sensorRefList, exposureList) lsfList = [self.calculateLsf(psf) for psf in psfList] else: psfList = [None]*len(sensorRefList) lsfList = [None]*len(sensorRefList) results = Struct(exposureList=exposureList, psfList=psfList, lsfList=lsfList) fiberTraceList = [sensorRef.get("fibertrace") for sensorRef in sensorRefList] detectorMapList = [sensorRef.get("detectormap") for sensorRef in sensorRefList] pfsConfig = sensorRefList[0].get("pfsConfig") results.fiberTraceList = fiberTraceList results.detectorMapList = detectorMapList results.pfsConfig = pfsConfig if self.config.doSubtractSky2d: results.sky2d = self.subtractSky2d.run(exposureList, pfsConfig, psfList, fiberTraceList, detectorMapList) if self.config.doExtractSpectra: originalList = [] spectraList = [] for exposure, fiberTraces, detectorMap in zip(exposureList, fiberTraceList, detectorMapList): spectra = self.extractSpectra.run(exposure.maskedImage, fiberTraces, detectorMap).spectra originalList.append(spectra) if self.config.doSubtractContinuum: continua = self.fitContinuum.run(spectra) exposure.maskedImage -= continua.makeImage(exposure.getBBox(), fiberTraces) spectra = self.extractSpectra.run(exposure.maskedImage, fiberTraces, detectorMap).spectra spectraList.append(spectra) results.originalList = originalList results.spectraList = spectraList self.write(sensorRefList, results) return results