def testTaskAPI(self): """Test that the Tasks work Checks both MeasureCrosstalkTask and the CrosstalkTask. """ # make exposure available to NullIsrTask # without NullIsrTask's `self` hiding this test class's `self` exposure = self.exposure class NullIsrTask(IsrTask): def runDataRef(self, dataRef): return Struct(exposure=exposure) config = MeasureCrosstalkTask.ConfigClass() config.isr.retarget(NullIsrTask) config.threshold = self.value - 1 measure = MeasureCrosstalkTask(config=config) fakeDataRef = Struct(dataId={'fake': 1}) coeff, coeffErr, coeffNum = measure.reduce( [measure.runDataRef(fakeDataRef)]) self.checkCoefficients(coeff, coeffErr, coeffNum) config = IsrTask.ConfigClass() config.crosstalk.minPixelToMask = self.value - 1 config.crosstalk.crosstalkMaskPlane = self.crosstalkStr isr = IsrTask(config=config) calib = CrosstalkCalib().fromDetector(self.exposure.getDetector(), coeffVector=coeff.transpose()) isr.crosstalk.run(self.exposure, crosstalk=calib) self.checkSubtracted(self.exposure)
def filterCrosstalkCalib(inCalib): """Apply valid constraints to the measured values. Any measured coefficient that is determined to be invalid is set to zero, and has the error set to nan. The validation is determined by checking that the measured coefficient is larger than the calculated standard error of the mean. Parameters ---------- inCalib : `lsst.ip.isr.CrosstalkCalib` Input calibration to filter. Returns ------- outCalib : `lsst.ip.isr.CrosstalkCalib` Filtered calibration. """ outCalib = CrosstalkCalib() outCalib.numAmps = inCalib.numAmps outCalib.coeffs = inCalib.coeffs outCalib.coeffs[~inCalib.coeffValid] = 0.0 outCalib.coeffErr = inCalib.coeffErr outCalib.coeffErr[~inCalib.coeffValid] = np.nan outCalib.coeffNum = inCalib.coeffNum outCalib.coeffValid = inCalib.coeffValid return outCalib
def test_interChip(self): """Test that passing an external exposure as the crosstalk source works. """ exposure = self.exposure ctSources = [self.ctSource] coeff = np.array(self.crosstalk).transpose() calib = CrosstalkCalib().fromDetector(exposure.getDetector(), coeffVector=coeff) # Now convert this into zero intra-chip, full inter-chip: calib.interChip['detector 2'] = coeff calib.coeffs = np.zeros_like(coeff) # Process and check as above config = IsrTask.ConfigClass() config.crosstalk.minPixelToMask = self.value - 1 config.crosstalk.crosstalkMaskPlane = self.crosstalkStr isr = IsrTask(config=config) isr.crosstalk.run(exposure, crosstalk=calib, crosstalkSources=ctSources) self.checkSubtracted(exposure)
def testDirectAPI(self): """Test that individual function calls work""" calib = CrosstalkCalib() calib.coeffs = np.array(self.crosstalk).transpose() calib.subtractCrosstalk(self.exposure, crosstalkCoeffs=calib.coeffs, minPixelToMask=self.value - 1, crosstalkStr=self.crosstalkStr) self.checkSubtracted(self.exposure) outPath = tempfile.mktemp( ) if outputName is None else "{}-isrCrosstalk".format(outputName) outPath += '.yaml' calib.writeText(outPath)
def test_crosstalkIO(self): """Test that crosstalk doesn't change on being converted to persistable formats. """ # Add the interchip crosstalk as in the previous test. exposure = self.exposure coeff = np.array(self.crosstalk).transpose() calib = CrosstalkCalib().fromDetector(exposure.getDetector(), coeffVector=coeff) # Now convert this into zero intra-chip, full inter-chip: calib.interChip['detector 2'] = coeff outPath = tempfile.mktemp() + '.yaml' calib.writeText(outPath) newCrosstalk = CrosstalkCalib().readText(outPath) self.assertEqual(calib, newCrosstalk) outPath = tempfile.mktemp() + '.fits' calib.writeFits(outPath) newCrosstalk = CrosstalkCalib().readFits(outPath) self.assertEqual(calib, newCrosstalk)
def testDirectAPI(self): """Test that individual function calls work""" config = MeasureCrosstalkTask.ConfigClass() measure = MeasureCrosstalkTask(config=config) ratios = measure.extractCrosstalkRatios(self.exposure, threshold=self.value - 1) coeff, coeffErr, coeffNum = measure.measureCrosstalkCoefficients( ratios) self.checkCoefficients(coeff, coeffErr, coeffNum) calib = CrosstalkCalib() calib.coeffs = coeff.transpose() calib.coeffErr = coeffErr.transpose() calib.coeffNum = coeffNum.transpose() calib.subtractCrosstalk(self.exposure, crosstalkCoeffs=coeff.transpose(), minPixelToMask=self.value - 1, crosstalkStr=self.crosstalkStr) self.checkSubtracted(self.exposure) outPath = tempfile.mktemp( ) if outputName is None else "{}-isrCrosstalk".format(outputName) outPath += '.yaml' calib.writeText(outPath)
def testTaskAPI(self): """Test that the Tasks work Checks both MeasureCrosstalkTask and the CrosstalkTask. """ # make exposure available to NullIsrTask # without NullIsrTask's `self` hiding this test class's `self` exposure = self.exposure class NullIsrTask(IsrTask): def runDataRef(self, dataRef): return Struct(exposure=exposure) coeff = np.array(self.crosstalk).transpose() config = IsrTask.ConfigClass() config.crosstalk.minPixelToMask = self.value - 1 config.crosstalk.crosstalkMaskPlane = self.crosstalkStr isr = IsrTask(config=config) calib = CrosstalkCalib().fromDetector(self.exposure.getDetector(), coeffVector=coeff) isr.crosstalk.run(self.exposure, crosstalk=calib) self.checkSubtracted(self.exposure)
def measureCrosstalkCoefficients(self, ratios, rejIter, rejSigma): """Measure crosstalk coefficients from the ratios. Given a list of ratios for each target/source amp combination, we measure a sigma clipped mean and error. The coefficient errors returned are the standard deviation of the final set of clipped input ratios. Parameters ---------- ratios : `dict` of `dict` of `numpy.ndarray` Catalog of arrays of ratios. rejIter : `int` Number of rejection iterations. rejSigma : `float` Rejection threshold (sigma). Returns ------- calib : `lsst.ip.isr.CrosstalkCalib` The output crosstalk calibration. Notes ----- The lsstDebug.Info() method can be rewritten for __name__ = `lsst.ip.isr.measureCrosstalk`, and supports the parameters: debug.display['measure'] : `bool` Display the CDF of the combined ratio measurements for a pair of source/target amplifiers from the final set of clipped input ratios. """ calib = CrosstalkCalib(nAmp=len(ratios)) # Calibration stores coefficients as a numpy ndarray. ordering = list(ratios.keys()) for ii, jj in itertools.product(range(calib.nAmp), range(calib.nAmp)): if ii == jj: values = [0.0] else: values = np.array(ratios[ordering[ii]][ordering[jj]]) values = values[np.abs(values) < 1.0] # Discard unreasonable values calib.coeffNum[ii][jj] = len(values) if len(values) == 0: self.log.warn("No values for matrix element %d,%d" % (ii, jj)) calib.coeffs[ii][jj] = np.nan calib.coeffErr[ii][jj] = np.nan calib.coeffValid[ii][jj] = False else: if ii != jj: for rej in range(rejIter): lo, med, hi = np.percentile(values, [25.0, 50.0, 75.0]) sigma = 0.741*(hi - lo) good = np.abs(values - med) < rejSigma*sigma if good.sum() == len(good): break values = values[good] calib.coeffs[ii][jj] = np.mean(values) if calib.coeffNum[ii][jj] == 1: calib.coeffErr[ii][jj] = np.nan else: correctionFactor = self.sigmaClipCorrection(rejSigma) calib.coeffErr[ii][jj] = np.std(values) * correctionFactor calib.coeffValid[ii][jj] = (np.abs(calib.coeffs[ii][jj]) > calib.coeffErr[ii][jj] / np.sqrt(calib.coeffNum[ii][jj])) if calib.coeffNum[ii][jj] > 1: self.debugRatios('measure', ratios, ordering[ii], ordering[jj], calib.coeffs[ii][jj], calib.coeffValid[ii][jj]) return calib
def run(self, inputExp, sourceExps=[]): """Measure pixel ratios between amplifiers in inputExp. Extract crosstalk ratios between different amplifiers. For pixels above ``config.threshold``, we calculate the ratio between each background-subtracted target amp and the source amp. We return a list of ratios for each pixel for each target/source combination, as nested dictionary containing the ratio. Parameters ---------- inputExp : `lsst.afw.image.Exposure` Input exposure to measure pixel ratios on. sourceExp : `list` [`lsst.afw.image.Exposure`], optional List of chips to use as sources to measure inter-chip crosstalk. Returns ------- results : `lsst.pipe.base.Struct` The results struct containing: ``outputRatios`` : `dict` [`dict` [`dict` [`dict` [`list`]]]] A catalog of ratio lists. The dictionaries are indexed such that: outputRatios[targetChip][sourceChip][targetAmp][sourceAmp] contains the ratio list for that combination. ``outputFluxes`` : `dict` [`dict` [`list`]] A catalog of flux lists. The dictionaries are indexed such that: outputFluxes[sourceChip][sourceAmp] contains the flux list used in the outputRatios. Notes ----- The lsstDebug.Info() method can be rewritten for __name__ = `lsst.cp.pipe.measureCrosstalk`, and supports the parameters: debug.display['extract'] : `bool` Display the exposure under consideration, with the pixels used for crosstalk measurement indicated by the DETECTED mask plane. debug.display['pixels'] : `bool` Display a plot of the ratio calculated for each pixel used in this exposure, split by amplifier pairs. The median value is listed for reference. """ outputRatios = defaultdict(lambda: defaultdict(dict)) outputFluxes = defaultdict(lambda: defaultdict(dict)) threshold = self.config.threshold badPixels = list(self.config.badMask) targetDetector = inputExp.getDetector() targetChip = targetDetector.getName() # Always look at the target chip first, then go to any other supplied exposures. sourceExtractExps = [inputExp] sourceExtractExps.extend(sourceExps) self.log.info("Measuring full detector background for target: %s", targetChip) targetIm = inputExp.getMaskedImage() FootprintSet(targetIm, Threshold(threshold), "DETECTED") detected = targetIm.getMask().getPlaneBitMask("DETECTED") bg = CrosstalkCalib.calculateBackground(targetIm, badPixels + ["DETECTED"]) self.debugView('extract', inputExp) for sourceExp in sourceExtractExps: sourceDetector = sourceExp.getDetector() sourceChip = sourceDetector.getName() sourceIm = sourceExp.getMaskedImage() bad = sourceIm.getMask().getPlaneBitMask(badPixels) self.log.info("Measuring crosstalk from source: %s", sourceChip) if sourceExp != inputExp: FootprintSet(sourceIm, Threshold(threshold), "DETECTED") detected = sourceIm.getMask().getPlaneBitMask("DETECTED") # The dictionary of amp-to-amp ratios for this pair of source->target detectors. ratioDict = defaultdict(lambda: defaultdict(list)) extractedCount = 0 for sourceAmp in sourceDetector: sourceAmpName = sourceAmp.getName() sourceAmpImage = sourceIm[sourceAmp.getBBox()] sourceMask = sourceAmpImage.mask.array select = ((sourceMask & detected > 0) & (sourceMask & bad == 0) & np.isfinite(sourceAmpImage.image.array)) count = np.sum(select) self.log.debug(" Source amplifier: %s", sourceAmpName) outputFluxes[sourceChip][sourceAmpName] = sourceAmpImage.image.array[select].tolist() for targetAmp in targetDetector: # iterate over targetExposure targetAmpName = targetAmp.getName() if sourceAmpName == targetAmpName and sourceChip == targetChip: ratioDict[sourceAmpName][targetAmpName] = [] continue self.log.debug(" Target amplifier: %s", targetAmpName) targetAmpImage = CrosstalkCalib.extractAmp(targetIm.image, targetAmp, sourceAmp, isTrimmed=self.config.isTrimmed) ratios = (targetAmpImage.array[select] - bg)/sourceAmpImage.image.array[select] ratioDict[targetAmpName][sourceAmpName] = ratios.tolist() extractedCount += count self.debugPixels('pixels', sourceAmpImage.image.array[select], targetAmpImage.array[select] - bg, sourceAmpName, targetAmpName) self.log.info("Extracted %d pixels from %s -> %s (targetBG: %f)", extractedCount, sourceChip, targetChip, bg) outputRatios[targetChip][sourceChip] = ratioDict return pipeBase.Struct( outputRatios=ddict2dict(outputRatios), outputFluxes=ddict2dict(outputFluxes) )
def run(self, inputExp, rawExp): ## run() method outputRatios = defaultdict(lambda: defaultdict(dict)) outputFluxes = defaultdict(lambda: defaultdict(dict)) outputZOffsets = defaultdict(lambda: defaultdict(dict)) outputYTilts = defaultdict(lambda: defaultdict(dict)) outputXTilts = defaultdict(lambda: defaultdict(dict)) badPixels = list(self.config.badMask) targetDetector = inputExp.getDetector() targetChip = targetDetector.getName() targetIm = inputExp.getMaskedImage() ## loop on sourceExp would go here sourceExp = inputExp ## Ignore other exposures for now sourceDetector = sourceExp.getDetector() sourceChip = sourceDetector.getName() sourceIm = sourceExp.getMaskedImage() bad = sourceIm.getMask().getPlaneBitMask(badPixels) self.log.info("Measuring crosstalk from source: %s", sourceChip) ratioDict = defaultdict(lambda: defaultdict(list)) zoffsetDict = defaultdict(lambda: defaultdict(list)) ytiltDict = defaultdict(lambda: defaultdict(list)) xtiltDict = defaultdict(lambda: defaultdict(list)) extractedCount = 0 for sourceAmp in sourceDetector: sourceAmpName = sourceAmp.getName() sourceAmpImage = sourceIm[sourceAmp.getBBox()] sourceMask = sourceAmpImage.mask.array sourceAmpArray = sourceAmpImage.image.array tested_angles = np.linspace(-np.pi / 2, np.pi / 2, 1000) edges = feature.canny(sourceAmpArray, sigma=self.config.cannySigma, low_threshold=self.config.thresholdLow, high_threshold=self.config.thresholdHigh) h, theta, d = hough_line(edges, theta=tested_angles) _, angle, dist = hough_line_peaks(h, theta, d) if len(angle) != 2: continue mean_angle = np.mean(angle) mean_dist = np.mean(dist) select = mixCrosstalk.satellite_mask(sourceAmpArray, mean_angle, mean_dist, width=self.config.maskWidth) signal = np.max(sourceAmpArray[select]) self.log.debug(" Source amplifier: %s", sourceAmpName) outputFluxes[sourceChip][sourceAmpName] = [float(signal)] for targetAmp in targetDetector: # iterate over targetExposure targetAmpName = targetAmp.getName() if sourceAmpName == targetAmpName and sourceChip == targetChip: ratioDict[sourceAmpName][targetAmpName] = [] continue self.log.debug(" Target amplifier: %s", targetAmpName) if self.config.correctNoiseCovariance: covariance = mixCrosstalk.calculate_covariance( rawExp, sourceAmp, targetAmp) else: noise = np.asarray( [[sourceAmp.getReadNoise() / sourceAmp.getGain(), 0.], [0., targetAmp.getReadNoise() / targetAmp.getGain()]]) covariance = np.square(noise) targetAmpImage = CrosstalkCalib.extractAmp( targetIm.image, targetAmp, sourceAmp, isTrimmed=self.config.isTrimmed) targetAmpArray = targetAmpImage.array results = mixCrosstalk.crosstalk_fit( sourceAmpArray, targetAmpArray, select, covariance=covariance, correct_covariance=self.config.correctNoiseCovariance, seed=189) ratioDict[targetAmpName][sourceAmpName] = [float(results[0])] zoffsetDict[targetAmpName][sourceAmpName] = [float(results[1])] ytiltDict[targetAmpName][sourceAmpName] = [float(results[2])] xtiltDict[targetAmpName][sourceAmpName] = [float(results[3])] extractedCount += 1 self.log.info("Extracted %d pixels from %s -> %s", extractedCount, sourceChip, targetChip) outputRatios[targetChip][sourceChip] = ratioDict outputZOffsets[targetChip][sourceChip] = zoffsetDict outputYTilts[targetChip][sourceChip] = ytiltDict outputXTilts[targetChip][sourceChip] = xtiltDict return pipeBase.Struct(outputRatios=ddict2dict(outputRatios), outputFluxes=ddict2dict(outputFluxes), outputZOffsets=ddict2dict(outputZOffsets), outputYTilts=ddict2dict(outputYTilts), outputXTilts=ddict2dict(outputXTilts))
def run(self, inputExp, rawExp): ## run() method outputRatios = defaultdict(lambda: defaultdict(dict)) outputFluxes = defaultdict(lambda: defaultdict(dict)) outputZOffsets = defaultdict(lambda: defaultdict(dict)) outputYTilts = defaultdict(lambda: defaultdict(dict)) outputXTilts = defaultdict(lambda: defaultdict(dict)) badPixels = list(self.config.badMask) targetDetector = inputExp.getDetector() targetChip = targetDetector.getName() targetIm = inputExp.getMaskedImage() ## loop on sourceExp would go here sourceExp = inputExp ## Ignore other exposures for now sourceDetector = sourceExp.getDetector() sourceChip = sourceDetector.getName() sourceIm = sourceExp.getMaskedImage() bad = sourceIm.getMask().getPlaneBitMask(badPixels) self.log.info("Measuring crosstalk from source: %s", sourceChip) ratioDict = defaultdict(lambda: defaultdict(list)) zoffsetDict = defaultdict(lambda: defaultdict(list)) ytiltDict = defaultdict(lambda: defaultdict(list)) xtiltDict = defaultdict(lambda: defaultdict(list)) extractedCount = 0 for sourceAmp in sourceDetector: sourceAmpName = sourceAmp.getName() sourceAmpImage = sourceIm[sourceAmp.getBBox()] sourceMask = sourceAmpImage.mask.array sourceAmpArray = sourceAmpImage.image.array columns = mixCrosstalk.find_bright_columns(sourceAmpArray, self.config.threshold) if len(columns) == 0: continue select = mixCrosstalk.rectangular_mask(sourceAmpArray, 1000, columns[0], ly=self.config.maskLengthY, lx=self.config.maskLengthX) signal = np.mean(sourceAmpArray[:, columns[0]]) self.log.debug(" Source amplifier: %s", sourceAmpName) outputFluxes[sourceChip][sourceAmpName] = [float(signal)] for targetAmp in targetDetector: # iterate over targetExposure targetAmpName = targetAmp.getName() if sourceAmpName == targetAmpName and sourceChip == targetChip: ratioDict[sourceAmpName][targetAmpName] = [] continue self.log.debug(" Target amplifier: %s", targetAmpName) if self.config.correctNoiseCovariance: covariance = mixCrosstalk.calculate_covariance( rawExp, sourceAmp, targetAmp) else: noise = np.asarray( [[sourceAmp.getReadNoise() / sourceAmp.getGain(), 0.], [0., targetAmp.getReadNoise() / targetAmp.getGain()]]) covariance = np.square(noise) targetAmpImage = CrosstalkCalib.extractAmp( targetIm.image, targetAmp, sourceAmp, isTrimmed=self.config.isTrimmed) targetAmpArray = targetAmpImage.array results = mixCrosstalk.crosstalk_fit( sourceAmpArray, targetAmpArray, select, covariance=covariance, correct_covariance=self.config.correctNoiseCovariance, seed=189) ratioDict[targetAmpName][sourceAmpName] = [float(results[0])] zoffsetDict[targetAmpName][sourceAmpName] = [float(results[1])] ytiltDict[targetAmpName][sourceAmpName] = [float(results[2])] xtiltDict[targetAmpName][sourceAmpName] = [float(results[3])] extractedCount += 1 self.log.info("Extracted %d pixels from %s -> %s", extractedCount, sourceChip, targetChip) outputRatios[targetChip][sourceChip] = ratioDict outputZOffsets[targetChip][sourceChip] = zoffsetDict outputYTilts[targetChip][sourceChip] = ytiltDict outputXTilts[targetChip][sourceChip] = xtiltDict return pipeBase.Struct(outputRatios=ddict2dict(outputRatios), outputFluxes=ddict2dict(outputFluxes), outputZOffsets=ddict2dict(outputZOffsets), outputYTilts=ddict2dict(outputYTilts), outputXTilts=ddict2dict(outputXTilts))