def run( rerun, # Rerun name frame1, # Frame number for input frame2, # Frame number for template diff, # Difference identifier for output patch, # Sky patch identifier config, # Configuration ): io = pipReadWrite.ReadWrite(suprimecam.SuprimecamMapper(rerun=rerun), ['visit'], config=config) diffProc = pipDiff.Diff(config=config) exp1 = io.inButler.get('warp', {'visit': frame1, 'patch': patch}) exp2 = io.inButler.get('warp', {'visit': frame2, 'patch': patch}) diffExp, sources, psf, apcorr, brightSources = diffProc.run(exp1, exp2) sources = afwDet.PersistableSourceVector(sources) diffProc.write(io.outButler, { 'diff': diff, 'patch': patch, 'filter': diffExp.getFilter().getName() }, { "diff": diffExp, "diffpsf": psf, "diffsources": sources })
class SourceMeasurementStageParallel(harnessStage.ParallelProcessing): """ Description: This stage wraps the measurement of sources on an exposure. The exposures to measure on should be in the clipboard along with the FootprintSet(s) to measure on those exposures. The keys for the exposures, and the FootprintSet(s) can be specified in the policy file. If not specified, default keys will be used """ def setup(self): self.log = Log(self.log, "SourceMeasurementStage - parallel") policyFile = pexPolicy.DefaultPolicyFile("meas_pipeline", "SourceMeasurementStageDictionary.paf", "policy") defPolicy = pexPolicy.Policy.createPolicy(policyFile, policyFile.getRepositoryPath(), True) if self.policy is None: self.policy = defPolicy else: self.policy.mergeDefaults(defPolicy.getDictionary()) def process(self, clipboard): """ Measure sources in the worker process """ self.log.log(Log.INFO, "Measuring Sources in process") #this may raise exceptions try: measurePolicy, exposure, psf, positiveDetection, negativeDetection = \ self.getClipboardData(clipboard) except pexExcept.LsstException, e: self.log.log(Log.FATAL, str(e)) # # Need to do something smart about merging positive and negative # detection sets. # # For now, assume they are disjoint sets, so merge is trivial # footprintLists = [] if positiveDetection: self.log.log(Log.DEBUG, "Positive FootprintSet found") isNegative = False footprintLists.append([positiveDetection.getFootprints(), isNegative]) if negativeDetection: self.log.log(Log.DEBUG, "Negative FootprintSet found") isNegative = True footprintLists.append([negativeDetection.getFootprints(), isNegative]) sourceSet = srcMeas.sourceMeasurement(exposure, psf, footprintLists, measurePolicy) # place SourceSet on the clipboard sourceKey = self.policy.get("outputKeys.sources") clipboard.put(sourceKey, sourceSet) clipboard.put(sourceKey + "_persistable", afwDet.PersistableSourceVector(sourceSet))
class SourceMeasurementPsfFluxStageParallel(harnessStage.ParallelProcessing): """ Description: This stage wraps the measurement of sources on an exposure. The exposures to measure on should be in the clipboard along with the FootprintSet(s) to measure on those exposures. The keys for the exposures, and the FootprintSet(s) can be specified in the policy file. If not specified, default keys will be used """ def setup(self): self.log = Log(self.log, "SourceMeasurementPsfFluxStage - parallel") policyFile = pexPolicy.DefaultPolicyFile( "meas_pipeline", "SourceMeasurementPsfFluxStageDictionary.paf", "policy") defPolicy = pexPolicy.Policy.createPolicy( policyFile, policyFile.getRepositoryPath(), True) if self.policy is None: self.policy = defPolicy else: self.policy.mergeDefaults(defPolicy.getDictionary()) def process(self, clipboard): """ Measure sources in the worker process """ self.log.log(Log.INFO, "Measuring Sources' psfFluxes in process") #this may raise exceptions try: exposure, sourceSet = self.getClipboardData(clipboard) except pexExcept.LsstException, e: self.log.log(Log.FATAL, str(e)) srcMeas.sourceMeasurementPsfFlux(exposure, sourceSet) # place SourceSet on the clipboard. We need to do this because of the _persistable version # even though there's already a sourceSet. Damn. sourceKey = self.policy.get("outputKeys.sourceSet") clipboard.put(sourceKey, sourceSet) clipboard.put(sourceKey + "_persistable", afwDet.PersistableSourceVector(sourceSet))
def write(self, dataId, exposure=None, psf=None, sources=None, brightSources=None, matches=None, matchMeta=None, **kwargs): """Write processed data. @param dataId Data identifier for butler @param exposure Exposure to write, or None @param psf PSF to write, or None @param sources Sources to write, or None @param matches Matches to write, or None """ if exposure is not None: self.log.log(self.log.INFO, "Writing exposure: %s" % (dataId)) self.outButler.put(exposure, 'calexp', dataId) if psf is not None: self.log.log(self.log.INFO, "Writing PSF: %s" % (dataId)) self.outButler.put(psf, 'psf', dataId) if sources is not None: self.log.log(self.log.INFO, "Writing sources: %s" % (dataId)) self.outButler.put(afwDet.PersistableSourceVector(sources), 'src', dataId) if matches is not None: try: self.log.log(self.log.INFO, "Writing matches: %s" % (dataId)) smv = afwDet.SourceMatchVector() for match in matches: smv.push_back(match) self.outButler.put( afwDet.PersistableSourceMatchVector(smv, matchMeta), 'icMatch', dataId) if brightSources is None: brightSources = afwDet.SourceSet() for match in matches: brightSources.push_back(match.second) except Exception, e: self.log.log(self.log.WARN, "Unable to write matches: %s" % e)
def process(self, clipboard): self.log.log(Log.INFO, "Making a persistable source vector in process") sourceSet = clipboard.get(self.policy.get("inputkeys.sourceSet")) clipboard.put(self.policy.get("outputKeys.persistable"), afwDet.PersistableSourceVector(sourceSet))
class ReadWrite(object): """ReadWrite provides I/O for pipette (LSST algorithms testing)""" def __init__( self, # ReadWrite mappers, # Mapper or mapper class to use ccdKeys, # Data keywords required to specify a CCD fileKeys=None, # Data keywords required to specify a file config=None, # Configuration ): """Initialisation @param mapper Data mapper (class or instance) for persistence @param config Configuration (for instantiating mapper) """ # if we got a list, it contains [inMapper, outMapper] if isinstance(mappers, list) and len(mappers) == 2: inMapper, outMapper = mappers # if we got a mapper, use it for both input and output elif (isinstance(mappers, dafPersist.Mapper) or issubclass(mappers, dafPersist.Mapper)): inMapper, outMapper = mappers, mappers # punt else: raise RuntimeError( "'mapper' must be a dafPersist.Mapper (or derived from), or a list containing two of them (in and out)." ) self.log = pexLog.Log(pexLog.getDefaultLog(), "ReadWrite") self.inMapper = initMapper(inMapper, config, self.log, inMap=True) self.ibf = dafPersist.ButlerFactory(mapper=self.inMapper) self.inButler = self.ibf.create() self.outMapper = initMapper(outMapper, config, self.log, inMap=False) self.obf = dafPersist.ButlerFactory(mapper=self.outMapper) self.outButler = self.obf.create() self.ccdKeys = ccdKeys if fileKeys is None: fileKeys = list(ccdKeys) if isinstance(fileKeys, basestring): fileKeys = [fileKeys] self.fileKeys = fileKeys return def lookup(self, dataId): """Lookup data for a CCD. @param dataId Data identifier for butler @returns Complete data identifiers """ for key in self.ccdKeys: if not dataId.has_key(key): raise KeyError("Data identifier does not contain keyword %s" % key) keys = self.inButler.queryMetadata('raw', self.ccdKeys, format=self.fileKeys, dataId=dataId) identifiers = list() for key in keys: ident = dict() if not isinstance(key, basestring) and hasattr(key, "__getitem__"): for index, name in enumerate(self.fileKeys): ident[name] = key[index] else: assert (len(self.fileKeys) == 1) ident[self.fileKeys[0]] = key identifiers.append(ident) return identifiers @timecall def readRaw(self, dataId): """Read raw data of a CCD. @param dataId Data identifier for butler @returns Raw exposures """ self.log.log(self.log.INFO, "Looking for: %s" % (dataId)) identifiers = self.lookup(dataId) if not identifiers: raise RuntimeError("No raw data found for dataId %s" % (dataId)) exposures = list() for ident in identifiers: ident.update(dataId) if not self.inButler.datasetExists('raw', ident): raise RuntimeError("Raw data does not exist for %s" % ident) self.log.log(self.log.DEBUG, "Reading: %s" % (ident)) exp = self.inButler.get('raw', ident) if isinstance(exp, afwImage.ExposureU): exp = exp.convertF() exposures.append(exp) return exposures def readMatches(self, dataId, ignore=False): """Read matches, sources and catalogue; combine. @param dataId Data identifier for butler @param ignore Ignore non-existent data? @returns Matches """ sources = self.read('icSrc', dataId, ignore=ignore) matches = self.read('icMatch', dataId, ignore=ignore) headers = self.read('calexp_md', dataId, ignore=ignore) output = [] for sourceList, matchList, header in zip(sources, matches, headers): wcs = afwImage.makeWcs(header) width, height = header.get('NAXIS1'), header.get('NAXIS2') matches = measAstrom.generateMatchesFromMatchList( matchList, sourceList.getSources(), wcs, width, height, log=self.log) output.append(matches) return output @timecall def read(self, which, dataId, ignore=False): """Read some data. @param which Type of data to read @param dataId Data identifier for butler @returns Raw exposures """ identifiers = self.lookup(dataId) data = list() for ident in identifiers: ident.update(dataId) for i, butler in enumerate([self.inButler, self.outButler]): if not butler.datasetExists(which, ident): if i == 1: if not ignore: raise RuntimeError( "Data type %s does not exist for %s" % (which, ident)) else: self.log.log(self.log.DEBUG, "Reading %s: %s" % (which, ident)) data.append(butler.get(which, ident)) break return data @timecall def detrends(self, dataId, config): """Read detrends for a CCD. @param dataId Data identifier for butler @param config Configuration (for which detrends to read) @returns Dict of lists for each detrend type """ identifiers = self.lookup(dataId) detrends = dict() do = config['do']['isr'] if not do['enabled']: return detrends for kind in ('bias', 'dark', 'flat'): if do[kind]: detList = list() for ident in identifiers: ident.update(dataId) if not self.inButler.datasetExists(kind, ident): raise RuntimeError( "Data type %s does not exist for %s" % (kind, ident)) self.log.log(self.log.DEBUG, "Reading %s for %s" % (kind, ident)) detrend = self.inButler.get(kind, ident) detList.append(detrend) detrends[kind] = detList # Fringe depends on the filter if do['fringe'] and config['fringe'].has_key('filters'): fringeList = list() for ident in identifiers: ident.update(dataId) filterList = self.inButler.queryMetadata( "raw", None, "filter", ident) assert len( filterList ) == 1, "Filter query is non-unique: %s" % filterList filtName = filterList[0] if filtName in config['fringe']['filters']: if not self.inButler.datasetExists('fringe', ident): raise RuntimeError( "Data type fringe does not exist for %s" % ident) self.log.log(self.log.DEBUG, "Reading fringe for %s" % (ident)) fringe = self.inButler.get("fringe", ident) fringeList.append(fringe) if len(fringeList) > 0: detrends['fringe'] = fringeList return detrends @timecall def write(self, dataId, exposure=None, psf=None, sources=None, brightSources=None, matches=None, matchMeta=None, **kwargs): """Write processed data. @param dataId Data identifier for butler @param exposure Exposure to write, or None @param psf PSF to write, or None @param sources Sources to write, or None @param matches Matches to write, or None """ if exposure is not None: self.log.log(self.log.INFO, "Writing exposure: %s" % (dataId)) self.outButler.put(exposure, 'calexp', dataId) if psf is not None: self.log.log(self.log.INFO, "Writing PSF: %s" % (dataId)) self.outButler.put(psf, 'psf', dataId) if sources is not None: self.log.log(self.log.INFO, "Writing sources: %s" % (dataId)) self.outButler.put(afwDet.PersistableSourceVector(sources), 'src', dataId) if matches is not None: try: self.log.log(self.log.INFO, "Writing matches: %s" % (dataId)) smv = afwDet.SourceMatchVector() for match in matches: smv.push_back(match) self.outButler.put( afwDet.PersistableSourceMatchVector(smv, matchMeta), 'icMatch', dataId) if brightSources is None: brightSources = afwDet.SourceSet() for match in matches: brightSources.push_back(match.second) except Exception, e: self.log.log(self.log.WARN, "Unable to write matches: %s" % e) if brightSources is not None: self.log.log(self.log.INFO, "Writing calibration sources: %s" % (dataId)) self.outButler.put(afwDet.PersistableSourceVector(brightSources), 'icSrc', dataId) return