def __call__(self, args): """Run the task on a single target. This implementation is nearly equivalent to the overridden one, but it never writes out metadata and always returns results. For memory efficiency reasons, the return value is exactly the one of |run|, rather than a :class:`~lsst.pipe.base.Struct` wrapped around it. """ data_ref, kwargs = args if self.log is None: self.log = Log.getDefaultLogger() if hasattr(data_ref, "dataId"): self.log.MDC("LABEL", str(data_ref.dataId)) elif isinstance(data_ref, (list, tuple)): self.log.MDC("LABEL", str([ref.dataId for ref in data_ref if hasattr(ref, "dataId")])) task = self.makeTask(args=args) result = None try: result = task.run(data_ref, **kwargs) except Exception, e: if self.doRaise: raise if hasattr(data_ref, "dataId"): task.log.fatal("Failed on dataId=%s: %s" % (data_ref.dataId, e)) elif isinstance(data_ref, (list, tuple)): task.log.fatal("Failed on dataId=[%s]: %s" % (",".join([str(_.dataId) for _ in data_ref]), e)) else: task.log.fatal("Failed on dataRef=%s: %s" % (data_ref, e)) if not isinstance(e, pipe_base.TaskError): traceback.print_exc(file=sys.stderr)
def parse_args(self, config, args=None, log=None, override=None): if args is None: args = sys.argv[1:] namespace = Namespace() namespace.config = config namespace.clobberConfig = False namespace.butler = None namespace.log = log if log is not None else Log.getDefaultLogger() namespace = super(MeasurementDebuggerArgumentParser, self).parse_args(args=args, namespace=namespace) del namespace.configfile return namespace
def _poolFunctionWrapper(function, arg): """Wrapper around function to catch exceptions that don't inherit from Exception Such exceptions aren't caught by multiprocessing, which causes the slave process to crash and you end up hitting the timeout. """ try: return function(arg) except Exception: raise # No worries except: # Need to wrap the exception with something multiprocessing will recognise cls, exc, tb = sys.exc_info() log = Log.getDefaultLogger() log.warn("Unhandled exception %s (%s):\n%s" % (cls.__name__, exc, traceback.format_exc())) raise Exception("Unhandled exception: %s (%s)" % (cls.__name__, exc))
def __init__(self, filenameList, healpix, nside): """!Constructor @param filenameList List of filenames; first is the multiindex, then follows the individual index files @param healpix Healpix number @param nside Healpix nside """ if len(filenameList) < 2: raise RuntimeError("Insufficient filenames provided for multiindex (%s): expected >= 2" % (filenameList,)) self._filenameList = filenameList self._healpix = int(healpix) self._nside = int(nside) self._mi = None self._loaded = False self.log = Log.getDefaultLogger()
def test_basic(self): """Perform basic correctness testing.""" ps = [] # Construct property sets for two exposures centered on the equator for center in ((0.0, 0.0), (180.0, 0.0)): props = daf_base.PropertySet() props.add("NAXIS1", 9) props.add("NAXIS2", 9) props.add("RADECSYS", "ICRS") props.add("EQUINOX", 2000.0) props.add("CTYPE1", "RA---TAN") props.add("CTYPE2", "DEC--TAN") props.add("CRPIX1", 5.0) props.add("CRPIX2", 5.0) props.add("CRVAL1", center[0]) props.add("CRVAL2", center[1]) props.add("CD1_1", 1.0) props.add("CD2_1", 0.0) props.add("CD1_2", 0.0) props.add("CD2_2", 1.0) ps.append(props) # Retain one as is, and create an exposure from the other inputs = [ps[0], afw_image.ExposureF(8, 8, afw_image.makeWcs(ps[1]))] # Test data-ids are just integers. refs = [MockDataRef(i, v) for i, v in enumerate(inputs)] config = IndexExposureConfig() config.allow_replace = True config.defer_writes = True config.init_statements = ['PRAGMA page_size = 4096'] database = sqlite3.connect(":memory:") # Avoid the command line parser. parsed_cmd = pipe_base.Struct( config=config, log=Log.getDefaultLogger(), doraise=True, clobberConfig=False, noBackupConfig=False, database=database, dstype="bogus", id=pipe_base.Struct(refList=refs), ) runner = IndexExposureRunner(IndexExposureTask, parsed_cmd) runner.run(parsed_cmd) # Re-ingest to test that allow_replace=True works. Toggle off # the deferred writes to test that as well. runner.config.defer_writes = False runner.run(parsed_cmd) # Re-ingest to test that allow_replace=False raises an exception. runner.config.allow_replace = False with self.assertRaises(Exception): runner.run(parsed_cmd) # Now, verify the contents of the database. First, check that # data ids are recoverable. data_ids = sorted( pickle.loads(str(r[0])) for r in database.execute("SELECT pickled_data_id FROM exposure")) self.assertEqual(data_ids, [0, 1]) # Next, run a spatial query and check that it returns the # expected results. center = sphgeom.UnitVector3d(sphgeom.LonLat.fromDegrees(4.0, 1.0)) circle = sphgeom.Circle(center, sphgeom.Angle.fromDegrees(1.5)) results = find_intersecting_exposures(database, circle) self.assertEqual(len(results), 1) info = results[0] # The first input exposure should have been returned, and # should intersect the query region self.assertEqual(info.data_id, 0) self.assertEqual(circle.relate(info.boundary), sphgeom.INTERSECTS) database.close()
def readSrc(self, dataRef): """Read source catalog etc for input dataRef The following are returned: Source catalog, matched list, and wcs will be read from 'src', 'srcMatch', and 'calexp_md', respectively. NOTE: If the detector has nQuarter%4 != 0 (i.e. it is rotated w.r.t the focal plane coordinate system), the (x, y) pixel values of the centroid slot for the source catalogs are rotated such that pixel (0, 0) is the LLC (i.e. the coordinate system expected by meas_mosaic). If color transformation information is given, it will be applied to the reference flux of the matched list. The source catalog and matched list will be converted to measMosaic's Source and SourceMatch and returned. The number of 'Source's in each cell defined by config.cellSize will be limited to brightest config.nStarPerCell. """ self.log = Log.getDefaultLogger() dataId = dataRef.dataId try: if not dataRef.datasetExists("src"): raise RuntimeError("no data for src %s" % (dataId)) if not dataRef.datasetExists("calexp_md"): raise RuntimeError("no data for calexp_md %s" % (dataId)) calexp_md = dataRef.get("calexp_md", immediate=True) detector = dataRef.get("camera")[dataRef.dataId["ccd"]] # OK for HSC; maybe not for other cameras wcs = afwGeom.makeSkyWcs(calexp_md) nQuarter = detector.getOrientation().getNQuarter() sources = dataRef.get("src", immediate=True, flags=afwTable.SOURCE_IO_NO_FOOTPRINTS) # Check if we are looking at HSC stack outputs: if so, no pixel rotation of sources is # required, but alias mapping must be set to associate HSC's schema with that of LSST. hscRun = mosaicUtils.checkHscStack(calexp_md) if hscRun is None: if nQuarter%4 != 0: dims = afwImage.bboxFromMetadata(calexp_md).getDimensions() sources = mosaicUtils.rotatePixelCoords(sources, dims.getX(), dims.getY(), nQuarter) # Set some alias maps for the source catalog where needed for # backwards compatibility if self.config.srcSchemaMap and hscRun: aliasMap = sources.schema.getAliasMap() for lsstName, otherName in self.config.srcSchemaMap.items(): aliasMap.set(lsstName, otherName) if self.config.flagsToAlias and "calib_psfUsed" in sources.schema: aliasMap = sources.schema.getAliasMap() for lsstName, otherName in self.config.flagsToAlias.items(): aliasMap.set(lsstName, otherName) refObjLoader = self.config.loadAstrom.apply(butler=dataRef.getButler()) srcMatch = dataRef.get("srcMatch", immediate=True) if hscRun is not None: # The reference object loader grows the bbox by the config parameter pixelMargin. This # is set to 50 by default but is not reflected by the radius parameter set in the # metadata, so some matches may reside outside the circle searched within this radius # Thus, increase the radius set in the metadata fed into joinMatchListWithCatalog() to # accommodate. matchmeta = srcMatch.table.getMetadata() rad = matchmeta.getDouble("RADIUS") matchmeta.setDouble("RADIUS", rad*1.05, "field radius in degrees, approximate, padded") matches = refObjLoader.joinMatchListWithCatalog(srcMatch, sources) # Set the aliap map for the matched sources (i.e. the [1] attribute schema for each match) if self.config.srcSchemaMap is not None and hscRun is not None: for mm in matches: aliasMap = mm[1].schema.getAliasMap() for lsstName, otherName in self.config.srcSchemaMap.items(): aliasMap.set(lsstName, otherName) if hscRun is not None: for slot in ("PsfFlux", "ModelFlux", "ApFlux", "GaussianFlux", "Centroid", "Shape"): getattr(matches[0][1].getTable(), "define" + slot)( getattr(sources, "get" + slot + "Definition")()) # For some reason, the CalibFlux slot in sources is coming up as centroid_sdss, so # set it to flux_naive explicitly for slot in ("CalibFlux", ): getattr(matches[0][1].getTable(), "define" + slot)("flux_naive") matches = [m for m in matches if m[0] is not None] refSchema = matches[0][0].schema if matches else None if self.cterm is not None and len(matches) != 0: # Add a "flux" field to the input schema of the first element # of the match and populate it with a colorterm correct flux. mapper = afwTable.SchemaMapper(refSchema) for key, field in refSchema: mapper.addMapping(key) fluxKey = mapper.editOutputSchema().addField("flux", type=float, doc="Reference flux") fluxErrKey = mapper.editOutputSchema().addField("fluxErr", type=float, doc="Reference flux uncertainty") table = afwTable.SimpleTable.make(mapper.getOutputSchema()) table.preallocate(len(matches)) for match in matches: newMatch = table.makeRecord() newMatch.assign(match[0], mapper) match[0] = newMatch # extract the matched refCat as a Catalog for the colorterm code refCat = afwTable.SimpleCatalog(matches[0].first.schema) refCat.reserve(len(matches)) for x in matches: record = refCat.addNew() record.assign(x.first) refMag, refMagErr = self.cterm.getCorrectedMagnitudes(refCat, afwImage.Filter(calexp_md).getName()) # NOTE: mosaic assumes fluxes are in Jy refFlux = (refMag*astropy.units.ABmag).to_value(astropy.units.Jy) refFluxErr = afwImage.fluxErrFromABMagErr(refMagErr, refMag) matches = [self.setCatFlux(m, flux, fluxKey, fluxErr, fluxErrKey) for m, flux, fluxErr in zip(matches, refFlux, refFluxErr) if flux == flux] else: filterName = afwImage.Filter(calexp_md).getName() refFluxField = measAlg.getRefFluxField(refSchema, filterName) refSchema.getAliasMap().set("flux", refFluxField) # LSST reads in a_net catalogs with flux in "janskys", so must convert back to DN. matches = mosaicUtils.matchJanskyToDn(matches) selSources = self.selectStars(sources, self.config.includeSaturated) selMatches = self.selectStars(matches, self.config.includeSaturated) retSrc = list() retMatch = list() if len(selMatches) > self.config.minNumMatch: naxis1, naxis2 = afwImage.bboxFromMetadata(calexp_md).getDimensions() if hscRun is None: if nQuarter%2 != 0: naxis1, naxis2 = naxis2, naxis1 bbox = afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(naxis1, naxis2)) cellSet = afwMath.SpatialCellSet(bbox, self.config.cellSize, self.config.cellSize) for s in selSources: if numpy.isfinite(s.getRa().asDegrees()): # get rid of NaN src = measMosaic.Source(s) src.setExp(dataId["visit"]) src.setChip(dataId["ccd"]) try: tmp = measMosaic.SpatialCellSource(src) cellSet.insertCandidate(tmp) except: self.log.info("FAILED TO INSERT CANDIDATE: visit=%d ccd=%d x=%f y=%f" % (dataRef.dataId["visit"], dataRef.dataId["ccd"], src.getX(), src.getY()) + " bbox=" + str(bbox)) for cell in cellSet.getCellList(): cell.sortCandidates() for i, cand in enumerate(cell): src = cand.getSource() retSrc.append(src) if i == self.config.nStarPerCell - 1: break for m in selMatches: if m[0] is not None and m[1] is not None: match = (measMosaic.Source(m[0], wcs), measMosaic.Source(m[1])) match[1].setExp(dataId["visit"]) match[1].setChip(dataId["ccd"]) retMatch.append(match) else: self.log.info("%8d %3d : %d/%d matches Suspicious to wrong match. Ignore this CCD" % (dataRef.dataId["visit"], dataRef.dataId["ccd"], len(selMatches), len(matches))) except Exception as e: self.log.warn("Failed to read %s: %s" % (dataId, e)) return dataId, [None, None, None] return dataId, [retSrc, retMatch, wcs]
def run(visit, rerun, config): mapper = getMapper() dataId = {'visit': visit, 'rerun': rerun} rrdir = mapper.getPath('outdir', dataId) if not os.path.exists(rrdir): print('Creating directory for ouputs:', rrdir) os.makedirs(rrdir) else: print('Output directory:', rrdir) io = pipReadWrite.ReadWrite(mapper, ['visit'], config=config) # ccdProc = pipProcCcd.ProcessCcd(config=config, Isr=NullISR, Calibrate=MyCalibrate) # raws = io.readRaw(dataId) # detrends = io.detrends(dataId, config) print('Reading exposure') # exposure = io.read('visitim', dataId) exposure = io.inButler.get('visitim', dataId) print('exposure is', exposure) print('size', exposure.getWidth(), 'x', exposure.getHeight()) # debug # mi = exposure.getMaskedImage() # img = mi.getImage() # var = mi.getVariance() # print('var at 90,100 is', var.get(90,100)) # print('img at 90,100 is', img.get(90,100)) # print('wcs is', exposure.getWcs()) wcs = exposure.getWcs() assert wcs # print('ccdProc.run()...') # raws = [exposure] # exposure, psf, apcorr, brightSources, sources, matches, matchMeta = ccdProc.run(raws, detrends) print('Calibrate()...') log = Log.getDefaultLogger() cal = MyCalibrate(config=config, log=log, Photometry=MyPhotometry) psf, sources, footprints = cal.run2(exposure) print('Photometry()...') phot = pipPhot.Photometry(config=config, log=log) sources, footprints = phot.run(exposure, psf) print('sources:', len(sources)) for s in sources: print(' ', s, s.getXAstrom(), s.getYAstrom(), s.getPsfFlux(), s.getIxx(), s.getIyy(), s.getIxy()) print('footprints:', footprints) # oh yeah, baby! fps = footprints.getFootprints() print(len(fps)) bb = [] for f in fps: print(' Footprint', f) print(' ', f.getBBox()) bbox = f.getBBox() bb.append( (bbox.getMinX(), bbox.getMinY(), bbox.getMaxX(), bbox.getMaxY())) print(' # peaks:', len(f.getPeaks())) for p in f.getPeaks(): print(' Peak', p) # print('psf', psf) # print('sources', sources) # print('footprints', footprints) # psf, apcorr, brightSources, matches, matchMeta = self.calibrate(exposure, defects=defects) # if self.config['do']['phot']: # sources, footprints = self.phot(exposure, psf, apcorr, wcs=exposure.getWcs()) # psf, wcs = self.fakePsf(exposure) # sources, footprints = self.phot(exposure, psf) # sources = self.rephot(exposure, footprints, psf, apcorr=apcorr) # model = calibrate['model'] # fwhm = calibrate['fwhm'] / wcs.pixelScale() # size = calibrate['size'] # psf = afwDet.createPsf(model, size, size, fwhm/(2*math.sqrt(2*math.log(2)))) # print('done!') print('writing output...') io.write(dataId, psf=psf, sources=sources) print('done!') print('Writing bounding-boxes...') io.outButler.put(bb, 'bb', dataId) # print('Writing footprints...') # io.outButler.put(fps, 'footprints', dataId) # serialize a python version of footprints & peaks; # commented out because footprintsToPython does not exist # pyfoots = footprintsToPython(fps) # print('Writing py footprints...') # io.outButler.put(pyfoots, 'pyfoots', dataId) return bb
def main(): from optparse import OptionParser parser = OptionParser(usage='%(program) [args] RA Dec radius') parser.add_option('-o', dest='outfn', help='FITS table output filename', default=None) (opt, args) = parser.parse_args() if len(args) != 3: parser.print_help() return -1 ra = float(args[0]) dec = float(args[1]) radius = float(args[2]) log = Log.getDefaultLogger() log.setLevel(Log.DEBUG) pol = policy.Policy() pol.set('matchThreshold', 30) solver = measAstrom.createSolver(pol, log) solver.setLogLevel(3) ids = solver.getIndexIdList() print('Index IDs:', ids) indexid = ids[0] idName = 'id' X = solver.getCatalogue(ra * afwGeom.degrees, dec * afwGeom.degrees, radius * afwGeom.degrees, '', idName, indexid) ref = X.refsources inds = X.inds print('Got', len(ref), 'reference catalog sources') print(' got indices:', len(inds)) print('Tag-along columns:') cols = solver.getTagAlongColumns(indexid) # print cols for c in cols: print(' column:', c.name, c.fitstype, c.ctype, c.units, c.arraysize) colnames = [c.name for c in cols] tagdata = [] for c in cols: fname = 'getTagAlong' + c.ctype func = getattr(solver, fname) data = func(indexid, c.name, inds) # print 'called', fname, 'to get', c.name, c.ctype, '(len %i)' % len(data) tagdata.append(data) if opt.outfn is None: # SSV print('ra dec', end=' ') for c in cols: if c.arraysize > 1: for a in len(c.arraysize): print(('%s_%i' % (c.name, a)), end=' ') else: print(c.name, end=' ') print() for i, r in enumerate(ref): print(r.getRa().asDegrees(), r.getDec().asDegrees(), end=' ') for c, d in zip(cols, tagdata): if c.arraysize > 1: for a in len(c.arraysize): print(d[c.arraysize * i + a], end=' ') else: print(d[i], end=' ') print() else: import pyfits import numpy as np fitscols = [] fitscols.append( pyfits.Column(name='RA', array=np.array([r.getRa().asDegrees() for r in ref]), format='D', unit='deg')) fitscols.append( pyfits.Column(name='DEC', array=np.array([r.getDec().asDegrees() for r in ref]), format='D', unit='deg')) for c, d in zip(cols, tagdata): fmap = { 'Int64': 'K', 'Int': 'J', 'Bool': 'L', 'Double': 'D', } if c.arraysize > 1: # May have to reshape the array as well... fitscols.append( pyfits.Column(name=c.name, array=np.array(d), format='%i%s' % (c.arraysize, fmap.get(c.ctype, 'D')))) else: fitscols.append( pyfits.Column(name=c.name, array=np.array(d), format=fmap.get(c.ctype, 'D'))) pyfits.new_table(fitscols).writeto(opt.outfn, clobber=True) print('Wrote FITS table', opt.outfn) return 0
def readSrc(self, dataRef): """Read source catalog etc for input dataRef The following are returned: Source catalog, matched list, and wcs will be read from 'src', 'srcMatch', and 'calexp_md', respectively. NOTE: If the detector has nQuarter%4 != 0 (i.e. it is rotated w.r.t the focal plane coordinate system), the (x, y) pixel values of the centroid slot for the source catalogs are rotated such that pixel (0, 0) is the LLC (i.e. the coordinate system expected by meas_mosaic). If color transformation information is given, it will be applied to the reference flux of the matched list. The source catalog and matched list will be converted to measMosaic's Source and SourceMatch and returned. The number of 'Source's in each cell defined by config.cellSize will be limited to brightest config.nStarPerCell. """ self.log = Log.getDefaultLogger() dataId = dataRef.dataId try: if not dataRef.datasetExists("src"): raise RuntimeError("no data for src %s" % (dataId)) if not dataRef.datasetExists("calexp_md"): raise RuntimeError("no data for calexp_md %s" % (dataId)) calexp_md = dataRef.get("calexp_md", immediate=True) detector = dataRef.get("camera")[dataRef.dataId[ "ccd"]] # OK for HSC; maybe not for other cameras wcs = afwGeom.makeSkyWcs(calexp_md) nQuarter = detector.getOrientation().getNQuarter() sources = dataRef.get("src", immediate=True, flags=afwTable.SOURCE_IO_NO_FOOTPRINTS) # Check if we are looking at HSC stack outputs: if so, no pixel rotation of sources is # required, but alias mapping must be set to associate HSC's schema with that of LSST. hscRun = mosaicUtils.checkHscStack(calexp_md) if hscRun is None: if nQuarter % 4 != 0: dims = afwImage.bboxFromMetadata(calexp_md).getDimensions() sources = mosaicUtils.rotatePixelCoords( sources, dims.getX(), dims.getY(), nQuarter) # Set the aliap map for the source catalog if self.config.srcSchemaMap is not None and hscRun is not None: aliasMap = sources.schema.getAliasMap() for lsstName, otherName in self.config.srcSchemaMap.items(): aliasMap.set(lsstName, otherName) refObjLoader = self.config.loadAstrom.apply( butler=dataRef.getButler()) srcMatch = dataRef.get("srcMatch", immediate=True) if hscRun is not None: # The reference object loader grows the bbox by the config parameter pixelMargin. This # is set to 50 by default but is not reflected by the radius parameter set in the # metadata, so some matches may reside outside the circle searched within this radius # Thus, increase the radius set in the metadata fed into joinMatchListWithCatalog() to # accommodate. matchmeta = srcMatch.table.getMetadata() rad = matchmeta.getDouble("RADIUS") matchmeta.setDouble( "RADIUS", rad * 1.05, "field radius in degrees, approximate, padded") matches = refObjLoader.joinMatchListWithCatalog(srcMatch, sources) # Set the aliap map for the matched sources (i.e. the [1] attribute schema for each match) if self.config.srcSchemaMap is not None and hscRun is not None: for mm in matches: aliasMap = mm[1].schema.getAliasMap() for lsstName, otherName in self.config.srcSchemaMap.items( ): aliasMap.set(lsstName, otherName) if hscRun is not None: for slot in ("PsfFlux", "ModelFlux", "ApFlux", "InstFlux", "Centroid", "Shape"): getattr(matches[0][1].getTable(), "define" + slot)(getattr( sources, "get" + slot + "Definition")()) # For some reason, the CalibFlux slot in sources is coming up as centroid_sdss, so # set it to flux_naive explicitly for slot in ("CalibFlux", ): getattr(matches[0][1].getTable(), "define" + slot)("flux_naive") matches = [m for m in matches if m[0] is not None] refSchema = matches[0][0].schema if matches else None if self.cterm is not None and len(matches) != 0: # Add a "flux" field to the input schema of the first element # of the match and populate it with a colorterm correct flux. mapper = afwTable.SchemaMapper(refSchema) for key, field in refSchema: mapper.addMapping(key) fluxKey = mapper.editOutputSchema().addField( "flux", type=float, doc="Reference flux") fluxSigmaKey = mapper.editOutputSchema().addField( "fluxSigma", type=float, doc="Reference flux uncertainty") table = afwTable.SimpleTable.make(mapper.getOutputSchema()) table.preallocate(len(matches)) for match in matches: newMatch = table.makeRecord() newMatch.assign(match[0], mapper) match[0] = newMatch primaryFluxKey = refSchema.find( refSchema.join(self.cterm.primary, "flux")).key secondaryFluxKey = refSchema.find( refSchema.join(self.cterm.secondary, "flux")).key primaryFluxSigmaKey = refSchema.find( refSchema.join(self.cterm.primary, "fluxSigma")).key secondaryFluxSigmaKey = refSchema.find( refSchema.join(self.cterm.secondary, "fluxSigma")).key refFlux1 = numpy.array( [m[0].get(primaryFluxKey) for m in matches]) refFlux2 = numpy.array( [m[0].get(secondaryFluxKey) for m in matches]) refFluxSigma1 = numpy.array( [m[0].get(primaryFluxSigmaKey) for m in matches]) refFluxSigma2 = numpy.array( [m[0].get(secondaryFluxSigmaKey) for m in matches]) refMag1 = -2.5 * numpy.log10(refFlux1) refMag2 = -2.5 * numpy.log10(refFlux2) refMag = self.cterm.transformMags(refMag1, refMag2) refFlux = numpy.power(10.0, -0.4 * refMag) refFluxSigma = self.cterm.propagateFluxErrors( refFluxSigma1, refFluxSigma2) matches = [ self.setCatFlux(m, flux, fluxKey, fluxSigma, fluxSigmaKey) for m, flux, fluxSigma in zip(matches, refFlux, refFluxSigma) if flux == flux ] else: filterName = afwImage.Filter(calexp_md).getName() refFluxField = measAlg.getRefFluxField(refSchema, filterName) refSchema.getAliasMap().set("flux", refFluxField) # LSST reads in a_net catalogs with flux in "janskys", so must convert back to DN. matches = mosaicUtils.matchJanskyToDn(matches) selSources = self.selectStars(sources, self.config.includeSaturated) selMatches = self.selectStars(matches, self.config.includeSaturated) retSrc = list() retMatch = list() if len(selMatches) > self.config.minNumMatch: naxis1, naxis2 = afwImage.bboxFromMetadata( calexp_md).getDimensions() if hscRun is None: if nQuarter % 2 != 0: naxis1, naxis2 = naxis2, naxis1 bbox = afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(naxis1, naxis2)) cellSet = afwMath.SpatialCellSet(bbox, self.config.cellSize, self.config.cellSize) for s in selSources: if numpy.isfinite(s.getRa().asDegrees()): # get rid of NaN src = measMosaic.Source(s) src.setExp(dataId["visit"]) src.setChip(dataId["ccd"]) try: tmp = measMosaic.SpatialCellSource(src) cellSet.insertCandidate(tmp) except: self.log.info( "FAILED TO INSERT CANDIDATE: visit=%d ccd=%d x=%f y=%f" % (dataRef.dataId["visit"], dataRef. dataId["ccd"], src.getX(), src.getY()) + " bbox=" + str(bbox)) for cell in cellSet.getCellList(): cell.sortCandidates() for i, cand in enumerate(cell): src = cand.getSource() retSrc.append(src) if i == self.config.nStarPerCell - 1: break for m in selMatches: if m[0] is not None and m[1] is not None: match = (measMosaic.Source(m[0], wcs), measMosaic.Source(m[1])) match[1].setExp(dataId["visit"]) match[1].setChip(dataId["ccd"]) retMatch.append(match) else: self.log.info( "%8d %3d : %d/%d matches Suspicious to wrong match. Ignore this CCD" % (dataRef.dataId["visit"], dataRef.dataId["ccd"], len(selMatches), len(matches))) except Exception as e: self.log.warn("Failed to read %s: %s" % (dataId, e)) return dataId, [None, None, None] return dataId, [retSrc, retMatch, wcs]
def plotsForField(inButler, keys, fixup, plots=None, prefix=''): if plots is None: plots = ['photom', 'matches', 'corr', 'distortion'] filters = inButler.queryMetadata('raw', 'filter', **keys) print('Filters:', filters) filterName = filters[0] psources = inButler.get('icSrc', **keys) # since the butler does lazy evaluation, we don't know if it fails until... try: print('Got sources', psources) except: print('"icSrc" not found. Trying "src" instead.') psources = inButler.get('src', **keys) print('Got sources', psources) pmatches = inButler.get('icMatch', **keys) sources = psources.getSources() calexp = inButler.get('calexp', **keys) wcs = calexp.getWcs() photocal = calexp.getCalib() zp = photocal.getMagnitude(1.) print('Zeropoint is', zp) # ref sources W, H = calexp.getWidth(), calexp.getHeight() log = Log.getDefaultLogger() log.setLevel(Log.DEBUG) kwargs = {} if fixup: # ugh, mask and offset req'd because source ids are assigned at write-time # and match list code made a deep copy before that. # (see svn+ssh://svn.lsstcorp.org/DMS/meas/astrom/tickets/1491-b r18027) kwargs['sourceIdMask'] = 0xffff kwargs['sourceIdOffset'] = -1 (matches, ref) = measAstrom.generateMatchesFromMatchList( pmatches, sources, wcs, W, H, returnRefs=True, log=log, **kwargs) print('Got', len(ref), 'reference catalog sources') # pull 'stargal' and 'referrs' arrays out of the reference sources fdict = maUtils.getDetectionFlags() starflag = int(fdict["STAR"]) stargal = [bool((r.getFlagForDetection() & starflag) > 0) for r in ref] referrs = [float(r.getPsfFluxErr() / r.getPsfFlux() * 2.5 / -np.log(10)) for r in ref] nstars = sum([1 for s in stargal if s]) print('Number of sources with STAR set:', nstars) visit = keys['visit'] raft = keys['raft'] sensor = keys['sensor'] prefix += 'imsim-v%i-r%s-s%s' % (visit, raft.replace(',', ''), sensor.replace(',', '')) if 'photom' in plots: print('photometry plots...') tt = 'LSST ImSim v%i r%s s%s' % (visit, raft.replace(',', ''), sensor.replace(',', '')) wcsPlots.plotPhotometry(sources, ref, matches, prefix, band=filterName, zp=zp, referrs=referrs, refstargal=stargal, title=tt) wcsPlots.plotPhotometry(sources, ref, matches, prefix, band=filterName, zp=zp, delta=True, referrs=referrs, refstargal=stargal, title=tt) # test w/ and w/o referrs and stargal. if False: wcsPlots.plotPhotometry(sources, ref, matches, prefix + 'A', band=filterName, zp=zp, title=tt) wcsPlots.plotPhotometry(sources, ref, matches, prefix + 'B', band=filterName, zp=zp, referrs=referrs, title=tt) wcsPlots.plotPhotometry(sources, ref, matches, prefix + 'C', band=filterName, zp=zp, refstargal=stargal, title=tt) wcsPlots.plotPhotometry(sources, ref, matches, prefix + 'A', band=filterName, zp=zp, delta=True, title=tt) wcsPlots.plotPhotometry(sources, ref, matches, prefix + 'B', band=filterName, zp=zp, delta=True, referrs=referrs, title=tt) wcsPlots.plotPhotometry(sources, ref, matches, prefix + 'C', band=filterName, zp=zp, delta=True, refstargal=stargal, title=tt) if 'matches' in plots: print('matches...') wcsPlots.plotMatches(sources, ref, matches, wcs, W, H, prefix) if 'distortion' in plots: print('distortion...') wcsPlots.plotDistortion(wcs, W, H, 400, prefix, 'SIP Distortion (exaggerated x 10)', exaggerate=10.) print('distortion...') wcsPlots.plotDistortion(wcs, W, H, 400, prefix, 'SIP Distortion (exaggerated x 100)', exaggerate=100., suffix='-distort2.')
# the GNU General Public License along with this program. If not, # see <http://www.lsstcorp.org/LegalNotices/>. # import math import sys import os import time import lsst.utils import lsst.geom import lsst.afw.image as afwImage import lsst.afw.math as afwMath from lsst.log import Log Log.getDefaultLogger().setLevel(Log.INFO) Log.getLogger("TRACE2.afw.math.convolve").setLevel(Log.DEBUG) MaxIter = 20 MaxTime = 1.0 # seconds afwdataDir = lsst.utils.getPackageDir("afwdata") InputMaskedImagePath = os.path.join(afwdataDir, "data", "med.fits") def getSpatialParameters(nKernelParams, func): """Get basic spatial parameters list You may wish to tweak it up for specific cases (especially the lower order terms) """
def run(visit, rerun, config): mapper = getMapper() dataId = {'visit': visit, 'rerun': rerun} rrdir = mapper.getPath('outdir', dataId) if not os.path.exists(rrdir): print('Creating directory for ouputs:', rrdir) os.makedirs(rrdir) else: print('Output directory:', rrdir) io = pipReadWrite.ReadWrite(mapper, ['visit'], config=config) # ccdProc = pipProcCcd.ProcessCcd(config=config, Isr=NullISR, Calibrate=MyCalibrate) # raws = io.readRaw(dataId) # detrends = io.detrends(dataId, config) print('Reading exposure') # exposure = io.read('visitim', dataId) exposure = io.inButler.get('visitim', dataId) print('exposure is', exposure) print('size', exposure.getWidth(), 'x', exposure.getHeight()) # debug # mi = exposure.getMaskedImage() # img = mi.getImage() # var = mi.getVariance() # print('var at 90,100 is', var.get(90,100)) # print('img at 90,100 is', img.get(90,100)) # print('wcs is', exposure.getWcs()) wcs = exposure.getWcs() assert wcs # print('ccdProc.run()...') # raws = [exposure] # exposure, psf, apcorr, brightSources, sources, matches, matchMeta = ccdProc.run(raws, detrends) print('Calibrate()...') log = Log.getDefaultLogger() cal = MyCalibrate(config=config, log=log, Photometry=MyPhotometry) psf, sources, footprints = cal.run2(exposure) print('Photometry()...') phot = pipPhot.Photometry(config=config, log=log) sources, footprints = phot.run(exposure, psf) print('sources:', len(sources)) for s in sources: print(' ', s, s.getXAstrom(), s.getYAstrom(), s.getPsfFlux(), s.getIxx(), s.getIyy(), s.getIxy()) print('footprints:', footprints) # oh yeah, baby! fps = footprints.getFootprints() print(len(fps)) bb = [] for f in fps: print(' Footprint', f) print(' ', f.getBBox()) bbox = f.getBBox() bb.append((bbox.getMinX(), bbox.getMinY(), bbox.getMaxX(), bbox.getMaxY())) print(' # peaks:', len(f.getPeaks())) for p in f.getPeaks(): print(' Peak', p) # print('psf', psf) # print('sources', sources) # print('footprints', footprints) # psf, apcorr, brightSources, matches, matchMeta = self.calibrate(exposure, defects=defects) # if self.config['do']['phot']: # sources, footprints = self.phot(exposure, psf, apcorr, wcs=exposure.getWcs()) # psf, wcs = self.fakePsf(exposure) # sources, footprints = self.phot(exposure, psf) # sources = self.rephot(exposure, footprints, psf, apcorr=apcorr) # model = calibrate['model'] # fwhm = calibrate['fwhm'] / wcs.pixelScale() # size = calibrate['size'] # psf = afwDet.createPsf(model, size, size, fwhm/(2*math.sqrt(2*math.log(2)))) # print('done!') print('writing output...') io.write(dataId, psf=psf, sources=sources) print('done!') print('Writing bounding-boxes...') io.outButler.put(bb, 'bb', dataId) # print('Writing footprints...') # io.outButler.put(fps, 'footprints', dataId) # serialize a python version of footprints & peaks; # commented out because footprintsToPython does not exist # pyfoots = footprintsToPython(fps) # print('Writing py footprints...') # io.outButler.put(pyfoots, 'pyfoots', dataId) return bb
def plotPixelResiduals(exposure, warpedTemplateExposure, diffExposure, kernelCellSet, kernel, background, testSources, config, origVariance=False, nptsFull=1e6, keepPlots=True, titleFs=14): """Plot diffim residuals for LOCAL and SPATIAL models. """ candidateResids = [] spatialResids = [] nonfitResids = [] for cell in kernelCellSet.getCellList(): for cand in cell.begin(True): # only look at good ones # Be sure if not (cand.getStatus() == afwMath.SpatialCellCandidate.GOOD): continue diffim = cand.getDifferenceImage(diffimLib.KernelCandidateF.ORIG) orig = cand.getScienceMaskedImage() ski = afwImage.ImageD(kernel.getDimensions()) kernel.computeImage(ski, False, int(cand.getXCenter()), int(cand.getYCenter())) sk = afwMath.FixedKernel(ski) sbg = background(int(cand.getXCenter()), int(cand.getYCenter())) sdiffim = cand.getDifferenceImage(sk, sbg) # trim edgs due to convolution bbox = kernel.shrinkBBox(diffim.getBBox()) tdiffim = diffim.Factory(diffim, bbox) torig = orig.Factory(orig, bbox) tsdiffim = sdiffim.Factory(sdiffim, bbox) if origVariance: candidateResids.append(np.ravel(tdiffim.getImage().getArray() / np.sqrt(torig.getVariance().getArray()))) spatialResids.append(np.ravel(tsdiffim.getImage().getArray() / np.sqrt(torig.getVariance().getArray()))) else: candidateResids.append(np.ravel(tdiffim.getImage().getArray() / np.sqrt(tdiffim.getVariance().getArray()))) spatialResids.append(np.ravel(tsdiffim.getImage().getArray() / np.sqrt(tsdiffim.getVariance().getArray()))) fullIm = diffExposure.getMaskedImage().getImage().getArray() fullMask = diffExposure.getMaskedImage().getMask().getArray() if origVariance: fullVar = exposure.getMaskedImage().getVariance().getArray() else: fullVar = diffExposure.getMaskedImage().getVariance().getArray() bitmaskBad = 0 bitmaskBad |= afwImage.Mask.getPlaneBitMask('NO_DATA') bitmaskBad |= afwImage.Mask.getPlaneBitMask('SAT') idx = np.where((fullMask & bitmaskBad) == 0) stride = int(len(idx[0])//nptsFull) sidx = idx[0][::stride], idx[1][::stride] allResids = fullIm[sidx]/np.sqrt(fullVar[sidx]) testFootprints = diffimTools.sourceToFootprintList(testSources, warpedTemplateExposure, exposure, config, Log.getDefaultLogger()) for fp in testFootprints: subexp = diffExposure.Factory(diffExposure, fp["footprint"].getBBox()) subim = subexp.getMaskedImage().getImage() if origVariance: subvar = afwImage.ExposureF(exposure, fp["footprint"].getBBox()).getMaskedImage().getVariance() else: subvar = subexp.getMaskedImage().getVariance() nonfitResids.append(np.ravel(subim.getArray()/np.sqrt(subvar.getArray()))) candidateResids = np.ravel(np.array(candidateResids)) spatialResids = np.ravel(np.array(spatialResids)) nonfitResids = np.ravel(np.array(nonfitResids)) try: import pylab from matplotlib.font_manager import FontProperties except ImportError as e: print("Unable to import pylab: %s" % e) return fig = pylab.figure() fig.clf() try: fig.canvas._tkcanvas._root().lift() # == Tk's raise, but raise is a python reserved word except Exception: # protect against API changes pass if origVariance: fig.suptitle("Diffim residuals: Normalized by sqrt(input variance)", fontsize=titleFs) else: fig.suptitle("Diffim residuals: Normalized by sqrt(diffim variance)", fontsize=titleFs) sp1 = pylab.subplot(221) sp2 = pylab.subplot(222, sharex=sp1, sharey=sp1) sp3 = pylab.subplot(223, sharex=sp1, sharey=sp1) sp4 = pylab.subplot(224, sharex=sp1, sharey=sp1) xs = np.arange(-5, 5.05, 0.1) ys = 1./np.sqrt(2*np.pi)*np.exp(-0.5*xs**2) sp1.hist(candidateResids, bins=xs, normed=True, alpha=0.5, label="N(%.2f, %.2f)" % (np.mean(candidateResids), np.var(candidateResids))) sp1.plot(xs, ys, "r-", lw=2, label="N(0,1)") sp1.set_title("Candidates: basis fit", fontsize=titleFs - 2) sp1.legend(loc=1, fancybox=True, shadow=True, prop=FontProperties(size=titleFs - 6)) sp2.hist(spatialResids, bins=xs, normed=True, alpha=0.5, label="N(%.2f, %.2f)" % (np.mean(spatialResids), np.var(spatialResids))) sp2.plot(xs, ys, "r-", lw=2, label="N(0,1)") sp2.set_title("Candidates: spatial fit", fontsize=titleFs - 2) sp2.legend(loc=1, fancybox=True, shadow=True, prop=FontProperties(size=titleFs - 6)) sp3.hist(nonfitResids, bins=xs, normed=True, alpha=0.5, label="N(%.2f, %.2f)" % (np.mean(nonfitResids), np.var(nonfitResids))) sp3.plot(xs, ys, "r-", lw=2, label="N(0,1)") sp3.set_title("Control sample: spatial fit", fontsize=titleFs - 2) sp3.legend(loc=1, fancybox=True, shadow=True, prop=FontProperties(size=titleFs - 6)) sp4.hist(allResids, bins=xs, normed=True, alpha=0.5, label="N(%.2f, %.2f)" % (np.mean(allResids), np.var(allResids))) sp4.plot(xs, ys, "r-", lw=2, label="N(0,1)") sp4.set_title("Full image (subsampled)", fontsize=titleFs - 2) sp4.legend(loc=1, fancybox=True, shadow=True, prop=FontProperties(size=titleFs - 6)) pylab.setp(sp1.get_xticklabels() + sp1.get_yticklabels(), fontsize=titleFs - 4) pylab.setp(sp2.get_xticklabels() + sp2.get_yticklabels(), fontsize=titleFs - 4) pylab.setp(sp3.get_xticklabels() + sp3.get_yticklabels(), fontsize=titleFs - 4) pylab.setp(sp4.get_xticklabels() + sp4.get_yticklabels(), fontsize=titleFs - 4) sp1.set_xlim(-5, 5) sp1.set_ylim(0, 0.5) fig.show() global keptPlots if keepPlots and not keptPlots: # Keep plots open when done def show(): print("%s: Please close plots when done." % __name__) try: pylab.show() except Exception: pass print("Plots closed, exiting...") import atexit atexit.register(show) keptPlots = True
def __call__(self, args): """Run the Task on a single target. Parameters ---------- args Arguments for Task.runDataRef() Returns ------- struct : `lsst.pipe.base.Struct` Contains these fields if ``doReturnResults`` is `True`: - ``dataRef``: the provided data reference. - ``metadata``: task metadata after execution of run. - ``result``: result returned by task run, or `None` if the task fails. - ``exitStatus``: 0 if the task completed successfully, 1 otherwise. If ``doReturnResults`` is `False` the struct contains: - ``exitStatus``: 0 if the task completed successfully, 1 otherwise. Notes ----- This default implementation assumes that the ``args`` is a tuple containing a data reference and a dict of keyword arguments. .. warning:: If you override this method and wish to return something when ``doReturnResults`` is `False`, then it must be picklable to support multiprocessing and it should be small enough that pickling and unpickling do not add excessive overhead. """ dataRef, kwargs = args if self.log is None: self.log = Log.getDefaultLogger() if hasattr(dataRef, "dataId"): self.log.MDC("LABEL", str(dataRef.dataId)) elif isinstance(dataRef, (list, tuple)): self.log.MDC("LABEL", str([ref.dataId for ref in dataRef if hasattr(ref, "dataId")])) task = self.makeTask(args=args) result = None # in case the task fails exitStatus = 0 # exit status for the shell if self.doRaise: result = self.runTask(task, dataRef, kwargs) else: try: result = self.runTask(task, dataRef, kwargs) except Exception as e: # The shell exit value will be the number of dataRefs returning # non-zero, so the actual value used here is lost. exitStatus = 1 # don't use a try block as we need to preserve the original exception eName = type(e).__name__ if hasattr(dataRef, "dataId"): task.log.fatal("Failed on dataId=%s: %s: %s", dataRef.dataId, eName, e) elif isinstance(dataRef, (list, tuple)): task.log.fatal("Failed on dataIds=[%s]: %s: %s", ", ".join(str(ref.dataId) for ref in dataRef), eName, e) else: task.log.fatal("Failed on dataRef=%s: %s: %s", dataRef, eName, e) if not isinstance(e, TaskError): traceback.print_exc(file=sys.stderr) # Ensure all errors have been logged and aren't hanging around in a buffer sys.stdout.flush() sys.stderr.flush() task.writeMetadata(dataRef) # remove MDC so it does not show up outside of task context self.log.MDCRemove("LABEL") if self.doReturnResults: return Struct( exitStatus=exitStatus, dataRef=dataRef, metadata=task.metadata, result=result, ) else: return Struct( exitStatus=exitStatus, )
def test_basic(self): """Perform basic correctness testing.""" ps = [] # Construct property sets for two exposures centered on the equator for center in ((0.0, 0.0), (180.0, 0.0)): props = daf_base.PropertySet() props.add("NAXIS1", 9) props.add("NAXIS2", 9) props.add("RADECSYS", "ICRS") props.add("EQUINOX", 2000.0) props.add("CTYPE1", "RA---TAN") props.add("CTYPE2", "DEC--TAN") props.add("CRPIX1", 5.0) props.add("CRPIX2", 5.0) props.add("CRVAL1", center[0]) props.add("CRVAL2", center[1]) props.add("CD1_1", 1.0) props.add("CD2_1", 0.0) props.add("CD1_2", 0.0) props.add("CD2_2", 1.0) ps.append(props) # Retain one as is, and create an exposure from the other inputs = [ ps[0], afw_image.ExposureF(8, 8, afw_image.makeWcs(ps[1])) ] # Test data-ids are just integers. refs = [MockDataRef(i, v) for i, v in enumerate(inputs)] config = IndexExposureConfig() config.allow_replace = True config.defer_writes = True config.init_statements = ['PRAGMA page_size = 4096'] database = sqlite3.connect(":memory:") # Avoid the command line parser. parsed_cmd = pipe_base.Struct( config=config, log=Log.getDefaultLogger(), doraise=True, clobberConfig=False, noBackupConfig=False, database=database, dstype="bogus", id=pipe_base.Struct(refList=refs), ) runner = IndexExposureRunner(IndexExposureTask, parsed_cmd) runner.run(parsed_cmd) # Re-ingest to test that allow_replace=True works. Toggle off # the deferred writes to test that as well. runner.config.defer_writes = False runner.run(parsed_cmd) # Re-ingest to test that allow_replace=False raises an exception. runner.config.allow_replace = False with self.assertRaises(Exception): runner.run(parsed_cmd) # Now, verify the contents of the database. First, check that # data ids are recoverable. data_ids = sorted(pickle.loads(str(r[0])) for r in database.execute( "SELECT pickled_data_id FROM exposure")) self.assertEqual(data_ids, [0, 1]) # Next, run a spatial query and check that it returns the # expected results. center = sphgeom.UnitVector3d(sphgeom.LonLat.fromDegrees(4.0, 1.0)) circle = sphgeom.Circle(center, sphgeom.Angle.fromDegrees(1.5)) results = find_intersecting_exposures(database, circle) self.assertEqual(len(results), 1) info = results[0] # The first input exposure should have been returned, and # should intersect the query region self.assertEqual(info.data_id, 0) self.assertEqual(circle.relate(info.boundary), sphgeom.INTERSECTS) database.close()
# the GNU General Public License along with this program. If not, # see <http://www.lsstcorp.org/LegalNotices/>. # import math import sys import os import time import lsst.utils import lsst.afw.image as afwImage import lsst.afw.math as afwMath import lsst.afw.geom as afwGeom from lsst.log import Log Log.getDefaultLogger().setLevel(Log.INFO) Log.getLogger("TRACE2.afw.math.convolve").setLevel(Log.DEBUG) MaxIter = 20 MaxTime = 1.0 # seconds afwdataDir = lsst.utils.getPackageDir("afwdata") InputMaskedImagePath = os.path.join(afwdataDir, "data", "med.fits") def getSpatialParameters(nKernelParams, func): """Get basic spatial parameters list You may wish to tweak it up for specific cases (especially the lower order terms) """
def plotsForField(inButler, keys, fixup, plots=None, prefix=''): if plots is None: plots = ['photom', 'matches', 'corr', 'distortion'] filters = inButler.queryMetadata('raw', 'filter', **keys) print('Filters:', filters) filterName = filters[0] try: psources = inButler.get('icSrc', **keys) print('Got sources', psources) except Exception: print('"icSrc" not found. Trying "src" instead.') psources = inButler.get('src', **keys) print('Got sources', psources) pmatches = inButler.get('icMatch', **keys) sources = psources.getSources() calexp = inButler.get('calexp', **keys) wcs = calexp.getWcs() photocal = calexp.getCalib() zp = photocal.getMagnitude(1.) print('Zeropoint is', zp) # ref sources W, H = calexp.getWidth(), calexp.getHeight() log = Log.getDefaultLogger() log.setLevel(Log.DEBUG) kwargs = {} if fixup: # ugh, mask and offset req'd because source ids are assigned at write-time # and match list code made a deep copy before that. # (see svn+ssh://svn.lsstcorp.org/DMS/meas/astrom/tickets/1491-b r18027) kwargs['sourceIdMask'] = 0xffff kwargs['sourceIdOffset'] = -1 (matches, ref) = measAstrom.generateMatchesFromMatchList( pmatches, sources, wcs, W, H, returnRefs=True, log=log, **kwargs) print('Got', len(ref), 'reference catalog sources') # pull 'stargal' and 'referrs' arrays out of the reference sources fdict = maUtils.getDetectionFlags() starflag = int(fdict["STAR"]) stargal = [bool((r.getFlagForDetection() & starflag) > 0) for r in ref] referrs = [float(r.getPsfInstFluxErr() / r.getPsfInstFlux() * 2.5 / -np.log(10)) for r in ref] nstars = sum([1 for s in stargal if s]) print('Number of sources with STAR set:', nstars) visit = keys['visit'] raft = keys['raft'] sensor = keys['sensor'] prefix += 'imsim-v%i-r%s-s%s' % (visit, raft.replace(',', ''), sensor.replace(',', '')) if 'photom' in plots: print('photometry plots...') tt = 'LSST ImSim v%i r%s s%s' % (visit, raft.replace(',', ''), sensor.replace(',', '')) wcsPlots.plotPhotometry(sources, ref, matches, prefix, band=filterName, zp=zp, referrs=referrs, refstargal=stargal, title=tt) wcsPlots.plotPhotometry(sources, ref, matches, prefix, band=filterName, zp=zp, delta=True, referrs=referrs, refstargal=stargal, title=tt) # test w/ and w/o referrs and stargal. if False: wcsPlots.plotPhotometry(sources, ref, matches, prefix + 'A', band=filterName, zp=zp, title=tt) wcsPlots.plotPhotometry(sources, ref, matches, prefix + 'B', band=filterName, zp=zp, referrs=referrs, title=tt) wcsPlots.plotPhotometry(sources, ref, matches, prefix + 'C', band=filterName, zp=zp, refstargal=stargal, title=tt) wcsPlots.plotPhotometry(sources, ref, matches, prefix + 'A', band=filterName, zp=zp, delta=True, title=tt) wcsPlots.plotPhotometry(sources, ref, matches, prefix + 'B', band=filterName, zp=zp, delta=True, referrs=referrs, title=tt) wcsPlots.plotPhotometry(sources, ref, matches, prefix + 'C', band=filterName, zp=zp, delta=True, refstargal=stargal, title=tt) if 'matches' in plots: print('matches...') wcsPlots.plotMatches(sources, ref, matches, wcs, W, H, prefix) if 'distortion' in plots: print('distortion...') wcsPlots.plotDistortion(wcs, W, H, 400, prefix, 'SIP Distortion (exaggerated x 10)', exaggerate=10.) print('distortion...') wcsPlots.plotDistortion(wcs, W, H, 400, prefix, 'SIP Distortion (exaggerated x 100)', exaggerate=100., suffix='-distort2.')
def __call__(self, args): """!Run the Task on a single target. This default implementation assumes that the 'args' is a tuple containing a data reference and a dict of keyword arguments. @warning if you override this method and wish to return something when doReturnResults is false, then it must be picklable to support multiprocessing and it should be small enough that pickling and unpickling do not add excessive overhead. @param args Arguments for Task.run() @return: - None if doReturnResults false - A pipe_base Struct containing these fields if doReturnResults true: - dataRef: the provided data reference - metadata: task metadata after execution of run - result: result returned by task run, or None if the task fails """ dataRef, kwargs = args if self.log is None: self.log = Log.getDefaultLogger() if hasattr(dataRef, "dataId"): self.log.MDC("LABEL", str(dataRef.dataId)) elif isinstance(dataRef, (list, tuple)): self.log.MDC( "LABEL", str([ref.dataId for ref in dataRef if hasattr(ref, "dataId")])) task = self.makeTask(args=args) result = None # in case the task fails exitStatus = 0 # exit status for the shell if self.doRaise: result = task.run(dataRef, **kwargs) else: try: result = task.run(dataRef, **kwargs) except Exception as e: exitStatus = 1 # n.b. The shell exit value is the number of dataRefs returning # non-zero, so the actual value used here is lost # don't use a try block as we need to preserve the original exception if hasattr(dataRef, "dataId"): task.log.fatal("Failed on dataId=%s: %s", dataRef.dataId, e) elif isinstance(dataRef, (list, tuple)): task.log.fatal( "Failed on dataId=[%s]: %s", ", ".join(str(ref.dataId) for ref in dataRef), e) else: task.log.fatal("Failed on dataRef=%s: %s", dataRef, e) if not isinstance(e, TaskError): traceback.print_exc(file=sys.stderr) task.writeMetadata(dataRef) # remove MDC so it does not show up outside of task context self.log.MDCRemove("LABEL") if self.doReturnResults: return Struct( exitStatus=exitStatus, dataRef=dataRef, metadata=task.metadata, result=result, ) else: return Struct(exitStatus=exitStatus, )
def printProcessStats(): """Print the process statistics to the log""" from lsst.log import Log log = Log.getDefaultLogger() log.info("Process stats for %s: %s" % (NODE, processStats()))
def __call__(self, args): """Run the Task on a single target. Parameters ---------- args Arguments for Task.runDataRef() Returns ------- struct : `lsst.pipe.base.Struct` Contains these fields if ``doReturnResults`` is `True`: - ``dataRef``: the provided data reference. - ``metadata``: task metadata after execution of run. - ``result``: result returned by task run, or `None` if the task fails. - ``exitStatus``: 0 if the task completed successfully, 1 otherwise. If ``doReturnResults`` is `False` the struct contains: - ``exitStatus``: 0 if the task completed successfully, 1 otherwise. Notes ----- This default implementation assumes that the ``args`` is a tuple containing a data reference and a dict of keyword arguments. .. warning:: If you override this method and wish to return something when ``doReturnResults`` is `False`, then it must be picklable to support multiprocessing and it should be small enough that pickling and unpickling do not add excessive overhead. """ dataRef, kwargs = args if self.log is None: self.log = Log.getDefaultLogger() if hasattr(dataRef, "dataId"): self.log.MDC("LABEL", str(dataRef.dataId)) elif isinstance(dataRef, (list, tuple)): self.log.MDC( "LABEL", str([ref.dataId for ref in dataRef if hasattr(ref, "dataId")])) task = self.makeTask(args=args) result = None # in case the task fails exitStatus = 0 # exit status for the shell if self.doRaise: result = self.runTask(task, dataRef, kwargs) else: try: result = self.runTask(task, dataRef, kwargs) except Exception as e: # The shell exit value will be the number of dataRefs returning # non-zero, so the actual value used here is lost. exitStatus = 1 # don't use a try block as we need to preserve the original exception eName = type(e).__name__ if hasattr(dataRef, "dataId"): task.log.fatal("Failed on dataId=%s: %s: %s", dataRef.dataId, eName, e) elif isinstance(dataRef, (list, tuple)): task.log.fatal( "Failed on dataIds=[%s]: %s: %s", ", ".join(str(ref.dataId) for ref in dataRef), eName, e) else: task.log.fatal("Failed on dataRef=%s: %s: %s", dataRef, eName, e) if not isinstance(e, TaskError): traceback.print_exc(file=sys.stderr) # Ensure all errors have been logged and aren't hanging around in a buffer sys.stdout.flush() sys.stderr.flush() task.writeMetadata(dataRef) # remove MDC so it does not show up outside of task context self.log.MDCRemove("LABEL") if self.doReturnResults: return Struct( exitStatus=exitStatus, dataRef=dataRef, metadata=task.metadata, result=result, ) else: return Struct(exitStatus=exitStatus, )