def run( rerun, # Rerun name frame1, # Frame number for input frame2, # Frame number for template diff, # Difference identifier for output patch, # Sky patch identifier config, # Configuration ): io = pipReadWrite.ReadWrite(suprimecam.SuprimecamMapper(rerun=rerun), ['visit'], config=config) diffProc = pipDiff.Diff(config=config) exp1 = io.inButler.get('warp', {'visit': frame1, 'patch': patch}) exp2 = io.inButler.get('warp', {'visit': frame2, 'patch': patch}) diffExp, sources, psf, apcorr, brightSources = diffProc.run(exp1, exp2) sources = afwDet.PersistableSourceVector(sources) diffProc.write(io.outButler, { 'diff': diff, 'patch': patch, 'filter': diffExp.getFilter().getName() }, { "diff": diffExp, "diffpsf": psf, "diffsources": sources })
def run( rerun, # Rerun name reference, # Reference stack name measureList, # List of stack names to measure skytile, # Sky tile number config, # Configuration ): io = pipReadWrite.ReadWrite(suprimecam.SuprimecamMapper, ['visit', 'ccd'], config=config) roots = config['roots'] basename = os.path.join(roots['output'], rerun) photProc = pipMultiPhot.MultiPhot(config=config) #import lsst.pex.logging as pexLog #pexLog.Log.getDefaultLog().setThreshold(pexLog.Log.DEBUG) refId = {'stack': reference, 'skytile': skytile, 'filter': "r"} refStack = photProc.read(io.inButler, refId, ['stack'])[0] measStackList = list() for name in measureList: dataId = {'stack': name, 'skytile': skytile, 'filter': "r"} measStack = refStack if dataId == refId else photProc.read( io.inButler, dataId, ['stack'])[0] measStackList.append(measStack) sourceList = photProc.run(refStack, measStackList) catPolicy = os.path.join(os.getenv("PIPETTE_DIR"), "policy", "catalog.paf") catalog = pipCatalog.Catalog(catPolicy, allowNonfinite=False) for name, sources in zip(measureList, sourceList): catalog.writeSources(basename + '-' + name + '.sources', sources, 'sources')
def run(rerun, # Rerun name frames, # Frame number ccds, # CCD number stack, # Stack identifier patch, # Patch identifier filter, # Filter name config, # Configuration coords, # Skycell centre coordinates scale, # Pixel scale sizes, # Skycell size ignore=False, # Ignore missing files? ): io = pipReadWrite.ReadWrite(hsc.HscSimMapper(rerun=rerun), ['visit', 'ccd'], config=config) roots = config['roots'] basename = os.path.join(roots['output'], rerun) stackProc = pipStack.Stack(config=config) identMatrix = list() for frame in frames: identList = list() for ccd in ccds: dataId = { 'visit': frame, 'ccd': ccd } identList.append(dataId) identMatrix.append(identList) exp = stackProc.run(identMatrix, io.inButler, coords[0], coords[1], scale, sizes[0], sizes[1], ignore=ignore) #stack.writeFits(basename + ".fits") stackProc.write(io.outButler, {'stack': stack, 'patch': patch, 'filter': filter}, {"stack": exp})
def run( rerun, # Rerun name frame, # Frame number ccds, # CCD number patch, # Sky patch identifier config, # Configuration coords, # Skycell centre coordinates scale, # Pixel scale sizes, # Skycell size ): io = pipReadWrite.ReadWrite(suprimecam.SuprimecamMapper(rerun=rerun), ['visit', 'ccd'], config=config) warpProc = pipWarp.Warp(config=config) identList = list() for ccd in ccds: dataId = {'visit': frame, 'ccd': ccd} identList.append(dataId) warp = warpProc.run(identList, io.inButler, coords[0], coords[1], scale, sizes[0], sizes[1]) warpProc.write(io.outButler, { 'visit': frame, 'patch': patch }, {"warp": warp})
def run( rerun, # Rerun name frames, # Frame number ccds, # CCD number mit, # MIT CCDs? config, # Configuration ): # log = pexLog.getDefaultLog() # log.setThreshold(log.DEBUG) mapper = suprimecam.SuprimecamMapper(rerun=rerun, mit=mit) io = pipReadWrite.ReadWrite(mapper, ['visit', 'ccd'], config=config) detProc = pipMaster.Master(config=config) identMatrix = list() for ccd in ccds: identList = list() for frame in frames: dataId = {'visit': frame, 'ccd': ccd} identList.append(dataId) identMatrix.append(identList) masterList = detProc.run(identMatrix, io.inButler, io.outButler) for master, ccd in zip(masterList, ccds): name = "%s-%d.fits" % (rerun, ccd) master.writeFits(name) print "Wrote %s" % name return
def run(args): camera = args.instrument rerun = args.rerun frame = args.frame ccd = args.ccd input = args.input output = args.output if camera.lower() in ('hsc'): Mapper = obsHsc.HscSimMapper elif camera.lower() in ("suprimecam", "suprime-cam", "sc", "suprimecam-mit", "sc-mit", "scmit", "suprimecam-old", "sc-old", "scold"): Mapper = obsSc.SuprimecamMapper else: raise RuntimeError("Unrecognised camera: %s" % camera) try: # LSST exceptions don't pickle io = pipReadWrite.ReadWrite(Mapper(rerun=rerun), ['visit', 'ccd']) exp = afwImage.ExposureF(input) image = exp.getMaskedImage() average = afwMath.makeStatistics(image, afwMath.MEANCLIP).getValue() md = io.read('calexp_md', {'visit': frame, 'ccd': ccd})[0] wcs = afwImage.makeWcs(md) image *= correction(image.getDimensions(), wcs) image *= average / afwMath.makeStatistics(image, afwMath.MEANCLIP).getValue() image.writeFits(output) except Exception, e: raise RuntimeError("Processing failed: %s" % e)
def run(visit, rerun, config): mapper = getMapper() dataId = {'visit': visit, 'rerun': rerun} #rrdir = mapper.getPath('outdir', dataId) #if not os.path.exists(rrdir): # print 'Creating directory for ouputs:', rrdir # os.makedirs(rrdir) io = pipReadWrite.ReadWrite(mapper, ['visit'], config=config) butler = io.inButler bb = butler.get('bb', dataId) print('Bounding-boxes:', bb) print(len(bb)) pyfoots = butler.get('pyfoots', dataId) foots, pks = footprintsFromPython(pyfoots) print('Footprints:') print(foots) print('Peaks:') print(pks) # HACK peaks fn = mapper.getPath('truesrc', dataId) srcs = pyfits.open(fn)[1].data x = srcs.field('x').astype(float) y = srcs.field('y').astype(float) print(x, y) pks = [] for foot in foots: thispks = [] bbox = foot.getBBox() bb = (bbox.getMinX(), bbox.getMinY(), bbox.getMaxX(), bbox.getMaxY()) print('Looking for sources for footprint with bbox', bb) for xi, yi in zip(x, y): if foot.contains(afwGeom.Point2I(int(round(xi)), int(round(yi)))): thispks.append(afwDet.Peak(xi, yi)) print(' Source at', (xi, yi), 'is inside footprint with bbox', bb) pks.append(thispks) print('Got', len(thispks), 'sources for this footprint') print('OVERRODE peaks', pks) exposureDatatype = 'visitim' exposure = butler.get(exposureDatatype, dataId) mi = exposure.getMaskedImage() print('MaskedImage:', mi) psf = butler.get('psf', dataId) print('PSF:', psf) testDeblend(foots, pks, mi, psf)
def plots(visit, rerun, config, bb=[]): mapper = getMapper() dataId = {'visit': visit, 'rerun': rerun} rrdir = mapper.getPath('outdir', dataId) if not os.path.exists(rrdir): raise RuntimeError('Rerun dir not found: "%s"' % rrdir) io = pipReadWrite.ReadWrite(mapper, ['visit'], config=config) butler = io.inButler import plotSources plotSources.plotSources(butler=butler, dataId=dataId, fn='src-v%04i-rr%04i.png' % (visit, rerun), bboxes=bb)
def run( rerun, # Rerun name visit, # Visit number snap, # Snap number raft, # Raft id sensor, # Sensor id config, # Configuration log=pexLog.Log.getDefaultLog() # Log object ): io = pipReadWrite.ReadWrite( lsstSim.LsstSimMapper, ['visit', 'snap', 'raft', 'sensor'], fileKeys=['visit', 'snap', 'raft', 'sensor', 'channel'], config=config) roots = config['roots'] basename = os.path.join( roots['output'], '%s-%d-%d-%s-%s' % (rerun, visit, snap, raft, sensor)) ccdProc = pipProcCcd.ProcessCcd(config=config, log=log) dataId = {'visit': visit, 'snap': snap, 'raft': raft, 'sensor': sensor} detrends = io.detrends(dataId, config) if len([x for x in detrends if x]): # We need to run at least part of the ISR raws = io.readRaw(dataId) else: io.fileKeys = ['visit', 'raft', 'sensor'] raws = io.read('calexp', dataId) detrends = None exposure, psf, brightSources, apcorr, sources, matches, matchMeta = ccdProc.run( raws, detrends) io.write(dataId, exposure=exposure, psf=psf, sources=sources, matches=matches, matchMeta=matchMeta) catPolicy = os.path.join(os.getenv("PIPETTE_DIR"), "policy", "catalog.paf") catalog = pipCatalog.Catalog(catPolicy, allowNonfinite=False) if sources is not None: catalog.writeSources(basename + '.sources', sources, 'sources') if matches is not None: catalog.writeMatches(basename + '.matches', matches, 'sources') return
def run( rerun, # Rerun name frame, # Frame number ccd, # CCD number config, # Configuration log=pexLog.Log.getDefaultLog(), # Log object ): roots = config['roots'] registry = os.path.join(roots['data'], 'registry.sqlite3') imapp = suprimecam.SuprimecamMapper(rerun=rerun, root=roots['data'], calibRoot=roots['calib']) omapp = suprimecam.SuprimecamMapper(rerun=rerun, root=roots['output'], calibRoot=roots['calib'], registry=registry) io = pipReadWrite.ReadWrite([imapp, omapp], ['visit', 'ccd'], config=config) ccdProc = pipProcCcd.ProcessCcd(config=config, Isr=pipSuprimeCam.IsrSuprimeCam, log=log) dataId = {'visit': frame, 'ccd': ccd} raws = io.readRaw(dataId) detrends = io.detrends(dataId, config) exposure, psf, apcorr, brightSources, sources, matches, matchMeta = ccdProc.run( raws, detrends) io.write(dataId, exposure=exposure, psf=psf, sources=sources, matches=matches, matchMeta=matchMeta) catPolicy = os.path.join(os.getenv("PIPETTE_DIR"), "policy", "catalog.paf") catalog = pipCatalog.Catalog(catPolicy, allowNonfinite=False) basename = os.path.join(roots['output'], '%s-%d%d' % (rerun, frame, ccd)) if sources is not None: catalog.writeSources(basename + '.sources', sources, 'sources') if matches is not None: catalog.writeMatches(basename + '.matches', matches, 'sources') return
def run( rerun, # Rerun name stack, # Stack identifier filter, # Filter name field, # Field name scale, # Scale, arcsec/pix ): io = pipReadWrite.ReadWrite(hsc.HscSimMapper(rerun=rerun), ['visit', 'ccd']) skyPolicy = io.inButler.get('skypolicy') print skyPolicy.toString() sky = skypix.QuadSpherePixelization( skyPolicy.get('resolutionPix'), skyPolicy.get('paddingArcsec') / 3600.0) dataId = {'filter': filter} if field is not None: dataId['field'] = field skytiles = io.inButler.queryMetadata('calexp', None, 'skyTile', dataId) for tile in skytiles: dataId['skyTile'] = tile visits = io.inButler.queryMetadata('calexp', None, 'visit', dataId) if len(visits) == 0: continue geom = sky.getGeometry(tile) bbox = geom.getBoundingBox() ra, dec = bbox.getCenter() # Degrees theta = bbox.getThetaExtent() # Width, degrees size = int(theta * 3600.0 / scale) # Size, pixels cmd = "hsc_stack.py --rerun " + rerun cmd += " --stack %d" % stack cmd += " --patch %d" % tile cmd += " --filter %s" % filter cmd += " --coords %f %f" % (ra, dec) cmd += " --scale %f" % scale cmd += " --sizes %d %d" % (size, size) cmd += " --frames %s" % ":".join(map(str, visits)) cmd += " --ignore" print cmd
def run(rerun, # Rerun name frame, # Frame number ccds, # CCD number patch, # Patch identifier config, # Configuration coords, # Skycell centre coordinates scale, # Pixel scale sizes, # Skycell size ): io = pipReadWrite.ReadWrite(hsc.HscSimMapper(rerun=rerun), ['visit', 'ccd'], config=config) roots = config['roots'] basename = os.path.join(roots['output'], '%s-%d' % (rerun, frame)) warpProc = pipWarp.Warp(config=config) identList = list() for ccd in ccds: dataId = { 'visit': frame, 'ccd': ccd } identList.append(dataId) warp = warpProc.run(identList, io.inButler, coords[0], coords[1], scale, sizes[0], sizes[1]) #warp.writeFits(basename + ".fits") warpProc.write(io.outButler, {'visit': frame, 'skytile': skytile}, {"warp": warp})
def run(rerun, # Rerun name visit, # Visit number ccd, # CCD number config, # Configuration log = pexLog.Log.getDefaultLog(), # Log object ): io = pipReadWrite.ReadWrite(cfht.CfhtMapper, ['visit', 'ccd'], fileKeys=['amp'], config=config) roots = config['roots'] basename = os.path.join(roots['output'], '%s-%d-%d' % (rerun, visit, ccd)) ccdProc = pipProcCcd.ProcessCcd(config=config, log=log) dataId = {'visit': visit, 'ccd': ccd} raws = io.readRaw(dataId) detrends = io.detrends(dataId, config) exposure, psf, brightSources, apcorr, sources, matches, matchMeta = ccdProc.run(raws, detrends) io.write(dataId, exposure=exposure, psf=psf, sources=sources, matches=matches, matchMeta=matchMeta) catPolicy = os.path.join(os.getenv("PIPETTE_DIR"), "policy", "catalog.paf") catalog = pipCatalog.Catalog(catPolicy, allowNonfinite=False) if sources is not None: catalog.writeSources(basename + '.sources', sources, 'sources') if matches is not None: catalog.writeMatches(basename + '.matches', matches, 'sources') return
def run(outName, rerun, frame1, frame2, config, matchTol=1.0, bright=None, ccd=None): io = pipReadWrite.ReadWrite(hscSim.HscSimMapper(rerun=rerun), ['visit'], fileKeys=['visit', 'ccd'], config=config) roots = config['roots'] outName = os.path.join(roots['output'], '%s.pdf' % outName) if outName is not None else None data1 = {'visit': frame1} data2 = {'visit': frame2} if ccd is not None: data1['ccd'] = ccd data2['ccd'] = ccd sources1 = io.read('src', data1, ignore=True) md1 = io.read('calexp_md', data1, ignore=True) sources2 = io.read('src', data2, ignore=True) md2 = io.read('calexp_md', data2, ignore=True) assert len(sources1) == len(md1) for i in range(len(sources1)): sources1[i] = filterSources(sources1[i], md1[i], bright) sources1 = concatenate(sources1) print len(sources1), "sources filtered from", frame1 assert len(sources2) == len(md2) for i in range(len(sources2)): sources2[i] = filterSources(sources2[i], md2[i], bright) sources2 = concatenate(sources2) print len(sources2), "sources filtered from", frame2 comp = pipCompare.Comparisons(sources1, sources2, matchTol=matchTol) print "%d matches" % comp.num ra = (comp['ra1'] + comp['ra2']) / 2.0 dec = (comp['dec1'] + comp['dec2']) / 2.0 if False: psfAvg = (comp['psf1'] + comp['psf2']) / 2.0 psfDiff = comp['psf1'] - comp['psf2'] apAvg = (comp['ap1'] + comp['ap2']) / 2.0 apDiff = comp['ap1'] - comp['ap2'] modelAvg = (comp['model1'] + comp['model2']) / 2.0 modelDiff = comp['model1'] - comp['model2'] else: psfAvg = (-2.5 * numpy.log10(comp['psf1']) - 2.5 * numpy.log10(comp['psf2'])) / 2.0 psfDiff = -2.5 * numpy.log10(comp['psf1']) + 2.5 * numpy.log10( comp['psf2']) apAvg = (-2.5 * numpy.log10(comp['ap1']) - 2.5 * numpy.log10(comp['ap2'])) / 2.0 apDiff = (-2.5 * numpy.log10(comp['ap1']) + 2.5 * numpy.log10(comp['ap2'])) modelAvg = (-2.5 * numpy.log10(comp['model1']) - 2.5 * numpy.log10(comp['model2'])) / 2.0 modelDiff = (-2.5 * numpy.log10(comp['model1']) + 2.5 * numpy.log10(comp['model2'])) plot = plotter.Plotter(outName) plot.xy(ra, dec, title="Detections") plot.xy(psfAvg, psfDiff, axis=[comp['psf1'].min(), comp['psf1'].max(), -0.25, 0.25], title="PSF photometry") plot.xy(apAvg, apDiff, axis=[comp['ap1'].min(), comp['ap1'].max(), -0.25, 0.25], title="Aperture photometry") plot.histogram(psfDiff, [-0.25, 0.25], title="PSF photometry") plot.histogram(apDiff, [-0.25, 0.25], title="Aperture photometry") plot.histogram(modelDiff, [-0.25, 0.25], title="Model photometry") plot.xy(psfAvg, modelAvg - psfAvg, title="PSF vs Model") plot.histogram(modelAvg - psfAvg, [-0.6, 0.2], bins=81, title="PSF vs Model") plot.xy2(ra, comp['distance'], dec, comp['distance'], axis1=[ra.min(), ra.max(), 0, matchTol], axis2=[dec.min(), dec.max(), 0, matchTol], title1="Right ascension", title2="Declination") plot.quivers(ra, dec, comp['ra1'] - comp['ra2'], comp['dec1'] - comp['dec2'], title="Astrometry", addUnitQuiver=1.0 / 3600.0) plot.close()
def run(visit, rerun, config): mapper = getMapper() dataId = {'visit': visit, 'rerun': rerun} rrdir = mapper.getPath('outdir', dataId) if not os.path.exists(rrdir): print('Creating directory for ouputs:', rrdir) os.makedirs(rrdir) else: print('Output directory:', rrdir) io = pipReadWrite.ReadWrite(mapper, ['visit'], config=config) # ccdProc = pipProcCcd.ProcessCcd(config=config, Isr=NullISR, Calibrate=MyCalibrate) # raws = io.readRaw(dataId) # detrends = io.detrends(dataId, config) print('Reading exposure') # exposure = io.read('visitim', dataId) exposure = io.inButler.get('visitim', dataId) print('exposure is', exposure) print('size', exposure.getWidth(), 'x', exposure.getHeight()) # debug # mi = exposure.getMaskedImage() # img = mi.getImage() # var = mi.getVariance() # print('var at 90,100 is', var.get(90,100)) # print('img at 90,100 is', img.get(90,100)) # print('wcs is', exposure.getWcs()) wcs = exposure.getWcs() assert wcs # print('ccdProc.run()...') # raws = [exposure] # exposure, psf, apcorr, brightSources, sources, matches, matchMeta = ccdProc.run(raws, detrends) print('Calibrate()...') log = Log.getDefaultLogger() cal = MyCalibrate(config=config, log=log, Photometry=MyPhotometry) psf, sources, footprints = cal.run2(exposure) print('Photometry()...') phot = pipPhot.Photometry(config=config, log=log) sources, footprints = phot.run(exposure, psf) print('sources:', len(sources)) for s in sources: print(' ', s, s.getXAstrom(), s.getYAstrom(), s.getPsfFlux(), s.getIxx(), s.getIyy(), s.getIxy()) print('footprints:', footprints) # oh yeah, baby! fps = footprints.getFootprints() print(len(fps)) bb = [] for f in fps: print(' Footprint', f) print(' ', f.getBBox()) bbox = f.getBBox() bb.append( (bbox.getMinX(), bbox.getMinY(), bbox.getMaxX(), bbox.getMaxY())) print(' # peaks:', len(f.getPeaks())) for p in f.getPeaks(): print(' Peak', p) # print('psf', psf) # print('sources', sources) # print('footprints', footprints) # psf, apcorr, brightSources, matches, matchMeta = self.calibrate(exposure, defects=defects) # if self.config['do']['phot']: # sources, footprints = self.phot(exposure, psf, apcorr, wcs=exposure.getWcs()) # psf, wcs = self.fakePsf(exposure) # sources, footprints = self.phot(exposure, psf) # sources = self.rephot(exposure, footprints, psf, apcorr=apcorr) # model = calibrate['model'] # fwhm = calibrate['fwhm'] / wcs.pixelScale() # size = calibrate['size'] # psf = afwDet.createPsf(model, size, size, fwhm/(2*math.sqrt(2*math.log(2)))) # print('done!') print('writing output...') io.write(dataId, psf=psf, sources=sources) print('done!') print('Writing bounding-boxes...') io.outButler.put(bb, 'bb', dataId) # print('Writing footprints...') # io.outButler.put(fps, 'footprints', dataId) # serialize a python version of footprints & peaks; # commented out because footprintsToPython does not exist # pyfoots = footprintsToPython(fps) # print('Writing py footprints...') # io.outButler.put(pyfoots, 'pyfoots', dataId) return bb
def run( rerun, # Rerun name frame, # Frame number ccd, # CCD number config, # Configuration log=pexLog.Log.getDefaultLog(), # Log object ): # Make our own mappers for now mapperArgs = {'rerun': rerun} # Arguments for mapper instantiation if config.has_key('roots'): roots = config['roots'] for key, value in { 'data': 'root', 'calib': 'calibRoot', 'output': 'outRoot' }.iteritems(): if roots.has_key(key): mapperArgs[value] = roots[key] camera = config['camera'] if camera.lower() in ("hsc"): mapper = obsHsc.HscSimMapper(**mapperArgs) ccdProc = pipCcd.ProcessCcd(config=config, Calibrate=CalibrateHscDc2, log=log) elif camera.lower() in ("suprimecam-mit", "sc-mit", "scmit", "suprimecam-old", "sc-old", "scold"): mapper = obsSc.SuprimecamMapper(mit=True, **mapperArgs) ccdProc = ProcessCcdSuprimeCam(config=config, log=log) elif camera.lower() in ("suprimecam", "suprime-cam", "sc"): mapper = obsSc.SuprimecamMapper(**mapperArgs) ccdProc = ProcessCcdSuprimeCam(config=config, log=log) io = pipReadWrite.ReadWrite(mapper, ['visit', 'ccd'], config=config) oldUmask = os.umask(2) if oldUmask != 2: io.log.log(io.log.WARN, "pipette umask started as: %s" % (os.umask(2))) dataId = {'visit': frame, 'ccd': ccd} raws = io.readRaw(dataId) detrends = io.detrends(dataId, config) if len([x for x in detrends if x]): # We need to run at least part of the ISR raws = io.readRaw(dataId) else: io.fileKeys = ['visit', 'ccd'] try: raws = io.read('calexp', dataId) config['do']['calibrate']['repair']['cosmicray'] = False except: raws = io.readRaw(dataId) detrends = None exposure, psf, apcorr, brightSources, sources, matches, matchMeta = ccdProc.run( raws, detrends) io.write(dataId, exposure=None, psf=psf, sources=None) catPolicy = os.path.join(os.getenv("PIPETTE_DIR"), "policy", "catalog.paf") catalog = pipCatalog.Catalog(catPolicy, allowNonfinite=False) deferredState = DeferredHSCState(dataId, io, matches, matchMeta, sources, brightSources, exposure) return deferredState