def main(): from optparse import OptionParser import sys # Otherwise plotting code can raise floating-point errors np.seterr(under='print') tune = [] def store_value (option, opt, value, parser): if opt == '--ntune': tune.append(('n',value)) elif opt == '--itune': tune.append(('i',value)) parser = OptionParser(usage=('%prog [options] [-r <run> -c <camcol> -f <field> -b <band>]')) parser.add_option('-r', '--run', dest='run', type='int') parser.add_option('-c', '--camcol', dest='camcol', type='int') parser.add_option('-f', '--field', dest='field', type='int') parser.add_option('-b', '--band', dest='band', help='SDSS Band (u, g, r, i, z)') parser.add_option('-R', '--radec', dest='radec', nargs=2, type=str, help='RA,Dec center: float degrees or hh:mm:ss +-dd:mm:ss') parser.add_option('-s', '--size', dest='pixsize', type=int, help='Pixel size when using RA,Dec center option') parser.add_option('--drfields', dest='drfields', help='FITS table of SDSS fields: default dr%ifields.fits, etc') parser.add_option('--dr8', dest='dr8', action='store_true', help='Use DR8? Default is DR7') parser.add_option('--dr9', dest='dr9', action='store_true', help='Use DR9? Default is DR7') parser.add_option('--curl', dest='curl', action='store_true', default=False, help='Use "curl", not "wget", to download files') parser.add_option('--ntune', action='callback', callback=store_value, type=int, help='Improve synthetic image by locally optimizing likelihood for nsteps iterations') parser.add_option('--itune', action='callback', callback=store_value, type=int, nargs=2, help='Optimizes each source individually') parser.add_option('--roi', dest='roi', type=int, nargs=4, help='Select an x0,x1,y0,y1 subset of the image') parser.add_option('--prefix', dest='prefix', help='Set output filename prefix; default is the SDSS RRRRRR-BC-FFFF string (run, band, camcol, field)') parser.add_option('-v', '--verbose', dest='verbose', action='count', default=0, help='Make more verbose') parser.add_option('-d','--debug',dest='debug',action='store_true',default=False,help="Trigger debug images") parser.add_option('--plotAll',dest='plotAll',action='store_true',default=False,help="Makes a plot for each source") parser.add_option('--no-arcsinh', dest='noarcsinh', action='store_true', help='Do not arcsinh-stretch plots') parser.add_option('--lbfgsb', dest='lbfgsb', action='store_true', default=False, help='Use L-BFGS-B optimization method') parser.add_option('--scale', dest='scale', type=int, help='Scale images down by this factor') parser.add_option('--unzip', dest='unzip', help='Save unzipped frame files in this directory') opt,args = parser.parse_args() if opt.verbose == 0: lvl = logging.INFO else: lvl = logging.DEBUG logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout) run = opt.run field = opt.field camcol = opt.camcol bands = [] if run is None and field is None and camcol is None and opt.band is None: # Demo mode... ## FIXME -- find a nice field here run = 1752 camcol = 3 field = 164 opt.band = 'r' opt.dr9 = True opt.curl = True tune.extend([('i',[1,1]), ('n',1)]) opt.roi = [100,600,100,600] print print 'Demo mode: grabbing Run/Camcol/Field/Band %i/%i/%i/%s' % (run, camcol, field, opt.band) print 'Using SDSS DR9 data' print if opt.band is None: parser.print_help() print print 'Must supply band (-b [u | g |r |i | z])' print sys.exit(-1) for char in opt.band: bands.append(char) if not char in ['u','g','r','i','z']: parser.print_help() print print 'Must supply band (u/g/r/i/z)' print sys.exit(-1) rerun = 0 usercf = (run is not None and field is not None and camcol is not None) userd = (opt.radec is not None and opt.pixsize is not None) if not (usercf or userd) or len(bands)==0: parser.print_help() print 'Must supply (--run, --camcol, --field) or (--radec, --size), and --band' sys.exit(-1) bandname = bands[0] #initial position and shape prefix = opt.prefix if userd: ra = opt.radec[0] dec = opt.radec[1] size = opt.pixsize if prefix is None: if usercf: prefix = '%06i-%i-%04i' % (run,camcol, field) else: prefix = '%s-%s' % (ra, dec) if userd: # convert h:m:s to deg try: ra = float(ra) except: ra = ra.replace(':',' ') ra = hmsstring2ra(ra) try: dec = float(dec) except: dec = dec.replace(':',' ') dec = dmsstring2dec(dec) sdss = None imkw = {} if opt.dr9: getim = st.get_tractor_image_dr9 getsrc = st.get_tractor_sources_dr9 drnum = 9 sdss = st.DR9() imkw.update(zrange=[-3,100], sdss=sdss) elif opt.dr8: getim = st.get_tractor_image_dr8 getsrc = st.get_tractor_sources_dr8 drnum = 8 sdss = st.DR8() imkw.update(zrange=[-3,100], sdss=sdss) else: getim = st.get_tractor_image getsrc = st.get_tractor_sources drnum = 7 imkw.update(useMags=True) if drnum in [8,9] and opt.unzip: sdss.saveUnzippedFiles(opt.unzip) tims = [] for bandname in bands: if userd: if not usercf: tfn = opt.drfields if tfn is None: tfn = 'dr%ifields.fits' % drnum rcfs = radec_to_sdss_rcf(ra, dec, radius=math.hypot(radius,13./2.), tablefn=tfn) if len(rcfs) == 0: print 'RA,Dec (%.3f,%.3f): found no overlapping SDSS fields in file %s' % (ra, dec, tfn) sys.exit(-1) if len(rcfs) > 1: print 'Found %i Run,Camcol,Fields overlapping RA,Dec (%.3f,%.3f): taking the first one' % (len(rcfs), ra, dec) rcfs = rcfs[0] run,camcol,field = rcfs[:3] imkw.update(roiradecsize = (ra, dec, opt.pixsize)) tim,tinf = getim(run, camcol, field, bandname, curl=opt.curl, roi=opt.roi, **imkw) tim.zr = tinf['zr'] if userd: opt.roi = tinf['roi'] tims.append(tim) if opt.scale: print 'Scaling images by', opt.scale tims = [st.scale_sdss_image(tim, opt.scale) for tim in tims] sources = getsrc(run, camcol, field, bandname, bands=bands, curl=opt.curl, roi=opt.roi) tractor = Tractor(tims, sources) sa = dict(debug=opt.debug, plotAll=opt.plotAll, roi=opt.roi) if opt.noarcsinh: sa.update(nlscale=0) elif opt.dr8: #sa.update(nlscale=1.) sa.update(chilo=-50., chihi=50.) for j,band in enumerate(bands): save('initial-%s-' % (band) + prefix, tractor, imgi=j, **sa) lnp0 = tractor.getLogProb() print 'Initial lnprob:', lnp0 for im in tractor.images: im.freezeAllParams() im.thawParam('sky') for count, each in enumerate(tune): if each[0]=='n': tractor.catalog.thawAllParams() tractor.images.thawParamsRecursive('sky') for i in range(each[1]): if opt.lbfgsb: tractor.optimize_lbfgsb() else: tractor.optimize() for j,band in enumerate(bands): save('tune-%d-%d-%s-' % (count+1, i+1,band) + prefix, tractor, imgi=j, **sa) lnp1 = tractor.getLogProb() print 'After optimization: lnprob', lnp1 print ' delta-lnprob:', lnp1 - lnp0 elif each[0]=='i': tractor.images.freezeParamsRecursive('sky') for i in range(each[1][0]): for j,src in enumerate(tractor.getCatalog()): tractor.catalog.freezeAllBut(src) if i < 6: print 'Freezing position' src.freezeParam('pos') print 'Optimizing:' for nm in tractor.getParamNames(): print nm for step in range(each[1][1]): if opt.lbfgsb: tractor.optimize_lbfgsb(plotfn='synt-opt-i%i-s%04i.png' % (i,j)) else: tractor.optimize() src.unfreezeParam('pos') for j,band in enumerate(bands): save('tune-%d-%d-%s-' % (count+1,i+1,band) + prefix, tractor, imgi=j, **sa) tractor.clearCache() lnp1 = tractor.getLogProb() print 'After optimization: lnprob', lnp1 print ' delta-lnprob:', lnp1 - lnp0 makeflipbook(opt, prefix,tune,len(tractor.getCatalog()),bands) print print 'Created flip-book flip-%s.pdf' % prefix
def general(name,ra,dec,remradius,fieldradius,threads=None,itune1=5,itune2=5,ntune=0,nocache=False,scale=1,ab=1.,angle=0.): #Radius should be in arcminutes if threads: mp = multiproc(nthreads=threads) else: mp = multiproc() IRLS_scale = 25. dr9 = True dr8 = False noarcsinh = False print name prefix = 'swapCG_%s' % (name.replace(' ', '_')) print 'Removal Radius', remradius print 'Field Radius', fieldradius print 'RA,Dec', ra, dec print os.getcwd() print ra,dec,math.hypot(fieldradius,13./2.) rcfs = radec_to_sdss_rcf(ra,dec,radius=math.hypot(fieldradius,13./2.),tablefn="dr9fields.fits") print 'RCFS:', rcfs print len(rcfs) assert(len(rcfs)>0) if 10 <= len(rcfs) < 20: scale = 2 elif 20 <= len(rcfs) < 40: scale = 4 elif 40 <= len(rcfs) < 80: scale = 8 assert(len(rcfs)<80) sras, sdecs, smags = tychoMatch(ra,dec,(fieldradius*1.5)/60.) imkw = dict(psf='kl-gm') if dr9: getim = st.get_tractor_image_dr9 getsrc = st.get_tractor_sources_dr9 imkw.update(zrange=[-3,100]) elif dr8: getim = st.get_tractor_image_dr8 getsrc = st.get_tractor_sources_dr8 imkw.update(zrange=[-3,100]) else: getim = st.get_tractor_image getsrc = st.get_tractor_sources_dr8 imkw.update(useMags=True) bands=['u','g','r','i','z'] #bands=['r'] bandname = 'r' flipBands = ['r'] print rcfs imsrcs = mp.map(get_ims_and_srcs, [(rcf + (bands, ra, dec, fieldradius*60./0.396, imkw, getim, getsrc)) for rcf in rcfs]) timgs = [] sources = [] allsources = [] for ims,s in imsrcs: if ims is None: continue if s is None: continue if scale > 1: for im in ims: timgs.append(st.scale_sdss_image(im,scale)) else: timgs.extend(ims) allsources.extend(s) sources.append(s) #rds = [rcf[3:5] for rcf in rcfs] #plotarea(ra, dec, fieldradius, name, prefix, timgs) #, rds) #lvl = logging.DEBUG #logging.basicConfig(level=lvl,format='%(message)s',stream=sys.stdout) tractor = st.Tractor(timgs, allsources, mp=mp) sa = dict(debug=True, plotAll=False,plotBands=False) if noarcsinh: sa.update(nlscale=0) elif dr8 or dr9: sa.update(chilo=-8.,chihi=8.) if nocache: tractor.cache = NullCache() sg.disable_galaxy_cache() zr = timgs[0].zr print "zr is: ",zr print bands print "Number of images: ", len(timgs) #for timg,band in zip(timgs,bands): # data = timg.getImage()/np.sqrt(timg.getInvvar()) # plt.hist(data,bins=100) # plt.savefig('hist-%s.png' % (band)) saveAll('initial-'+prefix, tractor,**sa) #plotInvvar('initial-'+prefix,tractor) for sra,sdec,smag in zip(sras,sdecs,smags): for img in tractor.getImages(): wcs = img.getWcs() starx,stary = wcs.positionToPixel(RaDecPos(sra,sdec)) starr=25*(2**(max(11-smag,0.))) if starx+starr<0. or starx-starr>img.getWidth() or stary+starr <0. or stary-starr>img.getHeight(): continue X,Y = np.meshgrid(np.arange(img.getWidth()), np.arange(img.getHeight())) R2 = (X - starx)**2 + (Y - stary)**2 img.getStarMask()[R2 < starr**2] = 0 for timgs,sources in imsrcs: timg = timgs[0] wcs = timg.getWcs() xtr,ytr = wcs.positionToPixel(RaDecPos(ra,dec)) xt = xtr yt = ytr r = ((remradius*60.))/.396 #radius in pixels for src in sources: xs,ys = wcs.positionToPixel(src.getPosition(),src) if (xs-xt)**2+(ys-yt)**2 <= r**2: #print "Removed:", src #print xs,ys tractor.removeSource(src) #saveAll('removed-'+prefix, tractor,**sa) newShape = sg.GalaxyShape((remradius*60.)/10.,ab,angle) newBright = ba.Mags(r=15.0,g=15.0,u=15.0,z=15.0,i=15.0,order=['u','g','r','i','z']) EG = st.ExpGalaxy(RaDecPos(ra,dec),newBright,newShape) print EG tractor.addSource(EG) saveAll('added-'+prefix,tractor,**sa) #print 'Tractor has', tractor.getParamNames() for im in tractor.images: im.freezeAllParams() im.thawParam('sky') tractor.catalog.freezeAllBut(EG) #print 'Tractor has', tractor.getParamNames() #print 'values', tractor.getParams() for i in range(itune1): tractor.optimize() tractor.changeInvvar(IRLS_scale) saveAll('itune1-%d-' % (i+1)+prefix,tractor,**sa) tractor.clearCache() sg.get_galaxy_cache().clear() gc.collect() print resource.getpagesize() print resource.getrusage(resource.RUSAGE_SELF)[2] CGPos = EG.getPosition() CGShape1 = EG.getShape().copy() CGShape2 = EG.getShape().copy() EGBright = EG.getBrightness() CGu = EGBright[0] + 0.75 CGg = EGBright[1] + 0.75 CGr = EGBright[2] + 0.75 CGi = EGBright[3] + 0.75 CGz = EGBright[4] + 0.75 CGBright1 = ba.Mags(r=CGr,g=CGg,u=CGu,z=CGz,i=CGi,order=['u','g','r','i','z']) CGBright2 = ba.Mags(r=CGr,g=CGg,u=CGu,z=CGz,i=CGi,order=['u','g','r','i','z']) print EGBright print CGBright1 CG = st.CompositeGalaxy(CGPos,CGBright1,CGShape1,CGBright2,CGShape2) tractor.removeSource(EG) tractor.addSource(CG) tractor.catalog.freezeAllBut(CG) print resource.getpagesize() print resource.getrusage(resource.RUSAGE_SELF)[2] for i in range(itune2): tractor.optimize() tractor.changeInvvar(IRLS_scale) saveAll('itune2-%d-' % (i+1)+prefix,tractor,**sa) tractor.clearCache() sg.get_galaxy_cache().clear() print resource.getpagesize() print resource.getrusage(resource.RUSAGE_SELF)[2] tractor.catalog.thawAllParams() for i in range(ntune): tractor.optimize() tractor.changeInvvar(IRLS_scale) saveAll('ntune-%d-' % (i+1)+prefix,tractor,**sa) #plotInvvar('final-'+prefix,tractor) sa.update(plotBands=True) saveAll('allBands-' + prefix,tractor,**sa) print "end of first round of optimization:", tractor.getLogLikelihood() print CG print CG.getPosition() print CGBright1 print CGBright2 print CGShape1 print CGShape2 print CGBright1+CGBright2 print CG.getBrightness() pfn = '%s.pickle' % prefix pickle_to_file(CG,pfn) makeflipbook(prefix,len(tractor.getImages()),itune1,itune2,ntune) # now SWAP exp and dev and DO IT AGAIN newCG = st.CompositeGalaxy(CG.getPosition, CG.brightnessDev.copy(), CG.shapeDev.copy(), CG.brightnessExp.copy(), CG.shapeExp.copy()) tractor.removeSource(CG) tractor.addSource(newCG) tractor.catalog.thawAllParams() for i in range(ntune): tractor.optimize() tractor.changeInvvar(IRLS_scale) saveAll('ntune-swap-%d-' % (i+1)+prefix,tractor,**sa) #plotInvvar('final-'+prefix,tractor) sa.update(plotBands=True) saveAll('allBands-swap-' + prefix,tractor,**sa) print "end of second (swapped) round of optimization:", tractor.getLogLikelihood() print newCG print newCG.getPosition() print newCG.getBrightness() pfn = '%s-swap.pickle' % prefix pickle_to_file(newCG,pfn) makeflipbook(prefix+"-swap",len(tractor.getImages()),itune1,itune2,ntune)
def general(name, ra, dec, remradius, fieldradius, threads=None, itune1=5, itune2=5, ntune=0, nocache=False, scale=1, ab=1., angle=0.): #Radius should be in arcminutes if threads: mp = multiproc(nthreads=threads) else: mp = multiproc() IRLS_scale = 25. dr9 = True dr8 = False noarcsinh = False print name prefix = 'swapCG_%s' % (name.replace(' ', '_')) print 'Removal Radius', remradius print 'Field Radius', fieldradius print 'RA,Dec', ra, dec print os.getcwd() print ra, dec, math.hypot(fieldradius, 13. / 2.) rcfs = radec_to_sdss_rcf(ra, dec, radius=math.hypot(fieldradius, 13. / 2.), tablefn="dr9fields.fits") print 'RCFS:', rcfs print len(rcfs) assert (len(rcfs) > 0) if 10 <= len(rcfs) < 20: scale = 2 elif 20 <= len(rcfs) < 40: scale = 4 elif 40 <= len(rcfs) < 80: scale = 8 assert (len(rcfs) < 80) sras, sdecs, smags = tychoMatch(ra, dec, (fieldradius * 1.5) / 60.) imkw = dict(psf='kl-gm') if dr9: getim = st.get_tractor_image_dr9 getsrc = st.get_tractor_sources_dr9 imkw.update(zrange=[-3, 100]) elif dr8: getim = st.get_tractor_image_dr8 getsrc = st.get_tractor_sources_dr8 imkw.update(zrange=[-3, 100]) else: getim = st.get_tractor_image getsrc = st.get_tractor_sources_dr8 imkw.update(useMags=True) bands = ['u', 'g', 'r', 'i', 'z'] #bands=['r'] bandname = 'r' flipBands = ['r'] print rcfs imsrcs = mp.map( get_ims_and_srcs, [(rcf + (bands, ra, dec, fieldradius * 60. / 0.396, imkw, getim, getsrc)) for rcf in rcfs]) timgs = [] sources = [] allsources = [] for ims, s in imsrcs: if ims is None: continue if s is None: continue if scale > 1: for im in ims: timgs.append(st.scale_sdss_image(im, scale)) else: timgs.extend(ims) allsources.extend(s) sources.append(s) #rds = [rcf[3:5] for rcf in rcfs] #plotarea(ra, dec, fieldradius, name, prefix, timgs) #, rds) #lvl = logging.DEBUG #logging.basicConfig(level=lvl,format='%(message)s',stream=sys.stdout) tractor = st.Tractor(timgs, allsources, mp=mp) sa = dict(debug=True, plotAll=False, plotBands=False) if noarcsinh: sa.update(nlscale=0) elif dr8 or dr9: sa.update(chilo=-8., chihi=8.) if nocache: tractor.cache = NullCache() sg.disable_galaxy_cache() zr = timgs[0].zr print "zr is: ", zr print bands print "Number of images: ", len(timgs) #for timg,band in zip(timgs,bands): # data = timg.getImage()/np.sqrt(timg.getInvvar()) # plt.hist(data,bins=100) # plt.savefig('hist-%s.png' % (band)) saveAll('initial-' + prefix, tractor, **sa) #plotInvvar('initial-'+prefix,tractor) for sra, sdec, smag in zip(sras, sdecs, smags): for img in tractor.getImages(): wcs = img.getWcs() starx, stary = wcs.positionToPixel(RaDecPos(sra, sdec)) starr = 25 * (2**(max(11 - smag, 0.))) if starx + starr < 0. or starx - starr > img.getWidth( ) or stary + starr < 0. or stary - starr > img.getHeight(): continue X, Y = np.meshgrid(np.arange(img.getWidth()), np.arange(img.getHeight())) R2 = (X - starx)**2 + (Y - stary)**2 img.getStarMask()[R2 < starr**2] = 0 for timgs, sources in imsrcs: timg = timgs[0] wcs = timg.getWcs() xtr, ytr = wcs.positionToPixel(RaDecPos(ra, dec)) xt = xtr yt = ytr r = ((remradius * 60.)) / .396 #radius in pixels for src in sources: xs, ys = wcs.positionToPixel(src.getPosition(), src) if (xs - xt)**2 + (ys - yt)**2 <= r**2: #print "Removed:", src #print xs,ys tractor.removeSource(src) #saveAll('removed-'+prefix, tractor,**sa) newShape = sg.GalaxyShape((remradius * 60.) / 10., ab, angle) newBright = ba.Mags(r=15.0, g=15.0, u=15.0, z=15.0, i=15.0, order=['u', 'g', 'r', 'i', 'z']) EG = st.ExpGalaxy(RaDecPos(ra, dec), newBright, newShape) print EG tractor.addSource(EG) saveAll('added-' + prefix, tractor, **sa) #print 'Tractor has', tractor.getParamNames() for im in tractor.images: im.freezeAllParams() im.thawParam('sky') tractor.catalog.freezeAllBut(EG) #print 'Tractor has', tractor.getParamNames() #print 'values', tractor.getParams() for i in range(itune1): tractor.optimize() tractor.changeInvvar(IRLS_scale) saveAll('itune1-%d-' % (i + 1) + prefix, tractor, **sa) tractor.clearCache() sg.get_galaxy_cache().clear() gc.collect() print resource.getpagesize() print resource.getrusage(resource.RUSAGE_SELF)[2] CGPos = EG.getPosition() CGShape1 = EG.getShape().copy() CGShape2 = EG.getShape().copy() EGBright = EG.getBrightness() CGu = EGBright[0] + 0.75 CGg = EGBright[1] + 0.75 CGr = EGBright[2] + 0.75 CGi = EGBright[3] + 0.75 CGz = EGBright[4] + 0.75 CGBright1 = ba.Mags(r=CGr, g=CGg, u=CGu, z=CGz, i=CGi, order=['u', 'g', 'r', 'i', 'z']) CGBright2 = ba.Mags(r=CGr, g=CGg, u=CGu, z=CGz, i=CGi, order=['u', 'g', 'r', 'i', 'z']) print EGBright print CGBright1 CG = st.CompositeGalaxy(CGPos, CGBright1, CGShape1, CGBright2, CGShape2) tractor.removeSource(EG) tractor.addSource(CG) tractor.catalog.freezeAllBut(CG) print resource.getpagesize() print resource.getrusage(resource.RUSAGE_SELF)[2] for i in range(itune2): tractor.optimize() tractor.changeInvvar(IRLS_scale) saveAll('itune2-%d-' % (i + 1) + prefix, tractor, **sa) tractor.clearCache() sg.get_galaxy_cache().clear() print resource.getpagesize() print resource.getrusage(resource.RUSAGE_SELF)[2] tractor.catalog.thawAllParams() for i in range(ntune): tractor.optimize() tractor.changeInvvar(IRLS_scale) saveAll('ntune-%d-' % (i + 1) + prefix, tractor, **sa) #plotInvvar('final-'+prefix,tractor) sa.update(plotBands=True) saveAll('allBands-' + prefix, tractor, **sa) print "end of first round of optimization:", tractor.getLogLikelihood() print CG print CG.getPosition() print CGBright1 print CGBright2 print CGShape1 print CGShape2 print CGBright1 + CGBright2 print CG.getBrightness() pfn = '%s.pickle' % prefix pickle_to_file(CG, pfn) makeflipbook(prefix, len(tractor.getImages()), itune1, itune2, ntune) # now SWAP exp and dev and DO IT AGAIN newCG = st.CompositeGalaxy(CG.getPosition, CG.brightnessDev.copy(), CG.shapeDev.copy(), CG.brightnessExp.copy(), CG.shapeExp.copy()) tractor.removeSource(CG) tractor.addSource(newCG) tractor.catalog.thawAllParams() for i in range(ntune): tractor.optimize() tractor.changeInvvar(IRLS_scale) saveAll('ntune-swap-%d-' % (i + 1) + prefix, tractor, **sa) #plotInvvar('final-'+prefix,tractor) sa.update(plotBands=True) saveAll('allBands-swap-' + prefix, tractor, **sa) print "end of second (swapped) round of optimization:", tractor.getLogLikelihood( ) print newCG print newCG.getPosition() print newCG.getBrightness() pfn = '%s-swap.pickle' % prefix pickle_to_file(newCG, pfn) makeflipbook(prefix + "-swap", len(tractor.getImages()), itune1, itune2, ntune)