def init(self): import os from astrometry.util.miscutils import point_in_poly from astrometry.net.wcs import TanWCS from scipy.ndimage.morphology import binary_dilation hp = self.healpix nside = self.nside topscale = self.version.topscale hpwcs, hpxy = get_healpix_wcs(nside, hp, topscale) H, W = int(hpwcs.get_height()), int(hpwcs.get_width()) bands = 3 enhI = np.zeros((H, W, bands), np.float32) enhW = np.zeros((H, W), np.float32) xx, yy = np.meshgrid(np.arange(W), np.arange(H)) # inside-healpix mask. enhM = point_in_poly(xx, yy, hpxy - 1.) # grow ? enhM = binary_dilation(enhM, np.ones((3, 3))) del xx del yy npix = np.sum(enhM) for b in range(bands): enhI[:, :, b][enhM] = np.random.permutation(npix) / float(npix) enhW[enhM] = 1e-3 mydir = self.get_dir() # print 'My directory:', mydir if not os.path.exists(mydir): # print 'Does not exist' try: os.makedirs(mydir) # print 'Created' except: import traceback print('Failed to create dir:') traceback.print_exc() pass tempfn = self.write_files(enhI, enhW, temp=True) dbwcs = TanWCS() dbwcs.set_from_tanwcs(hpwcs) dbwcs.save() with transaction.commit_on_success(): self.move_temp_files(tempfn) self.maxweight = 0. self.wcs = dbwcs self.save()
def init(self): import os from astrometry.util.miscutils import point_in_poly from astrometry.net.wcs import TanWCS from scipy.ndimage.morphology import binary_dilation hp = self.healpix nside = self.nside topscale = self.version.topscale hpwcs,hpxy = get_healpix_wcs(nside, hp, topscale) H,W = int(hpwcs.get_height()), int(hpwcs.get_width()) bands = 3 enhI = np.zeros((H,W,bands), np.float32) enhW = np.zeros((H,W), np.float32) xx,yy = np.meshgrid(np.arange(W), np.arange(H)) # inside-healpix mask. enhM = point_in_poly(xx, yy, hpxy-1.) # grow ? enhM = binary_dilation(enhM, np.ones((3,3))) del xx del yy npix = np.sum(enhM) for b in range(bands): enhI[:,:,b][enhM] = np.random.permutation(npix) / float(npix) enhW[enhM] = 1e-3 mydir = self.get_dir() # print 'My directory:', mydir if not os.path.exists(mydir): # print 'Does not exist' try: os.makedirs(mydir) # print 'Created' except: import traceback print 'Failed to create dir:' traceback.print_exc() pass tempfn = self.write_files(enhI, enhW, temp=True) dbwcs = TanWCS() dbwcs.set_from_tanwcs(hpwcs) dbwcs.save() with transaction.commit_on_success(): self.move_temp_files(tempfn) self.maxweight = 0. self.wcs = dbwcs self.save()
def main(): import optparse import logging import sys parser = optparse.OptionParser() parser.add_option('--threads', dest='threads', default=1, type=int, help='Use this many concurrent processors') parser.add_option('-v', '--verbose', dest='verbose', action='count', default=0, help='Make more verbose') parser.add_option('--grid', '-g', dest='gridn', type=int, default=5, help='Dust parameter grid size') parser.add_option('--steps', '-s', dest='steps', type=int, default=10, help='# Optimization step') parser.add_option('--suffix', dest='suffix', default='', help='Output file suffix') parser.add_option('--no-100', dest='no100', action='store_true', default=False, help='Omit PACS-100 data?') parser.add_option('--callgrind', dest='callgrind', action='store_true', default=False, help='Turn on callgrind around tractor.optimize()') parser.add_option('--resume', '-r', dest='resume', type=int, default=-1, help='Resume from a previous run at the given step?') parser.add_option('--zoom', dest='zoom', type=float, default=1, help='Scale down the model to only touch the (1/zoom x 1/zoom) central region of the images') parser.add_option('--damp', dest='damp', type=float, default=1., help='LSQR damping') opt,args = parser.parse_args() if opt.verbose == 0: lvl = logging.INFO log_init(2) else: lvl = logging.DEBUG log_init(3) logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout) if opt.threads > 1 and False: global dpool import debugpool dpool = debugpool.DebugPool(opt.threads) Time.add_measurement(debugpool.DebugPoolMeas(dpool)) mp = multiproc(pool=dpool) else: print('N threads', opt.threads) mp = multiproc(opt.threads)#, wrap_all=True) if opt.callgrind: import callgrind else: callgrind = None np.seterrcall(np_err_handler) np.seterr(all='call') #np.seterr(all='raise') if opt.resume > -1: pfn = 'herschel-%02i%s.pickle' % (opt.resume, opt.suffix) print('Reading from', pfn) tractor = unpickle_from_file(pfn) tractor.mp = mp ds = tractor.getCatalog()[0] print('DustSheet:', ds) # derivs = ds.getParamDerivatives(tim) # dim = np.zeros(tim.shape) # #for k,deriv in enumerate(derivs[:40]): # for k,deriv in enumerate(derivs[::10]): # dim[:,:] = 0 # deriv.addTo(dim) # plt.clf() # plt.imshow(dim, interpolation='nearest', origin='lower') # plt.savefig('deriv-%04i.png' % k) #tim = tractor.getImages()[0] # for it,tim in enumerate(tractor.getImages()): # X = ds._getTransformation(tim) # # #print 'X', X # keys = X.keys() # keys.sort() # # for k in keys[::10]: # # for k in keys[:40]: # for k in keys[::202]: # I,G,nil,nil = X[k] # rim = np.zeros_like(tim.getImage()) # rim.ravel()[I] = G # plt.clf() # plt.imshow(rim, interpolation='nearest', origin='lower') # plt.colorbar() # plt.savefig('rim-%i-%04i.png' % (it,k)) # print 'pix', k # sys.exit(0) makeplots(tractor, opt.resume, opt.suffix) step0 = opt.resume + 1 else: step0 = 0 tractor = create_tractor(opt) tractor.mp = mp # zero out invvar outside the model bounds. ds = tractor.getCatalog()[0] rd = ds.getRaDecCorners(margin=0.5) for i,tim in enumerate(tractor.getImages()): poly = np.array([tim.getWcs().positionToPixel( RaDecPos(rdi[0], rdi[1])) for rdi in rd]) poly = poly[:-1,:] print('Model bounding box in image', tim.name, 'coordinates:') #print poly.shape print(poly) H,W = tim.shape xx,yy = np.meshgrid(np.arange(W), np.arange(H)) inside = point_in_poly(xx, yy, poly) tim.inverr[inside == 0] = 0. plt.clf() for i,tim in enumerate(tractor.images): h,w = tim.shape rd = [tim.getWcs().pixelToPosition(x,y) for x,y in [(-0.5,-0.5),(w-0.5,-0.5),(w-0.5,h-0.5), (-0.5,h-0.5),(-0.5,-0.5)]] plt.plot([p.ra for p in rd], [p.dec for p in rd], '-', label=tim.name) rd = ds.getRaDecCorners(margin=0.5) plt.plot(rd[:,0], rd[:,1], 'k-', label='Grid') mh,mw = ds.shape r,d = ds.wcs.pixelxy2radec(1 + np.arange(mw), np.ones(mw)) plt.plot(r, d, 'k.') r,d = ds.wcs.pixelxy2radec(np.ones(mh), 1 + np.arange(mh)) plt.plot(r, d, 'k.') r,d = ds.wcs.pixelxy2radec(1 + np.arange(mw), np.zeros(mw)+mh) plt.plot(r, d, 'k.') r,d = ds.wcs.pixelxy2radec(np.zeros(mh)+mw, 1 + np.arange(mh)) plt.plot(r, d, 'k.') plt.legend() plt.savefig('radec.png') print('Precomputing transformations...') ds = tractor.getCatalog()[0] # Split the grid-spread matrix into strips... async_results = [] for im in tractor.getImages(): args = [] H,W = ds.shape dy = 10 y = 0 while y <= H: args.append((ds, im, y, min(H, y+dy))) y += dy async_results.append(mp.map_async(_map_trans, args)) # Glue to strips back together... XX = [] for ar in async_results: Xblocks = ar.get() X = Xblocks[0] for xi in Xblocks[1:]: X.update(xi) XX.append(X) for im,X in zip(tractor.getImages(), XX): ds._normalizeTransformation(im, X) ds._setTransformation(im, X) print('done precomputing.') # Plot the grid-spread functions. for itim,tim in enumerate(tractor.images): T = ds._getTransformation(tim) (I,G,nz,NZI) = T[0] plt.clf() g = np.zeros(tim.shape, np.float32) g.flat[I] = G plt.imshow(g, interpolation='nearest', origin='lower', cmap='hot') plt.colorbar() plt.title('Grid-spread function for cell 0, image %s' % tim.name) plt.savefig('gsf-%i.png' % itim) makeplots(tractor, 0, opt.suffix) pfn = 'herschel-%02i%s.pickle' % (0, opt.suffix) pickle_to_file(tractor, pfn) print('Wrote', pfn) for im in tractor.getImages(): im.freezeAllBut('sky') for i in range(step0, opt.steps): print('Step', i) if callgrind: callgrind.callgrind_start_instrumentation() tractor.optimize(damp=opt.damp, alphas=[1e-3, 1e-2, 0.1, 0.3, 1., 3., 10., 30., 100.]) if callgrind: callgrind.callgrind_stop_instrumentation() makeplots(tractor, 1 + i, opt.suffix) pfn = 'herschel-%02i%s.pickle' % (1 + i, opt.suffix) pickle_to_file(tractor, pfn) print('Wrote', pfn)
def main(): import optparse import logging import sys parser = optparse.OptionParser() parser.add_option('--threads', dest='threads', default=1, type=int, help='Use this many concurrent processors') parser.add_option('-v', '--verbose', dest='verbose', action='count', default=0, help='Make more verbose') parser.add_option('--grid', '-g', dest='gridn', type=int, default=5, help='Dust parameter grid size') parser.add_option('--steps', '-s', dest='steps', type=int, default=10, help='# Optimization step') parser.add_option('--suffix', dest='suffix', default='', help='Output file suffix') parser.add_option('--no-100', dest='no100', action='store_true', default=False, help='Omit PACS-100 data?') parser.add_option('--callgrind', dest='callgrind', action='store_true', default=False, help='Turn on callgrind around tractor.optimize()') parser.add_option('--resume', '-r', dest='resume', type=int, default=-1, help='Resume from a previous run at the given step?') parser.add_option('--zoom', dest='zoom', type=float, default=1, help='Scale down the model to only touch the (1/zoom x 1/zoom) central region of the images') parser.add_option('--damp', dest='damp', type=float, default=1., help='LSQR damping') opt,args = parser.parse_args() if opt.verbose == 0: lvl = logging.INFO log_init(2) else: lvl = logging.DEBUG log_init(3) logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout) if opt.threads > 1 and False: global dpool import debugpool dpool = debugpool.DebugPool(opt.threads) Time.add_measurement(debugpool.DebugPoolMeas(dpool)) mp = multiproc(pool=dpool) else: print 'N threads', opt.threads mp = multiproc(opt.threads)#, wrap_all=True) if opt.callgrind: import callgrind else: callgrind = None np.seterrcall(np_err_handler) np.seterr(all='call') #np.seterr(all='raise') if opt.resume > -1: pfn = 'herschel-%02i%s.pickle' % (opt.resume, opt.suffix) print 'Reading from', pfn tractor = unpickle_from_file(pfn) tractor.mp = mp ds = tractor.getCatalog()[0] print 'DustSheet:', ds # derivs = ds.getParamDerivatives(tim) # dim = np.zeros(tim.shape) # #for k,deriv in enumerate(derivs[:40]): # for k,deriv in enumerate(derivs[::10]): # dim[:,:] = 0 # deriv.addTo(dim) # plt.clf() # plt.imshow(dim, interpolation='nearest', origin='lower') # plt.savefig('deriv-%04i.png' % k) #tim = tractor.getImages()[0] # for it,tim in enumerate(tractor.getImages()): # X = ds._getTransformation(tim) # # #print 'X', X # keys = X.keys() # keys.sort() # # for k in keys[::10]: # # for k in keys[:40]: # for k in keys[::202]: # I,G,nil,nil = X[k] # rim = np.zeros_like(tim.getImage()) # rim.ravel()[I] = G # plt.clf() # plt.imshow(rim, interpolation='nearest', origin='lower') # plt.colorbar() # plt.savefig('rim-%i-%04i.png' % (it,k)) # print 'pix', k # sys.exit(0) makeplots(tractor, opt.resume, opt.suffix) step0 = opt.resume + 1 else: step0 = 0 tractor = create_tractor(opt) tractor.mp = mp # zero out invvar outside the model bounds. ds = tractor.getCatalog()[0] rd = ds.getRaDecCorners() for i,tim in enumerate(tractor.getImages()): poly = np.array([tim.getWcs().positionToPixel(RaDecPos(rdi[0], rdi[1])) for rdi in rd]) poly = poly[:-1,:] print 'Model bounding box in image', tim.name, 'coordinates:' print poly.shape print poly H,W = tim.shape xx,yy = np.meshgrid(np.arange(W), np.arange(H)) inside = point_in_poly(xx, yy, poly) iv = tim.getInvvar() iv[(inside == 0)] = 0. tim.setInvvar(iv) print 'Precomputing transformations...' ds = tractor.getCatalog()[0] # Split the grid-spread matrix into strips... async_results = [] for im in tractor.getImages(): args = [] H,W = ds.shape dy = 10 y = 0 while y <= H: args.append((ds, im, y, min(H, y+dy))) y += dy async_results.append(mp.map_async(_map_trans, args)) # Glue to strips back together... XX = [] for ar in async_results: Xblocks = ar.get() X = Xblocks[0] for xi in Xblocks[1:]: X.update(xi) XX.append(X) for im,X in zip(tractor.getImages(), XX): ds._normalizeTransformation(im, X) ds._setTransformation(im, X) print 'done precomputing.' makeplots(tractor, 0, opt.suffix) pfn = 'herschel-%02i%s.pickle' % (0, opt.suffix) pickle_to_file(tractor, pfn) print 'Wrote', pfn for im in tractor.getImages(): im.freezeAllBut('sky') for i in range(step0, opt.steps): if callgrind: callgrind.callgrind_start_instrumentation() tractor.optimize(damp=opt.damp, alphas=[1e-3, 1e-2, 0.1, 0.3, 1., 3., 10., 30., 100.]) if callgrind: callgrind.callgrind_stop_instrumentation() makeplots(tractor, 1 + i, opt.suffix) pfn = 'herschel-%02i%s.pickle' % (1 + i, opt.suffix) pickle_to_file(tractor, pfn) print 'Wrote', pfn
def apply_alignments(): from astrom_common import Affine T = fits_table('affines.fits') affs = Affine.fromTable(T) print('Read affines:', affs) ibright = dict([(fn.strip(), i) for fn in affs.filenames]) corners = {} for line in open('corners.txt').readlines(): line = line.strip() words = line.split() ras = np.array([float(words[i]) for i in [1, 3, 5, 7]]) decs = np.array([float(words[i]) for i in [2, 4, 6, 8]]) corners[words[0]] = (ras, decs) from astrometry.util.miscutils import point_in_poly fns = (glob('data/M31-*ST/proc_default/M31-*ST.phot.hdf5') + glob('data/M31-*ST/M31-*ST.phot.hdf5')) fns.sort() print('Files:', fns) veto_polys = [] for photfile in fns: basename = os.path.basename(photfile) basename = basename.replace('.phot.hdf5', '') print('Base name:', basename) brightfn = basename + '-bright.fits' ii = ibright[brightfn] aff = affs[ii] print('Reading', photfile) df = pd.read_hdf(photfile, key='data') ds = vaex.from_pandas(df) print(len(ds), 'rows') ra = ds.evaluate(ds['ra']) dec = ds.evaluate(ds['dec']) ra, dec = aff.apply(ra, dec) corner = corners[basename] Tleft = fits_table() Tleft.ra = ra Tleft.dec = dec Tleft.index = np.arange(len(Tleft)) ras, decs = corner poly = np.vstack((ras, decs)).T inside = point_in_poly(Tleft.ra, Tleft.dec, poly) print(np.sum(inside), 'of', len(Tleft), 'inside corners of this half-brick') inside_veto = np.zeros(len(Tleft), bool) for vp in veto_polys: inveto = point_in_poly(Tleft.ra, Tleft.dec, vp) inside_veto[inveto] = True print(np.sum(inside_veto), 'stars are inside the corners of previous half-bricks') print('inside:', type(inside)) inside[inveto] = False print(np.sum(inside), 'stars are uniquely in this half-brick') veto_polys.append(poly) outfn = 'out-%s.hdf5' % basename df[inside].to_hdf5(outfn, key='data', mode='w', format='table', complevel=9, complib='zlib') print('Wrote', outfn)
def apply_alignments(aff_fn, corners_fn, infns, pandas=True): from astrom_common import Affine T = fits_table(aff_fn) affs = Affine.fromTable(T) print('Read affines:', affs) ibright = dict([(fn.strip(), i) for i, fn in enumerate(T.filenames)]) corners = {} for line in open(corners_fn).readlines(): line = line.strip() words = line.split() ras = np.array([float(words[i]) for i in [1, 3, 5, 7]]) decs = np.array([float(words[i]) for i in [2, 4, 6, 8]]) corners[words[0]] = (ras, decs) from astrometry.util.miscutils import point_in_poly #fns1 = glob('data/M31-*ST/proc_default/M31-*ST.phot.hdf5') #fns2 = glob('data/M31-*ST/M31-*ST.phot.hdf5') #fns1.sort() #fns2.sort() #fns = fns1 + fns2 fns = infns print('Files:', fns) veto_polys = [] for photfile in fns: basename = os.path.basename(photfile) basename = basename.replace('.phot.hdf5', '') print('Base name:', basename) corner = corners[basename] ras, decs = corner poly = np.vstack((ras, decs)).T outfn2 = 'cut-%s.hdf5' % basename if os.path.exists(outfn2): print('File', outfn2, 'exists; skipping') veto_polys.append(poly) continue brightfn = basename + '-bright.fits' ii = ibright[brightfn] aff = affs[ii] print('Reading', photfile) if pandas: df = pd.read_hdf(photfile, key='data') ds = vaex.from_pandas(df) else: ds = vaex.open(photfile) def get_field(ds, col): if pandas: return ds.evaluate(ds[col]) else: return ds.evaluate(ds[col.upper()]) print(len(ds), 'rows') ra = get_field(ds, 'ra') dec = get_field(ds, 'dec') ra, dec = aff.apply(ra, dec) Tleft = fits_table() Tleft.ra = ra Tleft.dec = dec Tleft.index = np.arange(len(Tleft)) inside = point_in_poly(Tleft.ra, Tleft.dec, poly) print(np.sum(inside), 'of', len(Tleft), 'inside corners of this half-brick') inside_veto = np.zeros(len(Tleft), bool) for vp in veto_polys: inveto = point_in_poly(Tleft.ra, Tleft.dec, vp) inside_veto[inveto] = True print(np.sum(inside_veto), 'stars are inside the corners of previous half-bricks') print('inside:', type(inside), inside.dtype) inside[inside_veto] = False print(np.sum(inside), 'stars are uniquely in this half-brick') veto_polys.append(poly) outfn = 'out/out-%s.hdf5' % basename if pandas: df[inside].to_hdf(outfn, key='data', mode='w', format='table', complevel=9, complib='zlib') else: df = ds.take(np.flatnonzero(inside)).to_pandas_df() df.to_hdf(outfn, key='data', mode='w', format='table', complevel=9, complib='zlib') print('Wrote', outfn) outfn = 'cut/cut-%s.hdf5' % basename if pandas: df[np.logical_not(inside)].to_hdf(outfn, key='data', mode='w', format='table', complevel=9, complib='zlib') else: df = ds.take(np.flatnonzero(np.logical_not(inside))).to_pandas_df() df.to_hdf(outfn, key='data', mode='w', format='table', complevel=9, complib='zlib') print('Wrote', outfn)