def wrapper(off, full=False): t1 = time.time() off = self.unzip(off) chisq, amps, aicov = fun(off) t2 = time.time() if self.thumb_mapper and thumb_path and thumb_interval and self.i % thumb_interval == 0: thumbs = self.make_thumbs(off, amps) enmap.write_map(thumb_path % self.i, thumbs) if self.chisq0 is None: self.chisq0 = chisq if verbose: doff = off/utils.arcmin msg = "%4d %8.5f %8.5f" % (self.i,doff[0],doff[1]) msg += " %12.5e %7.3f" % (self.chisq0-chisq, t2-t1) famps = amps.reshape(-1,amps.shape[-1]) faicov = aicov.reshape((-1,)+aicov.shape[-2:]) for i in range(len(famps)): nsigma = np.sum(amatmul(faicov[i],famps[i])*famps[i])**0.5 msg += " %10.3f %8.1f" % (famps[i,0]/self.amp_unit, nsigma) print(msg) self.samples.offs.append(off) self.samples.amps.append(amps) self.samples.aicovs.append(aicov) self.samples.chisqs.append(chisq) self.i += 1 if not full: return chisq else: return chisq, amps, aicov
def solve(self, maxiter=100, cg_tol=1e-7, verbose=False, dump_dir=None): if np.sum(self.highres_mask) == 0: return None solver = cg.CG(self.A, self.rhs.reshape(-1), M=self.M) for i in range(maxiter): t1 = time.time() solver.step() t2 = time.time() if verbose: print "%5d %15.7e %5.2f" % (solver.i, solver.err, t2 - t1) if dump_dir is not None and solver.i in [1, 2, 5, 10, 20, 50 ] + range( 100, 10000, 100): m = enmap.ndmap(solver.x.reshape(self.shape), self.wcs) enmap.write_map(dump_dir + "/step%04d.fits" % solver.i, m) if solver.err < cg_tol: if dump_dir is not None: m = enmap.ndmap(solver.x.reshape(self.shape), self.wcs) enmap.write_map(dump_dir + "/step_final.fits", m) break tot_map = self.highres_mask * solver.x.reshape(self.shape) tot_div = self.highres_mask * self.tot_div # Get rid of the fourier padding ny, nx = tot_map.shape[-2:] tot_map = tot_map[..., :ny - self.ffpad[0], :nx - self.ffpad[1]] tot_div = tot_div[..., :ny - self.ffpad[0], :nx - self.ffpad[1]] return bunch.Bunch(map=tot_map, div=tot_div)
def monolithic(idir, ofile, verbose=True, slice=None, dtype=None): # Find the range of input tiles ipathfmt = idir + "/tile%(y)03d_%(x)03d.fits" itile1, itile2 = find_tile_range(ipathfmt) def read(fname): m = enmap.read_map(fname) if slice: m = eval("m" + slice) return m # Read the first and last tile to get the total dimensions m1 = read(ipathfmt % {"y": itile1[0], "x": itile1[1]}) m2 = read(ipathfmt % {"y": itile2[0] - 1, "x": itile2[1] - 1}) wy, wx = m1.shape[-2:] oshape = tuple( np.array(m1.shape[-2:]) * (itile2 - itile1 - 1) + np.array(m2.shape[-2:])) if dtype is None: dtype = m1.dtype omap = enmap.zeros(m1.shape[:-2] + oshape, m1.wcs, dtype) del m1, m2 # Now loop through all tiles and copy them in to the correct position for ty in range(itile1[0], itile2[0]): for tx in range(itile1[1], itile2[1]): m = read(ipathfmt % {"y": ty, "x": tx}) oy = ty - itile1[0] ox = tx - itile1[1] omap[..., oy * wy:(oy + 1) * wy, ox * wx:(ox + 1) * wx] = m if verbose: print ipathfmt % {"y": ty, "x": tx} enmap.write_map(ofile, omap)
def coadd_maps(imaps, ihits, omap, ohit, cont=False): # The first map will be used as a reference. All subsequent maps # must fit in its boundaries. if cont and os.path.exists(omap): return if args.verbose: print "Reading %s" % imaps[0] m = read_map(imaps[0]) if args.verbose: print"Reading %s" % ihits[0] w = apply_edge(apply_apod(apply_trim(read_div(ihits[0])))) wm = mul(w,m) for mif,wif in zip(imaps[1:],ihits[1:]): if args.verbose: print"Reading %s" % mif try: mi = read_map(mif) except IOError: if args.allow_missing: print "Can't read %s. Skipping" % mif continue else: raise if args.verbose: print"Reading %s" % wif wi = apply_edge(apply_apod(apply_trim(read_div(wif)))) # We may need to reproject maps if mi.shape != m.shape or str(mi.wcs.to_header()) != str(m.wcs.to_header()): mi = enmap.project(mi, m.shape, m.wcs, mode="constant") wi = enmap.project(wi, w.shape, w.wcs, mode="constant") w = add(w,wi) wm = add(wm,mul(wi,mi)) if args.verbose: print"Solving" m = solve(w,wm) if args.verbose: print"Writing %s" % omap enmap.write_map(omap, m) if args.verbose: print"Writing %s" % ohit enmap.write_map(ohit, w)
def wrapper(off, full=False): t1 = time.time() off = self.unzip(off) chisq, amps, aicov = fun(off) t2 = time.time() if self.thumb_mapper and thumb_path and thumb_interval and self.i % thumb_interval == 0: thumbs = self.make_thumbs(off, amps) enmap.write_map(thumb_path % self.i, thumbs) if self.chisq0 is None: self.chisq0 = chisq if verbose: doff = (off - self.off0)/utils.arcmin msg = "%4d %8.5f %8.5f" % (self.i,doff[0],doff[1]) msg += " %12.5e %7.3f" % (self.chisq0-chisq, t2-t1) famps, faicov = amps.reshape(-1), aicov.reshape(-1) for i in range(len(famps)): nsigma = (famps[i]**2*faicov[i])**0.5 msg += " %10.3f %8.1f" % (famps[i]/self.amp_unit, nsigma) print(msg) self.samples.offs.append(off) self.samples.amps.append(amps) self.samples.aicovs.append(aicov) self.samples.chisqs.append(chisq) self.i += 1 if not full: return chisq else: return chisq, amps, aicov
def dump_maps(ofile, tod, data, pos, amp, rad=args.radius*m2r, res=args.resolution*m2r): ncomp = amp.shape[1] dmaps, drhs, ddiv = make_maps(tod.astype(dtype), data, pos, ncomp, rad, res) dstack = stack_maps(drhs, ddiv, amp[:,0]) dmaps = np.concatenate((dmaps,[dstack]),0) dmaps = enmap.samewcs(dmaps, ddiv) enmap.write_map(ofile, dmaps[::-1])
def copy_mono(ifile, ofile, slice=None): if args.cont and os.path.exists(ofile): return if verbose: print("%3d copy_mono %s" % (comm.rank, ofile)) if args.dry: return tfile = ofile + ".tmp" map = read_map(ifile, slice=slice) enmap.write_map(tfile, map) shutil.move(tfile, ofile)
def copy_mono(ifile, ofile, slice=None): if args.cont and os.path.exists(ofile): return if verbose: print "%3d copy_mono %s" % (comm.rank, ofile) if args.dry: return tfile = ofile + ".tmp" map = read_map(ifile, slice=slice) enmap.write_map(tfile, map) shutil.move(tfile, ofile)
def add_maps(imaps, omap): if args.verbose: print "Reading %s" % imaps[0] m = nonan(enmap.read_map(imaps[0])) for mif in imaps[1:]: if args.verbose: print "Reading %s" % mif m += nonan(enmap.read_map(mif)) if args.verbose: "Writing %s" % omap enmap.write_map(omap, m)
def add_maps(imaps, omap): if args.verbose: print "Reading %s" % imaps[0] m = nonan(enmap.read_map(imaps[0]))*scales[0] for scale, mif in zip(scales[1:],imaps[1:]): if args.verbose: print "Reading %s" % mif m += nonan(enmap.read_map(mif))*scale if args.mean: m /= len(imaps) if args.verbose: "Writing %s" % omap enmap.write_map(omap, m)
def build_single(ifile, srcs, beam, ofile, mask_level=0, apod_size=16): imap = enmap.read_map(ifile) omap, oslice = pointsrcs.sim_srcs(imap.shape[-2:], imap.wcs, srcs, beam, return_padded=True) if mask_level: mask = omap > mask_level omap = 1-np.cos(np.minimum(1,ndimage.distance_transform_edt(1-mask)/16.0)*np.pi) omap = enmap.samewcs(omap, imap) omap = omap[oslice] enmap.write_map(ofile, omap)
def add_mono(ifiles, ofile, slice=None): if args.cont and os.path.exists(ofile): return if verbose: print "%3d add_mono %s" % (comm.rank, ofile) if args.dry: return tfile = ofile + ".tmp" omap = read_map(ifiles[0], slice=slice) for ifile in ifiles[1:]: omap += read_map(ifile, slice=slice) enmap.write_map(tfile, omap) shutil.move(tfile, ofile)
def read_map(fname, pbox, cname, write_cache=True, read_cache=True): if os.path.isdir(fname): fname = fname + "/tile%(y)03d_%(x)03d.fits" if read_cache and os.path.isfile(cname): map = enmap.read_map(cname).astype(dtype) else: map = retile.read_area(fname, pbox).astype(dtype) if write_cache: enmap.write_map(cname, map) #if map.ndim == 3: map = map[:1] return map
def add_maps(imaps, omap): if args.verbose: print "Reading %s" % imaps[0] m = nonan(enmap.read_map(imaps[0]))*scales[0] for scale, mif in zip(scales[1:],imaps[1:]): if args.verbose: print "Reading %s" % mif m2 = nonan(enmap.read_map(mif, geometry=(m.shape,m.wcs)))*scale n = min(len(m.preflat),len(m2.preflat)) m.preflat[:n] += m2.preflat[:n] if args.mean: m /= len(imaps) if args.verbose: "Writing %s" % omap enmap.write_map(omap, m)
def write(self, prefix, tag, ms): if not self.output: return for pid, (pattern, m) in enumerate(zip(self.patterns, ms)): oname = self.ofmt.format(name=self.name, pid=pid, el=pattern[0, 0] / utils.degree, az0=pattern[0, 1] / utils.degree, az1=pattern[1, 1] / utils.degree) oname = "%s%s_%s.%s" % (prefix, oname, tag, self.ext) if self.dof.comm.rank == 0: enmap.write_map(oname, m)
def retile(ipathfmt, opathfmt, itile1=(None,None), itile2=(None,None), otileoff=(0,0), otilenum=(None,None), ocorner=(-np.pi/2,-np.pi), otilesize=(675,675), comm=None, verbose=False, slice=None): """Given a set of tiles on disk with locations ipathfmt % {"y":...,"x":...}, retile them into a new tiling and write the result to opathfmt % {"y":...,"x":...}. The new tiling will have tile size given by otilesize[2]. Negative size means the tiling will to down/left instead of up/right. The corner of the tiling will be at sky coordinates ocorner[2] in radians. The new tiling will be pixel- compatible with the input tiling - w.g. the wcs will only differ by crpix. The output tiling will logically cover the whole sky, but only output tiles that overlap with input tiles will actually be written. This can be modified by using otileoff[2] and otilenum[2]. otileoff gives the tile indices of the corner tile, while otilenum indicates the number of tiles to write.""" # Set up mpi rank, size = (comm.rank, comm.size) if comm is not None else (0, 1) # Expand any scalars otilesize = np.zeros(2,int)+otilesize otileoff = np.zeros(2,int)+otileoff # Find the range of input tiles itile1, itile2 = find_tile_range(ipathfmt, itile1, itile2) # To fill in the rest of the information we need to know more # about the input tiling, so read the first tile ibase = enmap.read_map(ipathfmt % {"y":itile1[0],"x":itile1[1]}) if slice: ibase = eval("ibase"+slice) itilesize = ibase.shape[-2:] # Find the pixel position of our output corners according to the wcs. # This is the last place we need to do a coordinate transformation. # All the rest can be done in pure pixel logic. pixoff = np.round(ibase.sky2pix(ocorner)).astype(int) # Find the range of output tiles def pix2otile(pix, ioff, osize): return (pix-ioff)/osize otile1 = pix2otile(itile1*itilesize, pixoff, otilesize) otile2 = pix2otile(itile2*itilesize-1, pixoff, otilesize) otile1, otile2 = np.minimum(otile1,otile2), np.maximum(otile1,otile2) otile2 += 1 # We can now loop over output tiles cache = [None,None,None] oyx = [(oy,ox) for oy in range(otile1[0],otile2[0]) for ox in range(otile1[1],otile2[1])] for i in range(rank, len(oyx), size): otile = np.array(oyx[i]) # Find out which input tiles overlap with this output tile. # Our tile stretches from opix1:opix2 relative to the global input pixels opix1 = otile*otilesize + pixoff opix2 = (otile+1)*otilesize + pixoff # output tiles and input tiles may increase in opposite directions opix1, opix2 = np.minimum(opix1,opix2), np.maximum(opix1,opix2) try: omap = read_area(ipathfmt, [opix1,opix2],itile1=itile1, itile2=itile2,cache=cache, slice=slice) except IOError: continue oname = opathfmt % {"y":otile[0]+otileoff[0],"x":otile[1]+otileoff[1]} utils.mkdir(os.path.dirname(oname)) enmap.write_map(oname, omap) if verbose: print oname
def dump_maps(ofile, tod, data, pos, amp, rad=args.radius * m2r, res=args.resolution * m2r): ncomp = amp.shape[1] dmaps, drhs, ddiv = make_maps(tod.astype(dtype), data, pos, ncomp, rad, res) dstack = stack_maps(drhs, ddiv, amp[:, 0]) dmaps = np.concatenate((dmaps, [dstack]), 0) dmaps = enmap.samewcs(dmaps, ddiv) enmap.write_map(ofile, dmaps[::-1])
def solve_cg(eq, nmax=1000, ofmt=None, dump_interval=10): cg = CG(eq.A, eq.b, M=eq.M, dot=eq.dof.dot) while cg.i < nmax: with bench.mark("cg_step"): cg.step() dt = bench.stats["cg_step"]["time"].last L.info("CG step %5d %15.7e %6.1f %6.3f" % (cg.i, cg.err, dt, dt / len(eq.scans))) xmap, xjunk = eq.dof.unzip(cg.x) if ofmt and cg.i % dump_interval == 0 and myid == 0: enmap.write_map(ofmt % cg.i, eq.dof.unzip(cg.x)[0]) # Output benchmarking information bench.stats.write(benchfile) return cg.x
def add_mono(ifiles, ofile, slice=None, factors=None): if args.cont and os.path.exists(ofile): return if verbose: print("%3d add_mono %s" % (comm.rank, ofile)) if args.dry: return tfile = ofile + ".tmp" if factors is None: omap = read_map(ifiles[0], slice=slice) for ifile in ifiles[1:]: omap += read_map(ifile, slice=slice) else: omap = read_map(ifiles[0], slice=slice) * factors[0] for i in range(1, len(ifiles)): omap += read_map(ifiles[i], slice=slice) * factors[i] enmap.write_map(tfile, omap) shutil.move(tfile, ofile)
def build_single(ifile, srcs, beam, ofile, mask_level=0, apod_size=16): imap = enmap.read_map(ifile) omap, oslice = pointsrcs.sim_srcs(imap.shape[-2:], imap.wcs, srcs, beam, return_padded=True) if mask_level: mask = omap > mask_level omap = 1 - np.cos( np.minimum(1, ndimage.distance_transform_edt(1 - mask) / 16.0) * np.pi) omap = enmap.samewcs(omap, imap) omap = omap[oslice] enmap.write_map(ofile, omap)
def read_map(fname, pbox, name=None, cache_dir=None, dtype=None, read_cache=False): if os.path.isdir(fname): fname = fname + "/tile%(y)03d_%(x)03d.fits" if read_cache: map = enmap.read_map(cache_dir + "/" + name) if dtype is not None: map = map.astype(dtype) else: map = retile.read_area(fname, pbox).astype(dtype) if dtype is not None: map = map.astype(dtype) if cache_dir is not None and name is not None: enmap.write_map(cache_dir + "/" + name, map) #if map.ndim == 3: map = map[:1] return map
def write_map(name, map, ext="fits", merged=True): if not merged: # Write as individual tiles in directory of the specified name utils.mkdir(name) for pos, tile in zip(map.loc_pos, map.tiles): enmap.write_map( name + "/tile%03d_%03d.%s" % (tuple(pos) + (ext, )), tile) else: # Write to a single file. This currently creates the full map # in memory while writing. It is unclear how to avoid this # without bypassing pyfits or becoming super-slow. if map.comm.rank == 0: canvas = enmap.zeros(map.shape, map.wcs, map.dtype) else: canvas = None dmap2enmap(map, canvas) if map.comm.rank == 0: enmap.write_map(name, canvas)
def coadd_maps(imaps, ihits, omap, ohit, cont=False, ncomp=-1): # The first map will be used as a reference. All subsequent maps # must fit in its boundaries. if cont and os.path.exists(omap): return if args.verbose: print("Reading %s" % imaps[0]) if ncomp < 0: shape, wcs = enmap.read_map_geometry(imaps[0]) ncomp = 0 if len(shape) == 2 else shape[0] m = read_map(imaps[0], ncomp=ncomp) if args.verbose: print("Reading %s" % ihits[0]) w = apply_edge(apply_apod(apply_trim(read_div(ihits[0], ncomp=ncomp)))) if args.warn and np.any(w.preflat[0] < 0): print("Negative weight in %s" % ihits[0]) wm = mul(w, m) for i, (mif, wif) in enumerate(zip(imaps[1:], ihits[1:])): if args.verbose: print("Reading %s" % mif) try: mi = read_map(mif, m.shape, m.wcs, ncomp=ncomp) except (IOError, OSError): if args.allow_missing: print("Can't read %s. Skipping" % mif) continue else: raise if args.verbose: print("Reading %s" % wif) wi = apply_edge( apply_apod(apply_trim(read_div(wif, m.shape, m.wcs, ncomp=ncomp)))) if args.warn and np.any(wi.preflat[0] < 0): print("Negative weight in %s" % ihits[i + 1]) ## We may need to reproject maps #if mi.shape != m.shape or str(mi.wcs.to_header()) != str(m.wcs.to_header()): # mi = enmap.extract(mi, m.shape, m.wcs) # wi = enmap.extract(wi, w.shape, w.wcs) w = add(w, wi) wm = add(wm, mul(wi, mi)) if args.verbose: print("Solving") m = solve(w, wm) if args.verbose: print("Writing %s" % omap) enmap.write_map(omap, m) if args.verbose: print("Writing %s" % ohit) enmap.write_map(ohit, w)
def coadd_mono(imapfiles, idivfiles, omapfile, odivfile=None): if args.cont and os.path.exists(omapfile) and (odivfile is None or os.path.exists(odivfile)): return if verbose: print "%3d coadd_mono %s" % (comm.rank, omapfile) if args.dry: return omap, odiv = None, None tmapfile = omapfile + ".tmp" if odivfile: tdivfile = odivfile + ".tmp" for imapfile, idivfile in zip(imapfiles, idivfiles): imap = read_map(imapfile) idiv = read_map(idivfile, slice=".preflat[0]") if omap is None: omap, odiv = imap*0, idiv*0 omap += imap*idiv odiv += idiv mask = odiv > 0 omap[...,mask] /= odiv[mask] enmap.write_map(tmapfile, omap) if odivfile: enmap.write_map(tdivfile, odiv) shutil.move(tmapfile, omapfile) if odivfile: shutil.move(tdivfile, odivfile)
def wrapper(off): t1 = time.time() chisq, amps, aicov = fun(self.unzip(off)) t2 = time.time() if thumb_path and thumb_interval and self.i % thumb_interval == 0: tod2 = self.tod.copy() self.P.forward(tod2, amps, pmul=-1) thumbs = self.thumb_mapper.map(tod2) enmap.write_map(thumb_path % self.i, thumbs) del tod2, thumbs if self.chisq0 is None: self.chisq0 = chisq if verbose: msg = "%4d %6.3f %6.3f" % (self.i,off[0],off[1]) for i in range(len(amps)): nsigma = (amps[i,0]**2*aicov[i,0])**0.5 msg += " %7.3f %4.1f" % (amps[i,0]/self.amp_unit, nsigma) msg += " %12.5e %7.2f" % (self.chisq0-chisq, t2-t1) print(msg) self.i += 1 return chisq
def coadd_mono(imapfiles, idivfiles, omapfile, odivfile=None): if args.cont and os.path.exists(omapfile) and (odivfile is None or os.path.exists(odivfile)): return if verbose: print("%3d coadd_mono %s" % (comm.rank, omapfile)) if args.dry: return omap, odiv = None, None tmapfile = omapfile + ".tmp" if odivfile: tdivfile = odivfile + ".tmp" for imapfile, idivfile in zip(imapfiles, idivfiles): imap = read_map(imapfile) idiv = read_map(idivfile, slice=".preflat[0]") if omap is None: omap, odiv = imap * 0, idiv * 0 omap += imap * idiv odiv += idiv mask = odiv > 0 omap[..., mask] /= odiv[mask] enmap.write_map(tmapfile, omap) if odivfile: enmap.write_map(tdivfile, odiv) shutil.move(tmapfile, omapfile) if odivfile: shutil.move(tdivfile, odivfile)
def wrapper(off): t1 = time.time() chisq, amps, aicov = fun(self.unzip(off)) t2 = time.time() if thumb_path and thumb_interval and self.i % thumb_interval == 0: tod2 = self.tod.copy() self.P.forward(tod2, amps, pmul=-1) thumbs = self.thumb_mapper.map(tod2) enmap.write_map(thumb_path % self.i, thumbs) del tod2, thumbs if self.chisq0 is None: self.chisq0 = chisq if verbose: msg = "%4d %6.3f %6.3f" % (self.i, off[0], off[1]) for i in range(len(amps)): nsigma = (amps[i, 0]**2 * aicov[i, 0])**0.5 msg += " %7.3f %4.1f" % (amps[i, 0] / self.amp_unit, nsigma) msg += " %12.5e %7.2f" % (self.chisq0 - chisq, t2 - t1) print(msg) self.i += 1 return chisq
def __call__(fself, x): params = param_zipper.unzip(x) chisqs, amps, adiv = self.calc_chisqs_amps_cached( params.poss, params.taus) #print amps[433], adiv[433] chisq = np.sum(chisqs) if verbosity > 0 and fself.i % 10 == 0: if verbosity == 1: print("%6d %12.4f" % (fself.i, chisq / self.ndata)) else: print("%6d %12.4f" % (fself.i, chisq / self.ndata) + " %6.3f" * len(x) % tuple(x)) sys.stdout.flush() if False and fself.i % 10000 == 0: model = self.model(params.poss, amps, params.taus) map = enmap.enmap( [self.rdata.tod, model, self.rdata.tod - model]) enmap.write_map(args.odir + "/foo%06d.fits" % fself.i, map) fself.i += 1 return chisq
def get_coadded_tile(mapinfo, box, obeam=None, ncomp=1, dump_dir=None, verbose=False): if not overlaps_any(box, boxes): return None mapset = mapinfo.read(box, pad=pad, dtype=dtype, verbose=verbose, ncomp=ncomp) if mapset is None: return None if all([d.insufficient for d in mapset.datasets]): return None jointmap.sanitize_maps(mapset) jointmap.build_noise_model(mapset) if len(mapset.datasets) == 0: return None if all([d.insufficient for d in mapset.datasets]): return None jointmap.setup_beams(mapset) jointmap.setup_target_beam(mapset, obeam) jointmap.setup_filter(mapset, mode=args.filter_mode) jointmap.setup_background_spectrum(mapset) mask = jointmap.get_mask_insufficient(mapset) coadder = jointmap.Coadder(mapset) rhs = coadder.calc_rhs() if dump_dir: enmap.write_map(dump_dir + "/rhs.fits", rhs) enmap.write_map(dump_dir + "/ps_rhs.fits", np.abs(enmap.fft(rhs.preflat[0]))**2) map = coadder.calc_coadd(rhs, dump_dir=dump_dir, verbose=verbose)#, maxiter=1) if dump_dir: enmap.write_map(dump_dir + "/ps_map.fits", np.abs(enmap.fft(mapdiag(map)))**2) div = coadder.tot_div #C = 1/mapset.datasets[0].iN res = bunch.Bunch(rhs=rhs*mask, map=map*mask, div=div*mask)#, C=C) #res = bunch.Bunch(rhs=rhs, map=map, div=div)#, C=C) return res
def get_coadded_tile(mapinfo, box, obeam=None, ncomp=1, dump_dir=None, verbose=False): if not overlaps_any(box, boxes): return None mapset = mapinfo.read(box, pad=pad, dtype=dtype, verbose=verbose, ncomp=ncomp) if mapset is None: return None if all([d.insufficient for d in mapset.datasets]): return None jointmap.sanitize_maps(mapset) jointmap.build_noise_model(mapset) if len(mapset.datasets) == 0: return None if all([d.insufficient for d in mapset.datasets]): return None jointmap.setup_beams(mapset) jointmap.setup_target_beam(mapset, obeam) jointmap.setup_filter(mapset, mode=args.filter_mode) jointmap.setup_background_spectrum(mapset) mask = jointmap.get_mask_insufficient(mapset) if args.wiener: coadder = jointmap.Wiener(mapset) else: coadder = jointmap.Coadder(mapset) rhs = coadder.calc_rhs() if dump_dir: enmap.write_map(dump_dir + "/rhs.fits", rhs) enmap.write_map(dump_dir + "/ps_rhs.fits", np.abs(enmap.fft(rhs.preflat[0]))**2) map = coadder.calc_map(rhs, dump_dir=dump_dir, verbose=verbose, cg_tol=args.cg_tol)#, maxiter=1) if dump_dir: enmap.write_map(dump_dir + "/ps_map.fits", np.abs(enmap.fft(mapdiag(map)))**2) div = coadder.tot_div #C = 1/mapset.datasets[0].iN res = bunch.Bunch(rhs=rhs*mask, map=map*mask, div=div*mask)#, C=C) #res = bunch.Bunch(rhs=rhs, map=map, div=div)#, C=C) return res
def combine_srcmaps(imapfiles, isrcmapfiles, idivfiles, isrcdivfiles, omapfile, odivfile=None, isrcfiles=None): if args.cont and os.path.exists(omapfile) and (odivfile is None or os.path.exists(odivfile)): return if verbose: print("%3d combine_srcmaps %s" % (comm.rank, omapfile)) if args.dry: return omap, odiv = None, None tmapfile = omapfile + ".tmp" if odivfile: tdivfile = odivfile + ".tmp" nfile = len(imapfiles) for fi in range(nfile): imap = read_map(imapfiles[fi]) isrcmap = read_map(isrcmapfiles[fi]) if isrcfiles is not None: isrc = read_map(isrcfiles[fi]) imap += isrc isrcmap += isrc del isrc idiv = read_map(idivfiles[fi], slice=".preflat[0]") isrcdiv = read_map(isrcdivfiles[fi], slice=".preflat[0]") idiv -= isrcdiv if omap is None: omap, odiv = imap * 0, idiv * 0 omap += imap * idiv omap += isrcmap * isrcdiv odiv += idiv odiv += isrcdiv del imap, isrcmap, idiv, isrcdiv mask = odiv > 0 omap[..., mask] /= odiv[mask] enmap.write_map(tmapfile, omap) if odivfile: enmap.write_map(tdivfile, odiv) shutil.move(tmapfile, omapfile) if odivfile: shutil.move(tdivfile, odivfile)
def get_coadded_tile(mapinfo, box, obeam=None, ncomp=1, dump_dir=None, verbose=False): if not overlaps_any(np.sort(box, 0), boxes): return None mapset = mapinfo.read(box, pad=pad, dtype=dtype, verbose=verbose, ncomp=ncomp) if mapset is None: return None if all([d.insufficient for d in mapset.datasets]): return None jointmap.sanitize_maps(mapset, detrend=args.detrend) jointmap.build_noise_model(mapset) if len(mapset.datasets) == 0: return None if all([d.insufficient for d in mapset.datasets]): return None jointmap.setup_beams(mapset) jointmap.setup_target_beam(mapset, obeam) jointmap.setup_filter(mapset, mode=args.filter_mode) jointmap.setup_background_spectrum(mapset) mask = jointmap.get_mask_insufficient(mapset) if args.wiener: coadder = jointmap.Wiener(mapset) else: coadder = jointmap.Coadder(mapset) rhs = coadder.calc_rhs() if dump_dir: enmap.write_map(dump_dir + "/rhs.fits", rhs) enmap.write_map(dump_dir + "/ps_rhs.fits", np.abs(enmap.fft(rhs.preflat[0]))**2) with open(dump_dir + "/names.txt", "w") as nfile: for name in coadder.names: nfile.write(name + "\n") ls, weights = coadder.calc_debug_weights() np.savetxt( dump_dir + "/weights_1d.txt", np.concatenate( [ls[None], weights.reshape(-1, weights.shape[-1])], 0).T, fmt="%15.7e") ls, noisespecs = coadder.calc_debug_noise() np.savetxt( dump_dir + "/noisespecs_1d.txt", np.concatenate( [ls[None], noisespecs.reshape(-1, noisespecs.shape[-1])], 0).T, fmt="%15.7e") map = coadder.calc_map(rhs, dump_dir=dump_dir, verbose=verbose, cg_tol=args.cg_tol) #, maxiter=1) if dump_dir: enmap.write_map(dump_dir + "/ps_map.fits", np.abs(enmap.fft(mapdiag(map)))**2) div = coadder.tot_div #C = 1/mapset.datasets[0].iN res = bunch.Bunch(rhs=rhs * mask, map=map * mask, div=div * mask) #, C=C) #res = bunch.Bunch(rhs=rhs, map=map, div=div)#, C=C) return res
myscans, [], noise=False, hits=False) else: signal_map = mapmaking.SignalDmap(myscans, mysubs, imap, sys=args.sys) precon = mapmaking.PreconDmapBinned(signal_map, signal_cut, myscans, [], noise=False, hits=False) # We can now actually perform the postfilter L.info("Postfiltering") postfilter = mapmaking.PostPickup(myscans, signal_map, signal_cut, precon, daz=args.daz, nt=args.nt, weighted=args.weighted, deslope=args.deslope) omap = postfilter(imap) # And output the resulting map L.info("Writing") if not is_dmap: if comm.rank == 0: enmap.write_map(args.omap, omap) else: omap.write(args.omap)
nscan = comm.allreduce(nscan) det_hit = utils.allreduce(det_hit, comm) offsets = utils.allreduce(offsets, comm) offsets[det_hit > 0] /= det_hit[det_hit > 0][:, None] # Reduce to our actual set of detectors dets = np.where(det_hit > 0)[0] offsets = offsets[dets] det_hit = det_hit[dets] ndet = len(dets) # And output if comm.rank == 0: # Output our files proot = root + "el_%s_az_%s_%s" % tuple([str(w) for w in pat]) enmap.write_map(proot + "_rhs.fits", rhs) enmap.write_map(proot + "_hits.fits", hits) with h5py.File(proot + "_info.hdf", "w") as hfile: hfile["inspec"] = inspec hfile["srate"] = srate hfile["ids"] = np.array(pids) hfile["hits"] = hits hfile["pattern"] = pat hfile["speed"] = speed hfile["offsets"] = offsets / utils.degree # [det,{el,az}] in deg for k, v in site.items(): hfile["site/" + k] = v for k, v in hits.wcs.to_header().items(): hfile["wcs/" + k] = v tot_rhs += rhs tot_hits += hits
shape, wcs = jointmap.read_geometry(args.area) tshape = np.array([args.tsize,args.tsize]) ntile = np.floor((shape[-2:]+tshape-1)/tshape).astype(int) tyx = [(y,x) for y in range(ntile[0]-1,-1,-1) for x in range(ntile[1])] for i in range(comm.rank, len(tyx), comm.size): y, x = tyx[i] ofile_map = args.odir + "/map_padtile/tile%(y)03d_%(x)03d.fits" % {"y":y,"x":x} ofile_div = args.odir + "/div_padtile/tile%(y)03d_%(x)03d.fits" % {"y":y,"x":x} utils.mkdir(os.path.dirname(ofile_map)) utils.mkdir(os.path.dirname(ofile_div)) if args.cont and os.path.isfile(ofile_map): print "%3d skipping %3d %3d (already done)" % (comm.rank, y, x) continue print "%3d processing %3d %3d" % (comm.rank, y, x) tpos = np.array(tyx[i]) pbox = np.array([tpos*tshape,np.minimum((tpos+1)*tshape,shape[-2:])]) box = enmap.pix2sky(shape, wcs, pbox.T).T res = get_coadded_tile(mapinfo, box, obeam=obeam, ncomp=args.ncomp, verbose=args.verbose) if res is None: res = jointmap.make_dummy_tile((args.ncomp,)+shape[-2:], wcs, box, pad=pad, dtype=dtype) enmap.write_map(ofile_map, res.map) enmap.write_map(ofile_div, res.div) else: # Single arbitrary tile if not overlaps_any(bounds, boxes): print "No data in selected region" else: res = get_coadded_tile(mapinfo, bounds, obeam=obeam, ncomp=args.ncomp, dump_dir=args.odir, verbose=args.verbose) enmap.write_map(args.odir + "/map.fits", res.map) enmap.write_map(args.odir + "/div.fits", res.div) #enmap.write_map(args.odir + "/C.fits", res.C)
def write_padtile(ofile, map): enmap.write_map(ofile, map, extra={"PAD": args.pad, "EDGE": args.apod_edge})
pmap = pmat.PmatMap(scan, area) rhs = enmap.zeros((3,)+shape, wcs, dtype) div = rhs*0 with bench.mark("rhs"): pmap.backward(tod, rhs) with bench.mark("div"): pmap.backward(wtod,div) div = div[0] map = rhs.copy() map[:,div>0] /= div[div>0] map = map[0] # Crop the outermost pixel, where outside hits will have accumulated map, div, area = [m[...,1:-1,1:-1] for m in [map,div,area]] # Find the local scanning velocity at the source position scan_vel = find_scan_vel(scan, srcpos[:,sid], aspeed) hbox = scan.box dur = d.nsamp*d.srate el = np.mean(scan.box[:,2]) az = scan.box[:,1] sdata.append(bunch.Bunch( map=map, div=div, srcpos=srcpos[:,sid], sid=sid, vel=scan_vel, fknee=fknee, alpha=args.alpha, id=id, ctime=tref, dur=dur, el=el, az=az, site=d.site, off=d.point_correction)) del tod, wtod, d write_sdata(oname, sdata) if args.minimaps: for i, sdat in enumerate(sdata): enmap.write_map("%s/%s_srcmap_%03d.fits" % (args.odir, oid, sdat.sid), sdat.map)
L.info("Building source map") dump_maps(tdir + "/map.hdf", d.tod, d, pos_fid, amp_fid) if args.grid: L.info("Building pos grid") g = np.abs(args.grid) p = params.copy() if args.grid < 0: p.strong[:] = False grid = grid_pos(d, p, shape=(g, g)) grid -= np.max(grid) maxpos = grid.pix2sky(np.unravel_index(np.argmax(grid), grid.shape)) print np.max(grid), maxpos * 180 * 60 / np.pi if np.sum(maxpos**2)**0.5 <= pos_rel_max * 2 / 3: params.pos_rel[...] = maxpos enmap.write_map(tdir + "/grid.hdf", grid) L.info("Drawing %d samples" % args.nsamp) sampler = HybridSampler(d, params, dpos, dbeam, nstep=args.thin, prior=prior, dist=np.random.standard_cauchy) pos_rel = np.zeros([args.nsamp, 2]) beam_rel = np.zeros([args.nsamp, 3]) amp = np.zeros((args.nsamp, ) + params.amp_fid.shape) for i in range(-args.burnin, args.nsamp): if i <= 0 and (-i) % 25 == 0: sampler.adjust()
from enlib import enmap parser = argparse.ArgumentParser() parser.add_argument("omap") parser.add_argument("-D", "--diameter", type=float, default=30) parser.add_argument("-n", "--npix", type=int, default=800) parser.add_argument("--proj", type=str, default="cea") args = parser.parse_args() r = args.diameter * np.pi / 180 / 2 shape, wcs = enmap.geometry(pos=[[-r, -r], [r, r]], shape=(args.npix, args.npix), proj=args.proj) def linpos(n, b): return b[0] + (np.arange(n) + 0.5) * (b[1] - b[0]) / n alpha = enmap.zeros((2, ) + shape, wcs) pos = enmap.posmap(shape, wcs) # I'm not sure how to compute this in general, so here's a specialization # for cylindrical projections if args.proj == "cea" or args.proj == "car": dec = pos[0, :, 0] ra = pos[1, 0, :] lindec = linpos(shape[0], enmap.box(shape, wcs)[:, 0]) scale = 1 / np.cos(dec) - 1 alpha[0] = (lindec - dec)[:, None] alpha[1] = ra[None, :] * scale[:, None] enmap.write_map(args.omap, alpha * 180 * 60 / np.pi)
print "Processing row %5d/%d" % (r1, omap.shape[-2]) # Output map coordinates osub = omap[...,r1:r2,:] pmap = osub.posmap() # Coordinate transformation if args.rot: s1,s2 = args.rot.split(",") opos = coordinates.transform(s2, s1, pmap[::-1], pol=ncomp==3) pmap[...] = opos[1::-1] if len(opos) == 3: psi = -opos[2].copy() del opos # Switch to healpix convention theta = np.pi/2-pmap[0] phi = pmap[1] # Evaluate map at these locations if args.order == 0: pix = healpy.ang2pix(nside, theta, phi) osub[:] = imap[:,pix] elif args.order == 1: for i in range(ncomp): osub[i] = healpy.get_interp_val(imap[i], theta, phi) # Rotate polarization if necessary if args.rot and ncomp==3: osub[1:3] = enmap.rotate_pol(osub[1:3], psi) print "Writing" if args.scalar: omap = omap.preflat[0] enmap.write_map(args.ofile, omap) print "Done"
def fit_grid(self, verbose=False, grid_res=0.6*utils.arcmin, super=10): self.verbose = verbose t1 = time.time() if verbose: print "Building coarse likelihood grid" ngrid = int(np.round(2*self.lik.rmax/grid_res)) dchisqs, amps = self.likgrid(self.lik.rmax, ngrid, super=super, verbose=verbose) if np.all(dchisqs == 0): raise ValueError("featureless likelihood") if False and verbose: for i,s in enumerate(self.sdata): enmap.write_map("map_%d.fits"%i,s.map) enmap.write_map("div_%d.fits"%i,s.div) enmap.write_map("white_%d.fits"%i,s.map*s.div**0.5) enmap.write_map("pchisq_%d.fits"%i,s.map**2*s.div) enmap.write_map("pchisq_smooth_%d.fits%i",enmap.smooth_gauss(s.map**2*s.div,0.6*utils.arcmin)) enmap.write_map("dchisqs.fits",dchisqs) # Find local dchisq maxima maxmap = ndimage.maximum_filter(dchisqs, super) peaks = np.where((dchisqs==maxmap)*(maxmap>0)) maxvals = dchisqs[peaks] maxpos = dchisqs.pix2sky(peaks) # Why isn't this just amps[:,peaks] or similar? maxamps = amps.reshape(amps.shape[0],-1)[:,np.ravel_multi_index(peaks, amps.shape[-2:])] inds = np.argsort(maxvals)[::-1] maxvals = maxvals[inds] maxpos = maxpos[:,inds] maxamps = maxamps[:,inds] # Perform ML fit for the highest one dpos = optimize.fmin_powell(self.calc_chisq_wrapper, maxpos[:,0]/self.scale, disp=False)*self.scale res = self.calc_full_result(dpos, marginalize=False) if False and verbose: for i, m in enumerate(res.models): enmap.write_map("model_%d.fits"%i,m) resid = self.sdata[i].map-m enmap.write_map("resid_%d.fits"%i,resid) pchisq = resid**2*sdata[i].div pchisq_smooth = enmap.smooth_gauss(pchisq, 0.6*utils.arcmin) enmap.write_map("pchisq_smooth_resid.fits",pchisq_smooth) print np.sum((self.sdata[i].map-m)**2*self.sdata[i].div) - self.lik.chisq0 # Ideally we would integrate over the full likelihood, not # just the peaks. But the peaks have higher weight # and should be distributed representatively. Using just the # peaks makes it easy to compare with our ML-fit, which is also # a single point. So we loop over just the peaks here. maxvals = maxvals[1:] maxpos = maxpos[:,1:] maxamps = maxamps[:,1:] P = np.exp(0.5*(maxvals-res.dchisq)) P0 = 1/(1+np.sum(P)) P *= P0 # Marginalize over peaks res.dpos = P0*res.dpos + np.sum(P*maxpos,-1) off = maxpos-res.dpos[:,None] res.pcov = P0*res.pcov + np.sum(P*off[:,None]*off[None,:],-1) res.ddpos = np.diag(res.pcov)**0.5 res.pcorr = res.pcov[0,1]/res.ddpos[0]/res.ddpos[1] res.amps = P0*res.amps + np.sum(P*maxamps,-1) res.damps = (res.damps**2 + np.sum(P*(maxamps-res.amps[:,None])**2,-1))**0.5 # For the significance, we will use the difference from our peak to our # strongest competitor res.dchisq= res.dchisq - maxvals[0] # Base nsigma on the sources res.nsigma= max(0,res.dchisq)**0.5 res.time = time.time()-t1 return res
parser.add_argument("-r", "--res", type=float, default=0.1) parser.add_argument("-s", "--seed", type=int, default=0) parser.add_argument("--refsys-res",type=float, default=10) args = parser.parse_args() # Generate the inputs to our simulator, which should be enmaps. # First generate the simulated CMB res = args.res*utils.degree shape, wcs = pixie.fullsky_geometry(res, dims=(3,)) np.random.seed(args.seed) print colors.green + "Simulating reference blackbody" + colors.reset rshape, rwcs = pixie.fullsky_geometry(args.refsys_res*utils.degree, dims=(3,)) map_ref = pixie.sim_reference_blackbody(rshape, rwcs) extra = { "NAME": "REFERENCE", "BEAM": "NONE", "SPEC": "BLACK" } enmap.write_map(args.odir + "/map_ref.fits", map_ref, extra=extra) #map_ref_test = pixie.sim_reference_blackbody(rshape, rwcs, 19.6) #enmap.write_map(args.odir + "/map_ref_test.fits", map_ref_test, extra=extra) # Then project our dust map onto our target coordinates print colors.green + "Projecting dust model" + colors.reset heal_dust = pixie.read_healpix(args.idir + "/test_therm_600p0_512_v2.fits") map_dust = pixie.project_healpix(shape, wcs, heal_dust, rot="gal,ecl", verbose=True) extra = { "NAME": "DUST", "BEAM": "GAUSS", "FWHM": 0.07, "SPEC": "GRAY", "TBODY": 19.6, "BETA": 1.59,
import numpy as np, argparse from enlib import enmap from scipy import ndimage parser = argparse.ArgumentParser() parser.add_argument("pos") parser.add_argument("template") parser.add_argument("ofile") parser.add_argument("-r", "--radius", type=float, default=3) parser.add_argument("-c", "--columns", type=str, default="3,5,2") parser.add_argument("-t", "--threshold", type=float, default=0) args = parser.parse_args() cols = [int(w) for w in args.columns.split(",")] srcinfo = np.loadtxt(args.pos)[:,cols] pos = srcinfo[np.abs(srcinfo[:,2])>=args.threshold][:,:2] * np.pi/180 map = enmap.read_map(args.template) pix = map.sky2pix(pos.T).T.astype(int) pixrad = (map.area()/map.npix)**0.5 mrad = args.radius*np.pi/180/60/pixrad mask = enmap.zeros(map.shape[-2:], map.wcs)+1 mask[pix[:,0],pix[:,1]] = 0 mask = enmap.enmap(1.0*(ndimage.distance_transform_edt(mask) > mrad), map.wcs) enmap.write_map(args.ofile, mask)
dirname = os.path.dirname(fname) if len(dirname) == 0: dirname = "." base_ext = os.path.basename(fname) # Find the extension. Using the last dot does not work for .fits.gz. # Using the first dot in basename does not work for foo2.5_bar.fits. # Will therefore handle .gz as a special case. if base_ext.endswith(".gz"): dot = base_ext[:-3].rfind(".") else: dot = base_ext.rfind(".") if dot < 0: dot = len(base_ext) base = base_ext[:dot] ext = base_ext[dot+1:] return dirname, base, ext for ifile in args.ifiles: m = enmap.read_map(ifile) if args.slice: m = eval("m" + args.slice) N = m.shape[:-2] for i, comp in enumerate(m.preflat): I = np.unravel_index(i, N) if len(N) > 0 else [] if args.symmetric and np.any(np.sort(I) != I): continue ndigits = [get_num_digits(n) for n in N] compname = "_".join(["%0*d" % (ndig,ind) for ndig,ind in zip(ndigits,I)]) if len(N) > 0 else "" dir, base, ext = split_file_name(ifile) oname = dir + "/" + base + "_" + compname + "." + ext if args.verbose: print(oname) enmap.write_map(oname, comp)
smod = project_maps(fit.models, fit.poss_cel, shape, wcs) smap = enmap.samewcs([smap,smod,smap-smod],smap) # And build scaled coadd. When map is divided by a, div = ivar # is multiplied by a**2. Points in very noisy regions can have # large error bars in the amplitude, and hence might randomly # appear to have a very strong signal. We don't want to let these # dominate, so downweight points with atypically high variance. # FIXME: Need a better approach than this. This fixed some things, # but broke others. #beam_exposure = [] #for i, s in enumerate(sdata): # med_div = np.median(s.div[s.div!=0]) # profile = fit.models[i]/fit.amps[i] # avg_div = np.sum(s.div*profile)/np.sum(profile) # beam_exposure.append(avg_div/med_div) #beam_exposure = np.array(beam_exposure) #weight = np.minimum(1,beam_exposure*1.25)**4 weight = 1 trhs = np.sum(smap*sdiv*(fit.amps*weight)[None,:,None,None],1) tdiv = np.sum(sdiv*(fit.amps*weight)[:,None,None]**2,0) tdiv[tdiv==0] = np.inf tmap = trhs/tdiv # And write them enmap.write_map(args.odir + "/totmap_%s_%02d.fits" % (sdata[0].id,gi), tmap) for i in range(fit.nsrc): enmap.write_map(args.odir + "/shiftmap_%s_%03d.fits" % (sdata[i].id, sdata[i].sid), smap[:,i]) # Output unshifted map too omap = enmap.samewcs([sdata[i].map.preflat[0],fit.models[i],sdata[i].map.preflat[0]-fit.models[i]],sdata[i].map) enmap.write_map(args.odir + "/fitmap_%s_%03d.fits" % (sdata[i].id,sdata[i].sid), omap) f.close()
dhit = hit2+hit vmap = calc_map_block_ivar(dmap, bs) mask = (hit>quant(hit,qlim)*hitlim) & (hit2>quant(hit2,qlim)*hitlim) mask &= (hit<quant(hit,qlim)) & (hit2<quant(hit2,qlim)) # Reduce dhit and mask to vmap's resolution dhit = calc_map_block_mean(dhit, bs) mask = calc_map_block_mean(mask, bs)>0 ratio = utils.medmean(vmap[mask]/dhit[mask]) # And compute the sensitivity ratio *= get_bias(bs)**2 ratios.append(ratio) sens = (ratio*args.srate)**-0.5 print "%d-%d %7.2f" % (i+1,i, sens) map, hit = map2, hit2 # Ratio has units 1/(uK^2*sample), and gives us the conversion # factor between hitmaps and inverse variance maps, which we call # div maps by convention from tenki. ratio = np.mean(ratios) print "mean %7.2f" % (ratio*args.srate)**-0.5 if args.dry_run: sys.exit() # Ok, now that we have the mean conversion factor between hits and divs, # loop through all hits again, and output div maps for i, (hitfile, divfile) in enumerate(zip(hitfiles, divfiles)): hit = enmap.read_map(hitfile) div = hit*ratio print "Writing %s" % divfile enmap.write_map(divfile, div)
stats = sampler.build_stats() # But find the ML point exactly, so we don't have to waste too many samples pos_ml = optimize.fmin_powell(lik, stats.pos/utils.arcmin, disp=0)*utils.arcmin stats.pos_ml = pos_ml # Output to stdout and to our indiviudal files msg = format_stats(stats, thumb_data) f.write(msg + "\n") f.flush() #mpiwrite(f, msg + "\n") print msg sys.stdout.flush() if args.minimaps: diag = lik.diag_plot(stats.pos) enmap.write_map(args.odir + "/diag_%s.fits" % oname, diag) enmap.write_map(args.odir + "/lik_%s.fits" % oname, likmap) #if False: # find_minimum(lik) #if False: # likmap = map_likelihood(lik) # enmap.write_map(args.odir + "/grid_%s.fits" % thumb_data.id, likmap) #if False: # x0 = np.zeros(2) # x = optimize.fmin_powell(lik, x0) # off = x*utils.arcmin # diag= lik.diag_plot(off) # enmap.write_map(args.odir + "/diag_%s.fits" % thumb_data.id, diag) #if False: # sampler = Sampler(lik, verbose=True)
tyx = [(y,x) for y in range(ntile[0]-1,-1,-1) for x in range(ntile[1])] for i in range(comm.rank, len(tyx), comm.size): y, x = tyx[i] osuff = "_padtile%(y)03d_%(x)03d.fits" % {"y":y,"x":x} tags = [s.replace(":","_") for s in signals] nok = sum([os.path.isfile(args.odir + "/" + tag + osuff) for tag in tags]) if args.cont and nok == len(tags): print "%3d skipping %3d %3d (already done)" % (comm.rank, y, x) continue print "%3d processing %3d %3d" % (comm.rank, y, x) tpos = np.array(tyx[i]) pbox = np.array([tpos*tshape,np.minimum((tpos+1)*tshape,shape[-2:])]) box = enmap.pix2sky(shape, wcs, pbox.T).T res = get_filtered_tile(mapinfo, box, signals, verbose=False) for si, tag in enumerate(tags): if res is not None: snmap = res.signals[si].snmap else: snmap = jointmap.make_dummy_tile(shape, wcs, box, pad=pad, dtype=dtype).map enmap.write_map(args.odir + "/" + tag + osuff, snmap) else: # Single arbitrary tile if not overlaps_any(bounds, boxes): print "No data in selected region" else: res = get_filtered_tile(mapinfo, bounds, signals, verbose=True) for i, sig in enumerate(res.signals): enmap.write_map(args.odir + "/%s_snmap.fits" % sig.name, sig.snmap) enmap.write_map(args.odir + "/%s_alpha.fits" % sig.name, sig.alpha) enmap.write_map(args.odir + "/%s_dalpha.fits" % sig.name, sig.dalpha)
def output(res, dir): utils.mkdir(dir) for i, (tqu, teb, desc) in enumerate(zip(res.tqu, res.teb, res.desc)): enmap.write_map("%s/%02d_%s_tqu.hdf" % (dir,i+1,desc), tqu) enmap.write_map("%s/%02d_%s_teb.hdf" % (dir,i+1,desc), teb)
return enmap.samewcs(map[utils.transpose_inds(dets, nrow, ncol)], map) # Read our map, and give each row a weight pickup = enmap.read_map(args.pickup_map) pickup = reorder(pickup, nrow, ncol, d.dets) weight = np.median((pickup[:, 1:] - pickup[:, :-1])**2, -1) weight[weight > 0] = 1 / weight[weight > 0] # Find the output pixel for each input pixel baz = pickup[:1].posmap()[1, 0] bel = baz * 0 + args.el * utils.degree ipoint = np.array([baz, bel]) opoint = ipoint[:, None, :] + d.point_offset.T[:, :, None] opix = template.sky2pix(opoint[::-1]).astype(int) # [{y,x},ndet,naz] opix = np.rollaxis(opix, 1) # [ndet,{y,x},naz] omap = enmap.zeros((3, ) + template.shape[-2:], template.wcs) odiv = enmap.zeros((3, 3) + template.shape[-2:], template.wcs) for det in range(d.ndet): omap += utils.bin_multi(opix[det], template.shape[-2:], weight[det] * pickup[det]) * d.det_comps[det, :, None, None] odiv += utils.bin_multi( opix[det], template.shape[-2:], weight[det]) * d.det_comps[ det, :, None, None, None] * d.det_comps[det, None, :, None, None] odiv = enmap.samewcs(array_ops.eigpow(odiv, -1, axes=[0, 1]), odiv) omap = enmap.samewcs(array_ops.matmul(odiv, omap, axes=[0, 1]), omap) enmap.write_map(args.ofile, omap)
else: rhs_tot += rhs infos.append(bunch.Bunch(U=U,N=N,H=H,W=W,pattern=pattern,site=site,srate=srate,scale=scale,speed=speed)) rhs = utils.allreduce(rhs_tot, comm) #info = infos[0] #foo = rhs*info.H #enmap.write_map("test1.fits", foo) #bar = enmap.samewcs(info.U.apply(foo),foo) #enmap.write_map("test2.fits", bar) #foo = enmap.samewcs(info.U.trans(bar, foo),foo) #enmap.write_map("test3.fits", foo) #1/0 # # dof = zipper.ArrayZipper(rhs.copy()) A = Amat(dof, infos, comm) cg = enlib.cg.CG(A, dof.zip(rhs)) utils.mkdir(args.odir) for i in range(200): cg.step() if comm.rank == 0: #print np.std(cg.x[cg.x!=0]) if cg.i % 10 == 0: map = dof.unzip(cg.x) enmap.write_map(args.odir + "/map%03d.fits" % cg.i, map) L.info("%4d %15.7e" % (cg.i, cg.err))
# Go from detectors to y-pixel in input maps ypix = utils.transpose_inds(dets[i], nrow, ncol) vals.append(imaps[i,ypix,iaz]) vals = np.concatenate(vals) pos = np.concatenate(offs) # Write to appropriate position in array pix = np.maximum(0,np.minimum((np.array(shape[-2:])-1)[:,None],enmap.sky2pix(shape, wcs, pos.T).astype(np.int32))) m = enmap.zeros(shape[-2:],wcs) m[tuple(pix)] = vals # Grow by smoothing m = enmap.smooth_gauss(m, rad)/norm omap[iaz] = m # Truncate very low values refval = np.mean(np.abs(omap))*0.01 mask = np.any(np.abs(omap)>refval,0) omap[:,~mask] = 0 if not args.individual: # output a single file enmap.write_map(args.ofile, omap) else: # output one file for each. This lets us encode # each one's azimuth utils.mkdir(args.ofile) for i, (baz, m) in enumerate(zip(bazs, omap)): if np.all(m==0): continue print i m.wcs.wcs.crval[0] = baz/utils.degree enmap.write_map(args.ofile + "/map%04d.fits" % i, m)
def make_beam(nl, bsize): l = np.arange(nl) return np.exp(-l*(l+1)*bsize**2) for i in range(comm.rank, args.nsim, comm.size): if args.lensed: ps = powspec.read_camb_full_lens(args.powspec).astype(dtype) if args.beam: raise NotImplementedError("Beam not supported for lensed sims yet") if args.geometry == "curved": m, = lensing.rand_map(shape, wcs, ps, lmax=lmax, maplmax=maplmax, seed=(seed,i), verbose=verbose, dtype=dtype) else: maps = enmap.rand_map((shape[0]+1,)+shape[1:], wcs, ps) phi, unlensed = maps[0], maps[1:] m = lensing.lens_map_flat(unlensed, phi) else: ps = powspec.read_spectrum(args.powspec).astype(type) beam = make_beam(ps.shape[-1], args.beam*utils.arcmin*utils.fwhm) ps *= beam if args.geometry == "curved": m = curvedsky.rand_map(shape, wcs, ps, lmax=lmax, seed=(seed,i), method=args.method, direct=args.direct, dtype=dtype, verbose=verbose) else: m = enmap.rand_map(shape, wcs, ps) if args.nsim == 1: if verbose: print "Writing %s" % args.ofile enmap.write_map(args.ofile, m) else: if verbose: print "Writing %s" % (args.ofile % i) enmap.write_map(args.ofile % i, m)
rhs = enmap.zeros((ncomp,)+shape, area.wcs, dtype) div = enmap.zeros((ncomp,ncomp)+shape, area.wcs, dtype) junk = np.zeros(pcut.njunk, dtype) with bench.show("rhs"): tod *= ivar[:,None] pcut.backward(tod, junk) pmap.backward(tod, rhs) with bench.show("hits"): for i in range(ncomp): div[i,i] = 1 pmap.forward(tod, div[i]) tod *= ivar[:,None] pcut.backward(tod, junk) div[i] = 0 pmap.backward(tod, div[i]) with bench.show("map"): idiv = array_ops.eigpow(div, -1, axes=[0,1], lim=1e-5) map = enmap.map_mul(idiv, rhs) # Estimate central amplitude c = np.array(map.shape[-2:])/2 crad = 50 mcent = map[:,c[0]-crad:c[0]+crad,c[1]-crad:c[1]+crad] mcent = enmap.downgrade(mcent, 4) amp = np.max(mcent) print "%s amp %7.3f asens %7.3f" % (id, amp/1e6, asens) with bench.show("write"): enmap.write_map("%s%s_map.fits" % (prefix, bid), map) enmap.write_map("%s%s_rhs.fits" % (prefix, bid), rhs) enmap.write_map("%s%s_div.fits" % (prefix, bid), div) del d, scan, pmap, pcut, tod, map, rhs, div, idiv, junk
tod *= ivar[:,None] sampcut.gapfill_const(scan.cut, tod, inplace=True) for sid in tod_srcs[id]: src = srcs[sid] if src.type == "fixed": sys = "hor:%.6f_%.6f:cel/0_0:hor" % (src.ra/utils.degree, src.dec/utils.degree) elif src.type == "planet": sys = "hor:%s/0_0" % src.name else: raise ValueError("Invalid source type '%s'" % src.type) rhs = enmap.zeros((ncomp,)+shape, area.wcs, dtype) div = enmap.zeros((ncomp,ncomp)+shape, area.wcs, dtype) with bench.show("pmat %s" % sid): pmap = pmat.PmatMap(scan, area, sys=sys) with bench.show("rhs %s" % sid): pmap.backward(tod, rhs) with bench.show("hits"): for i in range(ncomp): div[i,i] = 1 pmap.forward(tod, div[i]) tod *= ivar[:,None] sampcut.gapfill_const(scan.cut, tod, inplace=True) div[i] = 0 pmap.backward(tod, div[i]) with bench.show("map %s" % sid): idiv = array_ops.eigpow(div, -1, axes=[0,1], lim=1e-5, fallback="scalar") map = enmap.map_mul(idiv, rhs) with bench.show("write"): enmap.write_map("%s%s_src%03d_map.fits" % (prefix, bid, sid), map) enmap.write_map("%s%s_src%03d_rhs.fits" % (prefix, bid, sid), rhs) enmap.write_map("%s%s_src%03d_div.fits" % (prefix, bid, sid), div) del rhs, div, idiv, map del d, scan, pmap, tod
ires = np.array([1,1./np.sin(R)])*res/args.supersample shape, wi = enmap.geometry(pos=[[np.pi/2-R,-np.pi],[np.pi/2,np.pi]], res=ires, proj="car") imap = enmap.zeros((ncomp,)+shape, wi) # Define SHT for interpolation pixels with dprint("construct sht"): minfo = curvedsky.map2minfo(imap) lmax_ideal = np.pi/res ps = ps[:,:,:lmax_ideal] lmax = ps.shape[-1] # We do not need all ms when centered on the pole. To reach 1e-10 relative # error up to R, we need mmax approx 9560*R in radians mmax = args.mmax or int(R*9560) ainfo = sharp.alm_info(lmax, mmax) sht = sharp.sht(minfo, ainfo) with dprint("curvedsky tot"): with dprint("rand alm"): alm = curvedsky.rand_alm(ps, ainfo=ainfo, seed=1, m_major=False) with dprint("alm2map"): sht.alm2map(alm[:1], imap[:1].reshape(1,-1)) if ncomp == 3: sht.alm2map(alm[1:3], imap[1:3,:].reshape(2,-1), spin=2) del alm # Make a test map to see if we can project between these with dprint("project"): omap = enmap.project(imap, omap.shape, omap.wcs, mode="constant", cval=np.nan) del imap enmap.write_map(args.omap, omap)
# This program computes a simple estimation of the amount of distortion the # flat-sky approximation involves import numpy as np, argparse from enlib import enmap parser = argparse.ArgumentParser() parser.add_argument("omap") parser.add_argument("-D", "--diameter", type=float, default=30) parser.add_argument("-n", "--npix", type=int, default=800) parser.add_argument("--proj", type=str, default="cea") args = parser.parse_args() r = args.diameter*np.pi/180/2 shape, wcs = enmap.geometry(pos=[[-r,-r],[r,r]], shape=(args.npix, args.npix), proj=args.proj) def linpos(n,b): return b[0] + (np.arange(n)+0.5)*(b[1]-b[0])/n alpha = enmap.zeros((2,)+shape, wcs) pos = enmap.posmap(shape, wcs) # I'm not sure how to compute this in general, so here's a specialization # for cylindrical projections if args.proj == "cea" or args.proj =="car": dec = pos[0,:,0] ra = pos[1,0,:] lindec= linpos(shape[0],enmap.box(shape,wcs)[:,0]) scale = 1/np.cos(dec)-1 alpha[0] = (lindec-dec)[:,None] alpha[1] = ra[None,:]*scale[:,None] enmap.write_map(args.omap, alpha*180*60/np.pi)
def combine_tiles(ipathfmt, opathfmt, combine=2, downsample=2, itile1=(None, None), itile2=(None, None), tyflip=False, txflip=False, pad_to=None, comm=None, verbose=False): """Given a set of tiles on disk at locaiton ipathfmt % {"y":...,"x"...}, combine them into larger tiles, downsample and write the result to opathfmt % {"y":...,"x":...}. x and y must be contiguous and start at 0. reftile[2] indicates the tile coordinates of the first valid input tile. This needs to be specified if not all tiles of the logical tiling are physically present. tyflip and txflip indicate if the tiles coordinate system is reversed relative to the pixel coordinates or not." """ # Expand combine and downsample to 2d combine = np.zeros(2, int) + combine downsample = np.zeros(2, int) + downsample if pad_to is not None: pad_to = np.zeros(2, int) + pad_to # Handle optional mpi rank, size = (comm.rank, comm.size) if comm is not None else (0, 1) # Find the range of input tiles itile1, itile2 = find_tile_range(ipathfmt, itile1, itile2) # Read the first tile to get its size information ibase = enmap.read_map(ipathfmt % {"y": itile1[0], "x": itile1[1]}) * 0 # Find the set of output tiles we need to consider otile1 = itile1 / combine otile2 = (itile2 - 1) / combine + 1 # And loop over them oyx = [(oy, ox) for oy in range(otile1[0], otile2[0]) for ox in range(otile1[1], otile2[1])] for i in range(rank, len(oyx), size): oy, ox = oyx[i] # Read in all associated tiles into a list of lists rows = [] for dy in range(combine[0]): iy = oy * combine[0] + dy if iy >= itile2[0]: continue cols = [] for dx in range(combine[1]): ix = ox * combine[1] + dx if ix >= itile2[1]: continue if iy < itile1[0] or ix < itile1[1]: # The first tiles are missing on disk, but are # logically a part of the tiling. Use ibase, # which has been zeroed out. cols.append(ibase) else: itname = ipathfmt % {"y": iy, "x": ix} cols.append(enmap.read_map(itname)) if txflip: cols = cols[::-1] rows.append(cols) # Stack them next to each other into a big tile if tyflip: rows = rows[::-1] omap = enmap.tile_maps(rows) # Downgrade if necessary if np.any(downsample > 1): omap = enmap.downgrade(omap, downsample) if pad_to is not None: # Padding happens towards the end of the tiling, # which depends on the flip status padding = np.array( [[0, 0], [pad_to[0] - omap.shape[-2], pad_to[1] - omap.shape[-1]]]) if tyflip: padding[:, 0] = padding[::-1, 0] if txflip: padding[:, 1] = padding[::-1, 1] omap = enmap.pad(omap, padding) # And output otname = opathfmt % {"y": oy, "x": ox} utils.mkdir(os.path.dirname(otname)) enmap.write_map(otname, omap) if verbose: print otname
rhs = enmap.zeros((ncomp,)+shape, area.wcs, dtype) div = enmap.zeros((ncomp,ncomp)+shape, area.wcs, dtype) junk = np.zeros(pcut.njunk, dtype) with bench.show("rhs"): tod *= ivar[:,None] pcut.backward(tod, junk) pmap.backward(tod, rhs) with bench.show("hits"): for i in range(ncomp): div[i,i] = 1 pmap.forward(tod, div[i]) tod *= ivar[:,None] pcut.backward(tod, junk) div[i] = 0 pmap.backward(tod, div[i]) with bench.show("map"): idiv = array_ops.eigpow(div, -1, axes=[0,1], lim=1e-5) map = enmap.map_mul(idiv, rhs) # Estimate central amplitude c = np.array(map.shape[-2:])/2 crad = 50 mcent = map[:,c[0]-crad:c[0]+crad,c[1]-crad:c[1]+crad] mcent = enmap.downgrade(mcent, 4) amp = np.max(mcent) print("%s amp %7.3f asens %7.3f" % (id, amp/1e6, asens)) with bench.show("write"): enmap.write_map("%s%s_map.fits" % (prefix, bid), map) enmap.write_map("%s%s_rhs.fits" % (prefix, bid), rhs) enmap.write_map("%s%s_div.fits" % (prefix, bid), div) del d, scan, pmap, pcut, tod, map, rhs, div, idiv, junk