def l(cseed,kseed,returnk=False,index=None): cname = fout_dir+"lensed_covseed_"+str(args.covseed).zfill(3)+"_cmbseed_"+str(cseed).zfill(5)+"_kseed_"+str(kseed).zfill(5)+".hdf" if unlensed: seedroot = (covseed)*Nsets*Nsims lensedt = parray_dat.get_unlensed_cmb(seed=seedroot+cseed,scalar=False) else: lensedt = enmap.read_map(cname)[0] if polsims else enmap.read_map(cname) # -- add beam and noise if you want -- if "noiseless" not in expf_name: assert index is not None if rank==0: print("Adding beam...") flensed = fftfast.fft(lensedt,axes=[-2,-1]) flensed *= parray_dat.lbeam lensedt = fftfast.ifft(flensed,axes=[-2,-1],normalize=True).real if rank==0: print("Adding noise...") seedroot = (covseed+1)*Nsets*Nsims # WARNING: noise sims will be correlated with CMB from the next covseed nseed = seedroot+index noise = parray_dat.get_noise_sim(seed=nseed) if paper: cents, noise1d = lbinner.bin(power(noise)[0]) mpibox.add_to_stats('noisett',noise1d) lensedt += noise lensedt = enmap.ndmap(lensedt,wcs_dat) if returnk: kname = fout_dir+"kappa_covseed_"+str(args.covseed).zfill(3)+"_kseed_"+str(kseed).zfill(5)+".hdf" return lensedt,enmap.read_map(kname) else: return lensedt
def read_tileset_geometry(ipathfmt, itile1=(None,None), itile2=(None,None)): itile1, itile2 = find_tile_range(ipathfmt, itile1, itile2) m1 = enmap.read_map(ipathfmt % {"y":itile1[0],"x":itile1[1]}) m2 = enmap.read_map(ipathfmt % {"y":itile2[0]-1,"x":itile2[1]-1}) wy,wx = m1.shape[-2:] oshape = tuple(np.array(m1.shape[-2:])*(itile2-itile1-1) + np.array(m2.shape[-2:])) return bunch.Bunch(shape=m1.shape[:-2]+oshape, wcs=m1.wcs, dtype=m1.dtype, tshape=m1.shape[-2:])
def add_maps(imaps, omap): if args.verbose: print "Reading %s" % imaps[0] m = nonan(enmap.read_map(imaps[0])) for mif in imaps[1:]: if args.verbose: print "Reading %s" % mif m += nonan(enmap.read_map(mif)) if args.verbose: "Writing %s" % omap enmap.write_map(omap, m)
def add_maps(imaps, omap): if args.verbose: print "Reading %s" % imaps[0] m = nonan(enmap.read_map(imaps[0]))*scales[0] for scale, mif in zip(scales[1:],imaps[1:]): if args.verbose: print "Reading %s" % mif m += nonan(enmap.read_map(mif))*scale if args.mean: m /= len(imaps) if args.verbose: "Writing %s" % omap enmap.write_map(omap, m)
def add_maps(imaps, omap): if args.verbose: print "Reading %s" % imaps[0] m = nonan(enmap.read_map(imaps[0]))*scales[0] for scale, mif in zip(scales[1:],imaps[1:]): if args.verbose: print "Reading %s" % mif m2 = nonan(enmap.read_map(mif, geometry=(m.shape,m.wcs)))*scale n = min(len(m.preflat),len(m2.preflat)) m.preflat[:n] += m2.preflat[:n] if args.mean: m /= len(imaps) if args.verbose: "Writing %s" % omap enmap.write_map(omap, m)
def read_maps(fmt, n, ntot=4): try: maps = en.read_map(fmt) if maps.ndim == ntot - 1: maps = en.enmap([maps] * n, maps.wcs) if maps.ndim != ntot: raise ValueError("Map %s must have %d dimensions" % (fmt, ntot)) return maps except (IOError, OSError): maps = [en.read_map(fmt % i) for i in range(n)] maps = en.ndmap(maps, maps[0].wcs) if maps.ndim != ntot: maps = maps.reshape(maps.shape[:-2] + (1, ) * (maps.ndim - ntot) + maps.shape[-2:]) return maps
def read_map(ifile, slice=None): if not os.path.isdir(ifile): map = enmap.read_map(ifile) if slice: map = eval("map" + slice) else: map = retile.read_monolithic(ifile, slice=slice, verbose=False) return map
def read_map(ifile, slice=None): if not os.path.isdir(ifile): map = enmap.read_map(ifile) if slice: map = eval("map"+slice) else: map = retile.read_monolithic(ifile, slice=slice, verbose=False) return map
def read_area(ipathfmt, opix, itile1=(None,None), itile2=(None,None), verbose=False, cache=None, slice=None): """Given a set of tiles on disk with locations ipathfmt % {"y":...,"x":...}, read the data corresponding to the pixel range opix[{from,to],{y,x}] in the full map.""" opix = np.asarray(opix) # Find the range of input tiles itile1, itile2 = find_tile_range(ipathfmt, itile1, itile2) # To fill in the rest of the information we need to know more # about the input tiling, so read the first tile if cache is None or cache[2] is None: geo = read_tileset_geometry(ipathfmt, itile1=itile1, itile2=itile2) else: geo = cache[2] if cache is not None: cache[2] = geo isize = geo.tshape osize = opix[1]-opix[0] omap = enmap.zeros(geo.shape[:-2]+tuple(osize), geo.wcs, geo.dtype) # Find out which input tiles overlap with this output tile. # Our tile stretches from opix1:opix2 relative to the global input pixels it1 = opix[0]/isize it2 = (opix[1]-1)/isize+1 noverlap = 0 for ity in range(it1[0],it2[0]): if ity < itile1[0] or ity >= itile2[0]: continue # Start/end of this tile in global input pixels ipy1, ipy2 = ity*isize[0], (ity+1)*isize[0] overlap = range_overlap(opix[:,0],[ipy1,ipy2]) oy1,oy2 = overlap-opix[0,0] iy1,iy2 = overlap-ipy1 for itx in range(it1[1],it2[1]): if itx < itile1[1] or itx >= itile2[1]: continue ipx1, ipx2 = itx*isize[1], (itx+1)*isize[1] overlap = range_overlap(opix[:,1],[ipx1,ipx2]) ox1,ox2 = overlap-opix[0,1] ix1,ix2 = overlap-ipx1 # Read the input tile and copy over iname = ipathfmt % {"y":ity,"x":itx} if cache is None or cache[0] != iname: imap = enmap.read_map(iname) if slice: imap = eval("imap"+slice) else: imap = cache[1] if cache is not None: cache[0], cache[1] = iname, imap if verbose: print iname # Edge input tiles may be smaller than the standard # size. ysub = isize[0]-imap.shape[-2] xsub = isize[1]-imap.shape[-1] # If the input map is too small, there may actually be # zero overlap. if oy2-ysub <= oy1 or ox2-xsub <= ox1: continue omap[...,oy1:oy2-ysub,ox1:ox2-xsub] = imap[...,iy1:iy2-ysub,ix1:ix2-xsub] noverlap += 1 if noverlap == 0: raise IOError("No tiles for tiling %s in range %s" % (ipathfmt, ",".join([":".join([str(p) for p in r]) for r in opix.T]))) # Set up the wcs for the output tile omap.wcs.wcs.crpix -= opix[0,::-1] return omap
def patch_array_from_config(Config, exp_name, shape, wcs, dimensionless=True, TCMB=2.7255e6, skip_real=False, skip_instrument=False): pa = fmaps.PatchArray(shape, wcs, dimensionless=dimensionless, TCMB=TCMB, skip_real=skip_real) if not (skip_instrument): try: bfile = Config.get(exp_name, "beam_file") ells, bls = np.loadtxt(bfile, delimiter=",", unpack=True, use_cols=[0, 1]) pa.add_1d_beam(ells, bls, fill_value="extrapolate") except: fwhm = Config.getfloat(exp_name, "beam") pa.add_gaussian_beam(fwhm) try: n2d_file_T = Config.get(exp_name, "noise_2d_file_T") n2d_file_P = Config.get(exp_name, "noise_2d_file_P") imapT = enmap.read_map(n2d_file_T) imapP = enmap.read_map(n2d_file_P) pa.add_noise_2d(nT=imapT, nP=imapP) except: noise_T = Config.getfloat(exp_name, "noise_T") noise_P = Config.getfloat(exp_name, "noise_P") lknee_T = Config.getfloat(exp_name, "lknee_T") lknee_P = Config.getfloat(exp_name, "lknee_P") alpha_T = Config.getfloat(exp_name, "alpha_T") alpha_P = Config.getfloat(exp_name, "alpha_P") pa.add_white_noise_with_atm(noise_T, noise_P, lknee_T, alpha_T, lknee_P, alpha_P) return pa
def flatFitsToHealpix(fitsFile, nside, downgrade=1): from enlib import enmap imap = enmap.read_map(fitsFile) if downgrade > 1: imap = enmap.downgrade(imap, args.downgrade) omap = imap.to_healpix(nside=args.nside) return omap
def build_single(ifile, srcs, beam, ofile, mask_level=0, apod_size=16): imap = enmap.read_map(ifile) omap, oslice = pointsrcs.sim_srcs(imap.shape[-2:], imap.wcs, srcs, beam, return_padded=True) if mask_level: mask = omap > mask_level omap = 1-np.cos(np.minimum(1,ndimage.distance_transform_edt(1-mask)/16.0)*np.pi) omap = enmap.samewcs(omap, imap) omap = omap[oslice] enmap.write_map(ofile, omap)
def read_map(fname, pbox, cname, write_cache=True, read_cache=True): if os.path.isdir(fname): fname = fname + "/tile%(y)03d_%(x)03d.fits" if read_cache and os.path.isfile(cname): map = enmap.read_map(cname).astype(dtype) else: map = retile.read_area(fname, pbox).astype(dtype) if write_cache: enmap.write_map(cname, map) #if map.ndim == 3: map = map[:1] return map
def read_div(fname): m = nonan(enmap.read_map(fname))*1.0 return m #return m.preflat[:1][None] if m.ndim == 2: res = enmap.zeros((padlen,padlen)+m.shape[-2:], m.wcs, m.dtype) for i in range(padlen): res[i,i] = m return res elif m.ndim == 4: return m else: raise ValueError("Wrong number of dimensions in div %s" % fname)
def retile(ipathfmt, opathfmt, itile1=(None,None), itile2=(None,None), otileoff=(0,0), otilenum=(None,None), ocorner=(-np.pi/2,-np.pi), otilesize=(675,675), comm=None, verbose=False, slice=None): """Given a set of tiles on disk with locations ipathfmt % {"y":...,"x":...}, retile them into a new tiling and write the result to opathfmt % {"y":...,"x":...}. The new tiling will have tile size given by otilesize[2]. Negative size means the tiling will to down/left instead of up/right. The corner of the tiling will be at sky coordinates ocorner[2] in radians. The new tiling will be pixel- compatible with the input tiling - w.g. the wcs will only differ by crpix. The output tiling will logically cover the whole sky, but only output tiles that overlap with input tiles will actually be written. This can be modified by using otileoff[2] and otilenum[2]. otileoff gives the tile indices of the corner tile, while otilenum indicates the number of tiles to write.""" # Set up mpi rank, size = (comm.rank, comm.size) if comm is not None else (0, 1) # Expand any scalars otilesize = np.zeros(2,int)+otilesize otileoff = np.zeros(2,int)+otileoff # Find the range of input tiles itile1, itile2 = find_tile_range(ipathfmt, itile1, itile2) # To fill in the rest of the information we need to know more # about the input tiling, so read the first tile ibase = enmap.read_map(ipathfmt % {"y":itile1[0],"x":itile1[1]}) if slice: ibase = eval("ibase"+slice) itilesize = ibase.shape[-2:] # Find the pixel position of our output corners according to the wcs. # This is the last place we need to do a coordinate transformation. # All the rest can be done in pure pixel logic. pixoff = np.round(ibase.sky2pix(ocorner)).astype(int) # Find the range of output tiles def pix2otile(pix, ioff, osize): return (pix-ioff)/osize otile1 = pix2otile(itile1*itilesize, pixoff, otilesize) otile2 = pix2otile(itile2*itilesize-1, pixoff, otilesize) otile1, otile2 = np.minimum(otile1,otile2), np.maximum(otile1,otile2) otile2 += 1 # We can now loop over output tiles cache = [None,None,None] oyx = [(oy,ox) for oy in range(otile1[0],otile2[0]) for ox in range(otile1[1],otile2[1])] for i in range(rank, len(oyx), size): otile = np.array(oyx[i]) # Find out which input tiles overlap with this output tile. # Our tile stretches from opix1:opix2 relative to the global input pixels opix1 = otile*otilesize + pixoff opix2 = (otile+1)*otilesize + pixoff # output tiles and input tiles may increase in opposite directions opix1, opix2 = np.minimum(opix1,opix2), np.maximum(opix1,opix2) try: omap = read_area(ipathfmt, [opix1,opix2],itile1=itile1, itile2=itile2,cache=cache, slice=slice) except IOError: continue oname = opathfmt % {"y":otile[0]+otileoff[0],"x":otile[1]+otileoff[1]} utils.mkdir(os.path.dirname(oname)) enmap.write_map(oname, omap) if verbose: print oname
def sim(cseed, kseed, skip_kappa_verif=False): if not (vonly): unlensed = parray_sim.get_unlensed_cmb(seed=seedroot + cseed, scalar=False) kappa = aio.kappa_from_config(Config, kappa_section, parray_sim, seed=seedroot + kseed) lensed = parray_sim.get_lensed(unlensed, order=lens_order, mode="spline", border="cyclic") luteb, dummy = sverif_cmb.add_power("unlensed", unlensed) llteb, dummy = sverif_cmb.add_power("lensed", lensed) if not (skip_kappa_verif): lk, dummy = sverif_kappa.add_power("kappa", kappa) cname = fout_dir + "lensed_covseed_" + str( args.covseed).zfill(3) + "_cmbseed_" + str(cseed).zfill( 5) + "_kseed_" + str(kseed).zfill(5) + ".hdf" kname = fout_dir + "kappa_covseed_" + str( args.covseed).zfill(3) + "_kseed_" + str(kseed).zfill(5) + ".hdf" if vonly: dlensed = enmap.read_map(cname) dkappa = enmap.read_map(kname) else: dlensed = lensed if abs(pixratio - 1.) < 1.e-3 else enmap.ndmap( resample.resample_fft(lensed, shape_dat), wcs_dat) dkappa = kappa if abs(pixratio - 1.) < 1.e-3 else enmap.ndmap( resample.resample_fft(kappa, shape_dat[-2:]), wcs_dat) dlensed.write(cname) dkappa.write(kname) dllteb, dummy = sverif_dcmb.add_power("dlensed", dlensed) if not (skip_kappa_verif): dlk, dummy = sverif_dkappa.add_power("dkappa", dkappa)
def build_single(ifile, srcs, beam, ofile, mask_level=0, apod_size=16): imap = enmap.read_map(ifile) omap, oslice = pointsrcs.sim_srcs(imap.shape[-2:], imap.wcs, srcs, beam, return_padded=True) if mask_level: mask = omap > mask_level omap = 1 - np.cos( np.minimum(1, ndimage.distance_transform_edt(1 - mask) / 16.0) * np.pi) omap = enmap.samewcs(omap, imap) omap = omap[oslice] enmap.write_map(ofile, omap)
def read_map(fname, pbox, name=None, cache_dir=None, dtype=None, read_cache=False): if os.path.isdir(fname): fname = fname + "/tile%(y)03d_%(x)03d.fits" if read_cache: map = enmap.read_map(cache_dir + "/" + name) if dtype is not None: map = map.astype(dtype) else: map = retile.read_area(fname, pbox).astype(dtype) if dtype is not None: map = map.astype(dtype) if cache_dir is not None and name is not None: enmap.write_map(cache_dir + "/" + name, map) #if map.ndim == 3: map = map[:1] return map
def read(self,y,x): if self.nphi: x = x % self.nphi for c in self.cache: if c[0] == (y,x): return c[1] else: while len(self.cache) >= self.ncache: del self.cache[0] fname = self.pathfmt % {"y":y,"x":x} if os.path.isfile(fname): m = enmap.read_map(fname) if self.crop: m = m[...,self.crop:-self.crop,self.crop:-self.crop] if self.ncomp: m = m.preflat extra = np.tile(m[:1]*0, (self.ncomp-len(m),1,1)) m = enmap.samewcs(np.concatenate([m,extra],0),m) else: m = None self.cache.append([(y,x),m]) return m
def enmap_from_config_section(Config, section, pol=False): analysis_section = section projection = Config.get(analysis_section, "projection") try: pt_file = Config.get(analysis_section, "patch_template") imap = enmap.read_map(pt_file) shape_dat = imap.shape wcs_dat = imap.wcs if pol and len(shape_dat) < 3: shape_dat = (3, shape_dat[0], shape_dat[1]) res = np.min(imap.extent() / imap.shape[-2:]) * 60. * 180. / np.pi except: pixel_analysis = Config.getfloat(analysis_section, "pixel_arcmin") try: width_analysis_deg = Config.getfloat(analysis_section, "patch_degrees_width") except: width_analysis_deg = Config.getfloat(analysis_section, "patch_arcmin_width") / 60. try: height_analysis_deg = Config.getfloat(analysis_section, "patch_degrees_height") except: height_analysis_deg = Config.getfloat(analysis_section, "patch_arcmin_height") / 60. ra_offset = Config.getfloat(analysis_section, "ra_offset") dec_offset = Config.getfloat(analysis_section, "dec_offset") shape_dat, wcs_dat = maps.rect_geometry( width_analysis_deg * 60., pixel_analysis, proj=projection, pol=pol, height_arcmin=height_analysis_deg * 60., xoffset_degree=ra_offset, yoffset_degree=dec_offset) return shape_dat, wcs_dat
if args.mode == "find": # Point source finding mode results = [] map_keys = ["map","snmap","model","resid","resid_snmap"] utils.mkdir(args.odir) # Mpi-parallelization over regions is simple, but a bit lazy. It means that there will be # no speedup for single-region maps for ri in range(comm.rank, len(regions), comm.size): reg_fid = regions[ri] reg_pad = dory.pad_region(reg_fid, args.pad) print "%3d region %3d/%d %5d %5d %6d %6d" % (comm.rank, ri+1, len(regions), reg_fid[0,0], reg_fid[1,0], reg_fid[0,1], reg_fid[1,1]) try: # We only use T to find sources in find mode for now. P usually has much lower S/N and # doesn't help much. Should add it later, though. imap = enmap.read_map(args.imap, pixbox=reg_pad).preflat[0] idiv = enmap.read_map(args.idiv, pixbox=reg_pad).preflat[0] if args.mask: mshape, mwcs = enmap.read_map_geometry(args.mask) mbox = enmap.pixbox_of(mwcs, imap.shape, imap.wcs) idiv *= (1-enmap.read_map(args.mask, pixbox=mbox).preflat[0]) if "debug" in args.output: dump_prefix = args.odir + "/region_%02d_" % ri else: dump_prefix = None result = dory.find_srcs(imap, idiv, beam, freq=args.freq, apod=args.apod, apod_margin=args.apod_margin, snmin=args.nsigma, verbose=args.verbose, dump=dump_prefix) # FIXME: artifacts are act-specific result = dory.prune_artifacts(result) except Exception as e: print "Exception for task %d region %d: %s" % (comm.rank, ri, e.message) raise # Write region output
parser = config.ArgumentParser(os.environ["HOME"]+"/.enkirc") parser.add_argument("area") parser.add_argument("sel") parser.add_argument("odir") parser.add_argument("-n", "--nstep", type=int, default=50) parser.add_argument("-m", "--method",type=str, default="messenger") parser.add_argument( "--ndet", type=int, default=None) parser.add_argument("-p", "--precompute", action="store_true") parser.add_argument("-o", "--ostep", type=int, default=10) args = parser.parse_args() utils.mkdir(args.odir) comm = mpi.COMM_WORLD dtype = np.float32 ncomp = 3 area = enmap.read_map(args.area) area = enmap.zeros((ncomp,)+area.shape[-2:],area.wcs,dtype) Tscale = 0.9 nstep = args.nstep downsample = config.get("downsample") filedb.init() ids = filedb.scans[args.sel] # Was 1e7 cooldown = sum([[10**j]*5 for j in range(6,0,-1)],[])+[1] # Read my scans njunk_tot = 0 cg_rhs = area*0 cg_rjunk = [] if args.precompute:
def vread(fname): print "Reading %s" % fname return enmap.read_map(fname)
def read_map(name, bbpix=None, bbox=None, tshape=None, comm=None): if comm is None: comm = mpi.COMM_WORLD if os.path.isdir(name): # Find the number of tiles in the map entries = os.listdir(name) nrow, ncol = 0, 0 for entry in entries: match = re.search(r'^tile(\d+)_(\d+).([^.]+)$', entry) if match: nrow = max(nrow, 1 + int(match.group(1))) ncol = max(ncol, 1 + int(match.group(2))) ext = match.group(3) # Build the list of tile files tfiles = [["" for c in range(ncol)] for r in range(nrow)] for entry in entries: match = re.search(r'^tile(\d+)_(\d+).([^.]+)$', entry) if match: tfiles[int(match.group(1))][int(match.group(2))] = entry if nrow == 0: raise IOError("'%s' is not a valid dmap file" % name) # Find the tile size and map extent tile1 = enmap.read_map(name + "/" + tfiles[0][0]) tile2 = enmap.read_map(name + "/" + tfiles[-1][-1]) npre, tshape = tile1.shape[:-2], tile1.shape[-2:] wcs = tile1.wcs shape = npre + (tile1.shape[-2] * (nrow - 1) + tile2.shape[-2], tile1.shape[-1] * (ncol - 1) + tile2.shape[-1]) dtype = tile1.dtype # Construct our dmap and read our tiles map = Dmap( geometry(shape, wcs, bbpix=bbpix, bbox=bbox, tshape=tshape, dtype=dtype, comm=comm)) for pos, tile in zip(map.loc_pos, map.tiles): tile[:] = enmap.read_map(name + "/" + tfiles[pos[0]][pos[1]]) else: # Map is in a single file. Get map info if comm.rank == 0: canvas = enmap.read_map(name) shape = comm.bcast(canvas.shape) wcs = WCS(comm.bcast(canvas.wcs.to_header_string())) dtype = comm.bcast(canvas.dtype) # Hack: Pickling changes the dtype from < to =, both of which are # equivalent. But mpi has problems with the former. canvas.dtype = dtype else: shape = comm.bcast(None) wcs = WCS(comm.bcast(None)) dtype = comm.bcast(None) canvas = None map = Dmap( geometry(shape, wcs, bbpix=bbpix, bbox=bbox, tshape=tshape, dtype=dtype, comm=comm)) # And send data to the tiles enmap2dmap(canvas, map) return map
shared = True utils.mkdir(args.odir) filedb.init() db = filedb.data ids = filedb.scans[args.query] myinds = range(len(ids))[myid::nproc] n = len(ids) # Optinal input map to subtract imap = None if args.imap: toks = args.imap.split(":") imap_sys, fname = ":".join(toks[:-1]), toks[-1] imap = bunch.Bunch(sys=imap_sys or None, map=enmap.read_map(fname)) for i in myinds: id = ids[i] entry = db[id] ofile = "%s/%s.hdf" % (args.odir, id) if os.path.isfile(ofile) and args.resume: continue t = [] t.append(time.time()) try: fields = ["gain", "tconst", "cut", "tod", "boresight", "noise_cut"] if args.spikecut: fields.append("spikes") if args.imap: fields += ["polangle", "point_offsets", "site"] d = data.read(entry, fields) t.append(time.time()) d = data.calibrate(d)
# Try to get about the same amount of data for each mpi task. # If we use distributed maps, we also try to make things as local as possible mycosts = [s.nsamp*s.ndet for s in myscans] if is_dmap: myboxes = [scanutils.calc_sky_bbox_scan(s, args.sys) for s in myscans] myinds, mysubs, mybbox = scanutils.distribute_scans(myinds, mycosts, myboxes, comm) else: myinds = scanutils.distribute_scans(myinds, mycosts, None, comm) L.info("Rereading shuffled scans") del myscans # scans do take up some space, even without the tod being read in myinds, myscans = scanutils.read_scans(ids, myinds, actscan.ACTScan, filedb.data, dets=args.dets, downsample=config.get("downsample")) L.info("Reading input map") if not is_dmap: imap = enmap.read_map(args.imap) else: imap = dmap.read_map(args.imap, bbox=mybbox, tshape=tshape, comm=comm) dtype = "=f" if imap.dtype == np.float32 else "=d" imap = imap.astype(dtype, copy=True) L.info("Initializing signals") signal_cut = mapmaking.SignalCut(myscans, imap.dtype, comm=comm) if not is_dmap: signal_map = mapmaking.SignalMap(myscans, imap, comm=comm) precon = mapmaking.PreconMapBinned(signal_map, signal_cut, myscans, [], noise=False, hits=False) else: signal_map = mapmaking.SignalDmap(myscans, mysubs, imap, sys=args.sys) precon = mapmaking.PreconDmapBinned(signal_map, signal_cut, myscans, [], noise=False, hits=False) # We can now actually perform the postfilter L.info("Postfiltering")
myinds, mysubs, mybbox = scanutils.distribute_scans( myinds, mycosts, myboxes, comm) else: myinds = scanutils.distribute_scans(myinds, mycosts, None, comm) L.info("Rereading shuffled scans") del myscans # scans do take up some space, even without the tod being read in myinds, myscans = scanutils.read_scans(ids, myinds, actscan.ACTScan, filedb.data, dets=args.dets, downsample=config.get("downsample")) L.info("Reading input map") if not is_dmap: imap = enmap.read_map(args.imap) else: imap = dmap.read_map(args.imap, bbox=mybbox, tshape=tshape, comm=comm) dtype = "=f" if imap.dtype == np.float32 else "=d" imap = imap.astype(dtype, copy=True) L.info("Initializing signals") signal_cut = mapmaking.SignalCut(myscans, imap.dtype, comm=comm) if not is_dmap: signal_map = mapmaking.SignalMap(myscans, imap, comm=comm) precon = mapmaking.PreconMapBinned(signal_map, signal_cut, myscans, [], noise=False, hits=False) else: signal_map = mapmaking.SignalDmap(myscans, mysubs, imap, sys=args.sys)
regions to cut. This mask can then be turned into a per-scanning-pattern cut list in mask2cut""" import numpy as np, argparse from scipy import ndimage from enlib import enmap, fft parser = argparse.ArgumentParser() parser.add_argument("imap") parser.add_argument("omap") parser.add_argument("-T", "--threshold", type=float, default=500) parser.add_argument("-H", "--high-threshold", type=float, default=4000) parser.add_argument("-W", "--minwidth", type=int, default=2) parser.add_argument("-w", "--widen", type=int, default=3) parser.add_argument("-m", "--maxmean", type=int, default=250) args = parser.parse_args() imap = enmap.read_map(args.imap) ngood = int(np.sum(np.any(imap!=0,1))) # Smooth a bit in az direction to reduce noise m = imap fm = fft.redft00(m) nf = fm.shape[1] fm[:,nf/4:] = 0 fft.redft00(fm, m, normalize=True) def fixwidth(row): row = ndimage.distance_transform_edt(row) > args.minwidth if np.any(row): row = ndimage.distance_transform_edt(1-row) < args.minwidth + args.widen return row
def read_map(fname): m = nonan(enmap.read_map(fname)) #return m.preflat[:1] return m.reshape(-1, m.shape[-2], m.shape[-1])
parser.add_argument("sel") parser.add_argument("objname") parser.add_argument("mask") parser.add_argument("odir") parser.add_argument("--margin", type=float, default=1) args = parser.parse_args() filedb.init() ids = filedb.scans[args.sel] comm = mpi.COMM_WORLD dtype= np.float32 ntod = len(ids) margin = args.margin*utils.degree imask = enmap.read_map(args.mask) # Expand to 3 components, as the pointing code expects that mask = enmap.zeros((3,)+imask.shape[-2:], imask.wcs, dtype) mask[0] = imask.reshape((-1,)+imask.shape[-2:])[0] del imask utils.mkdir(args.odir) # Each mpi tasks opens its own work file wfname = args.odir + "/work_%03d.hdf" % comm.rank mystats = [] with h5py.File(wfname, "w") as hfile: for ind in range(comm.rank*ntod//comm.size, (comm.rank+1)*ntod//comm.size): id = ids[ind] entry = filedb.data[id] # We only need pointing to build this cut
def read(fname): m = enmap.read_map(fname) if slice: m = eval("m" + slice) return m
import numpy as np, argparse from scipy import ndimage from enlib import enmap parser = argparse.ArgumentParser() parser.add_argument("ifile") parser.add_argument("ofile") parser.add_argument("-r", "--apod-radius", type=int, default=64) args = parser.parse_args() def make_apod(shape, rad): mask = np.zeros(shape[-2:]) mask[rad:-rad, rad:-rad] = 1 w = np.maximum(1 - ndimage.distance_transform_edt(1 - mask) / rad, 0) ** 3 return w teb = enmap.read_map(args.ifile) tqu = enmap.harm2map(enmap.fft(teb)) mask = make_apod(teb.shape, args.apod_radius) tqu_mask = tqu * mask[None] teb_mask = enmap.ifft(enmap.map2harm(tqu_mask)).real res = enmap.samewcs([teb, tqu, tqu_mask, teb_mask], teb) enmap.write_map(args.ofile, res)
shared = True utils.mkdir(args.odir) filedb.init() db = filedb.data ids = filedb.scans[args.query] myinds = range(len(ids))[myid::nproc] n = len(ids) # Optinal input map to subtract imap = None if args.imap: toks = args.imap.split(":") imap_sys, fname = ":".join(toks[:-1]), toks[-1] imap = bunch.Bunch(sys=imap_sys or None, map=enmap.read_map(fname)) for i in myinds: id = ids[i] entry = db[id] ofile = "%s/%s.hdf" % (args.odir, id) if os.path.isfile(ofile) and args.resume: continue t=[]; t.append(time.time()) try: fields = ["gain","tconst","cut","tod","boresight", "noise_cut"] if args.spikecut: fields.append("spikes") if args.imap: fields += ["polangle","point_offsets","site"] d = data.read(entry, fields) ; t.append(time.time()) d = data.calibrate(d) ; t.append(time.time()) if args.imap: # Make a full scan object, so we can perform pointing projection
dirname = os.path.dirname(fname) if len(dirname) == 0: dirname = "." base_ext = os.path.basename(fname) # Find the extension. Using the last dot does not work for .fits.gz. # Using the first dot in basename does not work for foo2.5_bar.fits. # Will therefore handle .gz as a special case. if base_ext.endswith(".gz"): dot = base_ext[:-3].rfind(".") else: dot = base_ext.rfind(".") if dot < 0: dot = len(base_ext) base = base_ext[:dot] ext = base_ext[dot+1:] return dirname, base, ext for ifile in args.ifiles: m = enmap.read_map(ifile) if args.slice: m = eval("m" + args.slice) N = m.shape[:-2] for i, comp in enumerate(m.preflat): I = np.unravel_index(i, N) if len(N) > 0 else [] if args.symmetric and np.any(np.sort(I) != I): continue ndigits = [get_num_digits(n) for n in N] compname = "_".join(["%0*d" % (ndig,ind) for ndig,ind in zip(ndigits,I)]) if len(N) > 0 else "" dir, base, ext = split_file_name(ifile) oname = dir + "/" + base + "_" + compname + "." + ext if args.verbose: print(oname) enmap.write_map(oname, comp)
import numpy as np, argparse, os from enlib import enmap, utils from enact import files parser = argparse.ArgumentParser() parser.add_argument("ifiles", nargs="+") parser.add_argument("ofile") args = parser.parse_args() maps = [] for ifile in args.ifiles: ibase = os.path.basename(ifile) print ibase data = enmap.read_map(ifile) maps.append(data) maps = enmap.samewcs(np.array(maps),maps[0]) print np.sum(maps**2) print maps.shape masked = np.ma.MaskedArray(maps, mask=maps==0) print np.sum(masked**2) medmap = np.ma.median(masked,0) medmap = enmap.samewcs(np.asarray(medmap),maps) # And run through again, subtracting it enmap.write_map(args.ofile, medmap)
# Given two maps, estimate the noise level their average would have. import numpy as np, argparse from enlib import enmap parser = argparse.ArgumentParser() parser.add_argument("ifiles", nargs=2) parser.add_argument("ofile") parser.add_argument("-b", "--binsize", type=int, default=3) parser.add_argument("-s", "--smooth", type=float, default=30) parser.add_argument("--div", type=float, default=1.0) args = parser.parse_args() b = args.binsize smooth = args.smooth * np.pi/180/60/(8*np.log(2)) m = [enmap.read_map(f) for f in args.ifiles] dm = (m[1]-m[0])/2 pixarea = dm.area()/np.product(dm.shape[-2:])*(180*60/np.pi)**2 # Compute standard deviation in bins dm = dm[...,:dm.shape[-2]/b*b,:dm.shape[-1]/b*b] dm_blocks = dm.reshape(dm.shape[:-2]+(dm.shape[-2]/b,b,dm.shape[-1]/b,b)) var = np.std(dm_blocks,axis=(-3,-1))**2*pixarea/args.div # This reshaping stuff messes up the wcs, which doesn't notice # that we now have bigger pixels. So correct that. var = enmap.samewcs(var, dm[...,::b,::b]) typ = np.median(var[var!=0]) var[~np.isfinite(var)] = 0 var = np.minimum(var,typ*1e6) svar = enmap.smooth_gauss(var, smooth) sigma = svar**0.5
def read_map(fname): m = enmap.read_map(fname) m = m[...,1:-1,1:-1] #m = eval("m" + args.slice) m = enmap.downgrade(m, args.downgrade) return m
# Project a single pickup map into horizontal coordinates import numpy as np, os from enlib import enmap, utils, config, array_ops from enact import actdata, filedb parser = config.ArgumentParser(os.environ["HOME"] + "/.enkirc") parser.add_argument("pickup_map") parser.add_argument("template") parser.add_argument("sel_repr") parser.add_argument("el", type=float) parser.add_argument("ofile") args = parser.parse_args() filedb.init() nrow, ncol = 33, 32 # Read our template, which represents the output horizontal coordinates template = enmap.read_map(args.template) # Use our representative selector to get focalplane offsets and polangles entry = filedb.data[filedb.scans[args.sel_repr][0]] d = actdata.read(entry, ["boresight", "point_offsets", "polangle"]) d.boresight[2] = args.el # In degrees, calibrated in next step d = actdata.calibrate(d, exclude=["autocut"]) def reorder(map, nrow, ncol, dets): return enmap.samewcs(map[utils.transpose_inds(dets, nrow, ncol)], map) # Read our map, and give each row a weight pickup = enmap.read_map(args.pickup_map) pickup = reorder(pickup, nrow, ncol, d.dets) weight = np.median((pickup[:, 1:] - pickup[:, :-1])**2, -1)
parser.add_argument("-b", "--bsize", type=float, default=1) parser.add_argument("-v", "--verbosity", type=int, default=1) parser.add_argument("-d", "--dets", type=str, default=None) args = parser.parse_args() comm = mpi.COMM_WORLD level = log.verbosity2level(args.verbosity) L = log.init(level=level, rank=comm.rank) dtype = np.float32 if config.get("map_bits") == 32 else np.float64 root = args.odir + "/" + (args.prefix + "_" if args.prefix else "") ncomp = 3 nbin = config.get("nmat_uncorr_nbin") filedb.init() utils.mkdir(args.odir) area = enmap.read_map(args.area) area = enmap.zeros((ncomp, ) + area.shape[-2:], area.wcs, dtype) # First loop through all the tods and find out which are in which scanning # pattern. L.info("Scanning tods") mypatids = {} ids = filedb.scans[args.sel] myinds = range(comm.rank, len(ids), comm.size) ndet_array = 0 for ind, d in scanutils.scan_iterator(ids, myinds, actscan.ACTScan, filedb.data): id = ids[ind] # Determine which scanning pattern we have el = np.mean(d.boresight[:, 2]) az1, az2 = utils.minmax(d.boresight[:, 1])
def read_helper(fname, shape=None, wcs=None): if shape is None: return enmap.read_map(fname) mshape, mwcs = enmap.read_map_geometry(fname) pixbox = enmap.pixbox_of(mwcs, shape, wcs) return enmap.read_map(fname, pixbox=pixbox)
import numpy as np, argparse, os, sys from enlib import enmap, pmat, config from enact import filedb, data parser = config.ArgumentParser(os.environ["HOME"] + "/.enkirc") parser.add_argument("id") parser.add_argument("area") parser.add_argument("--di", type=int, default=0, help="Index into array of accepted detectors to use.") args = parser.parse_args() dtype = np.float64 eqsys = config.get("map_eqsys") area = enmap.read_map(args.area).astype(dtype) area = enmap.zeros((3,)+area.shape[-2:], area.wcs, dtype) entry = filedb.data[args.id] # First get the raw samples d = data.read(entry, subdets=[args.di]) raw_tod = d.tod[0,d.sample_offset:d.cutafter].copy() raw_bore = d.boresight[:,d.sample_offset:d.cutafter].T # Then some calibrated samples d = data.calibrate(d) cal_tod = d.tod[0] cal_bore = d.boresight.T # Apply fourier-truncation to raw data raw_tod = raw_tod[:cal_tod.shape[0]] raw_bore = raw_bore[:cal_bore.shape[0]] # And a proper ACTScan scan = data.ACTScan(entry, subdets=[args.di]) # Detector pointing
debug_tile = None if args.debug_tile is None else [int(w) for w in args.debug_tile.split(",")] # Get the set of bounding boxes, after normalizing them boxes = np.sort(np.array([d.box for d in mapinfo.datasets]),-2) # Read the cmb power spectrum, which is an effective noise # component. T-only cl_path = os.path.join(os.path.dirname(args.config),config.cl_background) cl_bg = powspec.read_spectrum(cl_path)[0,0] # Read our mask. High-resolution and per-dataset masks would be handled otherwise. # This approach only works for low-resolution masks if config.mask_mode == "none": mask = None elif config.mask_mode == "lowres": mask_path = os.path.join(os.path.dirname(args.config), config.mask) mask = enmap.read_map(mask_path).preflat[0] else: raise ValueError("Unrecognized mask mode '%s'" % config.mask_mode) def overlaps_any(box, refboxes): box = np.sort(box, 0) rdec, rra = utils.moveaxis(refboxes - box[0,:], 2,0) wdec, wra = box[1] - box[0] rra -= np.floor(rra[:,0,None]/(2*np.pi)+0.5)*(2*np.pi) for i in range(-1,2): nra = rra + i*(2*np.pi) if np.any((nra[:,1]>0)&(nra[:,0]<wra)&(rdec[:,1]>0)&(rdec[:,0]<wdec)): return True return False def parse_bounds(bstr): res = [] toks = bstr.strip().split(",")
import orphics from enlib import enmap from astropy.io import fits import numpy as np from orphics import maps import orphics.io as io from orphics.io import Plotter as pl import matplotlib import matplotlib.pyplot as plt map90location = '/Users/Teva/maps and catalog data/f090_daynight_all_map_mono_deep56.fits' map150location = '/Users/Teva/maps and catalog data/f150_daynight_all_map_mono_deep56.fits' map217location = '/Users/Teva/maps and catalog data/HFI_SkyMap_217_2048_R2.02_full_cutout_h0.fits' map353location = '/Users/Teva/maps and catalog data/HFI_SkyMap_353_2048_R2.02_full_cutout_h0.fits' lmap90 = enmap.read_map(map90location) lmap150 = enmap.read_map(map150location) lmap217 = enmap.read_map(map217location) lmap353 = enmap.read_map(map353location) tmap90 = lmap90[0] tmap150 = lmap150[0] tmap217 = lmap217[0] tmap353 = lmap353[0] Ny, Nx = tmap90.shape #they have the same shape so it doesn't matter if we use the 90 or 150 map cat_location = '../maps and catalog data/act_confirmed_clusters.fits' #cat_location='../maps and catalog data/act_candidate_clusters.fits' #cat_location='../maps and catalog data/redmapper_dr8_public_v6.3_catalog.fits' out_dir = "./" hdulist = fits.open(cat_location)
ainfo = sharp.alm_info(lmax) sht = sharp.sht(minfo, ainfo) alm = np.zeros((ncomp,ainfo.nelem), dtype=ctype) # Perform the actual transform L.debug("T -> alm") print m.dtype, alm.dtype sht.map2alm(m[0], alm[0]) if ncomp == 3: L.debug("P -> alm") sht.map2alm(m[1:3],alm[1:3], spin=2) del m # Project down on each template for tfile in args.templates: L.info("Reading " + tfile) shape, wcs = enmap.read_map(tfile).geometry if args.rot and args.rot_method != "alm": # Rotate by displacing coordinates and then fixing the polarization L.debug("Computing pixel positions") pmap = enmap.posmap(shape, wcs) if args.rot: L.debug("Computing rotated positions") s1,s2 = args.rot.split(",") opos = coordinates.transform(s2, s1, pmap[::-1], pol=ncomp==3) pmap[...] = opos[1::-1] if len(opos) == 3: psi = -opos[2].copy() del opos L.debug("Projecting") res = curvedsky.alm2map_pos(alm, pmap) if args.rot and ncomp==3:
dtype = np.float32 ncol, nrow = 32, 33 nfile = len(args.ifiles)/2 rad = args.rad * utils.arcmin * utils.fwhm if not args.transpose: imapfiles = args.ifiles[:nfile] ilayfiles = args.ifiles[nfile:] else: imapfiles = args.ifiles[0::2] ilayfiles = args.ifiles[1::2] # Read in our focalplane layouts so we can define our output map bounds dets, offs, boxes, imaps = [], [], [], [] for i in range(nfile): det, off = files.read_point_template(ilayfiles[i]) imap = enmap.read_map(imapfiles[i]) if args.slice: imap = eval("imap"+args.slice) # We want y,x-ordering off = off[:,::-1] box = utils.minmax(off,0) dets.append(det) offs.append(off) boxes.append(box) imaps.append(imap) box = utils.bounding_box(boxes) box = utils.widen_box(box, rad*5, relative=False) # We assume that the two maps have the same pixelization imaps = enmap.samewcs(np.array(imaps), imaps[0]) # Downsample by averaging imaps = enmap.downgrade(imaps, (1,args.step))
parser.add_argument("ofile") parser.add_argument("-s", "--sigma", type=float, default=50) parser.add_argument("-n", "--nsamp", type=int, default=4) args = parser.parse_args() def sim_scan_fast(imap, sigma, rad, n): mask = np.zeros(imap.shape[-2:]) mask[rad:-rad,rad:-rad] = 1 w = np.maximum(1 - ndimage.distance_transform_edt(1-mask)/rad,0)**3 w = w[None,:,:]*np.array([1,0.5,0.5])[:,None,None]*sigma**-2*n w = enmap.samewcs(w, imap) m = imap + np.random.standard_normal(imap.shape)*w**-0.5 m[~np.isfinite(m)] = 0 return m, w template = enmap.read_map(args.template)[:,::2,::2] ps = powspec.read_spectrum(args.powspec, expand="diag") sim = enmap.rand_map(template.shape, template.wcs, ps) # Simulate noisy data m, w = sim_scan_fast(sim, args.sigma, 128, 1) ncomp = m.shape[0] iN = enmap.zeros((ncomp,ncomp)+w.shape[-2:]) for i in range(ncomp): iN[i,i] = w[i] iS = enmap.spec2flat(m.shape, m.wcs, ps, -1) # Draw samples sampler = gibbs.FieldSampler(iS, iN, m) res = enmap.zeros((1+args.nsamp,)+m.shape,m.wcs) res[0] = m
def load_fullsky(sim_root, prefix, k): uiqu = enmap.read_map(sim_root + prefix + "_" + str(k).zfill(2) + ".fits") return uiqu
import numpy as np, argparse from enlib import enmap, utils parser = argparse.ArgumentParser() parser.add_argument("imasks", nargs="+") args = parser.parse_args() maps = [enmap.read_map(fname) for fname in args.imasks] nrow, ncol = 33, 32 for r in range(nrow): for c in range(ncol): uid = c + ncol*r i = r + nrow*c # Loop through each map for this detector az_ranges = [] for mi, map in enumerate(maps): pix_ranges = utils.mask2range(map[i]>0) if len(pix_ranges) == 0: continue az_ranges.append(map.pix2sky([pix_ranges*0,pix_ranges])[1]/utils.degree) if len(az_ranges) == 0: az_ranges = np.zeros((1,0,2)) az_ranges = np.concatenate(az_ranges,0) order = np.argsort(az_ranges[:,0]) az_ranges = az_ranges[order] # And output our cut cut = "%3d " % uid + " ".join(["%.2f:%.2f" % tuple(a) for a in az_ranges]) print cut
parser.add_argument("area") parser.add_argument("sel") parser.add_argument("odir") parser.add_argument("tag", nargs="?") parser.add_argument("-R", "--dist", type=float, default=0.2) parser.add_argument("-e", "--equator", action="store_true") args = parser.parse_args() comm = mpi.COMM_WORLD filedb.init() ids = filedb.scans[args.sel] R = args.dist * utils.degree csize= 100 dtype= np.float32 area = enmap.read_map(args.area).astype(dtype) ncomp= 3 shape= area.shape[-2:] model_fknee = 10 model_alpha = 10 sys = "hor:"+args.planet if args.equator: sys += "/0_0" utils.mkdir(args.odir) prefix = args.odir + "/" if args.tag: prefix += args.tag + "_" for ind in range(comm.rank, len(ids), comm.size): id = ids[ind] bid = id.replace(":","_") entry = filedb.data[id] # Read the tod as usual
def map_to_blocks(map, n): m = map[...,:map.shape[-2]/n*n,:map.shape[-1]/n*n] m = m.reshape(m.shape[:-2]+(m.shape[-2]/n,n,m.shape[-1]/n,n)) return m def calc_map_block_ivar(map, n): m = map_to_blocks(map, n) vmap = np.var(m, axis=(-3,-1)) vmap[vmap!=0] = 1/vmap[vmap!=0] return enmap.samewcs(vmap, map[...,::n,::n]) def calc_map_block_mean(map, n): m = map_to_blocks(map, n) return enmap.samewcs(np.mean(m, axis=(-3,-1)), map[...,::n,::n]) print "Reading map %s" % (mapfiles[0]) map = enmap.read_map(mapfiles[0]).preflat[args.component] print "Reading hit %s" % (hitfiles[0]) hit = enmap.read_map(hitfiles[0]) # We assume that the hitcount maps are 2d def get_bias(bs): # Get bias factor we get from estimating the ratio # via medmean. Determined empirically based on white noise # simulations. We will also have biases from not actually having # white noise, but those will be map-dependent. biases = [0, 0, 0.74, 0.89, 0.94, 0.96, 0.97, 0.98, 0.99, 0.99, 0.99] return 1.0 if bs >= len(biases) else biases[bs] def quant(a, q): return np.percentile(a, q*100) qlim = 0.95 print "Measuring sensitivities"
from orphics.io import Plotter as pl import matplotlib import matplotlib.pyplot as plt import szar from szar import counts import scipy import healpy as hp map90location = '/Users/Teva/maps and catalog data/f090_night_all2_map_mono.fits' map150location = '/Users/Teva/maps and catalog data/f150_night_all2_map_mono.fits' # map217location='/Users/Teva/maps and catalog data/HFI_SkyMap_217_2048_R2.02_full.fits' # map353location='/Users/Teva/maps and catalog data/HFI_SkyMap_353_2048_R2.02_full.fits' # map545location='/Users/Teva/maps and catalog data/HFI_SkyMap_545_2048_R2.02_full.fits' # map857location='/Users/Teva/maps and catalog data/HFI_SkyMap_857_2048_R2.02_full.fits' lmap90 = enmap.read_map(map90location) lmap150 = enmap.read_map(map150location) # lmap217=hp.read_map(map217location) # lmap353=hp.read_map(map353location) # lmap545=hp.read_map(map545location) # lmap857=hp.read_map(map857location) tmap90 = lmap90[0] tmap150 = lmap150[0] # tmap217=lmap217[0] # tmap353=lmap353[0] # tmap545=lmap545[0] # tmap857=lmap857[0] Ny, Nx = tmap90.shape #they have the same shape so it doesn't matter if we use the 90 or 150 map #cat_location='../maps and catalog data/act_confirmed_clusters.fits'
import numpy as np, argparse from enlib import enmap from scipy import ndimage parser = argparse.ArgumentParser() parser.add_argument("pos") parser.add_argument("template") parser.add_argument("ofile") parser.add_argument("-r", "--radius", type=float, default=3) parser.add_argument("-c", "--columns", type=str, default="3,5,2") parser.add_argument("-t", "--threshold", type=float, default=0) args = parser.parse_args() cols = [int(w) for w in args.columns.split(",")] srcinfo = np.loadtxt(args.pos)[:,cols] pos = srcinfo[np.abs(srcinfo[:,2])>=args.threshold][:,:2] * np.pi/180 map = enmap.read_map(args.template) pix = map.sky2pix(pos.T).T.astype(int) pixrad = (map.area()/map.npix)**0.5 mrad = args.radius*np.pi/180/60/pixrad mask = enmap.zeros(map.shape[-2:], map.wcs)+1 mask[pix[:,0],pix[:,1]] = 0 mask = enmap.enmap(1.0*(ndimage.distance_transform_edt(mask) > mrad), map.wcs) enmap.write_map(args.ofile, mask)
hdeg=args.patch_height, yoffset=args.yoffset, mpi_comm=MPI.COMM_WORLD, nsims=args.Nsims, lmax=args.lmax, pix_intermediate=args.pix_inter, bin_lmax=args.bin_lmax) cmb = {} ikappa = {} mlist = ['e', 's', 'r'] mf = {} for m in mlist: if args.meanfield is not None: mf[m] = enmap.read_map(args.meanfield + "/meanfield_" + m + ".hdf") else: mf[m] = 0. for k, index in enumerate(pipe.tasks): cmb['s'], cmb['e'], ikappa['s'], ikappa['e'] = pipe.make_sim(index) cmb['r'] = pipe.rotator.rotate(cmb['s']) ikappa['r'] = pipe.rotator.rotate(ikappa['s'], order=5, mode="constant", cval=0.0, prefilter=True, mask_nan=True, safe=True)
# Point source handling: if src_handling is not None and src_handling["mode"] in ["inpaint","full"]: white_src_handler = mapmaking.SourceHandler(myscans, comm, dtype=dtype, **src_handling) else: white_src_handler = None # 1. Initialize signals L.info("Initializing signals") signals = [] for param in signal_params: effname = get_effname(param) active_scans = apply_scan_limits(myscans, param) if param["type"] == "cut": signal = mapmaking.SignalCut(active_scans, dtype=dtype, comm=comm, name=effname, ofmt=param["ofmt"], output=param["output"]=="yes") signal_cut = signal elif param["type"] == "map": area = enmap.read_map(param["value"]) area = enmap.zeros((args.ncomp,)+area.shape[-2:], area.wcs, dtype) signal = mapmaking.SignalMap(active_scans, area, comm=comm, name=effname, ofmt=param["ofmt"], output=param["output"]=="yes", sys=param["sys"], extra=setup_extra_transforms(param)) elif param["type"] == "fmap": area = enmap.read_map(param["value"]) area = enmap.zeros((args.ncomp,)+area.shape[-2:], area.wcs, dtype) signal = mapmaking.SignalMapFast(active_scans, area, comm=comm, name=effname, ofmt=param["ofmt"], output=param["output"]=="yes", sys=param["sys"], extra=setup_extra_transforms(param)) elif param["type"] == "dmap": area = dmap.read_map(param["value"], bbox=mybbox, tshape=tshape, comm=comm) area = dmap.zeros(area.geometry.aspre(args.ncomp).astype(dtype)) signal = mapmaking.SignalDmap(active_scans, mysubs, area, name=effname, ofmt=param["ofmt"], output=param["output"]=="yes", sys=param["sys"], extra=setup_extra_transforms(param)) elif param["type"] == "fdmap": area = dmap.read_map(param["value"], bbox=mybbox, tshape=tshape, comm=comm) area = dmap.zeros(area.geometry.aspre(args.ncomp).astype(dtype)) signal = mapmaking.SignalDmapFast(active_scans, mysubs, area, name=effname, ofmt=param["ofmt"], output=param["output"]=="yes", sys=param["sys"], extra=setup_extra_transforms(param)) elif param["type"] == "bmap":
def combine_tiles(ipathfmt, opathfmt, combine=2, downsample=2, itile1=(None, None), itile2=(None, None), tyflip=False, txflip=False, pad_to=None, comm=None, verbose=False): """Given a set of tiles on disk at locaiton ipathfmt % {"y":...,"x"...}, combine them into larger tiles, downsample and write the result to opathfmt % {"y":...,"x":...}. x and y must be contiguous and start at 0. reftile[2] indicates the tile coordinates of the first valid input tile. This needs to be specified if not all tiles of the logical tiling are physically present. tyflip and txflip indicate if the tiles coordinate system is reversed relative to the pixel coordinates or not." """ # Expand combine and downsample to 2d combine = np.zeros(2, int) + combine downsample = np.zeros(2, int) + downsample if pad_to is not None: pad_to = np.zeros(2, int) + pad_to # Handle optional mpi rank, size = (comm.rank, comm.size) if comm is not None else (0, 1) # Find the range of input tiles itile1, itile2 = find_tile_range(ipathfmt, itile1, itile2) # Read the first tile to get its size information ibase = enmap.read_map(ipathfmt % {"y": itile1[0], "x": itile1[1]}) * 0 # Find the set of output tiles we need to consider otile1 = itile1 / combine otile2 = (itile2 - 1) / combine + 1 # And loop over them oyx = [(oy, ox) for oy in range(otile1[0], otile2[0]) for ox in range(otile1[1], otile2[1])] for i in range(rank, len(oyx), size): oy, ox = oyx[i] # Read in all associated tiles into a list of lists rows = [] for dy in range(combine[0]): iy = oy * combine[0] + dy if iy >= itile2[0]: continue cols = [] for dx in range(combine[1]): ix = ox * combine[1] + dx if ix >= itile2[1]: continue if iy < itile1[0] or ix < itile1[1]: # The first tiles are missing on disk, but are # logically a part of the tiling. Use ibase, # which has been zeroed out. cols.append(ibase) else: itname = ipathfmt % {"y": iy, "x": ix} cols.append(enmap.read_map(itname)) if txflip: cols = cols[::-1] rows.append(cols) # Stack them next to each other into a big tile if tyflip: rows = rows[::-1] omap = enmap.tile_maps(rows) # Downgrade if necessary if np.any(downsample > 1): omap = enmap.downgrade(omap, downsample) if pad_to is not None: # Padding happens towards the end of the tiling, # which depends on the flip status padding = np.array( [[0, 0], [pad_to[0] - omap.shape[-2], pad_to[1] - omap.shape[-1]]]) if tyflip: padding[:, 0] = padding[::-1, 0] if txflip: padding[:, 1] = padding[::-1, 1] omap = enmap.pad(omap, padding) # And output otname = opathfmt % {"y": oy, "x": ox} utils.mkdir(os.path.dirname(otname)) enmap.write_map(otname, omap) if verbose: print otname
# Project a single pickup map into horizontal coordinates import numpy as np, os from enlib import enmap, utils, config, array_ops from enact import actdata, filedb parser = config.ArgumentParser(os.environ["HOME"] + "/.enkirc") parser.add_argument("pickup_map") parser.add_argument("template") parser.add_argument("sel_repr") parser.add_argument("el", type=float) parser.add_argument("ofile") args = parser.parse_args() filedb.init() nrow, ncol = 33, 32 # Read our template, which represents the output horizontal coordinates template = enmap.read_map(args.template) # Use our representative selector to get focalplane offsets and polangles entry = filedb.data[filedb.scans[args.sel_repr][0]] d = actdata.read(entry, ["boresight", "point_offsets", "polangle"]) d.boresight[2] = args.el # In degrees, calibrated in next step d = actdata.calibrate(d, exclude=["autocut"]) def reorder(map, nrow, ncol, dets): return enmap.samewcs(map[utils.transpose_inds(dets,nrow,ncol)],map) # Read our map, and give each row a weight pickup = enmap.read_map(args.pickup_map) pickup = reorder(pickup, nrow, ncol, d.dets) weight = np.median((pickup[:,1:]-pickup[:,:-1])**2,-1) weight[weight>0] = 1/weight[weight>0]