def l(cseed,kseed,returnk=False,index=None):
cname = fout_dir+"lensed_covseed_"+str(args.covseed).zfill(3)+"_cmbseed_"+str(cseed).zfill(5)+"_kseed_"+str(kseed).zfill(5)+".hdf"
if unlensed:
seedroot = (covseed)*Nsets*Nsims
lensedt = parray_dat.get_unlensed_cmb(seed=seedroot+cseed,scalar=False)
else:
lensedt = enmap.read_map(cname)[0] if polsims else enmap.read_map(cname)
# -- add beam and noise if you want --
if "noiseless" not in expf_name:
assert index is not None
if rank==0: print("Adding beam...")
flensed = fftfast.fft(lensedt,axes=[-2,-1])
flensed *= parray_dat.lbeam
lensedt = fftfast.ifft(flensed,axes=[-2,-1],normalize=True).real
if rank==0: print("Adding noise...")
seedroot = (covseed+1)*Nsets*Nsims # WARNING: noise sims will be correlated with CMB from the next covseed
nseed = seedroot+index
noise = parray_dat.get_noise_sim(seed=nseed)
if paper:
cents, noise1d = lbinner.bin(power(noise)[0])
mpibox.add_to_stats('noisett',noise1d)
lensedt += noise
lensedt = enmap.ndmap(lensedt,wcs_dat)
if returnk:
kname = fout_dir+"kappa_covseed_"+str(args.covseed).zfill(3)+"_kseed_"+str(kseed).zfill(5)+".hdf"
return lensedt,enmap.read_map(kname)
else:
return lensedt
def read_tileset_geometry(ipathfmt, itile1=(None,None), itile2=(None,None)):
itile1, itile2 = find_tile_range(ipathfmt, itile1, itile2)
m1 = enmap.read_map(ipathfmt % {"y":itile1[0],"x":itile1[1]})
m2 = enmap.read_map(ipathfmt % {"y":itile2[0]-1,"x":itile2[1]-1})
wy,wx = m1.shape[-2:]
oshape = tuple(np.array(m1.shape[-2:])*(itile2-itile1-1) + np.array(m2.shape[-2:]))
return bunch.Bunch(shape=m1.shape[:-2]+oshape, wcs=m1.wcs, dtype=m1.dtype,
tshape=m1.shape[-2:])
def add_maps(imaps, omap): if args.verbose: print "Reading %s" % imaps[0] m = nonan(enmap.read_map(imaps[0])) for mif in imaps[1:]: if args.verbose: print "Reading %s" % mif m += nonan(enmap.read_map(mif)) if args.verbose: "Writing %s" % omap enmap.write_map(omap, m)
def add_maps(imaps, omap): if args.verbose: print "Reading %s" % imaps[0] m = nonan(enmap.read_map(imaps[0]))*scales[0] for scale, mif in zip(scales[1:],imaps[1:]): if args.verbose: print "Reading %s" % mif m += nonan(enmap.read_map(mif))*scale if args.mean: m /= len(imaps) if args.verbose: "Writing %s" % omap enmap.write_map(omap, m)
def add_maps(imaps, omap): if args.verbose: print "Reading %s" % imaps[0] m = nonan(enmap.read_map(imaps[0]))*scales[0] for scale, mif in zip(scales[1:],imaps[1:]): if args.verbose: print "Reading %s" % mif m2 = nonan(enmap.read_map(mif, geometry=(m.shape,m.wcs)))*scale n = min(len(m.preflat),len(m2.preflat)) m.preflat[:n] += m2.preflat[:n] if args.mean: m /= len(imaps) if args.verbose: "Writing %s" % omap enmap.write_map(omap, m)
def read_maps(fmt, n, ntot=4):
try:
maps = en.read_map(fmt)
if maps.ndim == ntot - 1: maps = en.enmap([maps] * n, maps.wcs)
if maps.ndim != ntot:
raise ValueError("Map %s must have %d dimensions" % (fmt, ntot))
return maps
except (IOError, OSError):
maps = [en.read_map(fmt % i) for i in range(n)]
maps = en.ndmap(maps, maps[0].wcs)
if maps.ndim != ntot:
maps = maps.reshape(maps.shape[:-2] + (1, ) * (maps.ndim - ntot) +
maps.shape[-2:])
return maps
def read_map(ifile, slice=None):
if not os.path.isdir(ifile):
map = enmap.read_map(ifile)
if slice: map = eval("map" + slice)
else:
map = retile.read_monolithic(ifile, slice=slice, verbose=False)
return map
def read_map(ifile, slice=None):
if not os.path.isdir(ifile):
map = enmap.read_map(ifile)
if slice: map = eval("map"+slice)
else:
map = retile.read_monolithic(ifile, slice=slice, verbose=False)
return map
def read_area(ipathfmt, opix, itile1=(None,None), itile2=(None,None), verbose=False,
cache=None, slice=None):
"""Given a set of tiles on disk with locations ipathfmt % {"y":...,"x":...},
read the data corresponding to the pixel range opix[{from,to],{y,x}] in
the full map."""
opix = np.asarray(opix)
# Find the range of input tiles
itile1, itile2 = find_tile_range(ipathfmt, itile1, itile2)
# To fill in the rest of the information we need to know more
# about the input tiling, so read the first tile
if cache is None or cache[2] is None:
geo = read_tileset_geometry(ipathfmt, itile1=itile1, itile2=itile2)
else: geo = cache[2]
if cache is not None: cache[2] = geo
isize = geo.tshape
osize = opix[1]-opix[0]
omap = enmap.zeros(geo.shape[:-2]+tuple(osize), geo.wcs, geo.dtype)
# Find out which input tiles overlap with this output tile.
# Our tile stretches from opix1:opix2 relative to the global input pixels
it1 = opix[0]/isize
it2 = (opix[1]-1)/isize+1
noverlap = 0
for ity in range(it1[0],it2[0]):
if ity < itile1[0] or ity >= itile2[0]: continue
# Start/end of this tile in global input pixels
ipy1, ipy2 = ity*isize[0], (ity+1)*isize[0]
overlap = range_overlap(opix[:,0],[ipy1,ipy2])
oy1,oy2 = overlap-opix[0,0]
iy1,iy2 = overlap-ipy1
for itx in range(it1[1],it2[1]):
if itx < itile1[1] or itx >= itile2[1]: continue
ipx1, ipx2 = itx*isize[1], (itx+1)*isize[1]
overlap = range_overlap(opix[:,1],[ipx1,ipx2])
ox1,ox2 = overlap-opix[0,1]
ix1,ix2 = overlap-ipx1
# Read the input tile and copy over
iname = ipathfmt % {"y":ity,"x":itx}
if cache is None or cache[0] != iname:
imap = enmap.read_map(iname)
if slice: imap = eval("imap"+slice)
else: imap = cache[1]
if cache is not None:
cache[0], cache[1] = iname, imap
if verbose: print iname
# Edge input tiles may be smaller than the standard
# size.
ysub = isize[0]-imap.shape[-2]
xsub = isize[1]-imap.shape[-1]
# If the input map is too small, there may actually be
# zero overlap.
if oy2-ysub <= oy1 or ox2-xsub <= ox1: continue
omap[...,oy1:oy2-ysub,ox1:ox2-xsub] = imap[...,iy1:iy2-ysub,ix1:ix2-xsub]
noverlap += 1
if noverlap == 0:
raise IOError("No tiles for tiling %s in range %s" % (ipathfmt, ",".join([":".join([str(p) for p in r]) for r in opix.T])))
# Set up the wcs for the output tile
omap.wcs.wcs.crpix -= opix[0,::-1]
return omap
def patch_array_from_config(Config,
exp_name,
shape,
wcs,
dimensionless=True,
TCMB=2.7255e6,
skip_real=False,
skip_instrument=False):
pa = fmaps.PatchArray(shape,
wcs,
dimensionless=dimensionless,
TCMB=TCMB,
skip_real=skip_real)
if not (skip_instrument):
try:
bfile = Config.get(exp_name, "beam_file")
ells, bls = np.loadtxt(bfile,
delimiter=",",
unpack=True,
use_cols=[0, 1])
pa.add_1d_beam(ells, bls, fill_value="extrapolate")
except:
fwhm = Config.getfloat(exp_name, "beam")
pa.add_gaussian_beam(fwhm)
try:
n2d_file_T = Config.get(exp_name, "noise_2d_file_T")
n2d_file_P = Config.get(exp_name, "noise_2d_file_P")
imapT = enmap.read_map(n2d_file_T)
imapP = enmap.read_map(n2d_file_P)
pa.add_noise_2d(nT=imapT, nP=imapP)
except:
noise_T = Config.getfloat(exp_name, "noise_T")
noise_P = Config.getfloat(exp_name, "noise_P")
lknee_T = Config.getfloat(exp_name, "lknee_T")
lknee_P = Config.getfloat(exp_name, "lknee_P")
alpha_T = Config.getfloat(exp_name, "alpha_T")
alpha_P = Config.getfloat(exp_name, "alpha_P")
pa.add_white_noise_with_atm(noise_T, noise_P, lknee_T, alpha_T,
lknee_P, alpha_P)
return pa
def flatFitsToHealpix(fitsFile, nside, downgrade=1):
from enlib import enmap
imap = enmap.read_map(fitsFile)
if downgrade > 1:
imap = enmap.downgrade(imap, args.downgrade)
omap = imap.to_healpix(nside=args.nside)
return omap
def build_single(ifile, srcs, beam, ofile, mask_level=0, apod_size=16): imap = enmap.read_map(ifile) omap, oslice = pointsrcs.sim_srcs(imap.shape[-2:], imap.wcs, srcs, beam, return_padded=True) if mask_level: mask = omap > mask_level omap = 1-np.cos(np.minimum(1,ndimage.distance_transform_edt(1-mask)/16.0)*np.pi) omap = enmap.samewcs(omap, imap) omap = omap[oslice] enmap.write_map(ofile, omap)
def read_map(fname, pbox, cname, write_cache=True, read_cache=True):
if os.path.isdir(fname):
fname = fname + "/tile%(y)03d_%(x)03d.fits"
if read_cache and os.path.isfile(cname):
map = enmap.read_map(cname).astype(dtype)
else:
map = retile.read_area(fname, pbox).astype(dtype)
if write_cache: enmap.write_map(cname, map)
#if map.ndim == 3: map = map[:1]
return map
def read_div(fname):
m = nonan(enmap.read_map(fname))*1.0
return m
#return m.preflat[:1][None]
if m.ndim == 2:
res = enmap.zeros((padlen,padlen)+m.shape[-2:], m.wcs, m.dtype)
for i in range(padlen):
res[i,i] = m
return res
elif m.ndim == 4: return m
else: raise ValueError("Wrong number of dimensions in div %s" % fname)
def read_div(fname):
m = nonan(enmap.read_map(fname))*1.0
return m
#return m.preflat[:1][None]
if m.ndim == 2:
res = enmap.zeros((padlen,padlen)+m.shape[-2:], m.wcs, m.dtype)
for i in range(padlen):
res[i,i] = m
return res
elif m.ndim == 4: return m
else: raise ValueError("Wrong number of dimensions in div %s" % fname)
def retile(ipathfmt, opathfmt, itile1=(None,None), itile2=(None,None),
otileoff=(0,0), otilenum=(None,None), ocorner=(-np.pi/2,-np.pi),
otilesize=(675,675), comm=None, verbose=False, slice=None):
"""Given a set of tiles on disk with locations ipathfmt % {"y":...,"x":...},
retile them into a new tiling and write the result to opathfmt % {"y":...,"x":...}.
The new tiling will have tile size given by otilesize[2]. Negative size means the
tiling will to down/left instead of up/right. The corner of the tiling will
be at sky coordinates ocorner[2] in radians. The new tiling will be pixel-
compatible with the input tiling - w.g. the wcs will only differ by crpix.
The output tiling will logically cover the whole sky, but only output tiles
that overlap with input tiles will actually be written. This can be modified
by using otileoff[2] and otilenum[2]. otileoff gives the tile indices of the
corner tile, while otilenum indicates the number of tiles to write."""
# Set up mpi
rank, size = (comm.rank, comm.size) if comm is not None else (0, 1)
# Expand any scalars
otilesize = np.zeros(2,int)+otilesize
otileoff = np.zeros(2,int)+otileoff
# Find the range of input tiles
itile1, itile2 = find_tile_range(ipathfmt, itile1, itile2)
# To fill in the rest of the information we need to know more
# about the input tiling, so read the first tile
ibase = enmap.read_map(ipathfmt % {"y":itile1[0],"x":itile1[1]})
if slice: ibase = eval("ibase"+slice)
itilesize = ibase.shape[-2:]
# Find the pixel position of our output corners according to the wcs.
# This is the last place we need to do a coordinate transformation.
# All the rest can be done in pure pixel logic.
pixoff = np.round(ibase.sky2pix(ocorner)).astype(int)
# Find the range of output tiles
def pix2otile(pix, ioff, osize): return (pix-ioff)/osize
otile1 = pix2otile(itile1*itilesize, pixoff, otilesize)
otile2 = pix2otile(itile2*itilesize-1, pixoff, otilesize)
otile1, otile2 = np.minimum(otile1,otile2), np.maximum(otile1,otile2)
otile2 += 1
# We can now loop over output tiles
cache = [None,None,None]
oyx = [(oy,ox) for oy in range(otile1[0],otile2[0]) for ox in range(otile1[1],otile2[1])]
for i in range(rank, len(oyx), size):
otile = np.array(oyx[i])
# Find out which input tiles overlap with this output tile.
# Our tile stretches from opix1:opix2 relative to the global input pixels
opix1 = otile*otilesize + pixoff
opix2 = (otile+1)*otilesize + pixoff
# output tiles and input tiles may increase in opposite directions
opix1, opix2 = np.minimum(opix1,opix2), np.maximum(opix1,opix2)
try: omap = read_area(ipathfmt, [opix1,opix2],itile1=itile1, itile2=itile2,cache=cache, slice=slice)
except IOError: continue
oname = opathfmt % {"y":otile[0]+otileoff[0],"x":otile[1]+otileoff[1]}
utils.mkdir(os.path.dirname(oname))
enmap.write_map(oname, omap)
if verbose: print oname
def sim(cseed, kseed, skip_kappa_verif=False):
if not (vonly):
unlensed = parray_sim.get_unlensed_cmb(seed=seedroot + cseed,
scalar=False)
kappa = aio.kappa_from_config(Config,
kappa_section,
parray_sim,
seed=seedroot + kseed)
lensed = parray_sim.get_lensed(unlensed,
order=lens_order,
mode="spline",
border="cyclic")
luteb, dummy = sverif_cmb.add_power("unlensed", unlensed)
llteb, dummy = sverif_cmb.add_power("lensed", lensed)
if not (skip_kappa_verif):
lk, dummy = sverif_kappa.add_power("kappa", kappa)
cname = fout_dir + "lensed_covseed_" + str(
args.covseed).zfill(3) + "_cmbseed_" + str(cseed).zfill(
5) + "_kseed_" + str(kseed).zfill(5) + ".hdf"
kname = fout_dir + "kappa_covseed_" + str(
args.covseed).zfill(3) + "_kseed_" + str(kseed).zfill(5) + ".hdf"
if vonly:
dlensed = enmap.read_map(cname)
dkappa = enmap.read_map(kname)
else:
dlensed = lensed if abs(pixratio - 1.) < 1.e-3 else enmap.ndmap(
resample.resample_fft(lensed, shape_dat), wcs_dat)
dkappa = kappa if abs(pixratio - 1.) < 1.e-3 else enmap.ndmap(
resample.resample_fft(kappa, shape_dat[-2:]), wcs_dat)
dlensed.write(cname)
dkappa.write(kname)
dllteb, dummy = sverif_dcmb.add_power("dlensed", dlensed)
if not (skip_kappa_verif):
dlk, dummy = sverif_dkappa.add_power("dkappa", dkappa)
def build_single(ifile, srcs, beam, ofile, mask_level=0, apod_size=16):
imap = enmap.read_map(ifile)
omap, oslice = pointsrcs.sim_srcs(imap.shape[-2:],
imap.wcs,
srcs,
beam,
return_padded=True)
if mask_level:
mask = omap > mask_level
omap = 1 - np.cos(
np.minimum(1,
ndimage.distance_transform_edt(1 - mask) / 16.0) *
np.pi)
omap = enmap.samewcs(omap, imap)
omap = omap[oslice]
enmap.write_map(ofile, omap)
def read_map(fname,
pbox,
name=None,
cache_dir=None,
dtype=None,
read_cache=False):
if os.path.isdir(fname):
fname = fname + "/tile%(y)03d_%(x)03d.fits"
if read_cache:
map = enmap.read_map(cache_dir + "/" + name)
if dtype is not None: map = map.astype(dtype)
else:
map = retile.read_area(fname, pbox).astype(dtype)
if dtype is not None: map = map.astype(dtype)
if cache_dir is not None and name is not None:
enmap.write_map(cache_dir + "/" + name, map)
#if map.ndim == 3: map = map[:1]
return map
def read(self,y,x):
if self.nphi: x = x % self.nphi
for c in self.cache:
if c[0] == (y,x): return c[1]
else:
while len(self.cache) >= self.ncache:
del self.cache[0]
fname = self.pathfmt % {"y":y,"x":x}
if os.path.isfile(fname):
m = enmap.read_map(fname)
if self.crop:
m = m[...,self.crop:-self.crop,self.crop:-self.crop]
if self.ncomp:
m = m.preflat
extra = np.tile(m[:1]*0, (self.ncomp-len(m),1,1))
m = enmap.samewcs(np.concatenate([m,extra],0),m)
else:
m = None
self.cache.append([(y,x),m])
return m
def enmap_from_config_section(Config, section, pol=False):
analysis_section = section
projection = Config.get(analysis_section, "projection")
try:
pt_file = Config.get(analysis_section, "patch_template")
imap = enmap.read_map(pt_file)
shape_dat = imap.shape
wcs_dat = imap.wcs
if pol and len(shape_dat) < 3:
shape_dat = (3, shape_dat[0], shape_dat[1])
res = np.min(imap.extent() / imap.shape[-2:]) * 60. * 180. / np.pi
except:
pixel_analysis = Config.getfloat(analysis_section, "pixel_arcmin")
try:
width_analysis_deg = Config.getfloat(analysis_section,
"patch_degrees_width")
except:
width_analysis_deg = Config.getfloat(analysis_section,
"patch_arcmin_width") / 60.
try:
height_analysis_deg = Config.getfloat(analysis_section,
"patch_degrees_height")
except:
height_analysis_deg = Config.getfloat(analysis_section,
"patch_arcmin_height") / 60.
ra_offset = Config.getfloat(analysis_section, "ra_offset")
dec_offset = Config.getfloat(analysis_section, "dec_offset")
shape_dat, wcs_dat = maps.rect_geometry(
width_analysis_deg * 60.,
pixel_analysis,
proj=projection,
pol=pol,
height_arcmin=height_analysis_deg * 60.,
xoffset_degree=ra_offset,
yoffset_degree=dec_offset)
return shape_dat, wcs_dat
if args.mode == "find": # Point source finding mode results = [] map_keys = ["map","snmap","model","resid","resid_snmap"] utils.mkdir(args.odir) # Mpi-parallelization over regions is simple, but a bit lazy. It means that there will be # no speedup for single-region maps for ri in range(comm.rank, len(regions), comm.size): reg_fid = regions[ri] reg_pad = dory.pad_region(reg_fid, args.pad) print "%3d region %3d/%d %5d %5d %6d %6d" % (comm.rank, ri+1, len(regions), reg_fid[0,0], reg_fid[1,0], reg_fid[0,1], reg_fid[1,1]) try: # We only use T to find sources in find mode for now. P usually has much lower S/N and # doesn't help much. Should add it later, though. imap = enmap.read_map(args.imap, pixbox=reg_pad).preflat[0] idiv = enmap.read_map(args.idiv, pixbox=reg_pad).preflat[0] if args.mask: mshape, mwcs = enmap.read_map_geometry(args.mask) mbox = enmap.pixbox_of(mwcs, imap.shape, imap.wcs) idiv *= (1-enmap.read_map(args.mask, pixbox=mbox).preflat[0]) if "debug" in args.output: dump_prefix = args.odir + "/region_%02d_" % ri else: dump_prefix = None result = dory.find_srcs(imap, idiv, beam, freq=args.freq, apod=args.apod, apod_margin=args.apod_margin, snmin=args.nsigma, verbose=args.verbose, dump=dump_prefix) # FIXME: artifacts are act-specific result = dory.prune_artifacts(result) except Exception as e: print "Exception for task %d region %d: %s" % (comm.rank, ri, e.message) raise # Write region output
parser = config.ArgumentParser(os.environ["HOME"]+"/.enkirc")
parser.add_argument("area")
parser.add_argument("sel")
parser.add_argument("odir")
parser.add_argument("-n", "--nstep", type=int, default=50)
parser.add_argument("-m", "--method",type=str, default="messenger")
parser.add_argument( "--ndet", type=int, default=None)
parser.add_argument("-p", "--precompute", action="store_true")
parser.add_argument("-o", "--ostep", type=int, default=10)
args = parser.parse_args()
utils.mkdir(args.odir)
comm = mpi.COMM_WORLD
dtype = np.float32
ncomp = 3
area = enmap.read_map(args.area)
area = enmap.zeros((ncomp,)+area.shape[-2:],area.wcs,dtype)
Tscale = 0.9
nstep = args.nstep
downsample = config.get("downsample")
filedb.init()
ids = filedb.scans[args.sel]
# Was 1e7
cooldown = sum([[10**j]*5 for j in range(6,0,-1)],[])+[1]
# Read my scans
njunk_tot = 0
cg_rhs = area*0
cg_rjunk = []
if args.precompute:
def vread(fname): print "Reading %s" % fname return enmap.read_map(fname)
def read_map(name, bbpix=None, bbox=None, tshape=None, comm=None):
if comm is None: comm = mpi.COMM_WORLD
if os.path.isdir(name):
# Find the number of tiles in the map
entries = os.listdir(name)
nrow, ncol = 0, 0
for entry in entries:
match = re.search(r'^tile(\d+)_(\d+).([^.]+)$', entry)
if match:
nrow = max(nrow, 1 + int(match.group(1)))
ncol = max(ncol, 1 + int(match.group(2)))
ext = match.group(3)
# Build the list of tile files
tfiles = [["" for c in range(ncol)] for r in range(nrow)]
for entry in entries:
match = re.search(r'^tile(\d+)_(\d+).([^.]+)$', entry)
if match: tfiles[int(match.group(1))][int(match.group(2))] = entry
if nrow == 0: raise IOError("'%s' is not a valid dmap file" % name)
# Find the tile size and map extent
tile1 = enmap.read_map(name + "/" + tfiles[0][0])
tile2 = enmap.read_map(name + "/" + tfiles[-1][-1])
npre, tshape = tile1.shape[:-2], tile1.shape[-2:]
wcs = tile1.wcs
shape = npre + (tile1.shape[-2] *
(nrow - 1) + tile2.shape[-2], tile1.shape[-1] *
(ncol - 1) + tile2.shape[-1])
dtype = tile1.dtype
# Construct our dmap and read our tiles
map = Dmap(
geometry(shape,
wcs,
bbpix=bbpix,
bbox=bbox,
tshape=tshape,
dtype=dtype,
comm=comm))
for pos, tile in zip(map.loc_pos, map.tiles):
tile[:] = enmap.read_map(name + "/" + tfiles[pos[0]][pos[1]])
else:
# Map is in a single file. Get map info
if comm.rank == 0:
canvas = enmap.read_map(name)
shape = comm.bcast(canvas.shape)
wcs = WCS(comm.bcast(canvas.wcs.to_header_string()))
dtype = comm.bcast(canvas.dtype)
# Hack: Pickling changes the dtype from < to =, both of which are
# equivalent. But mpi has problems with the former.
canvas.dtype = dtype
else:
shape = comm.bcast(None)
wcs = WCS(comm.bcast(None))
dtype = comm.bcast(None)
canvas = None
map = Dmap(
geometry(shape,
wcs,
bbpix=bbpix,
bbox=bbox,
tshape=tshape,
dtype=dtype,
comm=comm))
# And send data to the tiles
enmap2dmap(canvas, map)
return map
shared = True
utils.mkdir(args.odir)
filedb.init()
db = filedb.data
ids = filedb.scans[args.query]
myinds = range(len(ids))[myid::nproc]
n = len(ids)
# Optinal input map to subtract
imap = None
if args.imap:
toks = args.imap.split(":")
imap_sys, fname = ":".join(toks[:-1]), toks[-1]
imap = bunch.Bunch(sys=imap_sys or None, map=enmap.read_map(fname))
for i in myinds:
id = ids[i]
entry = db[id]
ofile = "%s/%s.hdf" % (args.odir, id)
if os.path.isfile(ofile) and args.resume: continue
t = []
t.append(time.time())
try:
fields = ["gain", "tconst", "cut", "tod", "boresight", "noise_cut"]
if args.spikecut: fields.append("spikes")
if args.imap: fields += ["polangle", "point_offsets", "site"]
d = data.read(entry, fields)
t.append(time.time())
d = data.calibrate(d)
# Try to get about the same amount of data for each mpi task.
# If we use distributed maps, we also try to make things as local as possible
mycosts = [s.nsamp*s.ndet for s in myscans]
if is_dmap:
myboxes = [scanutils.calc_sky_bbox_scan(s, args.sys) for s in myscans]
myinds, mysubs, mybbox = scanutils.distribute_scans(myinds, mycosts, myboxes, comm)
else:
myinds = scanutils.distribute_scans(myinds, mycosts, None, comm)
L.info("Rereading shuffled scans")
del myscans # scans do take up some space, even without the tod being read in
myinds, myscans = scanutils.read_scans(ids, myinds, actscan.ACTScan,
filedb.data, dets=args.dets, downsample=config.get("downsample"))
L.info("Reading input map")
if not is_dmap: imap = enmap.read_map(args.imap)
else: imap = dmap.read_map(args.imap, bbox=mybbox, tshape=tshape, comm=comm)
dtype = "=f" if imap.dtype == np.float32 else "=d"
imap = imap.astype(dtype, copy=True)
L.info("Initializing signals")
signal_cut = mapmaking.SignalCut(myscans, imap.dtype, comm=comm)
if not is_dmap:
signal_map = mapmaking.SignalMap(myscans, imap, comm=comm)
precon = mapmaking.PreconMapBinned(signal_map, signal_cut, myscans, [], noise=False, hits=False)
else:
signal_map = mapmaking.SignalDmap(myscans, mysubs, imap, sys=args.sys)
precon = mapmaking.PreconDmapBinned(signal_map, signal_cut, myscans, [], noise=False, hits=False)
# We can now actually perform the postfilter
L.info("Postfiltering")
myinds, mysubs, mybbox = scanutils.distribute_scans(
myinds, mycosts, myboxes, comm)
else:
myinds = scanutils.distribute_scans(myinds, mycosts, None, comm)
L.info("Rereading shuffled scans")
del myscans # scans do take up some space, even without the tod being read in
myinds, myscans = scanutils.read_scans(ids,
myinds,
actscan.ACTScan,
filedb.data,
dets=args.dets,
downsample=config.get("downsample"))
L.info("Reading input map")
if not is_dmap: imap = enmap.read_map(args.imap)
else: imap = dmap.read_map(args.imap, bbox=mybbox, tshape=tshape, comm=comm)
dtype = "=f" if imap.dtype == np.float32 else "=d"
imap = imap.astype(dtype, copy=True)
L.info("Initializing signals")
signal_cut = mapmaking.SignalCut(myscans, imap.dtype, comm=comm)
if not is_dmap:
signal_map = mapmaking.SignalMap(myscans, imap, comm=comm)
precon = mapmaking.PreconMapBinned(signal_map,
signal_cut,
myscans, [],
noise=False,
hits=False)
else:
signal_map = mapmaking.SignalDmap(myscans, mysubs, imap, sys=args.sys)
regions to cut. This mask can then be turned into a per-scanning-pattern
cut list in mask2cut"""
import numpy as np, argparse
from scipy import ndimage
from enlib import enmap, fft
parser = argparse.ArgumentParser()
parser.add_argument("imap")
parser.add_argument("omap")
parser.add_argument("-T", "--threshold", type=float, default=500)
parser.add_argument("-H", "--high-threshold", type=float, default=4000)
parser.add_argument("-W", "--minwidth", type=int, default=2)
parser.add_argument("-w", "--widen", type=int, default=3)
parser.add_argument("-m", "--maxmean", type=int, default=250)
args = parser.parse_args()
imap = enmap.read_map(args.imap)
ngood = int(np.sum(np.any(imap!=0,1)))
# Smooth a bit in az direction to reduce noise
m = imap
fm = fft.redft00(m)
nf = fm.shape[1]
fm[:,nf/4:] = 0
fft.redft00(fm, m, normalize=True)
def fixwidth(row):
row = ndimage.distance_transform_edt(row) > args.minwidth
if np.any(row):
row = ndimage.distance_transform_edt(1-row) < args.minwidth + args.widen
return row
def read_map(fname): m = nonan(enmap.read_map(fname)) #return m.preflat[:1] return m.reshape(-1, m.shape[-2], m.shape[-1])
parser.add_argument("sel")
parser.add_argument("objname")
parser.add_argument("mask")
parser.add_argument("odir")
parser.add_argument("--margin", type=float, default=1)
args = parser.parse_args()
filedb.init()
ids = filedb.scans[args.sel]
comm = mpi.COMM_WORLD
dtype= np.float32
ntod = len(ids)
margin = args.margin*utils.degree
imask = enmap.read_map(args.mask)
# Expand to 3 components, as the pointing code expects that
mask = enmap.zeros((3,)+imask.shape[-2:], imask.wcs, dtype)
mask[0] = imask.reshape((-1,)+imask.shape[-2:])[0]
del imask
utils.mkdir(args.odir)
# Each mpi tasks opens its own work file
wfname = args.odir + "/work_%03d.hdf" % comm.rank
mystats = []
with h5py.File(wfname, "w") as hfile:
for ind in range(comm.rank*ntod//comm.size, (comm.rank+1)*ntod//comm.size):
id = ids[ind]
entry = filedb.data[id]
# We only need pointing to build this cut
def read(fname):
m = enmap.read_map(fname)
if slice: m = eval("m" + slice)
return m
import numpy as np, argparse
from scipy import ndimage
from enlib import enmap
parser = argparse.ArgumentParser()
parser.add_argument("ifile")
parser.add_argument("ofile")
parser.add_argument("-r", "--apod-radius", type=int, default=64)
args = parser.parse_args()
def make_apod(shape, rad):
mask = np.zeros(shape[-2:])
mask[rad:-rad, rad:-rad] = 1
w = np.maximum(1 - ndimage.distance_transform_edt(1 - mask) / rad, 0) ** 3
return w
teb = enmap.read_map(args.ifile)
tqu = enmap.harm2map(enmap.fft(teb))
mask = make_apod(teb.shape, args.apod_radius)
tqu_mask = tqu * mask[None]
teb_mask = enmap.ifft(enmap.map2harm(tqu_mask)).real
res = enmap.samewcs([teb, tqu, tqu_mask, teb_mask], teb)
enmap.write_map(args.ofile, res)
shared = True
utils.mkdir(args.odir)
filedb.init()
db = filedb.data
ids = filedb.scans[args.query]
myinds = range(len(ids))[myid::nproc]
n = len(ids)
# Optinal input map to subtract
imap = None
if args.imap:
toks = args.imap.split(":")
imap_sys, fname = ":".join(toks[:-1]), toks[-1]
imap = bunch.Bunch(sys=imap_sys or None, map=enmap.read_map(fname))
for i in myinds:
id = ids[i]
entry = db[id]
ofile = "%s/%s.hdf" % (args.odir, id)
if os.path.isfile(ofile) and args.resume: continue
t=[]; t.append(time.time())
try:
fields = ["gain","tconst","cut","tod","boresight", "noise_cut"]
if args.spikecut: fields.append("spikes")
if args.imap: fields += ["polangle","point_offsets","site"]
d = data.read(entry, fields) ; t.append(time.time())
d = data.calibrate(d) ; t.append(time.time())
if args.imap:
# Make a full scan object, so we can perform pointing projection
dirname = os.path.dirname(fname)
if len(dirname) == 0: dirname = "."
base_ext = os.path.basename(fname)
# Find the extension. Using the last dot does not work for .fits.gz.
# Using the first dot in basename does not work for foo2.5_bar.fits.
# Will therefore handle .gz as a special case.
if base_ext.endswith(".gz"):
dot = base_ext[:-3].rfind(".")
else:
dot = base_ext.rfind(".")
if dot < 0: dot = len(base_ext)
base = base_ext[:dot]
ext = base_ext[dot+1:]
return dirname, base, ext
for ifile in args.ifiles:
m = enmap.read_map(ifile)
if args.slice:
m = eval("m" + args.slice)
N = m.shape[:-2]
for i, comp in enumerate(m.preflat):
I = np.unravel_index(i, N) if len(N) > 0 else []
if args.symmetric and np.any(np.sort(I) != I):
continue
ndigits = [get_num_digits(n) for n in N]
compname = "_".join(["%0*d" % (ndig,ind) for ndig,ind in zip(ndigits,I)]) if len(N) > 0 else ""
dir, base, ext = split_file_name(ifile)
oname = dir + "/" + base + "_" + compname + "." + ext
if args.verbose: print(oname)
enmap.write_map(oname, comp)
import numpy as np, argparse, os
from enlib import enmap, utils
from enact import files
parser = argparse.ArgumentParser()
parser.add_argument("ifiles", nargs="+")
parser.add_argument("ofile")
args = parser.parse_args()
maps = []
for ifile in args.ifiles:
ibase = os.path.basename(ifile)
print ibase
data = enmap.read_map(ifile)
maps.append(data)
maps = enmap.samewcs(np.array(maps),maps[0])
print np.sum(maps**2)
print maps.shape
masked = np.ma.MaskedArray(maps, mask=maps==0)
print np.sum(masked**2)
medmap = np.ma.median(masked,0)
medmap = enmap.samewcs(np.asarray(medmap),maps)
# And run through again, subtracting it
enmap.write_map(args.ofile, medmap)
# Given two maps, estimate the noise level their average would have.
import numpy as np, argparse
from enlib import enmap
parser = argparse.ArgumentParser()
parser.add_argument("ifiles", nargs=2)
parser.add_argument("ofile")
parser.add_argument("-b", "--binsize", type=int, default=3)
parser.add_argument("-s", "--smooth", type=float, default=30)
parser.add_argument("--div", type=float, default=1.0)
args = parser.parse_args()
b = args.binsize
smooth = args.smooth * np.pi/180/60/(8*np.log(2))
m = [enmap.read_map(f) for f in args.ifiles]
dm = (m[1]-m[0])/2
pixarea = dm.area()/np.product(dm.shape[-2:])*(180*60/np.pi)**2
# Compute standard deviation in bins
dm = dm[...,:dm.shape[-2]/b*b,:dm.shape[-1]/b*b]
dm_blocks = dm.reshape(dm.shape[:-2]+(dm.shape[-2]/b,b,dm.shape[-1]/b,b))
var = np.std(dm_blocks,axis=(-3,-1))**2*pixarea/args.div
# This reshaping stuff messes up the wcs, which doesn't notice
# that we now have bigger pixels. So correct that.
var = enmap.samewcs(var, dm[...,::b,::b])
typ = np.median(var[var!=0])
var[~np.isfinite(var)] = 0
var = np.minimum(var,typ*1e6)
svar = enmap.smooth_gauss(var, smooth)
sigma = svar**0.5
def read_map(fname):
m = enmap.read_map(fname)
m = m[...,1:-1,1:-1]
#m = eval("m" + args.slice)
m = enmap.downgrade(m, args.downgrade)
return m
# Project a single pickup map into horizontal coordinates
import numpy as np, os
from enlib import enmap, utils, config, array_ops
from enact import actdata, filedb
parser = config.ArgumentParser(os.environ["HOME"] + "/.enkirc")
parser.add_argument("pickup_map")
parser.add_argument("template")
parser.add_argument("sel_repr")
parser.add_argument("el", type=float)
parser.add_argument("ofile")
args = parser.parse_args()
filedb.init()
nrow, ncol = 33, 32
# Read our template, which represents the output horizontal coordinates
template = enmap.read_map(args.template)
# Use our representative selector to get focalplane offsets and polangles
entry = filedb.data[filedb.scans[args.sel_repr][0]]
d = actdata.read(entry, ["boresight", "point_offsets", "polangle"])
d.boresight[2] = args.el # In degrees, calibrated in next step
d = actdata.calibrate(d, exclude=["autocut"])
def reorder(map, nrow, ncol, dets):
return enmap.samewcs(map[utils.transpose_inds(dets, nrow, ncol)], map)
# Read our map, and give each row a weight
pickup = enmap.read_map(args.pickup_map)
pickup = reorder(pickup, nrow, ncol, d.dets)
weight = np.median((pickup[:, 1:] - pickup[:, :-1])**2, -1)
parser.add_argument("-b", "--bsize", type=float, default=1)
parser.add_argument("-v", "--verbosity", type=int, default=1)
parser.add_argument("-d", "--dets", type=str, default=None)
args = parser.parse_args()
comm = mpi.COMM_WORLD
level = log.verbosity2level(args.verbosity)
L = log.init(level=level, rank=comm.rank)
dtype = np.float32 if config.get("map_bits") == 32 else np.float64
root = args.odir + "/" + (args.prefix + "_" if args.prefix else "")
ncomp = 3
nbin = config.get("nmat_uncorr_nbin")
filedb.init()
utils.mkdir(args.odir)
area = enmap.read_map(args.area)
area = enmap.zeros((ncomp, ) + area.shape[-2:], area.wcs, dtype)
# First loop through all the tods and find out which are in which scanning
# pattern.
L.info("Scanning tods")
mypatids = {}
ids = filedb.scans[args.sel]
myinds = range(comm.rank, len(ids), comm.size)
ndet_array = 0
for ind, d in scanutils.scan_iterator(ids, myinds, actscan.ACTScan,
filedb.data):
id = ids[ind]
# Determine which scanning pattern we have
el = np.mean(d.boresight[:, 2])
az1, az2 = utils.minmax(d.boresight[:, 1])
def read_helper(fname, shape=None, wcs=None):
if shape is None: return enmap.read_map(fname)
mshape, mwcs = enmap.read_map_geometry(fname)
pixbox = enmap.pixbox_of(mwcs, shape, wcs)
return enmap.read_map(fname, pixbox=pixbox)
import numpy as np, argparse, os, sys
from enlib import enmap, pmat, config
from enact import filedb, data
parser = config.ArgumentParser(os.environ["HOME"] + "/.enkirc")
parser.add_argument("id")
parser.add_argument("area")
parser.add_argument("--di", type=int, default=0, help="Index into array of accepted detectors to use.")
args = parser.parse_args()
dtype = np.float64
eqsys = config.get("map_eqsys")
area = enmap.read_map(args.area).astype(dtype)
area = enmap.zeros((3,)+area.shape[-2:], area.wcs, dtype)
entry = filedb.data[args.id]
# First get the raw samples
d = data.read(entry, subdets=[args.di])
raw_tod = d.tod[0,d.sample_offset:d.cutafter].copy()
raw_bore = d.boresight[:,d.sample_offset:d.cutafter].T
# Then some calibrated samples
d = data.calibrate(d)
cal_tod = d.tod[0]
cal_bore = d.boresight.T
# Apply fourier-truncation to raw data
raw_tod = raw_tod[:cal_tod.shape[0]]
raw_bore = raw_bore[:cal_bore.shape[0]]
# And a proper ACTScan
scan = data.ACTScan(entry, subdets=[args.di])
# Detector pointing
debug_tile = None if args.debug_tile is None else [int(w) for w in args.debug_tile.split(",")]
# Get the set of bounding boxes, after normalizing them
boxes = np.sort(np.array([d.box for d in mapinfo.datasets]),-2)
# Read the cmb power spectrum, which is an effective noise
# component. T-only
cl_path = os.path.join(os.path.dirname(args.config),config.cl_background)
cl_bg = powspec.read_spectrum(cl_path)[0,0]
# Read our mask. High-resolution and per-dataset masks would be handled otherwise.
# This approach only works for low-resolution masks
if config.mask_mode == "none": mask = None
elif config.mask_mode == "lowres":
mask_path = os.path.join(os.path.dirname(args.config), config.mask)
mask = enmap.read_map(mask_path).preflat[0]
else: raise ValueError("Unrecognized mask mode '%s'" % config.mask_mode)
def overlaps_any(box, refboxes):
box = np.sort(box, 0)
rdec, rra = utils.moveaxis(refboxes - box[0,:], 2,0)
wdec, wra = box[1] - box[0]
rra -= np.floor(rra[:,0,None]/(2*np.pi)+0.5)*(2*np.pi)
for i in range(-1,2):
nra = rra + i*(2*np.pi)
if np.any((nra[:,1]>0)&(nra[:,0]<wra)&(rdec[:,1]>0)&(rdec[:,0]<wdec)): return True
return False
def parse_bounds(bstr):
res = []
toks = bstr.strip().split(",")
import orphics from enlib import enmap from astropy.io import fits import numpy as np from orphics import maps import orphics.io as io from orphics.io import Plotter as pl import matplotlib import matplotlib.pyplot as plt map90location = '/Users/Teva/maps and catalog data/f090_daynight_all_map_mono_deep56.fits' map150location = '/Users/Teva/maps and catalog data/f150_daynight_all_map_mono_deep56.fits' map217location = '/Users/Teva/maps and catalog data/HFI_SkyMap_217_2048_R2.02_full_cutout_h0.fits' map353location = '/Users/Teva/maps and catalog data/HFI_SkyMap_353_2048_R2.02_full_cutout_h0.fits' lmap90 = enmap.read_map(map90location) lmap150 = enmap.read_map(map150location) lmap217 = enmap.read_map(map217location) lmap353 = enmap.read_map(map353location) tmap90 = lmap90[0] tmap150 = lmap150[0] tmap217 = lmap217[0] tmap353 = lmap353[0] Ny, Nx = tmap90.shape #they have the same shape so it doesn't matter if we use the 90 or 150 map cat_location = '../maps and catalog data/act_confirmed_clusters.fits' #cat_location='../maps and catalog data/act_candidate_clusters.fits' #cat_location='../maps and catalog data/redmapper_dr8_public_v6.3_catalog.fits' out_dir = "./" hdulist = fits.open(cat_location)
ainfo = sharp.alm_info(lmax)
sht = sharp.sht(minfo, ainfo)
alm = np.zeros((ncomp,ainfo.nelem), dtype=ctype)
# Perform the actual transform
L.debug("T -> alm")
print m.dtype, alm.dtype
sht.map2alm(m[0], alm[0])
if ncomp == 3:
L.debug("P -> alm")
sht.map2alm(m[1:3],alm[1:3], spin=2)
del m
# Project down on each template
for tfile in args.templates:
L.info("Reading " + tfile)
shape, wcs = enmap.read_map(tfile).geometry
if args.rot and args.rot_method != "alm":
# Rotate by displacing coordinates and then fixing the polarization
L.debug("Computing pixel positions")
pmap = enmap.posmap(shape, wcs)
if args.rot:
L.debug("Computing rotated positions")
s1,s2 = args.rot.split(",")
opos = coordinates.transform(s2, s1, pmap[::-1], pol=ncomp==3)
pmap[...] = opos[1::-1]
if len(opos) == 3: psi = -opos[2].copy()
del opos
L.debug("Projecting")
res = curvedsky.alm2map_pos(alm, pmap)
if args.rot and ncomp==3:
dtype = np.float32
ncol, nrow = 32, 33
nfile = len(args.ifiles)/2
rad = args.rad * utils.arcmin * utils.fwhm
if not args.transpose:
imapfiles = args.ifiles[:nfile]
ilayfiles = args.ifiles[nfile:]
else:
imapfiles = args.ifiles[0::2]
ilayfiles = args.ifiles[1::2]
# Read in our focalplane layouts so we can define our output map bounds
dets, offs, boxes, imaps = [], [], [], []
for i in range(nfile):
det, off = files.read_point_template(ilayfiles[i])
imap = enmap.read_map(imapfiles[i])
if args.slice: imap = eval("imap"+args.slice)
# We want y,x-ordering
off = off[:,::-1]
box = utils.minmax(off,0)
dets.append(det)
offs.append(off)
boxes.append(box)
imaps.append(imap)
box = utils.bounding_box(boxes)
box = utils.widen_box(box, rad*5, relative=False)
# We assume that the two maps have the same pixelization
imaps = enmap.samewcs(np.array(imaps), imaps[0])
# Downsample by averaging
imaps = enmap.downgrade(imaps, (1,args.step))
parser.add_argument("ofile")
parser.add_argument("-s", "--sigma", type=float, default=50)
parser.add_argument("-n", "--nsamp", type=int, default=4)
args = parser.parse_args()
def sim_scan_fast(imap, sigma, rad, n):
mask = np.zeros(imap.shape[-2:])
mask[rad:-rad,rad:-rad] = 1
w = np.maximum(1 - ndimage.distance_transform_edt(1-mask)/rad,0)**3
w = w[None,:,:]*np.array([1,0.5,0.5])[:,None,None]*sigma**-2*n
w = enmap.samewcs(w, imap)
m = imap + np.random.standard_normal(imap.shape)*w**-0.5
m[~np.isfinite(m)] = 0
return m, w
template = enmap.read_map(args.template)[:,::2,::2]
ps = powspec.read_spectrum(args.powspec, expand="diag")
sim = enmap.rand_map(template.shape, template.wcs, ps)
# Simulate noisy data
m, w = sim_scan_fast(sim, args.sigma, 128, 1)
ncomp = m.shape[0]
iN = enmap.zeros((ncomp,ncomp)+w.shape[-2:])
for i in range(ncomp): iN[i,i] = w[i]
iS = enmap.spec2flat(m.shape, m.wcs, ps, -1)
# Draw samples
sampler = gibbs.FieldSampler(iS, iN, m)
res = enmap.zeros((1+args.nsamp,)+m.shape,m.wcs)
res[0] = m
def load_fullsky(sim_root, prefix, k):
uiqu = enmap.read_map(sim_root + prefix + "_" + str(k).zfill(2) + ".fits")
return uiqu
import numpy as np, argparse
from enlib import enmap, utils
parser = argparse.ArgumentParser()
parser.add_argument("imasks", nargs="+")
args = parser.parse_args()
maps = [enmap.read_map(fname) for fname in args.imasks]
nrow, ncol = 33, 32
for r in range(nrow):
for c in range(ncol):
uid = c + ncol*r
i = r + nrow*c
# Loop through each map for this detector
az_ranges = []
for mi, map in enumerate(maps):
pix_ranges = utils.mask2range(map[i]>0)
if len(pix_ranges) == 0: continue
az_ranges.append(map.pix2sky([pix_ranges*0,pix_ranges])[1]/utils.degree)
if len(az_ranges) == 0: az_ranges = np.zeros((1,0,2))
az_ranges = np.concatenate(az_ranges,0)
order = np.argsort(az_ranges[:,0])
az_ranges = az_ranges[order]
# And output our cut
cut = "%3d " % uid + " ".join(["%.2f:%.2f" % tuple(a) for a in az_ranges])
print cut
parser.add_argument("area")
parser.add_argument("sel")
parser.add_argument("odir")
parser.add_argument("tag", nargs="?")
parser.add_argument("-R", "--dist", type=float, default=0.2)
parser.add_argument("-e", "--equator", action="store_true")
args = parser.parse_args()
comm = mpi.COMM_WORLD
filedb.init()
ids = filedb.scans[args.sel]
R = args.dist * utils.degree
csize= 100
dtype= np.float32
area = enmap.read_map(args.area).astype(dtype)
ncomp= 3
shape= area.shape[-2:]
model_fknee = 10
model_alpha = 10
sys = "hor:"+args.planet
if args.equator: sys += "/0_0"
utils.mkdir(args.odir)
prefix = args.odir + "/"
if args.tag: prefix += args.tag + "_"
for ind in range(comm.rank, len(ids), comm.size):
id = ids[ind]
bid = id.replace(":","_")
entry = filedb.data[id]
# Read the tod as usual
def map_to_blocks(map, n): m = map[...,:map.shape[-2]/n*n,:map.shape[-1]/n*n] m = m.reshape(m.shape[:-2]+(m.shape[-2]/n,n,m.shape[-1]/n,n)) return m def calc_map_block_ivar(map, n): m = map_to_blocks(map, n) vmap = np.var(m, axis=(-3,-1)) vmap[vmap!=0] = 1/vmap[vmap!=0] return enmap.samewcs(vmap, map[...,::n,::n]) def calc_map_block_mean(map, n): m = map_to_blocks(map, n) return enmap.samewcs(np.mean(m, axis=(-3,-1)), map[...,::n,::n]) print "Reading map %s" % (mapfiles[0]) map = enmap.read_map(mapfiles[0]).preflat[args.component] print "Reading hit %s" % (hitfiles[0]) hit = enmap.read_map(hitfiles[0]) # We assume that the hitcount maps are 2d def get_bias(bs): # Get bias factor we get from estimating the ratio # via medmean. Determined empirically based on white noise # simulations. We will also have biases from not actually having # white noise, but those will be map-dependent. biases = [0, 0, 0.74, 0.89, 0.94, 0.96, 0.97, 0.98, 0.99, 0.99, 0.99] return 1.0 if bs >= len(biases) else biases[bs] def quant(a, q): return np.percentile(a, q*100) qlim = 0.95 print "Measuring sensitivities"
from orphics.io import Plotter as pl import matplotlib import matplotlib.pyplot as plt import szar from szar import counts import scipy import healpy as hp map90location = '/Users/Teva/maps and catalog data/f090_night_all2_map_mono.fits' map150location = '/Users/Teva/maps and catalog data/f150_night_all2_map_mono.fits' # map217location='/Users/Teva/maps and catalog data/HFI_SkyMap_217_2048_R2.02_full.fits' # map353location='/Users/Teva/maps and catalog data/HFI_SkyMap_353_2048_R2.02_full.fits' # map545location='/Users/Teva/maps and catalog data/HFI_SkyMap_545_2048_R2.02_full.fits' # map857location='/Users/Teva/maps and catalog data/HFI_SkyMap_857_2048_R2.02_full.fits' lmap90 = enmap.read_map(map90location) lmap150 = enmap.read_map(map150location) # lmap217=hp.read_map(map217location) # lmap353=hp.read_map(map353location) # lmap545=hp.read_map(map545location) # lmap857=hp.read_map(map857location) tmap90 = lmap90[0] tmap150 = lmap150[0] # tmap217=lmap217[0] # tmap353=lmap353[0] # tmap545=lmap545[0] # tmap857=lmap857[0] Ny, Nx = tmap90.shape #they have the same shape so it doesn't matter if we use the 90 or 150 map #cat_location='../maps and catalog data/act_confirmed_clusters.fits'
import numpy as np, argparse
from enlib import enmap
from scipy import ndimage
parser = argparse.ArgumentParser()
parser.add_argument("pos")
parser.add_argument("template")
parser.add_argument("ofile")
parser.add_argument("-r", "--radius", type=float, default=3)
parser.add_argument("-c", "--columns", type=str, default="3,5,2")
parser.add_argument("-t", "--threshold", type=float, default=0)
args = parser.parse_args()
cols = [int(w) for w in args.columns.split(",")]
srcinfo = np.loadtxt(args.pos)[:,cols]
pos = srcinfo[np.abs(srcinfo[:,2])>=args.threshold][:,:2] * np.pi/180
map = enmap.read_map(args.template)
pix = map.sky2pix(pos.T).T.astype(int)
pixrad = (map.area()/map.npix)**0.5
mrad = args.radius*np.pi/180/60/pixrad
mask = enmap.zeros(map.shape[-2:], map.wcs)+1
mask[pix[:,0],pix[:,1]] = 0
mask = enmap.enmap(1.0*(ndimage.distance_transform_edt(mask) > mrad), map.wcs)
enmap.write_map(args.ofile, mask)
hdeg=args.patch_height,
yoffset=args.yoffset,
mpi_comm=MPI.COMM_WORLD,
nsims=args.Nsims,
lmax=args.lmax,
pix_intermediate=args.pix_inter,
bin_lmax=args.bin_lmax)
cmb = {}
ikappa = {}
mlist = ['e', 's', 'r']
mf = {}
for m in mlist:
if args.meanfield is not None:
mf[m] = enmap.read_map(args.meanfield + "/meanfield_" + m + ".hdf")
else:
mf[m] = 0.
for k, index in enumerate(pipe.tasks):
cmb['s'], cmb['e'], ikappa['s'], ikappa['e'] = pipe.make_sim(index)
cmb['r'] = pipe.rotator.rotate(cmb['s'])
ikappa['r'] = pipe.rotator.rotate(ikappa['s'],
order=5,
mode="constant",
cval=0.0,
prefilter=True,
mask_nan=True,
safe=True)
# Point source handling:
if src_handling is not None and src_handling["mode"] in ["inpaint","full"]:
white_src_handler = mapmaking.SourceHandler(myscans, comm, dtype=dtype, **src_handling)
else: white_src_handler = None
# 1. Initialize signals
L.info("Initializing signals")
signals = []
for param in signal_params:
effname = get_effname(param)
active_scans = apply_scan_limits(myscans, param)
if param["type"] == "cut":
signal = mapmaking.SignalCut(active_scans, dtype=dtype, comm=comm, name=effname, ofmt=param["ofmt"], output=param["output"]=="yes")
signal_cut = signal
elif param["type"] == "map":
area = enmap.read_map(param["value"])
area = enmap.zeros((args.ncomp,)+area.shape[-2:], area.wcs, dtype)
signal = mapmaking.SignalMap(active_scans, area, comm=comm, name=effname, ofmt=param["ofmt"], output=param["output"]=="yes", sys=param["sys"], extra=setup_extra_transforms(param))
elif param["type"] == "fmap":
area = enmap.read_map(param["value"])
area = enmap.zeros((args.ncomp,)+area.shape[-2:], area.wcs, dtype)
signal = mapmaking.SignalMapFast(active_scans, area, comm=comm, name=effname, ofmt=param["ofmt"], output=param["output"]=="yes", sys=param["sys"], extra=setup_extra_transforms(param))
elif param["type"] == "dmap":
area = dmap.read_map(param["value"], bbox=mybbox, tshape=tshape, comm=comm)
area = dmap.zeros(area.geometry.aspre(args.ncomp).astype(dtype))
signal = mapmaking.SignalDmap(active_scans, mysubs, area, name=effname, ofmt=param["ofmt"], output=param["output"]=="yes", sys=param["sys"], extra=setup_extra_transforms(param))
elif param["type"] == "fdmap":
area = dmap.read_map(param["value"], bbox=mybbox, tshape=tshape, comm=comm)
area = dmap.zeros(area.geometry.aspre(args.ncomp).astype(dtype))
signal = mapmaking.SignalDmapFast(active_scans, mysubs, area, name=effname, ofmt=param["ofmt"], output=param["output"]=="yes", sys=param["sys"], extra=setup_extra_transforms(param))
elif param["type"] == "bmap":
def combine_tiles(ipathfmt,
opathfmt,
combine=2,
downsample=2,
itile1=(None, None),
itile2=(None, None),
tyflip=False,
txflip=False,
pad_to=None,
comm=None,
verbose=False):
"""Given a set of tiles on disk at locaiton ipathfmt % {"y":...,"x"...},
combine them into larger tiles, downsample and write the result to
opathfmt % {"y":...,"x":...}. x and y must be contiguous and start at 0.
reftile[2] indicates the tile coordinates of the first valid input tile.
This needs to be specified if not all tiles of the logical tiling are
physically present.
tyflip and txflip indicate if the tiles coordinate system is reversed
relative to the pixel coordinates or not."
"""
# Expand combine and downsample to 2d
combine = np.zeros(2, int) + combine
downsample = np.zeros(2, int) + downsample
if pad_to is not None:
pad_to = np.zeros(2, int) + pad_to
# Handle optional mpi
rank, size = (comm.rank, comm.size) if comm is not None else (0, 1)
# Find the range of input tiles
itile1, itile2 = find_tile_range(ipathfmt, itile1, itile2)
# Read the first tile to get its size information
ibase = enmap.read_map(ipathfmt % {"y": itile1[0], "x": itile1[1]}) * 0
# Find the set of output tiles we need to consider
otile1 = itile1 / combine
otile2 = (itile2 - 1) / combine + 1
# And loop over them
oyx = [(oy, ox) for oy in range(otile1[0], otile2[0])
for ox in range(otile1[1], otile2[1])]
for i in range(rank, len(oyx), size):
oy, ox = oyx[i]
# Read in all associated tiles into a list of lists
rows = []
for dy in range(combine[0]):
iy = oy * combine[0] + dy
if iy >= itile2[0]: continue
cols = []
for dx in range(combine[1]):
ix = ox * combine[1] + dx
if ix >= itile2[1]: continue
if iy < itile1[0] or ix < itile1[1]:
# The first tiles are missing on disk, but are
# logically a part of the tiling. Use ibase,
# which has been zeroed out.
cols.append(ibase)
else:
itname = ipathfmt % {"y": iy, "x": ix}
cols.append(enmap.read_map(itname))
if txflip: cols = cols[::-1]
rows.append(cols)
# Stack them next to each other into a big tile
if tyflip: rows = rows[::-1]
omap = enmap.tile_maps(rows)
# Downgrade if necessary
if np.any(downsample > 1):
omap = enmap.downgrade(omap, downsample)
if pad_to is not None:
# Padding happens towards the end of the tiling,
# which depends on the flip status
padding = np.array(
[[0, 0],
[pad_to[0] - omap.shape[-2], pad_to[1] - omap.shape[-1]]])
if tyflip: padding[:, 0] = padding[::-1, 0]
if txflip: padding[:, 1] = padding[::-1, 1]
omap = enmap.pad(omap, padding)
# And output
otname = opathfmt % {"y": oy, "x": ox}
utils.mkdir(os.path.dirname(otname))
enmap.write_map(otname, omap)
if verbose: print otname
# Project a single pickup map into horizontal coordinates
import numpy as np, os
from enlib import enmap, utils, config, array_ops
from enact import actdata, filedb
parser = config.ArgumentParser(os.environ["HOME"] + "/.enkirc")
parser.add_argument("pickup_map")
parser.add_argument("template")
parser.add_argument("sel_repr")
parser.add_argument("el", type=float)
parser.add_argument("ofile")
args = parser.parse_args()
filedb.init()
nrow, ncol = 33, 32
# Read our template, which represents the output horizontal coordinates
template = enmap.read_map(args.template)
# Use our representative selector to get focalplane offsets and polangles
entry = filedb.data[filedb.scans[args.sel_repr][0]]
d = actdata.read(entry, ["boresight", "point_offsets", "polangle"])
d.boresight[2] = args.el # In degrees, calibrated in next step
d = actdata.calibrate(d, exclude=["autocut"])
def reorder(map, nrow, ncol, dets):
return enmap.samewcs(map[utils.transpose_inds(dets,nrow,ncol)],map)
# Read our map, and give each row a weight
pickup = enmap.read_map(args.pickup_map)
pickup = reorder(pickup, nrow, ncol, d.dets)
weight = np.median((pickup[:,1:]-pickup[:,:-1])**2,-1)
weight[weight>0] = 1/weight[weight>0]