def wrapper(off, full=False): t1 = time.time() off = self.unzip(off) chisq, amps, aicov = fun(off) t2 = time.time() if self.thumb_mapper and thumb_path and thumb_interval and self.i % thumb_interval == 0: thumbs = self.make_thumbs(off, amps) enmap.write_map(thumb_path % self.i, thumbs) if self.chisq0 is None: self.chisq0 = chisq if verbose: doff = off/utils.arcmin msg = "%4d %8.5f %8.5f" % (self.i,doff[0],doff[1]) msg += " %12.5e %7.3f" % (self.chisq0-chisq, t2-t1) famps = amps.reshape(-1,amps.shape[-1]) faicov = aicov.reshape((-1,)+aicov.shape[-2:]) for i in range(len(famps)): nsigma = np.sum(amatmul(faicov[i],famps[i])*famps[i])**0.5 msg += " %10.3f %8.1f" % (famps[i,0]/self.amp_unit, nsigma) print(msg) self.samples.offs.append(off) self.samples.amps.append(amps) self.samples.aicovs.append(aicov) self.samples.chisqs.append(chisq) self.i += 1 if not full: return chisq else: return chisq, amps, aicov
def solve(self, maxiter=100, cg_tol=1e-7, verbose=False, dump_dir=None):
if np.sum(self.highres_mask) == 0: return None
solver = cg.CG(self.A, self.rhs.reshape(-1), M=self.M)
for i in range(maxiter):
t1 = time.time()
solver.step()
t2 = time.time()
if verbose:
print "%5d %15.7e %5.2f" % (solver.i, solver.err, t2 - t1)
if dump_dir is not None and solver.i in [1, 2, 5, 10, 20, 50
] + range(
100, 10000, 100):
m = enmap.ndmap(solver.x.reshape(self.shape), self.wcs)
enmap.write_map(dump_dir + "/step%04d.fits" % solver.i, m)
if solver.err < cg_tol:
if dump_dir is not None:
m = enmap.ndmap(solver.x.reshape(self.shape), self.wcs)
enmap.write_map(dump_dir + "/step_final.fits", m)
break
tot_map = self.highres_mask * solver.x.reshape(self.shape)
tot_div = self.highres_mask * self.tot_div
# Get rid of the fourier padding
ny, nx = tot_map.shape[-2:]
tot_map = tot_map[..., :ny - self.ffpad[0], :nx - self.ffpad[1]]
tot_div = tot_div[..., :ny - self.ffpad[0], :nx - self.ffpad[1]]
return bunch.Bunch(map=tot_map, div=tot_div)
def monolithic(idir, ofile, verbose=True, slice=None, dtype=None):
# Find the range of input tiles
ipathfmt = idir + "/tile%(y)03d_%(x)03d.fits"
itile1, itile2 = find_tile_range(ipathfmt)
def read(fname):
m = enmap.read_map(fname)
if slice: m = eval("m" + slice)
return m
# Read the first and last tile to get the total dimensions
m1 = read(ipathfmt % {"y": itile1[0], "x": itile1[1]})
m2 = read(ipathfmt % {"y": itile2[0] - 1, "x": itile2[1] - 1})
wy, wx = m1.shape[-2:]
oshape = tuple(
np.array(m1.shape[-2:]) * (itile2 - itile1 - 1) +
np.array(m2.shape[-2:]))
if dtype is None: dtype = m1.dtype
omap = enmap.zeros(m1.shape[:-2] + oshape, m1.wcs, dtype)
del m1, m2
# Now loop through all tiles and copy them in to the correct position
for ty in range(itile1[0], itile2[0]):
for tx in range(itile1[1], itile2[1]):
m = read(ipathfmt % {"y": ty, "x": tx})
oy = ty - itile1[0]
ox = tx - itile1[1]
omap[..., oy * wy:(oy + 1) * wy, ox * wx:(ox + 1) * wx] = m
if verbose: print ipathfmt % {"y": ty, "x": tx}
enmap.write_map(ofile, omap)
def coadd_maps(imaps, ihits, omap, ohit, cont=False): # The first map will be used as a reference. All subsequent maps # must fit in its boundaries. if cont and os.path.exists(omap): return if args.verbose: print "Reading %s" % imaps[0] m = read_map(imaps[0]) if args.verbose: print"Reading %s" % ihits[0] w = apply_edge(apply_apod(apply_trim(read_div(ihits[0])))) wm = mul(w,m) for mif,wif in zip(imaps[1:],ihits[1:]): if args.verbose: print"Reading %s" % mif try: mi = read_map(mif) except IOError: if args.allow_missing: print "Can't read %s. Skipping" % mif continue else: raise if args.verbose: print"Reading %s" % wif wi = apply_edge(apply_apod(apply_trim(read_div(wif)))) # We may need to reproject maps if mi.shape != m.shape or str(mi.wcs.to_header()) != str(m.wcs.to_header()): mi = enmap.project(mi, m.shape, m.wcs, mode="constant") wi = enmap.project(wi, w.shape, w.wcs, mode="constant") w = add(w,wi) wm = add(wm,mul(wi,mi)) if args.verbose: print"Solving" m = solve(w,wm) if args.verbose: print"Writing %s" % omap enmap.write_map(omap, m) if args.verbose: print"Writing %s" % ohit enmap.write_map(ohit, w)
def coadd_maps(imaps, ihits, omap, ohit, cont=False): # The first map will be used as a reference. All subsequent maps # must fit in its boundaries. if cont and os.path.exists(omap): return if args.verbose: print "Reading %s" % imaps[0] m = read_map(imaps[0]) if args.verbose: print"Reading %s" % ihits[0] w = apply_edge(apply_apod(apply_trim(read_div(ihits[0])))) wm = mul(w,m) for mif,wif in zip(imaps[1:],ihits[1:]): if args.verbose: print"Reading %s" % mif try: mi = read_map(mif) except IOError: if args.allow_missing: print "Can't read %s. Skipping" % mif continue else: raise if args.verbose: print"Reading %s" % wif wi = apply_edge(apply_apod(apply_trim(read_div(wif)))) # We may need to reproject maps if mi.shape != m.shape or str(mi.wcs.to_header()) != str(m.wcs.to_header()): mi = enmap.project(mi, m.shape, m.wcs, mode="constant") wi = enmap.project(wi, w.shape, w.wcs, mode="constant") w = add(w,wi) wm = add(wm,mul(wi,mi)) if args.verbose: print"Solving" m = solve(w,wm) if args.verbose: print"Writing %s" % omap enmap.write_map(omap, m) if args.verbose: print"Writing %s" % ohit enmap.write_map(ohit, w)
def wrapper(off, full=False): t1 = time.time() off = self.unzip(off) chisq, amps, aicov = fun(off) t2 = time.time() if self.thumb_mapper and thumb_path and thumb_interval and self.i % thumb_interval == 0: thumbs = self.make_thumbs(off, amps) enmap.write_map(thumb_path % self.i, thumbs) if self.chisq0 is None: self.chisq0 = chisq if verbose: doff = (off - self.off0)/utils.arcmin msg = "%4d %8.5f %8.5f" % (self.i,doff[0],doff[1]) msg += " %12.5e %7.3f" % (self.chisq0-chisq, t2-t1) famps, faicov = amps.reshape(-1), aicov.reshape(-1) for i in range(len(famps)): nsigma = (famps[i]**2*faicov[i])**0.5 msg += " %10.3f %8.1f" % (famps[i]/self.amp_unit, nsigma) print(msg) self.samples.offs.append(off) self.samples.amps.append(amps) self.samples.aicovs.append(aicov) self.samples.chisqs.append(chisq) self.i += 1 if not full: return chisq else: return chisq, amps, aicov
def dump_maps(ofile, tod, data, pos, amp, rad=args.radius*m2r, res=args.resolution*m2r): ncomp = amp.shape[1] dmaps, drhs, ddiv = make_maps(tod.astype(dtype), data, pos, ncomp, rad, res) dstack = stack_maps(drhs, ddiv, amp[:,0]) dmaps = np.concatenate((dmaps,[dstack]),0) dmaps = enmap.samewcs(dmaps, ddiv) enmap.write_map(ofile, dmaps[::-1])
def copy_mono(ifile, ofile, slice=None):
if args.cont and os.path.exists(ofile): return
if verbose: print("%3d copy_mono %s" % (comm.rank, ofile))
if args.dry: return
tfile = ofile + ".tmp"
map = read_map(ifile, slice=slice)
enmap.write_map(tfile, map)
shutil.move(tfile, ofile)
def copy_mono(ifile, ofile, slice=None): if args.cont and os.path.exists(ofile): return if verbose: print "%3d copy_mono %s" % (comm.rank, ofile) if args.dry: return tfile = ofile + ".tmp" map = read_map(ifile, slice=slice) enmap.write_map(tfile, map) shutil.move(tfile, ofile)
def add_maps(imaps, omap): if args.verbose: print "Reading %s" % imaps[0] m = nonan(enmap.read_map(imaps[0])) for mif in imaps[1:]: if args.verbose: print "Reading %s" % mif m += nonan(enmap.read_map(mif)) if args.verbose: "Writing %s" % omap enmap.write_map(omap, m)
def add_maps(imaps, omap): if args.verbose: print "Reading %s" % imaps[0] m = nonan(enmap.read_map(imaps[0]))*scales[0] for scale, mif in zip(scales[1:],imaps[1:]): if args.verbose: print "Reading %s" % mif m += nonan(enmap.read_map(mif))*scale if args.mean: m /= len(imaps) if args.verbose: "Writing %s" % omap enmap.write_map(omap, m)
def build_single(ifile, srcs, beam, ofile, mask_level=0, apod_size=16): imap = enmap.read_map(ifile) omap, oslice = pointsrcs.sim_srcs(imap.shape[-2:], imap.wcs, srcs, beam, return_padded=True) if mask_level: mask = omap > mask_level omap = 1-np.cos(np.minimum(1,ndimage.distance_transform_edt(1-mask)/16.0)*np.pi) omap = enmap.samewcs(omap, imap) omap = omap[oslice] enmap.write_map(ofile, omap)
def add_mono(ifiles, ofile, slice=None): if args.cont and os.path.exists(ofile): return if verbose: print "%3d add_mono %s" % (comm.rank, ofile) if args.dry: return tfile = ofile + ".tmp" omap = read_map(ifiles[0], slice=slice) for ifile in ifiles[1:]: omap += read_map(ifile, slice=slice) enmap.write_map(tfile, omap) shutil.move(tfile, ofile)
def read_map(fname, pbox, cname, write_cache=True, read_cache=True):
if os.path.isdir(fname):
fname = fname + "/tile%(y)03d_%(x)03d.fits"
if read_cache and os.path.isfile(cname):
map = enmap.read_map(cname).astype(dtype)
else:
map = retile.read_area(fname, pbox).astype(dtype)
if write_cache: enmap.write_map(cname, map)
#if map.ndim == 3: map = map[:1]
return map
def add_maps(imaps, omap): if args.verbose: print "Reading %s" % imaps[0] m = nonan(enmap.read_map(imaps[0]))*scales[0] for scale, mif in zip(scales[1:],imaps[1:]): if args.verbose: print "Reading %s" % mif m2 = nonan(enmap.read_map(mif, geometry=(m.shape,m.wcs)))*scale n = min(len(m.preflat),len(m2.preflat)) m.preflat[:n] += m2.preflat[:n] if args.mean: m /= len(imaps) if args.verbose: "Writing %s" % omap enmap.write_map(omap, m)
def write(self, prefix, tag, ms):
if not self.output: return
for pid, (pattern, m) in enumerate(zip(self.patterns, ms)):
oname = self.ofmt.format(name=self.name,
pid=pid,
el=pattern[0, 0] / utils.degree,
az0=pattern[0, 1] / utils.degree,
az1=pattern[1, 1] / utils.degree)
oname = "%s%s_%s.%s" % (prefix, oname, tag, self.ext)
if self.dof.comm.rank == 0:
enmap.write_map(oname, m)
def retile(ipathfmt, opathfmt, itile1=(None,None), itile2=(None,None),
otileoff=(0,0), otilenum=(None,None), ocorner=(-np.pi/2,-np.pi),
otilesize=(675,675), comm=None, verbose=False, slice=None):
"""Given a set of tiles on disk with locations ipathfmt % {"y":...,"x":...},
retile them into a new tiling and write the result to opathfmt % {"y":...,"x":...}.
The new tiling will have tile size given by otilesize[2]. Negative size means the
tiling will to down/left instead of up/right. The corner of the tiling will
be at sky coordinates ocorner[2] in radians. The new tiling will be pixel-
compatible with the input tiling - w.g. the wcs will only differ by crpix.
The output tiling will logically cover the whole sky, but only output tiles
that overlap with input tiles will actually be written. This can be modified
by using otileoff[2] and otilenum[2]. otileoff gives the tile indices of the
corner tile, while otilenum indicates the number of tiles to write."""
# Set up mpi
rank, size = (comm.rank, comm.size) if comm is not None else (0, 1)
# Expand any scalars
otilesize = np.zeros(2,int)+otilesize
otileoff = np.zeros(2,int)+otileoff
# Find the range of input tiles
itile1, itile2 = find_tile_range(ipathfmt, itile1, itile2)
# To fill in the rest of the information we need to know more
# about the input tiling, so read the first tile
ibase = enmap.read_map(ipathfmt % {"y":itile1[0],"x":itile1[1]})
if slice: ibase = eval("ibase"+slice)
itilesize = ibase.shape[-2:]
# Find the pixel position of our output corners according to the wcs.
# This is the last place we need to do a coordinate transformation.
# All the rest can be done in pure pixel logic.
pixoff = np.round(ibase.sky2pix(ocorner)).astype(int)
# Find the range of output tiles
def pix2otile(pix, ioff, osize): return (pix-ioff)/osize
otile1 = pix2otile(itile1*itilesize, pixoff, otilesize)
otile2 = pix2otile(itile2*itilesize-1, pixoff, otilesize)
otile1, otile2 = np.minimum(otile1,otile2), np.maximum(otile1,otile2)
otile2 += 1
# We can now loop over output tiles
cache = [None,None,None]
oyx = [(oy,ox) for oy in range(otile1[0],otile2[0]) for ox in range(otile1[1],otile2[1])]
for i in range(rank, len(oyx), size):
otile = np.array(oyx[i])
# Find out which input tiles overlap with this output tile.
# Our tile stretches from opix1:opix2 relative to the global input pixels
opix1 = otile*otilesize + pixoff
opix2 = (otile+1)*otilesize + pixoff
# output tiles and input tiles may increase in opposite directions
opix1, opix2 = np.minimum(opix1,opix2), np.maximum(opix1,opix2)
try: omap = read_area(ipathfmt, [opix1,opix2],itile1=itile1, itile2=itile2,cache=cache, slice=slice)
except IOError: continue
oname = opathfmt % {"y":otile[0]+otileoff[0],"x":otile[1]+otileoff[1]}
utils.mkdir(os.path.dirname(oname))
enmap.write_map(oname, omap)
if verbose: print oname
def dump_maps(ofile,
tod,
data,
pos,
amp,
rad=args.radius * m2r,
res=args.resolution * m2r):
ncomp = amp.shape[1]
dmaps, drhs, ddiv = make_maps(tod.astype(dtype), data, pos, ncomp, rad,
res)
dstack = stack_maps(drhs, ddiv, amp[:, 0])
dmaps = np.concatenate((dmaps, [dstack]), 0)
dmaps = enmap.samewcs(dmaps, ddiv)
enmap.write_map(ofile, dmaps[::-1])
def solve_cg(eq, nmax=1000, ofmt=None, dump_interval=10):
cg = CG(eq.A, eq.b, M=eq.M, dot=eq.dof.dot)
while cg.i < nmax:
with bench.mark("cg_step"):
cg.step()
dt = bench.stats["cg_step"]["time"].last
L.info("CG step %5d %15.7e %6.1f %6.3f" %
(cg.i, cg.err, dt, dt / len(eq.scans)))
xmap, xjunk = eq.dof.unzip(cg.x)
if ofmt and cg.i % dump_interval == 0 and myid == 0:
enmap.write_map(ofmt % cg.i, eq.dof.unzip(cg.x)[0])
# Output benchmarking information
bench.stats.write(benchfile)
return cg.x
def add_mono(ifiles, ofile, slice=None, factors=None):
if args.cont and os.path.exists(ofile): return
if verbose: print("%3d add_mono %s" % (comm.rank, ofile))
if args.dry: return
tfile = ofile + ".tmp"
if factors is None:
omap = read_map(ifiles[0], slice=slice)
for ifile in ifiles[1:]:
omap += read_map(ifile, slice=slice)
else:
omap = read_map(ifiles[0], slice=slice) * factors[0]
for i in range(1, len(ifiles)):
omap += read_map(ifiles[i], slice=slice) * factors[i]
enmap.write_map(tfile, omap)
shutil.move(tfile, ofile)
def build_single(ifile, srcs, beam, ofile, mask_level=0, apod_size=16):
imap = enmap.read_map(ifile)
omap, oslice = pointsrcs.sim_srcs(imap.shape[-2:],
imap.wcs,
srcs,
beam,
return_padded=True)
if mask_level:
mask = omap > mask_level
omap = 1 - np.cos(
np.minimum(1,
ndimage.distance_transform_edt(1 - mask) / 16.0) *
np.pi)
omap = enmap.samewcs(omap, imap)
omap = omap[oslice]
enmap.write_map(ofile, omap)
def read_map(fname,
pbox,
name=None,
cache_dir=None,
dtype=None,
read_cache=False):
if os.path.isdir(fname):
fname = fname + "/tile%(y)03d_%(x)03d.fits"
if read_cache:
map = enmap.read_map(cache_dir + "/" + name)
if dtype is not None: map = map.astype(dtype)
else:
map = retile.read_area(fname, pbox).astype(dtype)
if dtype is not None: map = map.astype(dtype)
if cache_dir is not None and name is not None:
enmap.write_map(cache_dir + "/" + name, map)
#if map.ndim == 3: map = map[:1]
return map
def write_map(name, map, ext="fits", merged=True):
if not merged:
# Write as individual tiles in directory of the specified name
utils.mkdir(name)
for pos, tile in zip(map.loc_pos, map.tiles):
enmap.write_map(
name + "/tile%03d_%03d.%s" % (tuple(pos) + (ext, )), tile)
else:
# Write to a single file. This currently creates the full map
# in memory while writing. It is unclear how to avoid this
# without bypassing pyfits or becoming super-slow.
if map.comm.rank == 0:
canvas = enmap.zeros(map.shape, map.wcs, map.dtype)
else:
canvas = None
dmap2enmap(map, canvas)
if map.comm.rank == 0:
enmap.write_map(name, canvas)
def coadd_maps(imaps, ihits, omap, ohit, cont=False, ncomp=-1):
# The first map will be used as a reference. All subsequent maps
# must fit in its boundaries.
if cont and os.path.exists(omap): return
if args.verbose: print("Reading %s" % imaps[0])
if ncomp < 0:
shape, wcs = enmap.read_map_geometry(imaps[0])
ncomp = 0 if len(shape) == 2 else shape[0]
m = read_map(imaps[0], ncomp=ncomp)
if args.verbose: print("Reading %s" % ihits[0])
w = apply_edge(apply_apod(apply_trim(read_div(ihits[0], ncomp=ncomp))))
if args.warn and np.any(w.preflat[0] < 0):
print("Negative weight in %s" % ihits[0])
wm = mul(w, m)
for i, (mif, wif) in enumerate(zip(imaps[1:], ihits[1:])):
if args.verbose: print("Reading %s" % mif)
try:
mi = read_map(mif, m.shape, m.wcs, ncomp=ncomp)
except (IOError, OSError):
if args.allow_missing:
print("Can't read %s. Skipping" % mif)
continue
else:
raise
if args.verbose: print("Reading %s" % wif)
wi = apply_edge(
apply_apod(apply_trim(read_div(wif, m.shape, m.wcs, ncomp=ncomp))))
if args.warn and np.any(wi.preflat[0] < 0):
print("Negative weight in %s" % ihits[i + 1])
## We may need to reproject maps
#if mi.shape != m.shape or str(mi.wcs.to_header()) != str(m.wcs.to_header()):
# mi = enmap.extract(mi, m.shape, m.wcs)
# wi = enmap.extract(wi, w.shape, w.wcs)
w = add(w, wi)
wm = add(wm, mul(wi, mi))
if args.verbose: print("Solving")
m = solve(w, wm)
if args.verbose: print("Writing %s" % omap)
enmap.write_map(omap, m)
if args.verbose: print("Writing %s" % ohit)
enmap.write_map(ohit, w)
def coadd_mono(imapfiles, idivfiles, omapfile, odivfile=None): if args.cont and os.path.exists(omapfile) and (odivfile is None or os.path.exists(odivfile)): return if verbose: print "%3d coadd_mono %s" % (comm.rank, omapfile) if args.dry: return omap, odiv = None, None tmapfile = omapfile + ".tmp" if odivfile: tdivfile = odivfile + ".tmp" for imapfile, idivfile in zip(imapfiles, idivfiles): imap = read_map(imapfile) idiv = read_map(idivfile, slice=".preflat[0]") if omap is None: omap, odiv = imap*0, idiv*0 omap += imap*idiv odiv += idiv mask = odiv > 0 omap[...,mask] /= odiv[mask] enmap.write_map(tmapfile, omap) if odivfile: enmap.write_map(tdivfile, odiv) shutil.move(tmapfile, omapfile) if odivfile: shutil.move(tdivfile, odivfile)
def wrapper(off): t1 = time.time() chisq, amps, aicov = fun(self.unzip(off)) t2 = time.time() if thumb_path and thumb_interval and self.i % thumb_interval == 0: tod2 = self.tod.copy() self.P.forward(tod2, amps, pmul=-1) thumbs = self.thumb_mapper.map(tod2) enmap.write_map(thumb_path % self.i, thumbs) del tod2, thumbs if self.chisq0 is None: self.chisq0 = chisq if verbose: msg = "%4d %6.3f %6.3f" % (self.i,off[0],off[1]) for i in range(len(amps)): nsigma = (amps[i,0]**2*aicov[i,0])**0.5 msg += " %7.3f %4.1f" % (amps[i,0]/self.amp_unit, nsigma) msg += " %12.5e %7.2f" % (self.chisq0-chisq, t2-t1) print(msg) self.i += 1 return chisq
def coadd_mono(imapfiles, idivfiles, omapfile, odivfile=None):
if args.cont and os.path.exists(omapfile) and (odivfile is None or
os.path.exists(odivfile)):
return
if verbose: print("%3d coadd_mono %s" % (comm.rank, omapfile))
if args.dry: return
omap, odiv = None, None
tmapfile = omapfile + ".tmp"
if odivfile: tdivfile = odivfile + ".tmp"
for imapfile, idivfile in zip(imapfiles, idivfiles):
imap = read_map(imapfile)
idiv = read_map(idivfile, slice=".preflat[0]")
if omap is None: omap, odiv = imap * 0, idiv * 0
omap += imap * idiv
odiv += idiv
mask = odiv > 0
omap[..., mask] /= odiv[mask]
enmap.write_map(tmapfile, omap)
if odivfile: enmap.write_map(tdivfile, odiv)
shutil.move(tmapfile, omapfile)
if odivfile: shutil.move(tdivfile, odivfile)
def wrapper(off):
t1 = time.time()
chisq, amps, aicov = fun(self.unzip(off))
t2 = time.time()
if thumb_path and thumb_interval and self.i % thumb_interval == 0:
tod2 = self.tod.copy()
self.P.forward(tod2, amps, pmul=-1)
thumbs = self.thumb_mapper.map(tod2)
enmap.write_map(thumb_path % self.i, thumbs)
del tod2, thumbs
if self.chisq0 is None: self.chisq0 = chisq
if verbose:
msg = "%4d %6.3f %6.3f" % (self.i, off[0], off[1])
for i in range(len(amps)):
nsigma = (amps[i, 0]**2 * aicov[i, 0])**0.5
msg += " %7.3f %4.1f" % (amps[i, 0] / self.amp_unit,
nsigma)
msg += " %12.5e %7.2f" % (self.chisq0 - chisq, t2 - t1)
print(msg)
self.i += 1
return chisq
def __call__(fself, x):
params = param_zipper.unzip(x)
chisqs, amps, adiv = self.calc_chisqs_amps_cached(
params.poss, params.taus)
#print amps[433], adiv[433]
chisq = np.sum(chisqs)
if verbosity > 0 and fself.i % 10 == 0:
if verbosity == 1:
print("%6d %12.4f" % (fself.i, chisq / self.ndata))
else:
print("%6d %12.4f" % (fself.i, chisq / self.ndata) +
" %6.3f" * len(x) % tuple(x))
sys.stdout.flush()
if False and fself.i % 10000 == 0:
model = self.model(params.poss, amps, params.taus)
map = enmap.enmap(
[self.rdata.tod, model, self.rdata.tod - model])
enmap.write_map(args.odir + "/foo%06d.fits" % fself.i,
map)
fself.i += 1
return chisq
def get_coadded_tile(mapinfo, box, obeam=None, ncomp=1, dump_dir=None, verbose=False): if not overlaps_any(box, boxes): return None mapset = mapinfo.read(box, pad=pad, dtype=dtype, verbose=verbose, ncomp=ncomp) if mapset is None: return None if all([d.insufficient for d in mapset.datasets]): return None jointmap.sanitize_maps(mapset) jointmap.build_noise_model(mapset) if len(mapset.datasets) == 0: return None if all([d.insufficient for d in mapset.datasets]): return None jointmap.setup_beams(mapset) jointmap.setup_target_beam(mapset, obeam) jointmap.setup_filter(mapset, mode=args.filter_mode) jointmap.setup_background_spectrum(mapset) mask = jointmap.get_mask_insufficient(mapset) coadder = jointmap.Coadder(mapset) rhs = coadder.calc_rhs() if dump_dir: enmap.write_map(dump_dir + "/rhs.fits", rhs) enmap.write_map(dump_dir + "/ps_rhs.fits", np.abs(enmap.fft(rhs.preflat[0]))**2) map = coadder.calc_coadd(rhs, dump_dir=dump_dir, verbose=verbose)#, maxiter=1) if dump_dir: enmap.write_map(dump_dir + "/ps_map.fits", np.abs(enmap.fft(mapdiag(map)))**2) div = coadder.tot_div #C = 1/mapset.datasets[0].iN res = bunch.Bunch(rhs=rhs*mask, map=map*mask, div=div*mask)#, C=C) #res = bunch.Bunch(rhs=rhs, map=map, div=div)#, C=C) return res
def get_coadded_tile(mapinfo, box, obeam=None, ncomp=1, dump_dir=None, verbose=False): if not overlaps_any(box, boxes): return None mapset = mapinfo.read(box, pad=pad, dtype=dtype, verbose=verbose, ncomp=ncomp) if mapset is None: return None if all([d.insufficient for d in mapset.datasets]): return None jointmap.sanitize_maps(mapset) jointmap.build_noise_model(mapset) if len(mapset.datasets) == 0: return None if all([d.insufficient for d in mapset.datasets]): return None jointmap.setup_beams(mapset) jointmap.setup_target_beam(mapset, obeam) jointmap.setup_filter(mapset, mode=args.filter_mode) jointmap.setup_background_spectrum(mapset) mask = jointmap.get_mask_insufficient(mapset) if args.wiener: coadder = jointmap.Wiener(mapset) else: coadder = jointmap.Coadder(mapset) rhs = coadder.calc_rhs() if dump_dir: enmap.write_map(dump_dir + "/rhs.fits", rhs) enmap.write_map(dump_dir + "/ps_rhs.fits", np.abs(enmap.fft(rhs.preflat[0]))**2) map = coadder.calc_map(rhs, dump_dir=dump_dir, verbose=verbose, cg_tol=args.cg_tol)#, maxiter=1) if dump_dir: enmap.write_map(dump_dir + "/ps_map.fits", np.abs(enmap.fft(mapdiag(map)))**2) div = coadder.tot_div #C = 1/mapset.datasets[0].iN res = bunch.Bunch(rhs=rhs*mask, map=map*mask, div=div*mask)#, C=C) #res = bunch.Bunch(rhs=rhs, map=map, div=div)#, C=C) return res
def combine_srcmaps(imapfiles,
isrcmapfiles,
idivfiles,
isrcdivfiles,
omapfile,
odivfile=None,
isrcfiles=None):
if args.cont and os.path.exists(omapfile) and (odivfile is None or
os.path.exists(odivfile)):
return
if verbose: print("%3d combine_srcmaps %s" % (comm.rank, omapfile))
if args.dry: return
omap, odiv = None, None
tmapfile = omapfile + ".tmp"
if odivfile: tdivfile = odivfile + ".tmp"
nfile = len(imapfiles)
for fi in range(nfile):
imap = read_map(imapfiles[fi])
isrcmap = read_map(isrcmapfiles[fi])
if isrcfiles is not None:
isrc = read_map(isrcfiles[fi])
imap += isrc
isrcmap += isrc
del isrc
idiv = read_map(idivfiles[fi], slice=".preflat[0]")
isrcdiv = read_map(isrcdivfiles[fi], slice=".preflat[0]")
idiv -= isrcdiv
if omap is None: omap, odiv = imap * 0, idiv * 0
omap += imap * idiv
omap += isrcmap * isrcdiv
odiv += idiv
odiv += isrcdiv
del imap, isrcmap, idiv, isrcdiv
mask = odiv > 0
omap[..., mask] /= odiv[mask]
enmap.write_map(tmapfile, omap)
if odivfile: enmap.write_map(tdivfile, odiv)
shutil.move(tmapfile, omapfile)
if odivfile: shutil.move(tdivfile, odivfile)
def get_coadded_tile(mapinfo,
box,
obeam=None,
ncomp=1,
dump_dir=None,
verbose=False):
if not overlaps_any(np.sort(box, 0), boxes): return None
mapset = mapinfo.read(box,
pad=pad,
dtype=dtype,
verbose=verbose,
ncomp=ncomp)
if mapset is None: return None
if all([d.insufficient for d in mapset.datasets]): return None
jointmap.sanitize_maps(mapset, detrend=args.detrend)
jointmap.build_noise_model(mapset)
if len(mapset.datasets) == 0: return None
if all([d.insufficient for d in mapset.datasets]): return None
jointmap.setup_beams(mapset)
jointmap.setup_target_beam(mapset, obeam)
jointmap.setup_filter(mapset, mode=args.filter_mode)
jointmap.setup_background_spectrum(mapset)
mask = jointmap.get_mask_insufficient(mapset)
if args.wiener: coadder = jointmap.Wiener(mapset)
else: coadder = jointmap.Coadder(mapset)
rhs = coadder.calc_rhs()
if dump_dir:
enmap.write_map(dump_dir + "/rhs.fits", rhs)
enmap.write_map(dump_dir + "/ps_rhs.fits",
np.abs(enmap.fft(rhs.preflat[0]))**2)
with open(dump_dir + "/names.txt", "w") as nfile:
for name in coadder.names:
nfile.write(name + "\n")
ls, weights = coadder.calc_debug_weights()
np.savetxt(
dump_dir + "/weights_1d.txt",
np.concatenate(
[ls[None], weights.reshape(-1, weights.shape[-1])], 0).T,
fmt="%15.7e")
ls, noisespecs = coadder.calc_debug_noise()
np.savetxt(
dump_dir + "/noisespecs_1d.txt",
np.concatenate(
[ls[None],
noisespecs.reshape(-1, noisespecs.shape[-1])], 0).T,
fmt="%15.7e")
map = coadder.calc_map(rhs,
dump_dir=dump_dir,
verbose=verbose,
cg_tol=args.cg_tol) #, maxiter=1)
if dump_dir:
enmap.write_map(dump_dir + "/ps_map.fits",
np.abs(enmap.fft(mapdiag(map)))**2)
div = coadder.tot_div
#C = 1/mapset.datasets[0].iN
res = bunch.Bunch(rhs=rhs * mask, map=map * mask, div=div * mask) #, C=C)
#res = bunch.Bunch(rhs=rhs, map=map, div=div)#, C=C)
return res
myscans, [],
noise=False,
hits=False)
else:
signal_map = mapmaking.SignalDmap(myscans, mysubs, imap, sys=args.sys)
precon = mapmaking.PreconDmapBinned(signal_map,
signal_cut,
myscans, [],
noise=False,
hits=False)
# We can now actually perform the postfilter
L.info("Postfiltering")
postfilter = mapmaking.PostPickup(myscans,
signal_map,
signal_cut,
precon,
daz=args.daz,
nt=args.nt,
weighted=args.weighted,
deslope=args.deslope)
omap = postfilter(imap)
# And output the resulting map
L.info("Writing")
if not is_dmap:
if comm.rank == 0:
enmap.write_map(args.omap, omap)
else:
omap.write(args.omap)
nscan = comm.allreduce(nscan)
det_hit = utils.allreduce(det_hit, comm)
offsets = utils.allreduce(offsets, comm)
offsets[det_hit > 0] /= det_hit[det_hit > 0][:, None]
# Reduce to our actual set of detectors
dets = np.where(det_hit > 0)[0]
offsets = offsets[dets]
det_hit = det_hit[dets]
ndet = len(dets)
# And output
if comm.rank == 0:
# Output our files
proot = root + "el_%s_az_%s_%s" % tuple([str(w) for w in pat])
enmap.write_map(proot + "_rhs.fits", rhs)
enmap.write_map(proot + "_hits.fits", hits)
with h5py.File(proot + "_info.hdf", "w") as hfile:
hfile["inspec"] = inspec
hfile["srate"] = srate
hfile["ids"] = np.array(pids)
hfile["hits"] = hits
hfile["pattern"] = pat
hfile["speed"] = speed
hfile["offsets"] = offsets / utils.degree # [det,{el,az}] in deg
for k, v in site.items():
hfile["site/" + k] = v
for k, v in hits.wcs.to_header().items():
hfile["wcs/" + k] = v
tot_rhs += rhs
tot_hits += hits
shape, wcs = jointmap.read_geometry(args.area)
tshape = np.array([args.tsize,args.tsize])
ntile = np.floor((shape[-2:]+tshape-1)/tshape).astype(int)
tyx = [(y,x) for y in range(ntile[0]-1,-1,-1) for x in range(ntile[1])]
for i in range(comm.rank, len(tyx), comm.size):
y, x = tyx[i]
ofile_map = args.odir + "/map_padtile/tile%(y)03d_%(x)03d.fits" % {"y":y,"x":x}
ofile_div = args.odir + "/div_padtile/tile%(y)03d_%(x)03d.fits" % {"y":y,"x":x}
utils.mkdir(os.path.dirname(ofile_map))
utils.mkdir(os.path.dirname(ofile_div))
if args.cont and os.path.isfile(ofile_map):
print "%3d skipping %3d %3d (already done)" % (comm.rank, y, x)
continue
print "%3d processing %3d %3d" % (comm.rank, y, x)
tpos = np.array(tyx[i])
pbox = np.array([tpos*tshape,np.minimum((tpos+1)*tshape,shape[-2:])])
box = enmap.pix2sky(shape, wcs, pbox.T).T
res = get_coadded_tile(mapinfo, box, obeam=obeam, ncomp=args.ncomp, verbose=args.verbose)
if res is None: res = jointmap.make_dummy_tile((args.ncomp,)+shape[-2:], wcs, box, pad=pad, dtype=dtype)
enmap.write_map(ofile_map, res.map)
enmap.write_map(ofile_div, res.div)
else:
# Single arbitrary tile
if not overlaps_any(bounds, boxes):
print "No data in selected region"
else:
res = get_coadded_tile(mapinfo, bounds, obeam=obeam, ncomp=args.ncomp, dump_dir=args.odir, verbose=args.verbose)
enmap.write_map(args.odir + "/map.fits", res.map)
enmap.write_map(args.odir + "/div.fits", res.div)
#enmap.write_map(args.odir + "/C.fits", res.C)
def write_padtile(ofile, map):
enmap.write_map(ofile, map, extra={"PAD": args.pad, "EDGE": args.apod_edge})
pmap = pmat.PmatMap(scan, area)
rhs = enmap.zeros((3,)+shape, wcs, dtype)
div = rhs*0
with bench.mark("rhs"):
pmap.backward(tod, rhs)
with bench.mark("div"):
pmap.backward(wtod,div)
div = div[0]
map = rhs.copy()
map[:,div>0] /= div[div>0]
map = map[0]
# Crop the outermost pixel, where outside hits will have accumulated
map, div, area = [m[...,1:-1,1:-1] for m in [map,div,area]]
# Find the local scanning velocity at the source position
scan_vel = find_scan_vel(scan, srcpos[:,sid], aspeed)
hbox = scan.box
dur = d.nsamp*d.srate
el = np.mean(scan.box[:,2])
az = scan.box[:,1]
sdata.append(bunch.Bunch(
map=map, div=div, srcpos=srcpos[:,sid], sid=sid,
vel=scan_vel, fknee=fknee, alpha=args.alpha,
id=id, ctime=tref, dur=dur, el=el, az=az,
site=d.site, off=d.point_correction))
del tod, wtod, d
write_sdata(oname, sdata)
if args.minimaps:
for i, sdat in enumerate(sdata):
enmap.write_map("%s/%s_srcmap_%03d.fits" % (args.odir, oid, sdat.sid), sdat.map)
L.info("Building source map")
dump_maps(tdir + "/map.hdf", d.tod, d, pos_fid, amp_fid)
if args.grid:
L.info("Building pos grid")
g = np.abs(args.grid)
p = params.copy()
if args.grid < 0:
p.strong[:] = False
grid = grid_pos(d, p, shape=(g, g))
grid -= np.max(grid)
maxpos = grid.pix2sky(np.unravel_index(np.argmax(grid), grid.shape))
print np.max(grid), maxpos * 180 * 60 / np.pi
if np.sum(maxpos**2)**0.5 <= pos_rel_max * 2 / 3:
params.pos_rel[...] = maxpos
enmap.write_map(tdir + "/grid.hdf", grid)
L.info("Drawing %d samples" % args.nsamp)
sampler = HybridSampler(d,
params,
dpos,
dbeam,
nstep=args.thin,
prior=prior,
dist=np.random.standard_cauchy)
pos_rel = np.zeros([args.nsamp, 2])
beam_rel = np.zeros([args.nsamp, 3])
amp = np.zeros((args.nsamp, ) + params.amp_fid.shape)
for i in range(-args.burnin, args.nsamp):
if i <= 0 and (-i) % 25 == 0: sampler.adjust()
from enlib import enmap
parser = argparse.ArgumentParser()
parser.add_argument("omap")
parser.add_argument("-D", "--diameter", type=float, default=30)
parser.add_argument("-n", "--npix", type=int, default=800)
parser.add_argument("--proj", type=str, default="cea")
args = parser.parse_args()
r = args.diameter * np.pi / 180 / 2
shape, wcs = enmap.geometry(pos=[[-r, -r], [r, r]],
shape=(args.npix, args.npix),
proj=args.proj)
def linpos(n, b):
return b[0] + (np.arange(n) + 0.5) * (b[1] - b[0]) / n
alpha = enmap.zeros((2, ) + shape, wcs)
pos = enmap.posmap(shape, wcs)
# I'm not sure how to compute this in general, so here's a specialization
# for cylindrical projections
if args.proj == "cea" or args.proj == "car":
dec = pos[0, :, 0]
ra = pos[1, 0, :]
lindec = linpos(shape[0], enmap.box(shape, wcs)[:, 0])
scale = 1 / np.cos(dec) - 1
alpha[0] = (lindec - dec)[:, None]
alpha[1] = ra[None, :] * scale[:, None]
enmap.write_map(args.omap, alpha * 180 * 60 / np.pi)
print "Processing row %5d/%d" % (r1, omap.shape[-2])
# Output map coordinates
osub = omap[...,r1:r2,:]
pmap = osub.posmap()
# Coordinate transformation
if args.rot:
s1,s2 = args.rot.split(",")
opos = coordinates.transform(s2, s1, pmap[::-1], pol=ncomp==3)
pmap[...] = opos[1::-1]
if len(opos) == 3: psi = -opos[2].copy()
del opos
# Switch to healpix convention
theta = np.pi/2-pmap[0]
phi = pmap[1]
# Evaluate map at these locations
if args.order == 0:
pix = healpy.ang2pix(nside, theta, phi)
osub[:] = imap[:,pix]
elif args.order == 1:
for i in range(ncomp):
osub[i] = healpy.get_interp_val(imap[i], theta, phi)
# Rotate polarization if necessary
if args.rot and ncomp==3:
osub[1:3] = enmap.rotate_pol(osub[1:3], psi)
print "Writing"
if args.scalar: omap = omap.preflat[0]
enmap.write_map(args.ofile, omap)
print "Done"
def fit_grid(self, verbose=False, grid_res=0.6*utils.arcmin, super=10):
self.verbose = verbose
t1 = time.time()
if verbose: print "Building coarse likelihood grid"
ngrid = int(np.round(2*self.lik.rmax/grid_res))
dchisqs, amps = self.likgrid(self.lik.rmax, ngrid, super=super, verbose=verbose)
if np.all(dchisqs == 0):
raise ValueError("featureless likelihood")
if False and verbose:
for i,s in enumerate(self.sdata):
enmap.write_map("map_%d.fits"%i,s.map)
enmap.write_map("div_%d.fits"%i,s.div)
enmap.write_map("white_%d.fits"%i,s.map*s.div**0.5)
enmap.write_map("pchisq_%d.fits"%i,s.map**2*s.div)
enmap.write_map("pchisq_smooth_%d.fits%i",enmap.smooth_gauss(s.map**2*s.div,0.6*utils.arcmin))
enmap.write_map("dchisqs.fits",dchisqs)
# Find local dchisq maxima
maxmap = ndimage.maximum_filter(dchisqs, super)
peaks = np.where((dchisqs==maxmap)*(maxmap>0))
maxvals = dchisqs[peaks]
maxpos = dchisqs.pix2sky(peaks)
# Why isn't this just amps[:,peaks] or similar?
maxamps = amps.reshape(amps.shape[0],-1)[:,np.ravel_multi_index(peaks, amps.shape[-2:])]
inds = np.argsort(maxvals)[::-1]
maxvals = maxvals[inds]
maxpos = maxpos[:,inds]
maxamps = maxamps[:,inds]
# Perform ML fit for the highest one
dpos = optimize.fmin_powell(self.calc_chisq_wrapper, maxpos[:,0]/self.scale, disp=False)*self.scale
res = self.calc_full_result(dpos, marginalize=False)
if False and verbose:
for i, m in enumerate(res.models):
enmap.write_map("model_%d.fits"%i,m)
resid = self.sdata[i].map-m
enmap.write_map("resid_%d.fits"%i,resid)
pchisq = resid**2*sdata[i].div
pchisq_smooth = enmap.smooth_gauss(pchisq, 0.6*utils.arcmin)
enmap.write_map("pchisq_smooth_resid.fits",pchisq_smooth)
print np.sum((self.sdata[i].map-m)**2*self.sdata[i].div) - self.lik.chisq0
# Ideally we would integrate over the full likelihood, not
# just the peaks. But the peaks have higher weight
# and should be distributed representatively. Using just the
# peaks makes it easy to compare with our ML-fit, which is also
# a single point. So we loop over just the peaks here.
maxvals = maxvals[1:]
maxpos = maxpos[:,1:]
maxamps = maxamps[:,1:]
P = np.exp(0.5*(maxvals-res.dchisq))
P0 = 1/(1+np.sum(P))
P *= P0
# Marginalize over peaks
res.dpos = P0*res.dpos + np.sum(P*maxpos,-1)
off = maxpos-res.dpos[:,None]
res.pcov = P0*res.pcov + np.sum(P*off[:,None]*off[None,:],-1)
res.ddpos = np.diag(res.pcov)**0.5
res.pcorr = res.pcov[0,1]/res.ddpos[0]/res.ddpos[1]
res.amps = P0*res.amps + np.sum(P*maxamps,-1)
res.damps = (res.damps**2 + np.sum(P*(maxamps-res.amps[:,None])**2,-1))**0.5
# For the significance, we will use the difference from our peak to our
# strongest competitor
res.dchisq= res.dchisq - maxvals[0]
# Base nsigma on the sources
res.nsigma= max(0,res.dchisq)**0.5
res.time = time.time()-t1
return res
parser.add_argument("-r", "--res", type=float, default=0.1)
parser.add_argument("-s", "--seed", type=int, default=0)
parser.add_argument("--refsys-res",type=float, default=10)
args = parser.parse_args()
# Generate the inputs to our simulator, which should be enmaps.
# First generate the simulated CMB
res = args.res*utils.degree
shape, wcs = pixie.fullsky_geometry(res, dims=(3,))
np.random.seed(args.seed)
print colors.green + "Simulating reference blackbody" + colors.reset
rshape, rwcs = pixie.fullsky_geometry(args.refsys_res*utils.degree, dims=(3,))
map_ref = pixie.sim_reference_blackbody(rshape, rwcs)
extra = { "NAME": "REFERENCE", "BEAM": "NONE", "SPEC": "BLACK" }
enmap.write_map(args.odir + "/map_ref.fits", map_ref, extra=extra)
#map_ref_test = pixie.sim_reference_blackbody(rshape, rwcs, 19.6)
#enmap.write_map(args.odir + "/map_ref_test.fits", map_ref_test, extra=extra)
# Then project our dust map onto our target coordinates
print colors.green + "Projecting dust model" + colors.reset
heal_dust = pixie.read_healpix(args.idir + "/test_therm_600p0_512_v2.fits")
map_dust = pixie.project_healpix(shape, wcs, heal_dust, rot="gal,ecl", verbose=True)
extra = {
"NAME": "DUST",
"BEAM": "GAUSS",
"FWHM": 0.07,
"SPEC": "GRAY",
"TBODY": 19.6,
"BETA": 1.59,
import numpy as np, argparse
from enlib import enmap
from scipy import ndimage
parser = argparse.ArgumentParser()
parser.add_argument("pos")
parser.add_argument("template")
parser.add_argument("ofile")
parser.add_argument("-r", "--radius", type=float, default=3)
parser.add_argument("-c", "--columns", type=str, default="3,5,2")
parser.add_argument("-t", "--threshold", type=float, default=0)
args = parser.parse_args()
cols = [int(w) for w in args.columns.split(",")]
srcinfo = np.loadtxt(args.pos)[:,cols]
pos = srcinfo[np.abs(srcinfo[:,2])>=args.threshold][:,:2] * np.pi/180
map = enmap.read_map(args.template)
pix = map.sky2pix(pos.T).T.astype(int)
pixrad = (map.area()/map.npix)**0.5
mrad = args.radius*np.pi/180/60/pixrad
mask = enmap.zeros(map.shape[-2:], map.wcs)+1
mask[pix[:,0],pix[:,1]] = 0
mask = enmap.enmap(1.0*(ndimage.distance_transform_edt(mask) > mrad), map.wcs)
enmap.write_map(args.ofile, mask)
dirname = os.path.dirname(fname)
if len(dirname) == 0: dirname = "."
base_ext = os.path.basename(fname)
# Find the extension. Using the last dot does not work for .fits.gz.
# Using the first dot in basename does not work for foo2.5_bar.fits.
# Will therefore handle .gz as a special case.
if base_ext.endswith(".gz"):
dot = base_ext[:-3].rfind(".")
else:
dot = base_ext.rfind(".")
if dot < 0: dot = len(base_ext)
base = base_ext[:dot]
ext = base_ext[dot+1:]
return dirname, base, ext
for ifile in args.ifiles:
m = enmap.read_map(ifile)
if args.slice:
m = eval("m" + args.slice)
N = m.shape[:-2]
for i, comp in enumerate(m.preflat):
I = np.unravel_index(i, N) if len(N) > 0 else []
if args.symmetric and np.any(np.sort(I) != I):
continue
ndigits = [get_num_digits(n) for n in N]
compname = "_".join(["%0*d" % (ndig,ind) for ndig,ind in zip(ndigits,I)]) if len(N) > 0 else ""
dir, base, ext = split_file_name(ifile)
oname = dir + "/" + base + "_" + compname + "." + ext
if args.verbose: print(oname)
enmap.write_map(oname, comp)
smod = project_maps(fit.models, fit.poss_cel, shape, wcs) smap = enmap.samewcs([smap,smod,smap-smod],smap) # And build scaled coadd. When map is divided by a, div = ivar # is multiplied by a**2. Points in very noisy regions can have # large error bars in the amplitude, and hence might randomly # appear to have a very strong signal. We don't want to let these # dominate, so downweight points with atypically high variance. # FIXME: Need a better approach than this. This fixed some things, # but broke others. #beam_exposure = [] #for i, s in enumerate(sdata): # med_div = np.median(s.div[s.div!=0]) # profile = fit.models[i]/fit.amps[i] # avg_div = np.sum(s.div*profile)/np.sum(profile) # beam_exposure.append(avg_div/med_div) #beam_exposure = np.array(beam_exposure) #weight = np.minimum(1,beam_exposure*1.25)**4 weight = 1 trhs = np.sum(smap*sdiv*(fit.amps*weight)[None,:,None,None],1) tdiv = np.sum(sdiv*(fit.amps*weight)[:,None,None]**2,0) tdiv[tdiv==0] = np.inf tmap = trhs/tdiv # And write them enmap.write_map(args.odir + "/totmap_%s_%02d.fits" % (sdata[0].id,gi), tmap) for i in range(fit.nsrc): enmap.write_map(args.odir + "/shiftmap_%s_%03d.fits" % (sdata[i].id, sdata[i].sid), smap[:,i]) # Output unshifted map too omap = enmap.samewcs([sdata[i].map.preflat[0],fit.models[i],sdata[i].map.preflat[0]-fit.models[i]],sdata[i].map) enmap.write_map(args.odir + "/fitmap_%s_%03d.fits" % (sdata[i].id,sdata[i].sid), omap) f.close()
dhit = hit2+hit vmap = calc_map_block_ivar(dmap, bs) mask = (hit>quant(hit,qlim)*hitlim) & (hit2>quant(hit2,qlim)*hitlim) mask &= (hit<quant(hit,qlim)) & (hit2<quant(hit2,qlim)) # Reduce dhit and mask to vmap's resolution dhit = calc_map_block_mean(dhit, bs) mask = calc_map_block_mean(mask, bs)>0 ratio = utils.medmean(vmap[mask]/dhit[mask]) # And compute the sensitivity ratio *= get_bias(bs)**2 ratios.append(ratio) sens = (ratio*args.srate)**-0.5 print "%d-%d %7.2f" % (i+1,i, sens) map, hit = map2, hit2 # Ratio has units 1/(uK^2*sample), and gives us the conversion # factor between hitmaps and inverse variance maps, which we call # div maps by convention from tenki. ratio = np.mean(ratios) print "mean %7.2f" % (ratio*args.srate)**-0.5 if args.dry_run: sys.exit() # Ok, now that we have the mean conversion factor between hits and divs, # loop through all hits again, and output div maps for i, (hitfile, divfile) in enumerate(zip(hitfiles, divfiles)): hit = enmap.read_map(hitfile) div = hit*ratio print "Writing %s" % divfile enmap.write_map(divfile, div)
stats = sampler.build_stats() # But find the ML point exactly, so we don't have to waste too many samples pos_ml = optimize.fmin_powell(lik, stats.pos/utils.arcmin, disp=0)*utils.arcmin stats.pos_ml = pos_ml # Output to stdout and to our indiviudal files msg = format_stats(stats, thumb_data) f.write(msg + "\n") f.flush() #mpiwrite(f, msg + "\n") print msg sys.stdout.flush() if args.minimaps: diag = lik.diag_plot(stats.pos) enmap.write_map(args.odir + "/diag_%s.fits" % oname, diag) enmap.write_map(args.odir + "/lik_%s.fits" % oname, likmap) #if False: # find_minimum(lik) #if False: # likmap = map_likelihood(lik) # enmap.write_map(args.odir + "/grid_%s.fits" % thumb_data.id, likmap) #if False: # x0 = np.zeros(2) # x = optimize.fmin_powell(lik, x0) # off = x*utils.arcmin # diag= lik.diag_plot(off) # enmap.write_map(args.odir + "/diag_%s.fits" % thumb_data.id, diag) #if False: # sampler = Sampler(lik, verbose=True)
tyx = [(y,x) for y in range(ntile[0]-1,-1,-1) for x in range(ntile[1])]
for i in range(comm.rank, len(tyx), comm.size):
y, x = tyx[i]
osuff = "_padtile%(y)03d_%(x)03d.fits" % {"y":y,"x":x}
tags = [s.replace(":","_") for s in signals]
nok = sum([os.path.isfile(args.odir + "/" + tag + osuff) for tag in tags])
if args.cont and nok == len(tags):
print "%3d skipping %3d %3d (already done)" % (comm.rank, y, x)
continue
print "%3d processing %3d %3d" % (comm.rank, y, x)
tpos = np.array(tyx[i])
pbox = np.array([tpos*tshape,np.minimum((tpos+1)*tshape,shape[-2:])])
box = enmap.pix2sky(shape, wcs, pbox.T).T
res = get_filtered_tile(mapinfo, box, signals, verbose=False)
for si, tag in enumerate(tags):
if res is not None:
snmap = res.signals[si].snmap
else:
snmap = jointmap.make_dummy_tile(shape, wcs, box, pad=pad, dtype=dtype).map
enmap.write_map(args.odir + "/" + tag + osuff, snmap)
else:
# Single arbitrary tile
if not overlaps_any(bounds, boxes):
print "No data in selected region"
else:
res = get_filtered_tile(mapinfo, bounds, signals, verbose=True)
for i, sig in enumerate(res.signals):
enmap.write_map(args.odir + "/%s_snmap.fits" % sig.name, sig.snmap)
enmap.write_map(args.odir + "/%s_alpha.fits" % sig.name, sig.alpha)
enmap.write_map(args.odir + "/%s_dalpha.fits" % sig.name, sig.dalpha)
def output(res, dir):
utils.mkdir(dir)
for i, (tqu, teb, desc) in enumerate(zip(res.tqu, res.teb, res.desc)):
enmap.write_map("%s/%02d_%s_tqu.hdf" % (dir,i+1,desc), tqu)
enmap.write_map("%s/%02d_%s_teb.hdf" % (dir,i+1,desc), teb)
return enmap.samewcs(map[utils.transpose_inds(dets, nrow, ncol)], map)
# Read our map, and give each row a weight
pickup = enmap.read_map(args.pickup_map)
pickup = reorder(pickup, nrow, ncol, d.dets)
weight = np.median((pickup[:, 1:] - pickup[:, :-1])**2, -1)
weight[weight > 0] = 1 / weight[weight > 0]
# Find the output pixel for each input pixel
baz = pickup[:1].posmap()[1, 0]
bel = baz * 0 + args.el * utils.degree
ipoint = np.array([baz, bel])
opoint = ipoint[:, None, :] + d.point_offset.T[:, :, None]
opix = template.sky2pix(opoint[::-1]).astype(int) # [{y,x},ndet,naz]
opix = np.rollaxis(opix, 1) # [ndet,{y,x},naz]
omap = enmap.zeros((3, ) + template.shape[-2:], template.wcs)
odiv = enmap.zeros((3, 3) + template.shape[-2:], template.wcs)
for det in range(d.ndet):
omap += utils.bin_multi(opix[det], template.shape[-2:], weight[det] *
pickup[det]) * d.det_comps[det, :, None, None]
odiv += utils.bin_multi(
opix[det], template.shape[-2:], weight[det]) * d.det_comps[
det, :, None, None, None] * d.det_comps[det, None, :, None, None]
odiv = enmap.samewcs(array_ops.eigpow(odiv, -1, axes=[0, 1]), odiv)
omap = enmap.samewcs(array_ops.matmul(odiv, omap, axes=[0, 1]), omap)
enmap.write_map(args.ofile, omap)
else: rhs_tot += rhs
infos.append(bunch.Bunch(U=U,N=N,H=H,W=W,pattern=pattern,site=site,srate=srate,scale=scale,speed=speed))
rhs = utils.allreduce(rhs_tot, comm)
#info = infos[0]
#foo = rhs*info.H
#enmap.write_map("test1.fits", foo)
#bar = enmap.samewcs(info.U.apply(foo),foo)
#enmap.write_map("test2.fits", bar)
#foo = enmap.samewcs(info.U.trans(bar, foo),foo)
#enmap.write_map("test3.fits", foo)
#1/0
#
#
dof = zipper.ArrayZipper(rhs.copy())
A = Amat(dof, infos, comm)
cg = enlib.cg.CG(A, dof.zip(rhs))
utils.mkdir(args.odir)
for i in range(200):
cg.step()
if comm.rank == 0:
#print np.std(cg.x[cg.x!=0])
if cg.i % 10 == 0:
map = dof.unzip(cg.x)
enmap.write_map(args.odir + "/map%03d.fits" % cg.i, map)
L.info("%4d %15.7e" % (cg.i, cg.err))
# Go from detectors to y-pixel in input maps ypix = utils.transpose_inds(dets[i], nrow, ncol) vals.append(imaps[i,ypix,iaz]) vals = np.concatenate(vals) pos = np.concatenate(offs) # Write to appropriate position in array pix = np.maximum(0,np.minimum((np.array(shape[-2:])-1)[:,None],enmap.sky2pix(shape, wcs, pos.T).astype(np.int32))) m = enmap.zeros(shape[-2:],wcs) m[tuple(pix)] = vals # Grow by smoothing m = enmap.smooth_gauss(m, rad)/norm omap[iaz] = m # Truncate very low values refval = np.mean(np.abs(omap))*0.01 mask = np.any(np.abs(omap)>refval,0) omap[:,~mask] = 0 if not args.individual: # output a single file enmap.write_map(args.ofile, omap) else: # output one file for each. This lets us encode # each one's azimuth utils.mkdir(args.ofile) for i, (baz, m) in enumerate(zip(bazs, omap)): if np.all(m==0): continue print i m.wcs.wcs.crval[0] = baz/utils.degree enmap.write_map(args.ofile + "/map%04d.fits" % i, m)
def make_beam(nl, bsize):
l = np.arange(nl)
return np.exp(-l*(l+1)*bsize**2)
for i in range(comm.rank, args.nsim, comm.size):
if args.lensed:
ps = powspec.read_camb_full_lens(args.powspec).astype(dtype)
if args.beam:
raise NotImplementedError("Beam not supported for lensed sims yet")
if args.geometry == "curved":
m, = lensing.rand_map(shape, wcs, ps, lmax=lmax, maplmax=maplmax, seed=(seed,i), verbose=verbose, dtype=dtype)
else:
maps = enmap.rand_map((shape[0]+1,)+shape[1:], wcs, ps)
phi, unlensed = maps[0], maps[1:]
m = lensing.lens_map_flat(unlensed, phi)
else:
ps = powspec.read_spectrum(args.powspec).astype(type)
beam = make_beam(ps.shape[-1], args.beam*utils.arcmin*utils.fwhm)
ps *= beam
if args.geometry == "curved":
m = curvedsky.rand_map(shape, wcs, ps, lmax=lmax, seed=(seed,i), method=args.method, direct=args.direct, dtype=dtype, verbose=verbose)
else:
m = enmap.rand_map(shape, wcs, ps)
if args.nsim == 1:
if verbose: print "Writing %s" % args.ofile
enmap.write_map(args.ofile, m)
else:
if verbose: print "Writing %s" % (args.ofile % i)
enmap.write_map(args.ofile % i, m)
rhs = enmap.zeros((ncomp,)+shape, area.wcs, dtype)
div = enmap.zeros((ncomp,ncomp)+shape, area.wcs, dtype)
junk = np.zeros(pcut.njunk, dtype)
with bench.show("rhs"):
tod *= ivar[:,None]
pcut.backward(tod, junk)
pmap.backward(tod, rhs)
with bench.show("hits"):
for i in range(ncomp):
div[i,i] = 1
pmap.forward(tod, div[i])
tod *= ivar[:,None]
pcut.backward(tod, junk)
div[i] = 0
pmap.backward(tod, div[i])
with bench.show("map"):
idiv = array_ops.eigpow(div, -1, axes=[0,1], lim=1e-5)
map = enmap.map_mul(idiv, rhs)
# Estimate central amplitude
c = np.array(map.shape[-2:])/2
crad = 50
mcent = map[:,c[0]-crad:c[0]+crad,c[1]-crad:c[1]+crad]
mcent = enmap.downgrade(mcent, 4)
amp = np.max(mcent)
print "%s amp %7.3f asens %7.3f" % (id, amp/1e6, asens)
with bench.show("write"):
enmap.write_map("%s%s_map.fits" % (prefix, bid), map)
enmap.write_map("%s%s_rhs.fits" % (prefix, bid), rhs)
enmap.write_map("%s%s_div.fits" % (prefix, bid), div)
del d, scan, pmap, pcut, tod, map, rhs, div, idiv, junk
tod *= ivar[:,None]
sampcut.gapfill_const(scan.cut, tod, inplace=True)
for sid in tod_srcs[id]:
src = srcs[sid]
if src.type == "fixed": sys = "hor:%.6f_%.6f:cel/0_0:hor" % (src.ra/utils.degree, src.dec/utils.degree)
elif src.type == "planet": sys = "hor:%s/0_0" % src.name
else: raise ValueError("Invalid source type '%s'" % src.type)
rhs = enmap.zeros((ncomp,)+shape, area.wcs, dtype)
div = enmap.zeros((ncomp,ncomp)+shape, area.wcs, dtype)
with bench.show("pmat %s" % sid):
pmap = pmat.PmatMap(scan, area, sys=sys)
with bench.show("rhs %s" % sid):
pmap.backward(tod, rhs)
with bench.show("hits"):
for i in range(ncomp):
div[i,i] = 1
pmap.forward(tod, div[i])
tod *= ivar[:,None]
sampcut.gapfill_const(scan.cut, tod, inplace=True)
div[i] = 0
pmap.backward(tod, div[i])
with bench.show("map %s" % sid):
idiv = array_ops.eigpow(div, -1, axes=[0,1], lim=1e-5, fallback="scalar")
map = enmap.map_mul(idiv, rhs)
with bench.show("write"):
enmap.write_map("%s%s_src%03d_map.fits" % (prefix, bid, sid), map)
enmap.write_map("%s%s_src%03d_rhs.fits" % (prefix, bid, sid), rhs)
enmap.write_map("%s%s_src%03d_div.fits" % (prefix, bid, sid), div)
del rhs, div, idiv, map
del d, scan, pmap, tod
ires = np.array([1,1./np.sin(R)])*res/args.supersample
shape, wi = enmap.geometry(pos=[[np.pi/2-R,-np.pi],[np.pi/2,np.pi]], res=ires, proj="car")
imap = enmap.zeros((ncomp,)+shape, wi)
# Define SHT for interpolation pixels
with dprint("construct sht"):
minfo = curvedsky.map2minfo(imap)
lmax_ideal = np.pi/res
ps = ps[:,:,:lmax_ideal]
lmax = ps.shape[-1]
# We do not need all ms when centered on the pole. To reach 1e-10 relative
# error up to R, we need mmax approx 9560*R in radians
mmax = args.mmax or int(R*9560)
ainfo = sharp.alm_info(lmax, mmax)
sht = sharp.sht(minfo, ainfo)
with dprint("curvedsky tot"):
with dprint("rand alm"):
alm = curvedsky.rand_alm(ps, ainfo=ainfo, seed=1, m_major=False)
with dprint("alm2map"):
sht.alm2map(alm[:1], imap[:1].reshape(1,-1))
if ncomp == 3:
sht.alm2map(alm[1:3], imap[1:3,:].reshape(2,-1), spin=2)
del alm
# Make a test map to see if we can project between these
with dprint("project"):
omap = enmap.project(imap, omap.shape, omap.wcs, mode="constant", cval=np.nan)
del imap
enmap.write_map(args.omap, omap)
# This program computes a simple estimation of the amount of distortion the
# flat-sky approximation involves
import numpy as np, argparse
from enlib import enmap
parser = argparse.ArgumentParser()
parser.add_argument("omap")
parser.add_argument("-D", "--diameter", type=float, default=30)
parser.add_argument("-n", "--npix", type=int, default=800)
parser.add_argument("--proj", type=str, default="cea")
args = parser.parse_args()
r = args.diameter*np.pi/180/2
shape, wcs = enmap.geometry(pos=[[-r,-r],[r,r]], shape=(args.npix, args.npix), proj=args.proj)
def linpos(n,b): return b[0] + (np.arange(n)+0.5)*(b[1]-b[0])/n
alpha = enmap.zeros((2,)+shape, wcs)
pos = enmap.posmap(shape, wcs)
# I'm not sure how to compute this in general, so here's a specialization
# for cylindrical projections
if args.proj == "cea" or args.proj =="car":
dec = pos[0,:,0]
ra = pos[1,0,:]
lindec= linpos(shape[0],enmap.box(shape,wcs)[:,0])
scale = 1/np.cos(dec)-1
alpha[0] = (lindec-dec)[:,None]
alpha[1] = ra[None,:]*scale[:,None]
enmap.write_map(args.omap, alpha*180*60/np.pi)
def combine_tiles(ipathfmt,
opathfmt,
combine=2,
downsample=2,
itile1=(None, None),
itile2=(None, None),
tyflip=False,
txflip=False,
pad_to=None,
comm=None,
verbose=False):
"""Given a set of tiles on disk at locaiton ipathfmt % {"y":...,"x"...},
combine them into larger tiles, downsample and write the result to
opathfmt % {"y":...,"x":...}. x and y must be contiguous and start at 0.
reftile[2] indicates the tile coordinates of the first valid input tile.
This needs to be specified if not all tiles of the logical tiling are
physically present.
tyflip and txflip indicate if the tiles coordinate system is reversed
relative to the pixel coordinates or not."
"""
# Expand combine and downsample to 2d
combine = np.zeros(2, int) + combine
downsample = np.zeros(2, int) + downsample
if pad_to is not None:
pad_to = np.zeros(2, int) + pad_to
# Handle optional mpi
rank, size = (comm.rank, comm.size) if comm is not None else (0, 1)
# Find the range of input tiles
itile1, itile2 = find_tile_range(ipathfmt, itile1, itile2)
# Read the first tile to get its size information
ibase = enmap.read_map(ipathfmt % {"y": itile1[0], "x": itile1[1]}) * 0
# Find the set of output tiles we need to consider
otile1 = itile1 / combine
otile2 = (itile2 - 1) / combine + 1
# And loop over them
oyx = [(oy, ox) for oy in range(otile1[0], otile2[0])
for ox in range(otile1[1], otile2[1])]
for i in range(rank, len(oyx), size):
oy, ox = oyx[i]
# Read in all associated tiles into a list of lists
rows = []
for dy in range(combine[0]):
iy = oy * combine[0] + dy
if iy >= itile2[0]: continue
cols = []
for dx in range(combine[1]):
ix = ox * combine[1] + dx
if ix >= itile2[1]: continue
if iy < itile1[0] or ix < itile1[1]:
# The first tiles are missing on disk, but are
# logically a part of the tiling. Use ibase,
# which has been zeroed out.
cols.append(ibase)
else:
itname = ipathfmt % {"y": iy, "x": ix}
cols.append(enmap.read_map(itname))
if txflip: cols = cols[::-1]
rows.append(cols)
# Stack them next to each other into a big tile
if tyflip: rows = rows[::-1]
omap = enmap.tile_maps(rows)
# Downgrade if necessary
if np.any(downsample > 1):
omap = enmap.downgrade(omap, downsample)
if pad_to is not None:
# Padding happens towards the end of the tiling,
# which depends on the flip status
padding = np.array(
[[0, 0],
[pad_to[0] - omap.shape[-2], pad_to[1] - omap.shape[-1]]])
if tyflip: padding[:, 0] = padding[::-1, 0]
if txflip: padding[:, 1] = padding[::-1, 1]
omap = enmap.pad(omap, padding)
# And output
otname = opathfmt % {"y": oy, "x": ox}
utils.mkdir(os.path.dirname(otname))
enmap.write_map(otname, omap)
if verbose: print otname
rhs = enmap.zeros((ncomp,)+shape, area.wcs, dtype)
div = enmap.zeros((ncomp,ncomp)+shape, area.wcs, dtype)
junk = np.zeros(pcut.njunk, dtype)
with bench.show("rhs"):
tod *= ivar[:,None]
pcut.backward(tod, junk)
pmap.backward(tod, rhs)
with bench.show("hits"):
for i in range(ncomp):
div[i,i] = 1
pmap.forward(tod, div[i])
tod *= ivar[:,None]
pcut.backward(tod, junk)
div[i] = 0
pmap.backward(tod, div[i])
with bench.show("map"):
idiv = array_ops.eigpow(div, -1, axes=[0,1], lim=1e-5)
map = enmap.map_mul(idiv, rhs)
# Estimate central amplitude
c = np.array(map.shape[-2:])/2
crad = 50
mcent = map[:,c[0]-crad:c[0]+crad,c[1]-crad:c[1]+crad]
mcent = enmap.downgrade(mcent, 4)
amp = np.max(mcent)
print("%s amp %7.3f asens %7.3f" % (id, amp/1e6, asens))
with bench.show("write"):
enmap.write_map("%s%s_map.fits" % (prefix, bid), map)
enmap.write_map("%s%s_rhs.fits" % (prefix, bid), rhs)
enmap.write_map("%s%s_div.fits" % (prefix, bid), div)
del d, scan, pmap, pcut, tod, map, rhs, div, idiv, junk
