def solve(self, maxiter=100, cg_tol=1e-7, verbose=False, dump_dir=None):
if np.sum(self.highres_mask) == 0: return None
solver = cg.CG(self.A, self.rhs.reshape(-1), M=self.M)
for i in range(maxiter):
t1 = time.time()
solver.step()
t2 = time.time()
if verbose:
print "%5d %15.7e %5.2f" % (solver.i, solver.err, t2 - t1)
if dump_dir is not None and solver.i in [1, 2, 5, 10, 20, 50
] + range(
100, 10000, 100):
m = enmap.ndmap(solver.x.reshape(self.shape), self.wcs)
enmap.write_map(dump_dir + "/step%04d.fits" % solver.i, m)
if solver.err < cg_tol:
if dump_dir is not None:
m = enmap.ndmap(solver.x.reshape(self.shape), self.wcs)
enmap.write_map(dump_dir + "/step_final.fits", m)
break
tot_map = self.highres_mask * solver.x.reshape(self.shape)
tot_div = self.highres_mask * self.tot_div
# Get rid of the fourier padding
ny, nx = tot_map.shape[-2:]
tot_map = tot_map[..., :ny - self.ffpad[0], :nx - self.ffpad[1]]
tot_div = tot_div[..., :ny - self.ffpad[0], :nx - self.ffpad[1]]
return bunch.Bunch(map=tot_map, div=tot_div)
def calc_model_constrained(tod,
cut,
srate=400,
mask_scale=0.3,
lim=3e-4,
maxiter=50,
verbose=False):
# First do some simple gapfilling to avoid messing up the noise model
tod = sampcut.gapfill_linear(cut, tod, inplace=False)
ft = fft.rfft(tod) * tod.shape[1]**-0.5
iN = nmat_measure.detvecs_jon(ft, srate)
del ft
iV = iN.ivar * mask_scale
def A(x):
x = x.reshape(tod.shape)
Ax = iN.apply(x.copy())
Ax += sampcut.gapfill_const(cut, x * iV[:, None], 0, inplace=True)
return Ax.reshape(-1)
b = sampcut.gapfill_const(cut, tod * iV[:, None], 0,
inplace=True).reshape(-1)
x0 = sampcut.gapfill_linear(cut, tod).reshape(-1)
solver = cg.CG(A, b, x0)
while solver.i < maxiter and solver.err > lim:
solver.step()
if verbose:
print("%5d %15.7e" % (solver.i, solver.err))
res = solver.x.reshape(tod.shape)
res = smooth(res, srate)
return res
def solve(self, b, x0=None, verbose=False):
"""Solves the equation system (S"+N")x = b. This is done in
a single step if S and N are in the same domain. Otherwise,
conjugate gradients is used."""
mode = self.mode.lower()
if mode[0] == mode[1]:
# Can solve directly
return self.M(b)
else:
if x0 is None: x0 = b * 0
dof = DOF(Arg(default=self.d))
def wrap(fun):
return lambda x: dof.zip(fun(*dof.unzip(x)))
solver = cg.CG(wrap(self.A),
dof.zip(b),
x0=dof.zip(x0),
M=wrap(self.M))
for i in range(50):
#while solver.err > 1e-6:
#while solver.err_true > 10:
solver.step()
if verbose:
print "%5d %15.7e %15.7e" % (solver.i, solver.err,
solver.err_true)
return dof.unzip(solver.x)[0]
def cgsolve(Afun,
rhs,
x0=None,
Mfun=None,
tol=1e-14,
maxiter=1000000,
miniter=0,
callback=None):
if Mfun is None: Mfun = cg.default_M
solver = cg.CG(Afun, rhs, x0=x0, M=Mfun)
for i in range(maxiter):
solver.step()
if callback: callback(solver.err, solver.x)
if solver.err < tol and i > miniter: break
return solver.x
del tod
omap = utils.allreduce(omap, comm)
return np.concatenate([omap.reshape(-1), ojunk], 0)
def M(x):
map = x[:area.size].reshape(area.shape)
junk = x[area.size:]
omap = map * 0
omap[:] = enmap.map_mul(idiv, map)
ojunk = junk / jdiv
return np.concatenate([omap.reshape(-1), ojunk], 0)
def dot(x, y):
mprod = np.sum(x[:area.size] * y[:area.size])
jprod = np.sum(x[area.size:] * y[area.size:])
return mprod + comm.allreduce(jprod)
bin = enmap.map_mul(idiv, cg_rhs)
enmap.write_map(args.odir + "/map_bin.fits", bin)
b = np.concatenate([cg_rhs.reshape(-1), cg_rjunk], 0)
solver = cg.CG(A, b, M=M, dot=dot)
for i in range(nstep):
solver.step()
if comm.rank == 0:
print "%5d %15.7e" % (solver.i, solver.err)
if solver.i % args.ostep == 0:
map = enmap.samewcs(solver.x[:area.size].reshape(area.shape),
area)
enmap.write_map(args.odir + "/map%04d.fits" % solver.i, map)
def coadd_tile_data(datasets,
box,
odir,
ps_smoothing=10,
pad=0,
ref_beam=None,
cg_tol=1e-6,
dump=False,
verbose=False,
read_cache=False,
write_cache=False,
div_max_tol=100,
div_div_tol=1e-10):
# Load data for this box for each dataset
datasets, ffpad = read_data(datasets,
box,
odir,
pad=pad,
verbose=verbose,
read_cache=read_cache,
write_cache=write_cache)
# We might not find any data
if len(datasets) == 0: return None
# Find the smallest beam size of the datasets
bmin = np.min([beam_size(dataset.beam) for dataset in datasets])
# Subtract mean map from each split to get noise maps. Our noise
# model is HNH, where H is div**0.5 and N is the mean 2d noise spectrum
# after some smoothing
rhs, tot_div = None, None
tot_iN, tot_udiv = None, 0
for dataset in datasets:
nsplit = 0
dset_map, dset_div = None, None
for split in dataset.splits:
if dset_map is None:
dset_map = split.data.map * 0
dset_div = split.data.div * 0
dset_map += split.data.map * split.data.div
dset_div += split.data.div
# Form the mean map for this dataset
dset_map[:, dset_div > 0] /= dset_div[dset_div > 0]
if tot_div is None: tot_div = dset_div * 0
tot_div += dset_div
tshape, twcs, tdtype = dset_map.shape, dset_div.wcs, dset_div.dtype
# Then use it to build the diff maps and noise spectra
dset_ps = None
for split in dataset.splits:
if split.data.empty: continue
diff = split.data.map - dset_map
wdiff = diff * split.data.H
# What is the healthy area of wdiff? Wdiff should have variance
# 1 or above. This tells us how to upweight the power spectrum
# to take into account missing regions of the diff map.
ndown = 10
wvar = enmap.downgrade(wdiff**2, ndown)
goodfrac = np.sum(wvar > 1e-3) / float(wvar.size)
if goodfrac < 0.1: goodfrac = 0
#opre = odir + "/" + os.path.basename(split.map)[:-5]
#enmap.write_map(opre + "_diff.fits", diff)
#enmap.write_map(opre + "_wdiff.fits", wdiff)
#enmap.write_map(opre + "_wvar.fits", wvar)
ps = np.abs(map_fft(wdiff))**2
#enmap.write_map(opre + "_ps1.fits", ps)
# correct for unhit areas, which can't be whitened
#print "A", dataset.name, np.median(ps[ps>0]), medloop(ps), goodfrac
with utils.nowarn():
ps /= goodfrac
#print "B", dataset.name, np.median(ps[ps>0]), medloop(ps), goodfrac
#enmap.write_map(opre + "_ps2.fits", ps)
#enmap.write_map(opre + "_ps2d.fits", ps)
if dset_ps is None:
dset_ps = enmap.zeros(ps.shape, ps.wcs, ps.dtype)
dset_ps += ps
nsplit += 1
if nsplit < 2: continue
# With n splits, mean map has var 1/n, so diff has var (1-1/n) + (n-1)/n = 2*(n-1)/n
# Hence tot-ps has var 2*(n-1)
dset_ps /= 2 * (nsplit - 1)
#enmap.write_map(opre + "_ps2d_tot.fits", dset_ps)
dset_ps = smooth_pix(dset_ps, ps_smoothing)
#enmap.write_map(opre + "_ps2d_smooth.fits", dset_ps)
if np.all(np.isfinite(dset_ps)):
# Super-low values of the spectrum are not realistic. These appear
# due to beam/pixel smoothing in the planck maps. This will be
# mostly taken care of when processing the beams, as long as we don't
# let them get too small
dset_ps = np.maximum(dset_ps, 1e-7)
# Optionally cap the max dset_ps, this is mostly to speed up convergence
if args.max_ps:
dset_ps = np.minimum(dset_ps, args.max_ps)
# Our fourier-space inverse noise matrix is based on the inverse noise spectrum
iN = 1 / dset_ps
#enmap.write_map(opre + "_iN_raw.fits", iN)
else:
print "Setting weight of dataset %s to zero" % dataset.name
#print np.all(np.isfinite(dset_ps)), np.all(dset_ps>0)
iN = enmap.zeros(dset_ps.shape, dset_ps.wcs, dset_ps.dtype)
# Add any fourier-space masks to this
ly, lx = enmap.laxes(tshape, twcs)
lr = (ly[:, None]**2 + lx[None, :]**2)**0.5
if dataset.highpass:
kxmask = butter(lx, args.kxrad, -3)
kxmask = 1 - (1 - kxmask[None, :]) * (
np.abs(ly) < bmin * args.kx_ymax_scale)[:, None]
highpass = butter(lr, args.highpass, -10)
filter = highpass * kxmask
#print "filter weighting", dataset.name
del kxmask, highpass
else:
filter = 1
if not args.filter: iN *= filter
# We should deconvolve the relative beam from the maps,
# but that's numerically nasty. But it can be handled
# inversely. We want (BiNB + ...)x = (BiNB iB m + ...)
# where iB is the beam deconvolution operation in map space.
# Instead of actually doing that operation, we can compute two
# inverse noise matrixes: iN_A = BiNB for the left hand
# side and iN_b = BiN for the right hand side. That way we
# avoid dividing by any huge numbers.
# Add the relative beam
iN_A = iN.copy()
iN_b = iN.copy()
if ref_beam is not None:
rel_beam = beam_ratio(dataset.beam, ref_beam)
bspec = eval_beam(rel_beam, lr)
iN_A *= bspec**2
iN_b *= bspec
#moo = iN*0+filter
#enmap.write_map(opre + "_filter.fits", moo)
# Add filter to noise model if we're downweighting
# rather than filtering.
dataset.iN_A = iN_A
dataset.iN_b = iN_b
dataset.filter = filter
#print "A", opre
#enmap.write_map(opre + "_iN_A.fits", iN_A)
#enmap.write_map(opre + "_iN.fits", iN)
# Cap to avoid single crazy pixels
tot_div = np.maximum(tot_div, np.median(tot_div[tot_div > 0]) * 0.01)
tot_idiv = tot_div * 0
tot_idiv[tot_div > div_div_tol] = 1 / tot_div[tot_div > div_div_tol]
# Build the right-hand side. The right-hand side is
# sum(HNHm)
if rhs is None: rhs = enmap.zeros(tshape, twcs, tdtype)
for dataset in datasets:
i = 0
for split in dataset.splits:
if split.data.empty: continue
#print "MOO", dataset.name, np.max(split.data.map), np.min(split.data.map), np.max(split.data.div), np.min(split.data.div)
w = split.data.H * split.data.map
fw = map_fft(w)
fw *= dataset.iN_b
if args.filter: fw *= dataset.filter
w = map_ifft(fw) * split.data.H
#enmap.write_map(odir + "/%s_%02d_rhs.fits" % (dataset.name, i), w)
rhs += w
i += 1
del w, iN, iN_A, iN_b, filter
# Now solve the equation
def A(x):
global times
m = enmap.samewcs(x.reshape(rhs.shape), rhs)
res = m * 0
times[:] = 0
ntime = 0
for dataset in datasets:
for split in dataset.splits:
if split.data.empty: continue
t = [time.time()]
w = split.data.H * m
t.append(time.time())
fw = map_fft(w)
t.append(time.time())
fw *= dataset.iN_A
t.append(time.time())
w = map_ifft(fw)
t.append(time.time())
w *= split.data.H
t.append(time.time())
res += w
for i in range(1, len(t)):
times[i - 1] += t[i] - t[i - 1]
ntime += 1
#w = enmap.harm2map(dataset.iN_A*enmap.map2harm(w))
#w *= split.data.H
#res += w
del w
times /= ntime
return res.reshape(-1)
def M(x):
m = enmap.samewcs(x.reshape(rhs.shape), rhs)
res = m * tot_idiv
return res.reshape(-1)
solver = cg.CG(A, rhs.reshape(-1), M=M)
for i in range(1000):
t1 = time.time()
solver.step()
t2 = time.time()
if verbose:
print "%5d %15.7e %5.2f: %4.2f %4.2f %4.2f %4.2f %4.2f" % (
solver.i, solver.err, t2 - t1, times[0], times[1], times[2],
times[3], times[4]), np.std(solver.x)
if dump and solver.i in [1, 2, 5, 10, 20, 50] + range(100, 10000, 100):
m = enmap.samewcs(solver.x.reshape(rhs.shape), rhs)
enmap.write_map(odir + "/step%04d.fits" % solver.i, m)
if solver.err < cg_tol:
if dump:
m = enmap.samewcs(solver.x.reshape(rhs.shape), rhs)
enmap.write_map(odir + "/step_final.fits", m)
break
tot_map = enmap.samewcs(solver.x.reshape(rhs.shape), rhs)
# Get rid of the fourier padding
ny, nx = tot_map.shape[-2:]
tot_map = tot_map[..., :ny - ffpad[0], :nx - ffpad[1]]
tot_div = tot_div[..., :ny - ffpad[0], :nx - ffpad[1]]
return bunch.Bunch(map=tot_map, div=tot_div)