def __init__(self, icov, rhs, dof): self.Ai = icov self.b = rhs self.dof = dof if icov.size > 0: self.A = array_ops.eigpow(icov, -1.0) self.Aih = array_ops.eigpow(icov, 0.5) self.Ah = array_ops.eigpow(icov, -0.5) _,self.ldet = np.linalg.slogdet(self.Ai) self.x = self.A.dot(rhs) else: self.A, self.Aih, self.Ah = [icov.copy()]*3 self.x, self.ldet = rhs.copy(), 0
def get_samples(self): # Start with the noise if hasattr(self, "_tod") and self._tod is not None: return self._tod.copy() np.random.seed(self.seed) tod = np.zeros([self.ndet,self.nsamp]).astype(np.float32) if self.noise_scale != 0: tod = np.random.standard_normal([self.ndet,self.nsamp]).astype(np.float32)*self.noise_scale covs = array_ops.eigpow(self.noise.icovs, -0.5, axes=[-2,-1]) N12 = nmat.NmatBinned(covs, self.noise.bins, self.noise.dets) N12.apply(tod) else: tod[...] = 0 tod = tod.astype(np.float64) # And add the point sources for di in range(self.ndet): for i, (pos,amp,beam) in enumerate(zip(self.srcs.pos,self.srcs.amps,self.srcs.beam)): point = (self.boresight+self.offsets[di,None,:])[:,1:] point = coordinates.transform(self.sys, self.simsys, point.T, time=self.boresight[:,0]+self.mjd0, site=self.site).T r2 = np.sum((point-pos[None,:])**2,1)/beam**2 I = np.where(r2 < self.nsigma**2)[0] tod[di,I] += np.exp(-0.5*r2[I])*np.sum(amp*self.comps[di]) if hasattr(self, "_tod"): self._tod = tod.copy() return tod
def build_iN_constcorr_prior(data,
cmb=None,
lknee0=2000,
ivars=None,
constcov=False):
if ivars is None: ivars = data.ivars
N = enmap.zeros((data.n, data.n) + data.maps.shape[-2:], data.maps.wcs,
data.maps.dtype)
ref = np.mean(ivars, (-2, -1))
ref = np.maximum(ref, np.max(ref) * 1e-4)
norm = 1 / (ref / ivars.pixsize())
for i, freq in enumerate(data.freqs):
lknee = lknee0 * freq / 100
N[i, i] = (1 + (np.maximum(0.5, data.l) / lknee)**-3.5) * norm[i]
# Deconvolve the pixel window from the theoretical flat-at-high-l spectrum
N = N / data.wy[:, None]**2 / data.wx[None, :]**2
# Apply the beam-convolved cmb if available
if cmb is not None:
Bf = data.fconvs[:, None, None] * data.beams
N += cmb * Bf[:, None] * Bf[None, :]
del Bf
if not constcov:
N /= norm[:, None, None, None]**0.5 * norm[None, :, None, None]**0.5
iN = array_ops.eigpow(N, -1, axes=[0, 1])
return iN
def solve(w,m):
if w.ndim < 4: return m/w
elif w.ndim == 4:
# This is slower, but handles low-hit areas near the edge better
iw = array_ops.eigpow(w,-1,axes=[0,1])
return enmap.samewcs(array_ops.matmul(iw,m,axes=[0,1]), m)
#return array_ops.solve_masked(w,m,axes=[0,1])
else: raise NotImplementedError("Only 2d, 3d or 4d weight maps understood")
def solve(w,m):
if w.ndim < 4: return m/w
elif w.ndim == 4:
# This is slower, but handles low-hit areas near the edge better
iw = array_ops.eigpow(w,-1,axes=[0,1])
return enmap.samewcs(array_ops.matmul(iw,m,axes=[0,1]), m)
#return array_ops.solve_masked(w,m,axes=[0,1])
else: raise NotImplementedError("Only 2d, 3d or 4d weight maps understood")
def safe_invert(div): if div.shape[-1] == 1: idiv = div.copy() idiv[div==0] = 1 idiv[div!=0] = 1/idiv[div!=0] else: idiv = array_ops.eigpow(div, -1, axes=[-2,-1], lim=1e-3, fallback="scalar") return idiv
def get_covsqrt(ps, method="arrayops"):
if method == "multipow":
covsq = enmap.multi_pow(ps.copy(), 0.5)
elif method == "arrayops":
from enlib import array_ops
covsq = array_ops.eigpow(ps.copy(), 0.5, axes=[0, 1])
covsq[:, :, ps.modlmap() < 2] = 0
assert np.all(np.isfinite(covsq))
return enmap.enmap(covsq, ps.wcs)
def get_samples(self): np.random.seed(self.seed) tod = np.zeros([self.ndet,self.nsamp]).astype(self.map.dtype) self.pmat.forward(tod, self.map) if self.noise_scale: noise = np.random.standard_normal([self.ndet,self.nsamp]).astype(self.map.dtype)*self.noise_scale covs = array_ops.eigpow(self.noise.icovs, -0.5, axes=[-2,-1]) N12 = nmat.NmatBinned(covs, self.noise.bins, self.noise.dets) N12.apply(noise) tod += noise return tod
with bench.show("hits"):
for i in range(ncomp):
div[i, i] = 1
pmap.forward(tod, div[i]) # div -> tod.
tod_ones = tod.copy()
tod *= ivar[:, None]
pcut.backward(tod, junk) # tod -> junk.
pcut.backward(tod_ones, junk) # tod -> junk.
div[i] = 0
pmap.backward(tod, div[i]) # tod -> div.
pmap.backward(tod_ones, hits[i]) # tod -> hits.
with bench.show("map"):
idiv = array_ops.eigpow(div,
-1,
axes=[0, 1],
lim=1e-5,
fallback="scalar")
map = enmap.map_mul(idiv, rhs)
# Estimate central amplitude
c = np.array(map.shape[-2:]) // 2
crad = 50
mcent = map[:, c[0] - crad:c[0] + crad, c[1] - crad:c[1] + crad]
mcent = enmap.downgrade(mcent, 4)
amp = np.max(mcent)
print("%s amp %7.3e asens %7.3e" % (id, amp / 1e6, asens))
with bench.show("write"):
# Store maps in individual directories with id as name.
outdir = opj(prefix, entry.id)
if not os.path.isdir(outdir):
rhs = enmap.zeros((ncomp,)+shape, area.wcs, dtype)
div = enmap.zeros((ncomp,ncomp)+shape, area.wcs, dtype)
junk = np.zeros(pcut.njunk, dtype)
with bench.show("rhs"):
tod *= ivar[:,None]
pcut.backward(tod, junk)
pmap.backward(tod, rhs)
with bench.show("hits"):
for i in range(ncomp):
div[i,i] = 1
pmap.forward(tod, div[i])
tod *= ivar[:,None]
pcut.backward(tod, junk)
div[i] = 0
pmap.backward(tod, div[i])
with bench.show("map"):
idiv = array_ops.eigpow(div, -1, axes=[0,1], lim=1e-5)
map = enmap.map_mul(idiv, rhs)
# Estimate central amplitude
c = np.array(map.shape[-2:])/2
crad = 50
mcent = map[:,c[0]-crad:c[0]+crad,c[1]-crad:c[1]+crad]
mcent = enmap.downgrade(mcent, 4)
amp = np.max(mcent)
print("%s amp %7.3f asens %7.3f" % (id, amp/1e6, asens))
with bench.show("write"):
enmap.write_map("%s%s_map.fits" % (prefix, bid), map)
enmap.write_map("%s%s_rhs.fits" % (prefix, bid), rhs)
enmap.write_map("%s%s_div.fits" % (prefix, bid), div)
del d, scan, pmap, pcut, tod, map, rhs, div, idiv, junk
# Build div, which we need in both cases div = enmap.zeros((ncomp,)+area.shape,area.wcs,dtype) for i in range(ncomp): work = div[0]*0 work[i] = 1 for scan in scans: tod = np.zeros((scan.ndet,scan.nsamp),dtype) scan.pmap.forward(tod, work) if args.method == "cg": scan.noise.white(tod) else: tod /= scan.T scan.pcut.backward(tod, np.zeros(scan.pcut.njunk,dtype)) scan.pmap.backward(tod, div[i]) div = utils.allreduce(div, comm) #idiv = utils.eigpow(div,-1,[0,1]) idiv = array_ops.eigpow(div,-1,[0,1], lim=1e-6) if comm.rank == 0: enmap.write_map(args.odir + "/map_div.fits", div) enmap.write_map(args.odir + "/map_idiv.fits", idiv) del work, div # And the same for junk jdiv = np.full(njunk_tot, 1.0, dtype) for scan in scans: tod = np.zeros((scan.ndet,scan.nsamp),dtype) scan.pcut.forward(tod, jdiv[scan.cut_range[0]:scan.cut_range[1]]) if args.method == "cg": scan.noise.white(tod) else: tod /= scan.T scan.pcut.backward(tod, jdiv[scan.cut_range[0]:scan.cut_range[1]]) del tod
def pow(mat, exp, axes=[0, 1]):
return enmap.samewcs(array_ops.eigpow(mat, exp, axes=axes), mat, exp)
def getamps(W, V, d): rhs = (V.T*W[None]).dot(d) A = (V.T*W[None]).dot(V) iA = array_ops.eigpow(A,-1) return iA.dot(rhs)
def getamps(W, V, d): rhs = (V.T*W[None]).dot(d) A = (V.T*W[None]).dot(V) iA = array_ops.eigpow(A,-1) return iA.dot(rhs)
def pow(mat, exp, axes):
return array_ops.eigpow(mat, exp, axes=axes)
ndet = ft.shape[0]
bins = np.minimum(
(noise.bins * ft.shape[1] / noise.bins[-1, 1]).astype(int),
ft.shape[1] - 1)
nbin = len(bins)
cov_full = np.zeros([nbin, ndet, ndet])
for bi, b in enumerate(bins):
print "A", bi, b, np.mean(np.abs(ft[0, b[0]:b[1]])**
2)**0.5 / 20
cov_full[bi] = nmat_measure.measure_cov(ft[:, b[0]:b[1]])
cov_model = noise.covs
# Compute total noise and correlated noise per detector for the two
pow_full_tot = np.einsum("bii->bi", cov_full)
pow_model_tot = np.einsum("bii->bi", cov_model)
pow_full_ucorr = 1 / np.einsum("bii->bi",
array_ops.eigpow(cov_full, -1))
pow_model_ucorr = 1 / np.einsum(
"bii->bi", array_ops.eigpow(cov_model, -1))
corr_full = cov_full / (pow_full_tot[:, :, None] *
pow_full_tot[:, None, :])**0.5
corr_model = cov_model / (pow_model_tot[:, :, None] *
pow_model_tot[:, None, :])**0.5
# And write
hfile["full/corr"] = corr_full
hfile["full/tpow"] = pow_full_tot
hfile["full/upow"] = pow_full_ucorr
hfile["model/corr"] = corr_model
hfile["model/tpow"] = pow_model_tot
hfile["model/upow"] = pow_model_ucorr
t = np.array(t)
dt = t[1:] - t[:-1]
div = enmap.zeros((ncomp, ) + area.shape, area.wcs, dtype)
for i in range(ncomp):
work = div[0] * 0
work[i] = 1
for scan in scans:
tod = np.zeros((scan.ndet, scan.nsamp), dtype)
scan.pmap.forward(tod, work)
if args.method == "cg":
scan.noise.white(tod)
else:
tod /= scan.T
scan.pcut.backward(tod, np.zeros(scan.pcut.njunk, dtype))
scan.pmap.backward(tod, div[i])
div = utils.allreduce(div, comm)
#idiv = utils.eigpow(div,-1,[0,1])
idiv = array_ops.eigpow(div, -1, [0, 1], lim=1e-6)
if comm.rank == 0:
enmap.write_map(args.odir + "/map_div.fits", div)
enmap.write_map(args.odir + "/map_idiv.fits", idiv)
del work, div
# And the same for junk
jdiv = np.full(njunk_tot, 1.0, dtype)
for scan in scans:
tod = np.zeros((scan.ndet, scan.nsamp), dtype)
scan.pcut.forward(tod, jdiv[scan.cut_range[0]:scan.cut_range[1]])
if args.method == "cg":
scan.noise.white(tod)
else:
tod /= scan.T
scan.pcut.backward(tod, jdiv[scan.cut_range[0]:scan.cut_range[1]])
del tod
with h5py.File("%s/%s.hdf" % (args.odir, id),"w") as hfile:
nmat.write_nmat(hfile, noise) ; t.append(time.time())
if args.covtest:
# Measure full cov per bin
ndet = ft.shape[0]
bins = np.minimum((noise.bins*ft.shape[1]/noise.bins[-1,1]).astype(int),ft.shape[1]-1)
nbin = len(bins)
cov_full = np.zeros([nbin,ndet,ndet])
for bi, b in enumerate(bins):
print "A", bi, b, np.mean(np.abs(ft[0,b[0]:b[1]])**2)**0.5/20
cov_full[bi] = nmat_measure.measure_cov(ft[:,b[0]:b[1]])
cov_model= noise.covs
# Compute total noise and correlated noise per detector for the two
pow_full_tot = np.einsum("bii->bi",cov_full)
pow_model_tot = np.einsum("bii->bi",cov_model)
pow_full_ucorr = 1/np.einsum("bii->bi",array_ops.eigpow(cov_full,-1))
pow_model_ucorr = 1/np.einsum("bii->bi",array_ops.eigpow(cov_model,-1))
corr_full = cov_full/(pow_full_tot[:,:,None]*pow_full_tot[:,None,:])**0.5
corr_model = cov_model/(pow_model_tot[:,:,None]*pow_model_tot[:,None,:])**0.5
# And write
hfile["full/corr"] = corr_full
hfile["full/tpow"] = pow_full_tot
hfile["full/upow"] = pow_full_ucorr
hfile["model/corr"] = corr_model
hfile["model/tpow"] = pow_model_tot
hfile["model/upow"] = pow_model_ucorr
t = np.array(t)
dt= t[1:]-t[:-1]
except (errors.DataMissing, ValueError, errors.ModelError, AssertionError, np.linalg.LinAlgError) as e:
print "%3d/%d %25s skip (%s)" % (i+1,n,id, e.message)
#print entry
return enmap.samewcs(map[utils.transpose_inds(dets, nrow, ncol)], map)
# Read our map, and give each row a weight
pickup = enmap.read_map(args.pickup_map)
pickup = reorder(pickup, nrow, ncol, d.dets)
weight = np.median((pickup[:, 1:] - pickup[:, :-1])**2, -1)
weight[weight > 0] = 1 / weight[weight > 0]
# Find the output pixel for each input pixel
baz = pickup[:1].posmap()[1, 0]
bel = baz * 0 + args.el * utils.degree
ipoint = np.array([baz, bel])
opoint = ipoint[:, None, :] + d.point_offset.T[:, :, None]
opix = template.sky2pix(opoint[::-1]).astype(int) # [{y,x},ndet,naz]
opix = np.rollaxis(opix, 1) # [ndet,{y,x},naz]
omap = enmap.zeros((3, ) + template.shape[-2:], template.wcs)
odiv = enmap.zeros((3, 3) + template.shape[-2:], template.wcs)
for det in range(d.ndet):
omap += utils.bin_multi(opix[det], template.shape[-2:], weight[det] *
pickup[det]) * d.det_comps[det, :, None, None]
odiv += utils.bin_multi(
opix[det], template.shape[-2:], weight[det]) * d.det_comps[
det, :, None, None, None] * d.det_comps[det, None, :, None, None]
odiv = enmap.samewcs(array_ops.eigpow(odiv, -1, axes=[0, 1]), odiv)
omap = enmap.samewcs(array_ops.matmul(odiv, omap, axes=[0, 1]), omap)
enmap.write_map(args.ofile, omap)
tod *= ivar[:,None]
sampcut.gapfill_const(scan.cut, tod, inplace=True)
for sid in tod_srcs[id]:
src = srcs[sid]
if src.type == "fixed": sys = "hor:%.6f_%.6f:cel/0_0:hor" % (src.ra/utils.degree, src.dec/utils.degree)
elif src.type == "planet": sys = "hor:%s/0_0" % src.name
else: raise ValueError("Invalid source type '%s'" % src.type)
rhs = enmap.zeros((ncomp,)+shape, area.wcs, dtype)
div = enmap.zeros((ncomp,ncomp)+shape, area.wcs, dtype)
with bench.show("pmat %s" % sid):
pmap = pmat.PmatMap(scan, area, sys=sys)
with bench.show("rhs %s" % sid):
pmap.backward(tod, rhs)
with bench.show("hits"):
for i in range(ncomp):
div[i,i] = 1
pmap.forward(tod, div[i])
tod *= ivar[:,None]
sampcut.gapfill_const(scan.cut, tod, inplace=True)
div[i] = 0
pmap.backward(tod, div[i])
with bench.show("map %s" % sid):
idiv = array_ops.eigpow(div, -1, axes=[0,1], lim=1e-5, fallback="scalar")
map = enmap.map_mul(idiv, rhs)
with bench.show("write"):
enmap.write_map("%s%s_src%03d_map.fits" % (prefix, bid, sid), map)
enmap.write_map("%s%s_src%03d_rhs.fits" % (prefix, bid, sid), rhs)
enmap.write_map("%s%s_src%03d_div.fits" % (prefix, bid, sid), div)
del rhs, div, idiv, map
del d, scan, pmap, tod
rhs = enmap.zeros((ncomp,)+shape, area.wcs, dtype)
div = enmap.zeros((ncomp,ncomp)+shape, area.wcs, dtype)
junk = np.zeros(pcut.njunk, dtype)
with bench.show("rhs"):
tod *= ivar[:,None]
pcut.backward(tod, junk)
pmap.backward(tod, rhs)
with bench.show("hits"):
for i in range(ncomp):
div[i,i] = 1
pmap.forward(tod, div[i])
tod *= ivar[:,None]
pcut.backward(tod, junk)
div[i] = 0
pmap.backward(tod, div[i])
with bench.show("map"):
idiv = array_ops.eigpow(div, -1, axes=[0,1], lim=1e-5)
map = enmap.map_mul(idiv, rhs)
# Estimate central amplitude
c = np.array(map.shape[-2:])/2
crad = 50
mcent = map[:,c[0]-crad:c[0]+crad,c[1]-crad:c[1]+crad]
mcent = enmap.downgrade(mcent, 4)
amp = np.max(mcent)
print "%s amp %7.3f asens %7.3f" % (id, amp/1e6, asens)
with bench.show("write"):
enmap.write_map("%s%s_map.fits" % (prefix, bid), map)
enmap.write_map("%s%s_rhs.fits" % (prefix, bid), rhs)
enmap.write_map("%s%s_div.fits" % (prefix, bid), div)
del d, scan, pmap, pcut, tod, map, rhs, div, idiv, junk
d.boresight[2] = args.el # In degrees, calibrated in next step
d = actdata.calibrate(d, exclude=["autocut"])
def reorder(map, nrow, ncol, dets):
return enmap.samewcs(map[utils.transpose_inds(dets,nrow,ncol)],map)
# Read our map, and give each row a weight
pickup = enmap.read_map(args.pickup_map)
pickup = reorder(pickup, nrow, ncol, d.dets)
weight = np.median((pickup[:,1:]-pickup[:,:-1])**2,-1)
weight[weight>0] = 1/weight[weight>0]
# Find the output pixel for each input pixel
baz = pickup[:1].posmap()[1,0]
bel = baz*0 + args.el * utils.degree
ipoint = np.array([baz,bel])
opoint = ipoint[:,None,:] + d.point_offset.T[:,:,None]
opix = template.sky2pix(opoint[::-1]).astype(int) # [{y,x},ndet,naz]
opix = np.rollaxis(opix, 1) # [ndet,{y,x},naz]
omap = enmap.zeros((3,)+template.shape[-2:], template.wcs)
odiv = enmap.zeros((3,3)+template.shape[-2:], template.wcs)
for det in range(d.ndet):
omap += utils.bin_multi(opix[det], template.shape[-2:], weight[det]*pickup[det]) * d.det_comps[det,:,None,None]
odiv += utils.bin_multi(opix[det], template.shape[-2:], weight[det]) * d.det_comps[det,:,None,None,None] * d.det_comps[det,None,:,None,None]
odiv = enmap.samewcs(array_ops.eigpow(odiv, -1, axes=[0,1]), odiv)
omap = enmap.samewcs(array_ops.matmul(odiv, omap, axes=[0,1]), omap)
enmap.write_map(args.ofile, omap)
