def __init__(self, icov, rhs, dof): self.Ai = icov self.b = rhs self.dof = dof if icov.size > 0: self.A = array_ops.eigpow(icov, -1.0) self.Aih = array_ops.eigpow(icov, 0.5) self.Ah = array_ops.eigpow(icov, -0.5) _,self.ldet = np.linalg.slogdet(self.Ai) self.x = self.A.dot(rhs) else: self.A, self.Aih, self.Ah = [icov.copy()]*3 self.x, self.ldet = rhs.copy(), 0
def get_samples(self): # Start with the noise if hasattr(self, "_tod") and self._tod is not None: return self._tod.copy() np.random.seed(self.seed) tod = np.zeros([self.ndet,self.nsamp]).astype(np.float32) if self.noise_scale != 0: tod = np.random.standard_normal([self.ndet,self.nsamp]).astype(np.float32)*self.noise_scale covs = array_ops.eigpow(self.noise.icovs, -0.5, axes=[-2,-1]) N12 = nmat.NmatBinned(covs, self.noise.bins, self.noise.dets) N12.apply(tod) else: tod[...] = 0 tod = tod.astype(np.float64) # And add the point sources for di in range(self.ndet): for i, (pos,amp,beam) in enumerate(zip(self.srcs.pos,self.srcs.amps,self.srcs.beam)): point = (self.boresight+self.offsets[di,None,:])[:,1:] point = coordinates.transform(self.sys, self.simsys, point.T, time=self.boresight[:,0]+self.mjd0, site=self.site).T r2 = np.sum((point-pos[None,:])**2,1)/beam**2 I = np.where(r2 < self.nsigma**2)[0] tod[di,I] += np.exp(-0.5*r2[I])*np.sum(amp*self.comps[di]) if hasattr(self, "_tod"): self._tod = tod.copy() return tod
def build_iN_constcorr_prior(data, cmb=None, lknee0=2000, ivars=None, constcov=False): if ivars is None: ivars = data.ivars N = enmap.zeros((data.n, data.n) + data.maps.shape[-2:], data.maps.wcs, data.maps.dtype) ref = np.mean(ivars, (-2, -1)) ref = np.maximum(ref, np.max(ref) * 1e-4) norm = 1 / (ref / ivars.pixsize()) for i, freq in enumerate(data.freqs): lknee = lknee0 * freq / 100 N[i, i] = (1 + (np.maximum(0.5, data.l) / lknee)**-3.5) * norm[i] # Deconvolve the pixel window from the theoretical flat-at-high-l spectrum N = N / data.wy[:, None]**2 / data.wx[None, :]**2 # Apply the beam-convolved cmb if available if cmb is not None: Bf = data.fconvs[:, None, None] * data.beams N += cmb * Bf[:, None] * Bf[None, :] del Bf if not constcov: N /= norm[:, None, None, None]**0.5 * norm[None, :, None, None]**0.5 iN = array_ops.eigpow(N, -1, axes=[0, 1]) return iN
def solve(w,m): if w.ndim < 4: return m/w elif w.ndim == 4: # This is slower, but handles low-hit areas near the edge better iw = array_ops.eigpow(w,-1,axes=[0,1]) return enmap.samewcs(array_ops.matmul(iw,m,axes=[0,1]), m) #return array_ops.solve_masked(w,m,axes=[0,1]) else: raise NotImplementedError("Only 2d, 3d or 4d weight maps understood")
def safe_invert(div): if div.shape[-1] == 1: idiv = div.copy() idiv[div==0] = 1 idiv[div!=0] = 1/idiv[div!=0] else: idiv = array_ops.eigpow(div, -1, axes=[-2,-1], lim=1e-3, fallback="scalar") return idiv
def get_covsqrt(ps, method="arrayops"): if method == "multipow": covsq = enmap.multi_pow(ps.copy(), 0.5) elif method == "arrayops": from enlib import array_ops covsq = array_ops.eigpow(ps.copy(), 0.5, axes=[0, 1]) covsq[:, :, ps.modlmap() < 2] = 0 assert np.all(np.isfinite(covsq)) return enmap.enmap(covsq, ps.wcs)
def get_samples(self): np.random.seed(self.seed) tod = np.zeros([self.ndet,self.nsamp]).astype(self.map.dtype) self.pmat.forward(tod, self.map) if self.noise_scale: noise = np.random.standard_normal([self.ndet,self.nsamp]).astype(self.map.dtype)*self.noise_scale covs = array_ops.eigpow(self.noise.icovs, -0.5, axes=[-2,-1]) N12 = nmat.NmatBinned(covs, self.noise.bins, self.noise.dets) N12.apply(noise) tod += noise return tod
with bench.show("hits"): for i in range(ncomp): div[i, i] = 1 pmap.forward(tod, div[i]) # div -> tod. tod_ones = tod.copy() tod *= ivar[:, None] pcut.backward(tod, junk) # tod -> junk. pcut.backward(tod_ones, junk) # tod -> junk. div[i] = 0 pmap.backward(tod, div[i]) # tod -> div. pmap.backward(tod_ones, hits[i]) # tod -> hits. with bench.show("map"): idiv = array_ops.eigpow(div, -1, axes=[0, 1], lim=1e-5, fallback="scalar") map = enmap.map_mul(idiv, rhs) # Estimate central amplitude c = np.array(map.shape[-2:]) // 2 crad = 50 mcent = map[:, c[0] - crad:c[0] + crad, c[1] - crad:c[1] + crad] mcent = enmap.downgrade(mcent, 4) amp = np.max(mcent) print("%s amp %7.3e asens %7.3e" % (id, amp / 1e6, asens)) with bench.show("write"): # Store maps in individual directories with id as name. outdir = opj(prefix, entry.id) if not os.path.isdir(outdir):
rhs = enmap.zeros((ncomp,)+shape, area.wcs, dtype) div = enmap.zeros((ncomp,ncomp)+shape, area.wcs, dtype) junk = np.zeros(pcut.njunk, dtype) with bench.show("rhs"): tod *= ivar[:,None] pcut.backward(tod, junk) pmap.backward(tod, rhs) with bench.show("hits"): for i in range(ncomp): div[i,i] = 1 pmap.forward(tod, div[i]) tod *= ivar[:,None] pcut.backward(tod, junk) div[i] = 0 pmap.backward(tod, div[i]) with bench.show("map"): idiv = array_ops.eigpow(div, -1, axes=[0,1], lim=1e-5) map = enmap.map_mul(idiv, rhs) # Estimate central amplitude c = np.array(map.shape[-2:])/2 crad = 50 mcent = map[:,c[0]-crad:c[0]+crad,c[1]-crad:c[1]+crad] mcent = enmap.downgrade(mcent, 4) amp = np.max(mcent) print("%s amp %7.3f asens %7.3f" % (id, amp/1e6, asens)) with bench.show("write"): enmap.write_map("%s%s_map.fits" % (prefix, bid), map) enmap.write_map("%s%s_rhs.fits" % (prefix, bid), rhs) enmap.write_map("%s%s_div.fits" % (prefix, bid), div) del d, scan, pmap, pcut, tod, map, rhs, div, idiv, junk
# Build div, which we need in both cases div = enmap.zeros((ncomp,)+area.shape,area.wcs,dtype) for i in range(ncomp): work = div[0]*0 work[i] = 1 for scan in scans: tod = np.zeros((scan.ndet,scan.nsamp),dtype) scan.pmap.forward(tod, work) if args.method == "cg": scan.noise.white(tod) else: tod /= scan.T scan.pcut.backward(tod, np.zeros(scan.pcut.njunk,dtype)) scan.pmap.backward(tod, div[i]) div = utils.allreduce(div, comm) #idiv = utils.eigpow(div,-1,[0,1]) idiv = array_ops.eigpow(div,-1,[0,1], lim=1e-6) if comm.rank == 0: enmap.write_map(args.odir + "/map_div.fits", div) enmap.write_map(args.odir + "/map_idiv.fits", idiv) del work, div # And the same for junk jdiv = np.full(njunk_tot, 1.0, dtype) for scan in scans: tod = np.zeros((scan.ndet,scan.nsamp),dtype) scan.pcut.forward(tod, jdiv[scan.cut_range[0]:scan.cut_range[1]]) if args.method == "cg": scan.noise.white(tod) else: tod /= scan.T scan.pcut.backward(tod, jdiv[scan.cut_range[0]:scan.cut_range[1]]) del tod
def pow(mat, exp, axes=[0, 1]): return enmap.samewcs(array_ops.eigpow(mat, exp, axes=axes), mat, exp)
def getamps(W, V, d): rhs = (V.T*W[None]).dot(d) A = (V.T*W[None]).dot(V) iA = array_ops.eigpow(A,-1) return iA.dot(rhs)
def pow(mat, exp, axes): return array_ops.eigpow(mat, exp, axes=axes)
ndet = ft.shape[0] bins = np.minimum( (noise.bins * ft.shape[1] / noise.bins[-1, 1]).astype(int), ft.shape[1] - 1) nbin = len(bins) cov_full = np.zeros([nbin, ndet, ndet]) for bi, b in enumerate(bins): print "A", bi, b, np.mean(np.abs(ft[0, b[0]:b[1]])** 2)**0.5 / 20 cov_full[bi] = nmat_measure.measure_cov(ft[:, b[0]:b[1]]) cov_model = noise.covs # Compute total noise and correlated noise per detector for the two pow_full_tot = np.einsum("bii->bi", cov_full) pow_model_tot = np.einsum("bii->bi", cov_model) pow_full_ucorr = 1 / np.einsum("bii->bi", array_ops.eigpow(cov_full, -1)) pow_model_ucorr = 1 / np.einsum( "bii->bi", array_ops.eigpow(cov_model, -1)) corr_full = cov_full / (pow_full_tot[:, :, None] * pow_full_tot[:, None, :])**0.5 corr_model = cov_model / (pow_model_tot[:, :, None] * pow_model_tot[:, None, :])**0.5 # And write hfile["full/corr"] = corr_full hfile["full/tpow"] = pow_full_tot hfile["full/upow"] = pow_full_ucorr hfile["model/corr"] = corr_model hfile["model/tpow"] = pow_model_tot hfile["model/upow"] = pow_model_ucorr t = np.array(t) dt = t[1:] - t[:-1]
div = enmap.zeros((ncomp, ) + area.shape, area.wcs, dtype) for i in range(ncomp): work = div[0] * 0 work[i] = 1 for scan in scans: tod = np.zeros((scan.ndet, scan.nsamp), dtype) scan.pmap.forward(tod, work) if args.method == "cg": scan.noise.white(tod) else: tod /= scan.T scan.pcut.backward(tod, np.zeros(scan.pcut.njunk, dtype)) scan.pmap.backward(tod, div[i]) div = utils.allreduce(div, comm) #idiv = utils.eigpow(div,-1,[0,1]) idiv = array_ops.eigpow(div, -1, [0, 1], lim=1e-6) if comm.rank == 0: enmap.write_map(args.odir + "/map_div.fits", div) enmap.write_map(args.odir + "/map_idiv.fits", idiv) del work, div # And the same for junk jdiv = np.full(njunk_tot, 1.0, dtype) for scan in scans: tod = np.zeros((scan.ndet, scan.nsamp), dtype) scan.pcut.forward(tod, jdiv[scan.cut_range[0]:scan.cut_range[1]]) if args.method == "cg": scan.noise.white(tod) else: tod /= scan.T scan.pcut.backward(tod, jdiv[scan.cut_range[0]:scan.cut_range[1]]) del tod
with h5py.File("%s/%s.hdf" % (args.odir, id),"w") as hfile: nmat.write_nmat(hfile, noise) ; t.append(time.time()) if args.covtest: # Measure full cov per bin ndet = ft.shape[0] bins = np.minimum((noise.bins*ft.shape[1]/noise.bins[-1,1]).astype(int),ft.shape[1]-1) nbin = len(bins) cov_full = np.zeros([nbin,ndet,ndet]) for bi, b in enumerate(bins): print "A", bi, b, np.mean(np.abs(ft[0,b[0]:b[1]])**2)**0.5/20 cov_full[bi] = nmat_measure.measure_cov(ft[:,b[0]:b[1]]) cov_model= noise.covs # Compute total noise and correlated noise per detector for the two pow_full_tot = np.einsum("bii->bi",cov_full) pow_model_tot = np.einsum("bii->bi",cov_model) pow_full_ucorr = 1/np.einsum("bii->bi",array_ops.eigpow(cov_full,-1)) pow_model_ucorr = 1/np.einsum("bii->bi",array_ops.eigpow(cov_model,-1)) corr_full = cov_full/(pow_full_tot[:,:,None]*pow_full_tot[:,None,:])**0.5 corr_model = cov_model/(pow_model_tot[:,:,None]*pow_model_tot[:,None,:])**0.5 # And write hfile["full/corr"] = corr_full hfile["full/tpow"] = pow_full_tot hfile["full/upow"] = pow_full_ucorr hfile["model/corr"] = corr_model hfile["model/tpow"] = pow_model_tot hfile["model/upow"] = pow_model_ucorr t = np.array(t) dt= t[1:]-t[:-1] except (errors.DataMissing, ValueError, errors.ModelError, AssertionError, np.linalg.LinAlgError) as e: print "%3d/%d %25s skip (%s)" % (i+1,n,id, e.message) #print entry
return enmap.samewcs(map[utils.transpose_inds(dets, nrow, ncol)], map) # Read our map, and give each row a weight pickup = enmap.read_map(args.pickup_map) pickup = reorder(pickup, nrow, ncol, d.dets) weight = np.median((pickup[:, 1:] - pickup[:, :-1])**2, -1) weight[weight > 0] = 1 / weight[weight > 0] # Find the output pixel for each input pixel baz = pickup[:1].posmap()[1, 0] bel = baz * 0 + args.el * utils.degree ipoint = np.array([baz, bel]) opoint = ipoint[:, None, :] + d.point_offset.T[:, :, None] opix = template.sky2pix(opoint[::-1]).astype(int) # [{y,x},ndet,naz] opix = np.rollaxis(opix, 1) # [ndet,{y,x},naz] omap = enmap.zeros((3, ) + template.shape[-2:], template.wcs) odiv = enmap.zeros((3, 3) + template.shape[-2:], template.wcs) for det in range(d.ndet): omap += utils.bin_multi(opix[det], template.shape[-2:], weight[det] * pickup[det]) * d.det_comps[det, :, None, None] odiv += utils.bin_multi( opix[det], template.shape[-2:], weight[det]) * d.det_comps[ det, :, None, None, None] * d.det_comps[det, None, :, None, None] odiv = enmap.samewcs(array_ops.eigpow(odiv, -1, axes=[0, 1]), odiv) omap = enmap.samewcs(array_ops.matmul(odiv, omap, axes=[0, 1]), omap) enmap.write_map(args.ofile, omap)
tod *= ivar[:,None] sampcut.gapfill_const(scan.cut, tod, inplace=True) for sid in tod_srcs[id]: src = srcs[sid] if src.type == "fixed": sys = "hor:%.6f_%.6f:cel/0_0:hor" % (src.ra/utils.degree, src.dec/utils.degree) elif src.type == "planet": sys = "hor:%s/0_0" % src.name else: raise ValueError("Invalid source type '%s'" % src.type) rhs = enmap.zeros((ncomp,)+shape, area.wcs, dtype) div = enmap.zeros((ncomp,ncomp)+shape, area.wcs, dtype) with bench.show("pmat %s" % sid): pmap = pmat.PmatMap(scan, area, sys=sys) with bench.show("rhs %s" % sid): pmap.backward(tod, rhs) with bench.show("hits"): for i in range(ncomp): div[i,i] = 1 pmap.forward(tod, div[i]) tod *= ivar[:,None] sampcut.gapfill_const(scan.cut, tod, inplace=True) div[i] = 0 pmap.backward(tod, div[i]) with bench.show("map %s" % sid): idiv = array_ops.eigpow(div, -1, axes=[0,1], lim=1e-5, fallback="scalar") map = enmap.map_mul(idiv, rhs) with bench.show("write"): enmap.write_map("%s%s_src%03d_map.fits" % (prefix, bid, sid), map) enmap.write_map("%s%s_src%03d_rhs.fits" % (prefix, bid, sid), rhs) enmap.write_map("%s%s_src%03d_div.fits" % (prefix, bid, sid), div) del rhs, div, idiv, map del d, scan, pmap, tod
rhs = enmap.zeros((ncomp,)+shape, area.wcs, dtype) div = enmap.zeros((ncomp,ncomp)+shape, area.wcs, dtype) junk = np.zeros(pcut.njunk, dtype) with bench.show("rhs"): tod *= ivar[:,None] pcut.backward(tod, junk) pmap.backward(tod, rhs) with bench.show("hits"): for i in range(ncomp): div[i,i] = 1 pmap.forward(tod, div[i]) tod *= ivar[:,None] pcut.backward(tod, junk) div[i] = 0 pmap.backward(tod, div[i]) with bench.show("map"): idiv = array_ops.eigpow(div, -1, axes=[0,1], lim=1e-5) map = enmap.map_mul(idiv, rhs) # Estimate central amplitude c = np.array(map.shape[-2:])/2 crad = 50 mcent = map[:,c[0]-crad:c[0]+crad,c[1]-crad:c[1]+crad] mcent = enmap.downgrade(mcent, 4) amp = np.max(mcent) print "%s amp %7.3f asens %7.3f" % (id, amp/1e6, asens) with bench.show("write"): enmap.write_map("%s%s_map.fits" % (prefix, bid), map) enmap.write_map("%s%s_rhs.fits" % (prefix, bid), rhs) enmap.write_map("%s%s_div.fits" % (prefix, bid), div) del d, scan, pmap, pcut, tod, map, rhs, div, idiv, junk
d.boresight[2] = args.el # In degrees, calibrated in next step d = actdata.calibrate(d, exclude=["autocut"]) def reorder(map, nrow, ncol, dets): return enmap.samewcs(map[utils.transpose_inds(dets,nrow,ncol)],map) # Read our map, and give each row a weight pickup = enmap.read_map(args.pickup_map) pickup = reorder(pickup, nrow, ncol, d.dets) weight = np.median((pickup[:,1:]-pickup[:,:-1])**2,-1) weight[weight>0] = 1/weight[weight>0] # Find the output pixel for each input pixel baz = pickup[:1].posmap()[1,0] bel = baz*0 + args.el * utils.degree ipoint = np.array([baz,bel]) opoint = ipoint[:,None,:] + d.point_offset.T[:,:,None] opix = template.sky2pix(opoint[::-1]).astype(int) # [{y,x},ndet,naz] opix = np.rollaxis(opix, 1) # [ndet,{y,x},naz] omap = enmap.zeros((3,)+template.shape[-2:], template.wcs) odiv = enmap.zeros((3,3)+template.shape[-2:], template.wcs) for det in range(d.ndet): omap += utils.bin_multi(opix[det], template.shape[-2:], weight[det]*pickup[det]) * d.det_comps[det,:,None,None] odiv += utils.bin_multi(opix[det], template.shape[-2:], weight[det]) * d.det_comps[det,:,None,None,None] * d.det_comps[det,None,:,None,None] odiv = enmap.samewcs(array_ops.eigpow(odiv, -1, axes=[0,1]), odiv) omap = enmap.samewcs(array_ops.matmul(odiv, omap, axes=[0,1]), omap) enmap.write_map(args.ofile, omap)