def __init__(self, data, srcpos, srcamp, perdet=False, thumbs=False, N=None, method="fixamp"):
# Set up fiducial source model. These source parameters
# are not the same as those we will be optimizing.
with bench.show("PmatTot"):
self.P = PmatTot(data, srcpos, perdet=perdet)
with bench.show("NmatTot"):
self.N = N if N else NmatTot(data)
self.tod = data.tod # might only need the one below
with bench.show("Nmat apply"):
self.Nd = self.N.apply(self.tod.copy())
self.i = 0
# Initial values
self.amp0 = srcamp[:,None]
self.off0 = self.P.off0
self.chisq0 = None
# These are for internal mapmaking
self.thumb_mapper = None
if thumbs:
with bench.show("ThumbMapper"):
self.thumb_mapper = ThumbMapper(data, srcpos, self.P.pcut, self.N.nmat, perdet=perdet)
self.amp_unit, self.off_unit = 1e3, utils.arcmin
# Save samples from the wrapper, so we can use them to estimate uncertainty
self.samples = bunch.Bunch(offs=[], amps=[], aicovs=[], chisqs=[])
self.method = method
nside = args.nside
smooth_deg = args.smooth_deg
ch = SOChannel('LA',args.freq)
lmin = args.lmin
lmax = args.lmax
polcomb = args.polcomb
config = io.config_from_yaml("../input/config.yml")
mask = initialize_mask(nside,smooth_deg)
solint = SOLensInterface(mask)
thloc = "../data/" + config['theory_root']
theory = cosmology.loadTheorySpectraFromCAMB(thloc,get_dimensionless=False)
# norm dict
Als = {}
with bench.show("norm"):
ls,Als['TT'],Als['EE'],Als['EB'],Als['TE'],Als['TB'],al_mv_pol,al_mv,Al_te_hdv = initialize_norm(solint,ch,lmin,lmax)
Als['mv'] = al_mv
Als['mvpol'] = al_mv_pol
al_mv = Als[polcomb]
# Wiener filter
nls = al_mv * ls**2./4.
tclkk = theory.gCl('kk',ls)
wfilt = tclkk/(tclkk+nls)/ls**2.
wfilt[ls<50] = 0
wfilt[ls>500] = 0
narrays = len(aids)
cov = maps.SymMat(narrays, eshape[-2:])
def save_fn(x, a1, a2):
cov[a1, a2] = enmap.enmap(x, ewcs).copy()
print(comm.rank, ": Tile %d has arrays " % i, aids)
anisotropic_pairs = pipeline.get_aniso_pairs(aids, hybrids, friends)
def stack(x):
return enmap.enmap(np.stack(x), ewcs)
kcoadds = stack(kcoadds)
masks = stack(masks)
with bench.show("cov"):
ilc.build_cov(
names=qids,
kdiffs=kdiffs,
kcoadds=kcoadds,
fbeam=fbeam,
mask=masks,
lmins=lmins,
lmaxs=lmaxs,
freqs=freqs,
anisotropic_pairs=anisotropic_pairs,
delta_ell=args.delta_ell,
do_radial_fit=do_radial_fit,
save_fn=save_fn,
signal_bin_width=args.signal_bin_width,
signal_interp_order=args.signal_interp_order,
"""
# You need to specify the mask version (for noise sims) first
# These are different for s16 and non-s16, sorry about that
version = 'v4.0_mask_version_mr3c_20190215_pickupsub_190301'
#version = 'v4.0_mask_version_mr3c_20190215_pickupsub_190303' # for s16
# our test data set
season, array, patch, freq = ('s13', 'pa1', 'deep1', 'f150')
#season, array, patch, freq = ('s15', 'pa3', 'boss', 'f150')
# We initialize the sim generator with the mask version
simgen = simgen.SimGen(version=version)
# We can then get just signal = cmb + fg (and loop over season,patch,array,sim_num after the above initialization)
with bench.show("signal"):
simgen.get_signal(season,
patch,
array,
freq,
sim_num=0,
fgflux='15mjy',
add_poisson_srcs=False)
# Or get just cmb
#simgen.get_cmb(season, patch, array, freq, sim_num= 0)
# Or get just foregrounds
#simgen.get_fg(season, patch, array, freq, sim_num=0)
# Or get just phi map
#simgen.get_phi(season, patch, array, freq, sim_num=0)
# Or get just kappa map
#simgen.get_kappa(season, patch, array, freq, sim_num=0)
from math import ceil
import multiprocessing
# number of coordinates to transform
N = 96000013
# this should be obtained using multiprocessing.get_cpu_count()
Njobs = 12
th = np.random.uniform(0, 90, N)
phi = np.random.uniform(0, 90, N)
shape, wcs = enmap.rect_geometry(100., 0.5)
coords = np.array([th, phi])
with bench.show("serial"):
pix = enmap.pix2sky(shape, wcs, coords)
def chunks(l, n):
"""Yield successive n-sized chunks from l."""
for i in range(0, l.shape[-1], n):
yield l[:, i:i + n]
size_chunks = int(ceil(coords.shape[1] * 1. / Njobs))
# print coords.shape
# for i,x in enumerate(chunks(coords,size_chunks)):
# print i, x.shape
# sys.exit()
def test_massfn():
from szar import counts
import hmf
from cluster_toolkit import massfunction
zs = np.linspace(0., 3., 20)
ms = np.geomspace(1e14, 1e17, 200)
ks = np.geomspace(1e-3, 10, 101)
from enlib import bench
with bench.show("init"):
hcos = hm.HaloModel(zs, ks, ms=ms, mass_function="tinker")
dndM_ct2 = np.zeros((zs.size, ms.size))
for i, z in enumerate(zs):
h = hmf.MassFunction(z=z,
Mmin=np.log10(ms.min() * hcos.h),
Mmax=np.log10(ms.max() * hcos.h))
if i == 0: dndM_ct = np.zeros((zs.size, h.dndm.size))
dndM_ct[i, :] = h.dndm.copy()
dndM_ct2[i, :] = massfunction.dndM_at_M(ms * hcos.h,
hcos.ks_sigma2 / hcos.h,
hcos.sPzk[i] * hcos.h**3,
hcos.om0)
fsky = 0.4
hmf = counts.Halo_MF(counts.ClusterCosmology(hcos.params, skipCls=True),
np.log10(ms), zs)
nz_szar = hmf.N_of_z() * fsky
print(nz_szar, nz_szar.shape)
# sys.exit()
print(hcos.nzm.shape, hcos.bh.shape)
bh = hcos.bh
nzm = hcos.nzm
# ims,ins = np.loadtxt("data/tinker2008Fig5.txt",unpack=True,delimiter=',')
# pl = io.Plotter(xyscale='linlin')
# pl.add(ims,ins,ls="--")
# pl.add(np.log10(ms*hcos.h),np.log10(nzm[0,:]*ms**2./hcos.rho_matter_z(0.)))
# pl.done()
chis = hcos.results.angular_diameter_distance(hcos.zs) * (1 + hcos.zs)
nz = np.trapz(nzm, ms,
axis=-1) * 4. * np.pi * chis**2. / hcos.results.h_of_z(
hcos.zs) * fsky
nz_ct = np.trapz(dndM_ct, h.m,
axis=-1) * 4. * np.pi * chis**2. / hcos.results.h_of_z(
hcos.zs) * fsky * hcos.h**3.
nz_ct2 = np.trapz(dndM_ct2, ms,
axis=-1) * 4. * np.pi * chis**2. / hcos.results.h_of_z(
hcos.zs) * fsky * hcos.h**3.
pl = io.Plotter()
pl.add(zs, nz, label='hmvec')
pl.add(hmf.zarr, nz_szar, ls='--', label='szar')
pl.add(zs, nz_ct, ls='-.', label='hmf')
pl.add(zs, nz_ct2, ls='-.', label='ct')
pl.done()
n = np.trapz(nz, zs)
print(n)
n = np.trapz(nz_szar, hmf.zarr)
print(n)
n = np.trapz(nz_ct, zs)
print(n)
n = np.trapz(nz_ct2, zs)
print(n)
tspecs[1,i] = np.percentile(dhigh,15.86553,0)
tspecs[2,i] = np.percentile(dhigh,84.13447,0)
tspecs[3,i] = np.min(dhigh,0)
tspecs[4,i] = np.max(dhigh,0)
tspecs[5,i] = np.mean(dhigh,0)
tspecs[6,i] = bin(np.abs(np.mean(ft,0))**2, args.nbin)
del ps
# Normalize ft in bins, since we want correlations
for di in range(d.ndet):
ft[di] /= (dhigh[di]**0.5)[binds]
# Average correlation in bin
sps = np.abs(np.sum(ft,0))**2
tcorrs[i] = (bin(sps, args.nbin)-d.ndet)/(d.ndet**2-d.ndet)
del sps, ft, d
# Ok, we've gone through all the data in our chunk
with bench.show("Reduce"):
dspecs = utils.allreduce(dspecs, comm)
dzooms = utils.allreduce(dzooms, comm)
tspecs = utils.allreduce(tspecs, comm)
tcorrs = utils.allreduce(tcorrs, comm)
srates = utils.allreduce(srates, comm)
nhits = utils.allreduce(nhits, comm)
mce_fsamps = utils.allreduce(mce_fsamps, comm)
mce_params = utils.allreduce(mce_params, comm)
if comm.rank == 0:
# Get rid of empty tods
good = np.where(np.any(dspecs>0,(1,2)))[0]
if len(good) == 0:
print("No usable tods in chunk!")
continue
dspecs = dspecs[good]
if rank==0: print("Rank 0 starting ...")
for k,my_task in enumerate(my_tasks):
kamp = kamps[my_task]
kappa_template = lensing.nfw_kappa(kamp*1e15,bmodrmap,cc,overdensity=200.,critical=True,atClusterZ=True)
phi,_ = lensing.kappa_to_phi(kappa_template,bmodlmap,return_fphi=True)
grad_phi = enmap.grad(phi)
pos = posmap + grad_phi
alpha_pix = enmap.sky2pix(bshape,bwcs,pos, safe=False)
def do_the_thing():
return lensing.lens_cov(Ucov,alpha_pix,lens_order=lens_order,kbeam=kbeam,bshape=shape)
if rank==0:
with bench.show("rank 0 lensing cov"):
Scov = do_the_thing()
else:
Scov = do_the_thing()
np.save(cov_name(my_task),Scov)
if rank==0:
io.save_cols(GridName+"/amps.txt",(kamps,))
import json
save_dict = {"arc":args.arc,"pix":args.pix,"beam":args.beam}
with open(GridName+"/attribs.json",'w') as f:
f.write(json.dumps(save_dict))
nsigma)
msg += " %12.5e %7.2f" % (self.chisq0 - chisq, t2 - t1)
print(msg)
self.i += 1
return chisq
return wrapper
for ind in range(comm.rank, len(ids), comm.size):
id = ids[ind]
bid = id.replace(":", "_")
entry = filedb.data[id]
# Read the tod as usual
try:
with bench.show("read"):
d = actdata.read(entry)
with bench.show("calibrate"):
d = actdata.calibrate(d, exclude=["autocut"])
if d.ndet == 0 or d.nsamp < 2:
raise errors.DataMissing("no data in tod")
except errors.DataMissing as e:
print("Skipping %s (%s)" % (id, e))
continue
print("Processing %s" % id)
# Very simple white noise model
with bench.show("ivar"):
tod = d.tod
del d.tod
tod -= np.mean(tod, 1)[:, None]
tod = tod.astype(dtype)
from __future__ import print_function
from orphics import maps, io, cosmology, stats
from pixell import enmap, lensing, curvedsky as cs, utils
import numpy as np
import os, sys
from enlib import bench
shape, wcs = enmap.fullsky_geometry(res=1.0 * utils.arcmin)
lmax = 8000
theory = cosmology.default_theory()
ps = cosmology.enmap_power_from_orphics_theory(theory, lensed=False)
with bench.show('lens'):
uTquMap, lTquMap, pMap = lensing.rand_map((3, ) + shape,
wcs,
ps,
lmax=lmax,
output="ulp",
verbose=True,
phi_seed=0,
seed=0,
dtype=np.float32,
delta_theta=60 * utils.degree)
"""
dtheta = 10 deg : 273 s , 21 GB
dtheta = 20 deg : 271 s , 22 GB
dtheta = 40 deg : 268 s , 26 GB
dtheta = 60 deg : 267 s , 28 GB
OMP_NUM_THREADS=10
dtheta = 10 deg :
def make_circular_geometry(shape,
wcs,
context_arcmin,
hole_arcmin,
power2d,
buffer_factor=2,
verbose=False):
'''Makes the circular geometry matrices that need to be pre-calculated for later inpainting.
Arguments
---------
input2DPower - ndarray containing 2D power spectrum of a map. It need not already have been
downsampled to the shape of the stamp cutout
inputLy
inputLx - the fourier wavenumbers corresponding to the y and x axes of the stamp cutout
stampArc - the width in arcminutes of the stamp cut out
stampPxX - the pixel width in arcminutes of the stamp cut out in the x direction
stampPxY - the pixel width in arcminutes of the stamp cut out in the y direction
holeArc - the radius of the circular hole in arcminutes
bufferFactor - the pixel covariance matrix will be calculated on a periodic stamp larger by this
factor
verbose - True if you want more commentary
Returns
-------
meanMul - a matrix that has shape (nh,nc) where nh is the number of pixels in the hole and
nc is the number of pixels outside (in the "context"). It should be multiplied
by a vector (nc) containing pixels outside to get a vector (nh) for the mean
value of the pixels inside
covRoot - a (nh,nh) sqrt(covariance matrix) that can be used to generate a random realization
in the hole. This can be generated by multiplying the sqrt of cov by a vector (nh)
of standard normal variables. The generated vector should be added to the mean value
obtained using meanMul
pcov - the pixel-pixel covariance matrix used in intermediate steps, if you want to re-use it
targetTemplate - a liteMap template of the stamp cutout
m1 - a boolean array that can be used to select the hole region
m2 - a boolean array that can be used to select the context region
'''
arc = context_arcmin
res = maps.resolution(shape, wcs) * 60. * 180. / np.pi
bshape, bwcs = maps.rect_geometry(width_arcmin=arc * buffer_factor,
px_res_arcmin=res)
tshape, twcs = maps.rect_geometry(width_arcmin=arc, px_res_arcmin=res)
sny, snx = tshape
bmodlmap = enmap.modlmap(bshape, bwcs)
modlmap = enmap.modlmap(shape, wcs)
if verbose: print("Downsampling...")
Niy, Nix = shape[-2:]
Noy, Nox = bshape[-2:]
# print(bshape,tshape,power2d.shape)
# io.plot_img(np.fft.fftshift(np.log10(power2d)))
#out_power = resample.resample_fft(power2d,bshape[-2:],axes=[-2,-1])
#out_power = np.fft.ifftshift(resample.resample_fft(np.fft.fftshift(power2d),bshape[-2:],axes=[-2,-1]))
out_power = resample.resample_bin(
power2d, factors=[float(Noy) / Niy, float(Nox) / Nix], axes=[-2, -1])
# io.plot_img(np.fft.fftshift(np.log10(out_power)))
# print(out_power.shape)
if verbose: print("Starting slow part...")
d = maps.diagonal_cov(out_power)
with bench.show("pixcov"):
pcov = maps.pixcov(bshape, bwcs, d)[0, 0, :sny, :snx, :sny, :snx]
modrmap = enmap.modrmap(tshape, twcs)
m1 = np.where(modrmap.reshape(-1) < hole_arcmin * np.pi / 180. / 60.)[0]
m2 = np.where(modrmap.reshape(-1) >= hole_arcmin * np.pi / 180. / 60.)[0]
with bench.show("geom"):
meanMul, cov = get_geometry(pcov.reshape(sny * snx, sny * snx), m1, m2)
covRoot = stats.eig_pow(cov, 0.5)
return meanMul, covRoot, pcov, tshape, twcs, m1, m2
kbeam = maps.gauss_beam(fwhm, mc.modlmap)
ells = np.arange(0, 3000, 1)
lbeam = maps.gauss_beam(fwhm, ells)
ntt = np.nan_to_num((noise_t * np.pi / 180. / 60.)**2. / kbeam**2.)
nee = np.nan_to_num((noise_p * np.pi / 180. / 60.)**2. / kbeam**2.)
nbb = np.nan_to_num((noise_p * np.pi / 180. / 60.)**2. / kbeam**2.)
lntt = np.nan_to_num((noise_t * np.pi / 180. / 60.)**2. / lbeam**2.)
lnee = np.nan_to_num((noise_p * np.pi / 180. / 60.)**2. / lbeam**2.)
lnbb = np.nan_to_num((noise_p * np.pi / 180. / 60.)**2. / lbeam**2.)
ellmin = 20
ellmax = 3000
xmask = maps.mask_kspace(shape, wcs, lmin=ellmin, lmax=ellmax)
ymask = xmask
with bench.show("ALcalc"):
AL = mc.AL(pol,
xmask,
ymask,
ntt,
nee,
nbb,
theory=theory,
hdv=hdv,
cache=cache)
val = mc.NL_from_AL(AL)
bin_edges = np.arange(10, 2000, 40)
cents, nkk = stats.bin_in_annuli(val, mc.modlmap, bin_edges)
ls, hunls = np.loadtxt("../alhazen/data/hu_" + pol.lower() + ".csv",
# assuming that the search space only contain one source
omgs = np.linspace(60,80,32)
# omgs = np.linspace(60,80,100) # update: 220209
phis = np.linspace(0,2*np.pi,32)
# nsrc = len(omgs) * len(phis)
# srcs = np.zeros(7, nsrc, dtype=np.float64)
# srcs[0,:] = srcpos[0,None]
# srcs[1,:] = srcpos[1,None]
# srcs[2,:] = 1
# srcs[5:7,:] = 1
# Note that it is important for amp to be 1 to be able to extract the response as its flux.
# srcs= np.array([[srcpos[0], srcpos[1], 1, 0, 0, o, p, D[0]] for o in omgs for p in phis]).T
# srcs= np.array([[srcpos[0], srcpos[1], 1, 0, 0, omg, phi0, D]]).T # use original pointing
srcs= np.array([[srcpos[0], srcpos[1], 1, 0, 0, 70.6, 6.13, D]]).T # use close-enough value
# srcs= np.array([[srcpos[0], srcpos[1]]])
with bench.show("create pointing matrix"):
P = lib.PmatTotVar(scan, srcs, perdet=False, sys=sys)
# P = lib.PmatTot(scan, srcs[:,0], perdet=False, sys=sys)
# I should be able to use the same pointing matrix to do a search
# of pulsars by treating different period and phases as different
# sources, and estimating their amplitudes together. Let me give
# that a try below
# a factor to convert uK to mJy
beam_area = lib.get_beam_area(scan.beam)
_, uids = actdata.split_detname(scan.dets) # Argh, stupid detnames
freq = scan.array_info.info.nom_freq[uids[0]]
fluxconv = u.flux_factor(beam_area, freq*1e9)/1e3
print("fluxconv = ", fluxconv)
try:
lensed = load("lensed", index)
unlensed = load("unlensed", index)
kappa = load("kappa", index)
if rank == 0:
print("Rank 0 successfully loaded saved files.")
lensed.wcs = wcs # WCS is being saved wrong?! See orphics/scripts/enlib-issue.ipynb
unlensed.wcs = wcs
kappa.wcs = wcs
except:
# if rank==0:
# traceback.print_exc()
with bench.show("lensing"):
lensed, kappa, unlensed = lensing.rand_map(
shape,
wcs,
ps,
lmax=lmax,
maplmax=maplmax,
seed=(seed, index),
verbose=True if rank == 0 else False,
dtype=dtype,
output="lku")
save("lensed", lensed, index)
save("unlensed", unlensed, index)
save("kappa", kappa, index)
def noise_block_average(n2d,nsplits,delta_ell,lmin=300,lmax=8000,wnoise_annulus=500,bin_annulus=20,
lknee_guess=3000,alpha_guess=-4,nparams=None,
verbose=False,radial_fit=True,fill_lmax=None,fill_lmax_width=100,log=True,
isotropic_low_ell=True,allow_low_wnoise=False):
"""Find the empirical mean noise binned in blocks of dfact[0] x dfact[1] . Preserves noise anisotropy.
Most arguments are for the radial fitting part.
A radial fit is divided out before downsampling (by default by FFT) and then multplied back with the radial fit.
Watch for ringing in the final output.
n2d noise power
n2d -- the [...,Ny,Nx] 2d power to smooth
nsplits -- the number of splits from which the 2d noise power was estimated. This needs to be known
if log is True, in which case the power is log-transformed before smoothing, which changes the statistics
of the samples and hence needs a pre-determined correction based on the distribution of the original sample.
log -- whether to log transform before smoothing. Should only be used if the power is positive (so should not
be used e.g. if this is for the cross-noise of two components)
delta_ell -- the block width in ell units for the smoothing. The smoothing effectively gets done in blocks
of delta_ell x delta_ell.
radial_fit -- if True, divides out a fit to the 1d power
lmin -- lmin for the radial fit
lmax -- lmax for the radial fit (adjust based on resolution of map)
wnoise_annulus -- width of annulus in ell within which to estimate high ell white noise (adjust based on resolution)
bin_annulus -- width of 1d bins (IMPORTANT: adjust based on map size)
lknee_guess -- guess lknee for fit
alpha_guess -- guess alpha for fit
nparams -- optionally pass in a radial fit's parameters
verbose -- print more
fill_lmax -- fill power outside this lmax with the mean of the annulus between fill_lmax and fill_lmax_width
fill_lmax_width -- see above
isotropic_low_ell -- fill below lmin with an isotropic fit to the 1d power
allow_low_wnoise -- allow white noise level to be negative (for debugging)
"""
assert np.all(np.isfinite(n2d))
if log: assert np.all(n2d>0), "You can't log smooth a PS with negative or zero power. Use log=False for these."
shape,wcs = n2d.shape,n2d.wcs
modlmap = n2d.modlmap()
minell = maps.minimum_ell(shape,wcs)
Ny,Nx = shape[-2:]
if radial_fit:
with bench.show("radial fit"):
if nparams is None:
if verbose: print("Radial fitting...")
nparams = fit_noise_1d(n2d,lmin=lmin,lmax=lmax,wnoise_annulus=wnoise_annulus,
bin_annulus=bin_annulus,lknee_guess=lknee_guess,alpha_guess=alpha_guess,
allow_low_wnoise=allow_low_wnoise)
wfit,lfit,afit = nparams
nfitted = rednoise(modlmap,wfit,lfit,afit)
else:
nparams = None
nfitted = n2d*0 + 1
nfitted = np.maximum(nfitted,np.max(n2d)*1e-14)
nflat = enmap.enmap(n2d/nfitted,wcs) # flattened 2d noise power
fval = nflat[np.logical_and(modlmap>2,modlmap<2*minell)].mean()
nflat[modlmap<2] = fval
if fill_lmax is not None:
fill_avg = nflat[np.logical_and(modlmap>(fill_lmax-fill_lmax_width),modlmap<=fill_lmax)].mean()
nflat[modlmap>fill_lmax] = fill_avg
if verbose: print("Resampling...")
assert np.all(np.isfinite(nflat))
with bench.show("smooth ps grid"):
ndown = smooth_ps_grid(nflat, res=delta_ell, alpha=4, log=log, ndof=2*(nsplits-1))
# pshow(nflat)
# pshow(ndown)
outcov = ndown*nfitted
outcov[modlmap<minell] = 0
if fill_lmax is not None: outcov[modlmap>fill_lmax] = 0
assert np.all(np.isfinite(outcov))
if isotropic_low_ell:
with bench.show("isotropic low ell"):
if radial_fit:
ifunc = lambda ells,ell0,A,shell: (A*np.exp(-ell0/ells) + shell)
sel = np.logical_and(modlmap<=lmin,modlmap>=2)
ibin_edges = np.arange(minell,(lmin*2)+2*minell,2*minell)
ibinner = stats.bin2D(modlmap,ibin_edges)
cents,inls = ibinner.bin(nflat)
ys = inls
xs = cents
if radial_fit:
res,_ = curve_fit(ifunc,xs,ys,p0=[20,1,0],bounds=([2,0.,-np.inf],[lmin*2,np.inf,np.inf]))
outcov[sel] = ifunc(modlmap[sel],res[0],res[1],res[2])*nfitted[sel]
else:
deg = 5
res = np.polyfit(np.log(xs),np.log(ys*xs**2.),deg=deg)
assert res.size==(deg+1)
fitfunc = lambda x: sum([res[deg-p]*(x**p) for p in range(0,deg+1)[::-1]])
outcov[sel] = (np.exp(fitfunc(np.log(modlmap[sel])))/modlmap[sel]**2.)*nfitted[sel]
outcov[modlmap<2] = 0
# fbin_edges = np.arange(minell,lmax,bin_annulus)
# fbinner = stats.bin2D(modlmap,fbin_edges)
# cents, n1d = fbinner.bin(nflat)
# pl = io.Plotter(xyscale='loglog',xlabel='l',ylabel='D',scalefn=lambda x: x**2./2./np.pi)
# ells = np.arange(minell,2*lmin,1)
# if radial_fit:
# pl.add(ells,ifunc(ells,res[0],res[1],res[2]))
# else:
# pl.add(xs,ys,ls="--")
# pl.add(ells,np.exp(fitfunc(np.log(ells)))/ells**2.)
# pl.add(cents,n1d)
# pl.vline(x=100)
# pl.vline(x=200)
# pl.vline(x=300)
# pl.vline(x=500)
# t = "000"
# pl._ax.set_xlim(10,3000)
# pl.done(os.environ['WORK']+"/iso_fitnoise2_%s.png" % t)
# fbin_edges = np.arange(minell,lmax,bin_annulus)
# fbinner = stats.bin2D(modlmap,fbin_edges)
# cents, n1d = fbinner.bin(n2d)
# cents,dn1d = fbinner.bin(outcov)
# # cents,dn1d2 = fbinner.bin(nfitted)
# pl = io.Plotter(xyscale='linlog',xlabel='l',ylabel='D',scalefn=lambda x: x**2./2./np.pi)
# pl.add(cents,n1d)
# pl.add(cents,dn1d,ls="--")
# pl.vline(x=100)
# pl.vline(x=200)
# pl.vline(x=300)
# pl.vline(x=500)
# # pl.add(cents,dn1d2,ls="-.")
# t = "000"
# pl._ax.set_ylim(1e1,1e5)
# pl.done(os.environ['WORK']+"/fitnoise2_%s.png" % t)
# sys.exit()
return outcov,nfitted,nparams
for i in range(len(amps)):
nsigma = (amps[i,0]**2*aicov[i,0])**0.5
msg += " %7.3f %4.1f" % (amps[i,0]/self.amp_unit, nsigma)
msg += " %12.5e %7.2f" % (self.chisq0-chisq, t2-t1)
print(msg)
self.i += 1
return chisq
return wrapper
for ind in range(comm.rank, len(ids), comm.size):
id = ids[ind]
bid = id.replace(":","_")
entry = filedb.data[id]
# Read the tod as usual
try:
with bench.show("read"):
d = actdata.read(entry)
with bench.show("calibrate"):
d = actdata.calibrate(d, exclude=["autocut"])
if d.ndet == 0 or d.nsamp < 2: raise errors.DataMissing("no data in tod")
except errors.DataMissing as e:
print("Skipping %s (%s)" % (id, e))
continue
print("Processing %s" % id)
# Very simple white noise model
with bench.show("ivar"):
tod = d.tod
del d.tod
tod -= np.mean(tod,1)[:,None]
tod = tod.astype(dtype)
diff = tod[:,1:]-tod[:,:-1]
beam_fn = lambda x: dm.get_beam(
season=season, patch=patch, array=array, ells=x)
inpainted = []
for i in range(nsplits):
gtags = []
gdicts = {}
pcoords = []
for sindex, sid in enumerate(ids):
cutout = reproject.cutout(splits[i],
ra=np.deg2rad(ras[sid]),
dec=np.deg2rad(decs[sid]),
pad=1,
corner=False,
npix=noise_pix)
if np.std(cutout) < 1e-3: continue
with bench.show("geometry"):
pcov = pixcov.pcov_from_ivar(noise_pix,
np.deg2rad(decs[sid]),
np.deg2rad(ras[sid]),
ivars[i, 0],
cmb_theory_fn,
beam_fn,
iau=False)
gdicts[sid] = pixcov.make_geometry(
hole_radius=np.deg2rad(hole_radius / 60.),
n=noise_pix,
deproject=True,
iau=False,
pcov=pcov,
res=res)
gtags.append(sid)
jsim0 = pipeline.JointSim(qids,None,bandpassed=True)
comm,rank,my_tasks = mpi.distribute(len(qids))
bin_edges = np.arange(20,8000,20)
binner = stats.bin2D(modlmap,bin_edges)
jsim.update_signal_index(0,set_idx=0)
jsim0.update_signal_index(0,set_idx=0)
for task in my_tasks:
qid = qids[task]
with bench.show("signal"):
signal = jsim.compute_map(mask.shape,mask.wcs,qid,
include_cmb=True,include_tsz=True,
include_fgres=True,sht_beam=True) # !!!
signal0 = jsim0.compute_map(mask.shape,mask.wcs,qid,
include_cmb=True,include_tsz=True,
include_fgres=True,sht_beam=True)
enmap.write_map(os.environ['WORK']+"/temp_sig_%s.fits" % qid,signal[0])
enmap.write_map(os.environ['WORK']+"/temp_sig0_%s.fits" % qid,signal0[0])
comm.Barrier()
if rank==0:
print("Processing...")
kmaps = []
lpad=9000,
get_dimensionless=False)
wdeg = 45.
hdeg = 15.
yoffset = 60.
pix = 2.0
shape, wcs = enmap.rect_geometry(width_arcmin=wdeg * 60.,
px_res_arcmin=pix,
height_arcmin=hdeg * 60.,
yoffset_degree=yoffset)
ells = np.arange(0, 5000, 1)
ps = theory.lCl('TT', ells).reshape((1, 1, 5000))
with bench.show("Map generation"):
mg = enmap.MapGen(shape, wcs, ps)
taper, w2 = fmaps.get_taper(shape,
taper_percent=8.0,
pad_percent=2.0,
weight=None)
map_south = mg.get_map() * taper
with bench.show("Rotation init"):
r = fmaps.MapRotatorEquator(map_south.shape,
map_south.wcs,
wdeg,
hdeg,
verbose=True,
width_multiplier=0.6,
height_multiplier=1.2)
args.beam, " specified on command line.")
obeam = maps.gauss_beam(bbeam, bmodlmap)
nbeam = maps.gauss_beam(args.beam, bmodlmap)
beam_ratio = np.nan_to_num(nbeam / obeam)
cov = lensing.beam_cov(cov, beam_ratio)
try:
old_cores = os.environ["OMP_NUM_THREADS"]
except:
old_cores = "1"
import multiprocessing
num_cores = str(multiprocessing.cpu_count())
os.environ["OMP_NUM_THREADS"] = num_cores
Tcov = cov + Ncov + 5000 # !!!
with bench.show("covwork"):
s, logdet = np.linalg.slogdet(Tcov)
assert s > 0
cinv = pinv2(Tcov).astype(np.float64)
os.environ["OMP_NUM_THREADS"] = old_cores
for core in range(1, numcores):
comm.Send([cinv, mpi.MPI.DOUBLE], dest=core, tag=77)
comm.send(logdet, dest=core, tag=88)
else:
cinv = np.empty((np.prod(bshape), np.prod(bshape)),
dtype=np.float64)
comm.Recv([cinv, mpi.MPI.DOUBLE], source=0, tag=77)
logdet = comm.recv(source=0, tag=88)
cinvs.append(cinv)
import hmvec as hm
import numpy as np
from orphics import io
from enlib import bench
zs = np.linspace(0.1, 3., 4)[-1:]
ms = np.geomspace(2e10, 1e17, 200)
ks = np.geomspace(1e-4, 100, 1001)
with bench.show("num"):
hcos = hm.HaloCosmology(zs, ks, ms=ms, nfw_numeric=True)
opmm_1h = hcos.get_power_1halo_auto(name="nfw")
opmm_2h = hcos.get_power_2halo_auto(name="nfw")
hcos = hm.HaloCosmology(zs, ks, ms=ms, nfw_numeric=False)
apmm_1h = hcos.get_power_1halo_auto(name="nfw")
apmm_2h = hcos.get_power_2halo_auto(name="nfw")
pl = io.Plotter(xyscale='loglin')
for i, z in enumerate(zs):
pl.add(ks, (apmm_1h[i] - opmm_1h[i]) / opmm_1h[i], ls='--')
pl.add(ks, (apmm_2h[i] - opmm_2h[i]) / opmm_2h[i], ls='--')
pl.hline(y=0)
pl.done()
def test_lens_recon():
from orphics import lensing, io, cosmology, maps
from enlib import bench
deg = 10.
px = 2.0
tellmin = 100
tellmax = 3000
kellmin = 40
kellmax = 3000
grad_cut = None
bin_width = 80
beam_arcmin = 0.01
noise_uk_arcmin = 0.01
theory = cosmology.default_theory(lpad=30000)
shape, wcs = s.rect_geometry(width_deg=deg, px_res_arcmin=px)
flsims = lensing.FlatLensingSims(shape, wcs, theory, beam_arcmin,
noise_uk_arcmin)
kbeam = flsims.kbeam
modlmap = enmap.modlmap(shape, wcs)
fc = maps.FourierCalc(shape, wcs)
n2d = (noise_uk_arcmin * np.pi / 180. / 60.)**2. / flsims.kbeam**2.
tmask = s.mask_kspace(shape, wcs, lmin=tellmin, lmax=tellmax)
kmask = s.mask_kspace(shape, wcs, lmin=kellmin, lmax=kellmax)
with bench.show("orphics init"):
qest = lensing.qest(shape,
wcs,
theory,
noise2d=n2d,
kmask=tmask,
kmask_K=kmask,
pol=False,
grad_cut=grad_cut,
unlensed_equals_lensed=True,
bigell=30000)
bin_edges = np.arange(kellmin, kellmax, bin_width)
binner = s.bin2D(modlmap, bin_edges)
i = 0
unlensed, kappa, lensed, beamed, noise_map, observed = flsims.get_sim(
seed_cmb=(i, 1),
seed_kappa=(i, 2),
seed_noise=(i, 3),
lens_order=5,
return_intermediate=True)
kmap = enmap.fft(observed, normalize="phys")
# _,kmap,_ = fc.power2d(observed)
with bench.show("orphics"):
kkappa = qest.kappa_from_map("TT",
kmap / kbeam,
alreadyFTed=True,
returnFt=True)
pir2d, kinput = fc.f1power(kappa, kkappa)
pii2d = fc.f2power(kinput, kinput)
prr2d = fc.f2power(kkappa, kkappa)
cents, pir1d = binner.bin(pir2d)
cents, pii1d = binner.bin(pii2d)
cents, prr1d = binner.bin(prr2d)
feed_dict = {}
cltt = theory.lCl('TT', modlmap)
feed_dict['uC_T_T'] = theory.lCl('TT', modlmap)
feed_dict['tC_T_T'] = cltt + n2d
feed_dict['X'] = kmap / kbeam
feed_dict['Y'] = kmap / kbeam
with bench.show("symlens init"):
Al = s.A_l(shape,
wcs,
feed_dict,
"hdv",
"TT",
xmask=tmask,
ymask=tmask)
Nl = s.N_l_from_A_l_optimal(shape, wcs, Al)
with bench.show("symlens"):
ukappa = s.unnormalized_quadratic_estimator(shape,
wcs,
feed_dict,
"hdv",
"TT",
xmask=tmask,
ymask=tmask)
nkappa = Al * ukappa
pir2d2 = fc.f2power(nkappa, kinput)
cents, pir1d2 = binner.bin(pir2d2)
cents, Nlkk = binner.bin(qest.N.Nlkk['TT'])
cents, Nlkk2 = binner.bin(Nl)
pl = io.Plotter(xyscale='linlog')
pl.add(cents, pii1d, color='k', lw=3)
pl.add(cents, pir1d, label='orphics')
pl.add(cents, pir1d2, label='hdv symlens')
pl.add(cents, Nlkk, ls="--", label='orphics')
pl.add(cents, Nlkk2, ls="-.", label='symlens')
pl.done("ncomp.png")
def get_sim(healpix, homogeneous, white, scale_to_rms=None):
if not (healpix):
mask = enmap.read_map(
"/scratch/r/rbond/msyriac/data/depot/actlens/car_mask_lmax_3000_apodized_2.0_deg.fits"
)[0]
shape, wcs = mask.shape, mask.wcs
nside = None
w2 = wfactor(2, mask)
map2alm = lambda x, lmax: cs.map2alm(x, lmax=lmax)
else:
shape = None
wcs = None
mask = hp.read_map(
"/scratch/r/rbond/msyriac/data/depot/solenspipe/lensing_mask_nside_2048_apodized_4.0.fits"
)
nside = 2048
w2 = wfactor(2, mask, equal_area=True)
map2alm = lambda x, lmax: hp.map2alm(x, lmax=lmax)
tube = 'LT2'
with bench.show("init"):
nsim = mapsims.SONoiseSimulator \
( \
nside=nside,
shape=shape,
wcs=wcs,
ell_max=None,
return_uK_CMB=True,
sensitivity_mode="baseline",
apply_beam_correction=False,
apply_kludge_correction=True,
homogeneous=homogeneous,
no_power_below_ell=None,
rolloff_ell=50,
survey_efficiency=0.2,
full_covariance=True,
LA_years=5,
LA_noise_model="SOLatV3point1",
elevation=50,
SA_years=5,
SA_one_over_f_mode="pessimistic",
sky_fraction=None,
cache_hitmaps=True,
boolean_sky_fraction=False,
)
with bench.show("sim"):
omap = nsim.simulate(
tube,
output_units="uK_CMB",
seed=None,
nsplits=1,
mask_value=0,
atmosphere=not (white),
hitmap=None,
white_noise_rms=scale_to_rms,
)
# io.hplot(omap[0][0][0],f'{out_path}/mapsims_sim_homogeneous_{homogeneous}_white_{white}_scale_to_rms_{scale_to_rms}',downgrade=4,grid=True,ticks=20)
with bench.show("nells"):
ell, ps_T, ps_P, fsky, wnoise_power, hitmaps = nsim.get_noise_properties(
tube,
nsplits=1,
hitmap=None,
white_noise_rms=scale_to_rms,
atmosphere=not (white))
with bench.show("ivar"):
ivar = nsim.get_inverse_variance(tube,
output_units="uK_CMB",
hitmap=None,
white_noise_rms=scale_to_rms)
# Calculate raw power spectrum
imap = omap[0][0][0]
imap = imap * mask
alm = map2alm(imap, lmax=4000)
cls = hp.alm2cl(alm) / w2
ls = np.arange(len(cls))
pl = io.Plotter('Cell')
pl.add(ls, cls)
pl.add(ell, ps_T[0], ls='--')
pl.done(
f'{out_path}/mapsims_nells_homogeneous_{homogeneous}_white_{white}_scale_to_rms_{scale_to_rms}_healpix_{healpix}.png'
)
# Calculate whitened map power spectrum
imap = np.nan_to_num(omap[0][0][0]) * np.sqrt(
ivar[0] / nsim.pixarea_map) * mask
# io.hplot(imap,f'{out_path}/mapsims_wsim_homogeneous_{homogeneous}_white_{white}_scale_to_rms_{scale_to_rms}',downgrade=4,grid=True,ticks=20)
alm = map2alm(imap, lmax=4000)
cls = hp.alm2cl(alm) / w2
ls = np.arange(len(cls))
pl = io.Plotter('Cell')
pl.add(ls, cls)
pl.add(ell, ps_T[0] / wnoise_power[0])
pl.done(
f'{out_path}/mapsims_wnells_homogeneous_{homogeneous}_white_{white}_scale_to_rms_{scale_to_rms}_healpix_{healpix}.png'
)
for k, kamp in enumerate(kamps):
kappa_template = lensing.nfw_kappa(kamp * 1e15,
bmodrmap,
cc,
overdensity=200.,
critical=True,
atClusterZ=True)
phi, _ = lensing.kappa_to_phi(kappa_template, bmodlmap, return_fphi=True)
grad_phi = enmap.grad(phi)
pos = posmap + grad_phi
alpha_pix = enmap.sky2pix(bshape, bwcs, pos, safe=False)
#if k==0: io.plot_img(kappa_template)
with bench.show("lensing cov"):
Scov = lensing.lens_cov(Ucov,
alpha_pix,
lens_order=lens_order,
kbeam=kbeam,
bshape=shape)
Tcov = Scov + Ncov + 5000 # !!!
with bench.show("covwork"):
s, logdet = np.linalg.slogdet(Tcov)
assert s > 0
cinv = pinv2(Tcov).astype(np.float64)
cinvs.append(cinv)
logdets.append(logdet)
print(kamp, logdet)
# import cPickle as pickle
sky_fraction=c.fsky if c.homogenous else None,
)
ells, nlt, nlp = nsim.get_noise_spectra(c.tube, ncurve_sky_fraction=c.fsky)
if not (c.homogenous):
try:
if nside is not None:
mask = hp.read_map(c.mask)
else:
mask = enmap.read_map(c.mask)
except:
mask = None
else:
mask = None
with bench.show("sim"):
omap = nsim.simulate(c.tube,
seed=(1, ),
nsplits=1,
atmosphere=not (c.wnoise),
hitmap=mask)
if mask is None:
mask = 1
w2 = 1
else:
from solenspipe import wfactor
w2 = wfactor(2, mask, equal_area=not (nside is None))
ls, c11, c22, c12, zeros = get_spectra(c.tube, omap * mask, nside, res,
c.wnoise, w2)
b = sampcut.gapfill_const(cut, tod*iV[:,None], 0, inplace=True).reshape(-1)
x0 = sampcut.gapfill_linear(cut, tod).reshape(-1)
solver = cg.CG(A, b, x0)
while solver.i < maxiter and solver.err > lim:
solver.step()
if verbose:
print "%5d %15.7e" % (solver.i, solver.err)
return solver.x.reshape(tod.shape)
for ind in range(comm.rank, len(ids), comm.size):
id = ids[ind]
bid = id.replace(":","_")
entry = filedb.data[id]
# Read the tod as usual
try:
with bench.show("read"):
d = actdata.read(entry)
with bench.show("calibrate"):
d = actdata.calibrate(d, exclude=["autocut"])
# Replace the beam with our dummy beam
d.beam = beam
if d.ndet == 0 or d.nsamp < 2: raise errors.DataMissing("no data in tod")
except errors.DataMissing as e:
print "Skipping %s (%s)" % (id, e.message)
# Make a dummy output file so we can skip this tod in the future
with open("%s%s_empty.txt" % (prefix, bid),"w"): pass
continue
print "Processing %s [ndet:%d, nsamp:%d, nsrc:%d]" % (id, d.ndet, d.nsamp, len(tod_srcs[id]))
# Fill in representative ra, dec for planets for this tod
for sid in np.where(srcs.type == "planet")[0]:
srcs.ra[sid], srcs.dec[sid] = ephemeris.ephem_pos(srcs.name[sid], utils.ctime2mjd(d.boresight[0,d.nsamp//2]), dt=0)[:2]
for task in my_tasks:
sim_index = task
ind_str = str(set_id).zfill(2) + "_" + str(sim_index).zfill(4)
sim_version = "%s_%s" % (args.version, ind_str)
scratch = tutils.get_scratch_path(sim_version, args.region)
try:
os.makedirs(scratch)
except:
pass
"""
MAKE SIMS
"""
"""
SAVE COV
"""
print("Beginning covariance calculation...")
with bench.show("sim cov"):
sim_build(args.region, sim_version, args.overwrite)
"""
SAVE ILC
"""
print("done")
ilc_version = "map_%s_%s" % (args.version, ind_str)
with bench.show("sim ilc"):
print("starting")
sim_ilc(args.region, ilc_version, sim_version, args.overwrite)
savepath = tutils.get_save_path(sim_version, args.region)
shutil.rmtree(savepath)
parser.add_argument("--mask-kind", type=str, default="binary_apod",help='Mask kind')
parser.add_argument("--mask-patch", type=str, default=None,help='Mask patch')
parser.add_argument("--mask-pad", type=int, default=None,
help='Mask additional padding. No padding is applied to the extracted mask if any specified.')
parser.add_argument("--extract-mask", type=str, default=None,
help='Make sims on the big mask but do all the analysis on an extract of this version.')
parser.add_argument("--binary-percentile", type=float, default=10.,help='Binary percentile for sim masking.')
parser.add_argument("--season", type=str,help='Season')
parser.add_argument("--array", type=str,help='Array')
parser.add_argument("--patch", type=str,help='Patch')
args = parser.parse_args()
mask_patch = args.patch if args.mask_patch is None else args.mask_patch
with bench.show("region"):
if args.extract_mask is not None:
region_map = sints.get_act_mr3_crosslinked_mask(mask_patch,
version=args.extract_mask,
kind=args.mask_kind,
season=args.season)
else:
region_map = None
with bench.show("ngen init"):
ngen = noise.NoiseGen(args.version,extract_region=region_map,ncache=1)
with bench.show("make a sim"):
ngen.generate_sim(season=args.season,patch=args.patch,array=args.array,seed=1)
with bench.show("make another sim"):
covsqrt = noise.get_covsqrt(n2d_xflat_smoothed, args.covsqrt_kind)
print(np.shape(covsqrt))
del n2d_xflat_smoothed
ngen.save_covsqrt(covsqrt,
season=args.season,
patch=args.patch,
array=args.array,
coadd=coadd,
mask_patch=mask_patch)
if nsims > 0:
bin_edges = np.arange(40, 8000, 40)
p1ds = []
for i in range(nsims):
print("Sim %d of %d ..." % (i + 1, nsims))
with bench.show("simgen"):
sims = ngen.generate_sim(season=args.season,
patch=args.patch,
array=args.array,
seed=i,
mask_patch=mask_patch)
print(sims.nbytes / 1024. / 1024. / 1024., " GB", sims.shape,
sims.dtype)
if args.extract_mask is not None:
ivars2 = enmap.extract(ivars, eshape, ewcs)
modlmap = enmap.modlmap(eshape, ewcs)
else:
ivars2 = ivars
if args.debug and i == 0: noise.plot(pout + "_sims", sims)
if not (args.no_write):
pl.add(nu_ghz,trans/trans.max())
pl.done("band_%s.png" % qid)
if gauss_beam:
if 'p' in qid:
fwhm = dm.fwhms[array]
else:
fwhm = 1.4 * (150./cfreq)
lbeam = maps.gauss_beam(ells,fwhm)
else:
lbeam = tutils.get_kbeam(qid,ells,sanitize=False,planck_pixwin=False)
with bench.show("beamint"):
fbnus = maps.interp(ells,lbeam[None,:],fill_value=(lbeam[0],lbeam[-1]))
bnus = fbnus((cfreq/nu_ghz)*ells[:,None])[0].swapaxes(0,1)
bnus = bnus / bnus[:,:1]
pl = io.Plotter(xlabel='l',ylabel='b')
for i in range(bnus.shape[0]):
if trans[i]>1e-1: pl.add(ells,bnus[i]/lbeam)
pl.hline(y=1)
pl._ax.set_ylim(0.8,1.2)
pl.done("abeams_%s.png" % qid)
# with bench.show("huok"):
# cents,nls,ncoadd = get_nlkk_single("hu_ok",modlmap,asnoise-lcltt,apnoise,apnoise,tmask,pmask,pols=pols)
# plot(cents,nls,ncoadd,pols,tag="act")
# pols = ['TT','TE','ET','EE','EB','TB']
# with bench.show("hdv"):
# cents,nls,ncoadd = get_nlkk_single("hdv",modlmap,onoise,apnoise,apnoise,tmask,pmask,pols=pols)
# plot(cents,nls,ncoadd,pols,tag="act_only_all")
# pols = ['TT','TE','ET','EE','EB','TB']
# with bench.show("hdv cross-check"):
# cents,nls,ncoadd,pols = get_nlkk_mixed(modlmap,pnoise,onoise,asnoise*0.,apnoise,apnoise,tmask,pmask,pols)
# plot(cents,nls,ncoadd,pols,tag="act_smica_noilc_all")
pols = ['TT', 'TE', 'ET', 'EE', 'EB', 'TB']
with bench.show("hdv cross-check"):
cents, nls, ncoadd, pols = get_nlkk_mixed(modlmap, tcnoise - lcltt,
tsnoise - lcltt,
tscnoise - lcltt,
tenoise - lclee, tbnoise - lclbb,
tmask, pmask, pols)
plot(cents, nls, ncoadd, pols, tag="act_planck_ilc_all")
# pols = ['TT']
# with bench.show("hdv"):
# cents,nls,ncoadd = get_nlkk_single("hdv",modlmap,onoise,apnoise,apnoise,tmask,pmask,pols=pols)
# plot(cents,nls,ncoadd,pols,tag="act_only_tt")
# pols = ['TT']
# with bench.show("hdv cross-check"):
# cents,nls,ncoadd,pols = get_nlkk_mixed(modlmap,pnoise,onoise,asnoise*0.,apnoise,apnoise,tmask,pmask,pols)
sids = np.where(utils.point_in_polygon(srcpos.T, poly.T))[0]
sids = np.array(sorted(list(set(sids)&allowed)))
if len(sids) == 0:
print "%5d/%d %s has 0 srcs: skipping" % (ind+1, len(ids), id)
continue
nsrc = len(sids)
print "%5d/%d %s has %d srcs: %s" % (ind+1, len(ids), id,nsrc,", ".join(["%d (%.1f)" % (i,a) for i,a in zip(sids,amps[sids])]))
def skip(msg):
print "%s skipped: %s" % (id, msg)
with open(ename, "w") as f:
f.write(msg + "\n")
entry = filedb.data[id]
try:
with bench.show("read"):
d = actdata.read(entry)
with bench.show("calib"):
d = actdata.calibrate(d, exclude=["autocut"])
if d.ndet < 2 or d.nsamp < 1: raise errors.DataMissing("no data in tod")
except errors.DataMissing as e:
skip(e.message)
continue
tod = d.tod.astype(dtype)
del d.tod
# We need a white noise estimate per detector. How should we get this?
# 1. Use atmosphere-removed samples. But these have the source in them.
# But an individual detector is probably pretty noise dominated, so it might work.
# 2. Use standard noise model to estimate the noise level. A bit slow, but not that bad.
# 3. Use the running difference to estimate it. This is simple, but may place too
def make_sim(self, seed):
with bench.show(
"Lensing operation...") if self.rank == 0 else ignore():
full, kappa = lensing.rand_map(
self.fshape,
self.fwcs,
self.ps,
lmax=self.lmax,
maplmax=self.lmax,
seed=seed,
verbose=True if self.rank == 0 else False,
dtype=self.dtype,
output="lk")
alms = curvedsky.map2alm(full, lmax=self.lmax)
ps_data = hp.alm2cl(alms.astype(np.complex128))
del alms
self.mpibox.add_to_stats("fullsky_ps", ps_data)
south = full.submap(self.pos_south)
equator = full.submap(self.pos_eq)
ksouth = kappa.submap(self.pos_south)
kequator = kappa.submap(self.pos_eq)
del full
del kappa
if self.count == 0:
self.shape['s'], self.wcs['s'] = south.shape, south.wcs
self.shape['e'], self.wcs['e'] = equator.shape, equator.wcs
for m in ['s', 'e']:
self.taper[m], self.w2[m] = fmaps.get_taper(self.shape[m],
taper_percent=18.0,
pad_percent=4.0,
weight=None)
self.w4[m] = np.mean(self.taper[m]**4.)
self.w3[m] = np.mean(self.taper[m]**3.)
self.rotator = fmaps.MapRotatorEquator(
self.shape['s'],
self.wcs['s'],
self.wdeg,
self.hdeg,
width_multiplier=0.6,
height_multiplier=1.2,
downsample=True,
verbose=True if self.rank == 0 else False,
pix_target_override_arcmin=self.pix_intermediate)
self.taper['r'] = self.rotator.rotate(self.taper['s'])
self.w2['r'] = np.mean(self.taper['r']**2.)
self.w4['r'] = np.mean(self.taper['r']**4.)
self.w3['r'] = np.mean(self.taper['r']**3.)
self.shape['r'], self.wcs[
'r'] = self.rotator.shape_final, self.rotator.wcs_final
self.fc = {}
self.binner = {}
self.modlmap = {}
for m in ['s', 'e', 'r']:
self.fc[m] = fmaps.FourierCalc(self.shape[m], self.wcs[m])
self.modlmap[m] = enmap.modlmap(self.shape[m], self.wcs[m])
self.binner[m] = bin2D(self.modlmap[m], self.bin_edges)
self.cents = self.binner['s'].centers
self._init_qests()
self.count += 1
south *= self.taper['s']
equator *= self.taper['e']
ksouth *= self.taper['s']
kequator *= self.taper['e']
return south, equator, ksouth, kequator
# Then compute quantiles
tspecs[0,i] = np.median(dhigh,0)
tspecs[1,i] = np.percentile(dhigh,15.86553,0)
tspecs[2,i] = np.percentile(dhigh,84.13447,0)
tspecs[3,i] = np.min(dhigh,0)
tspecs[4,i] = np.max(dhigh,0)
del ps
# Normalize ft in bins, since we want correlations
for di in range(d.ndet):
ft[di] /= (dhigh[di]**0.5)[binds]
# Average correlation in bin
sps = np.abs(np.sum(ft,0))**2
tcorrs[i] = (bin(sps, args.nbin)-d.ndet)/(d.ndet**2-d.ndet)
del sps, ft, d
# Ok, we've gone through all the data in our chunk
with bench.show("Reduce"):
dspecs = utils.allreduce(dspecs, comm)
dzooms = utils.allreduce(dzooms, comm)
tspecs = utils.allreduce(tspecs, comm)
tcorrs = utils.allreduce(tcorrs, comm)
srates = utils.allreduce(srates, comm)
mce_fsamps = utils.allreduce(mce_fsamps, comm)
mce_params = utils.allreduce(mce_params, comm)
ofile = prefix + "specs%03d.hdf" % chunk
if comm.rank == 0:
# Get rid of empty tods
good = np.where(np.any(dspecs>0,(1,2)))[0]
if len(good) == 0:
print "No usable tods in chunk!"
continue
dspecs = dspecs[good]
Cov = Cov[:, :, modlmap < lmax1].reshape(
(narrays, narrays, modlmap[modlmap < lmax1].size))
s = stats.Stats(comm)
mask = enmap.ones(tsim.shape[-2:], tsim.wcs)
Cov = maps.SymMat(narrays, tsim.shape[-2:])
names = ["a%d" % i for i in range(narrays)]
def save_fn(tcov, a1, a2):
Cov[a1, a2] = tcov.copy()
for task in my_tasks:
with bench.show("sim gen"):
isim, isimnoise = tsim.get_sim(task)
with bench.show("ffts"):
iksplits = []
ikmaps = []
inkmaps = []
all_wins = []
for aindex, array in enumerate(tsim.arrays):
splits = isim[array]
noise_splits = isimnoise[array]
nsplits = tsim.nsplits[aindex]
wins = enmap.ones((nsplits, ) + tsim.shape[-2:], tsim.wcs)
all_wins.append(wins)
ksplits, kcoadd = kspace.process_splits(splits,
wins=wins,
mask=mask)
