We will MPI parallelize over pairs of arrays. This wastes FFTs, but is more memory efficient. (Each job only
holds 2 arrays in memory).
"""
njobs = len(qpairs)
comm,rank,my_tasks = mpi.distribute(njobs)
mask = sints.get_act_mr3_crosslinked_mask(args.region,
version=args.mask_version,
kind='binary_apod')
shape,wcs = mask.shape,mask.wcs
modlmap = mask.modlmap()
aspecs = tutils.ASpecs().get_specs
region = args.region
fbeam = lambda qname,x: tutils.get_kbeam(qname,x,sanitize=not(args.unsanitized_beam),planck_pixwin=True)
nbin_edges = np.arange(20,8000,100)
nbinner = stats.bin2D(modlmap,nbin_edges)
ncents = nbinner.centers
#cbin_edges = np.arange(20,8000,20)
cbin_edges = np.arange(20,8000,40)
cbinner = stats.bin2D(modlmap,cbin_edges)
fells = np.arange(lmax)
for task in my_tasks:
qids = qpairs[task]
qid1,qid2 = qids
ncents,n1d = np.loadtxt("%sn1d_%s_%s.txt" % (spath,qid1,qid2),unpack=True)
ncents,n1d1 = np.loadtxt("%sn1d_%s_%s.txt" % (spath,qid1,qid1),unpack=True)
ncents,n1d2 = np.loadtxt("%sn1d_%s_%s.txt" % (spath,qid2,qid2),unpack=True)
def __init__(self,qids,fg_res_version=None,ellmax=8101,bandpassed=True,no_act_color_correction=False,ccor_exp=-1):
self.qids = qids
self.alms = {}
ells = np.arange(ellmax)
self.ellmax = ellmax
self.cyy = fgs.power_y(ells)[None,None]
self.cyy[0,0][ells<2] = 0
narrays = len(qids)
if fg_res_version is not None:
fpath = sints.dconfig['actsims']['fg_res_path'] + "/" + fg_res_version + "/"
self.cfgres = np.zeros((narrays,narrays,ellmax))
for i in range(narrays):
for j in range(i,narrays):
qid1 = qids[i]
qid2 = qids[j]
clfilename = "%shfgcov_%s_%s.txt" % (fpath,qid1,qid2) #!!!! Changed fgcov to hfgcov # !!!!
clfilename_alt = "%shfgcov_%s_%s.txt" % (fpath,qid2,qid1) #!!!! Changed fgcov to hfgcov
try:
ls,cls = np.loadtxt(clfilename,unpack=True)
except:
ls,cls = np.loadtxt(clfilename_alt,unpack=True)
assert np.all(np.isclose(ls,ells))
self.cfgres[i,j] = cls.copy()
if i!=j:
self.cfgres[j,i] = cls.copy()
# self.cfgres[j,i] = self.cfgres[i,j] = cls.copy()*0 # !!!! REMOVED CORR
#if (qid1 in ['p01','p02']) or (qid2 in ['p01','p02']): self.cfgres[j,i] = self.cfgres[i,j] = 0 # !!!!
# self.cfgres[j,i][ls<300] = self.cfgres[i,j][ls<300] = 0 # !!!!
else:
self.cfgres = None
# get tSZ response, and also zero out parts of map that are not observed
aspecs = tutils.ASpecs().get_specs
bps = []
cfreqs = []
lbeams = []
ells = np.arange(0,35000)
for qid in qids:
dm = sints.models[sints.arrays(qid,'data_model')]()
if dm.name=='act_mr3':
season,array1,array2 = sints.arrays(qid,'season'),sints.arrays(qid,'array'),sints.arrays(qid,'freq')
array = '_'.join([array1,array2])
lbeam = tutils.get_kbeam(qid,ells,sanitize=True,planck_pixwin=False) # note no pixwin but doesnt matter since no ccorr for planck
elif dm.name=='planck_hybrid':
season,patch,array = None,None,sints.arrays(qid,'freq')
lbeam = None
else:
raise ValueError
lbeams.append(lbeam)
lmin,lmax,hybrid,radial,friend,cfreq,fgroup,wrfit = aspecs(qid)
if bandpassed:
bps.append("data/"+dm.get_bandpass_file_name(array))
else:
bps.append(cfreq)
cfreqs.append(cfreq)
if bandpassed:
if no_act_color_correction:
self.tsz_fnu = tfg.get_mix_bandpassed(bps, 'tSZ')
else:
self.tsz_fnu = tfg.get_mix_bandpassed(bps, 'tSZ',
ccor_cen_nus=cfreqs, ccor_beams=lbeams,
ccor_exps = [ccor_exp] * narrays)
else:
self.tsz_fnu = tfg.get_mix(bps, 'tSZ')
from __future__ import print_function
from orphics import maps,io,cosmology
from pixell import enmap
import numpy as np
import os,sys
from tilec import utils as tutils
qids = ['d56_%s' % str(i+1).zfill(2) for i in range(6)]
ells = np.arange(2,35000)
pl = io.Plotter(xyscale='linlog',xlabel='l',ylabel='B')
for i,qid in enumerate(qids):
lbeam = tutils.get_kbeam(qid,ells)
pl.add(ells,np.abs(lbeam),label=qid,color="C%d" % i,ls="-",alpha=0.7)
pl.add(ells,tutils.sanitize_beam(ells,lbeam),ls="--",color="C%d" % i)
pl._ax.set_ylim(1e-8,2)
pl.done("lbeam.png")
def build_and_save_ilc(arrays,region,version,cov_version,beam_version,
solutions,beams,chunk_size,
effective_freq,overwrite,maxval,unsanitized_beam=False,do_weights=False,
pa1_shift = None,
pa2_shift = None,
pa3_150_shift = None,
pa3_090_shift = None,
no_act_color_correction=False, ccor_exp = -1,
isotropize=False, isotropize_width=20):
print("Chunk size is ", chunk_size*64./8./1024./1024./1024., " GB.")
def warn(): print("WARNING: no bandpass file found. Assuming array ",dm.c['id']," has no response to CMB, tSZ and CIB.")
aspecs = tutils.ASpecs().get_specs
bandpasses = not(effective_freq)
savedir = tutils.get_save_path(version,region)
covdir = tutils.get_save_path(cov_version,region)
assert os.path.exists(covdir)
if not(overwrite):
assert not(os.path.exists(savedir)), \
"This version already exists on disk. Please use a different version identifier."
try: os.makedirs(savedir)
except:
if overwrite: pass
else: raise
mask = enmap.read_map(covdir+"tilec_mask.fits")
shape,wcs = mask.shape,mask.wcs
Ny,Nx = shape
modlmap = enmap.modlmap(shape,wcs)
arrays = arrays.split(',')
narrays = len(arrays)
kcoadds = []
kbeams = []
bps = []
names = []
lmins = []
lmaxs = []
shifts = []
cfreqs = []
lbeams = []
ells = np.arange(0,modlmap.max())
for i,qid in enumerate(arrays):
dm = sints.models[sints.arrays(qid,'data_model')](region=mask,calibrated=True)
lmin,lmax,hybrid,radial,friend,cfreq,fgroup,wrfit = aspecs(qid)
cfreqs.append(cfreq)
lmins.append(lmin)
lmaxs.append(lmax)
names.append(qid)
if dm.name=='act_mr3':
season,array1,array2 = sints.arrays(qid,'season'),sints.arrays(qid,'array'),sints.arrays(qid,'freq')
array = '_'.join([array1,array2])
elif dm.name=='planck_hybrid':
season,patch,array = None,None,sints.arrays(qid,'freq')
else:
raise ValueError
kcoadd_name = covdir + "kcoadd_%s.npy" % qid
kmask = maps.mask_kspace(shape,wcs,lmin=lmin,lmax=lmax)
kcoadd = enmap.enmap(np.load(kcoadd_name),wcs)
dtype = kcoadd.dtype
kcoadds.append(kcoadd.copy()*kmask)
kbeam = tutils.get_kbeam(qid,modlmap,sanitize=not(unsanitized_beam),version=beam_version,planck_pixwin=True)
if dm.name=='act_mr3':
lbeam = tutils.get_kbeam(qid,ells,sanitize=not(unsanitized_beam),version=beam_version,planck_pixwin=False) # note no pixwin but doesnt matter since no ccorr for planck
elif dm.name=='planck_hybrid':
lbeam = None
else:
raise ValueError
lbeams.append(lbeam)
kbeams.append(kbeam.copy())
if bandpasses:
try:
fname = dm.get_bandpass_file_name(array)
bps.append("data/"+fname)
if (pa1_shift is not None) and 'PA1' in fname:
shifts.append(pa1_shift)
elif (pa2_shift is not None) and 'PA2' in fname:
shifts.append(pa2_shift)
elif (pa3_150_shift is not None) and ('PA3' in fname) and ('150' in fname):
shifts.append(pa3_150_shift)
elif (pa3_090_shift is not None) and ('PA3' in fname) and ('090' in fname):
shifts.append(pa3_90_shift)
else:
shifts.append(0)
except:
warn()
bps.append(None)
else:
try: bps.append(cfreq)
except:
warn()
bps.append(None)
kcoadds = enmap.enmap(np.stack(kcoadds),wcs)
# Read Covmat
cov = maps.SymMat(narrays,shape[-2:])
for aindex1 in range(narrays):
for aindex2 in range(aindex1,narrays):
icov = enmap.enmap(np.load(covdir+"tilec_hybrid_covariance_%s_%s.npy" % (names[aindex1],names[aindex2])),wcs)
if isotropize:
bin_edges = np.append([0.],np.arange(min(lmins),modlmap.max(),isotropize_width))
binner = stats.bin2D(modlmap,bin_edges)
ls,c1d = binner.bin(icov)
icov = maps.interp(ls,c1d)(modlmap)
if aindex1==aindex2:
icov[modlmap<lmins[aindex1]] = maxval
icov[modlmap>lmaxs[aindex1]] = maxval
cov[aindex1,aindex2] = icov
cov.data = enmap.enmap(cov.data,wcs,copy=False)
covfunc = lambda sel: cov.to_array(sel,flatten=True)
assert cov.data.shape[0]==((narrays*(narrays+1))/2) # FIXME: generalize
assert np.all(np.isfinite(cov.data))
# Make responses
responses = {}
for comp in ['tSZ','CMB','CIB']:
if bandpasses:
if no_act_color_correction:
responses[comp] = tfg.get_mix_bandpassed(bps, comp, bandpass_shifts=shifts)
else:
responses[comp] = tfg.get_mix_bandpassed(bps, comp, bandpass_shifts=shifts,
ccor_cen_nus=cfreqs, ccor_beams=lbeams,
ccor_exps = [ccor_exp] * narrays)
else:
responses[comp] = tfg.get_mix(bps, comp)
ilcgen = ilc.chunked_ilc(modlmap,np.stack(kbeams),covfunc,chunk_size,responses=responses,invert=True)
# Initialize containers
solutions = solutions.split(',')
data = {}
kcoadds = kcoadds.reshape((narrays,Ny*Nx))
for solution in solutions:
data[solution] = {}
comps = solution.split('-')
data[solution]['comps'] = comps
if len(comps)<=2:
data[solution]['noise'] = enmap.zeros((Ny*Nx),wcs)
if len(comps)==2:
data[solution]['cnoise'] = enmap.zeros((Ny*Nx),wcs)
data[solution]['kmap'] = enmap.zeros((Ny*Nx),wcs,dtype=dtype) # FIXME: reduce dtype?
if do_weights and len(comps)<=2:
for qid in arrays:
data[solution]['weight_%s' % qid] = enmap.zeros((Ny*Nx),wcs)
for chunknum,(hilc,selchunk) in enumerate(ilcgen):
print("ILC on chunk ", chunknum+1, " / ",int(modlmap.size/chunk_size)+1," ...")
for solution in solutions:
comps = data[solution]['comps']
if len(comps)==1: # GENERALIZE
data[solution]['noise'][selchunk] = hilc.standard_noise(comps[0])
if do_weights: weight = hilc.standard_weight(comps[0])
data[solution]['kmap'][selchunk] = hilc.standard_map(kcoadds[...,selchunk],comps[0])
elif len(comps)==2:
data[solution]['noise'][selchunk] = hilc.constrained_noise(comps[0],comps[1])
data[solution]['cnoise'][selchunk] = hilc.cross_noise(comps[0],comps[1])
ret = hilc.constrained_map(kcoadds[...,selchunk],comps[0],comps[1],return_weight=do_weights)
if do_weights:
data[solution]['kmap'][selchunk],weight = ret
else:
data[solution]['kmap'][selchunk] = ret
elif len(comps)>2:
data[solution]['kmap'][selchunk] = np.nan_to_num(hilc.multi_constrained_map(kcoadds[...,selchunk],comps[0],*comps[1:]))
if len(comps)<=2 and do_weights:
for qind,qid in enumerate(arrays):
data[solution]['weight_%s' % qid][selchunk] = weight[qind]
del ilcgen,cov
# Reshape into maps
name_map = {'CMB':'cmb','tSZ':'comptony','CIB':'cib'}
beams = beams.split(',')
for solution,beam in zip(solutions,beams):
comps = "tilec_single_tile_"+region+"_"
comps = comps + name_map[data[solution]['comps'][0]]+"_"
if len(data[solution]['comps'])>1: comps = comps + "deprojects_"+ '_'.join([name_map[x] for x in data[solution]['comps'][1:]]) + "_"
comps = comps + version
if do_weights and len(data[solution]['comps'])<=2:
for qind,qid in enumerate(arrays):
enmap.write_map("%s/%s_%s_weight.fits" % (savedir,comps,qid), enmap.enmap(data[solution]['weight_%s' % qid].reshape((Ny,Nx)),wcs))
try:
noise = enmap.enmap(data[solution]['noise'].reshape((Ny,Nx)),wcs)
enmap.write_map("%s/%s_noise.fits" % (savedir,comps),noise)
except: pass
try:
cnoise = enmap.enmap(data[solution]['cnoise'].reshape((Ny,Nx)),wcs)
enmap.write_map("%s/%s_cross_noise.fits" % (savedir,comps),cnoise)
except: pass
ells = np.arange(0,modlmap.max(),1)
try:
fbeam = float(beam)
kbeam = maps.gauss_beam(modlmap,fbeam)
lbeam = maps.gauss_beam(ells,fbeam)
except:
qid = beam
bfunc = lambda x: tutils.get_kbeam(qid,x,version=beam_version,sanitize=not(unsanitized_beam),planck_pixwin=False)
kbeam = bfunc(modlmap)
lbeam = bfunc(ells)
kmap = enmap.enmap(data[solution]['kmap'].reshape((Ny,Nx)),wcs)
smap = enmap.ifft(kbeam*kmap,normalize='phys').real
enmap.write_map("%s/%s.fits" % (savedir,comps),smap)
io.save_cols("%s/%s_beam.txt" % (savedir,comps),(ells,lbeam),header="ell beam")
enmap.write_map(savedir+"/tilec_mask.fits",mask)
def calculate_yy(bin_edges,arrays,region,version,cov_versions,beam_version,
effective_freq,overwrite,maxval,unsanitized_beam=False,do_weights=False,
pa1_shift = None,
pa2_shift = None,
pa3_150_shift = None,
pa3_090_shift = None,
no_act_color_correction=False, ccor_exp = -1,
sim_splits=None,unblind=False,all_analytic=False,beta_samples=None):
"""
We calculate the yy power spectrum as follows.
We restrict the Fourier modes in our analysis to those within bin_edges.
This way we don't carry irrelevant pixels and thus speed up the ability to MC.
We accept two covariance versions in cov_versions, which correspond to
[act_covariance_from_split_0,act_covariance_from_split_1,other_covs].
Thus the ACT auto covariances are pre-calculated
"""
arrays = arrays.split(',')
narrays = len(arrays)
if sim_splits is not None: assert not(unblind)
def warn(): print("WARNING: no bandpass file found. Assuming array ",dm.c['id']," has no response to CMB, tSZ and CIB.")
aspecs = tutils.ASpecs().get_specs
bandpasses = not(effective_freq)
savedir = tutils.get_save_path(version,region)
assert len(cov_versions)==3
covdirs = [tutils.get_save_path(cov_versions[i],region) for i in range(3)]
for covdir in covdirs: assert os.path.exists(covdir)
if not(overwrite):
assert not(os.path.exists(savedir)), \
"This version already exists on disk. Please use a different version identifier."
try: os.makedirs(savedir)
except:
if overwrite: pass
else: raise
mask = enmap.read_map(covdir+"tilec_mask.fits")
from scipy.ndimage.filters import gaussian_filter as smooth
pm = enmap.read_map("/scratch/r/rbond/msyriac/data/planck/data/pr2/COM_Mask_Lensing_2048_R2.00_car_deep56_interp_order0.fits")
wcs = pm.wcs
mask = enmap.enmap(smooth(pm,sigma=10),wcs) * mask
shape,wcs = mask.shape,mask.wcs
Ny,Nx = shape
modlmap = enmap.modlmap(shape,wcs)
omodlmap = modlmap.copy()
ells = np.arange(0,modlmap.max())
minell = maps.minimum_ell(shape,wcs)
sel = np.where(np.logical_and(modlmap>=bin_edges[0]-minell,modlmap<=bin_edges[-1]+minell))
modlmap = modlmap[sel]
bps = []
lbeams = []
kbeams = []
shifts = []
cfreqs = []
lmins = []
lmaxs = []
names = []
for i,qid in enumerate(arrays):
dm = sints.models[sints.arrays(qid,'data_model')](region=mask,calibrated=True)
if dm.name=='act_mr3':
season,array1,array2 = sints.arrays(qid,'season'),sints.arrays(qid,'array'),sints.arrays(qid,'freq')
array = '_'.join([array1,array2])
elif dm.name=='planck_hybrid':
season,patch,array = None,None,sints.arrays(qid,'freq')
else:
raise ValueError
lmin,lmax,hybrid,radial,friend,cfreq,fgroup,wrfit = aspecs(qid)
lmins.append(lmin)
lmaxs.append(lmax)
names.append(qid)
cfreqs.append(cfreq)
if bandpasses:
try:
fname = dm.get_bandpass_file_name(array)
bps.append("data/"+fname)
if (pa1_shift is not None) and 'PA1' in fname:
shifts.append(pa1_shift)
elif (pa2_shift is not None) and 'PA2' in fname:
shifts.append(pa2_shift)
elif (pa3_150_shift is not None) and ('PA3' in fname) and ('150' in fname):
shifts.append(pa3_150_shift)
elif (pa3_090_shift is not None) and ('PA3' in fname) and ('090' in fname):
shifts.append(pa3_90_shift)
else:
shifts.append(0)
except:
warn()
bps.append(None)
else:
try: bps.append(cfreq)
except:
warn()
bps.append(None)
kbeam = tutils.get_kbeam(qid,modlmap,sanitize=not(unsanitized_beam),version=beam_version,planck_pixwin=True)
if dm.name=='act_mr3':
lbeam = tutils.get_kbeam(qid,ells,sanitize=not(unsanitized_beam),version=beam_version,planck_pixwin=False) # note no pixwin but doesnt matter since no ccorr for planck
elif dm.name=='planck_hybrid':
lbeam = None
else:
raise ValueError
lbeams.append(lbeam)
kbeams.append(kbeam.copy())
# Make responses
responses = {}
def _get_response(comp,param_override=None):
if bandpasses:
if no_act_color_correction:
r = tfg.get_mix_bandpassed(bps, comp, bandpass_shifts=shifts,
param_dict_override=param_override)
else:
r = tfg.get_mix_bandpassed(bps, comp, bandpass_shifts=shifts,
ccor_cen_nus=cfreqs, ccor_beams=lbeams,
ccor_exps = [ccor_exp] * narrays,
param_dict_override=param_override)
else:
r = tfg.get_mix(bps, comp,param_dict_override=param_override)
return r
for comp in ['tSZ','CMB','CIB']:
responses[comp] = _get_response(comp,None)
from tilec.utils import is_planck
ilcgens = []
okcoadds = []
for splitnum in range(2):
covdir = covdirs[splitnum]
kcoadds = []
for i,qid in enumerate(arrays):
lmin = lmins[i]
lmax = lmaxs[i]
if is_planck(qid):
dm = sints.models[sints.arrays(qid,'data_model')](region=mask,calibrated=True)
_,kcoadd,_ = kspace.process(dm,region,qid,mask,
skip_splits=True,
splits_fname=sim_splits[i] if sim_splits is not None else None,
inpaint=False,fn_beam = None,
plot_inpaint_path = None,
split_set=splitnum)
else:
kcoadd_name = covdir + "kcoadd_%s.npy" % qid
kcoadd = enmap.enmap(np.load(kcoadd_name),wcs)
kmask = maps.mask_kspace(shape,wcs,lmin=lmin,lmax=lmax)
dtype = kcoadd.dtype
kcoadds.append((kcoadd.copy()*kmask)[sel])
kcoadds = enmap.enmap(np.stack(kcoadds),wcs)
okcoadds.append(kcoadds.copy())
# Read Covmat
ctheory = ilc.CTheory(modlmap)
nells = kcoadds[0].size
cov = np.zeros((narrays,narrays,nells))
for aindex1 in range(narrays):
for aindex2 in range(aindex1,narrays):
qid1 = names[aindex1]
qid2 = names[aindex2]
if is_planck(names[aindex1]) or is_planck(names[aindex2]) or all_analytic:
lmin,lmax,hybrid,radial,friend,f1,fgroup,wrfit = aspecs(qid1)
lmin,lmax,hybrid,radial,friend,f2,fgroup,wrfit = aspecs(qid2)
# If both are Planck and same array, get white noise from last bin
icov = ctheory.get_theory_cls(f1,f2,a_cmb=1,a_gal=0.8)*kbeams[aindex1]*kbeams[aindex2]
if aindex1==aindex2:
pcov = enmap.enmap(np.load(covdirs[2]+"tilec_hybrid_covariance_%s_%s.npy" % (names[aindex1],names[aindex2])),wcs)
pbin_edges = np.append(np.arange(500,3000,200) ,[3000,4000,5000,5800])
pbinner = stats.bin2D(omodlmap,pbin_edges)
w = pbinner.bin(pcov)[1][-1]
icov = icov + w
else:
icov = np.load(covdir+"tilec_hybrid_covariance_%s_%s.npy" % (names[aindex1],names[aindex2]))[sel]
if aindex1==aindex2:
icov[modlmap<lmins[aindex1]] = maxval
icov[modlmap>lmaxs[aindex1]] = maxval
cov[aindex1,aindex2] = icov
cov[aindex2,aindex1] = icov
assert np.all(np.isfinite(cov))
ilcgen = ilc.HILC(modlmap,np.stack(kbeams),cov=cov,responses=responses,invert=True)
ilcgens.append(ilcgen)
solutions = ['tSZ','tSZ-CMB','tSZ-CIB']
ypowers = {}
w2 = np.mean(mask**2.)
binner = stats.bin2D(modlmap,bin_edges)
np.random.seed(100)
blinding = np.random.uniform(0.8,1.2) if not(unblind) else 1
def _get_ypow(sname,dname,dresponse=None,dcmb=False):
if dresponse is not None:
assert dname is not None
for splitnum in range(2):
ilcgens[splitnum].add_response(dname,dresponse)
ykmaps = []
for splitnum in range(2):
if dcmb:
assert dname is not None
ykmap = ilcgens[splitnum].multi_constrained_map(okcoadds[splitnum],sname,[dname,"CMB"])
else:
if dname is None:
ykmap = ilcgens[splitnum].standard_map(okcoadds[splitnum],sname)
else:
ykmap = ilcgens[splitnum].constrained_map(okcoadds[splitnum],sname,dname)
ykmaps.append(ykmap.copy())
ypower = (ykmaps[0]*ykmaps[1].conj()).real / w2
return binner.bin(ypower)[1] * blinding
# The usual solutions
for solution in solutions:
sols = solution.split('-')
if len(sols)==2:
sname = sols[0]
dname = sols[1]
elif len(sols)==1:
sname = sols[0]
dname = None
else:
raise ValueError
ypowers[solution] = _get_ypow(sname,dname,dresponse=None)
# The CIB SED samples
if beta_samples is not None:
y_bsamples = []
y_bsamples_cmb = []
for beta in beta_samples:
pdict = tfg.default_dict.copy()
pdict['beta_CIB'] = beta
response = _get_response("CIB",param_override=pdict)
y_bsamples.append( _get_ypow("tSZ","iCIB",dresponse=response,dcmb=False) )
y_bsamples_cmb.append( _get_ypow("tSZ","iCIB",dresponse=response,dcmb=True) )
else:
y_bsamples = None
y_bsamples_cmb = None
return binner.centers,ypowers,y_bsamples,y_bsamples_cmb
ctheory = ilc.CTheory(ells)
aspecs = tutils.ASpecs().get_specs
bin_edges = np.arange(100,lmax,100)
binner = stats.bin2D(modlmap,bin_edges)
pl = io.Plotter(xyscale='linlog',xlabel='l',ylabel='D',scalefn = lambda x: x**2./2./np.pi)
for i,qid in enumerate(arrays):
kcoadds = []
for splitnum in range(2):
dm = sints.models[sints.arrays(qid,'data_model')](region=mask,calibrated=True)
lmin,lmax,hybrid,radial,friend,cfreq,fgroup,wrfit = aspecs(qid)
cltt = ctheory.get_theory_cls(cfreq,cfreq,a_cmb=1,a_gal=0.8)
_,kcoadd,_ = kspace.process(dm,region,qid,mask,
skip_splits=True,
splits_fname=None,
inpaint=False,fn_beam = None,
plot_inpaint_path = None,
split_set=splitnum)
kbeam = tutils.get_kbeam(qid,modlmap,sanitize=False,planck_pixwin=True)
kcoadds.append(kcoadd/kbeam)
power = (kcoadds[0]*kcoadds[1].conj()).real / w2
cents,p1d = binner.bin(power)
pl.add(cents,p1d,color="C%d" % i,label=str(cfreq))
pl.add(ells,cltt,color="C%d" % i,ls='--')
pl._ax.set_ylim(1,1e8)
pl.done("planckspec.png")
def build_and_save_cov(arrays,region,version,mask_version,
signal_bin_width,signal_interp_order,delta_ell,
rfit_lmin,
overwrite,memory_intensive,uncalibrated,
sim_splits=None,skip_inpainting=False,theory_signal="none",
unsanitized_beam=False,save_all=False,plot_inpaint=False,
isotropic_override=False,split_set=None,save_extra=False):
save_scratch = not(memory_intensive)
savedir = tutils.get_save_path(version,region)
if save_scratch:
scratch = tutils.get_scratch_path(version,region)
covscratch = scratch
else:
covscratch = None
if not(overwrite):
assert not(os.path.exists(savedir)), \
"This version already exists on disk. Please use a different version identifier."
try: os.makedirs(savedir)
except:
if overwrite: pass
else: raise
if save_scratch:
try: os.makedirs(scratch)
except: pass
mask = sints.get_act_mr3_crosslinked_mask(region,
version=mask_version,
kind='binary_apod')
shape,wcs = mask.shape,mask.wcs
aspecs = tutils.ASpecs().get_specs
with bench.show("ffts"):
kcoadds = []
kdiffs = []
wins = []
lmins = []
lmaxs = []
hybrids = []
do_radial_fit = []
freqs = []
rfit_wnoise_widths = []
save_names = [] # to make sure nothing is overwritten
friends = {} # what arrays are each correlated with?
names = arrays.split(',')
print("Calculating FFTs for " , arrays)
fbeam = lambda qname,x: tutils.get_kbeam(qname,x,sanitize=not(unsanitized_beam),planck_pixwin=True)
for i,qid in enumerate(arrays.split(',')):
dm = sints.models[sints.arrays(qid,'data_model')](region=mask,calibrated=not(uncalibrated))
lmin,lmax,hybrid,radial,friend,cfreq,fgroup,wrfit = aspecs(qid)
print(f"{qid} lmin {lmin} lmax {lmax}")
assert (type(radial)==bool) or (type(radial)==np.bool_)
do_radial_fit.append(radial)
friends[qid] = friend
hybrids.append(hybrid)
freqs.append(fgroup)
rfit_wnoise_widths.append(wrfit)
kdiff,kcoadd,win = kspace.process(dm,region,qid,mask,
skip_splits=False,
splits_fname=sim_splits[i] if sim_splits is not None else None,
inpaint=not(skip_inpainting),fn_beam = lambda x: fbeam(qid,x),
plot_inpaint_path = savedir if plot_inpaint else None,
split_set=split_set)
print("Processed ",qid)
if save_scratch:
kcoadd_name = savedir + "kcoadd_%s.npy" % qid
kdiff_name = scratch + "kdiff_%s.npy" % qid
win_name = scratch + "win_%s.npy" % qid
assert win_name not in save_names
assert kcoadd_name not in save_names
assert kdiff_name not in save_names
np.save(win_name,win)
np.save(kcoadd_name,kcoadd)
np.save(kdiff_name,kdiff)
wins.append(win_name)
kcoadds.append(kcoadd_name)
kdiffs.append(kdiff_name)
save_names.append(win_name)
save_names.append(kcoadd_name)
save_names.append(kdiff_name)
print("Saved ",qid)
else:
wins.append(win.copy())
kcoadds.append(kcoadd.copy())
kdiffs.append(kdiff.copy())
lmins.append(lmin)
lmaxs.append(lmax)
if sim_splits is not None: del sim_splits
print("Done with ffts.")
# print("Exiting because I just want to see inpainted stuff.")
# sys.exit()
# Decide what pairs to do hybrid smoothing for
anisotropic_pairs = get_aniso_pairs(arrays.split(','),hybrids,friends)
print("Anisotropic pairs: ",anisotropic_pairs)
enmap.write_map(savedir+"tilec_mask.fits",mask)
save_fn = lambda x,a1,a2: np.save(savedir+"tilec_hybrid_covariance_%s_%s.npy" % (names[a1],names[a2]),enmap.enmap(x,wcs))
print("Building covariance...")
with bench.show("build cov"):
ilc.build_cov(names,kdiffs,kcoadds,fbeam,mask,lmins,lmaxs,freqs,anisotropic_pairs,
delta_ell,
do_radial_fit,save_fn,
signal_bin_width=signal_bin_width,
signal_interp_order=signal_interp_order,
rfit_lmaxes=lmaxs,
rfit_wnoise_widths=rfit_wnoise_widths,
rfit_lmin=rfit_lmin,
rfit_bin_width=None,
verbose=True,
debug_plots_loc=False,
separate_masks=False,theory_signal=theory_signal,scratch_dir=covscratch,
isotropic_override=isotropic_override,save_extra=save_extra,wins=wins)
if not(save_all): shutil.rmtree(scratch)
bsig = 0.15 * bcent
trans = np.exp(-(nu_ghz-bcent)**2./2./bsig**2.)
pl.add(nu_ghz,trans/trans.max())
pl.done("band_%s.png" % qid)
if gauss_beam:
if 'p' in qid:
fwhm = dm.fwhms[array]
else:
fwhm = 1.4 * (150./cfreq)
lbeam = maps.gauss_beam(ells,fwhm)
else:
lbeam = tutils.get_kbeam(qid,ells,sanitize=False,planck_pixwin=False)
with bench.show("beamint"):
fbnus = maps.interp(ells,lbeam[None,:],fill_value=(lbeam[0],lbeam[-1]))
bnus = fbnus((cfreq/nu_ghz)*ells[:,None])[0].swapaxes(0,1)
bnus = bnus / bnus[:,:1]
pl = io.Plotter(xlabel='l',ylabel='b')
for i in range(bnus.shape[0]):
if trans[i]>1e-1: pl.add(ells,bnus[i]/lbeam)
pl.hline(y=1)
pl._ax.set_ylim(0.8,1.2)
pl.done("abeams_%s.png" % qid)
from __future__ import print_function
from orphics import maps, io, cosmology
from pixell import enmap
import numpy as np
import os, sys
from tilec import utils as tutils
qids = 'boss_01,boss_02,boss_03,boss_04,d56_01,d56_02,d56_03,d56_04,d56_05,d56_06'.split(
',')
ells = np.arange(2, 8000, 1)
for qid in qids:
old = tutils.get_kbeam(qid,
ells,
sanitize=False,
version='190220',
planck_pixwin=False)
new = tutils.get_kbeam(qid,
ells,
sanitize=False,
version='190809',
planck_pixwin=False)
pl = io.Plotter(xlabel='l', ylabel='r')
pl.add(ells, (new - old) / old, label=qid)
pl.done("beamcomp_%s.png" % qid)
noise, ivars = ngen[dmname].generate_sim(season=season,
patch=patch,
array=farray,
seed=noise_seed,
apply_ivar=False)
ivars = ivars[0]
noise = noise[0]
splits = actnoise.apply_ivar_window(signal[None, None] + noise[None],
ivars[None])
nsplits = actnoise.apply_ivar_window(noise[None], ivars[None])
enmap.write_map("temp_planck_sim.fits", splits[0])
print(signal.shape, noise.shape, ivars.shape, splits.shape)
fbeam = lambda qname, x: tutils.get_kbeam(
qname, x, sanitize=True, planck_pixwin=True)
kdiff, kcoadd, win = kspace.process(ngen[dmname].dm,
region,
qid,
mask,
skip_splits=False,
splits_fname="temp_planck_sim.fits",
inpaint=False,
fn_beam=lambda x: fbeam(qid, x),
plot_inpaint_path=None,
split_set=None)
from actsims import noise as simnoise
ncov = simnoise.noise_power(kdiff,
skip_splits=False,
do_fft_splits=False)
kdiffs.append(kdiff.copy())
# ksplits.append(ksplit.copy())
lmin, lmax, hybrid, radial, friend, cfreq, fgroup = get_specs(qid)
freqs.append(fgroup)
dtype = kcoadd.dtype
kcoadds.append(kcoadd.copy())
masks.append(apod.copy())
lmins.append(lmin)
lmaxs.append(lmax)
hybrids.append(hybrid)
do_radial_fit.append(radial)
friends[qid] = friend
bps.append(cfreq) # change to bandpass file
kbeams.append(get_kbeam(qid, modlmap))
fbeam = lambda qname, x: get_kbeam(qname, x)
if len(aids) == 0: continue # this tile is empty
# Then build the covmat placeholder
narrays = len(aids)
cov = maps.SymMat(narrays, eshape[-2:])
def save_fn(x, a1, a2):
cov[a1, a2] = enmap.enmap(x, ewcs).copy()
print(comm.rank, ": Tile %d has arrays " % i, aids)
anisotropic_pairs = pipeline.get_aniso_pairs(aids, hybrids, friends)
def stack(x):
return enmap.enmap(np.stack(x), ewcs)
lexps2 = []
exp1 = {148: -1, 93: -1}
exp2 = {148: -0.8, 93: -0.9}
aspecs = tutils.ASpecs().get_specs
ells = np.arange(0, 8000, 1)
for qid in qids:
dm = sints.models[sints.arrays(qid, 'data_model')]()
lmin, lmax, hybrid, radial, friend, cfreq, fgroup, wrfit = aspecs(qid)
cfreqs.append(cfreq)
season, array1, array2 = sints.arrays(qid, 'season'), sints.arrays(
qid, 'array'), sints.arrays(qid, 'freq')
array = '_'.join([array1, array2])
bps.append("data/" + dm.get_bandpass_file_name(array))
lbeams.append(tutils.get_kbeam(qid, ells, sanitize=False))
lexps1.append(exp1[cfreq])
lexps2.append(exp2[cfreq])
r1 = tfg.get_mix_bandpassed(bps,
'tSZ',
ccor_cen_nus=cfreqs,
ccor_beams=lbeams,
ccor_exps=lexps1)
r2 = tfg.get_mix_bandpassed(bps,
'tSZ',
ccor_cen_nus=cfreqs,
ccor_beams=lbeams,
ccor_exps=lexps2)
qid2 = qids[j]
#if i!=j: continue # !!!
p2d = enmap.enmap(np.load(cfunc(qid1, qid2)), wcs)
modlmap = p2d.modlmap()
#print(p2d[modlmap<80])
modlmap = enmap.modlmap(shape, wcs)
bin_edges = np.append(np.append([0, 20, 40], np.arange(80, 500, 20)),
np.arange(500, 25000, 20))
#bin_edges = np.append(np.append([0,20,40],np.arange(80,300,20)),np.arange(300,25000,20))
binner = stats.bin2D(modlmap, bin_edges)
# b1 = tutils.get_kbeam(qid1,modlmap)
# b2 = tutils.get_kbeam(qid2,modlmap)
b1 = tutils.get_kbeam(qid1, modlmap,
planck_pixwin=True) / tutils.get_kbeam(
qid1, modlmap, planck_pixwin=False)
b2 = tutils.get_kbeam(qid2, modlmap,
planck_pixwin=True) / tutils.get_kbeam(
qid2, modlmap, planck_pixwin=False)
p2d = p2d / b1 / b2
k1 = enmap.enmap(np.load(kfunc(qid1)) / b1, wcs)
k2 = enmap.enmap(np.load(kfunc(qid2)) /
b2, wcs) if qid1 != qid2 else k1
kp2d = np.real(k1 * k2.conj()) / w2
#pl = io.Plotter(xyscale='linlog',xlabel='$\\ell$',ylabel='$C_{\\ell}$')
pl = io.Plotter(xyscale='loglog',
xlabel='$\\ell$',
tpath = os.environ[
'WORK'] + "/data/depot/tilec/map_v1.0.0_rc_joint_%s/" % region
"""
We compare masks
and make sure they are identical
"""
omask = enmap.read_map("%smask_s14&15_%s.fits" % (opath, region))
tmask = enmap.read_map("%stilec_mask.fits" % tpath)
assert np.all(np.isclose(omask, tmask))
modlmap = omask.modlmap()
bin_edges = np.arange(400, 8000, 200)
binner = stats.bin2D(modlmap, bin_edges)
Nplot = 300
kbeam = tutils.get_kbeam("d56_05", modlmap, sanitize=False)
w2 = np.mean(omask**2.)
ls, bells = np.loadtxt(
"%stilec_single_tile_%s_cmb_map_v1.0.0_rc_joint_beam.txt" %
(tpath, region),
unpack=True)
tkbeam = maps.interp(ls, bells)(modlmap)
ls, bells = np.loadtxt(
"%stilec_single_tile_%s_cmb_deprojects_comptony_map_v1.0.0_rc_joint_beam.txt"
% (tpath, region),
unpack=True)
tkbeam_nosz = maps.interp(ls, bells)(modlmap)
def pow(x):
k = enmap.fft(x, normalize='phys')
qid2 = arrays[qind2]
#if qid1[0]!='p' or qid2[0]!='p': continue
ufname = "/scratch/r/rbond/msyriac/dump/unsmoothed_beamed_%s_%s.fits" % (qid1,qid2)
sfname = "/scratch/r/rbond/msyriac/dump/smoothed_beamed_%s_%s.fits" % (qid1,qid2)
nfname = "/scratch/r/rbond/msyriac/dump/smoothed_noise_%s_%s.fits" % (qid1,qid2)
ucov = enmap.read_map(ufname)
scov = enmap.read_map(sfname)
ncov = enmap.read_map(nfname)
cents,up1d = binner.bin(ucov)
cents,sp1d = binner.bin(scov)
cents,np1d = binner.bin(ncov)
lmin,lmax,hybrid,radial,friend,f1,fgroup,wrfit = tutils.get_specs(qid1)
lmin,lmax,hybrid,radial,friend,f2,fgroup,wrfit = tutils.get_specs(qid2)
beam1 = tutils.get_kbeam(qid1,cents,sanitize=False,planck_pixwin=True)
beam2 = tutils.get_kbeam(qid2,cents,sanitize=False,planck_pixwin=True)
kc1 = "/scratch/r/rbond/msyriac/data/depot/tilec/test_v1.1.0_rc_deep56/kcoadd_%s.npy" % qid1
kc2 = "/scratch/r/rbond/msyriac/data/depot/tilec/test_v1.1.0_rc_deep56/kcoadd_%s.npy" % qid2
p2d = np.real(np.load(kc1)*np.load(kc2).conj())/w2
cents,pow1d = binner.bin(p2d)
p2d = np.load("/scratch/r/rbond/msyriac/data/depot/tilec/test_v1.1.0_rc_deep56/tilec_hybrid_covariance_%s_%s.npy" % (qid1,qid2))
cents,cov1d = binner.bin(p2d)
# fbeam1 = tutils.get_kbeam(qid1,ells,sanitize=False,planck_pixwin=True)
# fbeam2 = tutils.get_kbeam(qid2,ells,sanitize=False,planck_pixwin=True)
pl = io.Plotter(xyscale='loglog',scalefn = lambda x: x**2./2./np.pi,xlabel='l',ylabel='D')
def compute_map(self,oshape,owcs,qid,pixwin_taper_deg=0.3,pixwin_pad_deg=0.3,
include_cmb=True,include_tsz=True,include_fgres=True,sht_beam=True):
"""
1. get total alm
2. apply beam, and pixel window if Planck
3. ISHT
4. if ACT, apply a small taper and apply pixel window in Fourier space
"""
# pad to a slightly larger geometry
tot_pad_deg = pixwin_taper_deg + pixwin_pad_deg
res = maps.resolution(oshape,owcs)
pix = np.deg2rad(tot_pad_deg)/res
omap = enmap.pad(enmap.zeros((3,)+oshape,owcs),pix)
ishape,iwcs = omap.shape[-2:],omap.wcs
# get data model
dm = sints.models[sints.arrays(qid,'data_model')](region_shape=ishape,region_wcs=iwcs,calibrated=True)
# 1. get total alm
array_index = self.qids.index(qid)
tot_alm = int(include_cmb)*self.alms['cmb']
if include_tsz:
try:
assert self.tsz_fnu.ndim==2
tot_alm[0] = tot_alm[0] + hp.almxfl(self.alms['comptony'][0] ,self.tsz_fnu[array_index])
except:
tot_alm[0] = tot_alm[0] + self.alms['comptony'][0] * self.tsz_fnu[array_index]
if self.cfgres is not None: tot_alm[0] = tot_alm[0] + int(include_fgres)*self.alms['fgres'][array_index]
assert tot_alm.ndim==2
assert tot_alm.shape[0]==3
ells = np.arange(self.lmax+1)
# 2. get beam, and pixel window for Planck
if sht_beam:
beam = tutils.get_kbeam(qid,ells,sanitize=False,planck_pixwin=False) # NEVER SANITIZE THE BEAM IN A SIMULATION!!!
for i in range(3): tot_alm[i] = hp.almxfl(tot_alm[i],beam)
if dm.name=='planck_hybrid':
pixwint,pixwinp = hp.pixwin(nside=tutils.get_nside(qid),lmax=self.lmax,pol=True)
tot_alm[0] = hp.almxfl(tot_alm[0],pixwint)
tot_alm[1] = hp.almxfl(tot_alm[1],pixwinp)
tot_alm[2] = hp.almxfl(tot_alm[2],pixwinp)
# 3. ISHT
omap = curvedsky.alm2map(np.complex128(tot_alm),omap,spin=[0,2])
assert omap.ndim==3
assert omap.shape[0]==3
if not(sht_beam):
taper,_ = maps.get_taper_deg(ishape,iwcs,taper_width_degrees=pixwin_taper_deg,pad_width_degrees=pixwin_pad_deg)
modlmap = omap.modlmap()
beam = tutils.get_kbeam(qid,modlmap,sanitize=False,planck_pixwin=True)
kmap = enmap.fft(omap*taper,normalize='phys')
kmap = kmap * beam
# 4. if ACT, apply a small taper and apply pixel window in Fourier space
if dm.name=='act_mr3':
if sht_beam: taper,_ = maps.get_taper_deg(ishape,iwcs,taper_width_degrees=pixwin_taper_deg,pad_width_degrees=pixwin_pad_deg)
pwin = tutils.get_pixwin(ishape[-2:])
if sht_beam:
omap = maps.filter_map(omap*taper,pwin)
else:
kmap = kmap * pwin
if not(sht_beam): omap = enmap.ifft(kmap,normalize='phys').real
return enmap.extract(omap,(3,)+oshape[-2:],owcs)
xlabel='l',
ylabel='D',
scalefn=lambda x: x**2.)
qid1 = qids[i]
qid2 = qids[j]
array1 = sints.arrays(qid1, 'freq')
array2 = sints.arrays(qid2, 'freq')
splits1 = dm.get_splits([array1], ncomp=1, srcfree=True)[0, :, 0]
wins1 = dm.get_splits_ivar([array1])[0, :, 0]
splits2 = dm.get_splits([array2], ncomp=1, srcfree=True)[0, :, 0]
wins2 = dm.get_splits_ivar([array2])[0, :, 0]
fbeam1 = lambda x: tutils.get_kbeam(
qid1, x, sanitize=True, planck_pixwin=True)
fbeam2 = lambda x: tutils.get_kbeam(
qid2, x, sanitize=True, planck_pixwin=True)
ks1 = enmap.fft(splits1 * mask, normalize='phys') / fbeam1(modlmap)
ks2 = enmap.fft(splits2 * mask, normalize='phys') / fbeam2(modlmap)
p = ((ks1[0] * ks2[1].conj()).real +
(ks1[1] * ks2[0].conj()).real) / 2 / w2
cents, p1d = cbinner.bin(p)
ls, s1d = np.loadtxt(f"{fpath}/s1d_{qid1}_{qid2}.txt", unpack=True)
pl.add(ls,
s1d,
Ny,Nx = modlmap.shape
for seed in seeds:
for i in range(narrays):
for j in range(i,narrays):
pl = io.Plotter('Dell')
qid1 = qids[i]
qid2 = qids[j]
beam1 = tutils.get_kbeam(qid1,modlmap,sanitize=False,planck_pixwin=True)
beam2 = tutils.get_kbeam(qid2,modlmap,sanitize=False,planck_pixwin=True)
fname = f"/scratch/r/rbond/msyriac/data/scratch/tilec/test_sim_galtest_nofg_{version}_00_00{seed}_deep56/scovs_{i}_{j}.npy"
# if qid1=='p04' and qid2=='p04':
# pass
# else:
# continue
scov = enmap.enmap(np.load(fname),modlmap.wcs)/beam1/beam2
# io.power_crop(scov,500,"detp04p2d.png",ftrans=True)
# continue
cents,s1d = binner.bin(scov)
pl.add(cents,s1d,label='single array')