def load_survey_mask_core(qid):
# load mask by specifying qid
model, season, array, patch, freq = local.qid_info(qid)
if model == 'dr5':
mask = interfaces.get_binary_apodized_mask(qid)
elif model == 'act_mr3':
mask = interfaces.get_act_mr3_crosslinked_mask(patch,
version='padded_v1',
kind='binary_apod',
season=season,
array=array + "_" +
freq,
pad=None)
return mask
"""
This script calculates all ACT and Planck cross-spectra.
"""
import argparse
# Parse command line
parser = argparse.ArgumentParser(description='Do a thing.')
parser.add_argument("region", type=str,help='boss or deep56?')
parser.add_argument("--fft", action='store_true',help='A flag.')
args = parser.parse_args()
fftstr = "_fft" if args.fft else ""
mask_version = interfaces.dconfig['act_mr3']['default_mask_version']
mask = interfaces.get_act_mr3_crosslinked_mask(args.region)
identifier = args.region+"_"+mask_version
comm = mpi.MPI.COMM_WORLD
rank = comm.Get_rank()
numcores = comm.Get_size()
#bin_edges = np.arange(80,8000,100)
lefts,rights = np.loadtxt("data/BIN_ACTPOL_50_4_SC_low_ell_startAt2.txt",unpack=True,usecols=[0,1])
bin_edges = np.append(lefts,rights[-1:])
print(bin_edges)
# Calculate and store mcm
if not(args.fft):
with io.nostdout():
model = "act_mr3"
# Get file name convention
pout, cout, sout = noise.get_save_paths(model,
version,
coadd,
season=season,
patch=patch,
array=args.array,
mkdir=True,
overwrite=True,
mask_patch=mask_patch)
# Get data model
mask = sints.get_act_mr3_crosslinked_mask(mask_patch,
version=mask_version,
kind="binary_apod",
season=season,
array=args.array + "_f150",
pad=args.mask_pad)
# print(mask.shape)
mask = mask[400:-400, 5000:-5000]
mask *= enmap.apod(mask, 400)
# print(mask.shape)
noise.plot(pout + "_mask", mask, grid=True)
dm = sints.models[model](region=mask)
# Get a NoiseGen model
emask = mask
ngen = noise.NoiseGen(version=version,
model=model,
extract_region=emask,
ncache=1,
from __future__ import print_function
from orphics import maps, io, cosmology
from pixell import enmap
import numpy as np
import os, sys
import soapack.interfaces as sints
from enlib import bench
ntrials = 10
loc = "/scratch/r/rbond/msyriac/data/depot/tilec/temp/"
mask = sints.get_act_mr3_crosslinked_mask("boss")
for i in range(ntrials):
with bench.show("write"):
enmap.write_map(loc + "trial_%d.fits" % i, mask)
for i in range(ntrials):
with bench.show("read"):
mask = enmap.read_map(loc + "trial_%d.fits" % i)
for i in range(ntrials):
os.remove(loc + "trial_%d.fits" % i)
def build_and_save_cov(arrays,region,version,mask_version,
signal_bin_width,signal_interp_order,delta_ell,
rfit_lmin,
overwrite,memory_intensive,uncalibrated,
sim_splits=None,skip_inpainting=False,theory_signal="none",
unsanitized_beam=False,save_all=False,plot_inpaint=False,
isotropic_override=False,split_set=None,save_extra=False):
save_scratch = not(memory_intensive)
savedir = tutils.get_save_path(version,region)
if save_scratch:
scratch = tutils.get_scratch_path(version,region)
covscratch = scratch
else:
covscratch = None
if not(overwrite):
assert not(os.path.exists(savedir)), \
"This version already exists on disk. Please use a different version identifier."
try: os.makedirs(savedir)
except:
if overwrite: pass
else: raise
if save_scratch:
try: os.makedirs(scratch)
except: pass
mask = sints.get_act_mr3_crosslinked_mask(region,
version=mask_version,
kind='binary_apod')
shape,wcs = mask.shape,mask.wcs
aspecs = tutils.ASpecs().get_specs
with bench.show("ffts"):
kcoadds = []
kdiffs = []
wins = []
lmins = []
lmaxs = []
hybrids = []
do_radial_fit = []
freqs = []
rfit_wnoise_widths = []
save_names = [] # to make sure nothing is overwritten
friends = {} # what arrays are each correlated with?
names = arrays.split(',')
print("Calculating FFTs for " , arrays)
fbeam = lambda qname,x: tutils.get_kbeam(qname,x,sanitize=not(unsanitized_beam),planck_pixwin=True)
for i,qid in enumerate(arrays.split(',')):
dm = sints.models[sints.arrays(qid,'data_model')](region=mask,calibrated=not(uncalibrated))
lmin,lmax,hybrid,radial,friend,cfreq,fgroup,wrfit = aspecs(qid)
print(f"{qid} lmin {lmin} lmax {lmax}")
assert (type(radial)==bool) or (type(radial)==np.bool_)
do_radial_fit.append(radial)
friends[qid] = friend
hybrids.append(hybrid)
freqs.append(fgroup)
rfit_wnoise_widths.append(wrfit)
kdiff,kcoadd,win = kspace.process(dm,region,qid,mask,
skip_splits=False,
splits_fname=sim_splits[i] if sim_splits is not None else None,
inpaint=not(skip_inpainting),fn_beam = lambda x: fbeam(qid,x),
plot_inpaint_path = savedir if plot_inpaint else None,
split_set=split_set)
print("Processed ",qid)
if save_scratch:
kcoadd_name = savedir + "kcoadd_%s.npy" % qid
kdiff_name = scratch + "kdiff_%s.npy" % qid
win_name = scratch + "win_%s.npy" % qid
assert win_name not in save_names
assert kcoadd_name not in save_names
assert kdiff_name not in save_names
np.save(win_name,win)
np.save(kcoadd_name,kcoadd)
np.save(kdiff_name,kdiff)
wins.append(win_name)
kcoadds.append(kcoadd_name)
kdiffs.append(kdiff_name)
save_names.append(win_name)
save_names.append(kcoadd_name)
save_names.append(kdiff_name)
print("Saved ",qid)
else:
wins.append(win.copy())
kcoadds.append(kcoadd.copy())
kdiffs.append(kdiff.copy())
lmins.append(lmin)
lmaxs.append(lmax)
if sim_splits is not None: del sim_splits
print("Done with ffts.")
# print("Exiting because I just want to see inpainted stuff.")
# sys.exit()
# Decide what pairs to do hybrid smoothing for
anisotropic_pairs = get_aniso_pairs(arrays.split(','),hybrids,friends)
print("Anisotropic pairs: ",anisotropic_pairs)
enmap.write_map(savedir+"tilec_mask.fits",mask)
save_fn = lambda x,a1,a2: np.save(savedir+"tilec_hybrid_covariance_%s_%s.npy" % (names[a1],names[a2]),enmap.enmap(x,wcs))
print("Building covariance...")
with bench.show("build cov"):
ilc.build_cov(names,kdiffs,kcoadds,fbeam,mask,lmins,lmaxs,freqs,anisotropic_pairs,
delta_ell,
do_radial_fit,save_fn,
signal_bin_width=signal_bin_width,
signal_interp_order=signal_interp_order,
rfit_lmaxes=lmaxs,
rfit_wnoise_widths=rfit_wnoise_widths,
rfit_lmin=rfit_lmin,
rfit_bin_width=None,
verbose=True,
debug_plots_loc=False,
separate_masks=False,theory_signal=theory_signal,scratch_dir=covscratch,
isotropic_override=isotropic_override,save_extra=save_extra,wins=wins)
if not(save_all): shutil.rmtree(scratch)
array=array,
mkdir=True,
overwrite=args.overwrite,
mask_patch=mask_patch)
# Get mask
mask = sints.get_binary_apodized_mask(args.qid)
# Get data model
dm = sints.models[args.model](region=mask)
# Get a NoiseGen model
if args.extract_mask is not None:
emask = sints.get_act_mr3_crosslinked_mask(mask_patch,
version=args.extract_mask,
kind=args.mask_kind,
season=season,
array=array + "_f150")
eshape, ewcs = emask.shape, emask.wcs
else:
emask = mask
ngen = noise.NoiseGen(qid=qid,
version=version,
model=args.model,
extract_region=emask,
ncache=1,
verbose=True)
# Get split data
splits = enmap.enmap([dm.get_splits(q) for q in qid])
nsplits = dm.ainfo(args.qid, 'nsplits')
import healpy as hp
from pixell import enmap, curvedsky
from orphics import maps,io
from soapack import interfaces as sints
from tilec.pipeline import JointSim
qids = ['d56_03','p05']
jsim = JointSim(qids,None)
jsim.update_signal_index(0)
mask = sints.get_act_mr3_crosslinked_mask('deep6')
imap = jsim.compute_map(mask.shape,mask.wcs,'p05',include_cmb=True,include_tsz=True,include_fgres=True)
io.hplot(imap,'planck')
imap = jsim.compute_map(mask.shape,mask.wcs,'d56_03',include_cmb=True,include_tsz=True,include_fgres=True)
io.hplot(imap,'act')
from __future__ import print_function
from orphics import maps, io, cosmology
from pixell import enmap
import numpy as np
import os, sys
from soapack import interfaces as sints
mask = sints.get_act_mr3_crosslinked_mask("boss", version='padded_v1')
shape, wcs = mask.shape, mask.wcs
print(np.rad2deg(enmap.pix2sky(shape, wcs, (shape[-2], shape[-1]))))
sys.exit()
io.hplot(mask, "mask.png")
print(mask.shape)
dm = sints.PlanckHybrid(region=mask)
ivars = dm.get_splits_ivar(['100'], ncomp=1)
splits = dm.get_splits(['100'], ncomp=1)
# print(ivars.shape)
# print(ivars[ivars<=0].size/ivars.size*100.)
# print(ivars[ivars<=0].size)
# print(splits[ivars<=0])
# print(splits[ivars<=0].max())
# splits[ivars>0] = np.nan
# splits[ivars<=0] = 1
io.hplot(splits[0, 1, 0], "boss.png")
parser.add_argument("--mask-pad", type=int, default=None,
help='Mask additional padding. No padding is applied to the extracted mask if any specified.')
parser.add_argument("--extract-mask", type=str, default=None,
help='Make sims on the big mask but do all the analysis on an extract of this version.')
parser.add_argument("--binary-percentile", type=float, default=10.,help='Binary percentile for sim masking.')
parser.add_argument("--season", type=str,help='Season')
parser.add_argument("--array", type=str,help='Array')
parser.add_argument("--patch", type=str,help='Patch')
args = parser.parse_args()
mask_patch = args.patch if args.mask_patch is None else args.mask_patch
with bench.show("region"):
if args.extract_mask is not None:
region_map = sints.get_act_mr3_crosslinked_mask(mask_patch,
version=args.extract_mask,
kind=args.mask_kind,
season=args.season)
else:
region_map = None
with bench.show("ngen init"):
ngen = noise.NoiseGen(args.version,extract_region=region_map,ncache=1)
with bench.show("make a sim"):
ngen.generate_sim(season=args.season,patch=args.patch,array=args.array,seed=1)
with bench.show("make another sim"):
ngen.generate_sim(season=args.season,patch=args.patch,array=args.array,seed=2)
# Get file name convention
pout, cout, sout = noise.get_save_paths(args.model,
version,
coadd,
season=args.season,
patch=args.patch,
array=args.array,
mkdir=True,
overwrite=args.overwrite,
mask_patch=mask_patch)
# Get data model
mask = sints.get_act_mr3_crosslinked_mask(mask_patch,
version=args.mask_version,
kind=args.mask_kind,
season=args.season,
array=args.array + "_f150",
pad=args.mask_pad)
dm = sints.models[args.model](region=mask, calibrated=args.calibrated)
n2d = enmap.read_map(pout + '_n2d.fits')
covsq = enmap.multi_pow(n2d.copy(), 0.5)
covsq[:, :, n2d.modlmap() < 2] = 0
print('D', np.argwhere(~np.isfinite(covsq)))
covsq = enmap.enmap(covsq, n2d.wcs)
print('E', np.argwhere(~np.isfinite(covsq)))
#enmap.write_map(pout+'_covsqrt_tmp.fits',covsq)
#covsq = enmap.read_map(pout+'_covsqrt_tmp.fits')
#covsq = enmap.read_map(pout+'_covsqrt.fits')
from tilec import pipeline,utils as tutils
region = 'boss'
#region = 'deep56'
fg_res_version = 'test'
#qids = 'd56_01,d56_02,d56_03,d56_04,d56_05,d56_06,p01,p02,p03,p04,p05,p06,p07,p08'.split(',')
#qids = 'boss_01'.split(',')
qids = 'boss_01,boss_02,boss_03,boss_04,p01,p02,p03,p04,p05,p06,p07,p08'.split(',')
#qids = 'boss_04,p01,p02,p03,p04,p05,p06,p07,p08'.split(',')
#qids = 'boss_04,p08'.split(',')
#qids = 'boss_03,boss_04'.split(',')
#qids = 'd56_01,d56_02,d56_03,d56_04,d56_05,d56_06'.split(',')
#qids = 'd56_02,d56_03,d56_04'.split(',')
#qids = 'd56_04,p01,p02,p03,p04,p05,p06,p07,p08'.split(',')
mask = sints.get_act_mr3_crosslinked_mask(region,
kind='binary_apod')
modlmap = mask.modlmap()
jsim = pipeline.JointSim(qids,fg_res_version+"_"+region,bandpassed=True)
jsim0 = pipeline.JointSim(qids,None,bandpassed=True)
comm,rank,my_tasks = mpi.distribute(len(qids))
bin_edges = np.arange(20,8000,20)
binner = stats.bin2D(modlmap,bin_edges)
jsim.update_signal_index(0,set_idx=0)
jsim0.update_signal_index(0,set_idx=0)
from __future__ import print_function
import matplotlib
matplotlib.use('Agg')
from orphics import maps, io, cosmology, pixcov
from pixell import enmap, reproject
import numpy as np
import os, sys
import soapack.interfaces as sints
from enlib import bench
from actsims import noise
for season in ['s13', 's14', 's15']:
for apatch in ['boss', 'deep1', 'deep5', 'deep6', 'deep8', 'deep56']:
#for apatch in ['deep1','deep5','deep6','deep8','deep56','boss']:
mask = sints.get_act_mr3_crosslinked_mask(apatch)
ellcen = 5000
ellsig = 1000
modlmap = mask.modlmap()
ells = np.arange(0, modlmap.max())
mfilter = maps.interp(ells,
np.exp(-(ells - ellcen)**2. / 2. /
ellsig**2.))(modlmap)
for array in ['pa1_f150', 'pa2_f150', 'pa3_f150', 'pa3_f090']:
dm = sints.ACTmr3(calibrated=False, region=mask)
fname = "%s_%s_%s" % (season, apatch, array)
try:
splits = dm.get_splits(season=season,
patch=apatch,
arrays=[array],
ncomp=3,
srcfree=False)[0, :, :, ...]
centers = (bin_edges[1:] + bin_edges[:-1]) / 2.
w2 = np.mean(mask**2.)
modlmap = enmap.modlmap(map1.shape, map1.wcs)
binned_power = bin(power / w2 / beamf1(modlmap) / beamf2(modlmap), modlmap,
bin_edges)
return centers, binned_power
# # ACTxPlanck
# coadded ACT x coadded Planck
ACT_planck = {}
for patch in ['deep56', 'boss']:
mask = interfaces.get_act_mr3_crosslinked_mask(patch)
dma = interfaces.ACTmr3(region=mask)
dmp = interfaces.PlanckHybrid(region=mask)
# we loop over all pairs of Planck x ACT
for planckfreq in ['030', '044', '070', '100', '143', '217', '353',
'545']: # no '857'
planckbeam = lambda x: dmp.get_beam(x, planckfreq)
planckmap = get_planck_coadd(planckfreq, dmp)[0, :, :]
for actseason in ['s14', 's15']:
for array in ['pa1_f150', 'pa2_f150', 'pa3_f090', 'pa3_f150']:
try:
actbeam = lambda x: dma.get_beam(x, actseason, patch, array
)
actmap = dma.get_coadd(
actseason, patch, array, ncomp=1,
qmap = enmap.read_map("%sdataCoadd_combined_Q_s14&15_%s.fits" %
(opath, region))
umap = enmap.read_map("%sdataCoadd_combined_U_s14&15_%s.fits" %
(opath, region))
#oq_noise2d = pow(qmap)
#ou_noise2d = pow(umap)
_, oe_p2d, ob_p2d = get_pol_powers(tmap, qmap, umap, kbeam, w2)
io.power_crop(oe_p2d, Nplot, "oe_noise2d.png", ftrans=True)
#io.power_crop(ob_p2d,Nplot,"ob_noise2d.png",ftrans=True)
dmask = sints.get_act_mr3_crosslinked_mask(region='deep6')
dw2 = np.mean(dmask**2)
dm = sints.ACTmr3(calibrated=True, region=dmask)
imap = dm.get_coadd(season='s13',
array='pa1_f150',
patch='deep6',
srcfree=True)
tmap = imap[0] * dmask
qmap = imap[1] * dmask
umap = imap[2] * dmask
dbeam = tutils.get_kbeam("d6", dmask.modlmap(), sanitize=False)
_, de_p2d, db_p2d = get_pol_powers(tmap, qmap, umap, dbeam, dw2)
io.power_crop(de_p2d, Nplot, "de_noise2d.png", ftrans=True)
dbinner = stats.bin2D(dmask.modlmap(), bin_edges)
else:
act_arrays.append(qid)
do_act_only = (len(act_arrays)>0) and not(args.skip_only)
do_planck_only = (len(planck_arrays)>0) and not(args.skip_only)
act_arrays = ','.join(act_arrays)
planck_arrays = ','.join(planck_arrays)
print("Starting simulation for arrays %s of which %s are ACT and %s are Planck." % (args.arrays,act_arrays,planck_arrays))
# Generate each ACT and Planck sim and store kdiffs,kcoadd in memory
set_id = args.set_id
bandpasses = not(args.effective_freq)
gconfig = io.config_from_yaml("input/data.yml")
mask = sints.get_act_mr3_crosslinked_mask(args.region,
version=args.mask_version,
kind='binary_apod')
shape,wcs = mask.shape,mask.wcs
Ny,Nx = shape
modlmap = enmap.modlmap(shape,wcs)
ngen = {}
ngen['act_mr3'] = actnoise.NoiseGen(args.sim_version,model="act_mr3",extract_region=mask,ncache=0,verbose=True)
ngen['planck_hybrid'] = actnoise.NoiseGen(args.sim_version,model="planck_hybrid",extract_region=mask,ncache=0,verbose=True)
arrays = args.arrays.split(',')
narrays = len(arrays)
nsims = args.nsims
jsim = pipeline.JointSim(arrays,args.fg_res_version+"_"+args.region,
import numpy as np
import os,sys
from soapack import interfaces as sints
def get_coadd(imaps,wts,axis):
# sum(w*m)/sum(w)
twt = np.sum(wts,axis=axis)
retmap = np.sum(wts*imaps,axis=axis)/twt
retmap[~np.isfinite(retmap)] = 0
return retmap,twt
def get_npol(array):
if array=='545' or array=='857':return 1
else: return 3
mask = sints.get_act_mr3_crosslinked_mask('deep56',version='180323')
dm = sints.PlanckHybrid(region=mask)
bin_edges = np.arange(30,6000,40)
p1ds = {}
for array in dm.arrays:
splits = dm.get_splits(array,srcfree=False)[0]
ivars = dm.get_splits_ivar(array)[0]
coadd,wt = get_coadd(splits,ivars,axis=0)
npol = get_npol(array)
for i in range(npol):
cents,p1d = maps.binned_power(coadd[i],bin_edges=bin_edges,mask=mask)
p1ds[array+str(i)] = p1d.copy()
mivar = wt[i].mean()
print(array,mivar)
plt.rcParams["mathtext.fontset"] = "cm"
from orphics import maps,io,cosmology,stats
from pixell import enmap
import numpy as np
import os,sys
from soapack import interfaces as sints
from tilec import utils as tutils,covtools
from matplotlib.ticker import (MultipleLocator, FormatStrFormatter,
AutoMinorLocator)
region = 'deep56'
version = "noLFI_nohigh_test"
mask = sints.get_act_mr3_crosslinked_mask(region)
modlmap = mask.modlmap()
lmap = mask.lmap()
bin_edges = np.arange(20,6000,80)
binner = stats.bin2D(modlmap,bin_edges)
def pow(x,y=None):
k = enmap.fft(x,normalize='phys')
ky = enmap.fft(y,normalize='phys') if y is not None else k
p = (k*ky.conj()).real
cents,p1d = binner.bin(p)
return p,cents,p1d
seeds = [12]#,13]
qids = "d56_04,d56_05,d56_06,p04,p05,p06".split(',')
narrays = len(qids)
# fname = os.environ['WORK'] + "/data/boss/boss_dr12/galaxy_DR12v5_CMASS_North.fits"
# cols = catalogs.load_fits(fname,['RA','DEC'])
# iras2 = cols['RA']
# idecs2 = cols['DEC']
# iras = np.append(iras1,iras2)
# idecs = np.append(idecs1,idecs2)
fname = os.environ[
'WORK'] + "/data/boss/sdss_dr8/redmapper_dr8_public_v6.3_catalog.fits"
cols = catalogs.load_fits(fname, ['RA', 'DEC'])
iras = cols['RA']
idecs = cols['DEC']
mask1 = sints.get_act_mr3_crosslinked_mask(region1)
mask1[mask1 < 0.99] = 0
mask2 = sints.get_act_mr3_crosslinked_mask(region2)
mask2[mask2 < 0.99] = 0
dm1 = sints.ACTmr3(region=mask1, calibrated=True)
dm2 = sints.ACTmr3(region=mask2, calibrated=True)
wt1 = dm1.get_coadd_ivar("s15", region1, "pa2_f150")
wt2 = dm2.get_coadd_ivar("s15", region2, "pa2_f150")
ras1, decs1 = catalogs.select_based_on_mask(iras, idecs, mask1)
ras2, decs2 = catalogs.select_based_on_mask(iras, idecs, mask2)
tdir = '/scratch/r/rbond/msyriac/data/depot/tilec/v1.0.0_rc_20190919'
solution = 'tsz'
from __future__ import print_function
from orphics import maps,io,cosmology,stats
from pixell import enmap
import numpy as np
import os,sys
from soapack import interfaces as sints
from actsims import noise
froot = "/scratch/r/rbond/msyriac/data/scratch/tilec/test_lfi_v2_00_0000_deep56/"
kroot = "/scratch/r/rbond/msyriac/data/depot/tilec/test_lfi_v2_00_0000_deep56/"
droot = "/scratch/r/rbond/msyriac/data/depot/tilec/test_lfi_data_deep56/"
lroot = "/scratch/r/rbond/msyriac/data/depot/tilec/test_lfi_v3_00_0000_deep56/"
mask = sints.get_act_mr3_crosslinked_mask("deep56")
shape,wcs = mask.shape,mask.wcs
w2 = np.mean(mask**2.)
qids = 'p01,p02,p03'.split(',')
bin_edges = np.arange(20,6000,40)
modlmap = mask.modlmap()
binner = stats.bin2D(modlmap,bin_edges)
arraynames = {'p01':'030','p02':'044','p03':'070'}
dm = sints.PlanckHybrid(region=mask)
for qid in qids:
split = enmap.read_map("%ssplit_%s.fits" % (froot,qid))
#io.hplot(enmap.downgrade(split,4),"split_%s" % qid)
for patch in patches[survey]:
if 'deep' in patch:
pad = pads['deep']
elif patch == 'boss':
pad = pads['boss']
elif 'patch' in patch:
pad = pads['patch']
else:
raise ValueError
mpatch = patch
mask = sints.get_act_mr3_crosslinked_mask(
mpatch,
version=in_versions[survey],
kind='binary_apod',
season="s16" if survey == 'advact' else None,
array=None,
pad=pad)
print(survey, patch)
# FFT friendliness
Ny, Nx = mask.shape[-2:]
dNy = fft.fft_len(Ny, "above")
dNx = fft.fft_len(Nx, "above")
pny = dNy - Ny
pnx = dNx - Nx
pady1 = pny // 2
pady2 = pny - pady1
padx1 = pnx // 2
padx2 = pnx - padx1
mask = enmap.pad(mask, ((pady1, padx1), (pady2, padx2)))