def load_survey_mask_core(qid): # load mask by specifying qid model, season, array, patch, freq = local.qid_info(qid) if model == 'dr5': mask = interfaces.get_binary_apodized_mask(qid) elif model == 'act_mr3': mask = interfaces.get_act_mr3_crosslinked_mask(patch, version='padded_v1', kind='binary_apod', season=season, array=array + "_" + freq, pad=None) return mask
""" This script calculates all ACT and Planck cross-spectra. """ import argparse # Parse command line parser = argparse.ArgumentParser(description='Do a thing.') parser.add_argument("region", type=str,help='boss or deep56?') parser.add_argument("--fft", action='store_true',help='A flag.') args = parser.parse_args() fftstr = "_fft" if args.fft else "" mask_version = interfaces.dconfig['act_mr3']['default_mask_version'] mask = interfaces.get_act_mr3_crosslinked_mask(args.region) identifier = args.region+"_"+mask_version comm = mpi.MPI.COMM_WORLD rank = comm.Get_rank() numcores = comm.Get_size() #bin_edges = np.arange(80,8000,100) lefts,rights = np.loadtxt("data/BIN_ACTPOL_50_4_SC_low_ell_startAt2.txt",unpack=True,usecols=[0,1]) bin_edges = np.append(lefts,rights[-1:]) print(bin_edges) # Calculate and store mcm if not(args.fft): with io.nostdout():
model = "act_mr3" # Get file name convention pout, cout, sout = noise.get_save_paths(model, version, coadd, season=season, patch=patch, array=args.array, mkdir=True, overwrite=True, mask_patch=mask_patch) # Get data model mask = sints.get_act_mr3_crosslinked_mask(mask_patch, version=mask_version, kind="binary_apod", season=season, array=args.array + "_f150", pad=args.mask_pad) # print(mask.shape) mask = mask[400:-400, 5000:-5000] mask *= enmap.apod(mask, 400) # print(mask.shape) noise.plot(pout + "_mask", mask, grid=True) dm = sints.models[model](region=mask) # Get a NoiseGen model emask = mask ngen = noise.NoiseGen(version=version, model=model, extract_region=emask, ncache=1,
from __future__ import print_function from orphics import maps, io, cosmology from pixell import enmap import numpy as np import os, sys import soapack.interfaces as sints from enlib import bench ntrials = 10 loc = "/scratch/r/rbond/msyriac/data/depot/tilec/temp/" mask = sints.get_act_mr3_crosslinked_mask("boss") for i in range(ntrials): with bench.show("write"): enmap.write_map(loc + "trial_%d.fits" % i, mask) for i in range(ntrials): with bench.show("read"): mask = enmap.read_map(loc + "trial_%d.fits" % i) for i in range(ntrials): os.remove(loc + "trial_%d.fits" % i)
def build_and_save_cov(arrays,region,version,mask_version, signal_bin_width,signal_interp_order,delta_ell, rfit_lmin, overwrite,memory_intensive,uncalibrated, sim_splits=None,skip_inpainting=False,theory_signal="none", unsanitized_beam=False,save_all=False,plot_inpaint=False, isotropic_override=False,split_set=None,save_extra=False): save_scratch = not(memory_intensive) savedir = tutils.get_save_path(version,region) if save_scratch: scratch = tutils.get_scratch_path(version,region) covscratch = scratch else: covscratch = None if not(overwrite): assert not(os.path.exists(savedir)), \ "This version already exists on disk. Please use a different version identifier." try: os.makedirs(savedir) except: if overwrite: pass else: raise if save_scratch: try: os.makedirs(scratch) except: pass mask = sints.get_act_mr3_crosslinked_mask(region, version=mask_version, kind='binary_apod') shape,wcs = mask.shape,mask.wcs aspecs = tutils.ASpecs().get_specs with bench.show("ffts"): kcoadds = [] kdiffs = [] wins = [] lmins = [] lmaxs = [] hybrids = [] do_radial_fit = [] freqs = [] rfit_wnoise_widths = [] save_names = [] # to make sure nothing is overwritten friends = {} # what arrays are each correlated with? names = arrays.split(',') print("Calculating FFTs for " , arrays) fbeam = lambda qname,x: tutils.get_kbeam(qname,x,sanitize=not(unsanitized_beam),planck_pixwin=True) for i,qid in enumerate(arrays.split(',')): dm = sints.models[sints.arrays(qid,'data_model')](region=mask,calibrated=not(uncalibrated)) lmin,lmax,hybrid,radial,friend,cfreq,fgroup,wrfit = aspecs(qid) print(f"{qid} lmin {lmin} lmax {lmax}") assert (type(radial)==bool) or (type(radial)==np.bool_) do_radial_fit.append(radial) friends[qid] = friend hybrids.append(hybrid) freqs.append(fgroup) rfit_wnoise_widths.append(wrfit) kdiff,kcoadd,win = kspace.process(dm,region,qid,mask, skip_splits=False, splits_fname=sim_splits[i] if sim_splits is not None else None, inpaint=not(skip_inpainting),fn_beam = lambda x: fbeam(qid,x), plot_inpaint_path = savedir if plot_inpaint else None, split_set=split_set) print("Processed ",qid) if save_scratch: kcoadd_name = savedir + "kcoadd_%s.npy" % qid kdiff_name = scratch + "kdiff_%s.npy" % qid win_name = scratch + "win_%s.npy" % qid assert win_name not in save_names assert kcoadd_name not in save_names assert kdiff_name not in save_names np.save(win_name,win) np.save(kcoadd_name,kcoadd) np.save(kdiff_name,kdiff) wins.append(win_name) kcoadds.append(kcoadd_name) kdiffs.append(kdiff_name) save_names.append(win_name) save_names.append(kcoadd_name) save_names.append(kdiff_name) print("Saved ",qid) else: wins.append(win.copy()) kcoadds.append(kcoadd.copy()) kdiffs.append(kdiff.copy()) lmins.append(lmin) lmaxs.append(lmax) if sim_splits is not None: del sim_splits print("Done with ffts.") # print("Exiting because I just want to see inpainted stuff.") # sys.exit() # Decide what pairs to do hybrid smoothing for anisotropic_pairs = get_aniso_pairs(arrays.split(','),hybrids,friends) print("Anisotropic pairs: ",anisotropic_pairs) enmap.write_map(savedir+"tilec_mask.fits",mask) save_fn = lambda x,a1,a2: np.save(savedir+"tilec_hybrid_covariance_%s_%s.npy" % (names[a1],names[a2]),enmap.enmap(x,wcs)) print("Building covariance...") with bench.show("build cov"): ilc.build_cov(names,kdiffs,kcoadds,fbeam,mask,lmins,lmaxs,freqs,anisotropic_pairs, delta_ell, do_radial_fit,save_fn, signal_bin_width=signal_bin_width, signal_interp_order=signal_interp_order, rfit_lmaxes=lmaxs, rfit_wnoise_widths=rfit_wnoise_widths, rfit_lmin=rfit_lmin, rfit_bin_width=None, verbose=True, debug_plots_loc=False, separate_masks=False,theory_signal=theory_signal,scratch_dir=covscratch, isotropic_override=isotropic_override,save_extra=save_extra,wins=wins) if not(save_all): shutil.rmtree(scratch)
array=array, mkdir=True, overwrite=args.overwrite, mask_patch=mask_patch) # Get mask mask = sints.get_binary_apodized_mask(args.qid) # Get data model dm = sints.models[args.model](region=mask) # Get a NoiseGen model if args.extract_mask is not None: emask = sints.get_act_mr3_crosslinked_mask(mask_patch, version=args.extract_mask, kind=args.mask_kind, season=season, array=array + "_f150") eshape, ewcs = emask.shape, emask.wcs else: emask = mask ngen = noise.NoiseGen(qid=qid, version=version, model=args.model, extract_region=emask, ncache=1, verbose=True) # Get split data splits = enmap.enmap([dm.get_splits(q) for q in qid]) nsplits = dm.ainfo(args.qid, 'nsplits')
import healpy as hp from pixell import enmap, curvedsky from orphics import maps,io from soapack import interfaces as sints from tilec.pipeline import JointSim qids = ['d56_03','p05'] jsim = JointSim(qids,None) jsim.update_signal_index(0) mask = sints.get_act_mr3_crosslinked_mask('deep6') imap = jsim.compute_map(mask.shape,mask.wcs,'p05',include_cmb=True,include_tsz=True,include_fgres=True) io.hplot(imap,'planck') imap = jsim.compute_map(mask.shape,mask.wcs,'d56_03',include_cmb=True,include_tsz=True,include_fgres=True) io.hplot(imap,'act')
from __future__ import print_function from orphics import maps, io, cosmology from pixell import enmap import numpy as np import os, sys from soapack import interfaces as sints mask = sints.get_act_mr3_crosslinked_mask("boss", version='padded_v1') shape, wcs = mask.shape, mask.wcs print(np.rad2deg(enmap.pix2sky(shape, wcs, (shape[-2], shape[-1])))) sys.exit() io.hplot(mask, "mask.png") print(mask.shape) dm = sints.PlanckHybrid(region=mask) ivars = dm.get_splits_ivar(['100'], ncomp=1) splits = dm.get_splits(['100'], ncomp=1) # print(ivars.shape) # print(ivars[ivars<=0].size/ivars.size*100.) # print(ivars[ivars<=0].size) # print(splits[ivars<=0]) # print(splits[ivars<=0].max()) # splits[ivars>0] = np.nan # splits[ivars<=0] = 1 io.hplot(splits[0, 1, 0], "boss.png")
parser.add_argument("--mask-pad", type=int, default=None, help='Mask additional padding. No padding is applied to the extracted mask if any specified.') parser.add_argument("--extract-mask", type=str, default=None, help='Make sims on the big mask but do all the analysis on an extract of this version.') parser.add_argument("--binary-percentile", type=float, default=10.,help='Binary percentile for sim masking.') parser.add_argument("--season", type=str,help='Season') parser.add_argument("--array", type=str,help='Array') parser.add_argument("--patch", type=str,help='Patch') args = parser.parse_args() mask_patch = args.patch if args.mask_patch is None else args.mask_patch with bench.show("region"): if args.extract_mask is not None: region_map = sints.get_act_mr3_crosslinked_mask(mask_patch, version=args.extract_mask, kind=args.mask_kind, season=args.season) else: region_map = None with bench.show("ngen init"): ngen = noise.NoiseGen(args.version,extract_region=region_map,ncache=1) with bench.show("make a sim"): ngen.generate_sim(season=args.season,patch=args.patch,array=args.array,seed=1) with bench.show("make another sim"): ngen.generate_sim(season=args.season,patch=args.patch,array=args.array,seed=2)
# Get file name convention pout, cout, sout = noise.get_save_paths(args.model, version, coadd, season=args.season, patch=args.patch, array=args.array, mkdir=True, overwrite=args.overwrite, mask_patch=mask_patch) # Get data model mask = sints.get_act_mr3_crosslinked_mask(mask_patch, version=args.mask_version, kind=args.mask_kind, season=args.season, array=args.array + "_f150", pad=args.mask_pad) dm = sints.models[args.model](region=mask, calibrated=args.calibrated) n2d = enmap.read_map(pout + '_n2d.fits') covsq = enmap.multi_pow(n2d.copy(), 0.5) covsq[:, :, n2d.modlmap() < 2] = 0 print('D', np.argwhere(~np.isfinite(covsq))) covsq = enmap.enmap(covsq, n2d.wcs) print('E', np.argwhere(~np.isfinite(covsq))) #enmap.write_map(pout+'_covsqrt_tmp.fits',covsq) #covsq = enmap.read_map(pout+'_covsqrt_tmp.fits') #covsq = enmap.read_map(pout+'_covsqrt.fits')
from tilec import pipeline,utils as tutils region = 'boss' #region = 'deep56' fg_res_version = 'test' #qids = 'd56_01,d56_02,d56_03,d56_04,d56_05,d56_06,p01,p02,p03,p04,p05,p06,p07,p08'.split(',') #qids = 'boss_01'.split(',') qids = 'boss_01,boss_02,boss_03,boss_04,p01,p02,p03,p04,p05,p06,p07,p08'.split(',') #qids = 'boss_04,p01,p02,p03,p04,p05,p06,p07,p08'.split(',') #qids = 'boss_04,p08'.split(',') #qids = 'boss_03,boss_04'.split(',') #qids = 'd56_01,d56_02,d56_03,d56_04,d56_05,d56_06'.split(',') #qids = 'd56_02,d56_03,d56_04'.split(',') #qids = 'd56_04,p01,p02,p03,p04,p05,p06,p07,p08'.split(',') mask = sints.get_act_mr3_crosslinked_mask(region, kind='binary_apod') modlmap = mask.modlmap() jsim = pipeline.JointSim(qids,fg_res_version+"_"+region,bandpassed=True) jsim0 = pipeline.JointSim(qids,None,bandpassed=True) comm,rank,my_tasks = mpi.distribute(len(qids)) bin_edges = np.arange(20,8000,20) binner = stats.bin2D(modlmap,bin_edges) jsim.update_signal_index(0,set_idx=0) jsim0.update_signal_index(0,set_idx=0)
from __future__ import print_function import matplotlib matplotlib.use('Agg') from orphics import maps, io, cosmology, pixcov from pixell import enmap, reproject import numpy as np import os, sys import soapack.interfaces as sints from enlib import bench from actsims import noise for season in ['s13', 's14', 's15']: for apatch in ['boss', 'deep1', 'deep5', 'deep6', 'deep8', 'deep56']: #for apatch in ['deep1','deep5','deep6','deep8','deep56','boss']: mask = sints.get_act_mr3_crosslinked_mask(apatch) ellcen = 5000 ellsig = 1000 modlmap = mask.modlmap() ells = np.arange(0, modlmap.max()) mfilter = maps.interp(ells, np.exp(-(ells - ellcen)**2. / 2. / ellsig**2.))(modlmap) for array in ['pa1_f150', 'pa2_f150', 'pa3_f150', 'pa3_f090']: dm = sints.ACTmr3(calibrated=False, region=mask) fname = "%s_%s_%s" % (season, apatch, array) try: splits = dm.get_splits(season=season, patch=apatch, arrays=[array], ncomp=3, srcfree=False)[0, :, :, ...]
centers = (bin_edges[1:] + bin_edges[:-1]) / 2. w2 = np.mean(mask**2.) modlmap = enmap.modlmap(map1.shape, map1.wcs) binned_power = bin(power / w2 / beamf1(modlmap) / beamf2(modlmap), modlmap, bin_edges) return centers, binned_power # # ACTxPlanck # coadded ACT x coadded Planck ACT_planck = {} for patch in ['deep56', 'boss']: mask = interfaces.get_act_mr3_crosslinked_mask(patch) dma = interfaces.ACTmr3(region=mask) dmp = interfaces.PlanckHybrid(region=mask) # we loop over all pairs of Planck x ACT for planckfreq in ['030', '044', '070', '100', '143', '217', '353', '545']: # no '857' planckbeam = lambda x: dmp.get_beam(x, planckfreq) planckmap = get_planck_coadd(planckfreq, dmp)[0, :, :] for actseason in ['s14', 's15']: for array in ['pa1_f150', 'pa2_f150', 'pa3_f090', 'pa3_f150']: try: actbeam = lambda x: dma.get_beam(x, actseason, patch, array ) actmap = dma.get_coadd( actseason, patch, array, ncomp=1,
qmap = enmap.read_map("%sdataCoadd_combined_Q_s14&15_%s.fits" % (opath, region)) umap = enmap.read_map("%sdataCoadd_combined_U_s14&15_%s.fits" % (opath, region)) #oq_noise2d = pow(qmap) #ou_noise2d = pow(umap) _, oe_p2d, ob_p2d = get_pol_powers(tmap, qmap, umap, kbeam, w2) io.power_crop(oe_p2d, Nplot, "oe_noise2d.png", ftrans=True) #io.power_crop(ob_p2d,Nplot,"ob_noise2d.png",ftrans=True) dmask = sints.get_act_mr3_crosslinked_mask(region='deep6') dw2 = np.mean(dmask**2) dm = sints.ACTmr3(calibrated=True, region=dmask) imap = dm.get_coadd(season='s13', array='pa1_f150', patch='deep6', srcfree=True) tmap = imap[0] * dmask qmap = imap[1] * dmask umap = imap[2] * dmask dbeam = tutils.get_kbeam("d6", dmask.modlmap(), sanitize=False) _, de_p2d, db_p2d = get_pol_powers(tmap, qmap, umap, dbeam, dw2) io.power_crop(de_p2d, Nplot, "de_noise2d.png", ftrans=True) dbinner = stats.bin2D(dmask.modlmap(), bin_edges)
else: act_arrays.append(qid) do_act_only = (len(act_arrays)>0) and not(args.skip_only) do_planck_only = (len(planck_arrays)>0) and not(args.skip_only) act_arrays = ','.join(act_arrays) planck_arrays = ','.join(planck_arrays) print("Starting simulation for arrays %s of which %s are ACT and %s are Planck." % (args.arrays,act_arrays,planck_arrays)) # Generate each ACT and Planck sim and store kdiffs,kcoadd in memory set_id = args.set_id bandpasses = not(args.effective_freq) gconfig = io.config_from_yaml("input/data.yml") mask = sints.get_act_mr3_crosslinked_mask(args.region, version=args.mask_version, kind='binary_apod') shape,wcs = mask.shape,mask.wcs Ny,Nx = shape modlmap = enmap.modlmap(shape,wcs) ngen = {} ngen['act_mr3'] = actnoise.NoiseGen(args.sim_version,model="act_mr3",extract_region=mask,ncache=0,verbose=True) ngen['planck_hybrid'] = actnoise.NoiseGen(args.sim_version,model="planck_hybrid",extract_region=mask,ncache=0,verbose=True) arrays = args.arrays.split(',') narrays = len(arrays) nsims = args.nsims jsim = pipeline.JointSim(arrays,args.fg_res_version+"_"+args.region,
import numpy as np import os,sys from soapack import interfaces as sints def get_coadd(imaps,wts,axis): # sum(w*m)/sum(w) twt = np.sum(wts,axis=axis) retmap = np.sum(wts*imaps,axis=axis)/twt retmap[~np.isfinite(retmap)] = 0 return retmap,twt def get_npol(array): if array=='545' or array=='857':return 1 else: return 3 mask = sints.get_act_mr3_crosslinked_mask('deep56',version='180323') dm = sints.PlanckHybrid(region=mask) bin_edges = np.arange(30,6000,40) p1ds = {} for array in dm.arrays: splits = dm.get_splits(array,srcfree=False)[0] ivars = dm.get_splits_ivar(array)[0] coadd,wt = get_coadd(splits,ivars,axis=0) npol = get_npol(array) for i in range(npol): cents,p1d = maps.binned_power(coadd[i],bin_edges=bin_edges,mask=mask) p1ds[array+str(i)] = p1d.copy() mivar = wt[i].mean() print(array,mivar)
plt.rcParams["mathtext.fontset"] = "cm" from orphics import maps,io,cosmology,stats from pixell import enmap import numpy as np import os,sys from soapack import interfaces as sints from tilec import utils as tutils,covtools from matplotlib.ticker import (MultipleLocator, FormatStrFormatter, AutoMinorLocator) region = 'deep56' version = "noLFI_nohigh_test" mask = sints.get_act_mr3_crosslinked_mask(region) modlmap = mask.modlmap() lmap = mask.lmap() bin_edges = np.arange(20,6000,80) binner = stats.bin2D(modlmap,bin_edges) def pow(x,y=None): k = enmap.fft(x,normalize='phys') ky = enmap.fft(y,normalize='phys') if y is not None else k p = (k*ky.conj()).real cents,p1d = binner.bin(p) return p,cents,p1d seeds = [12]#,13] qids = "d56_04,d56_05,d56_06,p04,p05,p06".split(',') narrays = len(qids)
# fname = os.environ['WORK'] + "/data/boss/boss_dr12/galaxy_DR12v5_CMASS_North.fits" # cols = catalogs.load_fits(fname,['RA','DEC']) # iras2 = cols['RA'] # idecs2 = cols['DEC'] # iras = np.append(iras1,iras2) # idecs = np.append(idecs1,idecs2) fname = os.environ[ 'WORK'] + "/data/boss/sdss_dr8/redmapper_dr8_public_v6.3_catalog.fits" cols = catalogs.load_fits(fname, ['RA', 'DEC']) iras = cols['RA'] idecs = cols['DEC'] mask1 = sints.get_act_mr3_crosslinked_mask(region1) mask1[mask1 < 0.99] = 0 mask2 = sints.get_act_mr3_crosslinked_mask(region2) mask2[mask2 < 0.99] = 0 dm1 = sints.ACTmr3(region=mask1, calibrated=True) dm2 = sints.ACTmr3(region=mask2, calibrated=True) wt1 = dm1.get_coadd_ivar("s15", region1, "pa2_f150") wt2 = dm2.get_coadd_ivar("s15", region2, "pa2_f150") ras1, decs1 = catalogs.select_based_on_mask(iras, idecs, mask1) ras2, decs2 = catalogs.select_based_on_mask(iras, idecs, mask2) tdir = '/scratch/r/rbond/msyriac/data/depot/tilec/v1.0.0_rc_20190919' solution = 'tsz'
from __future__ import print_function from orphics import maps,io,cosmology,stats from pixell import enmap import numpy as np import os,sys from soapack import interfaces as sints from actsims import noise froot = "/scratch/r/rbond/msyriac/data/scratch/tilec/test_lfi_v2_00_0000_deep56/" kroot = "/scratch/r/rbond/msyriac/data/depot/tilec/test_lfi_v2_00_0000_deep56/" droot = "/scratch/r/rbond/msyriac/data/depot/tilec/test_lfi_data_deep56/" lroot = "/scratch/r/rbond/msyriac/data/depot/tilec/test_lfi_v3_00_0000_deep56/" mask = sints.get_act_mr3_crosslinked_mask("deep56") shape,wcs = mask.shape,mask.wcs w2 = np.mean(mask**2.) qids = 'p01,p02,p03'.split(',') bin_edges = np.arange(20,6000,40) modlmap = mask.modlmap() binner = stats.bin2D(modlmap,bin_edges) arraynames = {'p01':'030','p02':'044','p03':'070'} dm = sints.PlanckHybrid(region=mask) for qid in qids: split = enmap.read_map("%ssplit_%s.fits" % (froot,qid)) #io.hplot(enmap.downgrade(split,4),"split_%s" % qid)
for patch in patches[survey]: if 'deep' in patch: pad = pads['deep'] elif patch == 'boss': pad = pads['boss'] elif 'patch' in patch: pad = pads['patch'] else: raise ValueError mpatch = patch mask = sints.get_act_mr3_crosslinked_mask( mpatch, version=in_versions[survey], kind='binary_apod', season="s16" if survey == 'advact' else None, array=None, pad=pad) print(survey, patch) # FFT friendliness Ny, Nx = mask.shape[-2:] dNy = fft.fft_len(Ny, "above") dNx = fft.fft_len(Nx, "above") pny = dNy - Ny pnx = dNx - Nx pady1 = pny // 2 pady2 = pny - pady1 padx1 = pnx // 2 padx2 = pnx - padx1 mask = enmap.pad(mask, ((pady1, padx1), (pady2, padx2)))