def get_kbeam(qid, modlmap, sanitize=False, version=None, planck_pixwin=False, **kwargs): dmodel = sints.arrays(qid, 'data_model') season = sints.arrays(qid, 'season') region = sints.arrays(qid, 'region') array = sints.arrays(qid, 'array') freq = sints.arrays(qid, 'freq') dm = sints.models[dmodel]() gfreq = array + "_" + freq if not (is_planck(qid)) else freq if planck_pixwin and (qid in [ 'p01', 'p02', 'p03', 'p04', 'p05', 'p06', 'p07', 'p08' ]): nside = get_nside(qid) pixwin = hp.pixwin(nside=nside, pol=False) ls = np.arange(len(pixwin)) assert pixwin.ndim == 1 assert ls.size in [6144, 3072] pwin = maps.interp(ls, pixwin)(modlmap) else: pwin = 1. return dm.get_beam(modlmap, season=season, patch=region, array=gfreq, kind='normalized', sanitize=sanitize, version=version, **kwargs) * pwin
def get_splits_ivar(qid, extracter, ivar_unhit=1e-7, ivar_tol=20): dmodel = sints.arrays(qid, 'data_model') season = sints.arrays(qid, 'season') region = sints.arrays(qid, 'region') array = sints.arrays(qid, 'array') freq = sints.arrays(qid, 'freq') dm = sints.models[dmodel]() ivars = [] for i in range(dm.get_nsplits(season=season, patch=region, array=array)): omap = extracter( dm.get_split_ivar_fname(season=season, patch=region, array=array + "_" + freq if not (is_planck(qid)) else freq, splitnum=i)) if omap.ndim > 2: omap = omap[0] if not (np.any(omap > 0)): print( "Skipping %s as it seems to have an all zero ivar in this tile" % qid) return None omap[~np.isfinite(omap)] = 0 omap[omap < ivar_unhit] = 0 ref_div = robust_ref(omap) omap = np.minimum(omap, ref_div * ivar_tol) omap = filter_div(omap) eshape, ewcs = omap.shape, omap.wcs ivars.append(omap.copy()) return enmap.enmap(np.stack(ivars), ewcs)
def load_geometries(qids): geoms = {} for qid in qids: dmodel = sints.arrays(qid, 'data_model') season = sints.arrays(qid, 'season') region = sints.arrays(qid, 'region') array = sints.arrays(qid, 'array') freq = sints.arrays(qid, 'freq') dm = sints.models[dmodel]() print(season, region, array, freq) shape, wcs = enmap.read_map_geometry( dm.get_split_fname(season=season, patch=region, array=array + "_" + freq if not (is_planck(qid)) else freq, splitnum=0, srcfree=True)) geoms[qid] = shape[-2:], wcs return geoms
def get_splits(qid, extracter): dmodel = sints.arrays(qid, 'data_model') season = sints.arrays(qid, 'season') region = sints.arrays(qid, 'region') array = sints.arrays(qid, 'array') freq = sints.arrays(qid, 'freq') dm = sints.models[dmodel]() splits = [] for i in range(dm.get_nsplits(season=season, patch=region, array=array)): omap = extracter(dm.get_split_fname( season=season, patch=region, array=array + "_" + freq if not (is_planck(qid)) else freq, splitnum=i, srcfree=True), sel=np.s_[0, ...]) # sel assert np.all(np.isfinite(omap)) eshape, ewcs = omap.shape, omap.wcs splits.append(omap.copy()) return enmap.enmap(np.stack(splits), ewcs)
def build_and_save_ilc(arrays,region,version,cov_version,beam_version, solutions,beams,chunk_size, effective_freq,overwrite,maxval,unsanitized_beam=False,do_weights=False, pa1_shift = None, pa2_shift = None, pa3_150_shift = None, pa3_090_shift = None, no_act_color_correction=False, ccor_exp = -1, isotropize=False, isotropize_width=20): print("Chunk size is ", chunk_size*64./8./1024./1024./1024., " GB.") def warn(): print("WARNING: no bandpass file found. Assuming array ",dm.c['id']," has no response to CMB, tSZ and CIB.") aspecs = tutils.ASpecs().get_specs bandpasses = not(effective_freq) savedir = tutils.get_save_path(version,region) covdir = tutils.get_save_path(cov_version,region) assert os.path.exists(covdir) if not(overwrite): assert not(os.path.exists(savedir)), \ "This version already exists on disk. Please use a different version identifier." try: os.makedirs(savedir) except: if overwrite: pass else: raise mask = enmap.read_map(covdir+"tilec_mask.fits") shape,wcs = mask.shape,mask.wcs Ny,Nx = shape modlmap = enmap.modlmap(shape,wcs) arrays = arrays.split(',') narrays = len(arrays) kcoadds = [] kbeams = [] bps = [] names = [] lmins = [] lmaxs = [] shifts = [] cfreqs = [] lbeams = [] ells = np.arange(0,modlmap.max()) for i,qid in enumerate(arrays): dm = sints.models[sints.arrays(qid,'data_model')](region=mask,calibrated=True) lmin,lmax,hybrid,radial,friend,cfreq,fgroup,wrfit = aspecs(qid) cfreqs.append(cfreq) lmins.append(lmin) lmaxs.append(lmax) names.append(qid) if dm.name=='act_mr3': season,array1,array2 = sints.arrays(qid,'season'),sints.arrays(qid,'array'),sints.arrays(qid,'freq') array = '_'.join([array1,array2]) elif dm.name=='planck_hybrid': season,patch,array = None,None,sints.arrays(qid,'freq') else: raise ValueError kcoadd_name = covdir + "kcoadd_%s.npy" % qid kmask = maps.mask_kspace(shape,wcs,lmin=lmin,lmax=lmax) kcoadd = enmap.enmap(np.load(kcoadd_name),wcs) dtype = kcoadd.dtype kcoadds.append(kcoadd.copy()*kmask) kbeam = tutils.get_kbeam(qid,modlmap,sanitize=not(unsanitized_beam),version=beam_version,planck_pixwin=True) if dm.name=='act_mr3': lbeam = tutils.get_kbeam(qid,ells,sanitize=not(unsanitized_beam),version=beam_version,planck_pixwin=False) # note no pixwin but doesnt matter since no ccorr for planck elif dm.name=='planck_hybrid': lbeam = None else: raise ValueError lbeams.append(lbeam) kbeams.append(kbeam.copy()) if bandpasses: try: fname = dm.get_bandpass_file_name(array) bps.append("data/"+fname) if (pa1_shift is not None) and 'PA1' in fname: shifts.append(pa1_shift) elif (pa2_shift is not None) and 'PA2' in fname: shifts.append(pa2_shift) elif (pa3_150_shift is not None) and ('PA3' in fname) and ('150' in fname): shifts.append(pa3_150_shift) elif (pa3_090_shift is not None) and ('PA3' in fname) and ('090' in fname): shifts.append(pa3_90_shift) else: shifts.append(0) except: warn() bps.append(None) else: try: bps.append(cfreq) except: warn() bps.append(None) kcoadds = enmap.enmap(np.stack(kcoadds),wcs) # Read Covmat cov = maps.SymMat(narrays,shape[-2:]) for aindex1 in range(narrays): for aindex2 in range(aindex1,narrays): icov = enmap.enmap(np.load(covdir+"tilec_hybrid_covariance_%s_%s.npy" % (names[aindex1],names[aindex2])),wcs) if isotropize: bin_edges = np.append([0.],np.arange(min(lmins),modlmap.max(),isotropize_width)) binner = stats.bin2D(modlmap,bin_edges) ls,c1d = binner.bin(icov) icov = maps.interp(ls,c1d)(modlmap) if aindex1==aindex2: icov[modlmap<lmins[aindex1]] = maxval icov[modlmap>lmaxs[aindex1]] = maxval cov[aindex1,aindex2] = icov cov.data = enmap.enmap(cov.data,wcs,copy=False) covfunc = lambda sel: cov.to_array(sel,flatten=True) assert cov.data.shape[0]==((narrays*(narrays+1))/2) # FIXME: generalize assert np.all(np.isfinite(cov.data)) # Make responses responses = {} for comp in ['tSZ','CMB','CIB']: if bandpasses: if no_act_color_correction: responses[comp] = tfg.get_mix_bandpassed(bps, comp, bandpass_shifts=shifts) else: responses[comp] = tfg.get_mix_bandpassed(bps, comp, bandpass_shifts=shifts, ccor_cen_nus=cfreqs, ccor_beams=lbeams, ccor_exps = [ccor_exp] * narrays) else: responses[comp] = tfg.get_mix(bps, comp) ilcgen = ilc.chunked_ilc(modlmap,np.stack(kbeams),covfunc,chunk_size,responses=responses,invert=True) # Initialize containers solutions = solutions.split(',') data = {} kcoadds = kcoadds.reshape((narrays,Ny*Nx)) for solution in solutions: data[solution] = {} comps = solution.split('-') data[solution]['comps'] = comps if len(comps)<=2: data[solution]['noise'] = enmap.zeros((Ny*Nx),wcs) if len(comps)==2: data[solution]['cnoise'] = enmap.zeros((Ny*Nx),wcs) data[solution]['kmap'] = enmap.zeros((Ny*Nx),wcs,dtype=dtype) # FIXME: reduce dtype? if do_weights and len(comps)<=2: for qid in arrays: data[solution]['weight_%s' % qid] = enmap.zeros((Ny*Nx),wcs) for chunknum,(hilc,selchunk) in enumerate(ilcgen): print("ILC on chunk ", chunknum+1, " / ",int(modlmap.size/chunk_size)+1," ...") for solution in solutions: comps = data[solution]['comps'] if len(comps)==1: # GENERALIZE data[solution]['noise'][selchunk] = hilc.standard_noise(comps[0]) if do_weights: weight = hilc.standard_weight(comps[0]) data[solution]['kmap'][selchunk] = hilc.standard_map(kcoadds[...,selchunk],comps[0]) elif len(comps)==2: data[solution]['noise'][selchunk] = hilc.constrained_noise(comps[0],comps[1]) data[solution]['cnoise'][selchunk] = hilc.cross_noise(comps[0],comps[1]) ret = hilc.constrained_map(kcoadds[...,selchunk],comps[0],comps[1],return_weight=do_weights) if do_weights: data[solution]['kmap'][selchunk],weight = ret else: data[solution]['kmap'][selchunk] = ret elif len(comps)>2: data[solution]['kmap'][selchunk] = np.nan_to_num(hilc.multi_constrained_map(kcoadds[...,selchunk],comps[0],*comps[1:])) if len(comps)<=2 and do_weights: for qind,qid in enumerate(arrays): data[solution]['weight_%s' % qid][selchunk] = weight[qind] del ilcgen,cov # Reshape into maps name_map = {'CMB':'cmb','tSZ':'comptony','CIB':'cib'} beams = beams.split(',') for solution,beam in zip(solutions,beams): comps = "tilec_single_tile_"+region+"_" comps = comps + name_map[data[solution]['comps'][0]]+"_" if len(data[solution]['comps'])>1: comps = comps + "deprojects_"+ '_'.join([name_map[x] for x in data[solution]['comps'][1:]]) + "_" comps = comps + version if do_weights and len(data[solution]['comps'])<=2: for qind,qid in enumerate(arrays): enmap.write_map("%s/%s_%s_weight.fits" % (savedir,comps,qid), enmap.enmap(data[solution]['weight_%s' % qid].reshape((Ny,Nx)),wcs)) try: noise = enmap.enmap(data[solution]['noise'].reshape((Ny,Nx)),wcs) enmap.write_map("%s/%s_noise.fits" % (savedir,comps),noise) except: pass try: cnoise = enmap.enmap(data[solution]['cnoise'].reshape((Ny,Nx)),wcs) enmap.write_map("%s/%s_cross_noise.fits" % (savedir,comps),cnoise) except: pass ells = np.arange(0,modlmap.max(),1) try: fbeam = float(beam) kbeam = maps.gauss_beam(modlmap,fbeam) lbeam = maps.gauss_beam(ells,fbeam) except: qid = beam bfunc = lambda x: tutils.get_kbeam(qid,x,version=beam_version,sanitize=not(unsanitized_beam),planck_pixwin=False) kbeam = bfunc(modlmap) lbeam = bfunc(ells) kmap = enmap.enmap(data[solution]['kmap'].reshape((Ny,Nx)),wcs) smap = enmap.ifft(kbeam*kmap,normalize='phys').real enmap.write_map("%s/%s.fits" % (savedir,comps),smap) io.save_cols("%s/%s_beam.txt" % (savedir,comps),(ells,lbeam),header="ell beam") enmap.write_map(savedir+"/tilec_mask.fits",mask)
def build_and_save_cov(arrays,region,version,mask_version, signal_bin_width,signal_interp_order,delta_ell, rfit_lmin, overwrite,memory_intensive,uncalibrated, sim_splits=None,skip_inpainting=False,theory_signal="none", unsanitized_beam=False,save_all=False,plot_inpaint=False, isotropic_override=False,split_set=None,save_extra=False): save_scratch = not(memory_intensive) savedir = tutils.get_save_path(version,region) if save_scratch: scratch = tutils.get_scratch_path(version,region) covscratch = scratch else: covscratch = None if not(overwrite): assert not(os.path.exists(savedir)), \ "This version already exists on disk. Please use a different version identifier." try: os.makedirs(savedir) except: if overwrite: pass else: raise if save_scratch: try: os.makedirs(scratch) except: pass mask = sints.get_act_mr3_crosslinked_mask(region, version=mask_version, kind='binary_apod') shape,wcs = mask.shape,mask.wcs aspecs = tutils.ASpecs().get_specs with bench.show("ffts"): kcoadds = [] kdiffs = [] wins = [] lmins = [] lmaxs = [] hybrids = [] do_radial_fit = [] freqs = [] rfit_wnoise_widths = [] save_names = [] # to make sure nothing is overwritten friends = {} # what arrays are each correlated with? names = arrays.split(',') print("Calculating FFTs for " , arrays) fbeam = lambda qname,x: tutils.get_kbeam(qname,x,sanitize=not(unsanitized_beam),planck_pixwin=True) for i,qid in enumerate(arrays.split(',')): dm = sints.models[sints.arrays(qid,'data_model')](region=mask,calibrated=not(uncalibrated)) lmin,lmax,hybrid,radial,friend,cfreq,fgroup,wrfit = aspecs(qid) print(f"{qid} lmin {lmin} lmax {lmax}") assert (type(radial)==bool) or (type(radial)==np.bool_) do_radial_fit.append(radial) friends[qid] = friend hybrids.append(hybrid) freqs.append(fgroup) rfit_wnoise_widths.append(wrfit) kdiff,kcoadd,win = kspace.process(dm,region,qid,mask, skip_splits=False, splits_fname=sim_splits[i] if sim_splits is not None else None, inpaint=not(skip_inpainting),fn_beam = lambda x: fbeam(qid,x), plot_inpaint_path = savedir if plot_inpaint else None, split_set=split_set) print("Processed ",qid) if save_scratch: kcoadd_name = savedir + "kcoadd_%s.npy" % qid kdiff_name = scratch + "kdiff_%s.npy" % qid win_name = scratch + "win_%s.npy" % qid assert win_name not in save_names assert kcoadd_name not in save_names assert kdiff_name not in save_names np.save(win_name,win) np.save(kcoadd_name,kcoadd) np.save(kdiff_name,kdiff) wins.append(win_name) kcoadds.append(kcoadd_name) kdiffs.append(kdiff_name) save_names.append(win_name) save_names.append(kcoadd_name) save_names.append(kdiff_name) print("Saved ",qid) else: wins.append(win.copy()) kcoadds.append(kcoadd.copy()) kdiffs.append(kdiff.copy()) lmins.append(lmin) lmaxs.append(lmax) if sim_splits is not None: del sim_splits print("Done with ffts.") # print("Exiting because I just want to see inpainted stuff.") # sys.exit() # Decide what pairs to do hybrid smoothing for anisotropic_pairs = get_aniso_pairs(arrays.split(','),hybrids,friends) print("Anisotropic pairs: ",anisotropic_pairs) enmap.write_map(savedir+"tilec_mask.fits",mask) save_fn = lambda x,a1,a2: np.save(savedir+"tilec_hybrid_covariance_%s_%s.npy" % (names[a1],names[a2]),enmap.enmap(x,wcs)) print("Building covariance...") with bench.show("build cov"): ilc.build_cov(names,kdiffs,kcoadds,fbeam,mask,lmins,lmaxs,freqs,anisotropic_pairs, delta_ell, do_radial_fit,save_fn, signal_bin_width=signal_bin_width, signal_interp_order=signal_interp_order, rfit_lmaxes=lmaxs, rfit_wnoise_widths=rfit_wnoise_widths, rfit_lmin=rfit_lmin, rfit_bin_width=None, verbose=True, debug_plots_loc=False, separate_masks=False,theory_signal=theory_signal,scratch_dir=covscratch, isotropic_override=isotropic_override,save_extra=save_extra,wins=wins) if not(save_all): shutil.rmtree(scratch)
def compute_map(self,oshape,owcs,qid,pixwin_taper_deg=0.3,pixwin_pad_deg=0.3, include_cmb=True,include_tsz=True,include_fgres=True,sht_beam=True): """ 1. get total alm 2. apply beam, and pixel window if Planck 3. ISHT 4. if ACT, apply a small taper and apply pixel window in Fourier space """ # pad to a slightly larger geometry tot_pad_deg = pixwin_taper_deg + pixwin_pad_deg res = maps.resolution(oshape,owcs) pix = np.deg2rad(tot_pad_deg)/res omap = enmap.pad(enmap.zeros((3,)+oshape,owcs),pix) ishape,iwcs = omap.shape[-2:],omap.wcs # get data model dm = sints.models[sints.arrays(qid,'data_model')](region_shape=ishape,region_wcs=iwcs,calibrated=True) # 1. get total alm array_index = self.qids.index(qid) tot_alm = int(include_cmb)*self.alms['cmb'] if include_tsz: try: assert self.tsz_fnu.ndim==2 tot_alm[0] = tot_alm[0] + hp.almxfl(self.alms['comptony'][0] ,self.tsz_fnu[array_index]) except: tot_alm[0] = tot_alm[0] + self.alms['comptony'][0] * self.tsz_fnu[array_index] if self.cfgres is not None: tot_alm[0] = tot_alm[0] + int(include_fgres)*self.alms['fgres'][array_index] assert tot_alm.ndim==2 assert tot_alm.shape[0]==3 ells = np.arange(self.lmax+1) # 2. get beam, and pixel window for Planck if sht_beam: beam = tutils.get_kbeam(qid,ells,sanitize=False,planck_pixwin=False) # NEVER SANITIZE THE BEAM IN A SIMULATION!!! for i in range(3): tot_alm[i] = hp.almxfl(tot_alm[i],beam) if dm.name=='planck_hybrid': pixwint,pixwinp = hp.pixwin(nside=tutils.get_nside(qid),lmax=self.lmax,pol=True) tot_alm[0] = hp.almxfl(tot_alm[0],pixwint) tot_alm[1] = hp.almxfl(tot_alm[1],pixwinp) tot_alm[2] = hp.almxfl(tot_alm[2],pixwinp) # 3. ISHT omap = curvedsky.alm2map(np.complex128(tot_alm),omap,spin=[0,2]) assert omap.ndim==3 assert omap.shape[0]==3 if not(sht_beam): taper,_ = maps.get_taper_deg(ishape,iwcs,taper_width_degrees=pixwin_taper_deg,pad_width_degrees=pixwin_pad_deg) modlmap = omap.modlmap() beam = tutils.get_kbeam(qid,modlmap,sanitize=False,planck_pixwin=True) kmap = enmap.fft(omap*taper,normalize='phys') kmap = kmap * beam # 4. if ACT, apply a small taper and apply pixel window in Fourier space if dm.name=='act_mr3': if sht_beam: taper,_ = maps.get_taper_deg(ishape,iwcs,taper_width_degrees=pixwin_taper_deg,pad_width_degrees=pixwin_pad_deg) pwin = tutils.get_pixwin(ishape[-2:]) if sht_beam: omap = maps.filter_map(omap*taper,pwin) else: kmap = kmap * pwin if not(sht_beam): omap = enmap.ifft(kmap,normalize='phys').real return enmap.extract(omap,(3,)+oshape[-2:],owcs)
# for f in fs[qid]: # pl.vline(x=f) # pl.done("bpass_%s.png" % qid) # sys.exit() comp = 'tSZ' ells = np.arange(0,30000,1) gauss_beam = False gauss_band = False #False for qid in qids: dm = sints.models[sints.arrays(qid,'data_model')]() if dm.name=='act_mr3': season,array1,array2 = sints.arrays(qid,'season'),sints.arrays(qid,'array'),sints.arrays(qid,'freq') array = '_'.join([array1,array2]) elif dm.name=='planck_hybrid': season,patch,array = None,None,sints.arrays(qid,'freq') lmin,lmax,hybrid,radial,friend,cfreq,fgroup,wrfit = aspecs(qid) bpf = "data/"+dm.get_bandpass_file_name(array) nu_ghz, trans = np.loadtxt(bpf, usecols=(0,1), unpack=True) trans = trans/trans.max() sel = trans>1e-4 nu_ghz = nu_ghz[sel] trans = trans[sel]
bandpassed=True, no_act_color_correction=False, ccor_exp=-1) jsim.update_signal_index(sim_index, set_idx=set_id) qid = arrays[0] signal = jsim.compute_map(mask.shape, mask.wcs, qid, include_cmb=True, include_tsz=True, include_fgres=False, sht_beam=True) season, arrayname = None, sints.arrays(qid, 'freq') patch = region farray = arrayname.split('_')[0] noise_seed = seed_tracker.get_noise_seed(set_id, sim_index, ngen[dmname].dm, season, patch, farray, None) noise, ivars = ngen[dmname].generate_sim(season=season, patch=patch, array=farray, seed=noise_seed, apply_ivar=False) ivars = ivars[0] noise = noise[0] splits = actnoise.apply_ivar_window(signal[None, None] + noise[None], ivars[None]) nsplits = actnoise.apply_ivar_window(noise[None], ivars[None])
narrays = len(qids) cbin_edges = np.arange(20, 8000, 20) cbinner = stats.bin2D(modlmap, cbin_edges) fpath = "/scratch/r/rbond/msyriac/data/depot/actsims/fg_res/fgfit_deep56" for i in range(narrays): for j in range(i, narrays): pl = io.Plotter(xyscale='linlog', xlabel='l', ylabel='D', scalefn=lambda x: x**2.) qid1 = qids[i] qid2 = qids[j] array1 = sints.arrays(qid1, 'freq') array2 = sints.arrays(qid2, 'freq') splits1 = dm.get_splits([array1], ncomp=1, srcfree=True)[0, :, 0] wins1 = dm.get_splits_ivar([array1])[0, :, 0] splits2 = dm.get_splits([array2], ncomp=1, srcfree=True)[0, :, 0] wins2 = dm.get_splits_ivar([array2])[0, :, 0] fbeam1 = lambda x: tutils.get_kbeam( qid1, x, sanitize=True, planck_pixwin=True) fbeam2 = lambda x: tutils.get_kbeam( qid2, x, sanitize=True, planck_pixwin=True) ks1 = enmap.fft(splits1 * mask, normalize='phys') / fbeam1(modlmap) ks2 = enmap.fft(splits2 * mask, normalize='phys') / fbeam2(modlmap)
qids = ['d56_0%d' % x for x in range(1, 7)] + ['boss_0%d' % x for x in range(1, 5)] print(qids) bps = [] lbeams = [] cfreqs = [] lexps1 = [] lexps2 = [] exp1 = {148: -1, 93: -1} exp2 = {148: -0.8, 93: -0.9} aspecs = tutils.ASpecs().get_specs ells = np.arange(0, 8000, 1) for qid in qids: dm = sints.models[sints.arrays(qid, 'data_model')]() lmin, lmax, hybrid, radial, friend, cfreq, fgroup, wrfit = aspecs(qid) cfreqs.append(cfreq) season, array1, array2 = sints.arrays(qid, 'season'), sints.arrays( qid, 'array'), sints.arrays(qid, 'freq') array = '_'.join([array1, array2]) bps.append("data/" + dm.get_bandpass_file_name(array)) lbeams.append(tutils.get_kbeam(qid, ells, sanitize=False)) lexps1.append(exp1[cfreq]) lexps2.append(exp2[cfreq]) r1 = tfg.get_mix_bandpassed(bps, 'tSZ', ccor_cen_nus=cfreqs, ccor_beams=lbeams,
patches = ['deep56', 'boss'] aarrays = { 'deep56': ['d56_%s' % str(x).zfill(2) for x in range(1, 7)], 'boss': ['boss_%s' % str(x).zfill(2) for x in range(1, 4)] } planck_arrays = ['p0%d' % x for x in range(1, 9)] dmap = {'act_mr3': 'act', 'planck_hybrid': 'planck'} for patch in patches: act_arrays = aarrays[patch] allarrs = act_arrays + planck_arrays for i in range(len(allarrs)): for j in range(i, len(allarrs)): a1 = allarrs[i] a2 = allarrs[j] dm1 = dmap[sints.arrays(a1, 'data_model')] dm2 = dmap[sints.arrays(a2, 'data_model')] season1 = sints.arrays(a1, 'season') array1 = sints.arrays(a1, 'array') + "_" + sints.arrays(a1, 'freq') if dm1 == 'planck': array1 = array1.split('_')[1] season2 = sints.arrays(a2, 'season') array2 = sints.arrays(a2, 'array') + "_" + sints.arrays(a2, 'freq') if dm2 == 'planck': array2 = array2.split('_')[1] ls, Cls, Rcls, errs, pfit = s.get_spec(patch, dm1, dm2, season1=season1, array1=array1, season2=season2, array2=array2) sys.exit()
save_path = sints.dconfig['tilec']['save_path'] covdir = save_path + args.cov_version + "/" + args.region + "/" mask = enmap.read_map(covdir + "tilec_mask.fits") modlmap = mask.modlmap() names = args.arrays.split(',') narrays = len(names) arrays = names aspecs = tutils.ASpecs().get_specs lmins = [] lmaxs = [] for i, qid in enumerate(arrays): dm = sints.models[sints.arrays(qid, 'data_model')](region=mask, calibrated=True) lmin, lmax, hybrid, radial, friend, cfreq, fgroup = aspecs(qid) lmins.append(lmin) lmaxs.append(lmax) i = 0 for aindex1 in range(narrays): for aindex2 in range(aindex1, narrays): icov = enmap.read_map(covdir + "tilec_hybrid_covariance_%s_%s.hdf" % (names[aindex1], names[aindex2])) if aindex1 == aindex2: print(names[aindex1], names[aindex2], lmins[aindex1], lmaxs[aindex1]) # icov[modlmap<lmins[aindex1]] = 1e200 #np.inf # icov[modlmap>lmaxs[aindex1]] = 1e200 #np.inf
w2 = np.mean(mask**2.) modlmap = mask.modlmap() lmax = 5500 ells = np.arange(0,lmax,1) ctheory = ilc.CTheory(ells) aspecs = tutils.ASpecs().get_specs bin_edges = np.arange(100,lmax,100) binner = stats.bin2D(modlmap,bin_edges) pl = io.Plotter(xyscale='linlog',xlabel='l',ylabel='D',scalefn = lambda x: x**2./2./np.pi) for i,qid in enumerate(arrays): kcoadds = [] for splitnum in range(2): dm = sints.models[sints.arrays(qid,'data_model')](region=mask,calibrated=True) lmin,lmax,hybrid,radial,friend,cfreq,fgroup,wrfit = aspecs(qid) cltt = ctheory.get_theory_cls(cfreq,cfreq,a_cmb=1,a_gal=0.8) _,kcoadd,_ = kspace.process(dm,region,qid,mask, skip_splits=True, splits_fname=None, inpaint=False,fn_beam = None, plot_inpaint_path = None, split_set=splitnum) kbeam = tutils.get_kbeam(qid,modlmap,sanitize=False,planck_pixwin=True) kcoadds.append(kcoadd/kbeam) power = (kcoadds[0]*kcoadds[1].conj()).real / w2 cents,p1d = binner.bin(power) pl.add(cents,p1d,color="C%d" % i,label=str(cfreq))
def is_planck(qid): dmodel = sints.arrays(qid, 'data_model') return True if dmodel == 'planck_hybrid' else False
def calculate_yy(bin_edges,arrays,region,version,cov_versions,beam_version, effective_freq,overwrite,maxval,unsanitized_beam=False,do_weights=False, pa1_shift = None, pa2_shift = None, pa3_150_shift = None, pa3_090_shift = None, no_act_color_correction=False, ccor_exp = -1, sim_splits=None,unblind=False,all_analytic=False,beta_samples=None): """ We calculate the yy power spectrum as follows. We restrict the Fourier modes in our analysis to those within bin_edges. This way we don't carry irrelevant pixels and thus speed up the ability to MC. We accept two covariance versions in cov_versions, which correspond to [act_covariance_from_split_0,act_covariance_from_split_1,other_covs]. Thus the ACT auto covariances are pre-calculated """ arrays = arrays.split(',') narrays = len(arrays) if sim_splits is not None: assert not(unblind) def warn(): print("WARNING: no bandpass file found. Assuming array ",dm.c['id']," has no response to CMB, tSZ and CIB.") aspecs = tutils.ASpecs().get_specs bandpasses = not(effective_freq) savedir = tutils.get_save_path(version,region) assert len(cov_versions)==3 covdirs = [tutils.get_save_path(cov_versions[i],region) for i in range(3)] for covdir in covdirs: assert os.path.exists(covdir) if not(overwrite): assert not(os.path.exists(savedir)), \ "This version already exists on disk. Please use a different version identifier." try: os.makedirs(savedir) except: if overwrite: pass else: raise mask = enmap.read_map(covdir+"tilec_mask.fits") from scipy.ndimage.filters import gaussian_filter as smooth pm = enmap.read_map("/scratch/r/rbond/msyriac/data/planck/data/pr2/COM_Mask_Lensing_2048_R2.00_car_deep56_interp_order0.fits") wcs = pm.wcs mask = enmap.enmap(smooth(pm,sigma=10),wcs) * mask shape,wcs = mask.shape,mask.wcs Ny,Nx = shape modlmap = enmap.modlmap(shape,wcs) omodlmap = modlmap.copy() ells = np.arange(0,modlmap.max()) minell = maps.minimum_ell(shape,wcs) sel = np.where(np.logical_and(modlmap>=bin_edges[0]-minell,modlmap<=bin_edges[-1]+minell)) modlmap = modlmap[sel] bps = [] lbeams = [] kbeams = [] shifts = [] cfreqs = [] lmins = [] lmaxs = [] names = [] for i,qid in enumerate(arrays): dm = sints.models[sints.arrays(qid,'data_model')](region=mask,calibrated=True) if dm.name=='act_mr3': season,array1,array2 = sints.arrays(qid,'season'),sints.arrays(qid,'array'),sints.arrays(qid,'freq') array = '_'.join([array1,array2]) elif dm.name=='planck_hybrid': season,patch,array = None,None,sints.arrays(qid,'freq') else: raise ValueError lmin,lmax,hybrid,radial,friend,cfreq,fgroup,wrfit = aspecs(qid) lmins.append(lmin) lmaxs.append(lmax) names.append(qid) cfreqs.append(cfreq) if bandpasses: try: fname = dm.get_bandpass_file_name(array) bps.append("data/"+fname) if (pa1_shift is not None) and 'PA1' in fname: shifts.append(pa1_shift) elif (pa2_shift is not None) and 'PA2' in fname: shifts.append(pa2_shift) elif (pa3_150_shift is not None) and ('PA3' in fname) and ('150' in fname): shifts.append(pa3_150_shift) elif (pa3_090_shift is not None) and ('PA3' in fname) and ('090' in fname): shifts.append(pa3_90_shift) else: shifts.append(0) except: warn() bps.append(None) else: try: bps.append(cfreq) except: warn() bps.append(None) kbeam = tutils.get_kbeam(qid,modlmap,sanitize=not(unsanitized_beam),version=beam_version,planck_pixwin=True) if dm.name=='act_mr3': lbeam = tutils.get_kbeam(qid,ells,sanitize=not(unsanitized_beam),version=beam_version,planck_pixwin=False) # note no pixwin but doesnt matter since no ccorr for planck elif dm.name=='planck_hybrid': lbeam = None else: raise ValueError lbeams.append(lbeam) kbeams.append(kbeam.copy()) # Make responses responses = {} def _get_response(comp,param_override=None): if bandpasses: if no_act_color_correction: r = tfg.get_mix_bandpassed(bps, comp, bandpass_shifts=shifts, param_dict_override=param_override) else: r = tfg.get_mix_bandpassed(bps, comp, bandpass_shifts=shifts, ccor_cen_nus=cfreqs, ccor_beams=lbeams, ccor_exps = [ccor_exp] * narrays, param_dict_override=param_override) else: r = tfg.get_mix(bps, comp,param_dict_override=param_override) return r for comp in ['tSZ','CMB','CIB']: responses[comp] = _get_response(comp,None) from tilec.utils import is_planck ilcgens = [] okcoadds = [] for splitnum in range(2): covdir = covdirs[splitnum] kcoadds = [] for i,qid in enumerate(arrays): lmin = lmins[i] lmax = lmaxs[i] if is_planck(qid): dm = sints.models[sints.arrays(qid,'data_model')](region=mask,calibrated=True) _,kcoadd,_ = kspace.process(dm,region,qid,mask, skip_splits=True, splits_fname=sim_splits[i] if sim_splits is not None else None, inpaint=False,fn_beam = None, plot_inpaint_path = None, split_set=splitnum) else: kcoadd_name = covdir + "kcoadd_%s.npy" % qid kcoadd = enmap.enmap(np.load(kcoadd_name),wcs) kmask = maps.mask_kspace(shape,wcs,lmin=lmin,lmax=lmax) dtype = kcoadd.dtype kcoadds.append((kcoadd.copy()*kmask)[sel]) kcoadds = enmap.enmap(np.stack(kcoadds),wcs) okcoadds.append(kcoadds.copy()) # Read Covmat ctheory = ilc.CTheory(modlmap) nells = kcoadds[0].size cov = np.zeros((narrays,narrays,nells)) for aindex1 in range(narrays): for aindex2 in range(aindex1,narrays): qid1 = names[aindex1] qid2 = names[aindex2] if is_planck(names[aindex1]) or is_planck(names[aindex2]) or all_analytic: lmin,lmax,hybrid,radial,friend,f1,fgroup,wrfit = aspecs(qid1) lmin,lmax,hybrid,radial,friend,f2,fgroup,wrfit = aspecs(qid2) # If both are Planck and same array, get white noise from last bin icov = ctheory.get_theory_cls(f1,f2,a_cmb=1,a_gal=0.8)*kbeams[aindex1]*kbeams[aindex2] if aindex1==aindex2: pcov = enmap.enmap(np.load(covdirs[2]+"tilec_hybrid_covariance_%s_%s.npy" % (names[aindex1],names[aindex2])),wcs) pbin_edges = np.append(np.arange(500,3000,200) ,[3000,4000,5000,5800]) pbinner = stats.bin2D(omodlmap,pbin_edges) w = pbinner.bin(pcov)[1][-1] icov = icov + w else: icov = np.load(covdir+"tilec_hybrid_covariance_%s_%s.npy" % (names[aindex1],names[aindex2]))[sel] if aindex1==aindex2: icov[modlmap<lmins[aindex1]] = maxval icov[modlmap>lmaxs[aindex1]] = maxval cov[aindex1,aindex2] = icov cov[aindex2,aindex1] = icov assert np.all(np.isfinite(cov)) ilcgen = ilc.HILC(modlmap,np.stack(kbeams),cov=cov,responses=responses,invert=True) ilcgens.append(ilcgen) solutions = ['tSZ','tSZ-CMB','tSZ-CIB'] ypowers = {} w2 = np.mean(mask**2.) binner = stats.bin2D(modlmap,bin_edges) np.random.seed(100) blinding = np.random.uniform(0.8,1.2) if not(unblind) else 1 def _get_ypow(sname,dname,dresponse=None,dcmb=False): if dresponse is not None: assert dname is not None for splitnum in range(2): ilcgens[splitnum].add_response(dname,dresponse) ykmaps = [] for splitnum in range(2): if dcmb: assert dname is not None ykmap = ilcgens[splitnum].multi_constrained_map(okcoadds[splitnum],sname,[dname,"CMB"]) else: if dname is None: ykmap = ilcgens[splitnum].standard_map(okcoadds[splitnum],sname) else: ykmap = ilcgens[splitnum].constrained_map(okcoadds[splitnum],sname,dname) ykmaps.append(ykmap.copy()) ypower = (ykmaps[0]*ykmaps[1].conj()).real / w2 return binner.bin(ypower)[1] * blinding # The usual solutions for solution in solutions: sols = solution.split('-') if len(sols)==2: sname = sols[0] dname = sols[1] elif len(sols)==1: sname = sols[0] dname = None else: raise ValueError ypowers[solution] = _get_ypow(sname,dname,dresponse=None) # The CIB SED samples if beta_samples is not None: y_bsamples = [] y_bsamples_cmb = [] for beta in beta_samples: pdict = tfg.default_dict.copy() pdict['beta_CIB'] = beta response = _get_response("CIB",param_override=pdict) y_bsamples.append( _get_ypow("tSZ","iCIB",dresponse=response,dcmb=False) ) y_bsamples_cmb.append( _get_ypow("tSZ","iCIB",dresponse=response,dcmb=True) ) else: y_bsamples = None y_bsamples_cmb = None return binner.centers,ypowers,y_bsamples,y_bsamples_cmb
def __init__(self,qids,fg_res_version=None,ellmax=8101,bandpassed=True,no_act_color_correction=False,ccor_exp=-1): self.qids = qids self.alms = {} ells = np.arange(ellmax) self.ellmax = ellmax self.cyy = fgs.power_y(ells)[None,None] self.cyy[0,0][ells<2] = 0 narrays = len(qids) if fg_res_version is not None: fpath = sints.dconfig['actsims']['fg_res_path'] + "/" + fg_res_version + "/" self.cfgres = np.zeros((narrays,narrays,ellmax)) for i in range(narrays): for j in range(i,narrays): qid1 = qids[i] qid2 = qids[j] clfilename = "%shfgcov_%s_%s.txt" % (fpath,qid1,qid2) #!!!! Changed fgcov to hfgcov # !!!! clfilename_alt = "%shfgcov_%s_%s.txt" % (fpath,qid2,qid1) #!!!! Changed fgcov to hfgcov try: ls,cls = np.loadtxt(clfilename,unpack=True) except: ls,cls = np.loadtxt(clfilename_alt,unpack=True) assert np.all(np.isclose(ls,ells)) self.cfgres[i,j] = cls.copy() if i!=j: self.cfgres[j,i] = cls.copy() # self.cfgres[j,i] = self.cfgres[i,j] = cls.copy()*0 # !!!! REMOVED CORR #if (qid1 in ['p01','p02']) or (qid2 in ['p01','p02']): self.cfgres[j,i] = self.cfgres[i,j] = 0 # !!!! # self.cfgres[j,i][ls<300] = self.cfgres[i,j][ls<300] = 0 # !!!! else: self.cfgres = None # get tSZ response, and also zero out parts of map that are not observed aspecs = tutils.ASpecs().get_specs bps = [] cfreqs = [] lbeams = [] ells = np.arange(0,35000) for qid in qids: dm = sints.models[sints.arrays(qid,'data_model')]() if dm.name=='act_mr3': season,array1,array2 = sints.arrays(qid,'season'),sints.arrays(qid,'array'),sints.arrays(qid,'freq') array = '_'.join([array1,array2]) lbeam = tutils.get_kbeam(qid,ells,sanitize=True,planck_pixwin=False) # note no pixwin but doesnt matter since no ccorr for planck elif dm.name=='planck_hybrid': season,patch,array = None,None,sints.arrays(qid,'freq') lbeam = None else: raise ValueError lbeams.append(lbeam) lmin,lmax,hybrid,radial,friend,cfreq,fgroup,wrfit = aspecs(qid) if bandpassed: bps.append("data/"+dm.get_bandpass_file_name(array)) else: bps.append(cfreq) cfreqs.append(cfreq) if bandpassed: if no_act_color_correction: self.tsz_fnu = tfg.get_mix_bandpassed(bps, 'tSZ') else: self.tsz_fnu = tfg.get_mix_bandpassed(bps, 'tSZ', ccor_cen_nus=cfreqs, ccor_beams=lbeams, ccor_exps = [ccor_exp] * narrays) else: self.tsz_fnu = tfg.get_mix(bps, 'tSZ')
fnames = [] qids = "boss_01,boss_02,boss_03,boss_04,p01,p02,p03,p04,p05,p06,p07,p08".split( ',') ftsize = 18 pl = io.Plotter(xyscale='loglin', xlabel='$\\nu$ (GHz)', ylabel='$T(\\nu)$', figsize=(10, 3), ftsize=ftsize) lfi_done = False hfi_done = False act_done = False for qid in qids: dm = sints.models[sints.arrays(qid, 'data_model')]() if dm.name == 'act_mr3': season, array1, array2 = sints.arrays(qid, 'season'), sints.arrays( qid, 'array'), sints.arrays(qid, 'freq') array = '_'.join([array1, array2]) elif dm.name == 'planck_hybrid': season, patch, array = None, None, sints.arrays(qid, 'freq') fname = "data/" + dm.get_bandpass_file_name(array) if fname in fnames: continue fnames.append(fname) print(fname) nu, bp = np.loadtxt(fname, unpack=True, usecols=[0, 1]) if tutils.is_lfi(qid):
os.makedirs(scratch) except: pass """ MAKE SIMS """ jsim.update_signal_index(sim_index,set_idx=set_id) pa3_cache = {} # This assumes there are at most 2 pa3 arrays in the input sim_splits = [] for aindex in range(narrays): qid = arrays[aindex] fname = tutils.get_temp_split_fname(qid,args.region,sim_version) if not(args.use_cached_sims): dmname = sints.arrays(qid,'data_model') dm = sints.models[dmname](region=mask,calibrated=not(args.uncalibrated)) patch = args.region if dm.name=='act_mr3': season,array1,array2 = sints.arrays(qid,'season'),sints.arrays(qid,'array'),sints.arrays(qid,'freq') arrayname = array1 + "_" + array2 elif dm.name=='planck_hybrid': season,arrayname = None,sints.arrays(qid,'freq') with bench.show("signal"): # (npol,Ny,Nx) signal = jsim.compute_map(mask.shape,mask.wcs,qid, include_cmb=True,include_tsz=not(args.exclude_tsz), include_fgres=not(args.skip_fg),sht_beam=not(args.fft_beam))
def process(dm, patch, array_id, mask, skip_splits=False, splits_fname=None, inpaint=True, fn_beam=None, cache_inpaint_geometries=True, verbose=True, plot_inpaint_path=None, do_pol=False, split_set=None): """ Return (nsplits,Ny,Nx) fourier transform Return (Ny,Nx) fourier transform of coadd This function applies no corrections for masks. split_set : None by default means it returns all splits. Otherwise, can be 0 or 1, which determines which half of the splits to use. E.g. for ACT with 4 splits, 0 will return split 0,1 and 1 will return split 2,3. """ qid = array_id if dm.name == 'act_mr3': season, array1, array2 = sints.arrays(qid, 'season'), sints.arrays( qid, 'array'), sints.arrays(qid, 'freq') array = '_'.join([array1, array2]) pixwin = True elif dm.name == 'planck_hybrid': season, patch, array = None, None, sints.arrays(qid, 'freq') pixwin = False wins = dm.get_splits_ivar(season=season, patch=patch, arrays=[array], ncomp=None)[0, :, 0, :, :] if splits_fname is None: splits = dm.get_splits(season=season, patch=patch, arrays=[array], ncomp=3, srcfree=True)[0, :, :, :, :] else: splits = enmap.read_map(splits_fname) if split_set is not None: fsplits = splits.shape[0] assert fsplits == 2 or fsplits == 4 hsplits = fsplits // 2 assert split_set in [0, 1] if split_set == 0: wins = wins[:hsplits] splits = splits[:hsplits] elif split_set == 1: wins = wins[hsplits:] splits = splits[hsplits:] assert splits.ndim == 4 nsplits = splits.shape[0] if split_set is None: assert nsplits == 2 or nsplits == 4 else: assert nsplits == 1 or nsplits == 2 assert nsplits == wins.shape[0] # Inpaint if inpaint and dm.name == 'act_mr3': rsplits = [] for i in range(nsplits): result = inpainting.inpaint_map_white( splits[i], wins[i], fn_beam, union_sources_version=None, plots=False, cache_name="qid_%s_splitnum_%d" % (qid, i) if cache_inpaint_geometries else None, verbose=verbose) rsplits.append(result[0, :, :].copy()) if plot_inpaint_path is not None: io.hplot(splits[i][0, :, :], "%s/uninpainted_qid_%s_splitnum_%d" % (plot_inpaint_path, qid, i), grid=True, color='planck') io.hplot(result[0, :, :], "%s/inpainted_qid_%s_splitnum_%d" % (plot_inpaint_path, qid, i), grid=True, color='planck') # if do_pol: # for i in range(1,3): # io.hplot(splits[i][0,:,:],"%s/uninpainted_qid_%s_splitnum_%d" % (plot_inpaint_path,qid,i),grid=True,color='planck') # io.hplot(result[0,:,:],"%s/inpainted_qid_%s_splitnum_%d" % (plot_inpaint_path,qid,i),grid=True,color='planck') rsplits = enmap.enmap(np.stack(rsplits), splits.wcs) else: rsplits = splits[:, 0, :, :] kdiffs, kcoadd = process_splits(rsplits, wins, mask, skip_splits=skip_splits, do_fft_splits=False, pixwin=pixwin) return kdiffs, kcoadd, wins