def inpaint_map_const_cov(imap,mask,union_sources_version=None,noise_pix = 20,hole_radius = 3.,plots=False):
"""
Inpaints a map under the assumption of constant 2D Fourier covariance. This uses the average PS
of the full map for the noise model at each source location and thus does not handle inhomogenity.
Pros: no products needed other than map-maker outputs of map
Cons:
imap -- (npol,Ny,Nx)
ivar -- (Ny,Nx)
"""
ras,decs = sints.get_act_mr3f_union_sources(version=union_sources_version)
kmap = enmap.fft(mask*imap,normalize='phys')
gtags = []
gdicts = {}
pcoords = []
for i,(ra,dec) in enumerate(zip(ras,decs)):
sel = reproject.cutout(ivar, ra=np.deg2rad(ra), dec=np.deg2rad(dec), pad=1, corner=False,npix=noise_pix,return_slice=True)
if sel is None: continue
civar = ivar[sel]
if np.any(civar<=0): continue
modrmap = civar.modrmap()
modlmap = civar.modlmap()
res = maps.resolution(civar.shape,civar.wcs)
cimap = imap[sel]
print(ra,dec)
if plots:
for p in range(3): io.plot_img(cimap[p],os.environ['WORK']+"/cimap_%d_%s" % (p,str(i).zfill(2)))
mimap = cimap.copy()
mimap[...,modrmap<np.deg2rad(hole_radius/60.)] = np.nan
for p in range(3): io.plot_img(mimap[p],os.environ['WORK']+"/masked_cimap_%d_%s" % (p,str(i).zfill(2)))
scov = pixcov.scov_from_theory(modlmap,cmb_theory_fn,fn_beam,iau=False)
ncov = pixcov.ncov_from_ivar(civar)
pcov = scov + ncov
gdicts[i] = pixcov.make_geometry(hole_radius=np.deg2rad(hole_radius/60.),n=noise_pix,deproject=True,iau=False,pcov=pcov,res=res)
pcoords.append(np.array((dec,ra)))
gtags.append(i)
if len(gtags)>0:
pcoords = np.stack(pcoords).swapaxes(0,1)
result = pixcov.inpaint(imap,pcoords,deproject=True,iau=False,geometry_tags=gtags,geometry_dicts=gdicts,verbose=True)
if plots:
for i,(ra,dec) in enumerate(zip(ras,decs)):
sel = reproject.cutout(ivar, ra=np.deg2rad(ra), dec=np.deg2rad(dec), pad=1, corner=False,npix=noise_pix,return_slice=True)
if sel is None: continue
civar = ivar[sel]
if np.any(civar<=0): continue
modrmap = civar.modrmap()
modlmap = civar.modlmap()
res = maps.resolution(civar.shape,civar.wcs)
cimap = result[sel]
print("Inpainted ", ra,dec)
if plots:
for p in range(3): io.plot_img(cimap[p],os.environ['WORK']+"/inpainted_cimap_%d_%s" % (p,str(i).zfill(2)))
return result
def mask_map(imap, iys, ixs, hole_arc, hole_frac=0.6):
shape, wcs = imap.shape, imap.wcs
Ny, Nx = shape[-2:]
px = maps.resolution(shape, wcs) * 60. * 180. / np.pi
hole_n = int(round(hole_arc / px))
hole_ny = hole_nx = hole_n
oshape, owcs = enmap.geometry(pos=(0., 0.),
shape=(2 * hole_n, 2 * hole_n),
res=px * np.pi / 180. / 60.)
modrmap = enmap.modrmap(oshape, owcs)
mask = enmap.ones(shape, wcs)
for iy, ix in zip(iys, ixs):
if iy <= hole_n or ix <= hole_n or iy >= (Ny - hole_n) or ix >= (
Nx - hole_n):
continue
vslice = imap[np.int(iy - hole_ny):np.int(iy + hole_ny),
np.int(ix - hole_nx):np.int(ix + hole_nx)]
vslice[modrmap < (hole_frac * hole_arc) * np.pi / 180. /
60.] = np.nan # !!!! could cause a bias
mask[np.int(iy - hole_ny):np.int(iy + hole_ny),
np.int(ix - hole_nx):np.int(ix + hole_nx)][modrmap < hole_arc *
np.pi / 180. / 60.] = 0
return mask
def init_geometry(ishape,iwcs):
modlmap = enmap.modlmap(ishape,iwcs)
bin_edges = np.arange(args.kellmin,args.kellmax,args.dell)
binner = stats.bin2D(modlmap,bin_edges)
if args.beam<1e-5:
kbeam = None
else:
kbeam = maps.gauss_beam(modlmap,args.beam)
lmax = modlmap.max()
ells = np.arange(2,lmax,1)
wnoise_TT = ells*0.+(args.noise*(np.pi/180./60.))**2.
wnoise_PP = 2.*wnoise_TT
nT = modlmap*0.+(args.noise*(np.pi/180./60.))**2.
nP = 2.*nT
ncomp = 3 if pol else 1
ps = np.zeros((ncomp,ncomp,ells.size))
ps[0,0] = wnoise_TT
if pol:
ps[1,1] = wnoise_PP
ps[2,2] = wnoise_PP
oshape = (3,)+ishape if pol else ishape
if not(args.flat) and args.noise_pad>1.e-5:
# Pad noise sim geometry
pad_width_deg = args.noise_pad
pad_width = pad_width_deg * np.pi/180.
res = maps.resolution(oshape[-2:],iwcs)
pad_pixels = int(pad_width/res)
template = enmap.zeros(oshape,iwcs)
btemplate = enmap.pad(template,pad_pixels)
bshape,bwcs = btemplate.shape,btemplate.wcs
del template
del btemplate
ngen = maps.MapGen(bshape,bwcs,ps)
else:
ngen = maps.MapGen(oshape,iwcs,ps)
tmask = maps.mask_kspace(ishape,iwcs,lmin=args.tellmin,lmax=args.tellmax)
pmask = maps.mask_kspace(ishape,iwcs,lmin=args.pellmin,lmax=args.pellmax)
kmask = maps.mask_kspace(ishape,iwcs,lmin=args.kellmin,lmax=args.kellmax)
qest = lensing.qest(ishape,iwcs,theory,noise2d=nT,beam2d=kbeam,kmask=tmask,noise2d_P=nP,kmask_P=pmask,kmask_K=kmask,pol=pol,grad_cut=None,unlensed_equals_lensed=True)
taper,w2 = maps.get_taper_deg(ishape,iwcs,taper_width_degrees = args.taper_width,pad_width_degrees = args.pad_width)
fc = maps.FourierCalc(oshape,iwcs,iau=args.iau)
purifier = maps.Purify(ishape,iwcs,taper) if args.purify else None
return qest,ngen,kbeam,binner,taper,fc,purifier
def compute_map(self,oshape,owcs,qid,pixwin_taper_deg=0.3,pixwin_pad_deg=0.3,
include_cmb=True,include_tsz=True,include_fgres=True,sht_beam=True):
"""
1. get total alm
2. apply beam, and pixel window if Planck
3. ISHT
4. if ACT, apply a small taper and apply pixel window in Fourier space
"""
# pad to a slightly larger geometry
tot_pad_deg = pixwin_taper_deg + pixwin_pad_deg
res = maps.resolution(oshape,owcs)
pix = np.deg2rad(tot_pad_deg)/res
omap = enmap.pad(enmap.zeros((3,)+oshape,owcs),pix)
ishape,iwcs = omap.shape[-2:],omap.wcs
# get data model
dm = sints.models[sints.arrays(qid,'data_model')](region_shape=ishape,region_wcs=iwcs,calibrated=True)
# 1. get total alm
array_index = self.qids.index(qid)
tot_alm = int(include_cmb)*self.alms['cmb']
if include_tsz:
try:
assert self.tsz_fnu.ndim==2
tot_alm[0] = tot_alm[0] + hp.almxfl(self.alms['comptony'][0] ,self.tsz_fnu[array_index])
except:
tot_alm[0] = tot_alm[0] + self.alms['comptony'][0] * self.tsz_fnu[array_index]
if self.cfgres is not None: tot_alm[0] = tot_alm[0] + int(include_fgres)*self.alms['fgres'][array_index]
assert tot_alm.ndim==2
assert tot_alm.shape[0]==3
ells = np.arange(self.lmax+1)
# 2. get beam, and pixel window for Planck
if sht_beam:
beam = tutils.get_kbeam(qid,ells,sanitize=False,planck_pixwin=False) # NEVER SANITIZE THE BEAM IN A SIMULATION!!!
for i in range(3): tot_alm[i] = hp.almxfl(tot_alm[i],beam)
if dm.name=='planck_hybrid':
pixwint,pixwinp = hp.pixwin(nside=tutils.get_nside(qid),lmax=self.lmax,pol=True)
tot_alm[0] = hp.almxfl(tot_alm[0],pixwint)
tot_alm[1] = hp.almxfl(tot_alm[1],pixwinp)
tot_alm[2] = hp.almxfl(tot_alm[2],pixwinp)
# 3. ISHT
omap = curvedsky.alm2map(np.complex128(tot_alm),omap,spin=[0,2])
assert omap.ndim==3
assert omap.shape[0]==3
if not(sht_beam):
taper,_ = maps.get_taper_deg(ishape,iwcs,taper_width_degrees=pixwin_taper_deg,pad_width_degrees=pixwin_pad_deg)
modlmap = omap.modlmap()
beam = tutils.get_kbeam(qid,modlmap,sanitize=False,planck_pixwin=True)
kmap = enmap.fft(omap*taper,normalize='phys')
kmap = kmap * beam
# 4. if ACT, apply a small taper and apply pixel window in Fourier space
if dm.name=='act_mr3':
if sht_beam: taper,_ = maps.get_taper_deg(ishape,iwcs,taper_width_degrees=pixwin_taper_deg,pad_width_degrees=pixwin_pad_deg)
pwin = tutils.get_pixwin(ishape[-2:])
if sht_beam:
omap = maps.filter_map(omap*taper,pwin)
else:
kmap = kmap * pwin
if not(sht_beam): omap = enmap.ifft(kmap,normalize='phys').real
return enmap.extract(omap,(3,)+oshape[-2:],owcs)
def inpaint_map_white(imap,ivar,fn_beam,union_sources_version=None,noise_pix = 40,hole_radius = 6.,plots=False,cache_name=None,verbose=True):
"""
Inpaints a map under the assumption of inhomogenous but white uncorrelated instrument noise.
Pros: no products needed other than map-maker outputs of map and ivar, good inpainting despite crappy
noise model.
Cons: noise model has to be built for each source, so this is actually quite slow.
imap -- (npol,Ny,Nx)
ivar -- (Ny,Nx)
fn_beam -- lambda ells: beam(ells)
cache_name -- a unique string identifying the catalog+map/array/frequency/split combination to/from which the geometries are cached
"""
if cache_name is not None:
cache_name = cache_name + "_catversion_%s" % union_sources_version
try:
ras,decs,gtags,pcoords,gdicts = load_cached_inpaint_geometries(cache_name)
do_geoms = False
if verbose: print("actsims.inpaint: loaded cached geometries for ", cache_name)
except:
if verbose: print("actsims.inpaint: no cached geometries found for ", cache_name, ". Generating and saving...")
do_geoms = True
else:
do_geoms = True
if do_geoms:
ras,decs = sints.get_act_mr3f_union_sources(version=union_sources_version)
cmb_theory_fn = lambda s,l: cosmology.default_theory().lCl(s,l)
gtags = []
gdicts = {}
pcoords = []
for i,(ra,dec) in enumerate(zip(ras,decs)):
sel = reproject.cutout(ivar, ra=np.deg2rad(ra), dec=np.deg2rad(dec), pad=1, corner=False,npix=noise_pix,return_slice=True)
if sel is None: continue
civar = ivar[sel]
if np.any(civar<=0): continue
modrmap = civar.modrmap()
modlmap = civar.modlmap()
res = maps.resolution(civar.shape,civar.wcs)
cimap = imap[sel]
if verbose: print("actsims.inpaint: built noise model for source ",i," / ",len(ras))
scov = pixcov.scov_from_theory(modlmap,cmb_theory_fn,fn_beam,iau=False)
ncov = pixcov.ncov_from_ivar(civar)
pcov = scov + ncov
gdicts[i] = pixcov.make_geometry(hole_radius=np.deg2rad(hole_radius/60.),n=noise_pix,deproject=True,iau=False,pcov=pcov,res=res)
pcoords.append(np.array((dec,ra)))
gtags.append(i)
if len(gtags)>0:
pcoords = np.stack(pcoords).swapaxes(0,1)
if cache_name is not None:
save_cached_inpaint_geometries(cache_name,ras,decs,gtags,pcoords,gdicts)
if verbose: print("actsims.inpaint: cached geometries for ",cache_name)
if len(gtags)>0: result = pixcov.inpaint(imap,pcoords,deproject=True,iau=False,geometry_tags=gtags,geometry_dicts=gdicts,verbose=verbose)
if plots:
for i,(ra,dec) in enumerate(zip(ras,decs)):
sel = reproject.cutout(ivar, ra=np.deg2rad(ra), dec=np.deg2rad(dec), pad=1, corner=False,npix=noise_pix,return_slice=True)
if sel is None: continue
civar = ivar[sel]
if np.any(civar<=0): continue
modrmap = civar.modrmap()
modlmap = civar.modlmap()
res = maps.resolution(civar.shape,civar.wcs)
cimap = imap[sel]
for p in range(3): io.plot_img(cimap[p],os.environ['WORK']+"/cimap_%d_%s" % (p,str(i).zfill(2)))
mimap = cimap.copy()
mimap[...,modrmap<np.deg2rad(hole_radius/60.)] = np.nan
for p in range(3): io.plot_img(mimap[p],os.environ['WORK']+"/masked_cimap_%d_%s" % (p,str(i).zfill(2)))
cimap = result[sel]
for p in range(3): io.plot_img(cimap[p],os.environ['WORK']+"/inpainted_cimap_%d_%s" % (p,str(i).zfill(2)))
return result
def make_circular_geometry(shape,
wcs,
context_arcmin,
hole_arcmin,
power2d,
buffer_factor=2,
verbose=False):
'''Makes the circular geometry matrices that need to be pre-calculated for later inpainting.
Arguments
---------
input2DPower - ndarray containing 2D power spectrum of a map. It need not already have been
downsampled to the shape of the stamp cutout
inputLy
inputLx - the fourier wavenumbers corresponding to the y and x axes of the stamp cutout
stampArc - the width in arcminutes of the stamp cut out
stampPxX - the pixel width in arcminutes of the stamp cut out in the x direction
stampPxY - the pixel width in arcminutes of the stamp cut out in the y direction
holeArc - the radius of the circular hole in arcminutes
bufferFactor - the pixel covariance matrix will be calculated on a periodic stamp larger by this
factor
verbose - True if you want more commentary
Returns
-------
meanMul - a matrix that has shape (nh,nc) where nh is the number of pixels in the hole and
nc is the number of pixels outside (in the "context"). It should be multiplied
by a vector (nc) containing pixels outside to get a vector (nh) for the mean
value of the pixels inside
covRoot - a (nh,nh) sqrt(covariance matrix) that can be used to generate a random realization
in the hole. This can be generated by multiplying the sqrt of cov by a vector (nh)
of standard normal variables. The generated vector should be added to the mean value
obtained using meanMul
pcov - the pixel-pixel covariance matrix used in intermediate steps, if you want to re-use it
targetTemplate - a liteMap template of the stamp cutout
m1 - a boolean array that can be used to select the hole region
m2 - a boolean array that can be used to select the context region
'''
arc = context_arcmin
res = maps.resolution(shape, wcs) * 60. * 180. / np.pi
bshape, bwcs = maps.rect_geometry(width_arcmin=arc * buffer_factor,
px_res_arcmin=res)
tshape, twcs = maps.rect_geometry(width_arcmin=arc, px_res_arcmin=res)
sny, snx = tshape
bmodlmap = enmap.modlmap(bshape, bwcs)
modlmap = enmap.modlmap(shape, wcs)
if verbose: print("Downsampling...")
Niy, Nix = shape[-2:]
Noy, Nox = bshape[-2:]
# print(bshape,tshape,power2d.shape)
# io.plot_img(np.fft.fftshift(np.log10(power2d)))
#out_power = resample.resample_fft(power2d,bshape[-2:],axes=[-2,-1])
#out_power = np.fft.ifftshift(resample.resample_fft(np.fft.fftshift(power2d),bshape[-2:],axes=[-2,-1]))
out_power = resample.resample_bin(
power2d, factors=[float(Noy) / Niy, float(Nox) / Nix], axes=[-2, -1])
# io.plot_img(np.fft.fftshift(np.log10(out_power)))
# print(out_power.shape)
if verbose: print("Starting slow part...")
d = maps.diagonal_cov(out_power)
with bench.show("pixcov"):
pcov = maps.pixcov(bshape, bwcs, d)[0, 0, :sny, :snx, :sny, :snx]
modrmap = enmap.modrmap(tshape, twcs)
m1 = np.where(modrmap.reshape(-1) < hole_arcmin * np.pi / 180. / 60.)[0]
m2 = np.where(modrmap.reshape(-1) >= hole_arcmin * np.pi / 180. / 60.)[0]
with bench.show("geom"):
meanMul, cov = get_geometry(pcov.reshape(sny * snx, sny * snx), m1, m2)
covRoot = stats.eig_pow(cov, 0.5)
return meanMul, covRoot, pcov, tshape, twcs, m1, m2
from __future__ import print_function
from orphics import maps,io,cosmology,catalogs
from pixell import enmap,reproject
import numpy as np
import os,sys,shutil
from soapack import interfaces as sints
import healpy as hp
box = np.deg2rad([ [11,-173.5],
[13.5,-171] ])
rosat = enmap.read_map("/scratch/r/rbond/msyriac/data/for_sigurd/rosat_r7_boss.fits").submap(box)
print(rosat.shape,maps.resolution(rosat.shape,rosat.wcs) * 180.*60./np.pi)
version = "map_v1.0.0_rc_joint"
cversion = "v1.0.0_rc"
region = 'boss'
yname = "/scratch/r/rbond/msyriac/data/depot/tilec/v1.0.0_rc_20190919/%s_%s/tilec_single_tile_%s_comptony_%s.fits" % (version,region,region,version)
ymap = enmap.read_map(yname).submap(box)
print(ymap.center()*180./np.pi)
rosat = enmap.project(rosat,ymap.shape,ymap.wcs)
io.hplot(np.log10(rosat),'fig_virgo_rosat')
#io.hplot(enmap.smooth_gauss(ymap,np.deg2rad(3./60.)),'fig_virgo_act',color='gray') #,min=-1.25e-5,max=3.0e-5
io.hplot(ymap,'fig_virgo_act',color='gray') #,min=-1.25e-5,max=3.0e-5