def get_offset_result(res=1.,dtype=np.float64,seed=1):
shape,wcs = enmap.fullsky_geometry(res=np.deg2rad(res))
shape = (3,) + shape
obs_pos = enmap.posmap(shape, wcs)
np.random.seed(seed)
grad = enmap.enmap(np.random.random(shape),wcs)*1e-3
raw_pos = enmap.samewcs(lensing.offset_by_grad(obs_pos, grad, pol=shape[-3]>1, geodesic=True), obs_pos)
return obs_pos,grad,raw_pos
def solve(w, m):
if w.ndim < 4: return m / w
elif w.ndim == 4:
# This is slower, but handles low-hit areas near the edge better
iw = utils.eigpow(w, -1, axes=[0, 1])
return enmap.samewcs(array_ops.matmul(iw, m, axes=[0, 1]), m)
#return array_ops.solve_masked(w,m,axes=[0,1])
else:
raise NotImplementedError("Only 2d, 3d or 4d weight maps understood")
def coadd_map(map_list, ivar_list):
"""return coadded map from splits, input list of maps, where each map only contains one of I,Q or U"""
map_list = np.array(map_list)
ivar_list = np.array(ivar_list)
coadd_map = np.sum(map_list * ivar_list, axis=0)
coadd_map /= np.sum(ivar_list, axis=0)
coadd_map[~np.isfinite(coadd_map)] = 0
return enmap.samewcs(coadd_map, map_list[0])
def coadd_mapnew(map_list, ivar_list, a):
"""return coadded map from splits, the map in maplist contains I,Q,U
a=0,1,2 selects one of I Q U """
map_list = np.array(map_list)
ivar_list = np.array(ivar_list)
coadd_map = np.sum(map_list[:, a] * ivar_list, axis=0)
coadd_map /= np.sum(ivar_list, axis=0)
coadd_map[~np.isfinite(coadd_map)] = 0
return enmap.samewcs(coadd_map, map_list[0])
def build_cmb_2d(shape, wcs, cl_cmb, dtype=np.float32):
lmap = enmap.lmap(shape, wcs)
l = np.sum(lmap**2, 0)**0.5
cmb = enmap.samewcs(utils.interp(l, np.arange(cl_cmb.shape[-1]), cl_cmb),
l).astype(dtype)
# Rotate [TEB,EB] -> [TQU,TQU]. FIXME: not a perfect match
R = enmap.queb_rotmat(lmap, spin=2, inverse=True)
cmb[1:, :] = np.einsum("abyx,bcyx->acyx", R, cmb[1:, :])
cmb[:, 1:] = np.einsum("abyx,cbyx->acyx", cmb[:, 1:], R)
return cmb
def ivar_eff(split, ivar_list):
"""return effective invers variance map for split i and a list of inverse variance maps.
Inputs
splits:integer
ivar_list:list
Output
ndmap with same shape and wcs as individual inverse variance maps.
"""
ivar_list = np.array(ivar_list)
h_c = np.sum(ivar_list, axis=0)
w = h_c - ivar_list[split]
weight = 1 / (1 / ivar_list[split] - 1 / h_c)
weight[~np.isfinite(weight)] = 0
weight[weight < 0] = 0
return enmap.samewcs(weight, ivar_list[0])
# wmap = enmap.ifft(enmap.fft(wmap)*matched_filter).real**2
# lim = max(np.median(wmap)*tol1**2, np.max(wmap)*tol2**2)
# mask |= wmap > lim
# return mask
comm = mpi.COMM_WORLD
beam1d = get_beam(args.beam)
ifiles = sorted(sum([glob.glob(ifile) for ifile in args.ifiles], []))
for ind in range(comm.rank, len(ifiles), comm.size):
ifile = ifiles[ind]
if args.verbose: print(ifile)
ofile = args.odir + "/" + ifile
imap = enmap.read_map(ifile)
if args.mask is not None: mask = imap == args.mask
if args.apodize:
imap = imap.apod(args.apodize)
# We will apply a semi-matched-filter to T
l = np.maximum(1, imap.modlmap())
beam2d = enmap.samewcs(np.interp(l, np.arange(len(beam1d)), beam1d), imap)
matched_filter = (1 + (l / args.lknee)**args.alpha)**-1 * beam2d
fmap = enmap.map2harm(imap, iau=True)
fmap[0] *= matched_filter
omap = enmap.ifft(fmap).real
if args.mask is not None:
omap[mask] = 0
del mask
utils.mkdir(os.path.dirname(ofile))
enmap.write_map(ofile, omap)
del omap
lmin=300,
lmax=8000,
wnoise_annulus=500,
bin_annulus=20,
lknee_guess=3000,
alpha_guess=-4,
method="fft",
radial_fit=True)
#io.plot_img(np.fft.fftshift(np.log10(ndown)),"ndown.png",aspect='auto',lim=[-6,3])
#io.hplot(np.fft.fftshift(np.log10(ndown)),"hndown")
io.hplot(np.fft.fftshift((ndown)), "hndown")
io.plot_img(np.fft.fftshift(ndown / nfitted), "nunred.png", aspect='auto')
nmod = ndown / nfitted
enmap.write_map("anisotropy_template.fits", enmap.samewcs(nmod, npower))
shape, wcs = maps.rect_geometry(width_deg=50.,
height_deg=30,
px_res_arcmin=0.5)
rms = 10.0
lknee = 3000
alpha = -3
n2d = covtools.get_anisotropic_noise(shape, wcs, rms, lknee, alpha)
modlmap = enmap.modlmap(shape, wcs)
bin_edges = np.arange(100, 8000, 100)
binner = stats.bin2D(modlmap, bin_edges)
cents, n1d = binner.bin(n2d)
pl = io.Plotter(yscale='log', xlabel='l', ylabel='C')
pl.add(cents, n1d)
filename = op.join(input_dir, f)
imap = enmap.extract(enmap.read_map(filename), sample_sim_deep56.shape,
sample_sim_deep56.wcs)
if first:
omap = imap.copy()
first = False
else:
omap += imap
hits_map = omap
# seed the random number
np.random.seed(10)
# generate alpha with standard gaussian
alpha_0 = np.random.randn(*omap.shape) * alpha_error
alpha_map_0 = enmap.samewcs(alpha_0, omap)
alpha_map = alpha_map_0 * hits_map**-0.5
alpha_map[~np.isfinite(alpha_map)] = 0
enmap.write_map("relrot_deep56.fits", alpha_map)
# get Nhits map
first = True
for f in boss_files:
filename = op.join(input_dir, f)
imap = enmap.extract(enmap.read_map(filename), sample_sim_boss.shape,
sample_sim_boss.wcs)
if first:
omap = imap.copy()
first = False
else:
omap += imap
def mplot(img, savename=None, verbose=True, **kwargs):
from pixell import enmap
plot_img(enmap.samewcs(np.fft.fftshift(np.log10(img)), img),
filename=savename,
verbose=verbose,
**kwargs)
def fplot(img,savename=None,verbose=True,log=True,**kwargs):
from pixell import enmap
lfunc = np.log10 if log else lambda x: x
hplot(enmap.samewcs(np.fft.fftshift(lfunc(img)),img),savename=savename,verbose=verbose,**kwargs)
def mul(w, m):
if w.ndim < 4: return m * w
elif w.ndim == 4:
return enmap.samewcs(array_ops.matmul(w, m, axes=[0, 1]), m)
else:
raise NotImplementedError("Only 2d, 3d or 4d weight maps understood")
def smooth_ps_angular(ps2d, brel=5):
ps1d, l1d = ps2d.lbin(brel=brel)
#np.savetxt("ps1d.txt", np.concatenate([l1d[None], ps1d.reshape(-1,len(l1d))],0).T, fmt="%15.7e")
l = ps2d.modlmap()
return enmap.samewcs(utils.interp(l, l1d, ps1d), ps2d)
def search_maps(ifiles,
mode="find",
icat=None,
sel=None,
pixbox=None,
box=None,
templates=default_templates,
cl_cmb=None,
freq0=98.0,
nmat1="constcorr",
nmat2="constcorr",
snr1=5,
snr2=4,
comps="TQU",
dtype=np.float32,
apod=15 * utils.arcmin,
verbose=False,
sim_cat=None,
sim_noise=False,
mask=None):
"""Search the maps given by ifiles for objects, returning a bunch containing a catalog
of objects, etc.
Arguments:
* ifiles: A list of paths to mapdata files. These should be in µK units.
* mode: What operation to do. Either "find" or "fit". Defaults to "find"
* "find": Do a blind object search in the maps.
* "fit": Do forced photometry on the positions provided in the input catalog icat.
* sel, pixbox, box: These let you work with a subset of the maps. Same meaning as
in enmap.read_map. Default to None.
* templates: What spectral and spatial shapes to look for. This is a list of tuples
that will be passed to build_cases. Defaults to default_templates.
* cl_cmb: The CMB angular power spectrum. Ideally [TQU,TQU,nl] (note: not TEB).
Used to build the blind noise model.
* freq0: The reference frequency in GHz. This is used when reporting the overall flux
of multifrequency templates.
* nmat1: The noise model to use for the first pass of the search. These are
built from simple analytic models, not measured from the data.
"constcov": A constant covariance noise model. This is fast, but
purely harmonic, so it can't handle variations in hitcount.
"constcorr": A constant correlation noise model, where a constant
noise spectrum is modulated by the hitcount. Handles both correlations
and spatial variations, in a limited way. [default]
* nmat2: The noise model to use for the second pass of the search. These
are built from maps cleaned using the first pass. Defaults to "constcorr".
"none": Disables the second pass, returning the catalog found in the first pass
"constcov", "constcorr": As nmat1, but measured from the data. The power spectra
are smoothed isotropically for now.
* snr1: The S/N threshold used for the first pass in "find" mode. Defaults to 5.
* snr2. The S/N threshold used for the second pass in "find" mode. Defaults to 4.
* comps: Controls the Stokes parameters considered. Can be "T" or "TQU". Defaults to "TQU".
* dtype: The maps will be cast to this data type after reading in. Defaults to np.float32.
* apod: How much apodization is used at the edges (including edges of unhit areas),
in radians. This is necessary to avoid ringing artifacts in the fourier transforms.
Defaults to 15 arcmin.
* verbose: Whether to print what it's doing. Defaults to False.
* sim_noise: Whether to replace the raw data with noise. Currently a cmb-less constcorr
realization. Should fix this.
* sim_cat: A catalog to inject into the maps before the search. Defaults to None.
* mask: Path to a mask enmap that's True in bad regions and False in good regions.
Bad regions will be given zero weight and apodized.
Returns a bunch with the following members:
* cat: A catalog with the data type [("ra", "d"), ("dec", "d"), ("snr", "d", (ncomp,)),
("flux_tot", "d", (ncomp,)), ("dflux_tot", "d", (ncomp,)), ("flux", "d", (nfield,ncomp)),
("dflux", "d", (nfield,ncomp)), ("case", "i"), ("contam", "d", (nfield,))]
* maps: The maps that were searched [nfreq,ncomp,ny,nx]
* model: The best-fit model
* snr: The S/N ratio for the input maps [ny,nx]
* resid_snr: The S/N ratio after subtracting the best-fit model
* freqs: The frequencies, in GHz
* fconvs: The µK -> mJy/sr flux conversion factor for each frequency
* inds: The index of each entry in the output catalog in the input catalog.
Only relevant when mode == "fit".
"""
# Read in the total intensity data
if verbose: print("Reading T from %s" % str(ifiles))
data = read_data(ifiles,
sel=sel,
pixbox=pixbox,
box=box,
dtype=dtype,
apod=apod,
mask=mask)
data.freq0 = freq0
ncomp = len(comps)
nfield = len(data.maps)
cat_dtype = [("ra", "d"), ("dec", "d"), ("snr", "d", (ncomp, )),
("flux_tot", "d", (ncomp, )), ("dflux_tot", "d", (ncomp, )),
("flux", "d", (nfield, ncomp)),
("dflux", "d", (nfield, ncomp)), ("case", "i"),
("contam", "d", (nfield, ))]
cases = build_cases(data, templates)
# Get the part of the catalog inside our area
if mode == "fit":
inds = np.where(
np.any(data.ivars.at([icat.dec, icat.ra], order=0) > 0, 0))[0]
subicat = icat[inds]
else:
inds = None
# Abort if we have no data to process
if np.all(data.ivars == 0):
map_tot = enmap.zeros((nfield, ncomp) + data.maps.shape[-2:],
data.maps.wcs, dtype)
cat = np.zeros(0, cat_dtype)
return bunch.Bunch(cat=cat,
maps=map_tot,
model=map_tot,
snr=map_tot[0, 0],
resid_snr=map_tot[0, 0],
hits=map_tot[0, 0],
fconvs=data.fconvs,
freqs=data.freqs,
inds=inds)
# Build our noise model, based on a 1/l spectrum + cmb + a foreground penalty
cmb = build_cmb_2d(*data.maps.geometry, cl_cmb,
dtype=data.maps.dtype) if cl_cmb is not None else None
fg_var = build_foreground_var(data.maps)
nmat = build_nmat_prior(data,
type=nmat1,
fg_var=fg_var,
cmb=cmb[0, 0] if cmb is not None else None)
# Optionally inject signal. FIXME: This should be moved after nmat is defined,
# so we can let nmat handle the noise simulation
if sim_noise: data.maps = nmat.simulate() * data.apod
if sim_cat is not None:
inject_objects(data, cases, slice_cat_comp(sim_cat, 0))
# Total intensity
if mode == "find":
if verbose: print("1st pass T find")
res_t = find_objects(data,
cases,
nmat,
snmin=snr1,
resid=nmat2 == "none",
verbose=verbose)
elif mode == "fit":
if verbose: print("1st pass T measure")
res_t = measure_objects(data,
cases,
nmat,
slice_cat_comp(subicat, 0),
resid=nmat2 == "none",
verbose=verbose)
else:
raise ValueError("Unrecognized mode '%s'" % (mode))
if nmat2 != "none":
noise = data.maps - res_t.model
if "bad_mask" in res_t:
noise_apod = enmap.apod_mask(1 - res_t.bad_mask,
10 * utils.arcmin,
edge=False)
noise *= noise_apod
noise /= np.mean(noise_apod**2)
#enmap.write_map("noise.fits", noise)
del noise_apod
nmat = build_nmat_empirical(data, noise, fg_var=fg_var, type=nmat2)
if mode == "find":
if verbose: print("2nd pass T find")
res_t = find_objects(data,
cases,
nmat,
snmin=snr2,
resid=True,
verbose=verbose)
elif mode == "fit":
if verbose: print("2nd pass T measure")
res_t = measure_objects(data,
cases,
nmat,
slice_cat_comp(subicat, 0),
resid=True,
verbose=verbose)
else:
raise ValueError("Unrecognized mode '%s'" % (mode))
res = [res_t]
# Polarization is always "fit", since anything that would be found in polarization
# would definitely be found in total intensity
if comps == "T":
pass
elif comps == "TQU":
# Measure polarization too
for comp in [1, 2]:
if verbose:
print("Reading %s from %s" % (comps[comp], str(ifiles)))
data = read_data(ifiles,
sel=sel,
pixbox=pixbox,
box=box,
comp=comp,
apod=apod)
data.freq0 = freq0
# Optionally inject signal
if sim_cat is not None:
inject_objects(data, cases, slice_cat_comp(sim_cat, comp))
if verbose: print("1st pass %s measure" % comps[comp])
nmat = build_nmat_prior(data,
type=nmat1,
pol=True,
cmb=cmb[comp,
comp] if cmb is None else None)
res_p = measure_objects(data,
cases,
nmat,
res_t.cat,
verbose=verbose)
if nmat2 != "none":
if verbose: print("2nd pass %s measure" % comps[comp])
nmat = build_nmat_empirical(data,
noise_map=data.maps - res_p.model,
type=nmat2)
res_p = measure_objects(data,
cases,
nmat,
res_t.cat,
verbose=verbose)
res.append(res_p)
# First the catalog
cat = np.zeros(len(res_t.cat), cat_dtype).view(np.recarray)
cat.ra = res_t.cat.ra
cat.dec = res_t.cat.dec
cat.case = res_t.cat.case
cat.contam = res_t.cat.contam
for i in range(len(res)):
cat.snr[:, i] = res[i].cat.snr
cat.flux_tot[:, i] = res[i].cat.flux_tot
cat.dflux_tot[:, i] = res[i].cat.dflux_tot
cat.flux[:, :, i] = res[i].cat.flux
cat.dflux[:, :, i] = res[i].cat.dflux
# Then the maps
map_tot = enmap.samewcs(np.concatenate([r.maps[:, None] for r in res], 1),
data.maps)
model_tot = enmap.samewcs(
np.concatenate([r.model[:, None] for r in res], 1), data.maps)
result = bunch.Bunch(cat=cat,
maps=map_tot,
model=model_tot,
fconvs=data.fconvs,
freqs=data.freqs,
inds=inds)
# These only exist in "find" mode
for key in ["snr", "resid_snr", "hits"]:
result[key] = res_t[key] if key in res_t else None
return result
def fplot(img, savename=None, verbose=True, **kwargs):
hplot(enmap.samewcs(np.fft.fftshift(np.log10(img)), img),
savename=savename,
verbose=verbose,
**kwargs)
def fft(mappa):
return enmap.samewcs(enmap.fft(mappa, normalize='phys'), mappa)
