def calc_amp(self, profile): ivamp= np.sum(profile**2*self.div) if ivamp == 0: return 0, np.inf with utils.nowarn(): vamp = 1/ivamp amp = vamp*np.sum(profile*self.div*self.map) if ~np.isfinite(amp): amp = 0 return amp, vamp
def calc_amp(self, profile):
ivamp = np.sum(profile**2 * self.div)
if ivamp == 0: return 0, np.inf
with utils.nowarn():
vamp = 1 / ivamp
amp = vamp * np.sum(profile * self.div * self.map)
if ~np.isfinite(amp): amp = 0
return amp, vamp
def calc_delta_score(split_hits, bhits, mask): # fractional improvement is (split_hits + bhits)/split_hits -1 = bhits/split_hits # This will often lead to division by zero. That is not catastrophic, but loses # the ability to distinguish between multiple cases that would all fill in empty pixels. # So we cap the ratio to a large number. with utils.nowarn(): ratio = bhits/split_hits ratio[np.isnan(ratio)] = 0 ratio = np.minimum(ratio, 1000) return np.sum(ratio[:,mask],-1)
def calc_delta_score(split_hits, bhits, mask): # fractional improvement is (split_hits + bhits)/split_hits -1 = bhits/split_hits # This will often lead to division by zero. That is not catastrophic, but loses # the ability to distinguish between multiple cases that would all fill in empty pixels. # So we cap the ratio to a large number. with utils.nowarn(): ratio = bhits/split_hits ratio[np.isnan(ratio)] = 0 ratio = np.minimum(ratio, 1000) return np.sum(ratio[:,mask],-1)
def __init__(self, data, srcpos, pcut, nmat, perdet=False): pthumb = PmatThumbs(data, srcpos, perdet=perdet) twork = np.full(data.tod.shape, 1.0, data.tod.dtype) nmat.white(twork) div = enmap.zeros(pthumb.shape, pthumb.wcs, data.tod.dtype) junk = np.zeros(pcut.njunk,data.tod.dtype) pcut.backward(twork, junk) pthumb.backward(twork, div) div = div[:,0] self.pthumb, self.pcut, self.nmat = pthumb, pcut, nmat self.div = div with utils.nowarn(): self.idiv = 1/self.div self.idiv[~np.isfinite(self.idiv)] = 0
def __init__(self, data, srcpos, pcut, nmat, perdet=False): pthumb = PmatThumbs(data, srcpos, perdet=perdet) twork = np.full(data.tod.shape, 1.0, data.tod.dtype) nmat.white(twork) div = enmap.zeros(pthumb.shape, pthumb.wcs, data.tod.dtype) junk = np.zeros(pcut.njunk,data.tod.dtype) pcut.backward(twork, junk) pthumb.backward(twork, div) div = div[:,0] self.pthumb, self.pcut, self.nmat = pthumb, pcut, nmat self.div = div with utils.nowarn(): self.idiv = 1/self.div self.idiv[~np.isfinite(self.idiv)] = 0
def find_ref_pixs(divs, rcost=1.0, dcost=1.0):
"""rcost is cost per pixel away from center
dcost is cost per dB change in div value avay from median"""
# Find median nonzero div per map
ref_val = np.asarray(np.median(np.ma.array(divs, mask=divs==0),(-2,-1)))
with utils.nowarn():
val_off = 10*(np.log10(divs)-np.log10(ref_val[:,None,None]))
pix_map = divs.pixmap()
center = np.array(divs.shape[-2:])/2
dist_map= np.sum((pix_map-center[:,None,None])**2,-3)**0.5
cost = dist_map * rcost + np.abs(val_off)*dcost
# Sort each by cost. Will be [{y,x},map,order]
inds = np.array(np.unravel_index(np.argsort(cost.reshape(cost.shape[0],-1),1), cost.shape[-2:]))
inds = inds.T
# Want to return [order,map,{y,x}]
return inds
def find_ref_pixs(divs, rcost=1.0, dcost=1.0):
"""rcost is cost per pixel away from center
dcost is cost per dB change in div value avay from median"""
# Find median nonzero div per map
ref_val = np.asarray(np.median(np.ma.array(divs, mask=divs==0),(-2,-1)))
with utils.nowarn():
val_off = 10*(np.log10(divs)-np.log10(ref_val[:,None,None]))
pix_map = divs.pixmap()
center = np.array(divs.shape[-2:])/2
dist_map= np.sum((pix_map-center[:,None,None])**2,-3)**0.5
cost = dist_map * rcost + np.abs(val_off)*dcost
# Sort each by cost. Will be [{y,x},map,order]
inds = np.array(np.unravel_index(np.argsort(cost.reshape(cost.shape[0],-1),1), cost.shape[-2:]))
inds = inds.T
# Want to return [order,map,{y,x}]
return inds
def __init__(self, scan, model=None, window=None, filter=None):
model = config.get("noise_model", model)
window = config.get("tod_window", window)*scan.srate
nmat.apply_window(scan.tod, window)
self.nmat = nmat_measure.NmatBuildDelayed(model, cut=scan.cut_noiseest, spikes=scan.spikes)
self.nmat = self.nmat.update(scan.tod, scan.srate)
nmat.apply_window(scan.tod, window, inverse=True)
self.model, self.window = model, window
self.ivar = self.nmat.ivar
self.cut = scan.cut
# Optional extra filter
if filter:
freq = fft.rfftfreq(scan.nsamp, 1/scan.srate)
fknee, alpha = filter
with utils.nowarn():
self.filter = (1 + (freq/fknee)**-alpha)**-1
else: self.filter = None
def __init__(self, workspaces, template, comm=None):
"""Initialize a FastmapSolver for the equation system given by the workspace list
workspaces. The template argument specifies the output coordinate system. This
enmap have a wcs which is pixel-compatible with that used to build the workspaces."""
if comm is None: comm = mpi.COMM_WORLD
# Find the global coordinate offset needed to match our
# global wcs with the template wcs
corner = template.pix2sky([0, 0])
gwcs, offset = offset_wcs(workspaces[0].geometry.gwcs, corner)
# Prepare workspaces for solving in these coordinates
self.workspaces = []
for work in workspaces:
work = work.copy()
with utils.nowarn():
hdiv_norm = work.hdiv / np.sum(work.hdiv[0, 0],
-1)[None, None, :, None]
hdiv_norm[~np.isfinite(hdiv_norm)] = 0
work.hdiv_norm_sqrt = hdiv_norm[
0, 0]**0.5 # array_ops.eigpow(hdiv_norm, 0.5, [0,1])
# Update the global wcs and pixel coordinates
work.geometry.gwcs = gwcs
work.geometry.y0 -= offset[0]
work.geometry.xshifts -= offset[1]
# Set up our ponting matrix
work.pmat = PmatWorkspaceMap(work.geometry)
self.workspaces.append(work)
# Update our template to match the geometry we're actually using.
# If the original template was compatible, this will be a NOP geometry-wise
template = enmap.zeros((work.geometry.ncomp, ) + template.shape[-2:],
work.geometry.gwcs, work.geometry.dtype)
# Build a simple binned preconditioner
# FIXME: This just makes things worse
#idiv = enmap.zeros((work.geometry.ncomp,work.geometry.ncomp)+template.shape[-2:], work.geometry.gwcs, work.geometry.dtype)
#for work in self.workspaces:
# wmap = enmap.zeros(work.geometry.shape, work.geometry.lwcs, work.geometry.dtype)
# for i in range(work.geometry.ncomp):
# tmp = idiv[0]*0
# tmp[i] = 1
# work.pmat.forward(wmap, tmp)
# wmap[:] = array_ops.matmul(work.hdiv, wmap, [0,1])
# wmap[:] = array_ops.matmul(np.rollaxis(work.hdiv,1), wmap, [0,1])
# work.pmat.backward(wmap, idiv[i])
#self.prec = array_ops.eigpow(idiv, -1, axes=[0,1])
self.dof = zipper.ArrayZipper(template)
self.comm = comm
def write_package(fname, maps, divs, src_ids, d):
header = map_to_header(maps)
header["id"] = d.entry.id + ":" + entry.tag
header["off_x"] = d.point_correction[0] / utils.arcmin
header["off_y"] = d.point_correction[1] / utils.arcmin
header["t1"] = d.boresight[0, 0]
header["t2"] = d.boresight[0, -1]
meanoff = np.mean(d.point_offset, 0) / utils.degree
header["az1"] = np.min(d.boresight[1, :]) / utils.degree + meanoff[0]
header["az2"] = np.max(d.boresight[1, :]) / utils.degree + meanoff[1]
header["el"] = np.mean(d.boresight[2, ::100]) / utils.degree
hdu_maps = astropy.io.fits.PrimaryHDU(maps, header)
hdu_divs = astropy.io.fits.ImageHDU(divs, map_to_header(divs), name="div"),
hdu_ids = astropy.io.fits.TableHDU(src_ids, name="ids")
hdus = astropy.io.fits.HDUList([hdu_maps, hdu_divs, hdu_ids])
with utils.nowarn():
hdus.writeto(fname, clobber=True)
def __init__(self, workspaces, template, comm=None): """Initialize a FastmapSolver for the equation system given by the workspace list workspaces. The template argument specifies the output coordinate system. This enmap have a wcs which is pixel-compatible with that used to build the workspaces.""" if comm is None: comm = mpi.COMM_WORLD # Find the global coordinate offset needed to match our # global wcs with the template wcs corner = template.pix2sky([0,0]) gwcs, offset = offset_wcs(workspaces[0].geometry.gwcs, corner) # Prepare workspaces for solving in these coordinates self.workspaces = [] for work in workspaces: work = work.copy() with utils.nowarn(): hdiv_norm = work.hdiv / np.sum(work.hdiv[0,0],-1)[None,None,:,None] hdiv_norm[~np.isfinite(hdiv_norm)] = 0 work.hdiv_norm_sqrt = hdiv_norm[0,0]**0.5 # array_ops.eigpow(hdiv_norm, 0.5, [0,1]) # Update the global wcs and pixel coordinates work.geometry.gwcs = gwcs work.geometry.y0 -= offset[0] work.geometry.xshifts -= offset[1] # Set up our ponting matrix work.pmat = PmatWorkspaceMap(work.geometry) self.workspaces.append(work) # Update our template to match the geometry we're actually using. # If the original template was compatible, this will be a NOP geometry-wise template = enmap.zeros((work.geometry.ncomp,)+template.shape[-2:], work.geometry.gwcs, work.geometry.dtype) # Build a simple binned preconditioner # FIXME: This just makes things worse #idiv = enmap.zeros((work.geometry.ncomp,work.geometry.ncomp)+template.shape[-2:], work.geometry.gwcs, work.geometry.dtype) #for work in self.workspaces: # wmap = enmap.zeros(work.geometry.shape, work.geometry.lwcs, work.geometry.dtype) # for i in range(work.geometry.ncomp): # tmp = idiv[0]*0 # tmp[i] = 1 # work.pmat.forward(wmap, tmp) # wmap[:] = array_ops.matmul(work.hdiv, wmap, [0,1]) # wmap[:] = array_ops.matmul(np.rollaxis(work.hdiv,1), wmap, [0,1]) # work.pmat.backward(wmap, idiv[i]) #self.prec = array_ops.eigpow(idiv, -1, axes=[0,1]) self.dof = zipper.ArrayZipper(template) self.comm = comm
# in 1378883069.1378883122.ar1:
# 1.4179 0.0209 -3.8517 0.0303 9.7507 26.7
# 1.435 -3.864 13.217 38.4
# So that's a 44% higher S/N, corresponding to 2x more data. Not completely comparable, though,
# since the TOD-one didn't marginalize over position. The position also differs by almost a sigma,
# which it shouldn't considering that they share data. And I trust the tod-level one more.
# However, this takes 1-5 s per likelihood evaluation. A robust fit requires ~500 evaluations,
# which would be 8-42 minutes. And that's using 16 cores! That's too slow. So this one is
# useful for comparing with a faster methods for a few reference tods, but not in general.
# Currently N and P take similar time. Can optimize P more with some effort, but N is dominated
# by ffts, and can't improve much.
from __future__ import division, print_function
import numpy as np, time, astropy.io.fits, os, sys
from scipy import optimize, integrate
from enlib import utils
with utils.nowarn(): import h5py
from enlib import mpi, errors, fft, mapmaking, config, jointmap, pointsrcs
from enlib import pmat, coordinates, enmap, bench, bunch, nmat, sampcut, gapfill, wcsutils, array_ops
from enact import filedb, actdata, actscan, nmat_measure
config.set("downsample", 1, "Amount to downsample tod by")
config.set("gapfill", "linear", "Gapfiller to use. Can be 'linear' or 'joneig'")
config.default("pmat_interpol_pad", 10.0, "Number of arcminutes to pad the interpolation coordinate system by")
config.default("pmat_interpol_max_size", 4000000, "Maximum mesh size in pointing interpolation. Worst-case time and memory scale at most proportionally with this.")
parser = config.ArgumentParser(os.environ["HOME"] + "./enkirc")
parser.add_argument("mode", help="Mode to use. Can be srcs or planet. This sets up useful defaults for other arguments")
parser.add_argument("srcdb_or_planet")
parser.add_argument("sel")
parser.add_argument("odir")
parser.add_argument("-R", "--radius", type=float, default=12)
div = div.reshape((-1, ) + div.shape[-2:])[0]
# Individual pixel area
if args.area_model == "average":
pix_area = div * 0 + div.area() / div.size * (180 * 60 / np.pi)**2
else:
pos = div.posmap()
diffs = utils.rewind(pos[:, 1:, 1:] - pos[:, :-1, :-1], 0)
pix_area = np.abs(diffs[0] * diffs[1]) * np.cos(pos[0, :-1, :-1])
del diffs
# Go to square arcmins
pix_area /= utils.arcmin**2
# Pad to recover edge pixels
pix_area = np.concatenate([pix_area, pix_area[-1:]], 0)
pix_area = np.concatenate([pix_area, pix_area[:, -1:]], 1)
# Flatten everything
div = div.reshape(-1)
pix_area = pix_area.reshape(-1)
with utils.nowarn():
rms = (pix_area / div)**0.5 if not args.already_arcmin else div**-0.5
inds = np.argsort(rms, axis=None)
rms = rms[inds]
area = np.cumsum(pix_area[inds]) / 3600
mask = np.isfinite(rms)
rms, area = rms[mask], area[mask]
np.savetxt(args.ofile,
np.array([area[::args.thin], rms[::args.thin]]).T,
fmt="%9.3f %15.4f")
def butter(f, f0, alpha):
if f0 <= 0: return f * 0 + 1
with utils.nowarn():
return 1 / (1 + (np.abs(f) / f0)**alpha)
def analyze(self,
ref_beam=None,
mode="weight",
map_max=1e8,
div_tol=20,
apod_val=0.2,
apod_alpha=5,
apod_edge=120,
beam_tol=1e-4,
ps_spec_tol=0.5,
ps_smoothing=10,
filter_kxrad=20,
filter_highpass=200,
filter_kx_ymax_scale=1):
# Find the typical noise levels. We will use this to decide where
# divs and beams etc. can be truncated to improve convergence.
datasets = self.datasets
ncomp = max([
split.data.map.preflat.shape[0] for dataset in datasets
for split in dataset.splits
])
for dataset in datasets:
for split in dataset.splits:
split.ref_div = robust_ref(split.data.div)
# Avoid single, crazy pixels
split.data.div = np.minimum(split.data.div,
split.ref_div * div_tol)
split.data.div = filter_div(split.data.div)
split.data.map = np.maximum(
-map_max, np.minimum(map_max, split.data.map))
# Expand map to ncomp components
split.data.map = add_missing_comps(split.data.map, ncomp)
# Build apodization
apod = np.minimum(split.data.div / (split.ref_div * apod_val),
1.0)**apod_alpha
apod = apod.apod(apod_edge)
split.data.div *= apod
split.data.H = split.data.div**0.5
dataset.ref_div = np.sum(
[split.ref_div for split in dataset.splits])
tot_ref_div = np.sum([dataset.ref_div for dataset in datasets])
ly, lx = enmap.laxes(self.shape, self.wcs)
lr = (ly[:, None]**2 + lx[None, :]**2)**0.5
bmin = np.min([beam_size(dataset.beam) for dataset in datasets])
# Deconvolve all the relative beams. These should ideally include pixel windows.
# This could matter for planck
if ref_beam is not None:
for dataset in datasets:
rel_beam = beam_ratio(dataset.beam, ref_beam)
# Avoid division by zero
bspec = np.maximum(eval_beam(rel_beam, lr), 1e-10)
# We don't want to divide by tiny numbers, so we will cap the relative
# beam. The goal is just to make sure that the deconvolved noise ends up
# sufficiently high that anything beyond that is negligible. This will depend
# on the div ratios between the different datasets. We can stop deconvolving
# when beam*my_div << (tot_div-my_div). But deconvolving even by a factor
# 1000 leads to strange numberical errors
bspec = np.maximum(
bspec, beam_tol * (tot_ref_div / dataset.ref_div - 1))
bspec_dec = np.maximum(bspec, 0.1)
for split in dataset.splits:
split.data.map = map_ifft(
map_fft(split.data.map) / bspec_dec)
# In theory we don't need to worry about the beam any more by this point.
# But the pixel window might be unknown or missing. So we save the beam so
# we can make sure the noise model makes sense
dataset.bspec = bspec
# We classify this as a low-resolution dataset if we did an appreciable amount of
# deconvolution
dataset.lowres = np.min(bspec) < 0.5
# Can now build the noise model and rhs for each dataset.
# The noise model is N = HCH, where H = div**0.5 and C is the mean 2d noise spectrum
# of the whitened map, after some smoothing.
for dataset in datasets:
nsplit = 0
dset_map, dset_div = None, None
for split in dataset.splits:
if dset_map is None:
dset_map = split.data.map * 0
dset_div = split.data.div * 0
dset_map += split.data.map * split.data.div
dset_div += split.data.div
# Form the mean map for this dataset
dset_map[:, dset_div > 0] /= dset_div[dset_div > 0]
# Then use it to build the diff maps and noise spectra
dset_ps = None
#i=0
for split in dataset.splits:
if split.data.empty: continue
diff = split.data.map - dset_map
wdiff = diff * split.data.H
#i+=1
# What is the healthy area of wdiff? Wdiff should have variance
# 1 or above. This tells us how to upweight the power spectrum
# to take into account missing regions of the diff map.
ndown = 10
wvar = enmap.downgrade(wdiff**2, ndown)
goodfrac = np.sum(wvar > 1e-3) / float(wvar.size)
if goodfrac < 0.1: goodfrac = 0
ps = np.abs(map_fft(wdiff))**2
# correct for unhit areas, which can't be whitened
with utils.nowarn():
ps /= goodfrac
if dset_ps is None:
dset_ps = enmap.zeros(ps.shape, ps.wcs, ps.dtype)
dset_ps += ps
nsplit += 1
if nsplit < 2: continue
# With n splits, mean map has var 1/n, so diff has var (1-1/n) + (n-1)/n = 2*(n-1)/n
# Hence tot-ps has var 2*(n-1)
dset_ps /= 2 * (nsplit - 1)
dset_ps = smooth_pix(dset_ps, ps_smoothing)
# Use the beam we saved from earlier to make sure we don't have a remaining
# pixel window giving our high-l parts too high weight. If everything has
# been correctly deconvolved, we expect high-l dset_ps to go as
# 1/beam**2. The lower ls will realistically be no lower than this either.
# So we can simply take the max
dset_ps_ref = np.min(
np.maximum(dset_ps, dataset.bspec**-2 * ps_spec_tol * 0.1))
dset_ps = np.maximum(dset_ps,
dset_ps_ref * dataset.bspec**-2 * ps_spec_tol)
# Our fourier-space inverse noise matrix is the inverse of this
if np.all(np.isfinite(dset_ps)):
iN = 1 / dset_ps
else:
iN = enmap.zeros(dset_ps.shape, dset_ps.wcs, dset_ps.dtype)
# Add any fourier-space masks to this
if dataset.highpass:
kxmask = butter(lx, filter_kxrad, -5)
kxmask = 1 - (1 - kxmask[None, :]) * (
np.abs(ly) < bmin * filter_kx_ymax_scale)[:, None]
highpass = butter(lr, filter_highpass, -10)
filter = highpass * kxmask
del kxmask, highpass
else:
filter = 1
if mode != "filter": iN *= filter
dataset.iN = iN
dataset.filter = filter
self.mode = mode
def query(self, query=None, apply_default_query=True):
"""Query the database. The query takes the form
tag,tag,tag,...:sort[slice], where all tags must be satisfied for an id to
be returned. More general syntax is also available. For example,
(a+b>c)|foo&bar,cow. This follows standard python and numpy syntax,
except that , is treated as a lower-priority version of &."""
# Make a copy of self.data so we can't modify it without changing ourself
data = self.data.copy()
# First split off any sorting field or slice
if query is None: query = ""
toks = utils.split_outside(query, ":")
query, rest = toks[0], ":".join(toks[1:])
# Hack: Support id fields as tags, even if they contain
# illegal characters..
t1 = time.time()
for id in data["id"]:
if id not in query: continue
query = re.sub(r"""(?<!['"])\b%s\b""" % id, "(id=='%s')" % id,
query)
# Split into ,-separated fields. Fields starting with a "+"
# are taken to be tag markers, and are simply propagated to the
# resulting ids.
toks = utils.split_outside(query, ",")
fields, subid = [], []
override_ids = None
for tok in toks:
if len(tok) == 0: continue
if tok.startswith("+"):
# Tags starting with + will be interpreted as a subid specification
subid.append(tok[1:])
elif tok.startswith("/"):
# Tags starting with / will be interpreted as special query flags
if tok == "/all": apply_default_query = False
else: raise ValueError("Unknown query flag '%s'" % tok)
else:
# Normal field. Perform a few convenience transformations first.
if tok.startswith("@@"):
# Hack. *Force* the given ids to be returned, even if they aren't in the database.
override_ids = load_ids(tok[2:])
continue
elif tok.startswith("@"):
# Restrict dataset to those in the given file
tok = "file_contains('%s',id)" % tok[1:]
elif tok.startswith("~@"):
tok = "~file_contains('%s',id)" % tok[2:]
fields.append(tok)
if override_ids is not None:
# Append subids to our ids, and return immediately. All other fields
# and queries are ignored.
subs = np.array(",".join(subid))
subs = np.full(len(override_ids), subs, subs.dtype)
return append_subs(override_ids, subs)
# Apply our default queries here. These are things that we almost always
# want in our queries, and that it's tedious to have to specify manually
# each time. For example, this would be "selected" for act todinfo queries
if apply_default_query:
fields = fields + utils.split_outside(self.default_query, ",")
subid = ",".join(subid)
# Now evaluate our fields one by one. This is done so that
# function fields can inspect the current state at that point
for field in fields:
scope = np.__dict__.copy()
scope.update(data)
for name, functor in self.functors.iteritems():
scope[name] = functor(data)
with utils.nowarn():
hits = eval(field, scope)
# Restrict all fields to the result
data = dslice(data, hits)
# Split the rest into a sorting field and a slice
toks = rest.split("[")
if len(toks) == 1: sort, fsel, dsel = toks[0], "", ""
elif len(toks) == 2: sort, fsel, dsel = toks[0], "", "[" + toks[1]
else:
sort, fsel, dsel = toks[0], "[" + toks[1], "[" + "[".join(toks[2:])
if self.sort and not sort: sort = self.sort
if sort:
# Evaluate sorting field
field = data[sort]
field = eval("field" + fsel)
data = dslice(data, np.argsort(field))
# Finally apply the data slice
inds = np.arange(len(data["id"]))
inds = eval("inds" + dsel)
data = dslice(data, inds)
# Build our subid extensions and append them to ids
subs = np.array([merge_subid(subid, sub) for sub in data["subids"]])
ids = append_subs(data["id"], subs)
return ids
def coadd_tile_data(datasets,
box,
odir,
ps_smoothing=10,
pad=0,
ref_beam=None,
cg_tol=1e-6,
dump=False,
verbose=False,
read_cache=False,
write_cache=False,
div_max_tol=100,
div_div_tol=1e-10):
# Load data for this box for each dataset
datasets, ffpad = read_data(datasets,
box,
odir,
pad=pad,
verbose=verbose,
read_cache=read_cache,
write_cache=write_cache)
# We might not find any data
if len(datasets) == 0: return None
# Find the smallest beam size of the datasets
bmin = np.min([beam_size(dataset.beam) for dataset in datasets])
# Subtract mean map from each split to get noise maps. Our noise
# model is HNH, where H is div**0.5 and N is the mean 2d noise spectrum
# after some smoothing
rhs, tot_div = None, None
tot_iN, tot_udiv = None, 0
for dataset in datasets:
nsplit = 0
dset_map, dset_div = None, None
for split in dataset.splits:
if dset_map is None:
dset_map = split.data.map * 0
dset_div = split.data.div * 0
dset_map += split.data.map * split.data.div
dset_div += split.data.div
# Form the mean map for this dataset
dset_map[:, dset_div > 0] /= dset_div[dset_div > 0]
if tot_div is None: tot_div = dset_div * 0
tot_div += dset_div
tshape, twcs, tdtype = dset_map.shape, dset_div.wcs, dset_div.dtype
# Then use it to build the diff maps and noise spectra
dset_ps = None
for split in dataset.splits:
if split.data.empty: continue
diff = split.data.map - dset_map
wdiff = diff * split.data.H
# What is the healthy area of wdiff? Wdiff should have variance
# 1 or above. This tells us how to upweight the power spectrum
# to take into account missing regions of the diff map.
ndown = 10
wvar = enmap.downgrade(wdiff**2, ndown)
goodfrac = np.sum(wvar > 1e-3) / float(wvar.size)
if goodfrac < 0.1: goodfrac = 0
#opre = odir + "/" + os.path.basename(split.map)[:-5]
#enmap.write_map(opre + "_diff.fits", diff)
#enmap.write_map(opre + "_wdiff.fits", wdiff)
#enmap.write_map(opre + "_wvar.fits", wvar)
ps = np.abs(map_fft(wdiff))**2
#enmap.write_map(opre + "_ps1.fits", ps)
# correct for unhit areas, which can't be whitened
#print "A", dataset.name, np.median(ps[ps>0]), medloop(ps), goodfrac
with utils.nowarn():
ps /= goodfrac
#print "B", dataset.name, np.median(ps[ps>0]), medloop(ps), goodfrac
#enmap.write_map(opre + "_ps2.fits", ps)
#enmap.write_map(opre + "_ps2d.fits", ps)
if dset_ps is None:
dset_ps = enmap.zeros(ps.shape, ps.wcs, ps.dtype)
dset_ps += ps
nsplit += 1
if nsplit < 2: continue
# With n splits, mean map has var 1/n, so diff has var (1-1/n) + (n-1)/n = 2*(n-1)/n
# Hence tot-ps has var 2*(n-1)
dset_ps /= 2 * (nsplit - 1)
#enmap.write_map(opre + "_ps2d_tot.fits", dset_ps)
dset_ps = smooth_pix(dset_ps, ps_smoothing)
#enmap.write_map(opre + "_ps2d_smooth.fits", dset_ps)
if np.all(np.isfinite(dset_ps)):
# Super-low values of the spectrum are not realistic. These appear
# due to beam/pixel smoothing in the planck maps. This will be
# mostly taken care of when processing the beams, as long as we don't
# let them get too small
dset_ps = np.maximum(dset_ps, 1e-7)
# Optionally cap the max dset_ps, this is mostly to speed up convergence
if args.max_ps:
dset_ps = np.minimum(dset_ps, args.max_ps)
# Our fourier-space inverse noise matrix is based on the inverse noise spectrum
iN = 1 / dset_ps
#enmap.write_map(opre + "_iN_raw.fits", iN)
else:
print "Setting weight of dataset %s to zero" % dataset.name
#print np.all(np.isfinite(dset_ps)), np.all(dset_ps>0)
iN = enmap.zeros(dset_ps.shape, dset_ps.wcs, dset_ps.dtype)
# Add any fourier-space masks to this
ly, lx = enmap.laxes(tshape, twcs)
lr = (ly[:, None]**2 + lx[None, :]**2)**0.5
if dataset.highpass:
kxmask = butter(lx, args.kxrad, -3)
kxmask = 1 - (1 - kxmask[None, :]) * (
np.abs(ly) < bmin * args.kx_ymax_scale)[:, None]
highpass = butter(lr, args.highpass, -10)
filter = highpass * kxmask
#print "filter weighting", dataset.name
del kxmask, highpass
else:
filter = 1
if not args.filter: iN *= filter
# We should deconvolve the relative beam from the maps,
# but that's numerically nasty. But it can be handled
# inversely. We want (BiNB + ...)x = (BiNB iB m + ...)
# where iB is the beam deconvolution operation in map space.
# Instead of actually doing that operation, we can compute two
# inverse noise matrixes: iN_A = BiNB for the left hand
# side and iN_b = BiN for the right hand side. That way we
# avoid dividing by any huge numbers.
# Add the relative beam
iN_A = iN.copy()
iN_b = iN.copy()
if ref_beam is not None:
rel_beam = beam_ratio(dataset.beam, ref_beam)
bspec = eval_beam(rel_beam, lr)
iN_A *= bspec**2
iN_b *= bspec
#moo = iN*0+filter
#enmap.write_map(opre + "_filter.fits", moo)
# Add filter to noise model if we're downweighting
# rather than filtering.
dataset.iN_A = iN_A
dataset.iN_b = iN_b
dataset.filter = filter
#print "A", opre
#enmap.write_map(opre + "_iN_A.fits", iN_A)
#enmap.write_map(opre + "_iN.fits", iN)
# Cap to avoid single crazy pixels
tot_div = np.maximum(tot_div, np.median(tot_div[tot_div > 0]) * 0.01)
tot_idiv = tot_div * 0
tot_idiv[tot_div > div_div_tol] = 1 / tot_div[tot_div > div_div_tol]
# Build the right-hand side. The right-hand side is
# sum(HNHm)
if rhs is None: rhs = enmap.zeros(tshape, twcs, tdtype)
for dataset in datasets:
i = 0
for split in dataset.splits:
if split.data.empty: continue
#print "MOO", dataset.name, np.max(split.data.map), np.min(split.data.map), np.max(split.data.div), np.min(split.data.div)
w = split.data.H * split.data.map
fw = map_fft(w)
fw *= dataset.iN_b
if args.filter: fw *= dataset.filter
w = map_ifft(fw) * split.data.H
#enmap.write_map(odir + "/%s_%02d_rhs.fits" % (dataset.name, i), w)
rhs += w
i += 1
del w, iN, iN_A, iN_b, filter
# Now solve the equation
def A(x):
global times
m = enmap.samewcs(x.reshape(rhs.shape), rhs)
res = m * 0
times[:] = 0
ntime = 0
for dataset in datasets:
for split in dataset.splits:
if split.data.empty: continue
t = [time.time()]
w = split.data.H * m
t.append(time.time())
fw = map_fft(w)
t.append(time.time())
fw *= dataset.iN_A
t.append(time.time())
w = map_ifft(fw)
t.append(time.time())
w *= split.data.H
t.append(time.time())
res += w
for i in range(1, len(t)):
times[i - 1] += t[i] - t[i - 1]
ntime += 1
#w = enmap.harm2map(dataset.iN_A*enmap.map2harm(w))
#w *= split.data.H
#res += w
del w
times /= ntime
return res.reshape(-1)
def M(x):
m = enmap.samewcs(x.reshape(rhs.shape), rhs)
res = m * tot_idiv
return res.reshape(-1)
solver = cg.CG(A, rhs.reshape(-1), M=M)
for i in range(1000):
t1 = time.time()
solver.step()
t2 = time.time()
if verbose:
print "%5d %15.7e %5.2f: %4.2f %4.2f %4.2f %4.2f %4.2f" % (
solver.i, solver.err, t2 - t1, times[0], times[1], times[2],
times[3], times[4]), np.std(solver.x)
if dump and solver.i in [1, 2, 5, 10, 20, 50] + range(100, 10000, 100):
m = enmap.samewcs(solver.x.reshape(rhs.shape), rhs)
enmap.write_map(odir + "/step%04d.fits" % solver.i, m)
if solver.err < cg_tol:
if dump:
m = enmap.samewcs(solver.x.reshape(rhs.shape), rhs)
enmap.write_map(odir + "/step_final.fits", m)
break
tot_map = enmap.samewcs(solver.x.reshape(rhs.shape), rhs)
# Get rid of the fourier padding
ny, nx = tot_map.shape[-2:]
tot_map = tot_map[..., :ny - ffpad[0], :nx - ffpad[1]]
tot_div = tot_div[..., :ny - ffpad[0], :nx - ffpad[1]]
return bunch.Bunch(map=tot_map, div=tot_div)
import numpy as np, sys, os, ephem
from enlib import utils
with utils.nowarn(): import h5py
from enlib import config, pmat, mpi, errors, gapfill, enmap, bench, ephemeris
from enlib import fft, array_ops, sampcut, cg
from enact import filedb, actscan, actdata, cuts, nmat_measure
config.set("pmat_cut_type", "full")
parser = config.ArgumentParser()
parser.add_argument("sel")
parser.add_argument("srcs")
parser.add_argument("area")
parser.add_argument("odir")
parser.add_argument("tag", nargs="?")
parser.add_argument("-R", "--dist", type=float, default=4)
parser.add_argument("-y", "--ypad", type=float, default=3)
parser.add_argument("-s", "--src", type=int, default=None, help="Only analyze given source")
parser.add_argument("-c", "--cont", action="store_true")
parser.add_argument("-m", "--model",type=str, default="constrained")
parser.add_argument("--hit-tol", type=float, default=0.5)
args = parser.parse_args()
comm = mpi.COMM_WORLD
filedb.init()
R = args.dist * utils.arcmin
ypad = args.ypad * utils.arcmin
csize= 100
config.set("pmat_ptsrc_cell_res", 2*(R+ypad)/utils.arcmin)
config.set("pmat_ptsrc_rsigma", 5)
config.set("pmat_interpol_pad", 5+ypad/utils.arcmin)
dtype = np.float32
print fbin f1,f2 = [min(nfreq-1,int(i*fmax/dfreq/nbin)) for i in [fbin,fbin+1]] fsub = ft[:,f1:f2] cov = array_ops.measure_cov(fsub) std = np.diag(cov)**0.5 corr = cov / std[:,None] / std[None,:] myrhs = project_mat(pix, template, corr) mydiv = project_mat(pix, template) return fbin, myrhs, mydiv def collect(args): fbin, myrhs, mydiv = args rhs[fbin] += myrhs div[fbin] += mydiv p = multiprocessing.Pool(args.nmulti) for fbin in range(nbin): p.apply_async(handle_bin, [fbin], callback=collect) p.close() p.join() del ft # Collect the results if comm.rank == 0: print "Reducing" rhs = enmap.samewcs(utils.allreduce(rhs, comm), rhs) div = enmap.samewcs(utils.allreduce(div, comm), div) with utils.nowarn(): map = rhs/div if comm.rank == 0: print "Writing" enmap.write_map(args.ofile, map)
def read_catalog(fname):
with utils.nowarn():
cat = np.loadtxt(fname, usecols=range(8), ndmin=2).reshape(-1, 8)
cat[:, :2] = cat[:, 1::-1] * utils.degree
cat[:, -2:] *= ym
return cat
