def gapfill_helper(tod, cut):
method, context = config.get("gapfill"), config.get("gapfill_context")
gapfiller = {
"linear":gapfill.gapfill_linear,
"joneig":gapfill.gapfill_joneig,
}[method]
gapfiller(tod, cut, inplace=True, overlap=context)
def build_interpol(transform, box, id="none", posunit=1.0, sys=None):
sys = config.get("map_sys", sys)
# We widen the bounding box slightly to avoid samples falling outside it
# due to rounding errors.
box = utils.widen_box(np.array(box), 1e-3)
box[:,
1:] = utils.widen_box(box[:, 1:],
config.get("pmat_interpol_pad") * utils.arcmin,
relative=False)
acc = config.get("pmat_accuracy")
ip_size = config.get("pmat_interpol_max_size")
ip_time = config.get("pmat_interpol_max_time")
# Build pointing interpolator
errlim = np.array(
[1e-3 * posunit, 1e-3 * posunit, utils.arcmin, utils.arcmin]) * acc
ipol, obox, ok, err = interpol.build(transform,
interpol.ip_linear,
box,
errlim,
maxsize=ip_size,
maxtime=ip_time,
return_obox=True,
return_status=True)
if not ok and np.any(err > errlim):
print "Warning: Accuracy %g was specified, but only reached %g for tod %s" % (
acc, np.max(err / errlim) * acc, id)
return ipol, obox, err
def detvecs_oof(fourier, srate, dets=None, type=None, nbin=None, nmin=None):
nfreq = fourier.shape[1]
ndet = fourier.shape[0]
type = config.get("nmat_uncorr_type", type)
nbin = config.get("nmat_uncorr_nbin", nbin)
nmin = config.get("nmat_uncorr_nmin", nmin)
fknee = config.get("nmat_oof_fknee")
alpha = config.get("nmat_oof_alpha")
if type is "exp":
bins = utils.expbin(nfreq, nbin=nbin, nmin=nmin)
elif type is "lin":
bins = utils.linbin(nfreq, nbin=nbin, nmin=nmin)
else:
raise ValueError("No such power binning type '%s'" % type)
nbin = bins.shape[0] # expbin may not provide exactly what we want
# Dummy detector correlation stuff, so we can reuse the sharedvecs stuff
vecs = np.full([ndet, 0], 1)
Nu = np.zeros([nbin, ndet])
E = np.full([nbin, 0], 1e-10)
V = [vecs]
vinds = np.zeros(nbin, dtype=int)
for bi, b in enumerate(bins):
d = fourier[:, b[0]:b[1]]
Nu[bi] = measure_power(d)
# Replace Nu with functional form
f = np.mean(bins, 1) * srate / 2 / nfreq
Nu = np.mean(Nu**-1, 0)**-1 * (1 + (f / fknee)**alpha)[:, None]
return prepare_sharedvecs(Nu, V, E, bins, srate, dets, vinds)
def detvecs_scaled(ft,
srate,
dets=None,
bsize=None,
lim=None,
lim2=None,
spikecut=None):
bsize = config.get("nmat_scaled_bsize", bsize)
lim = config.get("nmat_scaled_spike", lim)
lim2 = config.get("nmat_scaled_smooth", lim2)
spikecut = config.get("nmat_scaled_spikecut", spikecut)
# First set up our sample points
mps = calc_mean_ps(ft)
bins, bins_spike = build_spec_bins(mps, bsize=bsize, lim=lim, lim2=lim2)
# Measure mean power per bin
brms = np.zeros(ft.shape[:-1] + (len(bins), ), mps.dtype)
for bi, b in enumerate(bins):
brms[:, bi] = np.mean(np.abs(ft[:, b[0]:b[1]])**2)**0.5
ft[:, b[0]:b[1]] /= brms[:, bi, None]
# Build interior noise model
if not spikecut: sbins_spike = None
nmat_inner = detvecs_jon(ft,
srate=srate,
dets=dets,
cut_bins=bins_spike,
cut_unit="inds")
# Undo scaling of ft in case caller still needs it
for bi, b in enumerate(bins):
ft[:, b[0]:b[1]] *= brms[:, bi, None]
return nmat.NmatScaled(brms, bins, nmat_inner)
def gapfill(arr, ranges, inplace=False, overlap=None):
gapfiller = {
"linear": gapfill_linear,
"joneig": gapfill_joneig,
}[config.get("gapfill")]
overlap = config.get("gapfill_context", overlap)
return gapfiller(arr, ranges, inplace=inplace, overlap=overlap)
def __init__(self,
scan,
template,
beam_offs,
beam_comps,
sys=None,
order=None):
# beam_offs has format [nbeam,ndet,{dt,dra,ddec,}], which allows
# each detector to have a separate beam. The dt part is pretty useless.
# beam_comps has format [nbeam,ndet,{T,Q,U}].
# Get the full box after taking all beam offsets into account
ibox = np.array([
np.min(scan.boresight, 0) + np.min(beam_offs, (0, 1)),
np.max(scan.boresight, 0) + np.max(beam_offs, (0, 1))
])
# Build our pointing interpolator
sys = config.get("map_sys", sys)
transform = pos2pix(scan, template, sys)
ipol, obox, err = build_interpol(transform, ibox, id=scan.entry.id)
self.rbox, self.nbox, self.yvals = extract_interpol_params(
ipol, template.dtype)
# And store our data
self.beam_offs, self.beam_comps = beam_offs, beam_comps
self.pixbox, self.nphi = build_pixbox(obox[:, :2], template)
self.scan, self.dtype = scan, template.dtype
self.order = config.get("pmat_map_order", order)
self.err = err
def get_samples(self, verbose=False):
"""Return the actual detector samples. Slow! Data is read from disk and
calibrated on the fly, so store the result if you need to reuse it."""
# Because we've read the tod_shape field earlier, we know that reading tod
# won't cause any additional truncation of the samples or detectors.
# tags is only needed here for read_combo support, but that is mostly broken
# anyway.
t1 = time.time()
self.d += actdata.read(self.entry,
fields=["tod", "tags"],
dets=self.d.dets)
#if debug_inject is not None: self.d.tod += debug_inject
t2 = time.time()
if verbose: print("read %-14s in %6.3f s" % ("tod", t2 - t1))
if config.get("tod_skip_deconv"): ops = ["tod_real"]
else: ops = ["tod"]
actdata.calibrate(self.d, operations=ops, verbose=verbose)
tod = self.d.tod
# Remove tod from our local d, so we won't end up hauling it around forever
del self.d.tod
# HWP resample if needed
if self.mapping is not None:
tod = np.ascontiguousarray(
utils.interpol(tod,
self.mapping.oimap[None],
order=1,
mask_nan=False))
method = config.get("downsample_method")
for s in self.sampslices:
tod = scan.slice_tod_samps(tod, s, method=method)
tod = np.ascontiguousarray(tod)
return tod
def get_samples(self, verbose=False):
"""Return the actual detector samples. Slow! Data is read from disk and
calibrated on the fly, so store the result if you need to reuse it."""
# Because we've read the tod_shape field earlier, we know that reading tod
# won't cause any additional truncation of the samples or detectors.
# tags is only needed here for read_combo support, but that is mostly broken
# anyway.
t1 = time.time()
self.d += actdata.read(self.entry, fields=["tod", "tags"], dets=self.d.dets)
t2 = time.time()
if verbose: print "read %-14s in %6.3f s" % ("tod", t2-t1)
if config.get("tod_skip_deconv"): ops = ["tod_real"]
else: ops = ["tod"]
actdata.calibrate(self.d, operations=ops, verbose=verbose)
tod = self.d.tod
# Remove tod from our local d, so we won't end up hauling it around forever
del self.d.tod
# HWP resample if needed
if self.mapping is not None:
tod = np.ascontiguousarray(utils.interpol(tod, self.mapping.oimap[None], order=1, mask_nan=False))
method = config.get("downsample_method")
for s in self.sampslices:
srange = slice(s.start, s.stop, np.sign(s.step) if s.step else None)
tod = tod[:,srange]
tod = resample.resample(tod, 1.0/np.abs(s.step or 1), method=method)
tod = np.ascontiguousarray(tod)
return tod
def detvecs_jon(ft, srate, dets=None, shared=False, cut_bins=None, apodization=None, cut_unit="freq", verbose=False):
"""Build a Detvecs noise matrix based on Jon's noise model.
ft is the *normalized* fourier-transform of a TOD: ft = fft.rfft(d)/nsamp.
srate is the sampling rate, dets is the list of detectors, shared specifies
whether the Detvecs object should use the compressed "shared" layout or not",
and cut_freq_ranges is a [nbin,{freq_from,freq_2}] array of frequencies
to completely cut."""
apodization = config.get("nmat_jon_apod", apodization) or None
downweight = config.get("nmat_jon_downweight")
spike_suppression = config.get("nmat_spike_suppression")
nfreq = ft.shape[1]
if cut_unit == "freq":
cut_bins = freq2ind(cut_bins, srate, nfreq, rfun=np.round)
# Construct our mode bins. Interestingly, we skip
# the f < 0.25 Hz area.
mbins = makebins([0.25, 4.0], srate, nfreq, 1000, rfun=np.round)[1:]
amp_thresholds = config.get("nmat_jon_amp_threshold")
amp_thresholds = extend_list([float(w) for w in amp_thresholds.split(",")], len(mbins))
single_threshold = config.get("nmat_jon_single_threshold")
# Ok, compute our modes, and then measure them in each bin.
# When using apodization, the vecs are not necessarily orthogonal,
# so don't rely on that.
vecs, weights = find_modes_jon(ft, mbins, amp_thresholds, single_threshold, apodization=apodization, verbose=verbose)
bin_edges = np.array([
0.10, 0.25, 0.35, 0.45, 0.55, 0.70, 0.85, 1.00,
1.20, 1.40, 1.70, 2.00, 2.40, 2.80, 3.40, 3.80,
4.60, 5.00, 5.50, 6.00, 6.50, 7.00, 8.00, 9.00, 10.0, 11.0,
12.0, 13.0, 14.0, 16.0, 18.0, 20.0, 22.0,
24.0, 26.0, 28.0, 30.0, 32.0, 36.5, 41.0,
45.0, 50.0, 55.0, 65.0, 70.0, 80.0, 90.0,
100., 110., 120., 130., 140., 150., 160., 170.,
180., 190.
])
# Cut bins that extend beyond our max frequency
bin_edges = bin_edges[bin_edges < srate/2 * 0.99]
bins = makebins(bin_edges, srate, nfreq, 2*vecs.shape[1], rfun=np.round)
bins, iscut = add_cut_bins(bins, cut_bins)
if downweight: white_scale = extend_list([1e-4, 0.25, 0.50, 1.00], len(bins))
else: white_scale = [1]*len(bins)
white_scale = np.asarray(white_scale)
white_scale[iscut] *= spike_suppression
if vecs.size == 0: raise errors.ModelError("Could not find any noise modes")
# Sharedvecs supports different sets of vecs per bin. But we only
# use a single group here. So every bin refers to the first group.
V = [vecs]
vinds = np.zeros(len(bins),dtype=int)
Nu, Nd, E = measure_detvecs_bin(ft, bins, vecs, mask=None, weights=weights)
# Apply white noise scaling
Nu /= white_scale[:,None]
if shared:
res = prepare_sharedvecs(Nu, V, E, bins, srate, dets, vinds)
else:
# Expand V so we have one block of vectors per bin
V = [V[i] for i in vinds]
res = prepare_detvecs(Nu, V, E, bins, srate, dets)
return res
def detvecs_jon(ft, srate, dets=None, shared=False, cut_bins=None, apodization=None, cut_unit="freq", verbose=False):
"""Build a Detvecs noise matrix based on Jon's noise model.
ft is the *normalized* fourier-transform of a TOD: ft = fft.rfft(d)/nsamp.
srate is the sampling rate, dets is the list of detectors, shared specifies
whether the Detvecs object should use the compressed "shared" layout or not",
and cut_freq_ranges is a [nbin,{freq_from,freq_2}] array of frequencies
to completely cut."""
apodization = config.get("nmat_jon_apod", apodization) or None
downweight = config.get("nmat_jon_downweight")
spike_suppression = config.get("nmat_spike_suppression")
nfreq = ft.shape[1]
if cut_unit == "freq":
cut_bins = freq2ind(cut_bins, srate, nfreq, rfun=np.round)
# Construct our mode bins. Interestingly, we skip
# the f < 0.25 Hz area.
mbins = makebins([0.25, 4.0], srate, nfreq, 1000, rfun=np.round)[1:]
amp_thresholds = config.get("nmat_jon_amp_threshold")
amp_thresholds = extend_list([float(w) for w in amp_thresholds.split(",")], len(mbins))
single_threshold = config.get("nmat_jon_single_threshold")
# Ok, compute our modes, and then measure them in each bin.
# When using apodization, the vecs are not necessarily orthogonal,
# so don't rely on that.
vecs, weights = find_modes_jon(ft, mbins, amp_thresholds, single_threshold, apodization=apodization, verbose=verbose)
bin_edges = np.array([
0.10, 0.25, 0.35, 0.45, 0.55, 0.70, 0.85, 1.00,
1.20, 1.40, 1.70, 2.00, 2.40, 2.80, 3.40, 3.80,
4.60, 5.00, 5.50, 6.00, 6.50, 7.00, 8.00, 9.00, 10.0, 11.0,
12.0, 13.0, 14.0, 16.0, 18.0, 20.0, 22.0,
24.0, 26.0, 28.0, 30.0, 32.0, 36.5, 41.0,
45.0, 50.0, 55.0, 65.0, 70.0, 80.0, 90.0,
100., 110., 120., 130., 140., 150., 160., 170.,
180., 190.
])
# Cut bins that extend beyond our max frequency
bin_edges = bin_edges[bin_edges < srate/2 * 0.99]
bins = makebins(bin_edges, srate, nfreq, 2*vecs.shape[1], rfun=np.round)
bins, iscut = add_cut_bins(bins, cut_bins)
if downweight: white_scale = extend_list([1e-4, 0.25, 0.50, 1.00], len(bins))
else: white_scale = [1]*len(bins)
white_scale = np.asarray(white_scale)
white_scale[iscut] *= spike_suppression
if vecs.size == 0: raise errors.ModelError("Could not find any noise modes")
# Sharedvecs supports different sets of vecs per bin. But we only
# use a single group here. So every bin refers to the first group.
V = [vecs]
vinds = np.zeros(len(bins),dtype=int)
Nu, Nd, E = measure_detvecs_bin(ft, bins, vecs, mask=None, weights=weights)
# Apply white noise scaling
Nu /= white_scale[:,None]
if shared:
res = prepare_sharedvecs(Nu, V, E, bins, srate, dets, vinds)
else:
# Expand V so we have one block of vectors per bin
V = [V[i] for i in vinds]
res = prepare_detvecs(Nu, V, E, bins, srate, dets)
return res
def parse_src_handling():
res = {}
res["mode"] = config.get("src_handling")
if res["mode"] == "none": return None
res["amplim"] = config.get("src_handling_lim")
res["srcs"] = None
srcfile = config.get("src_handling_list")
if srcfile:
res["srcs"] = pointsrcs.read(srcfile)
return res
def parse_src_handling():
res = {}
res["mode"] = config.get("src_handling")
if res["mode"] == "none": return None
res["amplim"] = config.get("src_handling_lim")
res["srcs"] = None
srcfile = config.get("src_handling_list")
if srcfile:
res["srcs"] = pointsrcs.read(srcfile)
return res
def __init__(self, data, model=None, window=None):
model = config.get("noise_model", model)
window = config.get("tod_window", window)*data.srate
nmat.apply_window(data.tod, window)
self.nmat = nmat_measure.NmatBuildDelayed(model, cut=data.cut_noiseest, spikes=data.spikes[:2].T)
self.nmat = self.nmat.update(data.tod, data.srate)
nmat.apply_window(data.tod, window, inverse=True)
self.model, self.window = model, window
self.ivar = self.nmat.ivar
self.cut = data.cut
def __init__(self, scan, template, sys=None):
sys = config.get("map_sys", sys)
downsamp = config.get("pmat_moby_downsamp", 20)
bore = scan.boresight.copy()
bore[:, 0] += utils.mjd2ctime(scan.mjd0)
opoint = get_moby_pointing(scan.entry,
bore,
scan.dets,
downgrade=downsamp)
# We will fit a polynomial to the pointing for each detector
box = np.array([[np.min(a), np.max(a)] for a in scan.boresight.T]).T
def scale(a, r):
return 2 * (a - r[0]) / (r[1] - r[0]) - 1
t = scale(scan.boresight[::downsamp, 0], box[:, 0])
az = scale(scan.boresight[::downsamp, 1], box[:, 1])
el = scale(scan.boresight[::downsamp, 2], box[:, 2])
basis = np.array(
[az**4, az**3, az**2, az**1, az**0, el**2, el, t**2, t, t * az]).T
denom = basis.T.dot(basis)
e, v = np.linalg.eigh(denom)
if (np.min(e) < 1e-8 * np.max(e)):
basis = np.concatenate([basis[:, :5], basis[:, 6:]], 1)
denom = basis.T.dot(basis)
# Convert from ra/dec to pixels, since we've confirmed that
# our ra/dec is the same as ninkasi to 0.01".
t1 = time.time()
pix = template.sky2pix(opoint[1::-1], safe=True)
t2 = time.time()
#rafit = np.linalg.solve(denom, basis.T.dot(opoint[0].T))
#decfit = np.linalg.solve(denom, basis.T.dot(opoint[1].T))
yfit = np.linalg.solve(denom, basis.T.dot(pix[0].T))
xfit = np.linalg.solve(denom, basis.T.dot(pix[1].T))
# Just use the same az and t as before for simplicity. The
# coefficients will be different, but the result will be
# the same.
basis = np.array([az**0, az**1, az**2, az**3, t]).T
denom = basis.T.dot(basis)
cosfit = np.linalg.solve(denom, basis.T.dot(opoint[2].T))
sinfit = np.linalg.solve(denom, basis.T.dot(opoint[3].T))
# Parameters for pmat
self.posfit = np.concatenate([yfit[:, :, None], xfit[:, :, None]], 2)
self.polfit = np.concatenate([cosfit[:, :, None], sinfit[:, :, None]],
2)
self.box = box
self.pixbox = np.array([[0, 0], template.shape[-2:]])
self.scan = scan
self.dtype = template.dtype
self.core = get_core(self.dtype)
def detvecs_simple(fourier,
srate,
dets=None,
type=None,
nbin=None,
nmin=None,
vecs=None,
eigs=None):
nfreq = fourier.shape[1]
ndet = fourier.shape[0]
type = config.get("nmat_uncorr_type", type)
nbin = config.get("nmat_uncorr_nbin", nbin)
nmin = config.get("nmat_uncorr_nmin", nmin)
whiten = config.get("nmat_uncorr_whiten")
if type is "exp":
bins = utils.expbin(nfreq, nbin=nbin, nmin=nmin)
elif type is "lin":
bins = utils.linbin(nfreq, nbin=nbin, nmin=nmin)
else:
raise ValueError("No such power binning type '%s'" % type)
nbin = bins.shape[0] # expbin may not provide exactly what we want
if vecs is None: vecs = np.full([ndet, 0], 1)
# Initialize our noise vectors with default values
vecs = np.asarray(vecs)
nvec = vecs.shape[-1]
Nu = np.zeros([nbin, ndet])
E = np.full([nbin, nvec], 1e-10)
V = [vecs]
vinds = np.zeros(nbin, dtype=int)
for bi, b in enumerate(bins):
d = fourier[:, b[0]:b[1]]
if vecs.size > 0:
# Measure amps when we have non-orthogonal vecs
rhs = vecs.T.dot(d)
div = vecs.T.dot(vecs)
amps = np.linalg.solve(div, rhs)
E[bi] = np.mean(np.abs(amps)**2, 1)
# Project out modes for every frequency individually
d -= vecs.dot(amps)
Nu[bi] = measure_power(d)
if whiten != 1:
white = np.percentile(Nu, 25, 0)
Nu = np.maximum((Nu - white) / whiten + white, white)
#Nu = (Nu/white)**whiten * white
# Override eigenvalues if necessary. This is useful
# for e.g. forcing total common mode subtraction.
# eigs must be broadcastable to [nbin,ndet]
if eigs is not None: E[:] = eigs
#return prepare_detvecs(Nd, V, E, bins, srate, dets)
return prepare_sharedvecs(Nu, V, E, bins, srate, dets, vinds)
def __init__(self, scan, template, sys=None, order=None):
sys = config.get("map_sys", sys)
transform = pos2pix(scan, template, sys)
ipol, obox = build_interpol(transform, scan.box, id=scan.entry.id)
self.rbox, self.nbox, self.yvals = extract_interpol_params(
ipol, template.dtype)
# Use obox to extract a pixel bounding box for this scan.
# These are the only pixels pmat needs to concern itself with.
# Reducing the number of pixels makes us more memory efficient
self.pixbox, self.nphi = build_pixbox(obox[:, :2], template)
self.scan, self.dtype = scan, template.dtype
self.core = get_core(self.dtype)
self.order = config.get("pmat_map_order", order)
def cut_mostly_cut_detectors(cuts, max_frac=None, max_nrange=None):
"""Mark detectors with too many cuts or too large cuts as completely cut."""
max_frac = config.get("cut_mostly_cut_frac", max_frac)
max_nrange = config.get("cut_mostly_cut_nrange", max_nrange)
cut_samps = cuts.sum(axis=1)
cut_nrange = cuts.nranges
bad = (cut_samps > cuts.nsamp * max_frac)
if max_nrange > 0:
bad |= cut_nrange > max_nrange
ocuts = []
for b in bad:
if b: ocuts.append(sampcut.full(1, cuts.nsamp))
else: ocuts.append(sampcut.empty(1, cuts.nsamp))
return sampcut.stack(ocuts)
def cut_mostly_cut_detectors(cuts, max_frac=None, max_nrange=None):
"""Mark detectors with too many cuts or too large cuts as completely cut."""
max_frac = config.get("cut_mostly_cut_frac", max_frac)
max_nrange = config.get("cut_mostly_cut_nrange", max_nrange)
cut_samps = cuts.sum(axis=1)
cut_nrange = cuts.nranges
bad = (cut_samps > cuts.nsamp*max_frac)
if max_nrange > 0:
bad |= cut_nrange > max_nrange
ocuts = []
for b in bad:
if b: ocuts.append(sampcut.full(1,cuts.nsamp))
else: ocuts.append(sampcut.empty(1,cuts.nsamp))
return sampcut.stack(ocuts)
def ground_cut(bore, det_offs, az_ranges=None, el_ranges=None):
az_ranges = np.array([[float(w) for w in tok.split(":")] for tok in config.get("cut_ground_az", az_ranges).split(",")])*np.pi/180
el_ranges = np.array([[float(w) for w in tok.split(":")] for tok in config.get("cut_ground_el", el_ranges).split(",")])*np.pi/180
n = bore.shape[1]
cuts = []
for di, doff in enumerate(det_offs):
p = bore[1:]+doff[:,None]
mask_az = np.full([n],False,dtype=bool)
for ar in az_ranges:
mask_az |= utils.between_angles(p[0], ar)
mask_el = np.full([n],False,dtype=bool)
for er in el_ranges:
mask_el |= utils.between_angles(p[1], er)
cuts.append(sampcut.from_mask(mask_az&mask_el))
return sampcut.stack(cuts)
def ground_cut(bore, det_offs, az_ranges=None, el_ranges=None):
az_ranges = np.array([[float(w) for w in tok.split(":")] for tok in config.get("cut_ground_az", az_ranges).split(",")])*np.pi/180
el_ranges = np.array([[float(w) for w in tok.split(":")] for tok in config.get("cut_ground_el", el_ranges).split(",")])*np.pi/180
n = bore.shape[1]
cuts = []
for di, doff in enumerate(det_offs):
p = bore[1:]+doff[:,None]
mask_az = np.full([n],False,dtype=bool)
for ar in az_ranges:
mask_az |= utils.between_angles(p[0], ar)
mask_el = np.full([n],False,dtype=bool)
for er in el_ranges:
mask_el |= utils.between_angles(p[1], er)
cuts.append(sampcut.from_mask(mask_az&mask_el))
return sampcut.stack(cuts)
def setup_filedb():
"""Create a default filedb based on the root, dataset and filedb config
variables. The result will be either a FormatDB or ExecDB based on the
format of the fildb file."""
override= config.get("file_override")
if override is "none": override = None
return execdb.ExecDB(cjoin(["root","dataset","filedb"]), cjoin(["root","filevars"]), override=override, root=cjoin(["root"]))
def setup_filedb():
"""Create a default filedb based on the root, dataset and filedb config
variables. The result will be either a FormatDB or ExecDB based on the
format of the fildb file."""
override= config.get("file_override")
if override is "none": override = None
return execdb.ExecDB(cjoin(["root","dataset","filedb"]), cjoin(["root","filevars"]), override=override, root=cjoin(["root"]))
def __init__(self, scan, params=None):
params = config.get("pmat_cut_type", params)
# Extract the cut parameters. E.g. poly:foo_secs -> [4,foo_samps]
par = np.array(self.parse_params(params, scan.srate))
# Meaning of cuts array: [:,{dets,offset,length,out_length,type,args..}]
self.cuts = np.zeros([scan.cut.nrange, 5 + len(par)], dtype=np.int32)
# Detector each cut belongs to
self.cuts[:, 0] = np.concatenate([
np.full(nr, i, np.int32) for i, nr in enumerate(scan.cut.nranges)
])
# Start of each cut
self.cuts[:, 1] = scan.cut.ranges[:, 0]
# Length of each cut
self.cuts[:, 2] = scan.cut.ranges[:, 1] - scan.cut.ranges[:, 0]
# Set up the parameter arguments
self.cuts[:, 5:] = par[None, :]
assert np.all(
self.cuts[:,
2] > 0), "Empty cut range detected in %s" % scan.entry.id
assert np.all(self.cuts[:, 1] >= 0) and np.all(
scan.cut.ranges[:, 1] <= scan.nsamp
), "Out of bounds cut range detected in %s" % scan.entry.id
if self.cuts.size > 0:
get_core(np.float32).measure_cuts(self.cuts.T)
self.cuts[:, 3] = utils.cumsum(self.cuts[:, 4])
# njunk is the number of cut parameters for *this scan*
self.njunk = np.sum(self.cuts[:, 4])
self.params = params
self.scan = scan
def __init__(self,
signal,
signal_cut,
scans,
weights,
noise=True,
hits=True):
"""Binned preconditioner: (P'W"P)", where W" is a white
nosie approximation of N". If noise=False, instead computes
(P'P)". If hits=True, also computes a hitcount map."""
ncomp = signal.area.shape[0]
self.div = enmap.zeros((ncomp, ) + signal.area.shape, signal.area.wcs,
signal.area.dtype)
calc_div_map(self.div, signal, signal_cut, scans, weights, noise=noise)
self.idiv = array_ops.svdpow(self.div,
-1,
axes=[0, 1],
lim=config.get("eig_limit"))
#self.idiv[:] = np.eye(3)[:,:,None,None]
if hits:
# Build hitcount map too
self.hits = signal.area.copy()
self.hits = calc_hits_map(self.hits, signal, signal_cut, scans)
else:
self.hits = None
self.signal = signal
def __init__(self,
signal,
signal_cut,
scans,
weights,
noise=True,
hits=True):
"""Binned preconditioner: (P'W"P)", where W" is a white
nosie approximation of N". If noise=False, instead computes
(P'P)". If hits=True, also computes a hitcount map."""
geom = signal.area.geometry.copy()
geom.pre = (signal.area.shape[0], ) + geom.pre
self.div = dmap.zeros(geom)
calc_div_map(self.div, signal, signal_cut, scans, weights, noise=noise)
self.idiv = self.div.copy()
for dtile in self.idiv.tiles:
dtile[:] = array_ops.svdpow(dtile,
-1,
axes=[0, 1],
lim=config.get("eig_limit"))
if hits:
# Build hitcount map too
self.hits = signal.area.copy()
self.hits = calc_hits_map(self.hits, signal, signal_cut, scans)
else:
self.hits = None
self.signal = signal
def gapfill_linear(arr, ranges, inplace=False, overlap=None):
"""Returns arr with the ranges given by ranges, which can be [:,{from,to}] or
a Rangelist, filled using linear interpolation."""
ranges = Rangelist(ranges, len(arr), copy=False)
overlap = config.get("gapfill_context", overlap)
if not inplace: arr = np.array(arr)
nr = len(ranges.ranges)
n = ranges.n
for i, (r1, r2) in enumerate(ranges.ranges):
left = max(0 if i == 0 else ranges.ranges[i - 1, 1], r1 - overlap)
right = min(n if i == nr - 1 else ranges.ranges[i + 1, 0],
r2 + overlap)
# If the cut coveres the whole array, fill with 0
if r1 == 0 and r2 == len(arr):
arr[r1:r2] = 0
# If it goes all the way to one end, use the value from one side
elif r1 == 0:
arr[r1:r2] = np.mean(arr[r2:right])
elif r2 == len(arr):
arr[r1:r2] = np.mean(arr[left:r1])
# Otherwise use linear interpolation
else:
arr[r1:r2] = np.linspace(np.mean(arr[left:r1]),
np.mean(arr[r2:right]),
r2 - r1 + 1,
endpoint=False)[1:]
return arr
def __init__(self, scan, model=None, window=None, filter=None):
model = config.get("noise_model", model)
window = config.get("tod_window", window)*scan.srate
nmat.apply_window(scan.tod, window)
self.nmat = nmat_measure.NmatBuildDelayed(model, cut=scan.cut_noiseest, spikes=scan.spikes)
self.nmat = self.nmat.update(scan.tod, scan.srate)
nmat.apply_window(scan.tod, window, inverse=True)
self.model, self.window = model, window
self.ivar = self.nmat.ivar
self.cut = scan.cut
# Optional extra filter
if filter:
freq = fft.rfftfreq(scan.nsamp, 1/scan.srate)
fknee, alpha = filter
with utils.nowarn():
self.filter = (1 + (freq/fknee)**-alpha)**-1
else: self.filter = None
def turnaround_cut(az, margin=None):
margin = config.get("cut_turnaround_margin", margin) * utils.degree
# Use percentile just in case there's some outliers (for example a scan that's a bit
# higher than the others.
az1 = np.percentile(az, 0.1)
az2 = np.percentile(az, 99.9)
mask = (az < az1) | (az > az2)
cut = sampcut.from_mask(mask)
return cut
def turnaround_cut(az, margin=None):
margin = config.get("cut_turnaround_margin", margin)*utils.degree
# Use percentile just in case there's some outliers (for example a scan that's a bit
# higher than the others.
az1 = np.percentile(az, 0.1)
az2 = np.percentile(az, 99.9)
mask = (az<az1)|(az>az2)
cut = sampcut.from_mask(mask)
return cut
def stationary_cut(az, tol=None):
"""Cut samples where the telescope isn't moving at the beginning
and end of the tod."""
tol = config.get("cut_stationary_tol", tol) * utils.degree
b1 = np.where(np.abs(az - az[0]) > tol)[0]
b2 = np.where(np.abs(az - az[-1]) > tol)[0]
if len(b1) == 0 or len(b2) == 0:
# Entire tod cut!
return sampcut.full(1, len(az))
else:
return sampcut.from_list([[[0, b1[0]], [b2[-1], len(az)]]], len(az))
def crop_fftlen(data, factors=None):
"""Slightly crop samples in order to make ffts faster. This should
be called at a point when the length won't be futher cropped by other
effects."""
if data.nsamp in [0, None]: raise errors.DataMissing("nsamp")
if data.nsamp < 0: raise errors.DataMissing("nsamp")
factors = config.get("fft_factors", factors)
if isinstance(factors, basestring): factors = [int(w) for w in factors.split(",")]
ncrop = fft.fft_len(data.nsamp, factors=factors)
data += dataset.DataField("fftlen", samples=[data.samples[0],data.samples[0]+ncrop])
return data
def detvecs_scaled(ft, srate, dets=None, bsize=None, lim=None, lim2=None, spikecut=None):
bsize = config.get("nmat_scaled_bsize", bsize)
lim = config.get("nmat_scaled_spike", lim)
lim2 = config.get("nmat_scaled_smooth", lim2)
spikecut = config.get("nmat_scaled_spikecut", spikecut)
# First set up our sample points
mps = calc_mean_ps(ft)
bins, bins_spike = build_spec_bins(mps, bsize=bsize, lim=lim, lim2=lim2)
# Measure mean power per bin
brms = np.zeros(ft.shape[:-1]+(len(bins),), mps.dtype)
for bi, b in enumerate(bins):
brms[:,bi] = np.mean(np.abs(ft[:,b[0]:b[1]])**2)**0.5
ft[:,b[0]:b[1]] /= brms[:,bi,None]
# Build interior noise model
if not spikecut: sbins_spike = None
nmat_inner = detvecs_jon(ft, srate=srate, dets=dets, cut_bins=bins_spike, cut_unit="inds")
# Undo scaling of ft in case caller still needs it
for bi, b in enumerate(bins):
ft[:,b[0]:b[1]] *= brms[:,bi,None]
return nmat.NmatScaled(brms, bins, nmat_inner)
def stationary_cut(az, tol=None):
"""Cut samples where the telescope isn't moving at the beginning
and end of the tod."""
tol = config.get("cut_stationary_tol", tol)*utils.degree
b1 = np.where(np.abs(az-az[0])>tol)[0]
b2 = np.where(np.abs(az-az[-1])>tol)[0]
if len(b1) == 0 or len(b2) == 0:
# Entire tod cut!
return sampcut.full(1,len(az))
else:
return sampcut.from_list([[[0,b1[0]],[b2[-1],len(az)]]],len(az))
def setup_params(category, predefined, defparams):
"""Set up parameters of a given category. With the following
convention.
1. If a name matches a default's name, unspecified properties use the default's
2. A name can be specified multiple times, e.g. -S sky:foo -S sky:bar. These
do not override each other - each will be used separately.
2. If a name that has a default is not specified manually, then a single
instance of that name is instantiated, with the default parameters."""
params = []
argdict = vars(args)
overrides = argdict[category]
counts = {}
if overrides:
for oval in overrides:
m = re.match(r'([^,:]+):(.*)', oval)
if m:
name, rest = m.groups()
desc = config.get("%s_%s_default" %
(category, name)) + ",use=yes," + rest
elif "," not in oval:
desc = config.get("%s_%s_default" %
(category, oval)) + ",use=yes"
else:
desc = "use=yes," + oval
param = parse_desc(desc, default=defparams)
name = param["name"]
if name in counts: counts[name] += 1
else: counts[name] = 1
param["i"] = counts[name]
params.append(param)
# For each predefined param, add it only if none of that name already exist
defaults = []
for p in predefined:
if not p in counts:
defaults.append(
parse_desc(config.get("%s_%s_default" % (category, p))))
defaults[-1]["i"] = 0
params = defaults + params
# Kill irrelevant parameters (those not in use)
params = [p for p in params if p["use"] != "no"]
return params
def get_samples(self):
"""Return the actual detector samples. Slow! Data is read from disk,
so store the result if you need to reuse it."""
with h5py.File(self.fname, "r") as hfile:
tod = hfile["tod"].value[self.subdets]
method = config.get("downsample_method")
for s in self.sampslices:
tod = resample.resample(tod, 1.0/np.abs(s.step or 1), method=method)
s = slice(s.start, s.stop, np.sign(s.step) if s.step else None)
tod = tod[:,s]
res = np.ascontiguousarray(tod)
return res
def get_samples(self, verbose=False):
"""Return the actual detector samples. Slow! Data is read from disk and
calibrated on the fly, so store the result if you need to reuse it."""
# Because we've read the tod_shape field earlier, we know that reading tod
# won't cause any additional truncation of the samples or detectors.
# tags is only needed here for read_combo support, but that is mostly broken
# anyway.
t1 = time.time()
self.d += actdata.read(self.entry,
fields=["tod", "tags"],
dets=self.d.dets)
t2 = time.time()
if verbose: print("read %-14s in %6.3f s" % ("tod", t2 - t1))
if config.get("tod_skip_deconv"): ops = ["tod_real"]
else: ops = ["tod"]
actdata.calibrate(self.d, operations=ops, verbose=verbose)
tod = self.d.tod
# Remove tod from our local d, so we won't end up hauling it around forever
del self.d.tod
# HWP resample if needed
if self.mapping is not None:
tod = np.ascontiguousarray(
utils.interpol(tod,
self.mapping.oimap[None],
order=1,
mask_nan=False))
method = config.get("downsample_method")
for s in self.sampslices:
srange = slice(s.start, s.stop,
np.sign(s.step) if s.step else None)
tod = tod[:, srange]
# make sure we get exactly the same length the cuts will be expecting
step = np.abs(s.step or 1)
olen = (tod.shape[1] + step - 1) // step
tod = resample.resample(tod,
float(olen) / tod.shape[1],
method=method)
tod = np.ascontiguousarray(tod)
return tod
def detvecs_simple(fourier, srate, dets=None, type=None, nbin=None, nmin=None, vecs=None, eigs=None):
nfreq = fourier.shape[1]
ndet = fourier.shape[0]
type = config.get("nmat_uncorr_type", type)
nbin = config.get("nmat_uncorr_nbin", nbin)
nmin = config.get("nmat_uncorr_nmin", nmin)
if type is "exp":
bins = utils.expbin(nfreq, nbin=nbin, nmin=nmin)
elif type is "lin":
bins = utils.linbin(nfreq, nbin=nbin, nmin=nmin)
else: raise ValueError("No such power binning type '%s'" % type)
nbin = bins.shape[0] # expbin may not provide exactly what we want
if vecs is None: vecs = np.full([ndet,0],1)
# Initialize our noise vectors with default values
vecs = np.asarray(vecs)
nvec = vecs.shape[-1]
Nu = np.zeros([nbin,ndet])
E = np.full([nbin,nvec],1e-10)
V = [vecs]
vinds = np.zeros(nbin,dtype=int)
for bi, b in enumerate(bins):
d = fourier[:,b[0]:b[1]]
if vecs.size > 0:
# Measure amps when we have non-orthogonal vecs
rhs = vecs.T.dot(d)
div = vecs.T.dot(vecs)
amps = np.linalg.solve(div,rhs)
E[bi] = np.mean(np.abs(amps)**2,1)
# Project out modes for every frequency individually
d -= vecs.dot(amps)
Nu[bi] = measure_power(d)
# Override eigenvalues if necessary. This is useful
# for e.g. forcing total common mode subtraction.
# eigs must be broadcastable to [nbin,ndet]
if eigs is not None: E[:] = eigs
#return prepare_detvecs(Nd, V, E, bins, srate, dets)
return prepare_sharedvecs(Nu, V, E, bins, srate, dets, vinds)
def read_hwp(entry):
dummy = dataset.DataSet([
dataset.DataField("hwp", 0),
dataset.DataField("hwp_id", "none"),
dataset.DataField("hwp_source", "none")])
epochs = try_read(files.read_hwp_epochs, "hwp_epochs", entry.hwp_epochs)
t, _, ar = entry.id.split(".")
t = float(t)
if ar not in epochs: return dummy
for epoch in epochs[ar]:
if t >= epoch[0] and t < epoch[1]:
# Ok, the HWP was active during this period. Try to read it. It can be in
# several different formats.
if entry.hwp_format == "tod":
# HWP angles in the tod, in the new, high-quality format
hwp, flags = try_read(files.read_hwp_angle, "hwp_tod_angles", entry.tod)
return dataset.DataSet([
dataset.DataField("hwp", hwp, samples=[0, hwp.size], sample_index=0),
dataset.DataField("hwp_id", epoch[2]),
dataset.DataField("hwp_source", "tod")])
elif entry.hwp_format == "raw":
# HWP angles in the tod, in the old, inaccurate format
hwp = try_read(files.read_hwp_raw, "hwp_raw_angles", entry.tod)
return dataset.DataSet([
dataset.DataField("hwp", hwp, samples=[0, hwp.size], sample_index=0),
dataset.DataField("hwp_id", epoch[2]),
dataset.DataField("hwp_source", "raw")])
elif entry.hwp_format == "external":
# HWP angles in external data files
try:
status = try_read(files.read_hwp_status, "hwp_status", entry.hwp_status)
except errors.DataMissing as e:
status = None
# If there weren't any external files, possibly fall back to raw angles
if status is None or get_dict_wild(status, entry.id, 0) != 1:
if config.get("hwp_fallback") == "raw":
hwp = try_read(files.read_hwp_raw, "hwp_raw_angles", entry.tod)
return dataset.DataSet([
dataset.DataField("hwp", hwp, samples=[0, hwp.size], sample_index=0),
dataset.DataField("hwp_id", epoch[2]),
dataset.DataField("hwp_source", "raw")])
else:
raise e if status is None else errors.DataMissing("Missing HWP angles!")
# Try to read the angles themselves
hwp = try_read(files.read_hwp_cleaned, "hwp_angles", entry.hwp)
return dataset.DataSet([
dataset.DataField("hwp", hwp, samples=[0,hwp.size], sample_index=0),
dataset.DataField("hwp_id", epoch[2]),
dataset.DataField("hwp_source","cleaned")])
# Not in any epoch, so return 0 hwp angle (which effectively turns it off)
return dummy
def setup_params(category, predefined, defparams):
"""Set up parameters of a given category. With the following
convention.
1. If a name matches a default's name, unspecified properties use the default's
2. A name can be specified multiple times, e.g. -S sky:foo -S sky:bar. These
do not override each other - each will be used separately.
2. If a name that has a default is not specified manually, then a single
instance of that name is instantiated, with the default parameters."""
params = []
argdict = vars(args)
overrides = argdict[category]
counts = {}
if overrides:
for oval in overrides:
m = re.match(r'([^,:]+):(.*)', oval)
if m:
name, rest = m.groups()
desc = config.get("%s_%s_default" % (category,name)) + ",use=yes," + rest
elif "," not in oval:
desc = config.get("%s_%s_default" % (category,oval)) + ",use=yes"
else:
desc = "use=yes,"+oval
param = parse_desc(desc, default=defparams)
name = param["name"]
if name in counts: counts[name] += 1
else: counts[name] = 1
param["i"] = counts[name]
params.append(param)
# For each predefined param, add it only if none of that name already exist
defaults = []
for p in predefined:
if not p in counts:
defaults.append(parse_desc(config.get("%s_%s_default" % (category, p))))
defaults[-1]["i"] = 0
params = defaults + params
# Kill irrelevant parameters (those not in use)
params = [p for p in params if p["use"] != "no"]
return params
def __init__(self, scan, template, sys=None, order=None):
self.sys = config.get("map_sys", sys)
# Build the pointing interpolator
self.trans = pos2pix(scan, template, self.sys)
self.poly = PolyInterpol(self.trans, scan.boresight, scan.offsets)
# Build the pixel shift information. This assumes ces-like scans in equ-like systems
self.sdir = get_scan_dir(scan.boresight[:, 1])
self.period = get_scan_period(scan.boresight[:, 1], scan.srate)
self.wbox, self.wshift = build_work_shift(self.trans, scan.box,
self.period)
self.nphi = int(np.round(np.abs(360. / template.wcs.wcs.cdelt[0])))
self.dtype = template.dtype
self.core = get_core(self.dtype)
self.scan = scan
self.order = 0
def __call__(self, ipos):
"""Transform ipos[{t,az,el},nsamp] into opix[{y,x,c,s},nsamp]."""
shape = ipos.shape[1:]
ipos = ipos.reshape(ipos.shape[0], -1)
time = self.scan.mjd0 + ipos[0] / utils.day2sec
# We support sidelobe mapping by passing the detector pointing "ipos" as the "boresight"
# pointing, which is only used in boresight-relative coordinates or sidelobe mapping.
# This actually results in a better coordinate system than if we had passed in the actual
# boresight, since we don't really want boresight-centered coordinates, we want detector
# centered coordinates.
opos = coordinates.transform(self.scan.sys,
self.sys,
ipos[1:],
time=time,
site=self.scan.site,
pol=True,
bore=ipos[1:])
# Parallax correction
sundist = config.get("pmat_parallax_au")
if sundist:
# Transform to a sun-centered coordinate system, assuming all objects
# are at a distance of sundist from the sun
opos[1::-1] = parallax.earth2sun(opos[1::-1], self.scan.mjd0,
sundist)
opix = np.zeros((4, ) + ipos.shape[1:])
if self.template is not None:
opix[:2] = self.template.sky2pix(opos[1::-1], safe=2)
# When mapping the full sky, angle wraps can't be hidden
# ouside the image. We must therefore unwind along each
# interpolation axis to avoid discontinuous interpolation
nx = int(np.round(np.abs(360 / self.template.wcs.wcs.cdelt[0])))
opix[1] = utils.unwind(opix[1].reshape(shape),
period=nx,
axes=range(len(shape))).reshape(-1)
# Prefer positive numbers
opix[1] -= np.floor(opix[1].reshape(-1)[0] / nx) * nx
# but not if they put everything outside our patch
if np.min(opix[1]) > self.template.shape[-1]:
opix[1] -= nx
else:
# If we have no template, output angles instead of pixels.
# Make sure the angles don't have any jumps in them
opix[:2] = opos[1::-1] # output order is dec,ra
opix[1] = utils.rewind(opix[1], self.ref_phi)
opix[2] = np.cos(2 * opos[2])
opix[3] = np.sin(2 * opos[2])
return opix.reshape((opix.shape[0], ) + shape)
parser.add_argument("-n", "--nstep", type=int, default=50)
parser.add_argument("-m", "--method",type=str, default="messenger")
parser.add_argument( "--ndet", type=int, default=None)
parser.add_argument("-p", "--precompute", action="store_true")
parser.add_argument("-o", "--ostep", type=int, default=10)
args = parser.parse_args()
utils.mkdir(args.odir)
comm = mpi.COMM_WORLD
dtype = np.float32
ncomp = 3
area = enmap.read_map(args.area)
area = enmap.zeros((ncomp,)+area.shape[-2:],area.wcs,dtype)
Tscale = 0.9
nstep = args.nstep
downsample = config.get("downsample")
filedb.init()
ids = filedb.scans[args.sel]
# Was 1e7
cooldown = sum([[10**j]*5 for j in range(6,0,-1)],[])+[1]
# Read my scans
njunk_tot = 0
cg_rhs = area*0
cg_rjunk = []
if args.precompute:
prec_NNmap = {lam: area*0 for lam in np.unique(cooldown)}
prec_NNjunk = {lam: [] for lam in np.unique(cooldown)}
scans = []
for ind in range(comm.rank, len(ids), comm.size):
parser.add_argument("--ntest", type=int, default=1000)
parser.add_argument("--interpolator", type=str, default="grad")
args = parser.parse_args()
# Hardcode an arbitrary site
site = bunch.Bunch(
lat = -22.9585,
lon = -67.7876,
alt = 5188.,
T = 273.15,
P = 550.,
hum = 0.2,
freq = 150.,
lapse= 0.0065)
acc = config.get("pmat_accuracy")
max_size = config.get("pmat_interpol_max_size")
max_time = config.get("pmat_interpol_max_time")
nel = int(np.rint((args.el2-args.el1)/args.delta_el))+1
naz = int(np.rint((args.az2-args.az1)/args.daz))+1
def hor2cel(hor):
shape = hor.shape[1:]
hor = hor.reshape(hor.shape[0],-1)
tmp = coordinates.transform("hor", "cel", hor[1:], time=hor[0]+t_mid, site=site, pol=True)
res = np.zeros((4,)+tmp.shape[1:])
res[0] = utils.rewind(tmp[0], tmp[0,0])
res[1] = tmp[1]
res[2] = np.cos(2*tmp[2])
res[3] = np.sin(2*tmp[2])
config.default(
"map_sys", "equ",
"The coordinate system of the maps. Can be eg. 'hor', 'equ' or 'gal'.")
config.default("map_dist", False, "Whether to use distributed maps")
parser = config.ArgumentParser()
parser.add_argument("sel", help="TOD selction query")
parser.add_argument("area", help="Geometry to map")
parser.add_argument("odir", help="Output directory")
parser.add_argument("prefix", nargs="?", help="Output file name prefix")
parser.add_argument("--dets", type=str, default=0, help="Detector slice")
args = parser.parse_args()
utils.mkdir(args.odir)
comm = mpi.COMM_WORLD
dtype = np.float32 if config.get("map_bits") == 32 else np.float64
ncomp = 3
tsize = 720
root = args.odir + "/" + (args.prefix + "_" if args.prefix else "")
down = config.get("downsample")
# Set up logging
utils.mkdir(root + ".log")
logfile = root + ".log/log%03d.txt" % comm.rank
log_level = log.verbosity2level(config.get("verbosity"))
L = log.init(level=log_level, file=logfile, rank=comm.rank, shared=True)
# Set up our geometry
shape, wcs = enmap.read_map_geometry(args.area)
shape = (ncomp, ) + shape[-2:]
msys = config.get("map_sys")
dist = config.get("map_dist")
# Filter parameters
from enact import actscan, nmat_measure, filedb, todinfo
config.default("map_bits", 32, "Bit-depth to use for maps and TOD")
config.default("verbosity", 1, "Verbosity for output. Higher means more verbose. 0 outputs only errors etc. 1 outputs INFO-level and 2 outputs DEBUG-level messages.")
parser = config.ArgumentParser(os.environ["HOME"] + "/.enkirc")
parser.add_argument("sel")
parser.add_argument("srcs")
parser.add_argument("area")
parser.add_argument("odir")
parser.add_argument("--nmax", type=int, default=10)
parser.add_argument("-s", "--src", type=int, default=None, help="Only analyze given source")
parser.add_argument("-c", "--cont", action="store_true")
args = parser.parse_args()
dtype = np.float32 if config.get("map_bits") == 32 else np.float64
comm = mpi.COMM_WORLD
tcomm = mpi.COMM_SELF
nmax = args.nmax
ncomp = 3
isys = "hor"
utils.mkdir(args.odir)
log_level = log.verbosity2level(config.get("verbosity"))
L = log.init(level=log_level, rank=comm.rank, shared=False)
# Get the source positions
srcs = np.loadtxt(args.srcs).T
# Read our area. Should be centered on 0,0
area = enmap.read_map(args.area)
fields = ["gain","tconst","cut","tod","boresight"]
if args.fields: fields = args.fields.split(",")
d = actdata.read(entry, fields=fields)
if absdets: d.restrict(dets=absdets)
if subdets: d.restrict(dets=d.dets[subdets])
if args.calib: d = actdata.calibrate(d, exclude=["autocut"])
elif args.manual_calib:
ops = args.manual_calib.split(",")
if "safe" in ops: d.boresight[1:] = utils.unwind(d.boresight[1:], period=360)
if "rad" in ops: d.boresight[1:] *= np.pi/180
if "bgap" in ops:
bad = (d.flags!=0)*(d.flags!=0x10)
for b in d.boresight: gapfill.gapfill_linear(b, bad, inplace=True)
if "gain" in ops: d.tod *= d.gain[:,None]
if "tgap" in ops:
gapfiller = {"copy":gapfill.gapfill_copy, "linear":gapfill.gapfill_linear}[config.get("gapfill")]
gapfiller(d.tod, d.cut, inplace=True)
if "slope" in ops:
utils.deslope(d.tod, w=8, inplace=True)
if "deconv" in ops:
d = actdata.calibrate(d, operations=["tod_fourier"])
if args.bin > 1:
d.tod = resample.downsample_bin(d.tod, steps=[args.bin])
d.boresight = resample.downsample_bin(d.boresight, steps=[args.bin])
d.flags = resample.downsample_bin(d.flags, steps=[args.bin])
oname = args.ofile
if len(ids) > 1: oname = "%s/%s.hdf" % (args.ofile, id)
with h5py.File(oname, "w") as hfile:
if "tod" in d: hfile["tod"] = d.tod
if "boresight" in d:
hfile["az"] = d.boresight[1]
def calibrate_cut(data, n=None):
n = [int(w) for w in config.get("pad_cuts", n).split(":")]
for name in ["cut","cut_basic","cut_noiseest","cut_quality"]:
if name in data:
data[name] = data[name].widen(n)
return data
import numpy as np, argparse, os, sys
from enlib import enmap, pmat, config
from enact import filedb, data
parser = config.ArgumentParser(os.environ["HOME"] + "/.enkirc")
parser.add_argument("id")
parser.add_argument("area")
parser.add_argument("--di", type=int, default=0, help="Index into array of accepted detectors to use.")
args = parser.parse_args()
dtype = np.float64
eqsys = config.get("map_eqsys")
area = enmap.read_map(args.area).astype(dtype)
area = enmap.zeros((3,)+area.shape[-2:], area.wcs, dtype)
entry = filedb.data[args.id]
# First get the raw samples
d = data.read(entry, subdets=[args.di])
raw_tod = d.tod[0,d.sample_offset:d.cutafter].copy()
raw_bore = d.boresight[:,d.sample_offset:d.cutafter].T
# Then some calibrated samples
d = data.calibrate(d)
cal_tod = d.tod[0]
cal_bore = d.boresight.T
# Apply fourier-truncation to raw data
raw_tod = raw_tod[:cal_tod.shape[0]]
raw_bore = raw_bore[:cal_bore.shape[0]]
# And a proper ACTScan
scan = data.ACTScan(entry, subdets=[args.di])
# Detector pointing
config.default("filedb", "filedb.txt", "File describing the location of the TOD and their metadata")
config.default("map_bits", 32, "Bit-depth to use for maps and TOD")
config.default("verbosity", 1, "Verbosity for output. Higher means more verbose. 0 outputs only errors etc. 1 outputs INFO-level and 2 outputs DEBUG-level messages.")
parser = config.ArgumentParser(os.environ["HOME"] + "/.enkirc")
parser.add_argument("filelist")
parser.add_argument("srcs")
parser.add_argument("odir")
parser.add_argument("--ncomp", type=int, default=3)
parser.add_argument("--ndet", type=int, default=0)
parser.add_argument("--minamp", type=float, default=100)
parser.add_argument("-c", action="store_true")
parser.add_argument("--oldformat", action="store_true")
args = parser.parse_args()
dtype = np.float32 if config.get("map_bits") == 32 else np.float64
comm = mpi.COMM_WORLD
myid = comm.rank
nproc = comm.size
filedb.init()
db = filedb.data
filelist = todinfo.get_tods(args.filelist, filedb.scans)
def compress_beam(sigma, phi):
c,s=np.cos(phi),np.sin(phi)
R = np.array([[c,-s],[s,c]])
C = np.diag(sigma**-2)
C = R.dot(C).dot(R.T)
return np.array([C[0,0],C[1,1],C[0,1]])
parser.add_argument("--daz", type=float, default=3.0)
parser.add_argument("--nt", type=int, default=10)
parser.add_argument("--dets", type=str, default=0)
parser.add_argument("--ntod", type=int, default=0)
parser.add_argument("-w", "--weighted", type=int, default=1)
parser.add_argument("-D", "--deslope", type=int, default=0)
args = parser.parse_args()
comm = mpi.COMM_WORLD
filedb.init()
ids = [line.split()[0] for line in open(args.idlist, "r")]
if args.ntod: ids = ids[:args.ntod]
is_dmap = os.path.isdir(args.imap)
log_level = log.verbosity2level(config.get("verbosity"))
L = log.init(level=log_level, rank=comm.rank)
tshape = (720, 720)
# Read in all our scans
L.info("Reading %d scans" % len(ids))
myinds = np.arange(len(ids))[comm.rank::comm.size]
myinds, myscans = scanutils.read_scans(ids,
myinds,
actscan.ACTScan,
filedb.data,
dets=args.dets,
downsample=config.get("downsample"))
myinds = np.array(myinds, int)
# Collect scan info. This currently fails if any task has empty myinds
import numpy as np, os, h5py
from enlib import config, mpi, coordinates, utils, errors, tagdb
from enact import filedb, actdata, todinfo
parser = config.ArgumentParser(os.environ["HOME"] + "/.enkirc")
parser.add_argument("tagfile")
parser.add_argument("sel", nargs="?", default="")
parser.add_argument("ofile")
args = parser.parse_args()
file_db = filedb.setup_filedb()
scan_db = todinfo.read(args.tagfile, vars={"root":config.get("root")})
comm = mpi.COMM_WORLD
# We want to process *all* tods, not just selected ones. Could also
# achieve this by adding /all to the selector.
scan_db = scan_db.select(scan_db.query(args.sel, apply_default_query=False))
ids = scan_db.ids
stats = []
for ind in range(comm.rank, len(ids), comm.size):
id = ids[ind]
entry = file_db[id]
try:
stats.append(todinfo.build_tod_stats(entry))
except (errors.DataMissing,AttributeError) as e:
print "%3d %4d/%d %5.1f%% Skipping %s (%s)" % (comm.rank, ind+1, len(ids), (ind+1)/float(len(ids))*100, id, str(e))
continue
print "%3d %4d/%d %5.1f%% %s" % (comm.rank, ind+1, len(ids), (ind+1)/float(len(ids))*100, id)
stats = todinfo.merge_tod_stats(stats)
if comm.rank == 0: print "Reducing"
parser.add_argument("-q", "--quiet", action="count", default=0)
parser.add_argument("-c", "--cont", action="store_true")
args = parser.parse_args()
comm = mpi.COMM_WORLD
filedb.init()
ids = filedb.scans[args.sel]
R = args.dist * utils.degree
csize= 100
verbose = args.verbose - args.quiet > 0
dtype= np.float32
model_fknee = 10
model_alpha = 10
sys = "hor:"+args.planet+"/0_0"
tod_sys = config.get("tod_sys")
utils.mkdir(args.odir)
prefix = args.odir + "/"
show = bench.show if verbose else bench.dummy
def estimate_ivar(tod):
tod -= np.mean(tod,1)[:,None]
tod = tod.astype(dtype)
diff = tod[:,1:]-tod[:,:-1]
diff = diff[:,:diff.shape[-1]/csize*csize].reshape(tod.shape[0],-1,csize)
ivar = 1/(np.median(np.mean(diff**2,-1),-1)/2**0.5)
return ivar
def estimate_atmosphere(tod, region_cut, srate, fknee, alpha):
model = gapfill.gapfill_joneig(tod, region_cut, inplace=False)
config.default("verbosity", 1, "Verbosity for output. Higher means more verbose. 0 outputs only errors etc. 1 outputs INFO-level and 2 outputs DEBUG-level messages.")
parser = config.ArgumentParser(os.environ["HOME"] + "/.enkirc")
parser.add_argument("filelist")
parser.add_argument("srcs")
parser.add_argument("odir")
parser.add_argument("-R", "--radius", type=float, default=5.0)
parser.add_argument("-r", "--resolution", type=float, default=0.25)
args = parser.parse_args()
comm = mpi4py.MPI.COMM_WORLD
myid = comm.rank
nproc = comm.size
log_level = log.verbosity2level(config.get("verbosity"))
L = log.init(level=log_level, rank=myid)
# Allow filelist to take the format filename:[slice]
toks = args.filelist.split(":")
filelist, fslice = toks[0], ":".join(toks[1:])
filelist = [line.split()[0] for line in open(filelist,"r") if line[0] != "#"]
filelist = eval("filelist"+fslice)
utils.mkdir(args.odir)
srcs = np.loadtxt(args.srcs).T
# create minimaps around each source
nsrc = srcs.shape[1]
ncomp = 1
n = int(np.round(2*args.radius/args.resolution))
def tod_end_cut(nsamp, srate, cut_secs=None):
"""Cut cut_secs seconds of data at each end of the tod"""
ncut = int(config.get("cut_tod_ends_nsec",cut_secs)*srate)
return sampcut.from_list([[[0,ncut],[nsamp-ncut,nsamp]]], nsamp)
parser.add_argument("-S", "--signal", action="append", help="Signals to solve for. For example -S sky:area.fits -S scan would solve for the sky map and scan pickup maps jointly, using area.fits as the map template.")
parser.add_argument("-F", "--filter", action="append")
parser.add_argument("-M", "--mapfilter", action="append")
parser.add_argument("--group-tods", action="store_true")
parser.add_argument("--individual", action="store_true")
parser.add_argument("--group", type=int, default=0)
parser.add_argument("--tod-debug", action="store_true")
parser.add_argument("--prepost", action="store_true")
parser.add_argument("--define-planets", type=str, default=None)
args = parser.parse_args()
if args.dump_config:
print config.to_str()
sys.exit(0)
dtype = np.float32 if config.get("map_bits") == 32 else np.float64
comm = mpi.COMM_WORLD
nmax = config.get("map_cg_nmax")
ext = config.get("map_format")
tshape= (720,720)
#tshape= (100,100)
resume= config.get("resume")
if args.define_planets:
for pdesc in args.define_planets.split(","):
name, elemfile = pdesc.split(":")
ephemeris.register_object(name, ephemeris.read_object(elemfile))
def parse_src_handling():
res = {}
res["mode"] = config.get("src_handling")
config.default("verbosity", 1, "Verbosity for output. Higher means more verbose. 0 outputs only errors etc. 1 outputs INFO-level and 2 outputs DEBUG-level messages.")
config.default("map_format", "fits", "File format to use when writing maps. Can be 'fits', 'fits.gz' or 'hdf'.")
config.default("tod_window", 5.0, "Number of samples to window the tod by on each end")
parser = config.ArgumentParser(os.environ["HOME"] + "/.enkirc")
parser.add_argument("sel")
parser.add_argument("area")
parser.add_argument("odir")
parser.add_argument("prefix",nargs="?")
parser.add_argument("--ndet", type=int, default=0, help="Max number of detectors")
args = parser.parse_args()
filedb.init()
utils.mkdir(args.odir)
root = args.odir + "/" + (args.prefix + "_" if args.prefix else "")
log_level = log.verbosity2level(config.get("verbosity"))
dtype = np.float32 if config.get("map_bits") == 32 else np.float64
area = enmap.read_map(args.area)
comm = mpi.COMM_WORLD
ids = filedb.scans[args.sel]
L = log.init(level=log_level, rank=comm.rank)
# Set up our output map.
osig = enmap.zeros((1,)+area.shape[-2:], area.wcs, dtype)
odiv = osig*0
sig_all = np.zeros(len(ids))
sig_med = sig_all*0
div_all, div_med = sig_all*0, sig_med*0
# Read in all our scans
for ind in range(comm.rank, len(ids), comm.size):
def __init__(self, scan, srcs, sys=None, tmul=None, pmul=None):
# We support a srcs which is either [nsrc,nparam], [nsrc,ndir,nparam] or [nsrc,ndir,ndet,nparam], where
# ndir is either 1 or 2 depending on whether one wants to separate different
# scanning directions.
srcs = np.array(srcs)
while srcs.ndim < 4:
srcs = srcs[:, None]
# srcs is [nsrc,ndir,ndet_or_1,{dec,ra,T,Q,U,ibeams}]
sys = config.get("map_sys", sys)
cres = config.get("pmat_ptsrc_cell_res") * utils.arcmin
nsrc, ndir, src_ndet = srcs.shape[:3]
self.scan = scan
maxcell = 50 # max numer of sources per cell
# Compute parallax displacement if necessary
sundist = config.get("pmat_parallax_au")
self.dpos = 0
if sundist:
# Transformation to a sun-centered system
self.dpos = parallax.earth2sun(srcs.T[:2],
self.scan.mjd0,
sundist,
diff=True).T
srcs[..., :2] += self.dpos
# Investigate the beam to find the max relevant radius
sigma_lim = config.get("pmat_ptsrc_rsigma")
value_lim = np.exp(-0.5 * sigma_lim**2)
rmax = np.where(scan.beam[1] >= value_lim)[0][-1] * scan.beam[0, 1]
rmul = max([
utils.expand_beam(src[-3:])[0][0]
for src in srcs.reshape(-1, srcs.shape[-1])
])
rmax *= rmul
# Build interpolator (dec,ra output ordering)
transform = build_pos_transform(scan, sys=config.get("map_sys", sys))
ipol, obox, err = build_interpol(transform,
scan.box,
scan.entry.id,
posunit=0.5 * utils.arcsec)
self.rbox, self.nbox, self.yvals = extract_interpol_params(
ipol, srcs.dtype)
self.cbox = obox[:, :2]
self.ref = np.mean(self.cbox, 0)
self.ncell, self.cells = pointsrcs.build_src_cells(self.cbox,
srcs[..., :2],
cres,
unwind=True)
## Build source hit grid
#cbox = obox[:,:2]
#cshape = tuple(np.ceil(((cbox[1]-cbox[0])/cres)).astype(int))
#self.ref = np.mean(cbox,0)
#srcs[...,:2] = utils.rewind(srcs[...,:2], self.ref)
## A cell is hit if it overlaps both horizontally and vertically
## with the point source +- rmax
## ncell is [ndir,nsrc_det,ncy,ncx]
#ncell = np.zeros((ndir,src_ndet)+cshape,dtype=np.int32)
## cells is [ndir,nsrc_det,ncy,ncx,maxcell]
#cells = np.zeros((ndir,src_ndet)+cshape+(maxcell,),dtype=np.int32)
#c0 = cbox[0]; inv_dc = cshape/(cbox[1]-cbox[0])
#for si in range(nsrc):
# for sdir in range(ndir):
# for sdi in range(src_ndet):
# src = srcs[si,sdir,sdi]
# i1 = (src[:2]-rmax-c0)*inv_dc
# i2 = (src[:2]+rmax-c0)*inv_dc+1 # +1 because this is a half-open interval
# # Truncate to edges - any source outside of our region
# # will be put on one of the edge cells
# i1 = np.maximum(i1.astype(int), 0)
# i2 = np.minimum(i2.astype(int), np.array(cshape)-1)
# #print si, sdir, i1, i2, cshape
# if np.any(i1 >= cshape) or np.any(i2 < 0): continue
# sel= (sdir,sdi,slice(i1[0],i2[0]),slice(i1[1],i2[1]))
# cells[sel][:,:,ncell[sel]] = si
# ncell[sel] += 1
#self.cells, self.ncell = cells, ncell
#print self.cells.shape, self.ncell.shape
self.rmax = rmax
self.tmul = 1 if tmul is None else tmul
self.pmul = 1 if pmul is None else pmul
self.err = err
def __init__(self, entry, subdets=None, d=None, verbose=False, dark=False):
self.fields = ["gain","mce_filter","tags","polangle","tconst","hwp","cut","point_offsets","boresight","site","tod_shape","array_info","beam","pointsrcs", "buddies"]
if dark: self.fields += ["dark"]
if config.get("noise_model") == "file":
self.fields += ["noise"]
else:
if config.get("cut_noise_whiteness"):
self.fields += ["noise_cut"]
if config.get("cut_spikes"):
self.fields += ["spikes"]
if d is None:
d = actdata.read(entry, self.fields, verbose=verbose)
d = actdata.calibrate(d, verbose=verbose)
if subdets is not None:
d.restrict(dets=d.dets[subdets])
if d.ndet == 0 or d.nsamp == 0: raise errors.DataMissing("No data in scan")
ndet = d.ndet
# Necessary components for Scan interface
self.mjd0 = utils.ctime2mjd(d.boresight[0,0])
self.boresight = np.ascontiguousarray(d.boresight.T.copy()) # [nsamp,{t,az,el}]
self.boresight[:,0] -= self.boresight[0,0]
self.offsets = np.zeros([ndet,self.boresight.shape[1]])
self.offsets[:,1:] = d.point_offset
self.cut = d.cut.copy()
self.cut_noiseest = d.cut_noiseest.copy()
self.comps = np.zeros([ndet,4])
self.beam = d.beam
self.pointsrcs = d.pointsrcs
self.comps = d.det_comps
self.hwp = d.hwp
self.hwp_phase = d.hwp_phase
self.dets = d.dets
self.dgrid = (d.array_info.nrow, d.array_info.ncol)
self.array_info = d.array_info
self.sys = config.get("tod_sys")
self.site = d.site
self.speed = d.speed
if "noise" in d:
self.noise = d.noise
else:
spikes = d.spikes[:2].T if "spikes" in d else None
self.noise = nmat_measure.NmatBuildDelayed(model = config.get("noise_model"), spikes=spikes,
cut=self.cut_noiseest)
if "dark_tod" in d:
self.dark_tod = d.dark_tod
if "dark_cut" in d:
self.dark_cut = d.dark_cut
if "buddy_comps" in d:
# Expand buddy_offs to {dt,daz,ddec}
self.buddy_comps = d.buddy_comps
self.buddy_offs = np.concatenate([d.buddy_offs[...,:1]*0,d.buddy_offs],-1)
self.autocut = d.autocut if "autocut" in d else []
# Implementation details. d is our DataSet, which we keep around in
# because we need it to read tod consistently later. It will *not*
# take part in any sample slicing operations, as that might make the
# delayed tod read inconsistent with the rest. It could take part in
# detector slicing as long as calibrate_tod operates on each detector
# independently. This is true now, but would not be so if we did stuff
# like common mode subtraction there. On the other hand, not doing this
# would prevent slicing before reading from giving any speedup or memory
# savings. I don't think allowing this should be a serious problem.
self.d = d
self.entry = entry
def fmt_id(entry):
if isinstance(entry, list): return "+".join([fmt_id(e) for e in entry])
else:
if entry.tag: return entry.id + ":" + entry.tag
else: return entry.id
self.id = fmt_id(entry)
self.sampslices = []
self.mapping = None
# FIXME: debug test
if config.get("dummy_cut") > 0:
nmax = int(config.get("dummy_cut_len"))
# Power law between 1 and nmax, with slope -1.
# C(w) = log(w)/log(nmax)
# P(w) = w**-1/log(nmax)
# w(C) = n**C
# Mean: (nmax-1)/log(nmax)
nmean = (nmax-1)/np.log(nmax)
ncut = int(self.nsamp * config.get("dummy_cut") / nmean)
cut_ranges = np.zeros([self.ndet, ncut, 2],int)
w = (nmax**np.random.uniform(0, 1, size=[self.ndet, ncut])).astype(int)
np.clip(w, 1, nmax)
cut_ranges[:,:,0] = np.random.uniform(0, self.nsamp, size=[self.ndet, ncut]).astype(int)
cut_ranges[:,:,0] = np.sort(cut_ranges[:,:,0],1)
cut_ranges[:,:,1] = cut_ranges[:,:,0] + w
np.clip(cut_ranges[:,:,1], 0, self.nsamp)
cut_dummy = sampcut.from_list(cut_ranges, self.nsamp)
print np.mean(w), nmean, nmax, ncut
print "cut fraction before", float(self.cut.sum())/self.cut.size
self.cut *= cut_dummy
print "cut fraction after", float(self.cut.sum())/self.cut.size
parser.add_argument("--p0", type=int, default=0)
parser.add_argument("-g,", "--group", type=int, default=1)
parser.add_argument("--dedark", action="store_true")
parser.add_argument("--demode", action="store_true")
parser.add_argument("--decommon", action="store_true")
parser.add_argument("-c", "--cont", action="store_true")
args = parser.parse_args()
filedb.init()
comm_world = mpi.COMM_WORLD
comm_group = comm_world.Split(comm_world.rank%args.nsub, comm_world.rank/args.nsub)
comm_sub = comm_world.Split(comm_world.rank/args.nsub, comm_world.rank%args.nsub)
ids = todinfo.get_tods(args.sel, filedb.scans)
tol = args.tol*utils.arcmin
daz = args.daz*utils.arcmin
dtype = np.float32 if config.get("map_bits") == 32 else np.float64
tods_per_map = args.group
utils.mkdir(args.odir)
root = args.odir + "/" + (args.prefix + "_" if args.prefix else "")
# Set up logging
utils.mkdir(root + "log")
logfile = root + "log/log%03d.txt" % comm_world.rank
log_level = log.verbosity2level(config.get("verbosity"))
L = log.init(level=log_level, file=logfile, rank=comm_world.rank, shared=False)
# Run through all tods to determine the scanning patterns
L.info("Detecting scanning patterns")
boxes = np.zeros([len(ids),2,2])
for ind in range(comm_world.rank, len(ids), comm_world.size):
id = ids[ind]
def autocut(d, turnaround=None, ground=None, sun=None, moon=None, max_frac=None, pickup=None):
"""Apply automatic cuts to calibrated data."""
if not config.get("autocut"): return d
ndet, nsamp = d.ndet, d.nsamp
if not ndet or not nsamp: return d
# Insert a cut into d if necessary
if "cut" not in d:
d += dataset.DataField("cut", sampcut.empty(ndet,nsamp))
# insert an autocut datafield, to keep track of how much data each
# automatic cut cost us
d += dataset.DataField("autocut", [])
def addcut(label, dcut, targets="c"):
# det ndet part here allows for broadcasting of cuts from 1-det to full-det
dn = dcut.sum()*d.ndet/dcut.ndet if dcut is not None else 0
if dn == 0: d.autocut.append([label,0,0])
else:
n0, dn = d.cut.sum(), dcut.sum()
dn = dn*d.cut.ndet/dcut.ndet
if "c" in targets: d.cut *= dcut
if "n" in targets: d.cut_noiseest *= dcut
if "b" in targets: d.cut_basic *= dcut
d.autocut.append([ label, dn, d.cut.sum() - n0 ]) # name, mycut, myeffect
if config.get("cut_tconst") and "tau" in d:
addcut("tconst", cuts.tconst_cut(nsamp, d.tau, config.get("cut_tconst")))
if config.get("cut_stationary") and "boresight" in d:
addcut("stationary", cuts.stationary_cut(d.boresight[1]))
if config.get("cut_tod_ends") and "srate" in d:
addcut("tod_ends", cuts.tod_end_cut(nsamp, d.srate))
if config.get("cut_turnaround", turnaround) and "boresight" in d:
addcut("turnaround",cuts.turnaround_cut(d.boresight[1]))
if config.get("cut_ground", ground) and "boresight" in d and "point_offset" in d:
addcut("ground", cuts.ground_cut(d.boresight, d.point_offset))
if config.get("cut_sun", sun) and "boresight" in d and "point_offset" in d and "site" in d:
addcut("avoidance",cuts.avoidance_cut(d.boresight, d.point_offset, d.site, "Sun", config.get("cut_sun_dist")*np.pi/180))
if config.get("cut_moon", moon) and "boresight" in d and "point_offset" in d and "site" in d:
addcut("moon",cuts.avoidance_cut(d.boresight, d.point_offset, d.site, "Moon", config.get("cut_moon_dist")*np.pi/180))
if config.get("cut_pickup", pickup) and "boresight" in d and "pickup_cut" in d:
addcut("pickup",cuts.pickup_cut(d.boresight[1], d.dets, d.pickup_cut))
if config.get("cut_mostly_cut"):
addcut("mostly_cut", cuts.cut_mostly_cut_detectors(d.cut_quality))
if config.get("cut_obj"):
objs = utils.split_outside(config.get("cut_obj"),",")
for obj in objs:
toks = obj.split(":")
objname = toks[0]
if objname.startswith("["):
objname = [float(w)*utils.degree for w in objname[1:-1].split(",")]
dist = 0.2*utils.degree
if len(toks) > 1: dist = float(toks[1])*utils.degree
# Hack: only cut for noise estimation purposes if dist is negative
targets = "cnb" if dist > 0 else "n"
addcut(obj, cuts.avoidance_cut(d.boresight, d.point_offset, d.site, objname, dist), targets=targets)
if config.get("cut_srcs"):
cpar = [tok.split(":") for tok in config.get("cut_srcs").split(",")]
names, lims = [], []
for par in cpar:
if par[0] in ["map","nmat"]:
names.append(par[0])
lims.append(float(par[1]))
if any(lims):
params = pointsrcs.src2param(d.pointsrcs)
params[:,5:7] = 1
params[:,7] = 0
# Only bother with sources that are actually strong enough
maxlim = max(lims+[0])
params = params[params[:,2]>maxlim]
cutlist = cuts.point_source_cut(d, params, lims)
for name, c in zip(names, cutlist):
if name == "map": addcut("point_srcs_m", c, "c")
elif name == "nmat": addcut("point_srcs_n", c, "n")
if config.get("cut_extra_srcs"):
srclist = np.loadtxt(config.get("cut_extra_srcs"), usecols=(0,1), ndmin=2)
srclim = float(config.get("cut_extra_lim"))
params = np.zeros([len(srclist),8])
params[:,:2] = srclist[:,1::-1]*utils.degree
params[:,2] = 1
params[:,5:7] = 1
c = cuts.point_source_cut(d, params, srclim)
addcut("point_srcs", c, "nbc")
# What fraction is cut?
cut_fraction = float(d.cut.sum())/d.cut.size
# Get rid of completely cut detectors
keep = np.where(d.cut.sum(axis=1) < nsamp)[0]
d.restrict(d.dets[keep])
ndet, nsamp = d.ndet, d.nsamp
def cut_all_if(label, condition):
if condition: dcut = sampcut.full(d.ndet, nsamp)
else: dcut = None
addcut(label, dcut)
cut_all_if("max_frac", config.get("cut_max_frac", max_frac) < cut_fraction)
if "srate" in d:
cut_all_if("tod_mindur", config.get("cut_tod_mindur") > nsamp/d.srate/60)
cut_all_if("tod_mindet", config.get("cut_tod_mindet") > ndet)
# Get rid of completely cut detectors again
keep = np.where(d.cut.sum(axis=1) < nsamp)[0]
d.restrict(dets=d.dets[keep])
return d
