def fit_phase_flat(tods, az, daz=1*utils.arcmin, cuts=None, niter=None, overlap=None, clean_tod=False, weight=None): # for the given tods[ndet,nsamp], cuts (multirange[ndet,nsamp]) and az[nsamp], if not clean_tod: tods = tods.copy() if daz is None: daz = 1*utils.arcmin if niter is None: niter = 3 if weight is None: weight = np.full(len(tods), 1.0, dtype=tods.dtype) elif weight is "auto": weight = 1/estimate_white_noise(tods) weight /= np.mean(weight) # Set up phase pixels amin = np.min(az) amax = np.max(az) naz = int((amax-amin)/daz)+1 pflat = pmat.PmatPhaseFlat(az, amin, daz, naz) # Output and work arrays phase = np.zeros((2,naz),tods.dtype) dphase = phase.copy() div = phase.copy() # Precompute div pflat.backward(tods*0+weight[:,None], div, -1) div[div==0] = 1 print np.mean(div) for i in range(niter): # Overall logic: gapfill -> bin -> subtract -> loop if cuts is not None: gapfill.gapfill_linear(tods, cuts, overlap=overlap, inplace=True) pflat.backward(tods*weight[:,None], dphase) dphase /= div phase += dphase pflat.forward(tods, -dphase) return phase
def calibrate_boresight(data):
"""Calibrate the boresight by converting to radians and
interpolating across missing samples linearly. Note that
this won't give reasonable results for gaps of length
similar to the scan period. Also adds a srate field containing
the sampling rate."""
require(data, ["boresight","flags"])
# Convert angles to radians
if data.nsamp in [0, None]: raise errors.DataMissing("nsamp")
if data.nsamp < 0: raise errors.DataMissing("nsamp")
a = data.boresight[1].copy()
data.boresight[1] = robust_unwind(data.boresight[1], period=360, tol=0.1)
data.boresight[1:]*= np.pi/180
#data.boresight[1:] = utils.unwind(data.boresight[1:] * np.pi/180)
# Find unreliable regions
bad_flag = (data.flags!=0)*(data.flags!=0x10)
bad_value = find_boresight_jumps(data.boresight)
bad_value |= find_elevation_outliers(data.boresight[2])
bad = bad_flag | bad_value
#bad += srate_mask(data.boresight[0])
# Interpolate through bad regions. For long regions, this won't
# work, so these should be cut.
# 1. Raise an exception
# 2. Construct a cut on the fly
# 3. Handle it in the autocuts.
# The latter is cleaner in my opinion
cut = sampcut.from_mask(bad)
gapfill.gapfill_linear(data.boresight, cut, inplace=True)
srate = 1/utils.medmean(data.boresight[0,1:]-data.boresight[0,:-1])
data += dataset.DataField("srate", srate)
# Get the scanning speed too
speed = calc_scan_speed(data.boresight[0], data.boresight[1])
data += dataset.DataField("speed", speed)
return data
def fit_common(tods,
cuts=None,
niter=None,
overlap=None,
clean_tod=False,
weight=None):
# for the given tods[ndet,nsamp], cuts (multirange[ndet,nsamp]) and az[nsamp],
if not clean_tod: tods = tods.copy()
if niter is None: niter = 3
if weight is None:
weight = np.full(len(tods), 1.0, dtype=tods.dtype)
elif weight is "auto":
weight = 1 / estimate_white_noise(tods)
weight /= np.mean(weight)
# Output and work arrays
res = tods[0] * 0
div = np.sum(tods * 0 + weight[:, None], 0)
for i in range(niter):
# Overall logic: gapfill -> bin -> subtract -> loop
if cuts is not None:
gapfill.gapfill_linear(tods, cuts, overlap=overlap, inplace=True)
delta = np.sum(tods * weight[:, None], 0)
delta /= div
res += delta
tods -= delta[None]
return res
def deglitch(tod, nsigma=10, width=15, padding=7, inplace=False): spikes = find_spikes(tod) return gapfill.gapfill_linear(tod, spikes, inplace=inplace)
elif args.dets is not None:
subdets = [int(w) for w in args.dets.split(",")]
fields = ["gain","tconst","cut","tod","boresight"]
if args.fields: fields = args.fields.split(",")
d = actdata.read(entry, fields=fields)
if absdets: d.restrict(dets=absdets)
if subdets: d.restrict(dets=d.dets[subdets])
if args.calib: d = actdata.calibrate(d, exclude=["autocut"])
elif args.manual_calib:
ops = args.manual_calib.split(",")
if "safe" in ops: d.boresight[1:] = utils.unwind(d.boresight[1:], period=360)
if "rad" in ops: d.boresight[1:] *= np.pi/180
if "bgap" in ops:
bad = (d.flags!=0)*(d.flags!=0x10)
for b in d.boresight: gapfill.gapfill_linear(b, bad, inplace=True)
if "gain" in ops: d.tod *= d.gain[:,None]
if "tgap" in ops:
gapfiller = {"copy":gapfill.gapfill_copy, "linear":gapfill.gapfill_linear}[config.get("gapfill")]
gapfiller(d.tod, d.cut, inplace=True)
if "slope" in ops:
utils.deslope(d.tod, w=8, inplace=True)
if "deconv" in ops:
d = actdata.calibrate(d, operations=["tod_fourier"])
if args.bin > 1:
d.tod = resample.downsample_bin(d.tod, steps=[args.bin])
d.boresight = resample.downsample_bin(d.boresight, steps=[args.bin])
d.flags = resample.downsample_bin(d.flags, steps=[args.bin])
oname = args.ofile
if len(ids) > 1: oname = "%s/%s.hdf" % (args.ofile, id)
with h5py.File(oname, "w") as hfile:
fields = ["gain", "tconst", "cut", "tod", "boresight"]
if args.fields: fields = args.fields.split(",")
d = actdata.read(entry, fields=fields)
if absdets: d.restrict(dets=absdets)
if subdets: d.restrict(dets=d.dets[subdets])
if args.calib: d = actdata.calibrate(d, exclude=["autocut"])
elif args.manual_calib:
ops = args.manual_calib.split(",")
if "safe" in ops:
d.boresight[1:] = utils.unwind(d.boresight[1:], period=360)
if "rad" in ops: d.boresight[1:] *= np.pi / 180
if "bgap" in ops:
bad = (d.flags != 0) * (d.flags != 0x10)
for b in d.boresight:
gapfill.gapfill_linear(b, bad, inplace=True)
if "gain" in ops: d.tod *= d.gain[:, None]
if "tgap" in ops:
gapfiller = {
"copy": gapfill.gapfill_copy,
"linear": gapfill.gapfill_linear
}[config.get("gapfill")]
gapfiller(d.tod, d.cut, inplace=True)
if "slope" in ops:
utils.deslope(d.tod, w=8, inplace=True)
if "deconv" in ops:
d = actdata.calibrate(d, operations=["tod_fourier"])
if args.bin > 1:
d.tod = resample.downsample_bin(d.tod, steps=[args.bin])
if "boresight" in d:
d.boresight = resample.downsample_bin(d.boresight,
psrc.forward(wtod, src_rhs * 0 + 1, tmul=0)))))
src_div = np.maximum(src_div, 0)
# Bias slightly towards input value. This helps avoid problems with sources
# that are hit by only a handful of samples, and are therefore super-uncertain.
src_amp_old = scan.srcparam[:, 2]
src_div_old = planet9.defmean(src_div[src_div > 0],
1e-5) * amp_prior
src_rhs += src_amp_old * src_div_old
src_div += src_div_old
with utils.nowarn():
src_amp = src_rhs / src_div
src_amp[~np.isfinite(src_amp)] = 0
# And subtract the point sources
psrc.forward(wtod, src_amp, tmul=0)
# Keep our gapfilling semi-consistent
gapfill.gapfill_linear(wtod, scan.cut, inplace=True)
scan.noise.apply(wtod)
tod -= wtod
del wtod, src_rhs, src_div
# Build our rhs map
apply_cut(tod)
signal.backward(scan, tod, work, mmul=0)
wrhs += work[0]
if bleh: sim_rhs += psim.backward(tod)
# Build our div map
work[:] = 0
work[0] = 1
signal.forward(scan, tod, work, tmul=0)
scan.noise.white(tod)
apply_cut(tod)
def filter_poly_jon(tod, az, weights=None, naz=None, nt=None, niter=None, cuts=None, hwp=None, nhwp=None, deslope=True, inplace=True, use_phase=None):
"""Fix naz Legendre polynomials in az and nt other polynomials
in t jointly. Then subtract the best fit from the data.
The subtraction is inplace, so tod is modified. If naz or nt are
negative, they are fit for, but not subtracted.
NOTE: This function may leave tod nonperiodic.
"""
naz = config.get("gfilter_jon_naz", naz)
nt = config.get("gfilter_jon_nt", nt)
nhwp= config.get("gfilter_jon_nhwp", nhwp)
niter = config.get("gfilter_jon_niter", niter)
use_phase = config.get("gfilter_jon_phase", use_phase)
if not inplace: tod = tod.copy()
do_gapfill = cuts is not None
# No point in iterating if we aren't gapfilling
if not do_gapfill: niter = 1
if hwp is None or np.all(hwp==0): nhwp = 0
naz, asign = np.abs(naz), np.sign(naz)
nt, tsign = np.abs(nt), np.sign(nt)
nhwp,hsign = np.abs(nhwp),np.sign(nhwp)
d = tod.reshape(-1,tod.shape[-1])
nsamp = d.shape[-1]
if naz == 0 and nt == 0 and nhwp == 0: return tod
B = []
# Set up our time basis
if nt > 0:
t = np.linspace(-1,1,nsamp,endpoint=False)
B.append(utils.build_legendre(t, nt))
if naz > 0:
if not use_phase:
# Set up our azimuth basis
B.append(utils.build_legendre(az, naz+1)[1:])
else:
# Set up phase basis. Vectors should be periodic
# in phase to avoid discontinuities. cossin is good for this.
phase = build_phase(az)*np.pi
B.append(utils.build_cossin(phase, naz))
if nhwp > 0:
B.append(utils.build_cossin(hwp, nhwp))
B = np.concatenate(B,0)
for it in range(niter):
if do_gapfill: gapfill.gapfill_linear(d, cuts, inplace=True)
# Solve for the best fit for each detector, [nbasis,ndet]
# B[b,n], d[d,n], amps[b,d]
if weights is None:
try:
amps = np.linalg.solve(B.dot(B.T),B.dot(d.T))
except np.linalg.LinAlgError as e:
print "LinAlgError in todfilter. Skipping"
continue
else:
w = weights.reshape(-1,weights.shape[-1])
amps = np.zeros([naz+nt+nhwp,d.shape[0]],dtype=tod.dtype)
for di in range(len(tod)):
try:
amps[:,di] = np.linalg.solve((B*w[di]).dot(B.T),B.dot(w[di]*d[di]))
except np.linalg.LinAlgError as e:
print "LinAlgError in todfilter di %d. Skipping" % di
continue
# Subtract the best fit, but skip some basis functions if requested
if tsign < 0: amps[:nt] = 0
if asign < 0: amps[nt:nt+naz] = 0
if hsign < 0: amps[nt+naz:nt+naz+nhwp] = 0
d -= amps.T.dot(B)
if do_gapfill: gapfill.gapfill(d, cuts, inplace=True)
# This filtering might have casued the tod to become
# non-continous, which would mess up future fourier transforms.
# So it's safest to deslope here.
if deslope: utils.deslope(tod, w=8, inplace=True)
res = d.reshape(tod.shape)
return res