def update(self, tod, srate):
# If we have noise estimation cuts, we must gapfill these
# before measuring the noise, and restore them afterwards
if self.cut is not None:
vals = self.cut.extract_samples(tod)
gapfill.gapfill(tod, self.cut, inplace=True)
try:
if self.model == "jon":
ft = fft.rfft(tod) * tod.shape[1]**-0.5
noise_model = detvecs_jon(ft, srate, cut_bins=self.spikes)
elif self.model == "uncorr":
ft = fft.rfft(tod) * tod.shape[1]**-0.5
noise_model = detvecs_simple(ft, srate)
elif self.model == "white":
noise_model = nmat.NoiseMatrix(len(tod))
elif self.model == "scaled":
ft = fft.rfft(tod) * tod.shape[1]**-0.5
noise_model = detvecs_scaled(ft, srate)
else:
raise ValueError("Unknown noise model '%s'" % self.model)
except (errors.ModelError, np.linalg.LinAlgError, AssertionError) as e:
print("Warning: Noise model fit failed for tod with shape %s. Assigning zero weight" % str(tod.shape))
noise_model = nmat.NmatNull(np.arange(tod.shape[0]))
if self.cut is not None:
self.cut.insert_samples(tod, vals)
return noise_model
def update(self, tod, srate):
# If we have noise estimation cuts, we must gapfill these
# before measuring the noise, and restore them afterwards
if self.cut is not None:
vals = self.cut.extract_samples(tod)
gapfill.gapfill(tod, self.cut, inplace=True)
try:
if self.model == "jon":
ft = fft.rfft(tod) * tod.shape[1]**-0.5
noise_model = detvecs_jon(ft, srate, cut_bins=self.spikes)
elif self.model == "uncorr":
ft = fft.rfft(tod) * tod.shape[1]**-0.5
noise_model = detvecs_simple(ft, srate)
elif self.model == "white":
noise_model = nmat.NoiseMatrix(len(tod))
elif self.model == "scaled":
ft = fft.rfft(tod) * tod.shape[1]**-0.5
noise_model = detvecs_scaled(ft, srate)
else:
raise ValueError("Unknown noise model '%s'" % self.model)
except (errors.ModelError, np.linalg.LinAlgError, AssertionError) as e:
print "Warning: Noise model fit failed for tod with shape %s. Assigning zero weight" % str(tod.shape)
noise_model = nmat.NmatNull(np.arange(tod.shape[0]))
if self.cut is not None:
self.cut.insert_samples(tod, vals)
return noise_model
def fit_basis(tods, basis, highpass=50, cuts=None, clean_tod=True): if not clean_tod: tods = tods.copy() def hpass(a, n): f = fft.rfft(a) f[...,:n] = 0 return fft.ifft(f,a.copy(),normalize=True) hdark = hpass(basis, highpass) for di in range(len(tods)): htod = hpass(tods[di], highpass) dark_tmp = hdark.copy() if cuts is not None: for ddi in range(len(hdark)): gapfill.gapfill(dark_tmp[ddi], cuts[di], inplace=True) fit = project(htod[None], dark_tmp)[0] # Subtract from original tod tods[di] -= fit return tods
def __call__(self, scan, tod):
gapfill.gapfill(tod, scan.cut, inplace=True)
def filter_poly_jon(tod, az, weights=None, naz=None, nt=None, niter=None, cuts=None, hwp=None, nhwp=None, deslope=True, inplace=True):
"""Fix naz Legendre polynomials in az and nt other polynomials
in t jointly. Then subtract the best fit from the data.
The subtraction is inplace, so tod is modified. If naz or nt are
negative, they are fit for, but not subtracted.
NOTE: This function may leave tod nonperiodic.
"""
#moomoo = tod[:8].copy()
naz = config.get("gfilter_jon_naz", naz)
nt = config.get("gfilter_jon_nt", nt)
nhwp= config.get("gfilter_jon_nhwp", nhwp)
niter = config.get("gfilter_jon_niter", niter)
if not inplace: tod = tod.copy()
do_gapfill = cuts is not None
#print "Mos", naz, nt, nhwp
#print hwp
# No point in iterating if we aren't gapfilling
if not do_gapfill: niter = 1
if hwp is None or np.all(hwp==0): nhwp = 0
naz, asign = np.abs(naz), np.sign(naz)
nt, tsign = np.abs(nt), np.sign(nt)
nhwp,hsign = np.abs(nhwp),np.sign(nhwp)
d = tod.reshape(-1,tod.shape[-1])
if naz == 0 and nt == 0 and nhwp == 0: return tod
# Build our set of basis functions. These are shared
# across iterations.
B = np.zeros([naz+nt+nhwp,d.shape[-1]],dtype=tod.dtype)
if naz > 0:
# Build azimuth basis as polynomials
x = utils.rescale(az,[-1,1])
for i in range(naz): B[i] = x**(i+1)
if nt > 0:
x = np.linspace(-1,1,d.shape[-1],endpoint=False)
for i in range(nt): B[naz+i] = x**i
if nhwp > 0:
# Use sin and cos to avoid discontinuities
c = np.cos(hwp)
s = np.sin(hwp)
for i in range(nhwp):
j = i/2+1
x = np.cos(j*hwp) if i%2 == 0 else np.sin(j*hwp)
B[naz+nt+i] = x
for it in range(niter):
if do_gapfill: gapfill.gapfill(d, cuts, inplace=True)
# Solve for the best fit for each detector, [nbasis,ndet]
# B[b,n], d[d,n], amps[b,d]
if weights is None:
try:
amps = np.linalg.solve(B.dot(B.T),B.dot(d.T))
except np.linalg.LinAlgError as e:
print "LinAlgError in todfilter. Skipping"
continue
else:
w = weights.reshape(-1,weights.shape[-1])
amps = np.zeros([naz+nt+nhwp,d.shape[0]],dtype=tod.dtype)
for di in range(len(tod)):
try:
amps[:,di] = np.linalg.solve((B*w[di]).dot(B.T),B.dot(w[di]*d[di]))
except np.linalg.LinAlgError as e:
print "LinAlgError in todfilter di %d. Skipping" % di
continue
#print "amps", amps[:,2]
# Subtract the best fit
if asign > 0: d -= amps[:naz].T.dot(B[:naz])
if tsign > 0: d -= amps[naz:naz+nt].T.dot(B[naz:naz+nt])
if hsign > 0: d -= amps[naz+nt:naz+nt+nhwp].T.dot(B[naz+nt:naz+nt+nhwp])
# Why was this necessary?
if do_gapfill: gapfill.gapfill(d, cuts, inplace=True)
if deslope: utils.deslope(tod, w=8, inplace=True)
res = d.reshape(tod.shape)
return res
def filter_poly_jon(tod,
az,
weights=None,
naz=None,
nt=None,
niter=None,
cuts=None,
hwp=None,
nhwp=None,
deslope=True,
inplace=True,
use_phase=None):
"""Fix naz Legendre polynomials in az and nt other polynomials
in t jointly. Then subtract the best fit from the data.
The subtraction is inplace, so tod is modified. If naz or nt are
negative, they are fit for, but not subtracted.
NOTE: This function may leave tod nonperiodic.
"""
naz = config.get("gfilter_jon_naz", naz)
nt = config.get("gfilter_jon_nt", nt)
nhwp = config.get("gfilter_jon_nhwp", nhwp)
niter = config.get("gfilter_jon_niter", niter)
use_phase = config.get("gfilter_jon_phase", use_phase)
if not inplace: tod = tod.copy()
do_gapfill = cuts is not None
# No point in iterating if we aren't gapfilling
if not do_gapfill: niter = 1
if hwp is None or np.all(hwp == 0): nhwp = 0
naz, asign = np.abs(naz), np.sign(naz)
nt, tsign = np.abs(nt), np.sign(nt)
nhwp, hsign = np.abs(nhwp), np.sign(nhwp)
d = tod.reshape(-1, tod.shape[-1])
nsamp = d.shape[-1]
if naz == 0 and nt == 0 and nhwp == 0: return tod
B = []
# Set up our time basis
if nt > 0:
B.append(np.full((1, nsamp), 1.0, d.dtype))
if nt > 1:
t = np.linspace(-1, 1, nsamp, endpoint=False)
B.append(utils.build_legendre(t, nt - 1))
if naz > 0:
if not use_phase:
# Set up our azimuth basis
B.append(utils.build_legendre(az, naz))
else:
# Set up phase basis. Vectors should be periodic
# in phase to avoid discontinuities. cossin is good for this.
phase = build_phase(az) * np.pi
B.append(utils.build_cossin(phase, naz))
if nhwp > 0:
B.append(utils.build_cossin(hwp, nhwp))
B = np.concatenate(B, 0)
for it in range(niter):
if do_gapfill: gapfill.gapfill(d, cuts, inplace=True)
# Solve for the best fit for each detector, [nbasis,ndet]
# B[b,n], d[d,n], amps[b,d]
if weights is None:
try:
amps = np.linalg.solve(B.dot(B.T), B.dot(d.T))
except np.linalg.LinAlgError as e:
print "LinAlgError in todfilter. Skipping"
continue
else:
w = weights.reshape(-1, weights.shape[-1])
amps = np.zeros([naz + nt + nhwp, d.shape[0]], dtype=tod.dtype)
for di in range(len(tod)):
try:
amps[:, di] = np.linalg.solve((B * w[di]).dot(B.T),
B.dot(w[di] * d[di]))
except np.linalg.LinAlgError as e:
print "LinAlgError in todfilter di %d. Skipping" % di
continue
# Subtract the best fit, but skip some basis functions if requested
if tsign < 0: amps[:nt] = 0
if asign < 0: amps[nt:nt + naz] = 0
if hsign < 0: amps[nt + naz:nt + naz + nhwp] = 0
d -= amps.T.dot(B)
if do_gapfill: gapfill.gapfill(d, cuts, inplace=True)
# This filtering might have casued the tod to become
# non-continous, which would mess up future fourier transforms.
# So it's safest to deslope here.
if deslope: utils.deslope(tod, w=8, inplace=True)
res = d.reshape(tod.shape)
return res