def filter_phase_blockwise(tods,
blocks,
az,
daz=None,
cuts=None,
niter=None,
deslope=True,
inplace=True,
weight="auto"):
"""Given a tod[ndet,nsamp], fit for a common azimuth phase signal
per block in blocks[nblock][dets], and subtract it. The binning size
is given in arc minutes."""
# Loop over and filter each block
#np.savetxt("moo0.txt", tods[0])
if not inplace: tods = tods.copy()
for bi, block in enumerate(blocks):
btod = np.ascontiguousarray(tods[block])
bcut = None if cuts is None else cuts[block]
phase = todops.fit_phase_flat(btod,
az,
daz=daz,
cuts=bcut,
niter=niter,
clean_tod=True,
weight=weight)
tods[block] = btod
if deslope: utils.deslope(tods, w=8, inplace=True)
#np.savetxt("moo.txt", tods[0])
#1/0
return tods
def filter_common_blockwise(tods,
blocks,
cuts=None,
niter=None,
deslope=True,
inplace=True,
weight="auto",
nmin=5):
# Loop over and filter each block
#np.savetxt("comtod0.txt", tods[0])
if not inplace: tods = tods.copy()
for bi, block in enumerate(blocks):
if len(block) < nmin: continue
btod = np.ascontiguousarray(tods[block])
bcut = None if cuts is None else cuts[block]
common = todops.fit_common(btod,
cuts=bcut,
niter=niter,
clean_tod=True,
weight=weight)
tods[block] = btod
#if 0 in block: np.savetxt("comcom.txt", common)
if deslope: utils.deslope(tods, w=8, inplace=True)
#np.savetxt("comtod1.txt", tods[0])
#1/0
return tods
def deproject_vecs(tods, dark, nmode=50, cuts=None, deslope=True, inplace=True): """Given a tod[ndet,nsamp] and a set of basis modes dark[nmode,nsamp], fit each tod in to the basis modes and subtract them from the tod. The fit ignores the lowest nmode fourier modes, and cut regions are approximately ignored.""" if not inplace: tods=tods.copy() todops.fit_basis(tods, dark, highpass=nmode, cuts=cuts, clean_tod=True) if deslope: utils.deslope(tods, w=8, inplace=True) return tods
def deproject_vecs_smooth(tods, dark, nmode=50, cuts=None, deslope=True, inplace=True): if not inplace: tods=tods.copy() dark = dark.copy() ftod = fft.rfft(tods) fdark = fft.rfft(dark) fdark = todops.smooth_basis_fourier(ftod, fdark) smooth= np.zeros((fdark.shape[0],dark.shape[1]),dtype=dark.dtype) fft.ifft(fdark, smooth, normalize=True) todops.fit_basis(tods, smooth, highpass=nmode, cuts=cuts, clean_tod=True) if deslope: utils.deslope(tods, w=8, inplace=True)
def calibrate_tod_real(data, nthread=None):
"""Apply gain to tod, fill gaps and deslope. We only gapfill
data that's bad enough that it should be excluded when estimating
the noise model."""
require(data, ["tod","gain","cut_basic"])
if data.tod.size == 0: raise errors.DataMissing("No tod samples")
data.tod = data.tod.astype(np.int32, copy=False)
data.tod /= 128
data.tod = data.tod * (data.gain[:,None]*8)
gapfill_helper(data.tod, data.cut_basic)
utils.deslope(data.tod, w=8, inplace=True)
return data
def calibrate_dark_real(data):
"""Calibrate dark detectors. Mostly desloping."""
#require(data, ["dark_tod","dark_cut"])
require(data, ["dark_tod"])
if data.dark_tod.size == 0: return data
data.dark_tod = data.dark_tod * 1.0
dark_cut = todops.find_spikes(data.dark_tod)
gapfill_helper(data.dark_tod, dark_cut)
utils.deslope(data.dark_tod, w=8, inplace=True)
# Add the cuts to the dataset
data += dataset.DataField("dark_cut", dark_cut, sample_index=1,
samples=data.datafields["dark_tod"].samples)
return data
if args.spikecut: fields.append("spikes")
if args.imap: fields += ["polangle", "point_offsets", "site"]
d = data.read(entry, fields)
t.append(time.time())
d = data.calibrate(d)
t.append(time.time())
if args.imap:
# Make a full scan object, so we can perform pointing projection
# operations
d.noise = None
scan = data.ACTScan(entry, d=d)
imap.map = imap.map.astype(d.tod.dtype, copy=False)
pmap = pmat.PmatMap(scan, imap.map, sys=imap.sys)
# Subtract input map from tod inplace
pmap.forward(d.tod, imap.map, tmul=1, mmul=-1)
utils.deslope(d.tod, w=8, inplace=True)
ft = fft.rfft(d.tod) * d.tod.shape[1]**-0.5
t.append(time.time())
spikes = d.spikes[:2].T if args.spikecut else None
if model == "old":
noise = nmat_measure.detvecs_old(ft, d.srate, d.dets)
elif model == "jon":
noise = nmat_measure.detvecs_jon(ft,
d.srate,
d.dets,
shared,
cut_bins=spikes)
elif model == "simple":
noise = nmat_measure.detvecs_simple(ft, d.srate, d.dets)
elif model == "joint":
noise = nmat_measure.detvecs_joint(ft,
if absdets: d.restrict(dets=absdets)
if subdets: d.restrict(dets=d.dets[subdets])
if args.calib: d = actdata.calibrate(d, exclude=["autocut"])
elif args.manual_calib:
ops = args.manual_calib.split(",")
if "safe" in ops: d.boresight[1:] = utils.unwind(d.boresight[1:], period=360)
if "rad" in ops: d.boresight[1:] *= np.pi/180
if "bgap" in ops:
bad = (d.flags!=0)*(d.flags!=0x10)
for b in d.boresight: gapfill.gapfill_linear(b, bad, inplace=True)
if "gain" in ops: d.tod *= d.gain[:,None]
if "tgap" in ops:
gapfiller = {"copy":gapfill.gapfill_copy, "linear":gapfill.gapfill_linear}[config.get("gapfill")]
gapfiller(d.tod, d.cut, inplace=True)
if "slope" in ops:
utils.deslope(d.tod, w=8, inplace=True)
if "deconv" in ops:
d = actdata.calibrate(d, operations=["tod_fourier"])
if args.bin > 1:
d.tod = resample.downsample_bin(d.tod, steps=[args.bin])
d.boresight = resample.downsample_bin(d.boresight, steps=[args.bin])
d.flags = resample.downsample_bin(d.flags, steps=[args.bin])
oname = args.ofile
if len(ids) > 1: oname = "%s/%s.hdf" % (args.ofile, id)
with h5py.File(oname, "w") as hfile:
if "tod" in d: hfile["tod"] = d.tod
if "boresight" in d:
hfile["az"] = d.boresight[1]
hfile["el"] = d.boresight[2]
hfile["dets"] = d.dets
try:
def filter_poly_jon(tod, az, weights=None, naz=None, nt=None, niter=None, cuts=None, hwp=None, nhwp=None, deslope=True, inplace=True):
"""Fix naz Legendre polynomials in az and nt other polynomials
in t jointly. Then subtract the best fit from the data.
The subtraction is inplace, so tod is modified. If naz or nt are
negative, they are fit for, but not subtracted.
NOTE: This function may leave tod nonperiodic.
"""
#moomoo = tod[:8].copy()
naz = config.get("gfilter_jon_naz", naz)
nt = config.get("gfilter_jon_nt", nt)
nhwp= config.get("gfilter_jon_nhwp", nhwp)
niter = config.get("gfilter_jon_niter", niter)
if not inplace: tod = tod.copy()
do_gapfill = cuts is not None
#print "Mos", naz, nt, nhwp
#print hwp
# No point in iterating if we aren't gapfilling
if not do_gapfill: niter = 1
if hwp is None or np.all(hwp==0): nhwp = 0
naz, asign = np.abs(naz), np.sign(naz)
nt, tsign = np.abs(nt), np.sign(nt)
nhwp,hsign = np.abs(nhwp),np.sign(nhwp)
d = tod.reshape(-1,tod.shape[-1])
if naz == 0 and nt == 0 and nhwp == 0: return tod
# Build our set of basis functions. These are shared
# across iterations.
B = np.zeros([naz+nt+nhwp,d.shape[-1]],dtype=tod.dtype)
if naz > 0:
# Build azimuth basis as polynomials
x = utils.rescale(az,[-1,1])
for i in range(naz): B[i] = x**(i+1)
if nt > 0:
x = np.linspace(-1,1,d.shape[-1],endpoint=False)
for i in range(nt): B[naz+i] = x**i
if nhwp > 0:
# Use sin and cos to avoid discontinuities
c = np.cos(hwp)
s = np.sin(hwp)
for i in range(nhwp):
j = i/2+1
x = np.cos(j*hwp) if i%2 == 0 else np.sin(j*hwp)
B[naz+nt+i] = x
for it in range(niter):
if do_gapfill: gapfill.gapfill(d, cuts, inplace=True)
# Solve for the best fit for each detector, [nbasis,ndet]
# B[b,n], d[d,n], amps[b,d]
if weights is None:
try:
amps = np.linalg.solve(B.dot(B.T),B.dot(d.T))
except np.linalg.LinAlgError as e:
print "LinAlgError in todfilter. Skipping"
continue
else:
w = weights.reshape(-1,weights.shape[-1])
amps = np.zeros([naz+nt+nhwp,d.shape[0]],dtype=tod.dtype)
for di in range(len(tod)):
try:
amps[:,di] = np.linalg.solve((B*w[di]).dot(B.T),B.dot(w[di]*d[di]))
except np.linalg.LinAlgError as e:
print "LinAlgError in todfilter di %d. Skipping" % di
continue
#print "amps", amps[:,2]
# Subtract the best fit
if asign > 0: d -= amps[:naz].T.dot(B[:naz])
if tsign > 0: d -= amps[naz:naz+nt].T.dot(B[naz:naz+nt])
if hsign > 0: d -= amps[naz+nt:naz+nt+nhwp].T.dot(B[naz+nt:naz+nt+nhwp])
# Why was this necessary?
if do_gapfill: gapfill.gapfill(d, cuts, inplace=True)
if deslope: utils.deslope(tod, w=8, inplace=True)
res = d.reshape(tod.shape)
return res
continue
# Read the data
entry = filedb.data[id]
try:
scan = actscan.ACTScan(entry, verbose=verbose >= 2)
if scan.ndet < 2 or scan.nsamp < 1:
raise errors.DataMissing("no data in tod")
except errors.DataMissing as e:
print("%s skipped: %s" % (id, e))
continue
# Apply downsampling
scan = scan[:, ::down]
# Prepeare our samples
scan.tod = scan.get_samples()
utils.deslope(scan.tod, w=5, inplace=True)
scan.tod = scan.tod.astype(dtype)
# Background subtraction
if args.sub:
Pmap = pmat.PmatMap(scan, background, sys=sys)
Pmap.forward(scan.tod, background, tmul=-1)
# Build the noise model
N = NmatTot(scan, model="uncorr", window=2.0, filter=highpass)
P = PmatTot(scan, srcpos[:, sids], perdet=True, sys=sys)
# rhs
N.apply(scan.tod)
rhs = P.backward(scan.tod, ncomp=1)
# div
def filter_poly_jon(tod,
az,
weights=None,
naz=None,
nt=None,
niter=None,
cuts=None,
hwp=None,
nhwp=None,
deslope=True,
inplace=True,
use_phase=None):
"""Fix naz Legendre polynomials in az and nt other polynomials
in t jointly. Then subtract the best fit from the data.
The subtraction is inplace, so tod is modified. If naz or nt are
negative, they are fit for, but not subtracted.
NOTE: This function may leave tod nonperiodic.
"""
naz = config.get("gfilter_jon_naz", naz)
nt = config.get("gfilter_jon_nt", nt)
nhwp = config.get("gfilter_jon_nhwp", nhwp)
niter = config.get("gfilter_jon_niter", niter)
use_phase = config.get("gfilter_jon_phase", use_phase)
if not inplace: tod = tod.copy()
do_gapfill = cuts is not None
# No point in iterating if we aren't gapfilling
if not do_gapfill: niter = 1
if hwp is None or np.all(hwp == 0): nhwp = 0
naz, asign = np.abs(naz), np.sign(naz)
nt, tsign = np.abs(nt), np.sign(nt)
nhwp, hsign = np.abs(nhwp), np.sign(nhwp)
d = tod.reshape(-1, tod.shape[-1])
nsamp = d.shape[-1]
if naz == 0 and nt == 0 and nhwp == 0: return tod
B = []
# Set up our time basis
if nt > 0:
B.append(np.full((1, nsamp), 1.0, d.dtype))
if nt > 1:
t = np.linspace(-1, 1, nsamp, endpoint=False)
B.append(utils.build_legendre(t, nt - 1))
if naz > 0:
if not use_phase:
# Set up our azimuth basis
B.append(utils.build_legendre(az, naz))
else:
# Set up phase basis. Vectors should be periodic
# in phase to avoid discontinuities. cossin is good for this.
phase = build_phase(az) * np.pi
B.append(utils.build_cossin(phase, naz))
if nhwp > 0:
B.append(utils.build_cossin(hwp, nhwp))
B = np.concatenate(B, 0)
for it in range(niter):
if do_gapfill: gapfill.gapfill(d, cuts, inplace=True)
# Solve for the best fit for each detector, [nbasis,ndet]
# B[b,n], d[d,n], amps[b,d]
if weights is None:
try:
amps = np.linalg.solve(B.dot(B.T), B.dot(d.T))
except np.linalg.LinAlgError as e:
print "LinAlgError in todfilter. Skipping"
continue
else:
w = weights.reshape(-1, weights.shape[-1])
amps = np.zeros([naz + nt + nhwp, d.shape[0]], dtype=tod.dtype)
for di in range(len(tod)):
try:
amps[:, di] = np.linalg.solve((B * w[di]).dot(B.T),
B.dot(w[di] * d[di]))
except np.linalg.LinAlgError as e:
print "LinAlgError in todfilter di %d. Skipping" % di
continue
# Subtract the best fit, but skip some basis functions if requested
if tsign < 0: amps[:nt] = 0
if asign < 0: amps[nt:nt + naz] = 0
if hsign < 0: amps[nt + naz:nt + naz + nhwp] = 0
d -= amps.T.dot(B)
if do_gapfill: gapfill.gapfill(d, cuts, inplace=True)
# This filtering might have casued the tod to become
# non-continous, which would mess up future fourier transforms.
# So it's safest to deslope here.
if deslope: utils.deslope(tod, w=8, inplace=True)
res = d.reshape(tod.shape)
return res
div = rhs.copy()
wrhs = signal.prepare(rhs)
wdiv = signal.prepare(div)
kvals = np.zeros(len(corr_pos), dtype)
freqs, bareas = np.zeros(ntod), np.zeros(ntod)
bleh = True and args.inject
if bleh:
sim_rhs = np.zeros(len(inject_params))
sim_div = np.zeros(len(inject_params))
for si, scan in zip(myinds, myscans):
L.debug("Processing %s" % scan.id)
# Read the tod
tod = scan.get_samples().astype(dtype)
tod = utils.deslope(tod)
if args.mapsub:
# Subtract the reference map. If the reference map is not source free,
# then this could reintroduce strong point sources that were cut earlier.
# To avoid this we do another round of gapfilling
signal.forward(scan, tod, refmap, tmul=1, mmul=-1)
gapfill.gapfill_joneig(tod, scan.cut, inplace=True)
# Inject simulated signal if requested
if args.inject:
dmjd = scan.mjd0 - mjd0
earth_pos = -ephemeris.ephem_vec("Sun", scan.mjd0)[:, 0]
# Set the position and amplitude uK of each simulated source
sim_srcs = np.zeros([len(inject_params), 8])
# TODO: inject and analyze with no displacement to see if interpolation is the
