def get_rangedata(id):
entry = filedb.data[id]
# Read the tod as usual
with show("read"):
d = actdata.read(entry)
with show("calibrate"):
# Don't apply time constant (and hence butterworth) deconvolution since we
# will fit these ourselves
d = actdata.calibrate(d, exclude=["autocut", "tod_fourier"])
if d.ndet == 0 or d.nsamp < 2: raise errors.DataMissing("no data in tod")
tod = d.tod
del d.tod
# Very simple white noise model
with show("noise"):
ivar = estimate_ivar(tod)
asens = np.sum(ivar)**-0.5 / d.srate**0.5
with show("planet mask"):
# Generate planet cut
planet_cut = cuts.avoidance_cut(d.boresight, d.point_offset, d.site,
args.planet, R)
with show("atmosphere"):
# Subtract atmospheric model
tod -= estimate_atmosphere(tod, planet_cut, d.srate, model_fknee,
model_alpha)
tod = tod.astype(dtype, copy=False)
with show("extract"):
# Should now be reasonably clean of correlated noise. Extract our range data
rdata = build_rangedata(tod, planet_cut, d, ivar)
return rdata
def get_rangedata(id):
entry = filedb.data[id]
# Read the tod as usual
with show("read"):
d = actdata.read(entry)
with show("calibrate"):
# Don't apply time constant (and hence butterworth) deconvolution since we
# will fit these ourselves
d = actdata.calibrate(d, exclude=["autocut","tod_fourier"])
if d.ndet == 0 or d.nsamp < 2: raise errors.DataMissing("no data in tod")
tod = d.tod; del d.tod
# Very simple white noise model
with show("noise"):
ivar = estimate_ivar(tod)
asens = np.sum(ivar)**-0.5 / d.srate**0.5
with show("planet mask"):
# Generate planet cut
planet_cut = cuts.avoidance_cut(d.boresight, d.point_offset, d.site, args.planet, R)
with show("atmosphere"):
# Subtract atmospheric model
tod -= estimate_atmosphere(tod, planet_cut, d.srate, model_fknee, model_alpha)
tod = tod.astype(dtype, copy=False)
with show("extract"):
# Should now be reasonably clean of correlated noise. Extract our range data
rdata = build_rangedata(tod, planet_cut, d, ivar)
return rdata
def cut_asteroids_scan(scan, asteroids, r=3 * utils.arcmin):
"""Cut the the samples that come within a distance of r in radians
from the given asteroids. scan is modified in-place"""
from enact import cuts as actcuts
import time
for ai, asteroid in enumerate(asteroids):
apos = asteroid(scan.mjd0)[:2]
bore = scan.boresight.T.copy()
bore[0] = utils.mjd2ctime(scan.mjd0) + bore[0]
t1 = time.time()
scan.cut *= actcuts.avoidance_cut(bore, scan.offsets[:, 1:], scan.site,
apos, r)
t2 = time.time()
return scan
print "Skipping %s (%s)" % (id, e.message)
continue
print "Processing %s" % id
# Very simple white noise model
with bench.show("ivar"):
tod = d.tod
del d.tod
tod -= np.mean(tod,1)[:,None]
tod = tod.astype(dtype)
diff = tod[:,1:]-tod[:,:-1]
diff = diff[:,:diff.shape[-1]/csize*csize].reshape(d.ndet,-1,csize)
ivar = 1/(np.median(np.mean(diff**2,-1),-1)/2**0.5)
del diff
# Generate planet cut
with bench.show("planet cut"):
planet_cut = cuts.avoidance_cut(d.boresight, d.point_offset, d.site,
args.planet, R)
# Subtract atmospheric model
with bench.show("atm model"):
model= gapfill.gapfill_joneig(tod, planet_cut, inplace=False)
# Estimate noise level
asens = np.sum(ivar)**-0.5 / d.srate**0.5
print asens
with bench.show("smooth"):
ft = fft.rfft(model)
freq = fft.rfftfreq(model.shape[-1])*d.srate
flt = 1/(1+(freq/model_fknee)**model_alpha)
ft *= flt
fft.ifft(ft, model, normalize=True)
del ft, flt, freq
with bench.show("atm subtract"):
tod -= model
def autocut(d, turnaround=None, ground=None, sun=None, moon=None, max_frac=None, pickup=None):
"""Apply automatic cuts to calibrated data."""
if not config.get("autocut"): return d
ndet, nsamp = d.ndet, d.nsamp
if not ndet or not nsamp: return d
# Insert a cut into d if necessary
if "cut" not in d:
d += dataset.DataField("cut", sampcut.empty(ndet,nsamp))
# insert an autocut datafield, to keep track of how much data each
# automatic cut cost us
d += dataset.DataField("autocut", [])
def addcut(label, dcut, targets="c"):
# det ndet part here allows for broadcasting of cuts from 1-det to full-det
dn = dcut.sum()*d.ndet/dcut.ndet if dcut is not None else 0
if dn == 0: d.autocut.append([label,0,0])
else:
n0, dn = d.cut.sum(), dcut.sum()
dn = dn*d.cut.ndet/dcut.ndet
if "c" in targets: d.cut *= dcut
if "n" in targets: d.cut_noiseest *= dcut
if "b" in targets: d.cut_basic *= dcut
d.autocut.append([ label, dn, d.cut.sum() - n0 ]) # name, mycut, myeffect
if config.get("cut_tconst") and "tau" in d:
addcut("tconst", cuts.tconst_cut(nsamp, d.tau, config.get("cut_tconst")))
if config.get("cut_stationary") and "boresight" in d:
addcut("stationary", cuts.stationary_cut(d.boresight[1]))
if config.get("cut_tod_ends") and "srate" in d:
addcut("tod_ends", cuts.tod_end_cut(nsamp, d.srate))
if config.get("cut_turnaround", turnaround) and "boresight" in d:
addcut("turnaround",cuts.turnaround_cut(d.boresight[1]))
if config.get("cut_ground", ground) and "boresight" in d and "point_offset" in d:
addcut("ground", cuts.ground_cut(d.boresight, d.point_offset))
if config.get("cut_sun", sun) and "boresight" in d and "point_offset" in d and "site" in d:
addcut("avoidance",cuts.avoidance_cut(d.boresight, d.point_offset, d.site, "Sun", config.get("cut_sun_dist")*np.pi/180))
if config.get("cut_moon", moon) and "boresight" in d and "point_offset" in d and "site" in d:
addcut("moon",cuts.avoidance_cut(d.boresight, d.point_offset, d.site, "Moon", config.get("cut_moon_dist")*np.pi/180))
if config.get("cut_pickup", pickup) and "boresight" in d and "pickup_cut" in d:
addcut("pickup",cuts.pickup_cut(d.boresight[1], d.dets, d.pickup_cut))
if config.get("cut_mostly_cut"):
addcut("mostly_cut", cuts.cut_mostly_cut_detectors(d.cut_quality))
if config.get("cut_obj"):
objs = utils.split_outside(config.get("cut_obj"),",")
for obj in objs:
toks = obj.split(":")
objname = toks[0]
if objname.startswith("["):
objname = [float(w)*utils.degree for w in objname[1:-1].split(",")]
dist = 0.2*utils.degree
if len(toks) > 1: dist = float(toks[1])*utils.degree
# Hack: only cut for noise estimation purposes if dist is negative
targets = "cnb" if dist > 0 else "n"
addcut(obj, cuts.avoidance_cut(d.boresight, d.point_offset, d.site, objname, dist), targets=targets)
if config.get("cut_srcs"):
cpar = [tok.split(":") for tok in config.get("cut_srcs").split(",")]
names, lims = [], []
for par in cpar:
if par[0] in ["map","nmat"]:
names.append(par[0])
lims.append(float(par[1]))
if any(lims):
params = pointsrcs.src2param(d.pointsrcs)
params[:,5:7] = 1
params[:,7] = 0
# Only bother with sources that are actually strong enough
maxlim = max(lims+[0])
params = params[params[:,2]>maxlim]
cutlist = cuts.point_source_cut(d, params, lims)
for name, c in zip(names, cutlist):
if name == "map": addcut("point_srcs_m", c, "c")
elif name == "nmat": addcut("point_srcs_n", c, "n")
if config.get("cut_extra_srcs"):
srclist = np.loadtxt(config.get("cut_extra_srcs"), usecols=(0,1), ndmin=2)
srclim = float(config.get("cut_extra_lim"))
params = np.zeros([len(srclist),8])
params[:,:2] = srclist[:,1::-1]*utils.degree
params[:,2] = 1
params[:,5:7] = 1
c = cuts.point_source_cut(d, params, srclim)
addcut("point_srcs", c, "nbc")
# What fraction is cut?
cut_fraction = float(d.cut.sum())/d.cut.size
# Get rid of completely cut detectors
keep = np.where(d.cut.sum(axis=1) < nsamp)[0]
d.restrict(d.dets[keep])
ndet, nsamp = d.ndet, d.nsamp
def cut_all_if(label, condition):
if condition: dcut = sampcut.full(d.ndet, nsamp)
else: dcut = None
addcut(label, dcut)
cut_all_if("max_frac", config.get("cut_max_frac", max_frac) < cut_fraction)
if "srate" in d:
cut_all_if("tod_mindur", config.get("cut_tod_mindur") > nsamp/d.srate/60)
cut_all_if("tod_mindet", config.get("cut_tod_mindet") > ndet)
# Get rid of completely cut detectors again
keep = np.where(d.cut.sum(axis=1) < nsamp)[0]
d.restrict(dets=d.dets[keep])
return d
print("Skipping %s (%s)" % (id, e))
continue
print("Processing %s" % id)
# Very simple white noise model
with bench.show("ivar"):
tod = d.tod
del d.tod
tod -= np.mean(tod,1)[:,None]
tod = tod.astype(dtype)
diff = tod[:,1:]-tod[:,:-1]
diff = diff[:,:diff.shape[-1]//csize*csize].reshape(d.ndet,-1,csize)
ivar = 1/(np.median(np.mean(diff**2,-1),-1)/2**0.5)
del diff
# Generate planet cut
with bench.show("planet cut"):
planet_cut = cuts.avoidance_cut(d.boresight, d.point_offset, d.site,
args.planet, R)
# Subtract atmospheric model
with bench.show("atm model"):
model= gapfill.gapfill_joneig(tod, planet_cut, inplace=False)
# Estimate noise level
asens = np.sum(ivar)**-0.5 / d.srate**0.5
print(asens)
with bench.show("smooth"):
ft = fft.rfft(model)
freq = fft.rfftfreq(model.shape[-1])*d.srate
flt = 1/(1+(freq/model_fknee)**model_alpha)
ft *= flt
fft.ifft(ft, model, normalize=True)
del ft, flt, freq
with bench.show("atm subtract"):
tod -= model