def pickup_cut(az, dets, pcut): """Cut samples as specified in the pcut struct, which works on hexed per tod per scanning direction.""" hex = det2hex(dets) dir = np.concatenate([[0],az[1:]<az[:-1]]) res = sampcut.empty(len(dets),len(az)) for cdir,chex,az1,az2,strength in pcut: myrange = sampcut.from_mask((az>=az1)&(az<az2)&(dir==cdir)) uncut = sampcut.empty(1, len(az)) mycut = [] for h in hex: if h == chex: mycut.append(myrange) else: mycut.append(uncut) res *= sampcut.stack(mycut) return res
def pickup_cut(az, dets, pcut):
"""Cut samples as specified in the pcut struct, which works on hexed per
tod per scanning direction."""
hex = det2hex(dets)
dir = np.concatenate([[0], az[1:] < az[:-1]])
res = sampcut.empty(len(dets), len(az))
for cdir, chex, az1, az2, strength in pcut:
myrange = sampcut.from_mask((az >= az1) & (az < az2) & (dir == cdir))
uncut = sampcut.empty(1, len(az))
mycut = []
for h in hex:
if h == chex: mycut.append(myrange)
else: mycut.append(uncut)
res *= sampcut.stack(mycut)
return res
def avoidance_cut_old(bore, det_offs, site, name_or_pos, margin):
"""Cut samples that get too close to the specified object
(e.g. "Sun" or "Moon") or celestial position ([ra,dec] in racians).
Margin specifies how much to avoid the object by."""
cmargin = np.cos(margin)
mjd = utils.ctime2mjd(bore[0])
obj_pos = coordinates.interpol_pos("cel", "hor", name_or_pos, mjd, site)
obj_rect = utils.ang2rect(obj_pos, zenith=False)
# Only cut if above horizon
above_horizon = obj_pos[1] > 0
if np.all(~above_horizon):
return sampcut.empty(det_offs.shape[0], bore.shape[1])
cuts = []
for di, off in enumerate(det_offs):
det_pos = bore[1:] + off[:, None]
#det_rect1 = utils.ang2rect(det_pos, zenith=False) # slow
# Not sure defining an ang2rect in array_ops is worth it.. It's ugly,
# (angle convention hard coded) and only gives factor 2 on laptop.
det_rect = array_ops.ang2rect(np.ascontiguousarray(det_pos.T)).T
cdist = np.sum(obj_rect * det_rect, 0)
# Cut samples above horizon that are too close
bad = (cdist > cmargin) & above_horizon
cuts.append(sampcut.from_mask(bad))
res = sampcut.stack(cuts)
return res
def read_cut(fname, permissive=True):
"""Read the act cut format, returning ids, cuts, offset, where cuts is a Multirange
object."""
nsamp, ndet, offset = None, None, None
dets, cuts = [], []
for line in utils.lines(fname):
if "=" in line:
# Header key-value pair
toks = line.split()
if toks[0] == "n_det": ndet = int(toks[2])
elif toks[0] == "n_samp": nsamp = int(toks[2])
elif toks[0] == "samp_offset": offset = int(toks[2])
else: continue # Ignore others
elif ":" in line:
parts = line.split(":")
uid = int(parts[0].split()[0])
if len(parts) > 1 and "(" in parts[1]:
toks = parts[1].split()
ranges = np.array([[int(w) for w in tok[1:-1].split(",")]
for tok in toks])
ranges = np.minimum(ranges, nsamp)
cuts.append(sampcut.from_list([ranges], nsamp))
# Handle uncut detectors
else:
cuts.append(sampcut.empty(1, nsamp))
dets.append(uid)
# Add any missing detectors if we are in permissive mode
if permissive:
missing = set(range(ndet)) - set(dets)
for uid in missing:
dets.append(uid)
cuts.append(sampcut.empty(1, nsamp))
# Filter out fully cut tods
odets, ocuts = [], []
for det, cut in zip(dets, cuts):
if cut.sum() < cut.nsamp:
odets.append(det)
ocuts.append(cut)
if len(ocuts) == 0: ocuts = sampcut.full(0, nsamp)
else: ocuts = sampcut.stack(ocuts)
return odets, ocuts, offset
def read_cut(fname, permissive=True):
"""Read the act cut format, returning ids, cuts, offset, where cuts is a Multirange
object."""
nsamp, ndet, offset = None, None, None
dets, cuts = [], []
for line in utils.lines(fname):
if "=" in line:
# Header key-value pair
toks = line.split()
if toks[0] == "n_det": ndet = int(toks[2])
elif toks[0] == "n_samp": nsamp = int(toks[2])
elif toks[0] == "samp_offset": offset = int(toks[2])
else: continue # Ignore others
elif ":" in line:
parts = line.split(":")
uid = int(parts[0].split()[0])
if len(parts) > 1 and "(" in parts[1]:
toks = parts[1].split()
ranges = np.array([[int(w) for w in tok[1:-1].split(",")] for tok in toks])
ranges = np.minimum(ranges, nsamp)
cuts.append(sampcut.from_list([ranges],nsamp))
# Handle uncut detectors
else:
cuts.append(sampcut.empty(1, nsamp))
dets.append(uid)
# Add any missing detectors if we are in permissive mode
if permissive:
missing = set(range(ndet))-set(dets)
for uid in missing:
dets.append(uid)
cuts.append(sampcut.empty(1,nsamp))
# Filter out fully cut tods
odets, ocuts = [], []
for det, cut in zip(dets, cuts):
if cut.sum() < cut.nsamp:
odets.append(det)
ocuts.append(cut)
if len(ocuts) == 0: ocuts = sampcut.full(0,nsamp)
else: ocuts = sampcut.stack(ocuts)
return odets, ocuts, offset
def cut_mostly_cut_detectors(cuts, max_frac=None, max_nrange=None):
"""Mark detectors with too many cuts or too large cuts as completely cut."""
max_frac = config.get("cut_mostly_cut_frac", max_frac)
max_nrange = config.get("cut_mostly_cut_nrange", max_nrange)
cut_samps = cuts.sum(axis=1)
cut_nrange = cuts.nranges
bad = (cut_samps > cuts.nsamp*max_frac)
if max_nrange > 0:
bad |= cut_nrange > max_nrange
ocuts = []
for b in bad:
if b: ocuts.append(sampcut.full(1,cuts.nsamp))
else: ocuts.append(sampcut.empty(1,cuts.nsamp))
return sampcut.stack(ocuts)
def cut_mostly_cut_detectors(cuts, max_frac=None, max_nrange=None):
"""Mark detectors with too many cuts or too large cuts as completely cut."""
max_frac = config.get("cut_mostly_cut_frac", max_frac)
max_nrange = config.get("cut_mostly_cut_nrange", max_nrange)
cut_samps = cuts.sum(axis=1)
cut_nrange = cuts.nranges
bad = (cut_samps > cuts.nsamp * max_frac)
if max_nrange > 0:
bad |= cut_nrange > max_nrange
ocuts = []
for b in bad:
if b: ocuts.append(sampcut.full(1, cuts.nsamp))
else: ocuts.append(sampcut.empty(1, cuts.nsamp))
return sampcut.stack(ocuts)
def avoidance_cut(bore, det_offs, site, name_or_pos, margin):
"""Cut samples that get too close to the specified object
(e.g. "Sun" or "Moon") or celestial position ([ra,dec] in racians).
Margin specifies how much to avoid the object by."""
cmargin = np.cos(margin)
mjd = utils.ctime2mjd(bore[0])
obj_pos = coordinates.interpol_pos("cel", "hor", name_or_pos, mjd, site)
obj_pos[0] = utils.rewind(obj_pos[0], bore[1])
cosel = np.cos(obj_pos[1])
# Only cut if above horizon
above_horizon = obj_pos[1] > 0
null_cut = sampcut.empty(det_offs.shape[0], bore.shape[1])
if np.all(~above_horizon): return null_cut
# Find center of array, and radius
arr_center = np.mean(det_offs, 0)
arr_rad = np.max(np.sum((det_offs - arr_center)**2, 1)**0.5)
def calc_mask(det_pos, rad, mask=slice(None)):
offs = (det_pos - obj_pos[:, mask])
offs[0] *= cosel[mask]
dists2 = np.sum(offs**2, 0)
return dists2 < rad**2
# Find samples that could possibly be near object for any detector
cand_mask = calc_mask(arr_center[:, None] + bore[1:], margin + arr_rad)
cand_mask &= above_horizon
cand_inds = np.where(cand_mask)[0]
if len(cand_inds) == 0: return null_cut
# Loop through all detectors and find out if each candidate actually intersects
cuts = []
for di, off in enumerate(det_offs):
det_pos = bore[1:, cand_inds] + off[:, None]
det_mask = calc_mask(det_pos, margin, cand_inds)
# Expand mask to full set
det_mask_full = np.zeros(bore.shape[1], bool)
det_mask_full[cand_inds] = det_mask
# And use this to build actual cuts
cuts.append(sampcut.from_mask(det_mask_full))
res = sampcut.stack(cuts)
return res
def avoidance_cut(bore, det_offs, site, name_or_pos, margin):
"""Cut samples that get too close to the specified object
(e.g. "Sun" or "Moon") or celestial position ([ra,dec] in racians).
Margin specifies how much to avoid the object by."""
cmargin = np.cos(margin)
mjd = utils.ctime2mjd(bore[0])
obj_pos = coordinates.interpol_pos("cel","hor",name_or_pos,mjd,site)
obj_pos[0] = utils.rewind(obj_pos[0], bore[1])
cosel = np.cos(obj_pos[1])
# Only cut if above horizon
above_horizon = obj_pos[1]>0
null_cut = sampcut.empty(det_offs.shape[0], bore.shape[1])
if np.all(~above_horizon): return null_cut
# Find center of array, and radius
arr_center = np.mean(det_offs,0)
arr_rad = np.max(np.sum((det_offs-arr_center)**2,1)**0.5)
def calc_mask(det_pos, rad, mask=slice(None)):
offs = (det_pos-obj_pos[:,mask])
offs[0] *= cosel[mask]
dists2= np.sum(offs**2,0)
return dists2 < rad**2
# Find samples that could possibly be near object for any detector
cand_mask = calc_mask(arr_center[:,None] + bore[1:], margin+arr_rad)
cand_mask &= above_horizon
cand_inds = np.where(cand_mask)[0]
if len(cand_inds) == 0: return null_cut
# Loop through all detectors and find out if each candidate actually intersects
cuts = []
for di, off in enumerate(det_offs):
det_pos = bore[1:,cand_inds]+off[:,None]
det_mask = calc_mask(det_pos, margin, cand_inds)
# Expand mask to full set
det_mask_full = np.zeros(bore.shape[1], bool)
det_mask_full[cand_inds] = det_mask
# And use this to build actual cuts
cuts.append(sampcut.from_mask(det_mask_full))
res = sampcut.stack(cuts)
return res
def avoidance_cut_old(bore, det_offs, site, name_or_pos, margin):
"""Cut samples that get too close to the specified object
(e.g. "Sun" or "Moon") or celestial position ([ra,dec] in racians).
Margin specifies how much to avoid the object by."""
cmargin = np.cos(margin)
mjd = utils.ctime2mjd(bore[0])
obj_pos = coordinates.interpol_pos("cel","hor",name_or_pos,mjd,site)
obj_rect = utils.ang2rect(obj_pos, zenith=False)
# Only cut if above horizon
above_horizon = obj_pos[1]>0
if np.all(~above_horizon): return sampcut.empty(det_offs.shape[0], bore.shape[1])
cuts = []
for di, off in enumerate(det_offs):
det_pos = bore[1:]+off[:,None]
#det_rect1 = utils.ang2rect(det_pos, zenith=False) # slow
# Not sure defining an ang2rect in array_ops is worth it.. It's ugly,
# (angle convention hard coded) and only gives factor 2 on laptop.
det_rect = array_ops.ang2rect(np.ascontiguousarray(det_pos.T)).T
cdist = np.sum(obj_rect*det_rect,0)
# Cut samples above horizon that are too close
bad = (cdist > cmargin) & above_horizon
cuts.append(sampcut.from_mask(bad))
res = sampcut.stack(cuts)
return res
def autocut(d, turnaround=None, ground=None, sun=None, moon=None, max_frac=None, pickup=None):
"""Apply automatic cuts to calibrated data."""
if not config.get("autocut"): return d
ndet, nsamp = d.ndet, d.nsamp
if not ndet or not nsamp: return d
# Insert a cut into d if necessary
if "cut" not in d:
d += dataset.DataField("cut", sampcut.empty(ndet,nsamp))
# insert an autocut datafield, to keep track of how much data each
# automatic cut cost us
d += dataset.DataField("autocut", [])
def addcut(label, dcut, targets="c"):
# det ndet part here allows for broadcasting of cuts from 1-det to full-det
dn = dcut.sum()*d.ndet/dcut.ndet if dcut is not None else 0
if dn == 0: d.autocut.append([label,0,0])
else:
n0, dn = d.cut.sum(), dcut.sum()
dn = dn*d.cut.ndet/dcut.ndet
if "c" in targets: d.cut *= dcut
if "n" in targets: d.cut_noiseest *= dcut
if "b" in targets: d.cut_basic *= dcut
d.autocut.append([ label, dn, d.cut.sum() - n0 ]) # name, mycut, myeffect
if config.get("cut_tconst") and "tau" in d:
addcut("tconst", cuts.tconst_cut(nsamp, d.tau, config.get("cut_tconst")))
if config.get("cut_stationary") and "boresight" in d:
addcut("stationary", cuts.stationary_cut(d.boresight[1]))
if config.get("cut_tod_ends") and "srate" in d:
addcut("tod_ends", cuts.tod_end_cut(nsamp, d.srate))
if config.get("cut_turnaround", turnaround) and "boresight" in d:
addcut("turnaround",cuts.turnaround_cut(d.boresight[1]))
if config.get("cut_ground", ground) and "boresight" in d and "point_offset" in d:
addcut("ground", cuts.ground_cut(d.boresight, d.point_offset))
if config.get("cut_sun", sun) and "boresight" in d and "point_offset" in d and "site" in d:
addcut("avoidance",cuts.avoidance_cut(d.boresight, d.point_offset, d.site, "Sun", config.get("cut_sun_dist")*np.pi/180))
if config.get("cut_moon", moon) and "boresight" in d and "point_offset" in d and "site" in d:
addcut("moon",cuts.avoidance_cut(d.boresight, d.point_offset, d.site, "Moon", config.get("cut_moon_dist")*np.pi/180))
if config.get("cut_pickup", pickup) and "boresight" in d and "pickup_cut" in d:
addcut("pickup",cuts.pickup_cut(d.boresight[1], d.dets, d.pickup_cut))
if config.get("cut_mostly_cut"):
addcut("mostly_cut", cuts.cut_mostly_cut_detectors(d.cut_quality))
if config.get("cut_obj"):
objs = utils.split_outside(config.get("cut_obj"),",")
for obj in objs:
toks = obj.split(":")
objname = toks[0]
if objname.startswith("["):
objname = [float(w)*utils.degree for w in objname[1:-1].split(",")]
dist = 0.2*utils.degree
if len(toks) > 1: dist = float(toks[1])*utils.degree
# Hack: only cut for noise estimation purposes if dist is negative
targets = "cnb" if dist > 0 else "n"
addcut(obj, cuts.avoidance_cut(d.boresight, d.point_offset, d.site, objname, dist), targets=targets)
if config.get("cut_srcs"):
cpar = [tok.split(":") for tok in config.get("cut_srcs").split(",")]
names, lims = [], []
for par in cpar:
if par[0] in ["map","nmat"]:
names.append(par[0])
lims.append(float(par[1]))
if any(lims):
params = pointsrcs.src2param(d.pointsrcs)
params[:,5:7] = 1
params[:,7] = 0
# Only bother with sources that are actually strong enough
maxlim = max(lims+[0])
params = params[params[:,2]>maxlim]
cutlist = cuts.point_source_cut(d, params, lims)
for name, c in zip(names, cutlist):
if name == "map": addcut("point_srcs_m", c, "c")
elif name == "nmat": addcut("point_srcs_n", c, "n")
if config.get("cut_extra_srcs"):
srclist = np.loadtxt(config.get("cut_extra_srcs"), usecols=(0,1), ndmin=2)
srclim = float(config.get("cut_extra_lim"))
params = np.zeros([len(srclist),8])
params[:,:2] = srclist[:,1::-1]*utils.degree
params[:,2] = 1
params[:,5:7] = 1
c = cuts.point_source_cut(d, params, srclim)
addcut("point_srcs", c, "nbc")
# What fraction is cut?
cut_fraction = float(d.cut.sum())/d.cut.size
# Get rid of completely cut detectors
keep = np.where(d.cut.sum(axis=1) < nsamp)[0]
d.restrict(d.dets[keep])
ndet, nsamp = d.ndet, d.nsamp
def cut_all_if(label, condition):
if condition: dcut = sampcut.full(d.ndet, nsamp)
else: dcut = None
addcut(label, dcut)
cut_all_if("max_frac", config.get("cut_max_frac", max_frac) < cut_fraction)
if "srate" in d:
cut_all_if("tod_mindur", config.get("cut_tod_mindur") > nsamp/d.srate/60)
cut_all_if("tod_mindet", config.get("cut_tod_mindet") > ndet)
# Get rid of completely cut detectors again
keep = np.where(d.cut.sum(axis=1) < nsamp)[0]
d.restrict(dets=d.dets[keep])
return d
mjd = utils.ctime2mjd(
utils.mjd2ctime(scan.mjd0) + scan.boresight[::100, 0])
try:
object_pos = coordinates.interpol_pos("cel",
"hor",
args.objname,
mjd,
site=scan.site)
except AttributeError as e:
print(
"Unexpected error in interpol_pos for %s. mid time was %.5f. message: %s. skipping"
% (id, mjd[len(mjd) // 2], e))
continue
visible = np.any(object_pos[1] >= -margin)
if not visible:
cut = sampcut.empty(d.ndet, d.nsamp)
else:
pmap = pmat.PmatMap(scan, mask, sys="sidelobe:%s" % args.objname)
# Build a tod to project onto.
tod = np.zeros((d.ndet, d.nsamp), dtype=dtype)
# And project
pmap.forward(tod, mask)
# Any nonzero samples should be cut
tod = tod != 0
cut = sampcut.from_mask(tod)
del tod
progress = 100.0 * (ind - comm.rank * ntod // comm.size) / (
(comm.rank + 1) * ntod // comm.size -
comm.rank * ntod // comm.size)
print("%3d %5.1f %s %6.4f %d %8.3f %8.3f" %
(comm.rank, progress, id, float(cut.sum()) / cut.size, visible,
sys = "hor", site = d.site)
scan.hwp_phase = np.zeros([len(bore),2])
bore_box = np.array([np.min(d.boresight,1),np.max(d.boresight,1)])
bore_corners = utils.box2corners(bore_box)
scan.entry = d.entry
# Is the source above the horizon? If not, it doesn't matter how close
# it is.
mjd = utils.ctime2mjd(utils.mjd2ctime(scan.mjd0)+scan.boresight[::100,0])
try:
object_pos = coordinates.interpol_pos("cel","hor", args.objname, mjd, site=scan.site)
except AttributeError as e:
print("Unexpected error in interpol_pos for %s. mid time was %.5f. message: %s. skipping" % (id, mjd[len(mjd)//2], e))
continue
visible = np.any(object_pos[1] >= -margin)
if not visible:
cut = sampcut.empty(d.ndet, d.nsamp)
else:
pmap = pmat.PmatMap(scan, mask, sys="sidelobe:%s" % args.objname)
# Build a tod to project onto.
tod = np.zeros((d.ndet, d.nsamp), dtype=dtype)
# And project
pmap.forward(tod, mask)
# Any nonzero samples should be cut
tod = tod != 0
cut = sampcut.from_mask(tod)
del tod
progress = 100.0*(ind-comm.rank*ntod//comm.size)/((comm.rank+1)*ntod//comm.size-comm.rank*ntod//comm.size)
print("%3d %5.1f %s %6.4f %d %8.3f %8.3f" % (comm.rank, progress, id, float(cut.sum())/cut.size, visible, memory.current()/1024.**3, memory.max()/1024.**3))
mystats.append([ind, float(cut.sum())/cut.size, visible])
# Add to my work file
_, uids = actdata.split_detname(d.dets)