def avoidance_cut_old(bore, det_offs, site, name_or_pos, margin):
"""Cut samples that get too close to the specified object
(e.g. "Sun" or "Moon") or celestial position ([ra,dec] in racians).
Margin specifies how much to avoid the object by."""
cmargin = np.cos(margin)
mjd = utils.ctime2mjd(bore[0])
obj_pos = coordinates.interpol_pos("cel", "hor", name_or_pos, mjd, site)
obj_rect = utils.ang2rect(obj_pos, zenith=False)
# Only cut if above horizon
above_horizon = obj_pos[1] > 0
if np.all(~above_horizon):
return sampcut.empty(det_offs.shape[0], bore.shape[1])
cuts = []
for di, off in enumerate(det_offs):
det_pos = bore[1:] + off[:, None]
#det_rect1 = utils.ang2rect(det_pos, zenith=False) # slow
# Not sure defining an ang2rect in array_ops is worth it.. It's ugly,
# (angle convention hard coded) and only gives factor 2 on laptop.
det_rect = array_ops.ang2rect(np.ascontiguousarray(det_pos.T)).T
cdist = np.sum(obj_rect * det_rect, 0)
# Cut samples above horizon that are too close
bad = (cdist > cmargin) & above_horizon
cuts.append(sampcut.from_mask(bad))
res = sampcut.stack(cuts)
return res
def find_spikes(tod, nsigma=10, width=15, padding=7, noise=None): res = [] if noise is None: noise = estimate_white_noise(tod)**0.5 for di, d in enumerate(ftod): smooth = scipy.signal.medfilt(d, width) bad = np.abs(d-smooth) > noise[di]*nsigma bad = sampcut.from_mask(bad).widen(padding) res.append(bad) res = sampcut.stack(res) res.data.reshape(tod.shape[:-1]) return res
def cut_mostly_cut_detectors(cuts, max_frac=None, max_nrange=None):
"""Mark detectors with too many cuts or too large cuts as completely cut."""
max_frac = config.get("cut_mostly_cut_frac", max_frac)
max_nrange = config.get("cut_mostly_cut_nrange", max_nrange)
cut_samps = cuts.sum(axis=1)
cut_nrange = cuts.nranges
bad = (cut_samps > cuts.nsamp*max_frac)
if max_nrange > 0:
bad |= cut_nrange > max_nrange
ocuts = []
for b in bad:
if b: ocuts.append(sampcut.full(1,cuts.nsamp))
else: ocuts.append(sampcut.empty(1,cuts.nsamp))
return sampcut.stack(ocuts)
def cut_mostly_cut_detectors(cuts, max_frac=None, max_nrange=None):
"""Mark detectors with too many cuts or too large cuts as completely cut."""
max_frac = config.get("cut_mostly_cut_frac", max_frac)
max_nrange = config.get("cut_mostly_cut_nrange", max_nrange)
cut_samps = cuts.sum(axis=1)
cut_nrange = cuts.nranges
bad = (cut_samps > cuts.nsamp * max_frac)
if max_nrange > 0:
bad |= cut_nrange > max_nrange
ocuts = []
for b in bad:
if b: ocuts.append(sampcut.full(1, cuts.nsamp))
else: ocuts.append(sampcut.empty(1, cuts.nsamp))
return sampcut.stack(ocuts)
def ground_cut(bore, det_offs, az_ranges=None, el_ranges=None):
az_ranges = np.array([[float(w) for w in tok.split(":")] for tok in config.get("cut_ground_az", az_ranges).split(",")])*np.pi/180
el_ranges = np.array([[float(w) for w in tok.split(":")] for tok in config.get("cut_ground_el", el_ranges).split(",")])*np.pi/180
n = bore.shape[1]
cuts = []
for di, doff in enumerate(det_offs):
p = bore[1:]+doff[:,None]
mask_az = np.full([n],False,dtype=bool)
for ar in az_ranges:
mask_az |= utils.between_angles(p[0], ar)
mask_el = np.full([n],False,dtype=bool)
for er in el_ranges:
mask_el |= utils.between_angles(p[1], er)
cuts.append(sampcut.from_mask(mask_az&mask_el))
return sampcut.stack(cuts)
def pickup_cut(az, dets, pcut): """Cut samples as specified in the pcut struct, which works on hexed per tod per scanning direction.""" hex = det2hex(dets) dir = np.concatenate([[0],az[1:]<az[:-1]]) res = sampcut.empty(len(dets),len(az)) for cdir,chex,az1,az2,strength in pcut: myrange = sampcut.from_mask((az>=az1)&(az<az2)&(dir==cdir)) uncut = sampcut.empty(1, len(az)) mycut = [] for h in hex: if h == chex: mycut.append(myrange) else: mycut.append(uncut) res *= sampcut.stack(mycut) return res
def pickup_cut(az, dets, pcut):
"""Cut samples as specified in the pcut struct, which works on hexed per
tod per scanning direction."""
hex = det2hex(dets)
dir = np.concatenate([[0], az[1:] < az[:-1]])
res = sampcut.empty(len(dets), len(az))
for cdir, chex, az1, az2, strength in pcut:
myrange = sampcut.from_mask((az >= az1) & (az < az2) & (dir == cdir))
uncut = sampcut.empty(1, len(az))
mycut = []
for h in hex:
if h == chex: mycut.append(myrange)
else: mycut.append(uncut)
res *= sampcut.stack(mycut)
return res
def ground_cut(bore, det_offs, az_ranges=None, el_ranges=None):
az_ranges = np.array([[float(w) for w in tok.split(":")] for tok in config.get("cut_ground_az", az_ranges).split(",")])*np.pi/180
el_ranges = np.array([[float(w) for w in tok.split(":")] for tok in config.get("cut_ground_el", el_ranges).split(",")])*np.pi/180
n = bore.shape[1]
cuts = []
for di, doff in enumerate(det_offs):
p = bore[1:]+doff[:,None]
mask_az = np.full([n],False,dtype=bool)
for ar in az_ranges:
mask_az |= utils.between_angles(p[0], ar)
mask_el = np.full([n],False,dtype=bool)
for er in el_ranges:
mask_el |= utils.between_angles(p[1], er)
cuts.append(sampcut.from_mask(mask_az&mask_el))
return sampcut.stack(cuts)
def read_cut(fname, permissive=True):
"""Read the act cut format, returning ids, cuts, offset, where cuts is a Multirange
object."""
nsamp, ndet, offset = None, None, None
dets, cuts = [], []
for line in utils.lines(fname):
if "=" in line:
# Header key-value pair
toks = line.split()
if toks[0] == "n_det": ndet = int(toks[2])
elif toks[0] == "n_samp": nsamp = int(toks[2])
elif toks[0] == "samp_offset": offset = int(toks[2])
else: continue # Ignore others
elif ":" in line:
parts = line.split(":")
uid = int(parts[0].split()[0])
if len(parts) > 1 and "(" in parts[1]:
toks = parts[1].split()
ranges = np.array([[int(w) for w in tok[1:-1].split(",")]
for tok in toks])
ranges = np.minimum(ranges, nsamp)
cuts.append(sampcut.from_list([ranges], nsamp))
# Handle uncut detectors
else:
cuts.append(sampcut.empty(1, nsamp))
dets.append(uid)
# Add any missing detectors if we are in permissive mode
if permissive:
missing = set(range(ndet)) - set(dets)
for uid in missing:
dets.append(uid)
cuts.append(sampcut.empty(1, nsamp))
# Filter out fully cut tods
odets, ocuts = [], []
for det, cut in zip(dets, cuts):
if cut.sum() < cut.nsamp:
odets.append(det)
ocuts.append(cut)
if len(ocuts) == 0: ocuts = sampcut.full(0, nsamp)
else: ocuts = sampcut.stack(ocuts)
return odets, ocuts, offset
def avoidance_cut(bore, det_offs, site, name_or_pos, margin):
"""Cut samples that get too close to the specified object
(e.g. "Sun" or "Moon") or celestial position ([ra,dec] in racians).
Margin specifies how much to avoid the object by."""
cmargin = np.cos(margin)
mjd = utils.ctime2mjd(bore[0])
obj_pos = coordinates.interpol_pos("cel", "hor", name_or_pos, mjd, site)
obj_pos[0] = utils.rewind(obj_pos[0], bore[1])
cosel = np.cos(obj_pos[1])
# Only cut if above horizon
above_horizon = obj_pos[1] > 0
null_cut = sampcut.empty(det_offs.shape[0], bore.shape[1])
if np.all(~above_horizon): return null_cut
# Find center of array, and radius
arr_center = np.mean(det_offs, 0)
arr_rad = np.max(np.sum((det_offs - arr_center)**2, 1)**0.5)
def calc_mask(det_pos, rad, mask=slice(None)):
offs = (det_pos - obj_pos[:, mask])
offs[0] *= cosel[mask]
dists2 = np.sum(offs**2, 0)
return dists2 < rad**2
# Find samples that could possibly be near object for any detector
cand_mask = calc_mask(arr_center[:, None] + bore[1:], margin + arr_rad)
cand_mask &= above_horizon
cand_inds = np.where(cand_mask)[0]
if len(cand_inds) == 0: return null_cut
# Loop through all detectors and find out if each candidate actually intersects
cuts = []
for di, off in enumerate(det_offs):
det_pos = bore[1:, cand_inds] + off[:, None]
det_mask = calc_mask(det_pos, margin, cand_inds)
# Expand mask to full set
det_mask_full = np.zeros(bore.shape[1], bool)
det_mask_full[cand_inds] = det_mask
# And use this to build actual cuts
cuts.append(sampcut.from_mask(det_mask_full))
res = sampcut.stack(cuts)
return res
def read_cut(fname, permissive=True):
"""Read the act cut format, returning ids, cuts, offset, where cuts is a Multirange
object."""
nsamp, ndet, offset = None, None, None
dets, cuts = [], []
for line in utils.lines(fname):
if "=" in line:
# Header key-value pair
toks = line.split()
if toks[0] == "n_det": ndet = int(toks[2])
elif toks[0] == "n_samp": nsamp = int(toks[2])
elif toks[0] == "samp_offset": offset = int(toks[2])
else: continue # Ignore others
elif ":" in line:
parts = line.split(":")
uid = int(parts[0].split()[0])
if len(parts) > 1 and "(" in parts[1]:
toks = parts[1].split()
ranges = np.array([[int(w) for w in tok[1:-1].split(",")] for tok in toks])
ranges = np.minimum(ranges, nsamp)
cuts.append(sampcut.from_list([ranges],nsamp))
# Handle uncut detectors
else:
cuts.append(sampcut.empty(1, nsamp))
dets.append(uid)
# Add any missing detectors if we are in permissive mode
if permissive:
missing = set(range(ndet))-set(dets)
for uid in missing:
dets.append(uid)
cuts.append(sampcut.empty(1,nsamp))
# Filter out fully cut tods
odets, ocuts = [], []
for det, cut in zip(dets, cuts):
if cut.sum() < cut.nsamp:
odets.append(det)
ocuts.append(cut)
if len(ocuts) == 0: ocuts = sampcut.full(0,nsamp)
else: ocuts = sampcut.stack(ocuts)
return odets, ocuts, offset
def avoidance_cut(bore, det_offs, site, name_or_pos, margin):
"""Cut samples that get too close to the specified object
(e.g. "Sun" or "Moon") or celestial position ([ra,dec] in racians).
Margin specifies how much to avoid the object by."""
cmargin = np.cos(margin)
mjd = utils.ctime2mjd(bore[0])
obj_pos = coordinates.interpol_pos("cel","hor",name_or_pos,mjd,site)
obj_pos[0] = utils.rewind(obj_pos[0], bore[1])
cosel = np.cos(obj_pos[1])
# Only cut if above horizon
above_horizon = obj_pos[1]>0
null_cut = sampcut.empty(det_offs.shape[0], bore.shape[1])
if np.all(~above_horizon): return null_cut
# Find center of array, and radius
arr_center = np.mean(det_offs,0)
arr_rad = np.max(np.sum((det_offs-arr_center)**2,1)**0.5)
def calc_mask(det_pos, rad, mask=slice(None)):
offs = (det_pos-obj_pos[:,mask])
offs[0] *= cosel[mask]
dists2= np.sum(offs**2,0)
return dists2 < rad**2
# Find samples that could possibly be near object for any detector
cand_mask = calc_mask(arr_center[:,None] + bore[1:], margin+arr_rad)
cand_mask &= above_horizon
cand_inds = np.where(cand_mask)[0]
if len(cand_inds) == 0: return null_cut
# Loop through all detectors and find out if each candidate actually intersects
cuts = []
for di, off in enumerate(det_offs):
det_pos = bore[1:,cand_inds]+off[:,None]
det_mask = calc_mask(det_pos, margin, cand_inds)
# Expand mask to full set
det_mask_full = np.zeros(bore.shape[1], bool)
det_mask_full[cand_inds] = det_mask
# And use this to build actual cuts
cuts.append(sampcut.from_mask(det_mask_full))
res = sampcut.stack(cuts)
return res
def simple_glitch_cut(tod, bsize=20, scales=[4], tol=100, dblock=32): """Look for dramatic glitches, processing each detector individually. Only meant for identifying nasty spikes that need to be gapfilled. scales controls the length scales considered. Something like [1,2,4,8,16] might be ideal for finding glitches with cosistent S/N on multiple length scales, but lower scales can take quite long. Let's try [4] for now. It takes about 2.5 seconds.""" cuts = [] ndet, nsamp = tod.shape for di1 in range(0, ndet, dblock): di2 = min(di1 + dblock, ndet) dtod = tod[di1:di2] bad = np.zeros(dtod.shape, bool) for si, scale in enumerate(scales): wtod = utils.block_reduce(dtod, scale, np.mean) # Remove a linear trend every 100 samples. This should get rid of the atmosphere meds = utils.block_reduce(wtod, bsize, np.median) rtod = wtod - utils.block_expand(meds, bsize, wtod.shape[-1], "linear") # Compute the typical rms rms = np.median(utils.block_reduce(rtod, bsize, np.std),-1) # Flag values that are much higher than this bad |= utils.block_expand(np.abs(rtod) > rms[:,None]*tol, scale, dtod.shape[-1], "nearest") cuts.append(sampcut.from_mask(bad)) return sampcut.stack(cuts)
def avoidance_cut_old(bore, det_offs, site, name_or_pos, margin):
"""Cut samples that get too close to the specified object
(e.g. "Sun" or "Moon") or celestial position ([ra,dec] in racians).
Margin specifies how much to avoid the object by."""
cmargin = np.cos(margin)
mjd = utils.ctime2mjd(bore[0])
obj_pos = coordinates.interpol_pos("cel","hor",name_or_pos,mjd,site)
obj_rect = utils.ang2rect(obj_pos, zenith=False)
# Only cut if above horizon
above_horizon = obj_pos[1]>0
if np.all(~above_horizon): return sampcut.empty(det_offs.shape[0], bore.shape[1])
cuts = []
for di, off in enumerate(det_offs):
det_pos = bore[1:]+off[:,None]
#det_rect1 = utils.ang2rect(det_pos, zenith=False) # slow
# Not sure defining an ang2rect in array_ops is worth it.. It's ugly,
# (angle convention hard coded) and only gives factor 2 on laptop.
det_rect = array_ops.ang2rect(np.ascontiguousarray(det_pos.T)).T
cdist = np.sum(obj_rect*det_rect,0)
# Cut samples above horizon that are too close
bad = (cdist > cmargin) & above_horizon
cuts.append(sampcut.from_mask(bad))
res = sampcut.stack(cuts)
return res
def stacker(cuts, axis): return sampcut.stack(cuts) fields.append(dataset.DataField(name, data, dets=build_detname(dets, entry), det_index=0, samples=samples, sample_index=1,