def get_glitch_flags(tod,
params={},
signal='signal',
merge=True,
overwrite=False,
name='glitches'):
""" Find glitches with fourier filtering
Translation from moby2 as starting point
Args:
tod (AxisManager): the tod
params (dictionary): Use to overwrite the default values
n_sig: significance of detection
t_glitch: Gaussian filter width
hp_fc: high pass filter cutoff
buffer: amount to buffer flags around found location
merge (bool): if true, add to tod.flags
name (string): name of flag to add to tod.flags
overwrite (bool): if true, write over flag. if false, don't
Returns:
flag: RangesMatrix object of glitches
"""
gparams = {
'n_sig': 10,
't_glitch': 0.002,
'hp_fc': 5.0,
'buffer': 200,
}
gparams.update(params)
params = gparams
# f-space filtering
filt = filters.high_pass_sine2(params['hp_fc']) * filters.gaussian_filter(
params['t_glitch'])
fvec = fourier_filter(tod,
filt,
detrend='linear',
signal_name=signal,
resize='zero_pad')
# get the threshods based on n_sig x nlev = n_sig x iqu x 0.741
fvec = np.abs(fvec)
thres = 0.741 * stats.iqr(fvec, axis=1) * params['n_sig']
# get flags
msk = fvec > thres[:, None]
flag = RangesMatrix([Ranges.from_bitmask(m) for m in msk])
flag.buffer(params['buffer'])
if merge:
if name in tod.flags and not overwrite:
raise ValueError(
'Flag name {} already exists in tod.flags'.format(name))
elif name in tod.flags:
tod.flags[name] = flag
else:
tod.flags.wrap(name, flag)
return flag
def get_glitch_flags(tod,
t_glitch=0.002,
hp_fc=0.5,
n_sig=10,
buffer=200,
signal=None,
merge=True,
overwrite=False,
name='glitches'):
""" Find glitches with fourier filtering
Translation from moby2 as starting point
Args:
tod (AxisManager): the tod
t_glitch (float): Gaussian filter width
hp_fc: high pass filter cutoff
n_sig (int or float): significance of detection
buffer (int): amount to buffer flags around found location
signal (str): if None, defaults to 'signal'
merge (bool): if true, add to tod.flags
name (string): name of flag to add to tod.flags
overwrite (bool): if true, write over flag. if false, don't
Returns:
flag: RangesMatrix object of glitches
"""
if signal is None:
signal = 'signal'
# f-space filtering
filt = filters.high_pass_sine2(cutoff=hp_fc) * filters.gaussian_filter(
t_sigma=0.002)
fvec = fourier_filter(tod,
filt,
detrend='linear',
signal_name=signal,
resize='zero_pad')
# get the threshods based on n_sig x nlev = n_sig x iqu x 0.741
fvec = np.abs(fvec)
thres = 0.741 * stats.iqr(fvec, axis=1) * n_sig
# get flags
msk = fvec > thres[:, None]
flag = RangesMatrix([Ranges.from_bitmask(m) for m in msk])
flag.buffer(buffer)
if merge:
if name in tod.flags and not overwrite:
raise ValueError(
'Flag name {} already exists in tod.flags'.format(name))
elif name in tod.flags:
tod.flags[name] = flag
else:
tod.flags.wrap(name, flag)
return flag
def wrap(self, name, data, axis_map=None, **kwargs):
"""See core.AxisManager for basic usage
If axis_map is None, the data better be (dets,), (samps,),
or (dets, samps). Will not work if dets.count == samps.count
"""
if axis_map is None:
if self[self._dets_name].count == self[self._samps_name].count:
raise ValueError("Cannot auto-detect axis_map when dets and "
"samps axes have equal lengths. axis_map "
"must be defined")
s = _get_shape(data)
if len(s) == 1:
if s[0] == self[self._dets_name].count:
## detector only flag. Turn into RangesMatrix
axis_map = [(0, self._dets_name)]
elif s[0] == self[self._samps_name].count:
axis_map = [(0, self.samps)]
else:
raise ValueError("FlagManager only takes data aligned with"
" dets and/or samps. Data of shape {}"
" is the wrong shape".format(s))
elif len(s) == 2:
if s[0] == self[self._dets_name].count and s[1] == self[
self._samps_name].count:
axis_map = [(0, self._dets_name), (1, self._samps_name)]
elif s[1] == self[self._dets_name].count and s[0] == self[
self._samps_name].count:
raise ValueError(
"FlagManager only takes 2D data aligned as"
" (dets, samps). Data of shape {}"
" is the wrong shape".format(s))
else:
raise ValueError(
"FlagManager only takes 2D data aligned as"
" (dets, samps). Data of shape {}"
" is the wrong shape".format(s))
else:
raise ValueError("FlagManager only takes data aligned with"
" dets and/or samps. Data of shape {}"
" is the wrong shape".format(s))
if len(axis_map) == 1 and axis_map[0][1] == self._dets_name:
### Change detector flags to RangesMatrix in the backend
x = Ranges(self.samps.count)
data = RangesMatrix([
Ranges.ones_like(x) if Y else Ranges.zeros_like(x)
for Y in data
])
axis_map = [(0, self._dets_name), (1, self._samps_name)]
super().wrap(name, data, axis_map, **kwargs)
def add_random_jumps(tod, params={}, signal='jumps', flag='true_jumps',
overwrite=False, verbose=False):
"""Add jumps (changes in DC level) to the data.
Args:
tod (AxisManager): the tod
params (dictionary): Use to overwrite the default values
n_jump is the expected number of jumps per detector per observation
sig_n_jump is the width of the expected distribution
h_jump is the expected higher of the jumps
sig_h_jump is the expected higher of the jumps
signal (string): name of the place to add the badness to the tod
flag (string): name of the flag to store where the glitches are
overwrite (bool): if true, write over signal. if false, add to signal
verbose (bool): print the number of glitches added
"""
gparams = {'n_jump' : 0.2, 'sig_n_jump' : 0.05,
'h_jump' : 0.1, 'sig_h_jump' : 0.3}
gparams.update(params)
params=gparams
n_tot = int(np.abs(params['sig_n_jump']*tod.dets.count*np.random.randn(1)
+ params['n_jump']*tod.dets.count))
places = np.random.randint( tod.dets.count*tod.samps.count, size=(n_tot,))
heights = np.random.randn( n_tot)*params['sig_h_jump'] + params['h_jump']
jumps = np.zeros( (tod.dets.count*tod.samps.count,) )
jumps[places] = heights
jumps = np.reshape( jumps, (tod.dets.count, tod.samps.count) )
truth = RangesMatrix( [Ranges.from_mask( g!=0) for g in jumps])
## actually make these jumps and not glitches
jumps = np.cumsum(jumps, axis=1)
if verbose:
print('Adding {} jumps to {} detectors'.format(n_tot,
np.sum( [len(t.ranges())>0 for t in truth]),))
if not signal in tod:
tod.wrap(signal, jumps,
[(0,tod.dets), (1,(tod.samps))] )
else:
if overwrite:
tod[signal] = jumps
else:
tod[signal] += jumps
if not flag in tod.flags:
tod.flags.wrap(flag, truth)
else:
if overwrite:
tod.flags[flag] = truth
else:
tod.flags[flag] += truth
def reduce(self,
flags=None,
method='union',
wrap=False,
new_flag=None,
remove_reduced=False):
"""Reduce (combine) flags in the FlagManager together.
Args:
flags: List of flags to collapse together. Uses their names.
If flags is None then all flags are reduced
method: How to collapse the data. Accepts 'union','intersect',
or function.
wrap: if True, add reduced flag to self
new_flag: name of new flag, required if wrap is True
remove_reduced: if True, remove all reduced flags from self
Returns:
out: reduced flag
"""
if flags is None:
## copy needed to no break things if removing flags
flags = self._fields.copy()
to_reduce = [self._fields[f] for f in flags]
if len(flags) == 0:
raise ValueError('Found zero flags to combine')
out = RangesMatrix(
[Ranges(self.samps.count) for det in self.dets.vals])
## need to add out to prevent flag ordering from causing errors
### (Ranges can't add to RangeMatrix, only other way around)
to_reduce[0] = out + to_reduce[0]
if method == 'union':
op = lambda x, y: x + y
elif method == 'intersect':
op = lambda x, y: x * y
else:
op = method
out = reduce(op, to_reduce)
# drop the fields if needed
if remove_reduced:
for f in flags:
self.move(f, None)
if wrap:
if new_flag is None:
raise ValueError("new_flag cannot be None if wrap is True")
self.wrap(new_flag, out)
return out
def test_mask(self):
"""Test conversion from/to boolean mask."""
# Make sure things work for various shapes.
for shape in [(10, 200), (10, 4, 200), (10, 0, 200), (0, 200),
(10, 4, 0), (200)]:
# Start from a mask.
m0 = (np.random.uniform(size=shape) > .8)
rm = RangesMatrix.from_mask(m0)
m1 = rm.mask()
self.assertEqual(rm.shape, m0.shape)
self.assertEqual(np.all(m0 == m1), True)
print(shape, m0.sum(), rm.shape)
def test_broadcast(self):
r0 = RangesMatrix.zeros((100, 1000))
self.assertCountEqual(r0.shape, (100, 1000))
self.assertCountEqual(r0[None, :, :].shape, (1, 100, 1000))
self.assertCountEqual(r0[:, None, :].shape, (100, 1, 1000))
# It should not be possible to pad or index beyond the
# outermost dimension. Ranges isn't very smart about this,
# but RangesMatrix can be.
with self.assertRaises(IndexError):
r0[:, :, None]
with self.assertRaises(IndexError):
r0[:, :, 0]
def add_random_glitches(tod, params={}, signal='glitches', flag='true_glitches',
overwrite=False, verbose=False):
"""Add glitches (spikes that just effect one data point) to the data.
Args:
tod (AxisManager): the tod
params (dictionary): Use to overwrite the default values
n_glitch: the expected number of glitches per detector per observation
sig_n_glitch: the width of the expected distribution
h_glitch: the expected higher of the glitches
sig_h_glitch: the expected higher of the glitches
signal (string): name of the place to add the badness to the tod
flag (string): name of the flag to store where the glitches are
overwrite (bool): if true, write over signal. if false, add to signal
verbose (bool): print the number of glitches added
"""
gparams = {'n_glitch' : 1, 'sig_n_glitch' : 0.1,
'h_glitch' : 10, 'sig_h_glitch' : 0.01}
gparams.update(params)
params=gparams
n_tot = int(np.abs(params['sig_n_glitch']*tod.dets.count*np.random.randn(1)
+ params['n_glitch']*tod.dets.count))
places = np.random.randint( tod.dets.count*tod.samps.count, size=(n_tot,))
heights = np.random.randn( n_tot)*params['sig_h_glitch'] + params['h_glitch']
glitches = np.zeros( (tod.dets.count*tod.samps.count,) )
glitches[places] = heights
glitches = np.reshape( glitches, (tod.dets.count, tod.samps.count) )
truth = RangesMatrix( [Ranges.from_mask( g!=0) for g in glitches])
if verbose:
print('Adding {} glitches to {} detectors'.format(n_tot,
np.sum( [len(t.ranges())>0 for t in truth]),))
if not signal in tod:
tod.wrap(signal, glitches,
[(0,tod.dets), (1,(tod.samps))] )
else:
if overwrite:
tod[signal] = glitches
else:
tod[signal] += glitches
if not flag in tod.flags:
tod.flags.wrap(flag, truth)
else:
if overwrite:
tod.flags[flag] = truth
else:
tod.flags[flag] += truth
def get_zeros(self, wrap=None):
"""
Return a correctly sized RangesMatrix for building cuts for the FlagManager
Args:
wrap: if not None, it is a string with which to add to the FlagManager
"""
out = RangesMatrix([
Ranges(self[self._samps_name].count)
for det in self[self._dets_name].vals
])
if not wrap is None:
self.wrap_dets_samps(wrap, out)
return self[wrap]
return out
def test_concat(self):
r0 = RangesMatrix.zeros((10, 100))
r1 = RangesMatrix.ones((10, 200))
r2 = RangesMatrix.ones((20, 100))
rc = RangesMatrix.concatenate([r0, r1], axis=1)
self.assertCountEqual(rc.shape, (10, 300))
self.assertEqual(rc[0].mask().sum(), r1[0].mask().sum())
rc = RangesMatrix.concatenate([r0, r2], axis=0)
self.assertCountEqual(rc.shape, (30, 100))
# These should fail due to shape incompat.
with self.assertRaises(ValueError):
rc = RangesMatrix.concatenate([r0, r1], axis=0)
with self.assertRaises(ValueError):
rc = RangesMatrix.concatenate([r0, r2], axis=1)
def test_matrix(self):
f = RangesMatrix.zeros((100, 100))
t = RangesMatrix.ones((100, 100))
self.assertCountEqual(f[0].mask(), ~(t[0].mask()))
self.assertCountEqual(f[0].mask(), (~t[0]).mask())
self.assertCountEqual(f[0].mask(), (~t)[0].mask())