def nothing(noth):
# If requested, remove the time gradient from all channels.
if remove_slope:
un_mask = sp.logical_not(ma.getmaskarray(NoiseData.data))
NoiseData.calc_time()
time = NoiseData.time
n_time = len(time)
# Test if the mask is the same for all slices. If it is, that greatly
# reduces the work as we only have to generate one set of polynomials.
all_masks_same = True
for jj in range(n_time):
if sp.all(un_mask[jj, ...] == un_mask[jj, 0, 0, 0]):
continue
else:
all_masks_same = False
break
if all_masks_same:
polys = misc.ortho_poly(time, 2, un_mask[:, 0, 0, 0], 0)
polys.shape = (2, len(time), 1, 1, 1)
else:
polys = misc.ortho_poly(time[:, None, None, None], 2, un_mask, 0)
# Subtract the slope mode (1th mode) out of the NoiseData.
slope_amps = sp.sum(polys[1, ...] * un_mask * NoiseData.data.filled(0),
0)
NoiseData.data -= polys[1, ...] * slope_amps
# Iteratively flag on sliding scale to get closer and closer to desired
# threshold.
n_time = Data.data.shape[0]
max_thres = sp.sqrt(n_time) / 2.
n_iter = 3
thresholds = (max_thres**(n_iter - 1 - sp.arange(n_iter)) *
thres**sp.arange(n_iter))**(1. / (n_iter - 1))
for threshold in thresholds:
# Get the deviation from the mean.
residuals = ma.anom(NoiseData.data, 0).filled(0)
# Get indices above the threshold.
mask = abs(residuals) > threshold * ma.std(NoiseData.data, 0)
# Mask the data.
Data.data[mask] = ma.masked
NoiseData.data[mask] = ma.masked
# Now flag for very noisey channels.
if max_noise_factor > 0:
vars = ma.var(NoiseData.data, 0)
mean_vars = ma.mean(vars, -1).filled(0)
bad_chans = vars.filled(0) > max_noise_factor * mean_vars[:, :, None]
Data.data[:, bad_chans] = ma.masked
NoiseData.data[:, bad_chans] = ma.masked
def nothing(noth):
# If requested, remove the time gradient from all channels.
if remove_slope:
un_mask = sp.logical_not(ma.getmaskarray(NoiseData.data))
NoiseData.calc_time()
time = NoiseData.time
n_time = len(time)
# Test if the mask is the same for all slices. If it is, that greatly
# reduces the work as we only have to generate one set of polynomials.
all_masks_same = True
for jj in range(n_time):
if sp.all(un_mask[jj,...] == un_mask[jj,0,0,0]):
continue
else:
all_masks_same = False
break
if all_masks_same:
polys = misc.ortho_poly(time, 2, un_mask[:,0,0,0], 0)
polys.shape = (2, len(time), 1, 1, 1)
else:
polys = misc.ortho_poly(time[:,None,None,None], 2, un_mask, 0)
# Subtract the slope mode (1th mode) out of the NoiseData.
slope_amps = sp.sum(polys[1,...] * un_mask * NoiseData.data.filled(0),
0)
NoiseData.data -= polys[1,...] * slope_amps
# Iteratively flag on sliding scale to get closer and closer to desired
# threshold.
n_time = Data.data.shape[0]
max_thres = sp.sqrt(n_time)/2.
n_iter = 3
thresholds = (max_thres ** (n_iter - 1 - sp.arange(n_iter))
* thres ** sp.arange(n_iter)) ** (1./(n_iter - 1))
for threshold in thresholds:
# Get the deviation from the mean.
residuals = ma.anom(NoiseData.data, 0).filled(0)
# Get indices above the threshold.
mask = abs(residuals) > threshold * ma.std(NoiseData.data, 0)
# Mask the data.
Data.data[mask] = ma.masked
NoiseData.data[mask] = ma.masked
# Now flag for very noisey channels.
if max_noise_factor > 0:
vars = ma.var(NoiseData.data, 0)
mean_vars = ma.mean(vars, -1).filled(0)
bad_chans = vars.filled(0) > max_noise_factor * mean_vars[:,:,None]
Data.data[:,bad_chans] = ma.masked
NoiseData.data[:,bad_chans] = ma.masked
def flag(Data, NoiseData, thres=3.0) :
"""Flags data for outliers using a signal subtracted data set.
Flags outliers of in a time stream data by looking at a version of the data
that has had the signal subtracted out of it. Each frequency channel,
polarization and cal state are treated separately.
Parameters
----------
Data : DataBlock Object
Data to be flaged. Upon exit, this object will have new flags.
NoiseData : DataBlock Object
Version of `Data` with the signal subtracted.
thres : float
Threshold for flagging in units of sigma (default is 3.0).
"""
# Get the deviation from the mean.
residuals = ma.anom(NoiseData.data, 0)
# Get indices above the threshold.
mask = abs(residuals) > thres*ma.std(NoiseData.data, 0)
# Mask the data.
Data.data[mask] = ma.masked
NoiseData.data[mask] = ma.masked
def measure(mode, x, y, x0, x1, thresh=0):
""" return the a measure of y in the window x0 to x1
"""
xt = x.view(numpy.ndarray) # strip Metaarray stuff -much faster!
v = y.view(numpy.ndarray)
xm = ma.masked_outside(xt, x0, x1).T
ym = ma.array(v, mask=ma.getmask(xm))
if mode == 'mean':
r1 = ma.mean(ym)
r2 = ma.std(ym)
if mode == 'max' or mode == 'maximum':
r1 = ma.max(ym)
r2 = xm[ma.argmax(ym)]
if mode == 'min' or mode == 'minimum':
r1 = ma.min(ym)
r2 = xm[ma.argmin(ym)]
if mode == 'median':
r1 = ma.median(ym)
r2 = 0
if mode == 'p2p': # peak to peak
r1 = ma.ptp(ym)
r2 = 0
if mode == 'std': # standard deviation
r1 = ma.std(ym)
r2 = 0
if mode == 'var': # variance
r1 = ma.var(ym)
r2 = 0
if mode == 'cumsum': # cumulative sum
r1 = ma.cumsum(ym) # Note: returns an array
r2 = 0
if mode == 'anom': # anomalies = difference from averge
r1 = ma.anom(ym) # returns an array
r2 = 0
if mode == 'sum':
r1 = ma.sum(ym)
r2 = 0
if mode == 'area' or mode == 'charge':
r1 = ma.sum(ym) / (ma.max(xm) - ma.min(xm))
r2 = 0
if mode == 'latency': # return first point that is > threshold
sm = ma.nonzero(ym > thresh)
r1 = -1 # use this to indicate no event detected
r2 = 0
if ma.count(sm) > 0:
r1 = sm[0][0]
r2 = len(sm[0])
if mode == 'count':
r1 = ma.count(ym)
r2 = 0
if mode == 'maxslope':
return (0, 0)
slope = numpy.array([])
win = ma.flatnotmasked_contiguous(ym)
st = int(len(win) / 20) # look over small ranges
for k in win: # move through the slope measurementwindow
tb = range(k - st, k + st) # get tb array
newa = numpy.array(self.dat[i][j, thisaxis, tb])
ppars = numpy.polyfit(
x[tb], ym[tb],
1) # do a linear fit - smooths the slope measures
slope = numpy.append(slope, ppars[0]) # keep track of max slope
r1 = numpy.amax(slope)
r2 = numpy.argmax(slope)
return (r1, r2)
def measure(mode, x, y, x0, x1, thresh = 0):
""" return the a measure of y in the window x0 to x1
"""
xt = x.view(numpy.ndarray) # strip Metaarray stuff -much faster!
v = y.view(numpy.ndarray)
xm = ma.masked_outside(xt, x0, x1).T
ym = ma.array(v, mask = ma.getmask(xm))
if mode == 'mean':
r1 = ma.mean(ym)
r2 = ma.std(ym)
if mode == 'max' or mode == 'maximum':
r1 = ma.max(ym)
r2 = xm[ma.argmax(ym)]
if mode == 'min' or mode == 'minimum':
r1 = ma.min(ym)
r2 = xm[ma.argmin(ym)]
if mode == 'median':
r1 = ma.median(ym)
r2 = 0
if mode == 'p2p': # peak to peak
r1 = ma.ptp(ym)
r2 = 0
if mode == 'std': # standard deviation
r1 = ma.std(ym)
r2 = 0
if mode == 'var': # variance
r1 = ma.var(ym)
r2 = 0
if mode == 'cumsum': # cumulative sum
r1 = ma.cumsum(ym) # Note: returns an array
r2 = 0
if mode == 'anom': # anomalies = difference from averge
r1 = ma.anom(ym) # returns an array
r2 = 0
if mode == 'sum':
r1 = ma.sum(ym)
r2 = 0
if mode == 'area' or mode == 'charge':
r1 = ma.sum(ym)/(ma.max(xm)-ma.min(xm))
r2 = 0
if mode == 'latency': # return first point that is > threshold
sm = ma.nonzero(ym > thresh)
r1 = -1 # use this to indicate no event detected
r2 = 0
if ma.count(sm) > 0:
r1 = sm[0][0]
r2 = len(sm[0])
if mode == 'count':
r1 = ma.count(ym)
r2 = 0
if mode == 'maxslope':
return(0,0)
slope = numpy.array([])
win = ma.flatnotmasked_contiguous(ym)
st = int(len(win)/20) # look over small ranges
for k in win: # move through the slope measurementwindow
tb = range(k-st, k+st) # get tb array
newa = numpy.array(self.dat[i][j, thisaxis, tb])
ppars = numpy.polyfit(x[tb], ym[tb], 1) # do a linear fit - smooths the slope measures
slope = numpy.append(slope, ppars[0]) # keep track of max slope
r1 = numpy.amax(slope)
r2 = numpy.argmax(slope)
return(r1, r2)
def measure(mode, x, y, x0, x1, thresh=0):
""" return the a measure of y in the window x0 to x1
"""
xm = ma.masked_outside(x, x0, x1) # .compressed()
ym = ma.array(y, mask=ma.getmask(xm)) # .compressed()
if mode == 'mean':
r1 = np.mean(ym)
r2 = np.std(ym)
if mode == 'max' or mode == 'maximum':
r1 = ma.max(ym)
r2 = xm[ma.argmax(ym)]
if mode == 'min' or mode == 'minimum':
r1 = ma.min(ym)
r2 = xm[ma.argmin(ym)]
if mode == 'minormax':
r1p = ma.max(ym)
r1n = ma.min(ym)
if ma.abs(r1p) > ma.abs(r1n):
r1 = r1p
r2 = xm[ma.argmax(ym)]
else:
r1 = r1n
r2 = xm[ma.argmin(ym)]
if mode == 'median':
r1 = ma.median(ym)
r2 = 0
if mode == 'p2p': # peak to peak
r1 = ma.ptp(ym)
r2 = 0
if mode == 'std': # standard deviation
r1 = ma.std(ym)
r2 = 0
if mode == 'var': # variance
r1 = ma.var(ym)
r2 = 0
if mode == 'cumsum': # cumulative sum
r1 = ma.cumsum(ym) # Note: returns an array
r2 = 0
if mode == 'anom': # anomalies = difference from averge
r1 = ma.anom(ym) # returns an array
r2 = 0
if mode == 'sum':
r1 = ma.sum(ym)
r2 = 0
if mode == 'area' or mode == 'charge':
r1 = ma.sum(ym) / (ma.max(xm) - ma.min(xm))
r2 = 0
if mode == 'latency': # return first point that is > threshold
sm = ma.nonzero(ym > thresh)
r1 = -1 # use this to indicate no event detected
r2 = 0
if ma.count(sm) > 0:
r1 = sm[0][0]
r2 = len(sm[0])
if mode == '1090': #measure 10-90% time, also returns max
r1 = ma.max(ym)
r2 = xm[ma.argmax(ym)]
y10 = 0.1 * r1
y90 = 0.9 * r1
sm1 = ma.nonzero(ym >= y10)
sm9 = ma.nonzero(ym >= y90)
r1 = xm[sm9] - xm[sm1]
if mode == 'count':
r1 = ma.count(ym)
r2 = 0
if mode == 'maxslope':
return (0, 0)
slope = np.array([])
win = ma.flatnotmasked_contiguous(ym)
st = int(len(win) / 20) # look over small ranges
for k in win: # move through the slope measurementwindow
tb = range(k - st, k + st) # get tb array
newa = np.array(self.dat[i][j, thisaxis, tb])
ppars = np.polyfit(
x[tb], ym[tb],
1) # do a linear fit - smooths the slope measures
slope = np.append(slope, ppars[0]) # keep track of max slope
r1 = np.amax(slope)
r2 = np.argmax(slope)
return (r1, r2)
def measure(mode, x, y, x0, x1, thresh=0):
""" return the a measure of y in the window x0 to x1
"""
xm = ma.masked_outside(x, x0, x1) # .compressed()
ym = ma.array(y, mask=ma.getmask(xm)) # .compressed()
if mode == "mean":
r1 = np.mean(ym)
r2 = np.std(ym)
if mode == "max" or mode == "maximum":
r1 = ma.max(ym)
r2 = xm[ma.argmax(ym)]
if mode == "min" or mode == "minimum":
r1 = ma.min(ym)
r2 = xm[ma.argmin(ym)]
if mode == "minormax":
r1p = ma.max(ym)
r1n = ma.min(ym)
if ma.abs(r1p) > ma.abs(r1n):
r1 = r1p
r2 = xm[ma.argmax(ym)]
else:
r1 = r1n
r2 = xm[ma.argmin(ym)]
if mode == "median":
r1 = ma.median(ym)
r2 = 0
if mode == "p2p": # peak to peak
r1 = ma.ptp(ym)
r2 = 0
if mode == "std": # standard deviation
r1 = ma.std(ym)
r2 = 0
if mode == "var": # variance
r1 = ma.var(ym)
r2 = 0
if mode == "cumsum": # cumulative sum
r1 = ma.cumsum(ym) # Note: returns an array
r2 = 0
if mode == "anom": # anomalies = difference from averge
r1 = ma.anom(ym) # returns an array
r2 = 0
if mode == "sum":
r1 = ma.sum(ym)
r2 = 0
if mode == "area" or mode == "charge":
r1 = ma.sum(ym) / (ma.max(xm) - ma.min(xm))
r2 = 0
if mode == "latency": # return first point that is > threshold
sm = ma.nonzero(ym > thresh)
r1 = -1 # use this to indicate no event detected
r2 = 0
if ma.count(sm) > 0:
r1 = sm[0][0]
r2 = len(sm[0])
if mode == "1090": # measure 10-90% time, also returns max
r1 = ma.max(ym)
r2 = xm[ma.argmax(ym)]
y10 = 0.1 * r1
y90 = 0.9 * r1
sm1 = ma.nonzero(ym >= y10)
sm9 = ma.nonzero(ym >= y90)
r1 = xm[sm9] - xm[sm1]
if mode == "count":
r1 = ma.count(ym)
r2 = 0
if mode == "maxslope":
return (0, 0)
slope = np.array([])
win = ma.flatnotmasked_contiguous(ym)
st = int(len(win) / 20) # look over small ranges
for k in win: # move through the slope measurementwindow
tb = range(k - st, k + st) # get tb array
newa = np.array(self.dat[i][j, thisaxis, tb])
ppars = np.polyfit(x[tb], ym[tb], 1) # do a linear fit - smooths the slope measures
slope = np.append(slope, ppars[0]) # keep track of max slope
r1 = np.amax(slope)
r2 = np.argmax(slope)
return (r1, r2)
def measure(mode, x, y, x0, x1, thresh=0, slopewin=1.0):
""" return the a measure of y in the window x0 to x1
"""
xm = ma.masked_outside(x, x0, x1)# .compressed()
ym = ma.array(y, mask = ma.getmask(xm))# .compressed()
if mode == 'mean':
r1 = np.mean(ym)
r2 = np.std(ym)
if mode == 'max' or mode == 'maximum':
r1 = ma.max(ym)
r2 = xm[ma.argmax(ym)]
if mode == 'min' or mode == 'minimum':
r1 = ma.min(ym)
r2 = xm[ma.argmin(ym)]
if mode == 'minormax':
r1p = ma.max(ym)
r1n = ma.min(ym)
if ma.abs(r1p) > ma.abs(r1n):
r1 = r1p
r2 = xm[ma.argmax(ym)]
else:
r1 = r1n
r2 = xm[ma.argmin(ym)]
if mode == 'median':
r1 = ma.median(ym)
r2 = 0
if mode == 'p2p': # peak to peak
r1 = ma.ptp(ym)
r2 = 0
if mode == 'std': # standard deviation
r1 = ma.std(ym)
r2 = 0
if mode == 'var': # variance
r1 = ma.var(ym)
r2 = 0
if mode == 'cumsum': # cumulative sum
r1 = ma.cumsum(ym) # Note: returns an array
r2 = 0
if mode == 'anom': # anomalies = difference from averge
r1 = ma.anom(ym) # returns an array
r2 = 0
if mode == 'sum':
r1 = ma.sum(ym)
r2 = 0
if mode == 'area' or mode == 'charge':
r1 = ma.sum(ym)/(ma.max(xm)-ma.min(xm))
r2 = 0
if mode == 'latency': # return first point that is > threshold
sm = ma.nonzero(ym > thresh)
r1 = -1 # use this to indicate no event detected
r2 = 0
if ma.count(sm) > 0:
r1 = sm[0][0]
r2 = len(sm[0])
if mode == '1090': #measure 10-90% time, also returns max
r1 = ma.max(ym)
r2 = xm[ma.argmax(ym)]
y10 = 0.1*r1
y90 = 0.9*r1
sm1 = ma.nonzero(ym >= y10)
sm9 = ma.nonzero(ym >= y90)
r1 = xm[sm9] - xm[sm1]
if mode == 'count':
r1 = ma.count(ym)
r2 = 0
if mode == 'maxslope':
slope = []
win = ma.flatnotmasked_contiguous(ym)
dt = x[1]-x[0]
st = int(slopewin/dt) # use slopewin duration window for fit.
print('st: ', st)
for k, w in enumerate(win): # move through the slope measurementwindow
tb = range(k-st, k+st) # get tb array
ppars = np.polyfit(x[tb], ym[tb], 1) # do a linear fit - smooths the slope measures
slope.append(ppars[0]) # keep track of max slope
r1 = np.max(slope)
r2 = np.argmax(slope)
return(r1, r2)
