def biweight(x, cst):
"""
Computes the biweight average and midvariance for a given 1D array.
Returns a tuple (biweight mean, biweight variance).
Parameters
----------
x: {ndarray}
Input Array
cst : {float}
Parameter controlling how outliers are censored.
Notes
-----
The function is restricted to 1D data only.
"""
assert (x.ndim == 1, "1D array only !")
xmed = ma.median(x, 0)
manom = x - xmed
mad = ma.median(ma.absolute(manom))
u_i = (manom/float(cst*mad))
u_i *= ma.less_equal(ma.absolute(u_i), 1.).astype(float)
w_i = (1-u_i**2)
if ma.count(w_i) > 0:
biw_m = xmed + ma.sum(manom * w_i**2)/ma.sum(w_i**2)
else:
biw_m = xmed
biw_sd = ma.sqrt(ma.count(x)*ma.sum(manom**2 * w_i**4))
biw_sd *= 1./ma.absolute(ma.sum(w_i * (1-5*u_i**2)))
return (biw_m, biw_sd.item())
def rate_of_change(data, v, cfg):
RoC = ma.masked_all_like(data[v])
RoC[1:] = ma.diff(data[v])
flag = np.zeros(data[v].shape, dtype='i1')
flag[np.nonzero(ma.absolute(RoC) <= cfg)] = 1
flag[np.nonzero(ma.absolute(RoC) > cfg)] = 4
flag[ma.getmaskarray(data[v])] = 9
return flag, RoC
def test_testUfuncs1(self):
# Test various functions such as sin, cos.
(x, y, a10, m1, m2, xm, ym, z, zm, xf, s) = self.d
assert_(eq(np.cos(x), cos(xm)))
assert_(eq(np.cosh(x), cosh(xm)))
assert_(eq(np.sin(x), sin(xm)))
assert_(eq(np.sinh(x), sinh(xm)))
assert_(eq(np.tan(x), tan(xm)))
assert_(eq(np.tanh(x), tanh(xm)))
with np.errstate(divide='ignore', invalid='ignore'):
assert_(eq(np.sqrt(abs(x)), sqrt(xm)))
assert_(eq(np.log(abs(x)), log(xm)))
assert_(eq(np.log10(abs(x)), log10(xm)))
assert_(eq(np.exp(x), exp(xm)))
assert_(eq(np.arcsin(z), arcsin(zm)))
assert_(eq(np.arccos(z), arccos(zm)))
assert_(eq(np.arctan(z), arctan(zm)))
assert_(eq(np.arctan2(x, y), arctan2(xm, ym)))
assert_(eq(np.absolute(x), absolute(xm)))
assert_(eq(np.equal(x, y), equal(xm, ym)))
assert_(eq(np.not_equal(x, y), not_equal(xm, ym)))
assert_(eq(np.less(x, y), less(xm, ym)))
assert_(eq(np.greater(x, y), greater(xm, ym)))
assert_(eq(np.less_equal(x, y), less_equal(xm, ym)))
assert_(eq(np.greater_equal(x, y), greater_equal(xm, ym)))
assert_(eq(np.conjugate(x), conjugate(xm)))
assert_(eq(np.concatenate((x, y)), concatenate((xm, ym))))
assert_(eq(np.concatenate((x, y)), concatenate((x, y))))
assert_(eq(np.concatenate((x, y)), concatenate((xm, y))))
assert_(eq(np.concatenate((x, y, x)), concatenate((x, ym, x))))
def test(self):
self.flags = {}
try:
threshold = self.cfg['threshold']
except KeyError:
module_logger.debug(
"Deprecated cfg format. It should contain a threshold item.")
threshold = self.cfg
try:
flag_good = self.cfg['flag_good']
except KeyError:
flag_good = 1
try:
flag_bad = self.cfg['flag_bad']
except KeyError:
flag_bad = 4
assert (np.size(threshold) == 1) \
and (threshold is not None) \
and (np.isfinite(threshold))
flag = np.zeros(self.data[self.varname].shape, dtype='i1')
feature = ma.absolute(self.features['rate_of_change'])
flag[np.nonzero(feature > threshold)] = self.flag_bad
flag[np.nonzero(feature <= threshold)] = self.flag_good
flag[ma.getmaskarray(self.data[self.varname])] = 9
self.flags['digit_roll_over'] = flag
def count_number_of_variables(self):
max_in_formula = 0
for clause in self.formula:
max_in_clause = max(absolute(clause))
if max_in_clause > max_in_formula:
max_in_formula = max_in_clause
return max_in_formula
def test_testUfuncs1(self):
# Test various functions such as sin, cos.
(x, y, a10, m1, m2, xm, ym, z, zm, xf, s) = self.d
assert_(eq(np.cos(x), cos(xm)))
assert_(eq(np.cosh(x), cosh(xm)))
assert_(eq(np.sin(x), sin(xm)))
assert_(eq(np.sinh(x), sinh(xm)))
assert_(eq(np.tan(x), tan(xm)))
assert_(eq(np.tanh(x), tanh(xm)))
with np.errstate(divide='ignore', invalid='ignore'):
assert_(eq(np.sqrt(abs(x)), sqrt(xm)))
assert_(eq(np.log(abs(x)), log(xm)))
assert_(eq(np.log10(abs(x)), log10(xm)))
assert_(eq(np.exp(x), exp(xm)))
assert_(eq(np.arcsin(z), arcsin(zm)))
assert_(eq(np.arccos(z), arccos(zm)))
assert_(eq(np.arctan(z), arctan(zm)))
assert_(eq(np.arctan2(x, y), arctan2(xm, ym)))
assert_(eq(np.absolute(x), absolute(xm)))
assert_(eq(np.equal(x, y), equal(xm, ym)))
assert_(eq(np.not_equal(x, y), not_equal(xm, ym)))
assert_(eq(np.less(x, y), less(xm, ym)))
assert_(eq(np.greater(x, y), greater(xm, ym)))
assert_(eq(np.less_equal(x, y), less_equal(xm, ym)))
assert_(eq(np.greater_equal(x, y), greater_equal(xm, ym)))
assert_(eq(np.conjugate(x), conjugate(xm)))
assert_(eq(np.concatenate((x, y)), concatenate((xm, ym))))
assert_(eq(np.concatenate((x, y)), concatenate((x, y))))
assert_(eq(np.concatenate((x, y)), concatenate((xm, y))))
assert_(eq(np.concatenate((x, y, x)), concatenate((x, ym, x))))
def get_histogram_centers(self):
diff_image = absolute(
cv2.blur(self.images[1], (5, 5)) -
cv2.blur(self.images[0], (5, 5)))
# diff_image = absolute(self.images[1] - self.images[0])
diff_image = cv2.cvtColor(diff_image, cv2.COLOR_BGR2GRAY)
ret, mask = cv2.threshold(diff_image, 127, 255, cv2.THRESH_BINARY)
mask_bool = mask.astype(np.bool)
nonzero_pos = nonzero(mask)
nonzero_num = len(nonzero_pos[0])
if FRACTION_MIN * self.num_pixels < nonzero_num < FRACTION_MAX * self.num_pixels:
data_set = np.vstack(nonzero_pos).T
mask_label, self.centers = make_kmeans(data_set)
mask_res = mask_bool.ravel()
mask_int = np.arange(len(mask_res))
mask0 = get_class_mask(mask_res, mask_label, mask_int, mask_bool,
0)
mask1 = get_class_mask(mask_res, mask_label, mask_int, mask_bool,
1)
self.histogram0 = get_histogram(self.images[1], mask0)
self.histogram1 = get_histogram(self.images[1], mask1)
if self._debug:
self._debug_output(mask, mask0, mask1)
def get_mask(self):
diff_image = absolute(
cv2.blur(self.images[1], (5, 5)) -
cv2.blur(self.images[0], (5, 5)))
# diff_image = absolute(self.images[1] - self.images[0])
diff_image = cv2.cvtColor(diff_image, cv2.COLOR_BGR2GRAY)
ret, mask = cv2.threshold(diff_image, 127, 255, cv2.THRESH_BINARY)
return mask
def log_linear_vinterp(T,P,levs):
'''
# Author Charles Doutriaux
# Version 1.1
# Expect 2D field here so there''s no reorder which I suspect to do a memory leak
# email: [email protected]
# Converts a field from sigma levels to pressure levels
# Log linear interpolation
# Input
# T : temperature on sigma levels
# P : pressure field from TOP (level 0) to BOTTOM (last level)
# levs : pressure levels to interplate to (same units as P)
# Output
# t : temperature on pressure levels (levs)
# External: Numeric'''
import numpy.ma as MA
## from numpy.oldnumeric.ma import ones,Float,greater,less,logical_and,where,equal,log,asarray,Float16
sh=P.shape
nsigma=sh[0] # Number of sigma levels
try:
nlev=len(levs) # Number of pressure levels
except:
nlev=1 # if only one level len(levs) would breaks
t=[]
for ilv in range(nlev): # loop through pressure levels
try:
lev=levs[ilv] # get value for the level
except:
lev=levs # only 1 level passed
# print ' ......... level:',lev
Pabv=MA.ones(P[0].shape,Numeric.Float)
Tabv=-Pabv # Temperature on sigma level Above
Tbel=-Pabv # Temperature on sigma level Below
Pbel=-Pabv # Pressure on sigma level Below
Pabv=-Pabv # Pressure on sigma level Above
for isg in range(1,nsigma): # loop from second sigma level to last one
## print 'Sigma level #',isg
a = MA.greater(P[isg], lev) # Where is the pressure greater than lev
b = MA.less(P[isg-1],lev) # Where is the pressure less than lev
# Now looks if the pressure level is in between the 2 sigma levels
# If yes, sets Pabv, Pbel and Tabv, Tbel
Pabv=MA.where(MA.logical_and(a,b),P[isg],Pabv) # Pressure on sigma level Above
Tabv=MA.where(MA.logical_and(a,b),T[isg],Tabv) # Temperature on sigma level Above
Pbel=MA.where(MA.logical_and(a,b),P[isg-1],Pbel) # Pressure on sigma level Below
Tbel=MA.where(MA.logical_and(a,b),T[isg-1],Tbel) # Temperature on sigma level Below
# end of for isg in range(1,nsigma)
# val=where(equal(Pbel,-1.),Pbel.missing_value,lev) # set to missing value if no data below lev if there is
tl=MA.masked_where(MA.equal(Pbel,-1.),MA.log(lev/MA.absolute(Pbel))/MA.log(Pabv/Pbel)*(Tabv-Tbel)+Tbel) # Interpolation
t.append(tl) # add a level to the output
# end of for ilv in range(nlev)
return asMA(t).astype(Numeric.Float32) # convert t to an array
def check_disjunction(clause, chromosome):
result = 0
for element in clause:
variable_index = absolute(element) - 1
if element > 0:
result |= chromosome[variable_index]
else:
result |= not chromosome[variable_index]
return result
def cum_rate_of_change(data, v, memory):
output = ma.masked_all_like(data[v])
output[1:] = ma.absolute(ma.diff(data[v]))
for i in range(2, output.size):
if output[i] < output[i-1]:
output[i] = (1 - memory) * output[i] + memory * output[i-1]
return output
def cum_rate_of_change(data, v, memory):
output = ma.masked_all_like(data[v])
output[1:] = ma.absolute(ma.diff(data[v]))
for i in range(2, output.size):
if output[i] < output[i - 1]:
output[i] = (1 - memory) * output[i] + memory * output[i - 1]
return output
def cum_rate_of_change(x, memory):
y = ma.fix_invalid(np.ones_like(x) * np.nan)
y[1:] = ma.absolute(ma.diff(x))
for i in range(2, y.size):
if y[i] < y[i - 1]:
y[i] = (1 - memory) * y[i] + memory * y[i - 1]
return y
def rate_of_change(data, v, cfg):
RoC = ma.masked_all_like(data[v])
RoC[1:] = ma.absolute(ma.diff(data[v]))
flag = np.zeros(data[v].shape, dtype='i1')
flag[np.nonzero(RoC <= cfg)] = 1
flag[np.nonzero(RoC > cfg)] = 4
flag[ma.getmaskarray(data[v])] = 9
return flag, RoC
def myfunction(d,mx,mn):
from numpy.ma import maximum,minimum,masked_where,absolute,greater,count
try:
d=masked_where(greater(absolute(d),9.9E19),d)
if count(d)==0 : return mx,mn
mx=float(maximum(mx,float(maximum(d))))
mn=float(minimum(mn,float(minimum(d))))
except:
for i in d:
mx,mn=myfunction(i,mx,mn)
return mx,mn
def myfunction(d, mx, mn):
from numpy.ma import maximum, minimum, masked_where, absolute, greater, count
try:
d = masked_where(greater(absolute(d), 9.9E19), d)
if count(d) == 0: return mx, mn
mx = float(maximum(mx, float(maximum(d))))
mn = float(minimum(mn, float(minimum(d))))
except:
for i in d:
mx, mn = myfunction(i, mx, mn)
return mx, mn
def woa_normbias(data, v, cfg):
if ('LATITUDE' in data.keys()) and ('LONGITUDE' in data.keys()):
if 'datetime' in data.keys():
d = data['datetime']
elif ('datetime' in data.attributes):
d0 = data.attributes['datetime']
if ('timeS' in data.keys()):
d = [d0 + timedelta(seconds=s) for s in data['timeS']]
else:
d = [data.attributes['datetime']]*len(data['LATITUDE']),
woa = woa_track_from_file(
d,
data['LATITUDE'],
data['LONGITUDE'],
cfg['file'],
varnames=cfg['vars'])
elif ('LATITUDE' in data.attributes.keys()) and \
('LONGITUDE' in data.attributes.keys()) and \
('PRES' in data.keys()):
woa = woa_profile(v,
data.attributes['datetime'],
data.attributes['LATITUDE'],
data.attributes['LONGITUDE'],
data['PRES'],
cfg)
if woa is None:
# self.logger.warn("%s - WOA is not available at this site" %
# self.name)
flag = np.zeros(data[v].shape, dtype='i1')
woa_normbias = ma.masked_all(data[v].shape)
return flag, woa_normbias
woa_bias = ma.absolute(data[v] - woa['woa_an'])
woa_normbias = woa_bias/woa['woa_sd']
flag = np.zeros(data[v].shape, dtype='i1')
ind = np.nonzero(woa_normbias <= cfg['sigma_threshold'])
flag[ind] = 1 # cfg['flag_good']
ind = np.nonzero(woa_normbias > cfg['sigma_threshold'])
flag[ind] = 3 # cfg['flag_bad']
# Flag as 9 any masked input value
flag[ma.getmaskarray(data[v])] = 9
return flag, woa_normbias
def _make_verts(self, U, V):
uv = ma.asarray(U+V*1j)
a = ma.absolute(uv)
if self.scale is None:
sn = max(10, math.sqrt(self.N))
scale = 1.8 * a.mean() * sn / self.span # crude auto-scaling
self.scale = scale
length = a/(self.scale*self.width)
X, Y = self._h_arrows(length)
# There seems to be a ma bug such that indexing
# a masked array with one element converts it to
# an ndarray.
theta = np.angle(ma.asarray(uv[..., np.newaxis]).filled(0))
xy = (X+Y*1j) * np.exp(1j*theta)*self.width
xy = xy[:,:,np.newaxis]
XY = ma.concatenate((xy.real, xy.imag), axis=2)
return XY
def test(self):
self.flags = {}
try:
threshold = self.cfg["threshold"]
except:
print("Deprecated cfg format. It should contain a threshold item.")
threshold = self.cfg
assert ((np.size(threshold) == 1) and (threshold is not None)
and (np.isfinite(threshold)))
flag = np.zeros(self.data[self.varname].shape, dtype="i1")
feature = ma.absolute(self.features["cum_rate_of_change"])
flag[np.nonzero(feature > threshold)] = self.flag_bad
flag[np.nonzero(feature <= threshold)] = self.flag_good
flag[ma.getmaskarray(self.data[self.varname])] = 9
self.flags["cum_rate_of_change"] = flag
def _make_verts(self, U, V):
uv = ma.asarray(U + V * 1j)
a = ma.absolute(uv)
if self.scale is None:
sn = max(10, math.sqrt(self.N))
scale = 1.8 * a.mean() * sn / self.span # crude auto-scaling
self.scale = scale
length = a / (self.scale * self.width)
X, Y = self._h_arrows(length)
# There seems to be a ma bug such that indexing
# a masked array with one element converts it to
# an ndarray.
theta = np.angle(ma.asarray(uv[..., np.newaxis]).filled(0))
xy = (X + Y * 1j) * np.exp(1j * theta) * self.width
xy = xy[:, :, np.newaxis]
XY = ma.concatenate((xy.real, xy.imag), axis=2)
return XY
def woa_normbias(data, v, cfg):
if ('LATITUDE' in data.keys()) and ('LONGITUDE' in data.keys()):
if 'datetime' in data.keys():
d = data['datetime']
elif ('datetime' in data.attributes):
d0 = data.attributes['datetime']
if ('timeS' in data.keys()):
d = [d0 + timedelta(seconds=s) for s in data['timeS']]
else:
d = [data.attributes['datetime']] * len(data['LATITUDE']),
woa = woa_track_from_file(d,
data['LATITUDE'],
data['LONGITUDE'],
cfg['file'],
varnames=cfg['vars'])
elif ('LATITUDE' in data.attributes.keys()) and \
('LONGITUDE' in data.attributes.keys()) and \
('PRES' in data.keys()):
woa = woa_profile(v, data.attributes['datetime'],
data.attributes['LATITUDE'],
data.attributes['LONGITUDE'], data['PRES'], cfg)
if woa is None:
# self.logger.warn("%s - WOA is not available at this site" %
# self.name)
flag = np.zeros(data[v].shape, dtype='i1')
woa_normbias = ma.masked_all(data[v].shape)
return flag, woa_normbias
woa_bias = ma.absolute(data[v] - woa['woa_an'])
woa_normbias = woa_bias / woa['woa_sd']
flag = np.zeros(data[v].shape, dtype='i1')
ind = np.nonzero(woa_normbias <= cfg['sigma_threshold'])
flag[ind] = 1 # cfg['flag_good']
ind = np.nonzero(woa_normbias > cfg['sigma_threshold'])
flag[ind] = 3 # cfg['flag_bad']
# Flag as 9 any masked input value
flag[ma.getmaskarray(data[v])] = 9
return flag, woa_normbias
def corr_proba(r, ndata, ndataset=2, dof=False):
"""Probability of rejecting correlations
- **r**: Correlation coefficient
- **ndata**: Number of records use for correlations
- **ndataset**, optional: Number of datasets (1 for autocorrelations, else 2) [default: 2]
.. todo::
This must be rewritten using :mod:`scipy.stats`
"""
# TODO: use scipy for betai and _gamma?
from genutil.salstat import betai,_gammaln
# Basic tests
ndata = MA.masked_equal(ndata,0,copy=0)
r = MV2.masked_where(MA.equal(MA.absolute(r),1.),r,copy=0)
# Degree of freedom
if dof:
df = ndata
else:
df = ndata-2-ndataset
# Advanced test: prevent extreme values by locally decreasing the dof
reduc = N.ones(r.shape)
z = None
while z is None or MA.count(MA.masked_greater(z,-600.)):
if z is not None:
imax = MA.argmin(z.ravel())
reduc.flat[imax] += 1
dfr = df/reduc
t = r*MV2.sqrt(dfr/((1.0-r)* (1.0+r)))
a = 0.5*dfr
b = 0.5
x = df/(dfr+t**2)
z = _gammaln(a+b)-_gammaln(a)-_gammaln(b)+a*MA.log(x)+b*MA.log(1.0-x)
# Perfom the test and format the variable
prob = MV2.masked_array(betai(a,b,x),axes=r.getAxisList())*100
prob.id = 'corr_proba' ; prob.name = prob.id
prob.long_name = 'Probability of rejection'
prob.units = '%'
return prob
def _make_verts(self, U, V):
uv = ma.asarray(U + V * 1j)
a = ma.absolute(uv)
if self.scale is None:
sn = max(10, math.sqrt(self.N))
scale = 1.8 * a.mean() * sn / self.span # crude auto-scaling
self.scale = scale
length = a / (self.scale * self.width)
X, Y = self._h_arrows(length)
if self.angles == 'xy':
theta = self._angles(U, V).filled(0)[:, np.newaxis]
elif self.angles == 'uv':
theta = np.angle(ma.asarray(uv[..., np.newaxis]).filled(0))
else:
theta = ma.asarray(self.angles * np.pi / 180.0).filled(0)
xy = (X + Y * 1j) * np.exp(1j * theta) * self.width
xy = xy[:, :, np.newaxis]
XY = ma.concatenate((xy.real, xy.imag), axis=2)
return XY
def corr_proba(r, ndata, ndataset=2, dof=False):
"""Probability of rejecting correlations
- **r**: Correlation coefficient
- **ndata**: Number of records use for correlations
- **ndataset**, optional: Number of datasets (1 for autocorrelations, else 2) [default: 2]
.. todo::
This must be rewritten using :mod:`scipy.stats`
"""
# Basic tests
ndata = MA.masked_equal(ndata,0,copy=0)
r = MV2.masked_where(MA.equal(MA.absolute(r),1.),r,copy=0)
# Degree of freedom
if dof:
df = ndata
else:
df = ndata-2-ndataset
# Advanced test: prevent extreme values by locally decreasing the dof
reduc = N.ones(r.shape)
z = None
while z is None or MA.count(MA.masked_greater(z,-600.)):
if z is not None:
imax = MA.argmin(z.ravel())
reduc.flat[imax] += 1
dfr = df/reduc
t = r*MV2.sqrt(dfr/((1.0-r)* (1.0+r)))
a = 0.5*dfr
b = 0.5
x = df/(dfr+t**2)
z = _gammaln(a+b)-_gammaln(a)-_gammaln(b)+a*MA.log(x)+b*MA.log(1.0-x)
# Perfom the test and format the variable
prob = MV2.masked_array(betai(a,b,x),axes=r.getAxisList())*100
prob.id = 'corr_proba' ; prob.name = prob.id
prob.long_name = 'Probability of rejection'
prob.units = '%'
return prob
def _make_verts(self, U, V):
uv = ma.asarray(U+V*1j)
a = ma.absolute(uv)
if self.scale is None:
sn = max(10, math.sqrt(self.N))
scale = 1.8 * a.mean() * sn / self.span # crude auto-scaling
self.scale = scale
length = a/(self.scale*self.width)
X, Y = self._h_arrows(length)
if self.angles == 'xy':
theta = self._angles(U, V).filled(0)[:,np.newaxis]
elif self.angles == 'uv':
theta = np.angle(ma.asarray(uv[..., np.newaxis]).filled(0))
else:
theta = ma.asarray(self.angles*np.pi/180.0).filled(0)
xy = (X+Y*1j) * np.exp(1j*theta)*self.width
xy = xy[:,:,np.newaxis]
XY = ma.concatenate((xy.real, xy.imag), axis=2)
return XY
def setcolorinterrupt(self):
val = self.stream.readAll().data()[-5000:]
self.timer.stop()
if val:
from numpy import fft
from numpy.ma import absolute
val = [i - 128 for i in val]
fur = absolute(fft.fft(val))
freq = fft.fftfreq(len(val), d=1 / 44100)
fur = fur[1:int(len(fur) / 2)]
freq = freq[1:int(len(freq) / 2)]
switch = {
'Smooth': self.smooth,
'Change': self.change,
'Flash': self.flash,
'Strob': self.strob
}
switch[self.mode](fur, freq)
timeout = 50 if self.mode == 'Smooth' else 200
self.timer.start(timeout)
def test(self):
self.flags = {}
try:
threshold = self.cfg["threshold"]
except KeyError:
module_logger.debug(
"Deprecated cfg format. It should contain a threshold item.")
threshold = self.cfg
assert ((np.size(threshold) == 1) and (threshold is not None)
and (np.isfinite(threshold)))
feature = ma.absolute(self.features["rate_of_change"])
flag = np.zeros(np.shape(self.data[self.varname]), dtype="i1")
flag[feature > threshold] = self.flag_bad
flag[feature <= threshold] = self.flag_good
x = np.atleast_1d(self.data[self.varname])
flag[ma.getmaskarray(x) | ~np.isfinite(x)] = 9
self.flags["digit_roll_over"] = flag
def tukey53H(x):
"""Spike test Tukey 53H from Goring & Nikora 2002
"""
N = len(x)
u1 = ma.masked_all(N)
for n in range(N-4):
if x[n:n+5].any():
u1[n+2] = ma.median(x[n:n+5])
u2 = ma.masked_all(N)
for n in range(N-2):
if u1[n:n+3].any():
u2[n+1] = ma.median(u1[n:n+3])
u3 = ma.masked_all(N)
u3[1:-1] = 0.25*(u2[:-2] + 2*u2[1:-1] + u2[2:])
Delta = ma.absolute(x-u3)
return Delta
def tukey53H(x):
"""Spike test Tukey 53H from Goring & Nikora 2002
"""
N = len(x)
u1 = ma.masked_all(N)
for n in range(N - 4):
if x[n:n + 5].any():
u1[n + 2] = ma.median(x[n:n + 5])
u2 = ma.masked_all(N)
for n in range(N - 2):
if u1[n:n + 3].any():
u2[n + 1] = ma.median(u1[n:n + 3])
u3 = ma.masked_all(N)
u3[1:-1] = 0.25 * (u2[:-2] + 2 * u2[1:-1] + u2[2:])
Delta = ma.absolute(x - u3)
return Delta
def evaluate(self, v, cfg):
self.flags[v] = {}
# Apply common flag for all points.
if 'common' in self.flags:
N = self.input[v].shape
for f in self.flags['common']:
self.flags[v][f] = self.flags['common'][f] * \
np.ones(N, dtype='i1')
if self.saveauxiliary:
if v not in self.auxiliary.keys():
self.auxiliary[v] = {}
if 'platform_identification' in cfg:
logging.warn("Sorry I'm not ready to evaluate platform_identification()")
if 'valid_geolocation' in cfg:
logging.warn("Sorry I'm not ready to evaluate valid_geolocation()")
if 'valid_speed' in cfg:
# Think about. ARGO also has a test valid_speed, but that is
# in respect to sucessive profiles. How is the best way to
# distinguish them here?
try:
if self.saveauxiliary:
self.flags[v]['valid_speed'], \
self.auxiliary[v]['valid_speed'] = \
possible_speed(self.input, cfg['valid_speed'])
except:
print("Fail on valid_speed")
if 'global_range' in cfg:
self.flags[v]['global_range'] = global_range(
self.input, v, cfg['global_range'])
if 'regional_range' in cfg:
logging.warn("Sorry, I'm no ready to evaluate regional_range()")
if 'pressure_increasing' in cfg:
logging.warn("Sorry, I'm no ready to evaluate pressure_increasing()")
if 'profile_envelop' in cfg:
self.flags[v]['profile_envelop'] = profile_envelop(
self.input, cfg['profile_envelop'], v)
if 'gradient' in cfg:
y = Gradient(self.input, v, cfg['gradient'])
y.test()
if self.saveauxiliary:
self.auxiliary[v]['gradient'] = y.features['gradient']
self.flags[v]['gradient'] = y.flags['gradient']
if 'gradient_depthconditional' in cfg:
cfg_tmp = cfg['gradient_depthconditional']
g = gradient(self.input[v])
flag = np.zeros(g.shape, dtype='i1')
# Flag as 9 any masked input value
flag[ma.getmaskarray(self.input[v])] = 9
# ---- Shallow zone -----------------
threshold = cfg_tmp['shallow_max']
flag[np.nonzero( \
(self['PRES'] <= cfg_tmp['pressure_threshold']) & \
(g > threshold))] \
= 4
flag[np.nonzero( \
(self['PRES'] <= cfg_tmp['pressure_threshold']) & \
(g <= threshold))] \
= 1
# ---- Deep zone --------------------
threshold = cfg_tmp['deep_max']
flag[np.nonzero( \
(self['PRES'] > cfg_tmp['pressure_threshold']) & \
(g > threshold))] \
= 4
flag[np.nonzero( \
(self['PRES'] > cfg_tmp['pressure_threshold']) & \
(g <= threshold))] \
= 1
self.flags[v]['gradient_depthconditional'] = flag
if 'spike' in cfg:
y = Spike(self.input, v, cfg['spike'])
y.test()
if self.saveauxiliary:
self.auxiliary[v]['spike'] = y.features['spike']
self.flags[v]['spike'] = y.flags['spike']
if 'spike_depthconditional' in cfg:
cfg_tmp = cfg['spike_depthconditional']
s = spike(self.input[v])
flag = np.zeros(s.shape, dtype='i1')
# Flag as 9 any masked input value
flag[ma.getmaskarray(self.input[v])] = 9
# ---- Shallow zone -----------------
threshold = cfg_tmp['shallow_max']
flag[np.nonzero( \
(self['PRES'] <= cfg_tmp['pressure_threshold']) & \
(s > threshold))] \
= 4
flag[np.nonzero( \
(self['PRES'] <= cfg_tmp['pressure_threshold']) & \
(s <= threshold))] \
= 1
# ---- Deep zone --------------------
threshold = cfg_tmp['deep_max']
flag[np.nonzero( \
(self['PRES'] > cfg_tmp['pressure_threshold']) & \
(s > threshold))] \
= 4
flag[np.nonzero( \
(self['PRES'] > cfg_tmp['pressure_threshold']) & \
(s <= threshold))] \
= 1
self.flags[v]['spike_depthconditional'] = flag
if 'stuck_value' in cfg:
logging.warn("Sorry I'm not ready to evaluate stuck_value()")
if 'grey_list' in cfg:
logging.warn("Sorry I'm not ready to evaluate grey_list()")
if 'gross_sensor_drift' in cfg:
logging.warn("Sorry I'm not ready to evaluate gross_sensor_drift()")
if 'frozen_profile' in cfg:
logging.warn("Sorry I'm not ready to evaluate frozen_profile()")
if 'deepest_pressure' in cfg:
logging.warn("Sorry I'm not ready to evaluate deepest_pressure()")
if 'tukey53H_norm' in cfg:
y = Tukey53H(self.input, v, cfg['tukey53H_norm'])
y.test()
if self.saveauxiliary:
self.auxiliary[v]['tukey53H_norm'] = \
y.features['tukey53H_norm']
self.flags[v]['tukey53H_norm'] = y.flags['tukey53H_norm']
#if 'spike_depthsmooth' in cfg:
# from maud.window_func import _weight_hann as wfunc
# cfg_tmp = cfg['spike_depthsmooth']
# cfg_tmp['dzwindow'] = 10
# smooth = ma.masked_all(self.input[v].shape)
# z = ped['pressure']
# for i in range(len(self.input[v])):
# ind = np.nonzero(ma.absolute(z-z[i]) < \
# cfg_tmp['dzwindow'])[0]
# ind = ind[ind != i]
# w = wfunc(z[ind]-z[i], cfg_tmp['dzwindow'])
# smooth[i] = (T[ind]*w).sum()/w.sum()
# ARGO, test #12. (10C, 5PSU)
if 'digit_roll_over' in cfg:
threshold = cfg['digit_roll_over']
s = step(self.input[v])
if self.saveauxiliary:
self.auxiliary[v]['step'] = s
flag = np.zeros(s.shape, dtype='i1')
# Flag as 9 any masked input value
flag[ma.getmaskarray(self.input[v])] = 9
flag[np.nonzero(ma.absolute(s) > threshold)] = 4
flag[np.nonzero(ma.absolute(s) <= threshold)] = 1
self.flags[v]['digit_roll_over'] = flag
if 'bin_spike' in cfg:
y = Bin_Spike(self.input, v, cfg['bin_spike'])
# y.test()
if self.saveauxiliary:
self.auxiliary[v]['bin_spike'] = y.features['bin_spike']
# self.flags[v]['bin_spike'] = y.flags['bin_spike']
if 'density_inversion' in cfg:
try:
if self.saveauxiliary:
self.flags[v]['density_inversion'], \
self.auxiliary[v]['density_step'] = \
density_inversion(
self.input,
cfg['density_inversion'],
saveaux=True)
else:
self.flags[v]['density_inversion'] = density_inversion(
self.input, cfg['density_inversion'])
except:
print("Fail on density_inversion")
if 'woa_normbias' in cfg:
y = WOA_NormBias(self.input, v, cfg['woa_normbias'])
# self.attributes)
y.test()
if self.saveauxiliary:
for f in y.features:
self.auxiliary[v][f] = y.features[f]
self.flags[v]['woa_normbias'] = y.flags['woa_normbias']
#if 'pstep' in cfg:
# ind = np.isfinite(self.input[v])
# ind = ma.getmaskarray(self.input[v])
# if self.saveauxiliary:
# self.auxiliary[v]['pstep'] = ma.concatenate(
# [ma.masked_all(1),
# np.diff(self.input['PRES'][ind])])
if 'rate_of_change' in cfg:
self.flags[v]['rate_of_change'], RoC = \
rate_of_change(self.input, v, cfg['rate_of_change'])
if self.saveauxiliary:
self.auxiliary[v]['rate_of_change'] = RoC
if 'cum_rate_of_change' in cfg:
x = cum_rate_of_change(self.input, v,
cfg['cum_rate_of_change']['memory'])
self.flags[v]['cum_rate_of_change'] = np.zeros(x.shape, dtype='i1')
self.flags[v]['cum_rate_of_change'][
np.nonzero(x <= cfg['cum_rate_of_change']['threshold'])
] = 1
self.flags[v]['cum_rate_of_change'][
np.nonzero(x > cfg['cum_rate_of_change']['threshold'])
] = 4
self.flags[v]['cum_rate_of_change'][
ma.getmaskarray(self.input[v])] = 9
# FIXME: the Anomaly Detection and Fuzzy require some features
# to be estimated previously. Generalize this.
if 'anomaly_detection' in cfg:
features = {}
for f in cfg['anomaly_detection']['features']:
if f == 'spike':
features['spike'] = spike(self.input[v])
elif f == 'gradient':
features['gradient'] = gradient(self.input[v])
elif f == 'tukey53H_norm':
features['tukey53H_norm'] = tukey53H_norm(self.input[v])
elif f == 'rate_of_change':
RoC = ma.masked_all_like(self.input[v])
RoC[1:] = ma.absolute(ma.diff(self.input[v]))
features['rate_of_change'] = RoC
elif (f == 'woa_normbias'):
y = WOA_NormBias(self.input, v, {})
features['woa_normbias'] = \
np.abs(y.features['woa_normbias'])
else:
logging.error("Sorry, I can't evaluate anomaly_detection with: %s" % f)
self.flags[v]['anomaly_detection'] = \
anomaly_detection(features, cfg['anomaly_detection'])
if 'morello2014' in cfg:
self.flags[v]['morello2014'] = morello2014(
features=self.auxiliary[v],
cfg=cfg['morello2014'])
if 'fuzzylogic' in cfg:
features = {}
for f in cfg['fuzzylogic']['features']:
if f == 'spike':
features['spike'] = spike(self.input[v])
elif f == 'gradient':
features['gradient'] = gradient(self.input[v])
elif f == 'tukey53H_norm':
features['tukey53H_norm'] = tukey53H_norm(self.input[v],
k=1.5)
elif f == 'rate_of_change':
RoC = ma.masked_all_like(data[v])
RoC[1:] = ma.absolute(ma.diff(data[v]))
features['rate_of_change'] = RoC
elif (f == 'woa_normbias'):
y = WOA_NormBias(self.input, v, {})
features['woa_normbias'] = \
np.abs(y.features['woa_normbias'])
else:
logging.error("Sorry, I can't evaluate fuzzylogic with: %s" % f)
self.flags[v]['fuzzylogic'] = fuzzylogic(
features=features,
cfg=cfg['fuzzylogic'])
self.flags[v]['overall'] = combined_flag(self.flags[v])
pylab.plot(x[:100],Y[:100], 'o') pylab.plot(x[:100],y1[:100], 'k--', lw=2) pylab.plot(x[:100],y2[:100], 'k-.', lw=2) pylab.plot(x[:100],y1[:100]+y2[:100], 'k', lw=2) pylab.plot(x[:100],f1[:100], 'r', lw=1) pylab.plot(x[:100],f2[:100], 'g', lw=1) pylab.plot(x[:100],f3[:100], 'b', lw=1) pylab.axhline(y=Y.mean(), c='0.7') pylab.figure() T=1./np.fft.fftfreq(N) ff = np.fft.fft(Y) ff1 = np.fft.fft(f1) ff2 = np.fft.fft(f2) ff3 = np.fft.fft(f3) pylab.plot(T[1:N/2], ma.absolute(ff[1:N/2]), 'k') pylab.plot(T[1:N/2], ma.absolute(ff1[1:N/2]),'r') pylab.plot(T[1:N/2], ma.absolute(ff2[1:N/2]),'g') pylab.plot(T[1:N/2], ma.absolute(ff3[1:N/2]),'b') pylab.show() pylab.show()
def amp(a_single_subRegion, special_parameters):
# TODO: this 'special_parameters' maybe be useful
amplitude = array(absolute(a_single_subRegion))
return amplitude
def evaluate(self, v, cfg):
self.flags[v] = {}
if self.saveauxiliary:
if v not in self.auxiliary.keys():
self.auxiliary[v] = {}
if 'platform_identification' in cfg:
logging.warn(
"Sorry I'm not ready to evaluate platform_identification()")
if 'valid_geolocation' in cfg:
logging.warn("Sorry I'm not ready to evaluate valid_geolocation()")
if 'valid_speed' in cfg:
# Think about. ARGO also has a test valid_speed, but that is
# in respect to sucessive profiles. How is the best way to
# distinguish them here?
try:
if self.saveauxiliary:
self.flags[v]['valid_speed'], \
self.auxiliary[v]['valid_speed'] = \
possible_speed(self.input, cfg['valid_speed'])
except:
print("Fail on valid_speed")
if 'global_range' in cfg:
self.flags[v]['global_range'] = np.zeros(self.input[v].shape,
dtype='i1')
# Flag as 9 any masked input value
self.flags[v]['global_range'][ma.getmaskarray(self.input[v])] = 9
ind = (self.input[v] >= cfg['global_range']['minval']) & \
(self.input[v] <= cfg['global_range']['maxval'])
self.flags[v]['global_range'][np.nonzero(ind)] = 1
ind = (self.input[v] < cfg['global_range']['minval']) | \
(self.input[v] > cfg['global_range']['maxval'])
self.flags[v]['global_range'][np.nonzero(ind)] = 4
if 'regional_range' in cfg:
logging.warn("Sorry, I'm no ready to evaluate regional_range()")
if 'pressure_increasing' in cfg:
logging.warn(
"Sorry, I'm no ready to evaluate pressure_increasing()")
if 'profile_envelop' in cfg:
self.flags[v]['profile_envelop'] = profile_envelop(
self.input, cfg['profile_envelop'], v)
if 'gradient' in cfg:
threshold = cfg['gradient']
g = gradient(self.input[v])
if self.saveauxiliary:
self.auxiliary[v]['gradient'] = g
flag = np.zeros(g.shape, dtype='i1')
flag[ma.getmaskarray(self.input[v])] = 9
flag[np.nonzero(g > threshold)] = 4
flag[np.nonzero(g <= threshold)] = 1
self.flags[v]['gradient'] = flag
if 'gradient_depthconditional' in cfg:
cfg_tmp = cfg['gradient_depthconditional']
g = gradient(self.input[v])
flag = np.zeros(g.shape, dtype='i1')
# Flag as 9 any masked input value
flag[ma.getmaskarray(self.input[v])] = 9
# ---- Shallow zone -----------------
threshold = cfg_tmp['shallow_max']
flag[np.nonzero( \
(self['PRES'] <= cfg_tmp['pressure_threshold']) & \
(g > threshold))] \
= 4
flag[np.nonzero( \
(self['PRES'] <= cfg_tmp['pressure_threshold']) & \
(g <= threshold))] \
= 1
# ---- Deep zone --------------------
threshold = cfg_tmp['deep_max']
flag[np.nonzero( \
(self['PRES'] > cfg_tmp['pressure_threshold']) & \
(g > threshold))] \
= 4
flag[np.nonzero( \
(self['PRES'] > cfg_tmp['pressure_threshold']) & \
(g <= threshold))] \
= 1
self.flags[v]['gradient_depthconditional'] = flag
if 'spike' in cfg:
threshold = cfg['spike']
s = spike(self.input[v])
if self.saveauxiliary:
self.auxiliary[v]['spike'] = s
flag = np.zeros(s.shape, dtype='i1')
# Flag as 9 any masked input value
flag[ma.getmaskarray(self.input[v])] = 9
flag[np.nonzero(s > threshold)] = 4
flag[np.nonzero(s <= threshold)] = 1
self.flags[v]['spike'] = flag
if 'spike_depthconditional' in cfg:
cfg_tmp = cfg['spike_depthconditional']
s = spike(self.input[v])
flag = np.zeros(s.shape, dtype='i1')
# Flag as 9 any masked input value
flag[ma.getmaskarray(self.input[v])] = 9
# ---- Shallow zone -----------------
threshold = cfg_tmp['shallow_max']
flag[np.nonzero( \
(self['PRES'] <= cfg_tmp['pressure_threshold']) & \
(g > threshold))] \
= 4
flag[np.nonzero( \
(self['PRES'] <= cfg_tmp['pressure_threshold']) & \
(g <= threshold))] \
= 1
# ---- Deep zone --------------------
threshold = cfg_tmp['deep_max']
flag[np.nonzero( \
(self['PRES'] > cfg_tmp['pressure_threshold']) & \
(g > threshold))] \
= 4
flag[np.nonzero( \
(self['PRES'] > cfg_tmp['pressure_threshold']) & \
(g <= threshold))] \
= 1
self.flags[v]['spike_depthconditional'] = flag
if 'stuck_value' in cfg:
logging.warn("Sorry I'm not ready to evaluate stuck_value()")
if 'grey_list' in cfg:
logging.warn("Sorry I'm not ready to evaluate grey_list()")
if 'gross_sensor_drift' in cfg:
logging.warn(
"Sorry I'm not ready to evaluate gross_sensor_drift()")
if 'frozen_profile' in cfg:
logging.warn("Sorry I'm not ready to evaluate frozen_profile()")
if 'deepest_pressure' in cfg:
logging.warn("Sorry I'm not ready to evaluate deepest_pressure()")
if 'tukey53H_norm' in cfg:
"""
I slightly modified the Goring & Nikora 2002. It is
expected that CTD profiles has a typical depth
structure, with a range between surface and bottom.
"""
s = tukey53H_norm(self.input[v],
k=cfg['tukey53H_norm']['k'],
l=cfg['tukey53H_norm']['l'])
if self.saveauxiliary:
self.auxiliary[v]['tukey53H_norm'] = s
threshold = cfg['tukey53H_norm']['k']
flag = np.zeros(s.shape, dtype='i1')
# Flag as 9 any masked input value
flag[ma.getmaskarray(self.input[v])] = 9
flag[np.nonzero(s > threshold)] = 4
flag[np.nonzero(s <= threshold)] = 1
self.flags[v]['tukey53H_norm'] = flag
#if 'spike_depthsmooth' in cfg:
# from maud.window_func import _weight_hann as wfunc
# cfg_tmp = cfg['spike_depthsmooth']
# cfg_tmp['dzwindow'] = 10
# smooth = ma.masked_all(self.input[v].shape)
# z = ped['pressure']
# for i in range(len(self.input[v])):
# ind = np.nonzero(ma.absolute(z-z[i]) < \
# cfg_tmp['dzwindow'])[0]
# ind = ind[ind != i]
# w = wfunc(z[ind]-z[i], cfg_tmp['dzwindow'])
# smooth[i] = (T[ind]*w).sum()/w.sum()
# ARGO, test #12. (10C, 5PSU)
if 'digit_roll_over' in cfg:
threshold = cfg['digit_roll_over']
s = step(self.input[v])
if self.saveauxiliary:
self.auxiliary[v]['step'] = s
flag = np.zeros(s.shape, dtype='i1')
# Flag as 9 any masked input value
flag[ma.getmaskarray(self.input[v])] = 9
flag[np.nonzero(ma.absolute(s) > threshold)] = 4
flag[np.nonzero(ma.absolute(s) <= threshold)] = 1
self.flags[v]['digit_roll_over'] = flag
if 'bin_spike' in cfg:
bin = bin_spike(self.input[v], cfg['bin_spike'])
if self.saveauxiliary:
self.auxiliary[v]['bin_spike'] = bin
if 'density_inversion' in cfg:
try:
if self.saveauxiliary:
self.flags[v]['density_inversion'], \
self.auxiliary[v]['density_step'] = \
density_inversion(
self.input,
cfg['density_inversion'],
saveaux=True)
else:
self.flags[v]['density_inversion'] = density_inversion(
self.input, cfg['density_inversion'])
except:
print("Fail on density_inversion")
if 'woa_normbias' in cfg:
if self.saveauxiliary:
self.flags[v]['woa_normbias'], \
self.auxiliary[v]['woa_relbias'] = \
woa_normbias(self.input, v, cfg['woa_normbias'])
#for k in woa.keys():
# self.auxiliary[v][k] = woa[k]
#self.auxiliary[v]['woa_bias'] = woa_bias
#self.auxiliary[v]['woa_relbias'] = woa_bias/woa['woa_sd']
else:
self.flags[v]['woa_normbias'], \
tmp = \
woa_normbias(self.input, v, cfg['woa_normbias'])
del (tmp)
#if 'pstep' in cfg:
# ind = np.isfinite(self.input[v])
# ind = ma.getmaskarray(self.input[v])
# if self.saveauxiliary:
# self.auxiliary[v]['pstep'] = ma.concatenate(
# [ma.masked_all(1),
# np.diff(self.input['PRES'][ind])])
if 'RoC' in cfg:
x = ma.concatenate([ma.masked_all(1), ma.diff(self.input[v])])
if self.saveauxiliary:
self.auxiliary[v]['RoC'] = x
self.flags[v]['RoC'] = np.zeros(x.shape, dtype='i1')
self.flags[v]['RoC'][np.nonzero(x <= cfg['RoC'])] = 1
self.flags[v]['RoC'][np.nonzero(x > cfg['RoC'])] = 4
self.flags[v]['RoC'][ma.getmaskarray(self.input[v])] = 9
if 'anomaly_detection' in cfg:
features = {}
for f in cfg['anomaly_detection']['features']:
if f == 'spike':
features['spike'] = spike(self.input[v])
elif f == 'gradient':
features['gradient'] = gradient(self.input[v])
elif f == 'tukey53H_norm':
features['tukey53H_norm'] = tukey53H_norm(self.input[v],
k=1.5)
else:
logging.error(
"Sorry, I can't evaluate anomaly_detection with: %s" %
f)
self.flags[v]['anomaly_detection'] = \
anomaly_detection(features, cfg['anomaly_detection'])
if 'fuzzylogic' in cfg:
self.flags[v]['fuzzylogic'] = fuzzylogic(self.auxiliary[v], v,
cfg['fuzzylogic'])
def evaluate(self, v, cfg):
self.flags[v] = {}
if self.saveauxiliary:
if v not in self.auxiliary.keys():
self.auxiliary[v] = {}
if 'platform_identification' in cfg:
logging.warn("Sorry I'm not ready to evaluate platform_identification()")
if 'valid_geolocation' in cfg:
logging.warn("Sorry I'm not ready to evaluate valid_geolocation()")
if 'valid_speed' in cfg:
# Think about. ARGO also has a test valid_speed, but that is
# in respect to sucessive profiles. How is the best way to
# distinguish them here?
try:
if self.saveauxiliary:
self.flags[v]['valid_speed'], \
self.auxiliary[v]['valid_speed'] = \
possible_speed(self.input, cfg['valid_speed'])
except:
print("Fail on valid_speed")
if 'global_range' in cfg:
self.flags[v]['global_range'] = np.zeros(self.input[v].shape,
dtype='i1')
# Flag as 9 any masked input value
self.flags[v]['global_range'][ma.getmaskarray(self.input[v])] = 9
ind = (self.input[v] >= cfg['global_range']['minval']) & \
(self.input[v] <= cfg['global_range']['maxval'])
self.flags[v]['global_range'][np.nonzero(ind)] = 1
ind = (self.input[v] < cfg['global_range']['minval']) | \
(self.input[v] > cfg['global_range']['maxval'])
self.flags[v]['global_range'][np.nonzero(ind)] = 4
if 'regional_range' in cfg:
logging.warn("Sorry, I'm no ready to evaluate regional_range()")
if 'pressure_increasing' in cfg:
logging.warn("Sorry, I'm no ready to evaluate pressure_increasing()")
if 'profile_envelop' in cfg:
self.flags[v]['profile_envelop'] = profile_envelop(
self.input, cfg['profile_envelop'], v)
if 'gradient' in cfg:
threshold = cfg['gradient']
g = gradient(self.input[v])
if self.saveauxiliary:
self.auxiliary[v]['gradient'] = g
flag = np.zeros(g.shape, dtype='i1')
flag[ma.getmaskarray(self.input[v])] = 9
flag[np.nonzero(g > threshold)] = 4
flag[np.nonzero(g <= threshold)] = 1
self.flags[v]['gradient'] = flag
if 'gradient_depthconditional' in cfg:
cfg_tmp = cfg['gradient_depthconditional']
g = gradient(self.input[v])
flag = np.zeros(g.shape, dtype='i1')
# Flag as 9 any masked input value
flag[ma.getmaskarray(self.input[v])] = 9
# ---- Shallow zone -----------------
threshold = cfg_tmp['shallow_max']
flag[np.nonzero( \
(self['PRES'] <= cfg_tmp['pressure_threshold']) & \
(g > threshold))] \
= 4
flag[np.nonzero( \
(self['PRES'] <= cfg_tmp['pressure_threshold']) & \
(g <= threshold))] \
= 1
# ---- Deep zone --------------------
threshold = cfg_tmp['deep_max']
flag[np.nonzero( \
(self['PRES'] > cfg_tmp['pressure_threshold']) & \
(g > threshold))] \
= 4
flag[np.nonzero( \
(self['PRES'] > cfg_tmp['pressure_threshold']) & \
(g <= threshold))] \
= 1
self.flags[v]['gradient_depthconditional'] = flag
if 'spike' in cfg:
threshold = cfg['spike']
s = spike(self.input[v])
if self.saveauxiliary:
self.auxiliary[v]['spike'] = s
flag = np.zeros(s.shape, dtype='i1')
# Flag as 9 any masked input value
flag[ma.getmaskarray(self.input[v])] = 9
flag[np.nonzero(s > threshold)] = 4
flag[np.nonzero(s <= threshold)] = 1
self.flags[v]['spike'] = flag
if 'spike_depthconditional' in cfg:
cfg_tmp = cfg['spike_depthconditional']
s = spike(self.input[v])
flag = np.zeros(s.shape, dtype='i1')
# Flag as 9 any masked input value
flag[ma.getmaskarray(self.input[v])] = 9
# ---- Shallow zone -----------------
threshold = cfg_tmp['shallow_max']
flag[np.nonzero( \
(self['PRES'] <= cfg_tmp['pressure_threshold']) & \
(g > threshold))] \
= 4
flag[np.nonzero( \
(self['PRES'] <= cfg_tmp['pressure_threshold']) & \
(g <= threshold))] \
= 1
# ---- Deep zone --------------------
threshold = cfg_tmp['deep_max']
flag[np.nonzero( \
(self['PRES'] > cfg_tmp['pressure_threshold']) & \
(g > threshold))] \
= 4
flag[np.nonzero( \
(self['PRES'] > cfg_tmp['pressure_threshold']) & \
(g <= threshold))] \
= 1
self.flags[v]['spike_depthconditional'] = flag
if 'stuck_value' in cfg:
logging.warn("Sorry I'm not ready to evaluate stuck_value()")
if 'grey_list' in cfg:
logging.warn("Sorry I'm not ready to evaluate grey_list()")
if 'gross_sensor_drift' in cfg:
logging.warn("Sorry I'm not ready to evaluate gross_sensor_drift()")
if 'frozen_profile' in cfg:
logging.warn("Sorry I'm not ready to evaluate frozen_profile()")
if 'deepest_pressure' in cfg:
logging.warn("Sorry I'm not ready to evaluate deepest_pressure()")
if 'tukey53H_norm' in cfg:
"""
I slightly modified the Goring & Nikora 2002. It is
expected that CTD profiles has a typical depth
structure, with a range between surface and bottom.
"""
s = tukey53H_norm(self.input[v],
k = cfg['tukey53H_norm']['k'],
l = cfg['tukey53H_norm']['l'])
if self.saveauxiliary:
self.auxiliary[v]['tukey53H_norm'] = s
threshold = cfg['tukey53H_norm']['k']
flag = np.zeros(s.shape, dtype='i1')
# Flag as 9 any masked input value
flag[ma.getmaskarray(self.input[v])] = 9
flag[np.nonzero(s > threshold)] = 4
flag[np.nonzero(s <= threshold)] = 1
self.flags[v]['tukey53H_norm'] = flag
#if 'spike_depthsmooth' in cfg:
# from maud.window_func import _weight_hann as wfunc
# cfg_tmp = cfg['spike_depthsmooth']
# cfg_tmp['dzwindow'] = 10
# smooth = ma.masked_all(self.input[v].shape)
# z = ped['pressure']
# for i in range(len(self.input[v])):
# ind = np.nonzero(ma.absolute(z-z[i]) < \
# cfg_tmp['dzwindow'])[0]
# ind = ind[ind != i]
# w = wfunc(z[ind]-z[i], cfg_tmp['dzwindow'])
# smooth[i] = (T[ind]*w).sum()/w.sum()
# ARGO, test #12. (10C, 5PSU)
if 'digit_roll_over' in cfg:
threshold = cfg['digit_roll_over']
s = step(self.input[v])
if self.saveauxiliary:
self.auxiliary[v]['step'] = s
flag = np.zeros(s.shape, dtype='i1')
# Flag as 9 any masked input value
flag[ma.getmaskarray(self.input[v])] = 9
flag[np.nonzero(ma.absolute(s) > threshold)] = 4
flag[np.nonzero(ma.absolute(s) <= threshold)] = 1
self.flags[v]['digit_roll_over'] = flag
if 'bin_spike' in cfg:
bin = bin_spike(self.input[v], cfg['bin_spike'])
if self.saveauxiliary:
self.auxiliary[v]['bin_spike'] = bin
if 'density_inversion' in cfg:
try:
if self.saveauxiliary:
self.flags[v]['density_inversion'], \
self.auxiliary[v]['density_step'] = \
density_inversion(
self.input,
cfg['density_inversion'],
saveaux=True)
else:
self.flags[v]['density_inversion'] = density_inversion(
self.input, cfg['density_inversion'])
except:
print("Fail on density_inversion")
if 'woa_normbias' in cfg:
if self.saveauxiliary:
self.flags[v]['woa_normbias'], \
self.auxiliary[v]['woa_relbias'] = \
woa_normbias(self.input, v, cfg['woa_normbias'])
#for k in woa.keys():
# self.auxiliary[v][k] = woa[k]
#self.auxiliary[v]['woa_bias'] = woa_bias
#self.auxiliary[v]['woa_relbias'] = woa_bias/woa['woa_sd']
else:
self.flags[v]['woa_normbias'], \
tmp = \
woa_normbias(self.input, v, cfg['woa_normbias'])
del(tmp)
#if 'pstep' in cfg:
# ind = np.isfinite(self.input[v])
# ind = ma.getmaskarray(self.input[v])
# if self.saveauxiliary:
# self.auxiliary[v]['pstep'] = ma.concatenate(
# [ma.masked_all(1),
# np.diff(self.input['PRES'][ind])])
if 'RoC' in cfg:
x = ma.concatenate([ma.masked_all(1), ma.diff(self.input[v])])
if self.saveauxiliary:
self.auxiliary[v]['RoC'] = x
self.flags[v]['RoC'] = np.zeros(x.shape, dtype='i1')
self.flags[v]['RoC'][np.nonzero(x <= cfg['RoC'])] = 1
self.flags[v]['RoC'][np.nonzero(x > cfg['RoC'])] = 4
self.flags[v]['RoC'][ma.getmaskarray(self.input[v])] = 9
if 'anomaly_detection' in cfg:
features = {}
for f in cfg['anomaly_detection']['features']:
if f == 'spike':
features['spike'] = spike(self.input[v])
elif f == 'gradient':
features['gradient'] = gradient(self.input[v])
elif f == 'tukey53H_norm':
features['tukey53H_norm'] = tukey53H_norm(self.input[v],
k=1.5)
else:
logging.error("Sorry, I can't evaluate anomaly_detection with: %s" % f)
self.flags[v]['anomaly_detection'] = \
anomaly_detection(features, cfg['anomaly_detection'])
if 'fuzzylogic' in cfg:
self.flags[v]['fuzzylogic'] = fuzzylogic(
self.auxiliary[v],
v,
cfg['fuzzylogic'])
def log_linear_vinterp(T, P, levs):
'''
# Author Charles Doutriaux
# Version 1.1
# Expect 2D field here so there''s no reorder which I suspect to do a memory leak
# email: [email protected]
# Converts a field from sigma levels to pressure levels
# Log linear interpolation
# Input
# T : temperature on sigma levels
# P : pressure field from TOP (level 0) to BOTTOM (last level)
# levs : pressure levels to interplate to (same units as P)
# Output
# t : temperature on pressure levels (levs)
# External: Numeric'''
import numpy.ma as MA
## from numpy.oldnumeric.ma import ones,Float,greater,less,logical_and,where,equal,log,asarray,Float16
sh = P.shape
nsigma = sh[0] # Number of sigma levels
try:
nlev = len(levs) # Number of pressure levels
except:
nlev = 1 # if only one level len(levs) would breaks
t = []
for ilv in range(nlev): # loop through pressure levels
try:
lev = levs[ilv] # get value for the level
except:
lev = levs # only 1 level passed
# print ' ......... level:',lev
Pabv = MA.ones(P[0].shape, Numeric.Float)
Tabv = -Pabv # Temperature on sigma level Above
Tbel = -Pabv # Temperature on sigma level Below
Pbel = -Pabv # Pressure on sigma level Below
Pabv = -Pabv # Pressure on sigma level Above
for isg in range(1,
nsigma): # loop from second sigma level to last one
## print 'Sigma level #',isg
a = MA.greater(P[isg],
lev) # Where is the pressure greater than lev
b = MA.less(P[isg - 1], lev) # Where is the pressure less than lev
# Now looks if the pressure level is in between the 2 sigma levels
# If yes, sets Pabv, Pbel and Tabv, Tbel
Pabv = MA.where(MA.logical_and(a, b), P[isg],
Pabv) # Pressure on sigma level Above
Tabv = MA.where(MA.logical_and(a, b), T[isg],
Tabv) # Temperature on sigma level Above
Pbel = MA.where(MA.logical_and(a, b), P[isg - 1],
Pbel) # Pressure on sigma level Below
Tbel = MA.where(MA.logical_and(a, b), T[isg - 1],
Tbel) # Temperature on sigma level Below
# end of for isg in range(1,nsigma)
# val=where(equal(Pbel,-1.),Pbel.missing_value,lev) # set to missing value if no data below lev if there is
tl = MA.masked_where(
MA.equal(Pbel, -1.),
MA.log(lev / MA.absolute(Pbel)) / MA.log(Pabv / Pbel) *
(Tabv - Tbel) + Tbel) # Interpolation
t.append(tl) # add a level to the output
# end of for ilv in range(nlev)
return asMA(t).astype(Numeric.Float32) # convert t to an array
def evaluate(self, v, cfg):
self.flags[v] = {}
# Apply common flag for all points.
if 'common' in self.flags:
N = self.input[v].shape
for f in self.flags['common']:
self.flags[v][f] = self.flags['common'][f] * \
np.ones(N, dtype='i1')
if self.saveauxiliary:
if v not in self.auxiliary.keys():
self.auxiliary[v] = {}
if 'platform_identification' in cfg:
logging.warn("Sorry I'm not ready to evaluate platform_identification()")
if 'valid_geolocation' in cfg:
logging.warn("Sorry I'm not ready to evaluate valid_geolocation()")
if 'valid_speed' in cfg:
# Think about. ARGO also has a test valid_speed, but that is
# in respect to sucessive profiles. How is the best way to
# distinguish them here?
try:
if self.saveauxiliary:
self.flags[v]['valid_speed'], \
self.auxiliary[v]['valid_speed'] = \
possible_speed(self.input, cfg['valid_speed'])
except:
print("Fail on valid_speed")
if 'global_range' in cfg:
self.flags[v]['global_range'] = global_range(
self.input, v, cfg['global_range'])
if 'regional_range' in cfg:
logging.warn("Sorry, I'm no ready to evaluate regional_range()")
if 'pressure_increasing' in cfg:
logging.warn("Sorry, I'm no ready to evaluate pressure_increasing()")
if 'profile_envelop' in cfg:
self.flags[v]['profile_envelop'] = profile_envelop(
self.input, cfg['profile_envelop'], v)
if 'gradient' in cfg:
y = Gradient(self.input, v, cfg['gradient'])
y.test()
if self.saveauxiliary:
self.auxiliary[v]['gradient'] = y.features['gradient']
self.flags[v]['gradient'] = y.flags['gradient']
if 'gradient_depthconditional' in cfg:
cfg_tmp = cfg['gradient_depthconditional']
g = gradient(self.input[v])
flag = np.zeros(g.shape, dtype='i1')
# Flag as 9 any masked input value
flag[ma.getmaskarray(self.input[v])] = 9
# ---- Shallow zone -----------------
threshold = cfg_tmp['shallow_max']
flag[np.nonzero( \
(self['PRES'] <= cfg_tmp['pressure_threshold']) & \
(g > threshold))] \
= 4
flag[np.nonzero( \
(self['PRES'] <= cfg_tmp['pressure_threshold']) & \
(g <= threshold))] \
= 1
# ---- Deep zone --------------------
threshold = cfg_tmp['deep_max']
flag[np.nonzero( \
(self['PRES'] > cfg_tmp['pressure_threshold']) & \
(g > threshold))] \
= 4
flag[np.nonzero( \
(self['PRES'] > cfg_tmp['pressure_threshold']) & \
(g <= threshold))] \
= 1
self.flags[v]['gradient_depthconditional'] = flag
if 'spike' in cfg:
y = Spike(self.input, v, cfg['spike'])
y.test()
if self.saveauxiliary:
self.auxiliary[v]['spike'] = y.features['spike']
self.flags[v]['spike'] = y.flags['spike']
if 'spike_depthconditional' in cfg:
cfg_tmp = cfg['spike_depthconditional']
s = spike(self.input[v])
flag = np.zeros(s.shape, dtype='i1')
# Flag as 9 any masked input value
flag[ma.getmaskarray(self.input[v])] = 9
# ---- Shallow zone -----------------
threshold = cfg_tmp['shallow_max']
flag[np.nonzero( \
(self['PRES'] <= cfg_tmp['pressure_threshold']) & \
(s > threshold))] \
= 4
flag[np.nonzero( \
(self['PRES'] <= cfg_tmp['pressure_threshold']) & \
(s <= threshold))] \
= 1
# ---- Deep zone --------------------
threshold = cfg_tmp['deep_max']
flag[np.nonzero( \
(self['PRES'] > cfg_tmp['pressure_threshold']) & \
(s > threshold))] \
= 4
flag[np.nonzero( \
(self['PRES'] > cfg_tmp['pressure_threshold']) & \
(s <= threshold))] \
= 1
self.flags[v]['spike_depthconditional'] = flag
if 'stuck_value' in cfg:
logging.warn("Sorry I'm not ready to evaluate stuck_value()")
if 'grey_list' in cfg:
logging.warn("Sorry I'm not ready to evaluate grey_list()")
if 'gross_sensor_drift' in cfg:
logging.warn("Sorry I'm not ready to evaluate gross_sensor_drift()")
if 'frozen_profile' in cfg:
logging.warn("Sorry I'm not ready to evaluate frozen_profile()")
if 'deepest_pressure' in cfg:
logging.warn("Sorry I'm not ready to evaluate deepest_pressure()")
if 'tukey53H_norm' in cfg:
y = Tukey53H(self.input, v, cfg['tukey53H_norm'])
y.test()
if self.saveauxiliary:
self.auxiliary[v]['tukey53H_norm'] = \
y.features['tukey53H_norm']
self.flags[v]['tukey53H_norm'] = y.flags['tukey53H_norm']
#if 'spike_depthsmooth' in cfg:
# from maud.window_func import _weight_hann as wfunc
# cfg_tmp = cfg['spike_depthsmooth']
# cfg_tmp['dzwindow'] = 10
# smooth = ma.masked_all(self.input[v].shape)
# z = ped['pressure']
# for i in range(len(self.input[v])):
# ind = np.nonzero(ma.absolute(z-z[i]) < \
# cfg_tmp['dzwindow'])[0]
# ind = ind[ind != i]
# w = wfunc(z[ind]-z[i], cfg_tmp['dzwindow'])
# smooth[i] = (T[ind]*w).sum()/w.sum()
# ARGO, test #12. (10C, 5PSU)
if 'digit_roll_over' in cfg:
threshold = cfg['digit_roll_over']
s = step(self.input[v])
if self.saveauxiliary:
self.auxiliary[v]['step'] = s
flag = np.zeros(s.shape, dtype='i1')
# Flag as 9 any masked input value
flag[ma.getmaskarray(self.input[v])] = 9
flag[np.nonzero(ma.absolute(s) > threshold)] = 4
flag[np.nonzero(ma.absolute(s) <= threshold)] = 1
self.flags[v]['digit_roll_over'] = flag
if 'bin_spike' in cfg:
y = Bin_Spike(self.input, v, cfg['bin_spike'])
# y.test()
if self.saveauxiliary:
self.auxiliary[v]['bin_spike'] = y.features['bin_spike']
# self.flags[v]['bin_spike'] = y.flags['bin_spike']
if 'density_inversion' in cfg:
try:
if self.saveauxiliary:
self.flags[v]['density_inversion'], \
self.auxiliary[v]['density_step'] = \
density_inversion(
self.input,
cfg['density_inversion'],
saveaux=True)
else:
self.flags[v]['density_inversion'] = density_inversion(
self.input, cfg['density_inversion'])
except:
print("Fail on density_inversion")
if 'woa_normbias' in cfg:
y = WOA_NormBias(self.input, v, cfg['woa_normbias'])
# self.attributes)
y.test()
if self.saveauxiliary:
for f in y.features:
self.auxiliary[v][f] = y.features[f]
self.flags[v]['woa_normbias'] = y.flags['woa_normbias']
#if 'pstep' in cfg:
# ind = np.isfinite(self.input[v])
# ind = ma.getmaskarray(self.input[v])
# if self.saveauxiliary:
# self.auxiliary[v]['pstep'] = ma.concatenate(
# [ma.masked_all(1),
# np.diff(self.input['PRES'][ind])])
if 'rate_of_change' in cfg:
self.flags[v]['rate_of_change'], RoC = \
rate_of_change(self.input, v, cfg['rate_of_change'])
if self.saveauxiliary:
self.auxiliary[v]['rate_of_change'] = RoC
if 'cum_rate_of_change' in cfg:
x = cum_rate_of_change(self.input, v,
cfg['cum_rate_of_change']['memory'])
self.flags[v]['cum_rate_of_change'] = np.zeros(x.shape, dtype='i1')
self.flags[v]['cum_rate_of_change'][
np.nonzero(x <= cfg['cum_rate_of_change']['threshold'])
] = 1
self.flags[v]['cum_rate_of_change'][
np.nonzero(x > cfg['cum_rate_of_change']['threshold'])
] = 4
self.flags[v]['cum_rate_of_change'][
ma.getmaskarray(self.input[v])] = 9
# FIXME: the Anomaly Detection and Fuzzy require some features
# to be estimated previously. Generalize this.
if 'anomaly_detection' in cfg:
features = {}
for f in cfg['anomaly_detection']['features']:
if f == 'spike':
features['spike'] = spike(self.input[v])
elif f == 'gradient':
features['gradient'] = gradient(self.input[v])
elif f == 'tukey53H_norm':
features['tukey53H_norm'] = tukey53H_norm(self.input[v])
elif f == 'rate_of_change':
RoC = ma.masked_all_like(self.input[v])
RoC[1:] = ma.absolute(ma.diff(self.input[v]))
features['rate_of_change'] = RoC
elif (f == 'woa_normbias'):
y = WOA_NormBias(self.input, v, {})
features['woa_normbias'] = \
np.abs(y.features['woa_normbias'])
else:
logging.error("Sorry, I can't evaluate anomaly_detection with: %s" % f)
self.flags[v]['anomaly_detection'] = \
anomaly_detection(features, cfg['anomaly_detection'])
if 'morello2014' in cfg:
self.flags[v]['morello2014'] = morello2014(
features=self.auxiliary[v],
cfg=cfg['morello2014'])
if 'fuzzylogic' in cfg:
features = {}
for f in cfg['fuzzylogic']['features']:
if f == 'spike':
features['spike'] = spike(self.input[v])
elif f == 'gradient':
features['gradient'] = gradient(self.input[v])
elif f == 'tukey53H_norm':
features['tukey53H_norm'] = tukey53H_norm(self.input[v],
k=1.5)
elif f == 'rate_of_change':
RoC = ma.masked_all_like(data[v])
RoC[1:] = ma.absolute(ma.diff(data[v]))
features['rate_of_change'] = RoC
elif (f == 'woa_normbias'):
y = WOA_NormBias(self.input, v, {})
features['woa_normbias'] = \
np.abs(y.features['woa_normbias'])
else:
logging.error("Sorry, I can't evaluate fuzzylogic with: %s" % f)
self.flags[v]['fuzzylogic'] = fuzzylogic(
features=features,
cfg=cfg['fuzzylogic'])
self.flags[v]['overall'] = combined_flag(self.flags[v])
