class TestIndTop:
x = np.array([1, 3, 8, 4, -99, 1, 0.5, 0])
mx = ma.masked_array(x, mask=[0, 0, 0, 0, 1, 0, 0, 0])
dx = ma.diff(mx).filled(0)
peak_position = 2
n_prof = x.shape[0]
search_range = 5
threshold = 2
def test_ind_top_0(self):
assert droplet.ind_top(self.dx, self.peak_position, self.n_prof,
self.search_range, self.threshold) == 3
def test_ind_top_1(self):
search_range = 100
assert droplet.ind_top(self.dx, self.peak_position, self.n_prof,
search_range, self.threshold) == 3
def test_ind_top_2(self):
huge_threshold = 5000
assert droplet.ind_top(self.dx, self.peak_position, self.n_prof,
self.search_range, huge_threshold) == 7
def test_ind_top_3(self):
small_threshold = 1.01
assert droplet.ind_top(self.dx, self.peak_position, self.n_prof,
self.search_range, small_threshold) == 3
def test_ind_top_4(self):
mx = ma.masked_array(self.x, mask=[1, 0, 1, 0, 1, 0, 1, 0])
dx = ma.diff(mx).filled(0)
with pytest.raises(IndexError):
droplet.ind_top(dx, self.peak_position, self.n_prof,
self.search_range, self.threshold)
def ifreq(a_single_subRegion, special_parameters):
# TODO: this 'special_parameters' maybe be useful
phase = array(angle(a_single_subRegion))
ifreq = array(diff(phase))
return ifreq
class TestIndBase:
x = np.array([0, 0.5, 1, -99, 4, 8, 5])
mx = ma.masked_array(x, mask=[0, 0, 0, 1, 0, 0, 0])
dx = ma.diff(mx).filled(0)
peak_position = 5
search_range = 4
threshold = 2
def test_ind_base_0(self):
assert droplet.ind_base(self.dx, self.peak_position, self.search_range,
self.threshold) == 4
def test_ind_base_1(self):
search_range = 100
assert droplet.ind_base(self.dx, self.peak_position, search_range,
self.threshold) == 4
def test_ind_base_2(self):
huge_threshold = 5000
assert droplet.ind_base(self.dx, self.peak_position, self.search_range,
huge_threshold) == 1
def test_ind_base_3(self):
small_threshold = 1.01
assert droplet.ind_base(self.dx, self.peak_position, self.search_range,
small_threshold) == 4
def test_ind_base_4(self):
mx = ma.masked_array(self.x, mask=[1, 0, 1, 1, 1, 0, 0])
dx = ma.diff(mx).filled(0)
with pytest.raises(IndexError):
droplet.ind_base(dx, self.peak_position, self.search_range,
self.threshold)
def _find_profiles_with_undetected_melting(bits: list) -> np.ndarray:
drizzle_and_falling = _find_drizzle_and_falling(*bits[:3])
transition = ma.diff(drizzle_and_falling, axis=1)
is_transition = ma.any(transition, axis=1)
is_melting_layer = ma.any(bits[3], axis=1)
is_undetected_melting = is_transition & ~is_melting_layer
is_undetected_melting[is_undetected_melting == 0] = ma.masked
return is_undetected_melting.astype(int)
def y_grad_GridCor_Rho(RomsFile, RomsGrd, varname):
"""
Compute Grid Correction for gradients in y direction
"""
#load roms file
RomsNC = nc4(RomsFile, 'r')
#check if variable supplied or needs to load
if type(varname) == str:
_var = RomsNC.variables[varname][:]
else:
_var = varname
#get land mask
Mask_rho = ma.getmask(_var)
Mask_w = ma.concatenate((Mask_rho, Mask_rho[:, -2:-1, :, :]), axis=1)
#compute depth at rho points
_rhodepth = ma.array(dep._set_depth_T(RomsFile, None, 'rho', \
RomsNC.variables['h'][:], \
RomsNC.variables['zeta'][:]), mask = Mask_rho)
#compute depth at w points
_wdepth = ma.array(dep._set_depth_T(RomsFile, None, 'w', \
RomsNC.variables['h'][:], \
RomsNC.variables['zeta'][:]), mask = Mask_w)
#depth difference in vertical [rho points]
dz_z = np.diff(_wdepth, n=1, axis=1)
#depth difference between adjacent rho points [v points]
_dz_y = np.diff(_rhodepth, n=1, axis=2)
#shift to rho points
dz_y = GridShift.Vpt_to_Rho(_dz_y)
#compute difference [w points]
_dvar_z = ma.diff(_var, n=1, axis=1)
#shift to rho points
dvar_z = GridShift.Wpt_to_Rho(_dvar_z)
#distance between rho points in x and y directions
_y_dist = rt.rho_dist_grd(RomsGrd)[1]
#repeat over depth and time space and add mask
dy = ma.array(rt.AddDepthTime(RomsFile, _y_dist), mask=Mask_rho)
#vertical gradient [rho points]
dvar_dz = dvar_z / dz_z
#correction for roms grid
dv_dyCor = dvar_dz * (dz_y / dy)
return dv_dyCor
def cum_rate_of_change(data, v, memory):
output = ma.masked_all_like(data[v])
output[1:] = ma.absolute(ma.diff(data[v]))
for i in range(2, output.size):
if output[i] < output[i-1]:
output[i] = (1 - memory) * output[i] + memory * output[i-1]
return output
def descentPrate(data):
"""
It's probably a good idea to smooth it with a window of 2-5 seconds.
After binned, the data will be probably groupped in bins of 1dbar,
but the raw data might have more than one records per second, which
might have plenty spikes. I'm looking here for inadequate casts
lowered too fast, or maybe bad weather and a rolling vessel.
Consider to create another test looking for excessive ups and downs.
"""
assert ('timeS' in data.keys()), "timeS is not available"
assert ('PRES' in data.keys()), "pressure is not available"
assert data['timeS'].shape == data['PRES'].shape, \
"t and p have different sizes"
dt = ma.diff(data['timeS'])
dp = ma.diff(data['PRES'])
y = ma.append(ma.masked_all(1, dtype='i'), dp/dt)
return y
def descentPrate(data):
"""
It's probably a good idea to smooth it with a window of 2-5 seconds.
After binned, the data will be probably groupped in bins of 1dbar,
but the raw data might have more than one records per second, which
might have plenty spikes. I'm looking here for inadequate casts
lowered too fast, or maybe bad weather and a rolling vessel.
Consider to create another test looking for excessive ups and downs.
"""
assert ('timeS' in data.keys()), "timeS is not available"
assert ('PRES' in data.keys()), "pressure is not available"
assert data['timeS'].shape == data['PRES'].shape, \
"t and p have different sizes"
dt = ma.diff(data['timeS'])
dp = ma.diff(data['PRES'])
y = ma.append(ma.masked_all(1, dtype='i'), dp / dt)
return y
def cum_rate_of_change(x, memory):
y = ma.fix_invalid(np.ones_like(x) * np.nan)
y[1:] = ma.absolute(ma.diff(x))
for i in range(2, y.size):
if y[i] < y[i - 1]:
y[i] = (1 - memory) * y[i] + memory * y[i - 1]
return y
def cum_rate_of_change(data, v, memory):
output = ma.masked_all_like(data[v])
output[1:] = ma.absolute(ma.diff(data[v]))
for i in range(2, output.size):
if output[i] < output[i - 1]:
output[i] = (1 - memory) * output[i] + memory * output[i - 1]
return output
def rate_of_change(data, v, cfg):
RoC = ma.masked_all_like(data[v])
RoC[1:] = ma.absolute(ma.diff(data[v]))
flag = np.zeros(data[v].shape, dtype='i1')
flag[np.nonzero(RoC <= cfg)] = 1
flag[np.nonzero(RoC > cfg)] = 4
flag[ma.getmaskarray(data[v])] = 9
return flag, RoC
def speed(data):
"""
"""
assert ('timeS' in data.keys()), \
"Missing timeS in input data"
assert ('LATITUDE' in data.keys()), \
"Missing LATITUDE in input data"
assert ('LONGITUDE' in data.keys()), \
"Missing LONGITUDE in input data"
dL = haversine(data['LATITUDE'][:-1], data['LONGITUDE'][:-1],
data['LATITUDE'][1:], data['LONGITUDE'][1:])
dt = ma.diff(data['timeS'])
speed = ma.append(ma.masked_array([0], [True]), dL / dt)
return speed
def x_grad_u(RomsFile, RomsGrd, varname):
"""
compute x-gradient on u points
"""
if type(varname) == str:
#load roms file
RomsNC = nc4(RomsFile, 'r')
#load variable
_var = RomsNC.variables[varname][:]
else:
_var = varname #[u points]
#get mask
Mask = ma.getmask(_var)
#gradient in x direction [rho points]
dvar_rho = ma.diff(_var, n=1, axis=3)
#pad
mask_pad = ma.notmasked_edges(dvar_rho, axis=3)
dvar_ = dvar_rho
dvar_[:, :, :, mask_pad[0][3][0] - 1] = dvar_rho[:, :, :,
mask_pad[0][3][0]]
dvar_[:, :, :, mask_pad[1][3][0] + 1] = dvar_rho[:, :, :,
mask_pad[1][3][0]]
dvar_pad = ma.concatenate((dvar_rho[:,:,:,0:1], dvar_rho, \
dvar_rho[:,:,:,-2:-1]), axis = 3)
#shift to u points
dvar_u = GridShift.Rho_to_Upt(dvar_pad)
#dx [rho points]
x_dist = rt.rho_dist_grd(RomsGrd)[0]
#repeat over depth and time and apply mask
_dx = rt.AddDepthTime(RomsFile, x_dist)
dx = ma.array(GridShift.Rho_to_Upt(_dx), mask=Mask)
#gradient
dvar_dx = dvar_u / dx
return dvar_dx
def speed(data):
"""
"""
assert ('timeS' in data.keys()), \
"Missing timeS in input data"
assert ('LATITUDE' in data.keys()), \
"Missing LATITUDE in input data"
assert ('LONGITUDE' in data.keys()), \
"Missing LONGITUDE in input data"
dL = haversine(data['LATITUDE'][:-1], data['LONGITUDE'][:-1],
data['LATITUDE'][1:], data['LONGITUDE'][1:])
dt = ma.diff(data['timeS'])
speed = ma.append(ma.masked_array([0], [True]),
dL/dt)
return speed
def speed(data):
"""
"""
assert "timeS" in data.keys(), "Missing timeS in input data"
assert "LATITUDE" in data.keys(), "Missing LATITUDE in input data"
assert "LONGITUDE" in data.keys(), "Missing LONGITUDE in input data"
dL = haversine(
data["LATITUDE"][:-1],
data["LONGITUDE"][:-1],
data["LATITUDE"][1:],
data["LONGITUDE"][1:],
)
dt = ma.diff(data["timeS"])
speed = ma.append(ma.masked_array([0], [True]), dL / dt)
return speed
def y_grad_v(RomsFile, RomsGrd, varname):
"""
Compute y-gradient on v points
"""
if type(varname) == str:
#load roms file
RomsNC = nc4(RomsFile, 'r')
#load variable
_var = RomsNC.variables[varname][:]
else:
_var = varname #[v points]
#get mask
Mask = ma.getmask(_var)
#compute difference [rho points]
dvar_rho = ma.diff(_var, n=1, axis=2)
#pad
mask_pad = ma.notmasked_edges(dvar_rho, axis=2)
dvar_ = dvar_rho
dvar_[:, :, mask_pad[0][2][0] - 1, :] = dvar_rho[:, :,
mask_pad[0][2][0], :]
dvar_[:, :, mask_pad[1][2][0] + 1, :] = dvar_rho[:, :,
mask_pad[1][2][0], :]
dvar_pad = ma.concatenate((dvar_[:,:,0:1, :],\
dvar_, \
dvar_[:,:,-2:-1, :]), axis = 2)
#shift to V points
dvar_v = GridShift.Rho_to_Vpt(dvar_pad)
#dy
y_dist = rt.rho_dist_grd(RomsGrd)[1]
dy = rt.AddDepthTime(RomsFile, y_dist)
dy_v = ma.array(GridShift.Rho_to_Vpt(dy), mask=Mask)
#compute gradient
dvar_dy = dvar_v / dy_v
return dvar_dy
def estimate_cell_edges(x):
"""Convert one-dimensional vector x of size n into n + 1, where the input
describes the centres of the cells, and the output is an estimate of the
edges of the cell"""
# centres (with extra centres padded at the ends by linear interpolation)
dx = ma.diff(x)
x_c = ma.hstack((x[0] - atleast_1d(dx[0]), x,
x[-1] + atleast_1d(dx[-1])))
# _f is notation from MITgcm (implies faces)
x_f = (x_c[1:] + x_c[:-1])/2
dx_c = np.diff(x_c)
# Catch nan or masked values and estimate edge using dx from previous or
# next cell
nan_before = ma.where(
ma.logical_and(nan_or_masked(x_f[:-1]), ~nan_or_masked(x_f[1:])))[0]
nan_after = ma.where(
ma.logical_and(~nan_or_masked(x_f[:-1]), nan_or_masked(x_f[1:])))[0]
x_f[nan_before] = x_f[nan_before + 1] - dx_c[nan_before + 1]
x_f[nan_after + 1] = x_f[nan_after] + dx_c[nan_after]
return x_f
@author: Jasen
"""
import os
os.chdir('/Users/Jasen/Documents/GitHub/ROMS_Budget/')
from netCDF4 import Dataset as nc4
import numpy.ma as ma
import obs_depth_JJ as dep
DiaFile = '/Users/Jasen/Documents/Data/ROMS_WCOF/DiagnosticFiles/ocean_dia_jasen.nc'
AvgFile = '/Users/Jasen/Documents/Data/ROMS_WCOF/DiagnosticFiles/ocean_avg_jasen.nc'
GrdFile = '/Users/Jasen/Documents/Data/ROMS_WCOF/DiagnosticFiles/grd_wcofs4_visc200_50-50-50_diff200_spherical.nc'
DiaNC = nc4(DiaFile, 'r')
AvgNC = nc4(AvgFile, 'r')
GrdNC = nc4(GrdFile, 'r')
#variable from average file
salt = AvgNC.variables['salt'][:]
#depth at w points
romsvars = {'h' : AvgNC.variables['h'][:], \
'zeta' : AvgNC.variables['zeta'][:]}
# update to include Vstretching = 4 (spherical?)
depth_w = dep._set_depth_T(AvgFile, None, 'w', romsvars['h'], romsvars['zeta'])
delta = ma.diff(depth_w, n=1, axis=1)
#variables from
return var_prime
#load salt_rate from diagnostic file
Diag = nc4(DiagFile, 'r')
salt_rate = Diag.variables['salt_rate'][:]
#from average file, load salt and vertical thickness
Avg = nc4(AvgFile, 'r')
salt = Avg.variables['salt'][:]
#Compute vertical thickness at w points
depth_domain = dep._set_depth_T(AvgFile, None, 'w', Avg.variables['h'][:],\
Avg.variables['zeta'][:])
delta = ma.diff(depth_domain, n=1, axis=1)
#from history files compute time derivative of thickness of cell
Hist = nc4(HistFile, 'r')
depth_hist = dep._set_depth_T(HistFile, None, 'w', Hist.variables['h'][:], \
Hist.variables['zeta'][:])
delta_hist = ma.diff(depth_hist, n=1, axis=1)
ddelta_dt = ma.diff(delta_hist, \
n = 1, axis = 0)/ma.diff(Hist.variables['ocean_time'][:],\
n = 1, axis = 0)
#time deriv of salt, less change in cell volume
ds_dt = salt_rate - salt / delta * ddelta_dt
def test_ind_top_4(self):
mx = ma.masked_array(self.x, mask=[1, 0, 1, 0, 1, 0, 1, 0])
dx = ma.diff(mx).filled(0)
with pytest.raises(IndexError):
droplet.ind_top(dx, self.peak_position, self.n_prof,
self.search_range, self.threshold)
time = Avg.variables['ocean_time'][:]
#load dA
dA_xy = ma.array(dff.dA_top(Avg), mask=RhoMask)
#integrate time derivative of s_prime^2
SprInt = np.empty(time.shape[0])
SprInt.fill(np.nan)
for t in range(time.shape[0]):
#avg file variables at t
salt = ma.array(Avg.variables['salt'][t, :, :, :], mask=RhoMask)
s_prime = salt - salt.mean()
#diff and mask are in opposite order
deltaA = ma.array(ma.diff(dep._set_depth(AvgFile, None, 'w', \
Avg.variables['h'][:], \
Avg.variables['zeta'][t,:,:]),
n = 1, axis = 0), \
mask = RhoMask)
#diag file variables at t
salt_rate = ma.array(Diag.variables['salt_rate'][t, :, :, :], \
mask = RhoMask)
#hist variables at t and t+1
deltaH0 = ma.array(ma.diff(dep._set_depth(HistFile, None, 'w', \
Hist.variables['h'][:], \
Hist.variables['zeta'][t,:,:]),
n = 1, axis = 0), \
mask = RhoMask)
deltaH1 = ma.array(ma.diff(dep._set_depth(HistFile, None, 'w', \
def _fix_undetected_melting_layer(bits: list) -> np.ndarray:
melting_layer = bits[3]
drizzle_and_falling = _find_drizzle_and_falling(*bits[:3])
transition = ma.diff(drizzle_and_falling, axis=1) == -1
melting_layer[:, 1:][transition] = True
return melting_layer
def step(x):
y = ma.masked_all(x.shape, dtype=x.dtype)
y[1:] = ma.diff(x)
return y
def evaluate(self, v, cfg):
self.flags[v] = {}
if self.saveauxiliary:
if v not in self.auxiliary.keys():
self.auxiliary[v] = {}
if 'platform_identification' in cfg:
logging.warn("Sorry I'm not ready to evaluate platform_identification()")
if 'valid_geolocation' in cfg:
logging.warn("Sorry I'm not ready to evaluate valid_geolocation()")
if 'valid_speed' in cfg:
# Think about. ARGO also has a test valid_speed, but that is
# in respect to sucessive profiles. How is the best way to
# distinguish them here?
try:
if self.saveauxiliary:
self.flags[v]['valid_speed'], \
self.auxiliary[v]['valid_speed'] = \
possible_speed(self.input, cfg['valid_speed'])
except:
print("Fail on valid_speed")
if 'global_range' in cfg:
self.flags[v]['global_range'] = np.zeros(self.input[v].shape,
dtype='i1')
# Flag as 9 any masked input value
self.flags[v]['global_range'][ma.getmaskarray(self.input[v])] = 9
ind = (self.input[v] >= cfg['global_range']['minval']) & \
(self.input[v] <= cfg['global_range']['maxval'])
self.flags[v]['global_range'][np.nonzero(ind)] = 1
ind = (self.input[v] < cfg['global_range']['minval']) | \
(self.input[v] > cfg['global_range']['maxval'])
self.flags[v]['global_range'][np.nonzero(ind)] = 4
if 'regional_range' in cfg:
logging.warn("Sorry, I'm no ready to evaluate regional_range()")
if 'pressure_increasing' in cfg:
logging.warn("Sorry, I'm no ready to evaluate pressure_increasing()")
if 'profile_envelop' in cfg:
self.flags[v]['profile_envelop'] = profile_envelop(
self.input, cfg['profile_envelop'], v)
if 'gradient' in cfg:
threshold = cfg['gradient']
g = gradient(self.input[v])
if self.saveauxiliary:
self.auxiliary[v]['gradient'] = g
flag = np.zeros(g.shape, dtype='i1')
flag[ma.getmaskarray(self.input[v])] = 9
flag[np.nonzero(g > threshold)] = 4
flag[np.nonzero(g <= threshold)] = 1
self.flags[v]['gradient'] = flag
if 'gradient_depthconditional' in cfg:
cfg_tmp = cfg['gradient_depthconditional']
g = gradient(self.input[v])
flag = np.zeros(g.shape, dtype='i1')
# Flag as 9 any masked input value
flag[ma.getmaskarray(self.input[v])] = 9
# ---- Shallow zone -----------------
threshold = cfg_tmp['shallow_max']
flag[np.nonzero( \
(self['PRES'] <= cfg_tmp['pressure_threshold']) & \
(g > threshold))] \
= 4
flag[np.nonzero( \
(self['PRES'] <= cfg_tmp['pressure_threshold']) & \
(g <= threshold))] \
= 1
# ---- Deep zone --------------------
threshold = cfg_tmp['deep_max']
flag[np.nonzero( \
(self['PRES'] > cfg_tmp['pressure_threshold']) & \
(g > threshold))] \
= 4
flag[np.nonzero( \
(self['PRES'] > cfg_tmp['pressure_threshold']) & \
(g <= threshold))] \
= 1
self.flags[v]['gradient_depthconditional'] = flag
if 'spike' in cfg:
threshold = cfg['spike']
s = spike(self.input[v])
if self.saveauxiliary:
self.auxiliary[v]['spike'] = s
flag = np.zeros(s.shape, dtype='i1')
# Flag as 9 any masked input value
flag[ma.getmaskarray(self.input[v])] = 9
flag[np.nonzero(s > threshold)] = 4
flag[np.nonzero(s <= threshold)] = 1
self.flags[v]['spike'] = flag
if 'spike_depthconditional' in cfg:
cfg_tmp = cfg['spike_depthconditional']
s = spike(self.input[v])
flag = np.zeros(s.shape, dtype='i1')
# Flag as 9 any masked input value
flag[ma.getmaskarray(self.input[v])] = 9
# ---- Shallow zone -----------------
threshold = cfg_tmp['shallow_max']
flag[np.nonzero( \
(self['PRES'] <= cfg_tmp['pressure_threshold']) & \
(g > threshold))] \
= 4
flag[np.nonzero( \
(self['PRES'] <= cfg_tmp['pressure_threshold']) & \
(g <= threshold))] \
= 1
# ---- Deep zone --------------------
threshold = cfg_tmp['deep_max']
flag[np.nonzero( \
(self['PRES'] > cfg_tmp['pressure_threshold']) & \
(g > threshold))] \
= 4
flag[np.nonzero( \
(self['PRES'] > cfg_tmp['pressure_threshold']) & \
(g <= threshold))] \
= 1
self.flags[v]['spike_depthconditional'] = flag
if 'stuck_value' in cfg:
logging.warn("Sorry I'm not ready to evaluate stuck_value()")
if 'grey_list' in cfg:
logging.warn("Sorry I'm not ready to evaluate grey_list()")
if 'gross_sensor_drift' in cfg:
logging.warn("Sorry I'm not ready to evaluate gross_sensor_drift()")
if 'frozen_profile' in cfg:
logging.warn("Sorry I'm not ready to evaluate frozen_profile()")
if 'deepest_pressure' in cfg:
logging.warn("Sorry I'm not ready to evaluate deepest_pressure()")
if 'tukey53H_norm' in cfg:
"""
I slightly modified the Goring & Nikora 2002. It is
expected that CTD profiles has a typical depth
structure, with a range between surface and bottom.
"""
s = tukey53H_norm(self.input[v],
k = cfg['tukey53H_norm']['k'],
l = cfg['tukey53H_norm']['l'])
if self.saveauxiliary:
self.auxiliary[v]['tukey53H_norm'] = s
threshold = cfg['tukey53H_norm']['k']
flag = np.zeros(s.shape, dtype='i1')
# Flag as 9 any masked input value
flag[ma.getmaskarray(self.input[v])] = 9
flag[np.nonzero(s > threshold)] = 4
flag[np.nonzero(s <= threshold)] = 1
self.flags[v]['tukey53H_norm'] = flag
#if 'spike_depthsmooth' in cfg:
# from maud.window_func import _weight_hann as wfunc
# cfg_tmp = cfg['spike_depthsmooth']
# cfg_tmp['dzwindow'] = 10
# smooth = ma.masked_all(self.input[v].shape)
# z = ped['pressure']
# for i in range(len(self.input[v])):
# ind = np.nonzero(ma.absolute(z-z[i]) < \
# cfg_tmp['dzwindow'])[0]
# ind = ind[ind != i]
# w = wfunc(z[ind]-z[i], cfg_tmp['dzwindow'])
# smooth[i] = (T[ind]*w).sum()/w.sum()
# ARGO, test #12. (10C, 5PSU)
if 'digit_roll_over' in cfg:
threshold = cfg['digit_roll_over']
s = step(self.input[v])
if self.saveauxiliary:
self.auxiliary[v]['step'] = s
flag = np.zeros(s.shape, dtype='i1')
# Flag as 9 any masked input value
flag[ma.getmaskarray(self.input[v])] = 9
flag[np.nonzero(ma.absolute(s) > threshold)] = 4
flag[np.nonzero(ma.absolute(s) <= threshold)] = 1
self.flags[v]['digit_roll_over'] = flag
if 'bin_spike' in cfg:
bin = bin_spike(self.input[v], cfg['bin_spike'])
if self.saveauxiliary:
self.auxiliary[v]['bin_spike'] = bin
if 'density_inversion' in cfg:
try:
if self.saveauxiliary:
self.flags[v]['density_inversion'], \
self.auxiliary[v]['density_step'] = \
density_inversion(
self.input,
cfg['density_inversion'],
saveaux=True)
else:
self.flags[v]['density_inversion'] = density_inversion(
self.input, cfg['density_inversion'])
except:
print("Fail on density_inversion")
if 'woa_normbias' in cfg:
if self.saveauxiliary:
self.flags[v]['woa_normbias'], \
self.auxiliary[v]['woa_relbias'] = \
woa_normbias(self.input, v, cfg['woa_normbias'])
#for k in woa.keys():
# self.auxiliary[v][k] = woa[k]
#self.auxiliary[v]['woa_bias'] = woa_bias
#self.auxiliary[v]['woa_relbias'] = woa_bias/woa['woa_sd']
else:
self.flags[v]['woa_normbias'], \
tmp = \
woa_normbias(self.input, v, cfg['woa_normbias'])
del(tmp)
#if 'pstep' in cfg:
# ind = np.isfinite(self.input[v])
# ind = ma.getmaskarray(self.input[v])
# if self.saveauxiliary:
# self.auxiliary[v]['pstep'] = ma.concatenate(
# [ma.masked_all(1),
# np.diff(self.input['PRES'][ind])])
if 'RoC' in cfg:
x = ma.concatenate([ma.masked_all(1), ma.diff(self.input[v])])
if self.saveauxiliary:
self.auxiliary[v]['RoC'] = x
self.flags[v]['RoC'] = np.zeros(x.shape, dtype='i1')
self.flags[v]['RoC'][np.nonzero(x <= cfg['RoC'])] = 1
self.flags[v]['RoC'][np.nonzero(x > cfg['RoC'])] = 4
self.flags[v]['RoC'][ma.getmaskarray(self.input[v])] = 9
if 'anomaly_detection' in cfg:
features = {}
for f in cfg['anomaly_detection']['features']:
if f == 'spike':
features['spike'] = spike(self.input[v])
elif f == 'gradient':
features['gradient'] = gradient(self.input[v])
elif f == 'tukey53H_norm':
features['tukey53H_norm'] = tukey53H_norm(self.input[v],
k=1.5)
else:
logging.error("Sorry, I can't evaluate anomaly_detection with: %s" % f)
self.flags[v]['anomaly_detection'] = \
anomaly_detection(features, cfg['anomaly_detection'])
if 'fuzzylogic' in cfg:
self.flags[v]['fuzzylogic'] = fuzzylogic(
self.auxiliary[v],
v,
cfg['fuzzylogic'])
def evaluate(self, v, cfg):
self.flags[v] = {}
if self.saveauxiliary:
if v not in self.auxiliary.keys():
self.auxiliary[v] = {}
if 'platform_identification' in cfg:
logging.warn(
"Sorry I'm not ready to evaluate platform_identification()")
if 'valid_geolocation' in cfg:
logging.warn("Sorry I'm not ready to evaluate valid_geolocation()")
if 'valid_speed' in cfg:
# Think about. ARGO also has a test valid_speed, but that is
# in respect to sucessive profiles. How is the best way to
# distinguish them here?
try:
if self.saveauxiliary:
self.flags[v]['valid_speed'], \
self.auxiliary[v]['valid_speed'] = \
possible_speed(self.input, cfg['valid_speed'])
except:
print("Fail on valid_speed")
if 'global_range' in cfg:
self.flags[v]['global_range'] = np.zeros(self.input[v].shape,
dtype='i1')
# Flag as 9 any masked input value
self.flags[v]['global_range'][ma.getmaskarray(self.input[v])] = 9
ind = (self.input[v] >= cfg['global_range']['minval']) & \
(self.input[v] <= cfg['global_range']['maxval'])
self.flags[v]['global_range'][np.nonzero(ind)] = 1
ind = (self.input[v] < cfg['global_range']['minval']) | \
(self.input[v] > cfg['global_range']['maxval'])
self.flags[v]['global_range'][np.nonzero(ind)] = 4
if 'regional_range' in cfg:
logging.warn("Sorry, I'm no ready to evaluate regional_range()")
if 'pressure_increasing' in cfg:
logging.warn(
"Sorry, I'm no ready to evaluate pressure_increasing()")
if 'profile_envelop' in cfg:
self.flags[v]['profile_envelop'] = profile_envelop(
self.input, cfg['profile_envelop'], v)
if 'gradient' in cfg:
threshold = cfg['gradient']
g = gradient(self.input[v])
if self.saveauxiliary:
self.auxiliary[v]['gradient'] = g
flag = np.zeros(g.shape, dtype='i1')
flag[ma.getmaskarray(self.input[v])] = 9
flag[np.nonzero(g > threshold)] = 4
flag[np.nonzero(g <= threshold)] = 1
self.flags[v]['gradient'] = flag
if 'gradient_depthconditional' in cfg:
cfg_tmp = cfg['gradient_depthconditional']
g = gradient(self.input[v])
flag = np.zeros(g.shape, dtype='i1')
# Flag as 9 any masked input value
flag[ma.getmaskarray(self.input[v])] = 9
# ---- Shallow zone -----------------
threshold = cfg_tmp['shallow_max']
flag[np.nonzero( \
(self['PRES'] <= cfg_tmp['pressure_threshold']) & \
(g > threshold))] \
= 4
flag[np.nonzero( \
(self['PRES'] <= cfg_tmp['pressure_threshold']) & \
(g <= threshold))] \
= 1
# ---- Deep zone --------------------
threshold = cfg_tmp['deep_max']
flag[np.nonzero( \
(self['PRES'] > cfg_tmp['pressure_threshold']) & \
(g > threshold))] \
= 4
flag[np.nonzero( \
(self['PRES'] > cfg_tmp['pressure_threshold']) & \
(g <= threshold))] \
= 1
self.flags[v]['gradient_depthconditional'] = flag
if 'spike' in cfg:
threshold = cfg['spike']
s = spike(self.input[v])
if self.saveauxiliary:
self.auxiliary[v]['spike'] = s
flag = np.zeros(s.shape, dtype='i1')
# Flag as 9 any masked input value
flag[ma.getmaskarray(self.input[v])] = 9
flag[np.nonzero(s > threshold)] = 4
flag[np.nonzero(s <= threshold)] = 1
self.flags[v]['spike'] = flag
if 'spike_depthconditional' in cfg:
cfg_tmp = cfg['spike_depthconditional']
s = spike(self.input[v])
flag = np.zeros(s.shape, dtype='i1')
# Flag as 9 any masked input value
flag[ma.getmaskarray(self.input[v])] = 9
# ---- Shallow zone -----------------
threshold = cfg_tmp['shallow_max']
flag[np.nonzero( \
(self['PRES'] <= cfg_tmp['pressure_threshold']) & \
(g > threshold))] \
= 4
flag[np.nonzero( \
(self['PRES'] <= cfg_tmp['pressure_threshold']) & \
(g <= threshold))] \
= 1
# ---- Deep zone --------------------
threshold = cfg_tmp['deep_max']
flag[np.nonzero( \
(self['PRES'] > cfg_tmp['pressure_threshold']) & \
(g > threshold))] \
= 4
flag[np.nonzero( \
(self['PRES'] > cfg_tmp['pressure_threshold']) & \
(g <= threshold))] \
= 1
self.flags[v]['spike_depthconditional'] = flag
if 'stuck_value' in cfg:
logging.warn("Sorry I'm not ready to evaluate stuck_value()")
if 'grey_list' in cfg:
logging.warn("Sorry I'm not ready to evaluate grey_list()")
if 'gross_sensor_drift' in cfg:
logging.warn(
"Sorry I'm not ready to evaluate gross_sensor_drift()")
if 'frozen_profile' in cfg:
logging.warn("Sorry I'm not ready to evaluate frozen_profile()")
if 'deepest_pressure' in cfg:
logging.warn("Sorry I'm not ready to evaluate deepest_pressure()")
if 'tukey53H_norm' in cfg:
"""
I slightly modified the Goring & Nikora 2002. It is
expected that CTD profiles has a typical depth
structure, with a range between surface and bottom.
"""
s = tukey53H_norm(self.input[v],
k=cfg['tukey53H_norm']['k'],
l=cfg['tukey53H_norm']['l'])
if self.saveauxiliary:
self.auxiliary[v]['tukey53H_norm'] = s
threshold = cfg['tukey53H_norm']['k']
flag = np.zeros(s.shape, dtype='i1')
# Flag as 9 any masked input value
flag[ma.getmaskarray(self.input[v])] = 9
flag[np.nonzero(s > threshold)] = 4
flag[np.nonzero(s <= threshold)] = 1
self.flags[v]['tukey53H_norm'] = flag
#if 'spike_depthsmooth' in cfg:
# from maud.window_func import _weight_hann as wfunc
# cfg_tmp = cfg['spike_depthsmooth']
# cfg_tmp['dzwindow'] = 10
# smooth = ma.masked_all(self.input[v].shape)
# z = ped['pressure']
# for i in range(len(self.input[v])):
# ind = np.nonzero(ma.absolute(z-z[i]) < \
# cfg_tmp['dzwindow'])[0]
# ind = ind[ind != i]
# w = wfunc(z[ind]-z[i], cfg_tmp['dzwindow'])
# smooth[i] = (T[ind]*w).sum()/w.sum()
# ARGO, test #12. (10C, 5PSU)
if 'digit_roll_over' in cfg:
threshold = cfg['digit_roll_over']
s = step(self.input[v])
if self.saveauxiliary:
self.auxiliary[v]['step'] = s
flag = np.zeros(s.shape, dtype='i1')
# Flag as 9 any masked input value
flag[ma.getmaskarray(self.input[v])] = 9
flag[np.nonzero(ma.absolute(s) > threshold)] = 4
flag[np.nonzero(ma.absolute(s) <= threshold)] = 1
self.flags[v]['digit_roll_over'] = flag
if 'bin_spike' in cfg:
bin = bin_spike(self.input[v], cfg['bin_spike'])
if self.saveauxiliary:
self.auxiliary[v]['bin_spike'] = bin
if 'density_inversion' in cfg:
try:
if self.saveauxiliary:
self.flags[v]['density_inversion'], \
self.auxiliary[v]['density_step'] = \
density_inversion(
self.input,
cfg['density_inversion'],
saveaux=True)
else:
self.flags[v]['density_inversion'] = density_inversion(
self.input, cfg['density_inversion'])
except:
print("Fail on density_inversion")
if 'woa_normbias' in cfg:
if self.saveauxiliary:
self.flags[v]['woa_normbias'], \
self.auxiliary[v]['woa_relbias'] = \
woa_normbias(self.input, v, cfg['woa_normbias'])
#for k in woa.keys():
# self.auxiliary[v][k] = woa[k]
#self.auxiliary[v]['woa_bias'] = woa_bias
#self.auxiliary[v]['woa_relbias'] = woa_bias/woa['woa_sd']
else:
self.flags[v]['woa_normbias'], \
tmp = \
woa_normbias(self.input, v, cfg['woa_normbias'])
del (tmp)
#if 'pstep' in cfg:
# ind = np.isfinite(self.input[v])
# ind = ma.getmaskarray(self.input[v])
# if self.saveauxiliary:
# self.auxiliary[v]['pstep'] = ma.concatenate(
# [ma.masked_all(1),
# np.diff(self.input['PRES'][ind])])
if 'RoC' in cfg:
x = ma.concatenate([ma.masked_all(1), ma.diff(self.input[v])])
if self.saveauxiliary:
self.auxiliary[v]['RoC'] = x
self.flags[v]['RoC'] = np.zeros(x.shape, dtype='i1')
self.flags[v]['RoC'][np.nonzero(x <= cfg['RoC'])] = 1
self.flags[v]['RoC'][np.nonzero(x > cfg['RoC'])] = 4
self.flags[v]['RoC'][ma.getmaskarray(self.input[v])] = 9
if 'anomaly_detection' in cfg:
features = {}
for f in cfg['anomaly_detection']['features']:
if f == 'spike':
features['spike'] = spike(self.input[v])
elif f == 'gradient':
features['gradient'] = gradient(self.input[v])
elif f == 'tukey53H_norm':
features['tukey53H_norm'] = tukey53H_norm(self.input[v],
k=1.5)
else:
logging.error(
"Sorry, I can't evaluate anomaly_detection with: %s" %
f)
self.flags[v]['anomaly_detection'] = \
anomaly_detection(features, cfg['anomaly_detection'])
if 'fuzzylogic' in cfg:
self.flags[v]['fuzzylogic'] = fuzzylogic(self.auxiliary[v], v,
cfg['fuzzylogic'])
def mdiff(array: np.ndarray) -> float:
"""Returns median difference of 1-D array."""
return float(ma.median(ma.diff(array)))
def rate_of_change(x):
y = ma.fix_invalid(np.ones_like(x) * np.nan)
y[1:] = ma.diff(x)
return y
def evaluate(self, v, cfg):
self.flags[v] = {}
# Apply common flag for all points.
if 'common' in self.flags:
N = self.input[v].shape
for f in self.flags['common']:
self.flags[v][f] = self.flags['common'][f] * \
np.ones(N, dtype='i1')
if self.saveauxiliary:
if v not in self.auxiliary.keys():
self.auxiliary[v] = {}
if 'platform_identification' in cfg:
logging.warn("Sorry I'm not ready to evaluate platform_identification()")
if 'valid_geolocation' in cfg:
logging.warn("Sorry I'm not ready to evaluate valid_geolocation()")
if 'valid_speed' in cfg:
# Think about. ARGO also has a test valid_speed, but that is
# in respect to sucessive profiles. How is the best way to
# distinguish them here?
try:
if self.saveauxiliary:
self.flags[v]['valid_speed'], \
self.auxiliary[v]['valid_speed'] = \
possible_speed(self.input, cfg['valid_speed'])
except:
print("Fail on valid_speed")
if 'global_range' in cfg:
self.flags[v]['global_range'] = global_range(
self.input, v, cfg['global_range'])
if 'regional_range' in cfg:
logging.warn("Sorry, I'm no ready to evaluate regional_range()")
if 'pressure_increasing' in cfg:
logging.warn("Sorry, I'm no ready to evaluate pressure_increasing()")
if 'profile_envelop' in cfg:
self.flags[v]['profile_envelop'] = profile_envelop(
self.input, cfg['profile_envelop'], v)
if 'gradient' in cfg:
y = Gradient(self.input, v, cfg['gradient'])
y.test()
if self.saveauxiliary:
self.auxiliary[v]['gradient'] = y.features['gradient']
self.flags[v]['gradient'] = y.flags['gradient']
if 'gradient_depthconditional' in cfg:
cfg_tmp = cfg['gradient_depthconditional']
g = gradient(self.input[v])
flag = np.zeros(g.shape, dtype='i1')
# Flag as 9 any masked input value
flag[ma.getmaskarray(self.input[v])] = 9
# ---- Shallow zone -----------------
threshold = cfg_tmp['shallow_max']
flag[np.nonzero( \
(self['PRES'] <= cfg_tmp['pressure_threshold']) & \
(g > threshold))] \
= 4
flag[np.nonzero( \
(self['PRES'] <= cfg_tmp['pressure_threshold']) & \
(g <= threshold))] \
= 1
# ---- Deep zone --------------------
threshold = cfg_tmp['deep_max']
flag[np.nonzero( \
(self['PRES'] > cfg_tmp['pressure_threshold']) & \
(g > threshold))] \
= 4
flag[np.nonzero( \
(self['PRES'] > cfg_tmp['pressure_threshold']) & \
(g <= threshold))] \
= 1
self.flags[v]['gradient_depthconditional'] = flag
if 'spike' in cfg:
y = Spike(self.input, v, cfg['spike'])
y.test()
if self.saveauxiliary:
self.auxiliary[v]['spike'] = y.features['spike']
self.flags[v]['spike'] = y.flags['spike']
if 'spike_depthconditional' in cfg:
cfg_tmp = cfg['spike_depthconditional']
s = spike(self.input[v])
flag = np.zeros(s.shape, dtype='i1')
# Flag as 9 any masked input value
flag[ma.getmaskarray(self.input[v])] = 9
# ---- Shallow zone -----------------
threshold = cfg_tmp['shallow_max']
flag[np.nonzero( \
(self['PRES'] <= cfg_tmp['pressure_threshold']) & \
(s > threshold))] \
= 4
flag[np.nonzero( \
(self['PRES'] <= cfg_tmp['pressure_threshold']) & \
(s <= threshold))] \
= 1
# ---- Deep zone --------------------
threshold = cfg_tmp['deep_max']
flag[np.nonzero( \
(self['PRES'] > cfg_tmp['pressure_threshold']) & \
(s > threshold))] \
= 4
flag[np.nonzero( \
(self['PRES'] > cfg_tmp['pressure_threshold']) & \
(s <= threshold))] \
= 1
self.flags[v]['spike_depthconditional'] = flag
if 'stuck_value' in cfg:
logging.warn("Sorry I'm not ready to evaluate stuck_value()")
if 'grey_list' in cfg:
logging.warn("Sorry I'm not ready to evaluate grey_list()")
if 'gross_sensor_drift' in cfg:
logging.warn("Sorry I'm not ready to evaluate gross_sensor_drift()")
if 'frozen_profile' in cfg:
logging.warn("Sorry I'm not ready to evaluate frozen_profile()")
if 'deepest_pressure' in cfg:
logging.warn("Sorry I'm not ready to evaluate deepest_pressure()")
if 'tukey53H_norm' in cfg:
y = Tukey53H(self.input, v, cfg['tukey53H_norm'])
y.test()
if self.saveauxiliary:
self.auxiliary[v]['tukey53H_norm'] = \
y.features['tukey53H_norm']
self.flags[v]['tukey53H_norm'] = y.flags['tukey53H_norm']
#if 'spike_depthsmooth' in cfg:
# from maud.window_func import _weight_hann as wfunc
# cfg_tmp = cfg['spike_depthsmooth']
# cfg_tmp['dzwindow'] = 10
# smooth = ma.masked_all(self.input[v].shape)
# z = ped['pressure']
# for i in range(len(self.input[v])):
# ind = np.nonzero(ma.absolute(z-z[i]) < \
# cfg_tmp['dzwindow'])[0]
# ind = ind[ind != i]
# w = wfunc(z[ind]-z[i], cfg_tmp['dzwindow'])
# smooth[i] = (T[ind]*w).sum()/w.sum()
# ARGO, test #12. (10C, 5PSU)
if 'digit_roll_over' in cfg:
threshold = cfg['digit_roll_over']
s = step(self.input[v])
if self.saveauxiliary:
self.auxiliary[v]['step'] = s
flag = np.zeros(s.shape, dtype='i1')
# Flag as 9 any masked input value
flag[ma.getmaskarray(self.input[v])] = 9
flag[np.nonzero(ma.absolute(s) > threshold)] = 4
flag[np.nonzero(ma.absolute(s) <= threshold)] = 1
self.flags[v]['digit_roll_over'] = flag
if 'bin_spike' in cfg:
y = Bin_Spike(self.input, v, cfg['bin_spike'])
# y.test()
if self.saveauxiliary:
self.auxiliary[v]['bin_spike'] = y.features['bin_spike']
# self.flags[v]['bin_spike'] = y.flags['bin_spike']
if 'density_inversion' in cfg:
try:
if self.saveauxiliary:
self.flags[v]['density_inversion'], \
self.auxiliary[v]['density_step'] = \
density_inversion(
self.input,
cfg['density_inversion'],
saveaux=True)
else:
self.flags[v]['density_inversion'] = density_inversion(
self.input, cfg['density_inversion'])
except:
print("Fail on density_inversion")
if 'woa_normbias' in cfg:
y = WOA_NormBias(self.input, v, cfg['woa_normbias'])
# self.attributes)
y.test()
if self.saveauxiliary:
for f in y.features:
self.auxiliary[v][f] = y.features[f]
self.flags[v]['woa_normbias'] = y.flags['woa_normbias']
#if 'pstep' in cfg:
# ind = np.isfinite(self.input[v])
# ind = ma.getmaskarray(self.input[v])
# if self.saveauxiliary:
# self.auxiliary[v]['pstep'] = ma.concatenate(
# [ma.masked_all(1),
# np.diff(self.input['PRES'][ind])])
if 'rate_of_change' in cfg:
self.flags[v]['rate_of_change'], RoC = \
rate_of_change(self.input, v, cfg['rate_of_change'])
if self.saveauxiliary:
self.auxiliary[v]['rate_of_change'] = RoC
if 'cum_rate_of_change' in cfg:
x = cum_rate_of_change(self.input, v,
cfg['cum_rate_of_change']['memory'])
self.flags[v]['cum_rate_of_change'] = np.zeros(x.shape, dtype='i1')
self.flags[v]['cum_rate_of_change'][
np.nonzero(x <= cfg['cum_rate_of_change']['threshold'])
] = 1
self.flags[v]['cum_rate_of_change'][
np.nonzero(x > cfg['cum_rate_of_change']['threshold'])
] = 4
self.flags[v]['cum_rate_of_change'][
ma.getmaskarray(self.input[v])] = 9
# FIXME: the Anomaly Detection and Fuzzy require some features
# to be estimated previously. Generalize this.
if 'anomaly_detection' in cfg:
features = {}
for f in cfg['anomaly_detection']['features']:
if f == 'spike':
features['spike'] = spike(self.input[v])
elif f == 'gradient':
features['gradient'] = gradient(self.input[v])
elif f == 'tukey53H_norm':
features['tukey53H_norm'] = tukey53H_norm(self.input[v])
elif f == 'rate_of_change':
RoC = ma.masked_all_like(self.input[v])
RoC[1:] = ma.absolute(ma.diff(self.input[v]))
features['rate_of_change'] = RoC
elif (f == 'woa_normbias'):
y = WOA_NormBias(self.input, v, {})
features['woa_normbias'] = \
np.abs(y.features['woa_normbias'])
else:
logging.error("Sorry, I can't evaluate anomaly_detection with: %s" % f)
self.flags[v]['anomaly_detection'] = \
anomaly_detection(features, cfg['anomaly_detection'])
if 'morello2014' in cfg:
self.flags[v]['morello2014'] = morello2014(
features=self.auxiliary[v],
cfg=cfg['morello2014'])
if 'fuzzylogic' in cfg:
features = {}
for f in cfg['fuzzylogic']['features']:
if f == 'spike':
features['spike'] = spike(self.input[v])
elif f == 'gradient':
features['gradient'] = gradient(self.input[v])
elif f == 'tukey53H_norm':
features['tukey53H_norm'] = tukey53H_norm(self.input[v],
k=1.5)
elif f == 'rate_of_change':
RoC = ma.masked_all_like(data[v])
RoC[1:] = ma.absolute(ma.diff(data[v]))
features['rate_of_change'] = RoC
elif (f == 'woa_normbias'):
y = WOA_NormBias(self.input, v, {})
features['woa_normbias'] = \
np.abs(y.features['woa_normbias'])
else:
logging.error("Sorry, I can't evaluate fuzzylogic with: %s" % f)
self.flags[v]['fuzzylogic'] = fuzzylogic(
features=features,
cfg=cfg['fuzzylogic'])
self.flags[v]['overall'] = combined_flag(self.flags[v])
def step(x):
y = ma.masked_all(x.shape, dtype=x.dtype)
y[1:] = ma.diff(x)
return y
def evaluate(self, v, cfg):
self.flags[v] = {}
# Apply common flag for all points.
if 'common' in self.flags:
N = self.input[v].shape
for f in self.flags['common']:
self.flags[v][f] = self.flags['common'][f] * \
np.ones(N, dtype='i1')
if self.saveauxiliary:
if v not in self.auxiliary.keys():
self.auxiliary[v] = {}
if 'platform_identification' in cfg:
logging.warn("Sorry I'm not ready to evaluate platform_identification()")
if 'valid_geolocation' in cfg:
logging.warn("Sorry I'm not ready to evaluate valid_geolocation()")
if 'valid_speed' in cfg:
# Think about. ARGO also has a test valid_speed, but that is
# in respect to sucessive profiles. How is the best way to
# distinguish them here?
try:
if self.saveauxiliary:
self.flags[v]['valid_speed'], \
self.auxiliary[v]['valid_speed'] = \
possible_speed(self.input, cfg['valid_speed'])
except:
print("Fail on valid_speed")
if 'global_range' in cfg:
self.flags[v]['global_range'] = global_range(
self.input, v, cfg['global_range'])
if 'regional_range' in cfg:
logging.warn("Sorry, I'm no ready to evaluate regional_range()")
if 'pressure_increasing' in cfg:
logging.warn("Sorry, I'm no ready to evaluate pressure_increasing()")
if 'profile_envelop' in cfg:
self.flags[v]['profile_envelop'] = profile_envelop(
self.input, cfg['profile_envelop'], v)
if 'gradient' in cfg:
y = Gradient(self.input, v, cfg['gradient'])
y.test()
if self.saveauxiliary:
self.auxiliary[v]['gradient'] = y.features['gradient']
self.flags[v]['gradient'] = y.flags['gradient']
if 'gradient_depthconditional' in cfg:
cfg_tmp = cfg['gradient_depthconditional']
g = gradient(self.input[v])
flag = np.zeros(g.shape, dtype='i1')
# Flag as 9 any masked input value
flag[ma.getmaskarray(self.input[v])] = 9
# ---- Shallow zone -----------------
threshold = cfg_tmp['shallow_max']
flag[np.nonzero( \
(self['PRES'] <= cfg_tmp['pressure_threshold']) & \
(g > threshold))] \
= 4
flag[np.nonzero( \
(self['PRES'] <= cfg_tmp['pressure_threshold']) & \
(g <= threshold))] \
= 1
# ---- Deep zone --------------------
threshold = cfg_tmp['deep_max']
flag[np.nonzero( \
(self['PRES'] > cfg_tmp['pressure_threshold']) & \
(g > threshold))] \
= 4
flag[np.nonzero( \
(self['PRES'] > cfg_tmp['pressure_threshold']) & \
(g <= threshold))] \
= 1
self.flags[v]['gradient_depthconditional'] = flag
if 'spike' in cfg:
y = Spike(self.input, v, cfg['spike'])
y.test()
if self.saveauxiliary:
self.auxiliary[v]['spike'] = y.features['spike']
self.flags[v]['spike'] = y.flags['spike']
if 'spike_depthconditional' in cfg:
cfg_tmp = cfg['spike_depthconditional']
s = spike(self.input[v])
flag = np.zeros(s.shape, dtype='i1')
# Flag as 9 any masked input value
flag[ma.getmaskarray(self.input[v])] = 9
# ---- Shallow zone -----------------
threshold = cfg_tmp['shallow_max']
flag[np.nonzero( \
(self['PRES'] <= cfg_tmp['pressure_threshold']) & \
(s > threshold))] \
= 4
flag[np.nonzero( \
(self['PRES'] <= cfg_tmp['pressure_threshold']) & \
(s <= threshold))] \
= 1
# ---- Deep zone --------------------
threshold = cfg_tmp['deep_max']
flag[np.nonzero( \
(self['PRES'] > cfg_tmp['pressure_threshold']) & \
(s > threshold))] \
= 4
flag[np.nonzero( \
(self['PRES'] > cfg_tmp['pressure_threshold']) & \
(s <= threshold))] \
= 1
self.flags[v]['spike_depthconditional'] = flag
if 'stuck_value' in cfg:
logging.warn("Sorry I'm not ready to evaluate stuck_value()")
if 'grey_list' in cfg:
logging.warn("Sorry I'm not ready to evaluate grey_list()")
if 'gross_sensor_drift' in cfg:
logging.warn("Sorry I'm not ready to evaluate gross_sensor_drift()")
if 'frozen_profile' in cfg:
logging.warn("Sorry I'm not ready to evaluate frozen_profile()")
if 'deepest_pressure' in cfg:
logging.warn("Sorry I'm not ready to evaluate deepest_pressure()")
if 'tukey53H_norm' in cfg:
y = Tukey53H(self.input, v, cfg['tukey53H_norm'])
y.test()
if self.saveauxiliary:
self.auxiliary[v]['tukey53H_norm'] = \
y.features['tukey53H_norm']
self.flags[v]['tukey53H_norm'] = y.flags['tukey53H_norm']
#if 'spike_depthsmooth' in cfg:
# from maud.window_func import _weight_hann as wfunc
# cfg_tmp = cfg['spike_depthsmooth']
# cfg_tmp['dzwindow'] = 10
# smooth = ma.masked_all(self.input[v].shape)
# z = ped['pressure']
# for i in range(len(self.input[v])):
# ind = np.nonzero(ma.absolute(z-z[i]) < \
# cfg_tmp['dzwindow'])[0]
# ind = ind[ind != i]
# w = wfunc(z[ind]-z[i], cfg_tmp['dzwindow'])
# smooth[i] = (T[ind]*w).sum()/w.sum()
# ARGO, test #12. (10C, 5PSU)
if 'digit_roll_over' in cfg:
threshold = cfg['digit_roll_over']
s = step(self.input[v])
if self.saveauxiliary:
self.auxiliary[v]['step'] = s
flag = np.zeros(s.shape, dtype='i1')
# Flag as 9 any masked input value
flag[ma.getmaskarray(self.input[v])] = 9
flag[np.nonzero(ma.absolute(s) > threshold)] = 4
flag[np.nonzero(ma.absolute(s) <= threshold)] = 1
self.flags[v]['digit_roll_over'] = flag
if 'bin_spike' in cfg:
y = Bin_Spike(self.input, v, cfg['bin_spike'])
# y.test()
if self.saveauxiliary:
self.auxiliary[v]['bin_spike'] = y.features['bin_spike']
# self.flags[v]['bin_spike'] = y.flags['bin_spike']
if 'density_inversion' in cfg:
try:
if self.saveauxiliary:
self.flags[v]['density_inversion'], \
self.auxiliary[v]['density_step'] = \
density_inversion(
self.input,
cfg['density_inversion'],
saveaux=True)
else:
self.flags[v]['density_inversion'] = density_inversion(
self.input, cfg['density_inversion'])
except:
print("Fail on density_inversion")
if 'woa_normbias' in cfg:
y = WOA_NormBias(self.input, v, cfg['woa_normbias'])
# self.attributes)
y.test()
if self.saveauxiliary:
for f in y.features:
self.auxiliary[v][f] = y.features[f]
self.flags[v]['woa_normbias'] = y.flags['woa_normbias']
#if 'pstep' in cfg:
# ind = np.isfinite(self.input[v])
# ind = ma.getmaskarray(self.input[v])
# if self.saveauxiliary:
# self.auxiliary[v]['pstep'] = ma.concatenate(
# [ma.masked_all(1),
# np.diff(self.input['PRES'][ind])])
if 'rate_of_change' in cfg:
self.flags[v]['rate_of_change'], RoC = \
rate_of_change(self.input, v, cfg['rate_of_change'])
if self.saveauxiliary:
self.auxiliary[v]['rate_of_change'] = RoC
if 'cum_rate_of_change' in cfg:
x = cum_rate_of_change(self.input, v,
cfg['cum_rate_of_change']['memory'])
self.flags[v]['cum_rate_of_change'] = np.zeros(x.shape, dtype='i1')
self.flags[v]['cum_rate_of_change'][
np.nonzero(x <= cfg['cum_rate_of_change']['threshold'])
] = 1
self.flags[v]['cum_rate_of_change'][
np.nonzero(x > cfg['cum_rate_of_change']['threshold'])
] = 4
self.flags[v]['cum_rate_of_change'][
ma.getmaskarray(self.input[v])] = 9
# FIXME: the Anomaly Detection and Fuzzy require some features
# to be estimated previously. Generalize this.
if 'anomaly_detection' in cfg:
features = {}
for f in cfg['anomaly_detection']['features']:
if f == 'spike':
features['spike'] = spike(self.input[v])
elif f == 'gradient':
features['gradient'] = gradient(self.input[v])
elif f == 'tukey53H_norm':
features['tukey53H_norm'] = tukey53H_norm(self.input[v])
elif f == 'rate_of_change':
RoC = ma.masked_all_like(self.input[v])
RoC[1:] = ma.absolute(ma.diff(self.input[v]))
features['rate_of_change'] = RoC
elif (f == 'woa_normbias'):
y = WOA_NormBias(self.input, v, {})
features['woa_normbias'] = \
np.abs(y.features['woa_normbias'])
else:
logging.error("Sorry, I can't evaluate anomaly_detection with: %s" % f)
self.flags[v]['anomaly_detection'] = \
anomaly_detection(features, cfg['anomaly_detection'])
if 'morello2014' in cfg:
self.flags[v]['morello2014'] = morello2014(
features=self.auxiliary[v],
cfg=cfg['morello2014'])
if 'fuzzylogic' in cfg:
features = {}
for f in cfg['fuzzylogic']['features']:
if f == 'spike':
features['spike'] = spike(self.input[v])
elif f == 'gradient':
features['gradient'] = gradient(self.input[v])
elif f == 'tukey53H_norm':
features['tukey53H_norm'] = tukey53H_norm(self.input[v],
k=1.5)
elif f == 'rate_of_change':
RoC = ma.masked_all_like(data[v])
RoC[1:] = ma.absolute(ma.diff(data[v]))
features['rate_of_change'] = RoC
elif (f == 'woa_normbias'):
y = WOA_NormBias(self.input, v, {})
features['woa_normbias'] = \
np.abs(y.features['woa_normbias'])
else:
logging.error("Sorry, I can't evaluate fuzzylogic with: %s" % f)
self.flags[v]['fuzzylogic'] = fuzzylogic(
features=features,
cfg=cfg['fuzzylogic'])
self.flags[v]['overall'] = combined_flag(self.flags[v])
def mdiff(x):
"""Returns median difference of 1-D array."""
return ma.median(ma.diff(x))
