def biweight(x, cst): """ Computes the biweight average and midvariance for a given 1D array. Returns a tuple (biweight mean, biweight variance). Parameters ---------- x: {ndarray} Input Array cst : {float} Parameter controlling how outliers are censored. Notes ----- The function is restricted to 1D data only. """ assert (x.ndim == 1, "1D array only !") xmed = ma.median(x, 0) manom = x - xmed mad = ma.median(ma.absolute(manom)) u_i = (manom/float(cst*mad)) u_i *= ma.less_equal(ma.absolute(u_i), 1.).astype(float) w_i = (1-u_i**2) if ma.count(w_i) > 0: biw_m = xmed + ma.sum(manom * w_i**2)/ma.sum(w_i**2) else: biw_m = xmed biw_sd = ma.sqrt(ma.count(x)*ma.sum(manom**2 * w_i**4)) biw_sd *= 1./ma.absolute(ma.sum(w_i * (1-5*u_i**2))) return (biw_m, biw_sd.item())
def rate_of_change(data, v, cfg): RoC = ma.masked_all_like(data[v]) RoC[1:] = ma.diff(data[v]) flag = np.zeros(data[v].shape, dtype='i1') flag[np.nonzero(ma.absolute(RoC) <= cfg)] = 1 flag[np.nonzero(ma.absolute(RoC) > cfg)] = 4 flag[ma.getmaskarray(data[v])] = 9 return flag, RoC
def test_testUfuncs1(self): # Test various functions such as sin, cos. (x, y, a10, m1, m2, xm, ym, z, zm, xf, s) = self.d assert_(eq(np.cos(x), cos(xm))) assert_(eq(np.cosh(x), cosh(xm))) assert_(eq(np.sin(x), sin(xm))) assert_(eq(np.sinh(x), sinh(xm))) assert_(eq(np.tan(x), tan(xm))) assert_(eq(np.tanh(x), tanh(xm))) with np.errstate(divide='ignore', invalid='ignore'): assert_(eq(np.sqrt(abs(x)), sqrt(xm))) assert_(eq(np.log(abs(x)), log(xm))) assert_(eq(np.log10(abs(x)), log10(xm))) assert_(eq(np.exp(x), exp(xm))) assert_(eq(np.arcsin(z), arcsin(zm))) assert_(eq(np.arccos(z), arccos(zm))) assert_(eq(np.arctan(z), arctan(zm))) assert_(eq(np.arctan2(x, y), arctan2(xm, ym))) assert_(eq(np.absolute(x), absolute(xm))) assert_(eq(np.equal(x, y), equal(xm, ym))) assert_(eq(np.not_equal(x, y), not_equal(xm, ym))) assert_(eq(np.less(x, y), less(xm, ym))) assert_(eq(np.greater(x, y), greater(xm, ym))) assert_(eq(np.less_equal(x, y), less_equal(xm, ym))) assert_(eq(np.greater_equal(x, y), greater_equal(xm, ym))) assert_(eq(np.conjugate(x), conjugate(xm))) assert_(eq(np.concatenate((x, y)), concatenate((xm, ym)))) assert_(eq(np.concatenate((x, y)), concatenate((x, y)))) assert_(eq(np.concatenate((x, y)), concatenate((xm, y)))) assert_(eq(np.concatenate((x, y, x)), concatenate((x, ym, x))))
def test(self): self.flags = {} try: threshold = self.cfg['threshold'] except KeyError: module_logger.debug( "Deprecated cfg format. It should contain a threshold item.") threshold = self.cfg try: flag_good = self.cfg['flag_good'] except KeyError: flag_good = 1 try: flag_bad = self.cfg['flag_bad'] except KeyError: flag_bad = 4 assert (np.size(threshold) == 1) \ and (threshold is not None) \ and (np.isfinite(threshold)) flag = np.zeros(self.data[self.varname].shape, dtype='i1') feature = ma.absolute(self.features['rate_of_change']) flag[np.nonzero(feature > threshold)] = self.flag_bad flag[np.nonzero(feature <= threshold)] = self.flag_good flag[ma.getmaskarray(self.data[self.varname])] = 9 self.flags['digit_roll_over'] = flag
def count_number_of_variables(self): max_in_formula = 0 for clause in self.formula: max_in_clause = max(absolute(clause)) if max_in_clause > max_in_formula: max_in_formula = max_in_clause return max_in_formula
def get_histogram_centers(self): diff_image = absolute( cv2.blur(self.images[1], (5, 5)) - cv2.blur(self.images[0], (5, 5))) # diff_image = absolute(self.images[1] - self.images[0]) diff_image = cv2.cvtColor(diff_image, cv2.COLOR_BGR2GRAY) ret, mask = cv2.threshold(diff_image, 127, 255, cv2.THRESH_BINARY) mask_bool = mask.astype(np.bool) nonzero_pos = nonzero(mask) nonzero_num = len(nonzero_pos[0]) if FRACTION_MIN * self.num_pixels < nonzero_num < FRACTION_MAX * self.num_pixels: data_set = np.vstack(nonzero_pos).T mask_label, self.centers = make_kmeans(data_set) mask_res = mask_bool.ravel() mask_int = np.arange(len(mask_res)) mask0 = get_class_mask(mask_res, mask_label, mask_int, mask_bool, 0) mask1 = get_class_mask(mask_res, mask_label, mask_int, mask_bool, 1) self.histogram0 = get_histogram(self.images[1], mask0) self.histogram1 = get_histogram(self.images[1], mask1) if self._debug: self._debug_output(mask, mask0, mask1)
def get_mask(self): diff_image = absolute( cv2.blur(self.images[1], (5, 5)) - cv2.blur(self.images[0], (5, 5))) # diff_image = absolute(self.images[1] - self.images[0]) diff_image = cv2.cvtColor(diff_image, cv2.COLOR_BGR2GRAY) ret, mask = cv2.threshold(diff_image, 127, 255, cv2.THRESH_BINARY) return mask
def log_linear_vinterp(T,P,levs): ''' # Author Charles Doutriaux # Version 1.1 # Expect 2D field here so there''s no reorder which I suspect to do a memory leak # email: [email protected] # Converts a field from sigma levels to pressure levels # Log linear interpolation # Input # T : temperature on sigma levels # P : pressure field from TOP (level 0) to BOTTOM (last level) # levs : pressure levels to interplate to (same units as P) # Output # t : temperature on pressure levels (levs) # External: Numeric''' import numpy.ma as MA ## from numpy.oldnumeric.ma import ones,Float,greater,less,logical_and,where,equal,log,asarray,Float16 sh=P.shape nsigma=sh[0] # Number of sigma levels try: nlev=len(levs) # Number of pressure levels except: nlev=1 # if only one level len(levs) would breaks t=[] for ilv in range(nlev): # loop through pressure levels try: lev=levs[ilv] # get value for the level except: lev=levs # only 1 level passed # print ' ......... level:',lev Pabv=MA.ones(P[0].shape,Numeric.Float) Tabv=-Pabv # Temperature on sigma level Above Tbel=-Pabv # Temperature on sigma level Below Pbel=-Pabv # Pressure on sigma level Below Pabv=-Pabv # Pressure on sigma level Above for isg in range(1,nsigma): # loop from second sigma level to last one ## print 'Sigma level #',isg a = MA.greater(P[isg], lev) # Where is the pressure greater than lev b = MA.less(P[isg-1],lev) # Where is the pressure less than lev # Now looks if the pressure level is in between the 2 sigma levels # If yes, sets Pabv, Pbel and Tabv, Tbel Pabv=MA.where(MA.logical_and(a,b),P[isg],Pabv) # Pressure on sigma level Above Tabv=MA.where(MA.logical_and(a,b),T[isg],Tabv) # Temperature on sigma level Above Pbel=MA.where(MA.logical_and(a,b),P[isg-1],Pbel) # Pressure on sigma level Below Tbel=MA.where(MA.logical_and(a,b),T[isg-1],Tbel) # Temperature on sigma level Below # end of for isg in range(1,nsigma) # val=where(equal(Pbel,-1.),Pbel.missing_value,lev) # set to missing value if no data below lev if there is tl=MA.masked_where(MA.equal(Pbel,-1.),MA.log(lev/MA.absolute(Pbel))/MA.log(Pabv/Pbel)*(Tabv-Tbel)+Tbel) # Interpolation t.append(tl) # add a level to the output # end of for ilv in range(nlev) return asMA(t).astype(Numeric.Float32) # convert t to an array
def check_disjunction(clause, chromosome): result = 0 for element in clause: variable_index = absolute(element) - 1 if element > 0: result |= chromosome[variable_index] else: result |= not chromosome[variable_index] return result
def cum_rate_of_change(data, v, memory): output = ma.masked_all_like(data[v]) output[1:] = ma.absolute(ma.diff(data[v])) for i in range(2, output.size): if output[i] < output[i-1]: output[i] = (1 - memory) * output[i] + memory * output[i-1] return output
def cum_rate_of_change(data, v, memory): output = ma.masked_all_like(data[v]) output[1:] = ma.absolute(ma.diff(data[v])) for i in range(2, output.size): if output[i] < output[i - 1]: output[i] = (1 - memory) * output[i] + memory * output[i - 1] return output
def cum_rate_of_change(x, memory): y = ma.fix_invalid(np.ones_like(x) * np.nan) y[1:] = ma.absolute(ma.diff(x)) for i in range(2, y.size): if y[i] < y[i - 1]: y[i] = (1 - memory) * y[i] + memory * y[i - 1] return y
def rate_of_change(data, v, cfg): RoC = ma.masked_all_like(data[v]) RoC[1:] = ma.absolute(ma.diff(data[v])) flag = np.zeros(data[v].shape, dtype='i1') flag[np.nonzero(RoC <= cfg)] = 1 flag[np.nonzero(RoC > cfg)] = 4 flag[ma.getmaskarray(data[v])] = 9 return flag, RoC
def myfunction(d,mx,mn): from numpy.ma import maximum,minimum,masked_where,absolute,greater,count try: d=masked_where(greater(absolute(d),9.9E19),d) if count(d)==0 : return mx,mn mx=float(maximum(mx,float(maximum(d)))) mn=float(minimum(mn,float(minimum(d)))) except: for i in d: mx,mn=myfunction(i,mx,mn) return mx,mn
def myfunction(d, mx, mn): from numpy.ma import maximum, minimum, masked_where, absolute, greater, count try: d = masked_where(greater(absolute(d), 9.9E19), d) if count(d) == 0: return mx, mn mx = float(maximum(mx, float(maximum(d)))) mn = float(minimum(mn, float(minimum(d)))) except: for i in d: mx, mn = myfunction(i, mx, mn) return mx, mn
def woa_normbias(data, v, cfg): if ('LATITUDE' in data.keys()) and ('LONGITUDE' in data.keys()): if 'datetime' in data.keys(): d = data['datetime'] elif ('datetime' in data.attributes): d0 = data.attributes['datetime'] if ('timeS' in data.keys()): d = [d0 + timedelta(seconds=s) for s in data['timeS']] else: d = [data.attributes['datetime']]*len(data['LATITUDE']), woa = woa_track_from_file( d, data['LATITUDE'], data['LONGITUDE'], cfg['file'], varnames=cfg['vars']) elif ('LATITUDE' in data.attributes.keys()) and \ ('LONGITUDE' in data.attributes.keys()) and \ ('PRES' in data.keys()): woa = woa_profile(v, data.attributes['datetime'], data.attributes['LATITUDE'], data.attributes['LONGITUDE'], data['PRES'], cfg) if woa is None: # self.logger.warn("%s - WOA is not available at this site" % # self.name) flag = np.zeros(data[v].shape, dtype='i1') woa_normbias = ma.masked_all(data[v].shape) return flag, woa_normbias woa_bias = ma.absolute(data[v] - woa['woa_an']) woa_normbias = woa_bias/woa['woa_sd'] flag = np.zeros(data[v].shape, dtype='i1') ind = np.nonzero(woa_normbias <= cfg['sigma_threshold']) flag[ind] = 1 # cfg['flag_good'] ind = np.nonzero(woa_normbias > cfg['sigma_threshold']) flag[ind] = 3 # cfg['flag_bad'] # Flag as 9 any masked input value flag[ma.getmaskarray(data[v])] = 9 return flag, woa_normbias
def _make_verts(self, U, V): uv = ma.asarray(U+V*1j) a = ma.absolute(uv) if self.scale is None: sn = max(10, math.sqrt(self.N)) scale = 1.8 * a.mean() * sn / self.span # crude auto-scaling self.scale = scale length = a/(self.scale*self.width) X, Y = self._h_arrows(length) # There seems to be a ma bug such that indexing # a masked array with one element converts it to # an ndarray. theta = np.angle(ma.asarray(uv[..., np.newaxis]).filled(0)) xy = (X+Y*1j) * np.exp(1j*theta)*self.width xy = xy[:,:,np.newaxis] XY = ma.concatenate((xy.real, xy.imag), axis=2) return XY
def test(self): self.flags = {} try: threshold = self.cfg["threshold"] except: print("Deprecated cfg format. It should contain a threshold item.") threshold = self.cfg assert ((np.size(threshold) == 1) and (threshold is not None) and (np.isfinite(threshold))) flag = np.zeros(self.data[self.varname].shape, dtype="i1") feature = ma.absolute(self.features["cum_rate_of_change"]) flag[np.nonzero(feature > threshold)] = self.flag_bad flag[np.nonzero(feature <= threshold)] = self.flag_good flag[ma.getmaskarray(self.data[self.varname])] = 9 self.flags["cum_rate_of_change"] = flag
def _make_verts(self, U, V): uv = ma.asarray(U + V * 1j) a = ma.absolute(uv) if self.scale is None: sn = max(10, math.sqrt(self.N)) scale = 1.8 * a.mean() * sn / self.span # crude auto-scaling self.scale = scale length = a / (self.scale * self.width) X, Y = self._h_arrows(length) # There seems to be a ma bug such that indexing # a masked array with one element converts it to # an ndarray. theta = np.angle(ma.asarray(uv[..., np.newaxis]).filled(0)) xy = (X + Y * 1j) * np.exp(1j * theta) * self.width xy = xy[:, :, np.newaxis] XY = ma.concatenate((xy.real, xy.imag), axis=2) return XY
def woa_normbias(data, v, cfg): if ('LATITUDE' in data.keys()) and ('LONGITUDE' in data.keys()): if 'datetime' in data.keys(): d = data['datetime'] elif ('datetime' in data.attributes): d0 = data.attributes['datetime'] if ('timeS' in data.keys()): d = [d0 + timedelta(seconds=s) for s in data['timeS']] else: d = [data.attributes['datetime']] * len(data['LATITUDE']), woa = woa_track_from_file(d, data['LATITUDE'], data['LONGITUDE'], cfg['file'], varnames=cfg['vars']) elif ('LATITUDE' in data.attributes.keys()) and \ ('LONGITUDE' in data.attributes.keys()) and \ ('PRES' in data.keys()): woa = woa_profile(v, data.attributes['datetime'], data.attributes['LATITUDE'], data.attributes['LONGITUDE'], data['PRES'], cfg) if woa is None: # self.logger.warn("%s - WOA is not available at this site" % # self.name) flag = np.zeros(data[v].shape, dtype='i1') woa_normbias = ma.masked_all(data[v].shape) return flag, woa_normbias woa_bias = ma.absolute(data[v] - woa['woa_an']) woa_normbias = woa_bias / woa['woa_sd'] flag = np.zeros(data[v].shape, dtype='i1') ind = np.nonzero(woa_normbias <= cfg['sigma_threshold']) flag[ind] = 1 # cfg['flag_good'] ind = np.nonzero(woa_normbias > cfg['sigma_threshold']) flag[ind] = 3 # cfg['flag_bad'] # Flag as 9 any masked input value flag[ma.getmaskarray(data[v])] = 9 return flag, woa_normbias
def corr_proba(r, ndata, ndataset=2, dof=False): """Probability of rejecting correlations - **r**: Correlation coefficient - **ndata**: Number of records use for correlations - **ndataset**, optional: Number of datasets (1 for autocorrelations, else 2) [default: 2] .. todo:: This must be rewritten using :mod:`scipy.stats` """ # TODO: use scipy for betai and _gamma? from genutil.salstat import betai,_gammaln # Basic tests ndata = MA.masked_equal(ndata,0,copy=0) r = MV2.masked_where(MA.equal(MA.absolute(r),1.),r,copy=0) # Degree of freedom if dof: df = ndata else: df = ndata-2-ndataset # Advanced test: prevent extreme values by locally decreasing the dof reduc = N.ones(r.shape) z = None while z is None or MA.count(MA.masked_greater(z,-600.)): if z is not None: imax = MA.argmin(z.ravel()) reduc.flat[imax] += 1 dfr = df/reduc t = r*MV2.sqrt(dfr/((1.0-r)* (1.0+r))) a = 0.5*dfr b = 0.5 x = df/(dfr+t**2) z = _gammaln(a+b)-_gammaln(a)-_gammaln(b)+a*MA.log(x)+b*MA.log(1.0-x) # Perfom the test and format the variable prob = MV2.masked_array(betai(a,b,x),axes=r.getAxisList())*100 prob.id = 'corr_proba' ; prob.name = prob.id prob.long_name = 'Probability of rejection' prob.units = '%' return prob
def _make_verts(self, U, V): uv = ma.asarray(U + V * 1j) a = ma.absolute(uv) if self.scale is None: sn = max(10, math.sqrt(self.N)) scale = 1.8 * a.mean() * sn / self.span # crude auto-scaling self.scale = scale length = a / (self.scale * self.width) X, Y = self._h_arrows(length) if self.angles == 'xy': theta = self._angles(U, V).filled(0)[:, np.newaxis] elif self.angles == 'uv': theta = np.angle(ma.asarray(uv[..., np.newaxis]).filled(0)) else: theta = ma.asarray(self.angles * np.pi / 180.0).filled(0) xy = (X + Y * 1j) * np.exp(1j * theta) * self.width xy = xy[:, :, np.newaxis] XY = ma.concatenate((xy.real, xy.imag), axis=2) return XY
def corr_proba(r, ndata, ndataset=2, dof=False): """Probability of rejecting correlations - **r**: Correlation coefficient - **ndata**: Number of records use for correlations - **ndataset**, optional: Number of datasets (1 for autocorrelations, else 2) [default: 2] .. todo:: This must be rewritten using :mod:`scipy.stats` """ # Basic tests ndata = MA.masked_equal(ndata,0,copy=0) r = MV2.masked_where(MA.equal(MA.absolute(r),1.),r,copy=0) # Degree of freedom if dof: df = ndata else: df = ndata-2-ndataset # Advanced test: prevent extreme values by locally decreasing the dof reduc = N.ones(r.shape) z = None while z is None or MA.count(MA.masked_greater(z,-600.)): if z is not None: imax = MA.argmin(z.ravel()) reduc.flat[imax] += 1 dfr = df/reduc t = r*MV2.sqrt(dfr/((1.0-r)* (1.0+r))) a = 0.5*dfr b = 0.5 x = df/(dfr+t**2) z = _gammaln(a+b)-_gammaln(a)-_gammaln(b)+a*MA.log(x)+b*MA.log(1.0-x) # Perfom the test and format the variable prob = MV2.masked_array(betai(a,b,x),axes=r.getAxisList())*100 prob.id = 'corr_proba' ; prob.name = prob.id prob.long_name = 'Probability of rejection' prob.units = '%' return prob
def _make_verts(self, U, V): uv = ma.asarray(U+V*1j) a = ma.absolute(uv) if self.scale is None: sn = max(10, math.sqrt(self.N)) scale = 1.8 * a.mean() * sn / self.span # crude auto-scaling self.scale = scale length = a/(self.scale*self.width) X, Y = self._h_arrows(length) if self.angles == 'xy': theta = self._angles(U, V).filled(0)[:,np.newaxis] elif self.angles == 'uv': theta = np.angle(ma.asarray(uv[..., np.newaxis]).filled(0)) else: theta = ma.asarray(self.angles*np.pi/180.0).filled(0) xy = (X+Y*1j) * np.exp(1j*theta)*self.width xy = xy[:,:,np.newaxis] XY = ma.concatenate((xy.real, xy.imag), axis=2) return XY
def setcolorinterrupt(self): val = self.stream.readAll().data()[-5000:] self.timer.stop() if val: from numpy import fft from numpy.ma import absolute val = [i - 128 for i in val] fur = absolute(fft.fft(val)) freq = fft.fftfreq(len(val), d=1 / 44100) fur = fur[1:int(len(fur) / 2)] freq = freq[1:int(len(freq) / 2)] switch = { 'Smooth': self.smooth, 'Change': self.change, 'Flash': self.flash, 'Strob': self.strob } switch[self.mode](fur, freq) timeout = 50 if self.mode == 'Smooth' else 200 self.timer.start(timeout)
def test(self): self.flags = {} try: threshold = self.cfg["threshold"] except KeyError: module_logger.debug( "Deprecated cfg format. It should contain a threshold item.") threshold = self.cfg assert ((np.size(threshold) == 1) and (threshold is not None) and (np.isfinite(threshold))) feature = ma.absolute(self.features["rate_of_change"]) flag = np.zeros(np.shape(self.data[self.varname]), dtype="i1") flag[feature > threshold] = self.flag_bad flag[feature <= threshold] = self.flag_good x = np.atleast_1d(self.data[self.varname]) flag[ma.getmaskarray(x) | ~np.isfinite(x)] = 9 self.flags["digit_roll_over"] = flag
def tukey53H(x): """Spike test Tukey 53H from Goring & Nikora 2002 """ N = len(x) u1 = ma.masked_all(N) for n in range(N-4): if x[n:n+5].any(): u1[n+2] = ma.median(x[n:n+5]) u2 = ma.masked_all(N) for n in range(N-2): if u1[n:n+3].any(): u2[n+1] = ma.median(u1[n:n+3]) u3 = ma.masked_all(N) u3[1:-1] = 0.25*(u2[:-2] + 2*u2[1:-1] + u2[2:]) Delta = ma.absolute(x-u3) return Delta
def tukey53H(x): """Spike test Tukey 53H from Goring & Nikora 2002 """ N = len(x) u1 = ma.masked_all(N) for n in range(N - 4): if x[n:n + 5].any(): u1[n + 2] = ma.median(x[n:n + 5]) u2 = ma.masked_all(N) for n in range(N - 2): if u1[n:n + 3].any(): u2[n + 1] = ma.median(u1[n:n + 3]) u3 = ma.masked_all(N) u3[1:-1] = 0.25 * (u2[:-2] + 2 * u2[1:-1] + u2[2:]) Delta = ma.absolute(x - u3) return Delta
def evaluate(self, v, cfg): self.flags[v] = {} # Apply common flag for all points. if 'common' in self.flags: N = self.input[v].shape for f in self.flags['common']: self.flags[v][f] = self.flags['common'][f] * \ np.ones(N, dtype='i1') if self.saveauxiliary: if v not in self.auxiliary.keys(): self.auxiliary[v] = {} if 'platform_identification' in cfg: logging.warn("Sorry I'm not ready to evaluate platform_identification()") if 'valid_geolocation' in cfg: logging.warn("Sorry I'm not ready to evaluate valid_geolocation()") if 'valid_speed' in cfg: # Think about. ARGO also has a test valid_speed, but that is # in respect to sucessive profiles. How is the best way to # distinguish them here? try: if self.saveauxiliary: self.flags[v]['valid_speed'], \ self.auxiliary[v]['valid_speed'] = \ possible_speed(self.input, cfg['valid_speed']) except: print("Fail on valid_speed") if 'global_range' in cfg: self.flags[v]['global_range'] = global_range( self.input, v, cfg['global_range']) if 'regional_range' in cfg: logging.warn("Sorry, I'm no ready to evaluate regional_range()") if 'pressure_increasing' in cfg: logging.warn("Sorry, I'm no ready to evaluate pressure_increasing()") if 'profile_envelop' in cfg: self.flags[v]['profile_envelop'] = profile_envelop( self.input, cfg['profile_envelop'], v) if 'gradient' in cfg: y = Gradient(self.input, v, cfg['gradient']) y.test() if self.saveauxiliary: self.auxiliary[v]['gradient'] = y.features['gradient'] self.flags[v]['gradient'] = y.flags['gradient'] if 'gradient_depthconditional' in cfg: cfg_tmp = cfg['gradient_depthconditional'] g = gradient(self.input[v]) flag = np.zeros(g.shape, dtype='i1') # Flag as 9 any masked input value flag[ma.getmaskarray(self.input[v])] = 9 # ---- Shallow zone ----------------- threshold = cfg_tmp['shallow_max'] flag[np.nonzero( \ (self['PRES'] <= cfg_tmp['pressure_threshold']) & \ (g > threshold))] \ = 4 flag[np.nonzero( \ (self['PRES'] <= cfg_tmp['pressure_threshold']) & \ (g <= threshold))] \ = 1 # ---- Deep zone -------------------- threshold = cfg_tmp['deep_max'] flag[np.nonzero( \ (self['PRES'] > cfg_tmp['pressure_threshold']) & \ (g > threshold))] \ = 4 flag[np.nonzero( \ (self['PRES'] > cfg_tmp['pressure_threshold']) & \ (g <= threshold))] \ = 1 self.flags[v]['gradient_depthconditional'] = flag if 'spike' in cfg: y = Spike(self.input, v, cfg['spike']) y.test() if self.saveauxiliary: self.auxiliary[v]['spike'] = y.features['spike'] self.flags[v]['spike'] = y.flags['spike'] if 'spike_depthconditional' in cfg: cfg_tmp = cfg['spike_depthconditional'] s = spike(self.input[v]) flag = np.zeros(s.shape, dtype='i1') # Flag as 9 any masked input value flag[ma.getmaskarray(self.input[v])] = 9 # ---- Shallow zone ----------------- threshold = cfg_tmp['shallow_max'] flag[np.nonzero( \ (self['PRES'] <= cfg_tmp['pressure_threshold']) & \ (s > threshold))] \ = 4 flag[np.nonzero( \ (self['PRES'] <= cfg_tmp['pressure_threshold']) & \ (s <= threshold))] \ = 1 # ---- Deep zone -------------------- threshold = cfg_tmp['deep_max'] flag[np.nonzero( \ (self['PRES'] > cfg_tmp['pressure_threshold']) & \ (s > threshold))] \ = 4 flag[np.nonzero( \ (self['PRES'] > cfg_tmp['pressure_threshold']) & \ (s <= threshold))] \ = 1 self.flags[v]['spike_depthconditional'] = flag if 'stuck_value' in cfg: logging.warn("Sorry I'm not ready to evaluate stuck_value()") if 'grey_list' in cfg: logging.warn("Sorry I'm not ready to evaluate grey_list()") if 'gross_sensor_drift' in cfg: logging.warn("Sorry I'm not ready to evaluate gross_sensor_drift()") if 'frozen_profile' in cfg: logging.warn("Sorry I'm not ready to evaluate frozen_profile()") if 'deepest_pressure' in cfg: logging.warn("Sorry I'm not ready to evaluate deepest_pressure()") if 'tukey53H_norm' in cfg: y = Tukey53H(self.input, v, cfg['tukey53H_norm']) y.test() if self.saveauxiliary: self.auxiliary[v]['tukey53H_norm'] = \ y.features['tukey53H_norm'] self.flags[v]['tukey53H_norm'] = y.flags['tukey53H_norm'] #if 'spike_depthsmooth' in cfg: # from maud.window_func import _weight_hann as wfunc # cfg_tmp = cfg['spike_depthsmooth'] # cfg_tmp['dzwindow'] = 10 # smooth = ma.masked_all(self.input[v].shape) # z = ped['pressure'] # for i in range(len(self.input[v])): # ind = np.nonzero(ma.absolute(z-z[i]) < \ # cfg_tmp['dzwindow'])[0] # ind = ind[ind != i] # w = wfunc(z[ind]-z[i], cfg_tmp['dzwindow']) # smooth[i] = (T[ind]*w).sum()/w.sum() # ARGO, test #12. (10C, 5PSU) if 'digit_roll_over' in cfg: threshold = cfg['digit_roll_over'] s = step(self.input[v]) if self.saveauxiliary: self.auxiliary[v]['step'] = s flag = np.zeros(s.shape, dtype='i1') # Flag as 9 any masked input value flag[ma.getmaskarray(self.input[v])] = 9 flag[np.nonzero(ma.absolute(s) > threshold)] = 4 flag[np.nonzero(ma.absolute(s) <= threshold)] = 1 self.flags[v]['digit_roll_over'] = flag if 'bin_spike' in cfg: y = Bin_Spike(self.input, v, cfg['bin_spike']) # y.test() if self.saveauxiliary: self.auxiliary[v]['bin_spike'] = y.features['bin_spike'] # self.flags[v]['bin_spike'] = y.flags['bin_spike'] if 'density_inversion' in cfg: try: if self.saveauxiliary: self.flags[v]['density_inversion'], \ self.auxiliary[v]['density_step'] = \ density_inversion( self.input, cfg['density_inversion'], saveaux=True) else: self.flags[v]['density_inversion'] = density_inversion( self.input, cfg['density_inversion']) except: print("Fail on density_inversion") if 'woa_normbias' in cfg: y = WOA_NormBias(self.input, v, cfg['woa_normbias']) # self.attributes) y.test() if self.saveauxiliary: for f in y.features: self.auxiliary[v][f] = y.features[f] self.flags[v]['woa_normbias'] = y.flags['woa_normbias'] #if 'pstep' in cfg: # ind = np.isfinite(self.input[v]) # ind = ma.getmaskarray(self.input[v]) # if self.saveauxiliary: # self.auxiliary[v]['pstep'] = ma.concatenate( # [ma.masked_all(1), # np.diff(self.input['PRES'][ind])]) if 'rate_of_change' in cfg: self.flags[v]['rate_of_change'], RoC = \ rate_of_change(self.input, v, cfg['rate_of_change']) if self.saveauxiliary: self.auxiliary[v]['rate_of_change'] = RoC if 'cum_rate_of_change' in cfg: x = cum_rate_of_change(self.input, v, cfg['cum_rate_of_change']['memory']) self.flags[v]['cum_rate_of_change'] = np.zeros(x.shape, dtype='i1') self.flags[v]['cum_rate_of_change'][ np.nonzero(x <= cfg['cum_rate_of_change']['threshold']) ] = 1 self.flags[v]['cum_rate_of_change'][ np.nonzero(x > cfg['cum_rate_of_change']['threshold']) ] = 4 self.flags[v]['cum_rate_of_change'][ ma.getmaskarray(self.input[v])] = 9 # FIXME: the Anomaly Detection and Fuzzy require some features # to be estimated previously. Generalize this. if 'anomaly_detection' in cfg: features = {} for f in cfg['anomaly_detection']['features']: if f == 'spike': features['spike'] = spike(self.input[v]) elif f == 'gradient': features['gradient'] = gradient(self.input[v]) elif f == 'tukey53H_norm': features['tukey53H_norm'] = tukey53H_norm(self.input[v]) elif f == 'rate_of_change': RoC = ma.masked_all_like(self.input[v]) RoC[1:] = ma.absolute(ma.diff(self.input[v])) features['rate_of_change'] = RoC elif (f == 'woa_normbias'): y = WOA_NormBias(self.input, v, {}) features['woa_normbias'] = \ np.abs(y.features['woa_normbias']) else: logging.error("Sorry, I can't evaluate anomaly_detection with: %s" % f) self.flags[v]['anomaly_detection'] = \ anomaly_detection(features, cfg['anomaly_detection']) if 'morello2014' in cfg: self.flags[v]['morello2014'] = morello2014( features=self.auxiliary[v], cfg=cfg['morello2014']) if 'fuzzylogic' in cfg: features = {} for f in cfg['fuzzylogic']['features']: if f == 'spike': features['spike'] = spike(self.input[v]) elif f == 'gradient': features['gradient'] = gradient(self.input[v]) elif f == 'tukey53H_norm': features['tukey53H_norm'] = tukey53H_norm(self.input[v], k=1.5) elif f == 'rate_of_change': RoC = ma.masked_all_like(data[v]) RoC[1:] = ma.absolute(ma.diff(data[v])) features['rate_of_change'] = RoC elif (f == 'woa_normbias'): y = WOA_NormBias(self.input, v, {}) features['woa_normbias'] = \ np.abs(y.features['woa_normbias']) else: logging.error("Sorry, I can't evaluate fuzzylogic with: %s" % f) self.flags[v]['fuzzylogic'] = fuzzylogic( features=features, cfg=cfg['fuzzylogic']) self.flags[v]['overall'] = combined_flag(self.flags[v])
pylab.plot(x[:100],Y[:100], 'o') pylab.plot(x[:100],y1[:100], 'k--', lw=2) pylab.plot(x[:100],y2[:100], 'k-.', lw=2) pylab.plot(x[:100],y1[:100]+y2[:100], 'k', lw=2) pylab.plot(x[:100],f1[:100], 'r', lw=1) pylab.plot(x[:100],f2[:100], 'g', lw=1) pylab.plot(x[:100],f3[:100], 'b', lw=1) pylab.axhline(y=Y.mean(), c='0.7') pylab.figure() T=1./np.fft.fftfreq(N) ff = np.fft.fft(Y) ff1 = np.fft.fft(f1) ff2 = np.fft.fft(f2) ff3 = np.fft.fft(f3) pylab.plot(T[1:N/2], ma.absolute(ff[1:N/2]), 'k') pylab.plot(T[1:N/2], ma.absolute(ff1[1:N/2]),'r') pylab.plot(T[1:N/2], ma.absolute(ff2[1:N/2]),'g') pylab.plot(T[1:N/2], ma.absolute(ff3[1:N/2]),'b') pylab.show() pylab.show()
def amp(a_single_subRegion, special_parameters): # TODO: this 'special_parameters' maybe be useful amplitude = array(absolute(a_single_subRegion)) return amplitude
def evaluate(self, v, cfg): self.flags[v] = {} if self.saveauxiliary: if v not in self.auxiliary.keys(): self.auxiliary[v] = {} if 'platform_identification' in cfg: logging.warn( "Sorry I'm not ready to evaluate platform_identification()") if 'valid_geolocation' in cfg: logging.warn("Sorry I'm not ready to evaluate valid_geolocation()") if 'valid_speed' in cfg: # Think about. ARGO also has a test valid_speed, but that is # in respect to sucessive profiles. How is the best way to # distinguish them here? try: if self.saveauxiliary: self.flags[v]['valid_speed'], \ self.auxiliary[v]['valid_speed'] = \ possible_speed(self.input, cfg['valid_speed']) except: print("Fail on valid_speed") if 'global_range' in cfg: self.flags[v]['global_range'] = np.zeros(self.input[v].shape, dtype='i1') # Flag as 9 any masked input value self.flags[v]['global_range'][ma.getmaskarray(self.input[v])] = 9 ind = (self.input[v] >= cfg['global_range']['minval']) & \ (self.input[v] <= cfg['global_range']['maxval']) self.flags[v]['global_range'][np.nonzero(ind)] = 1 ind = (self.input[v] < cfg['global_range']['minval']) | \ (self.input[v] > cfg['global_range']['maxval']) self.flags[v]['global_range'][np.nonzero(ind)] = 4 if 'regional_range' in cfg: logging.warn("Sorry, I'm no ready to evaluate regional_range()") if 'pressure_increasing' in cfg: logging.warn( "Sorry, I'm no ready to evaluate pressure_increasing()") if 'profile_envelop' in cfg: self.flags[v]['profile_envelop'] = profile_envelop( self.input, cfg['profile_envelop'], v) if 'gradient' in cfg: threshold = cfg['gradient'] g = gradient(self.input[v]) if self.saveauxiliary: self.auxiliary[v]['gradient'] = g flag = np.zeros(g.shape, dtype='i1') flag[ma.getmaskarray(self.input[v])] = 9 flag[np.nonzero(g > threshold)] = 4 flag[np.nonzero(g <= threshold)] = 1 self.flags[v]['gradient'] = flag if 'gradient_depthconditional' in cfg: cfg_tmp = cfg['gradient_depthconditional'] g = gradient(self.input[v]) flag = np.zeros(g.shape, dtype='i1') # Flag as 9 any masked input value flag[ma.getmaskarray(self.input[v])] = 9 # ---- Shallow zone ----------------- threshold = cfg_tmp['shallow_max'] flag[np.nonzero( \ (self['PRES'] <= cfg_tmp['pressure_threshold']) & \ (g > threshold))] \ = 4 flag[np.nonzero( \ (self['PRES'] <= cfg_tmp['pressure_threshold']) & \ (g <= threshold))] \ = 1 # ---- Deep zone -------------------- threshold = cfg_tmp['deep_max'] flag[np.nonzero( \ (self['PRES'] > cfg_tmp['pressure_threshold']) & \ (g > threshold))] \ = 4 flag[np.nonzero( \ (self['PRES'] > cfg_tmp['pressure_threshold']) & \ (g <= threshold))] \ = 1 self.flags[v]['gradient_depthconditional'] = flag if 'spike' in cfg: threshold = cfg['spike'] s = spike(self.input[v]) if self.saveauxiliary: self.auxiliary[v]['spike'] = s flag = np.zeros(s.shape, dtype='i1') # Flag as 9 any masked input value flag[ma.getmaskarray(self.input[v])] = 9 flag[np.nonzero(s > threshold)] = 4 flag[np.nonzero(s <= threshold)] = 1 self.flags[v]['spike'] = flag if 'spike_depthconditional' in cfg: cfg_tmp = cfg['spike_depthconditional'] s = spike(self.input[v]) flag = np.zeros(s.shape, dtype='i1') # Flag as 9 any masked input value flag[ma.getmaskarray(self.input[v])] = 9 # ---- Shallow zone ----------------- threshold = cfg_tmp['shallow_max'] flag[np.nonzero( \ (self['PRES'] <= cfg_tmp['pressure_threshold']) & \ (g > threshold))] \ = 4 flag[np.nonzero( \ (self['PRES'] <= cfg_tmp['pressure_threshold']) & \ (g <= threshold))] \ = 1 # ---- Deep zone -------------------- threshold = cfg_tmp['deep_max'] flag[np.nonzero( \ (self['PRES'] > cfg_tmp['pressure_threshold']) & \ (g > threshold))] \ = 4 flag[np.nonzero( \ (self['PRES'] > cfg_tmp['pressure_threshold']) & \ (g <= threshold))] \ = 1 self.flags[v]['spike_depthconditional'] = flag if 'stuck_value' in cfg: logging.warn("Sorry I'm not ready to evaluate stuck_value()") if 'grey_list' in cfg: logging.warn("Sorry I'm not ready to evaluate grey_list()") if 'gross_sensor_drift' in cfg: logging.warn( "Sorry I'm not ready to evaluate gross_sensor_drift()") if 'frozen_profile' in cfg: logging.warn("Sorry I'm not ready to evaluate frozen_profile()") if 'deepest_pressure' in cfg: logging.warn("Sorry I'm not ready to evaluate deepest_pressure()") if 'tukey53H_norm' in cfg: """ I slightly modified the Goring & Nikora 2002. It is expected that CTD profiles has a typical depth structure, with a range between surface and bottom. """ s = tukey53H_norm(self.input[v], k=cfg['tukey53H_norm']['k'], l=cfg['tukey53H_norm']['l']) if self.saveauxiliary: self.auxiliary[v]['tukey53H_norm'] = s threshold = cfg['tukey53H_norm']['k'] flag = np.zeros(s.shape, dtype='i1') # Flag as 9 any masked input value flag[ma.getmaskarray(self.input[v])] = 9 flag[np.nonzero(s > threshold)] = 4 flag[np.nonzero(s <= threshold)] = 1 self.flags[v]['tukey53H_norm'] = flag #if 'spike_depthsmooth' in cfg: # from maud.window_func import _weight_hann as wfunc # cfg_tmp = cfg['spike_depthsmooth'] # cfg_tmp['dzwindow'] = 10 # smooth = ma.masked_all(self.input[v].shape) # z = ped['pressure'] # for i in range(len(self.input[v])): # ind = np.nonzero(ma.absolute(z-z[i]) < \ # cfg_tmp['dzwindow'])[0] # ind = ind[ind != i] # w = wfunc(z[ind]-z[i], cfg_tmp['dzwindow']) # smooth[i] = (T[ind]*w).sum()/w.sum() # ARGO, test #12. (10C, 5PSU) if 'digit_roll_over' in cfg: threshold = cfg['digit_roll_over'] s = step(self.input[v]) if self.saveauxiliary: self.auxiliary[v]['step'] = s flag = np.zeros(s.shape, dtype='i1') # Flag as 9 any masked input value flag[ma.getmaskarray(self.input[v])] = 9 flag[np.nonzero(ma.absolute(s) > threshold)] = 4 flag[np.nonzero(ma.absolute(s) <= threshold)] = 1 self.flags[v]['digit_roll_over'] = flag if 'bin_spike' in cfg: bin = bin_spike(self.input[v], cfg['bin_spike']) if self.saveauxiliary: self.auxiliary[v]['bin_spike'] = bin if 'density_inversion' in cfg: try: if self.saveauxiliary: self.flags[v]['density_inversion'], \ self.auxiliary[v]['density_step'] = \ density_inversion( self.input, cfg['density_inversion'], saveaux=True) else: self.flags[v]['density_inversion'] = density_inversion( self.input, cfg['density_inversion']) except: print("Fail on density_inversion") if 'woa_normbias' in cfg: if self.saveauxiliary: self.flags[v]['woa_normbias'], \ self.auxiliary[v]['woa_relbias'] = \ woa_normbias(self.input, v, cfg['woa_normbias']) #for k in woa.keys(): # self.auxiliary[v][k] = woa[k] #self.auxiliary[v]['woa_bias'] = woa_bias #self.auxiliary[v]['woa_relbias'] = woa_bias/woa['woa_sd'] else: self.flags[v]['woa_normbias'], \ tmp = \ woa_normbias(self.input, v, cfg['woa_normbias']) del (tmp) #if 'pstep' in cfg: # ind = np.isfinite(self.input[v]) # ind = ma.getmaskarray(self.input[v]) # if self.saveauxiliary: # self.auxiliary[v]['pstep'] = ma.concatenate( # [ma.masked_all(1), # np.diff(self.input['PRES'][ind])]) if 'RoC' in cfg: x = ma.concatenate([ma.masked_all(1), ma.diff(self.input[v])]) if self.saveauxiliary: self.auxiliary[v]['RoC'] = x self.flags[v]['RoC'] = np.zeros(x.shape, dtype='i1') self.flags[v]['RoC'][np.nonzero(x <= cfg['RoC'])] = 1 self.flags[v]['RoC'][np.nonzero(x > cfg['RoC'])] = 4 self.flags[v]['RoC'][ma.getmaskarray(self.input[v])] = 9 if 'anomaly_detection' in cfg: features = {} for f in cfg['anomaly_detection']['features']: if f == 'spike': features['spike'] = spike(self.input[v]) elif f == 'gradient': features['gradient'] = gradient(self.input[v]) elif f == 'tukey53H_norm': features['tukey53H_norm'] = tukey53H_norm(self.input[v], k=1.5) else: logging.error( "Sorry, I can't evaluate anomaly_detection with: %s" % f) self.flags[v]['anomaly_detection'] = \ anomaly_detection(features, cfg['anomaly_detection']) if 'fuzzylogic' in cfg: self.flags[v]['fuzzylogic'] = fuzzylogic(self.auxiliary[v], v, cfg['fuzzylogic'])
def evaluate(self, v, cfg): self.flags[v] = {} if self.saveauxiliary: if v not in self.auxiliary.keys(): self.auxiliary[v] = {} if 'platform_identification' in cfg: logging.warn("Sorry I'm not ready to evaluate platform_identification()") if 'valid_geolocation' in cfg: logging.warn("Sorry I'm not ready to evaluate valid_geolocation()") if 'valid_speed' in cfg: # Think about. ARGO also has a test valid_speed, but that is # in respect to sucessive profiles. How is the best way to # distinguish them here? try: if self.saveauxiliary: self.flags[v]['valid_speed'], \ self.auxiliary[v]['valid_speed'] = \ possible_speed(self.input, cfg['valid_speed']) except: print("Fail on valid_speed") if 'global_range' in cfg: self.flags[v]['global_range'] = np.zeros(self.input[v].shape, dtype='i1') # Flag as 9 any masked input value self.flags[v]['global_range'][ma.getmaskarray(self.input[v])] = 9 ind = (self.input[v] >= cfg['global_range']['minval']) & \ (self.input[v] <= cfg['global_range']['maxval']) self.flags[v]['global_range'][np.nonzero(ind)] = 1 ind = (self.input[v] < cfg['global_range']['minval']) | \ (self.input[v] > cfg['global_range']['maxval']) self.flags[v]['global_range'][np.nonzero(ind)] = 4 if 'regional_range' in cfg: logging.warn("Sorry, I'm no ready to evaluate regional_range()") if 'pressure_increasing' in cfg: logging.warn("Sorry, I'm no ready to evaluate pressure_increasing()") if 'profile_envelop' in cfg: self.flags[v]['profile_envelop'] = profile_envelop( self.input, cfg['profile_envelop'], v) if 'gradient' in cfg: threshold = cfg['gradient'] g = gradient(self.input[v]) if self.saveauxiliary: self.auxiliary[v]['gradient'] = g flag = np.zeros(g.shape, dtype='i1') flag[ma.getmaskarray(self.input[v])] = 9 flag[np.nonzero(g > threshold)] = 4 flag[np.nonzero(g <= threshold)] = 1 self.flags[v]['gradient'] = flag if 'gradient_depthconditional' in cfg: cfg_tmp = cfg['gradient_depthconditional'] g = gradient(self.input[v]) flag = np.zeros(g.shape, dtype='i1') # Flag as 9 any masked input value flag[ma.getmaskarray(self.input[v])] = 9 # ---- Shallow zone ----------------- threshold = cfg_tmp['shallow_max'] flag[np.nonzero( \ (self['PRES'] <= cfg_tmp['pressure_threshold']) & \ (g > threshold))] \ = 4 flag[np.nonzero( \ (self['PRES'] <= cfg_tmp['pressure_threshold']) & \ (g <= threshold))] \ = 1 # ---- Deep zone -------------------- threshold = cfg_tmp['deep_max'] flag[np.nonzero( \ (self['PRES'] > cfg_tmp['pressure_threshold']) & \ (g > threshold))] \ = 4 flag[np.nonzero( \ (self['PRES'] > cfg_tmp['pressure_threshold']) & \ (g <= threshold))] \ = 1 self.flags[v]['gradient_depthconditional'] = flag if 'spike' in cfg: threshold = cfg['spike'] s = spike(self.input[v]) if self.saveauxiliary: self.auxiliary[v]['spike'] = s flag = np.zeros(s.shape, dtype='i1') # Flag as 9 any masked input value flag[ma.getmaskarray(self.input[v])] = 9 flag[np.nonzero(s > threshold)] = 4 flag[np.nonzero(s <= threshold)] = 1 self.flags[v]['spike'] = flag if 'spike_depthconditional' in cfg: cfg_tmp = cfg['spike_depthconditional'] s = spike(self.input[v]) flag = np.zeros(s.shape, dtype='i1') # Flag as 9 any masked input value flag[ma.getmaskarray(self.input[v])] = 9 # ---- Shallow zone ----------------- threshold = cfg_tmp['shallow_max'] flag[np.nonzero( \ (self['PRES'] <= cfg_tmp['pressure_threshold']) & \ (g > threshold))] \ = 4 flag[np.nonzero( \ (self['PRES'] <= cfg_tmp['pressure_threshold']) & \ (g <= threshold))] \ = 1 # ---- Deep zone -------------------- threshold = cfg_tmp['deep_max'] flag[np.nonzero( \ (self['PRES'] > cfg_tmp['pressure_threshold']) & \ (g > threshold))] \ = 4 flag[np.nonzero( \ (self['PRES'] > cfg_tmp['pressure_threshold']) & \ (g <= threshold))] \ = 1 self.flags[v]['spike_depthconditional'] = flag if 'stuck_value' in cfg: logging.warn("Sorry I'm not ready to evaluate stuck_value()") if 'grey_list' in cfg: logging.warn("Sorry I'm not ready to evaluate grey_list()") if 'gross_sensor_drift' in cfg: logging.warn("Sorry I'm not ready to evaluate gross_sensor_drift()") if 'frozen_profile' in cfg: logging.warn("Sorry I'm not ready to evaluate frozen_profile()") if 'deepest_pressure' in cfg: logging.warn("Sorry I'm not ready to evaluate deepest_pressure()") if 'tukey53H_norm' in cfg: """ I slightly modified the Goring & Nikora 2002. It is expected that CTD profiles has a typical depth structure, with a range between surface and bottom. """ s = tukey53H_norm(self.input[v], k = cfg['tukey53H_norm']['k'], l = cfg['tukey53H_norm']['l']) if self.saveauxiliary: self.auxiliary[v]['tukey53H_norm'] = s threshold = cfg['tukey53H_norm']['k'] flag = np.zeros(s.shape, dtype='i1') # Flag as 9 any masked input value flag[ma.getmaskarray(self.input[v])] = 9 flag[np.nonzero(s > threshold)] = 4 flag[np.nonzero(s <= threshold)] = 1 self.flags[v]['tukey53H_norm'] = flag #if 'spike_depthsmooth' in cfg: # from maud.window_func import _weight_hann as wfunc # cfg_tmp = cfg['spike_depthsmooth'] # cfg_tmp['dzwindow'] = 10 # smooth = ma.masked_all(self.input[v].shape) # z = ped['pressure'] # for i in range(len(self.input[v])): # ind = np.nonzero(ma.absolute(z-z[i]) < \ # cfg_tmp['dzwindow'])[0] # ind = ind[ind != i] # w = wfunc(z[ind]-z[i], cfg_tmp['dzwindow']) # smooth[i] = (T[ind]*w).sum()/w.sum() # ARGO, test #12. (10C, 5PSU) if 'digit_roll_over' in cfg: threshold = cfg['digit_roll_over'] s = step(self.input[v]) if self.saveauxiliary: self.auxiliary[v]['step'] = s flag = np.zeros(s.shape, dtype='i1') # Flag as 9 any masked input value flag[ma.getmaskarray(self.input[v])] = 9 flag[np.nonzero(ma.absolute(s) > threshold)] = 4 flag[np.nonzero(ma.absolute(s) <= threshold)] = 1 self.flags[v]['digit_roll_over'] = flag if 'bin_spike' in cfg: bin = bin_spike(self.input[v], cfg['bin_spike']) if self.saveauxiliary: self.auxiliary[v]['bin_spike'] = bin if 'density_inversion' in cfg: try: if self.saveauxiliary: self.flags[v]['density_inversion'], \ self.auxiliary[v]['density_step'] = \ density_inversion( self.input, cfg['density_inversion'], saveaux=True) else: self.flags[v]['density_inversion'] = density_inversion( self.input, cfg['density_inversion']) except: print("Fail on density_inversion") if 'woa_normbias' in cfg: if self.saveauxiliary: self.flags[v]['woa_normbias'], \ self.auxiliary[v]['woa_relbias'] = \ woa_normbias(self.input, v, cfg['woa_normbias']) #for k in woa.keys(): # self.auxiliary[v][k] = woa[k] #self.auxiliary[v]['woa_bias'] = woa_bias #self.auxiliary[v]['woa_relbias'] = woa_bias/woa['woa_sd'] else: self.flags[v]['woa_normbias'], \ tmp = \ woa_normbias(self.input, v, cfg['woa_normbias']) del(tmp) #if 'pstep' in cfg: # ind = np.isfinite(self.input[v]) # ind = ma.getmaskarray(self.input[v]) # if self.saveauxiliary: # self.auxiliary[v]['pstep'] = ma.concatenate( # [ma.masked_all(1), # np.diff(self.input['PRES'][ind])]) if 'RoC' in cfg: x = ma.concatenate([ma.masked_all(1), ma.diff(self.input[v])]) if self.saveauxiliary: self.auxiliary[v]['RoC'] = x self.flags[v]['RoC'] = np.zeros(x.shape, dtype='i1') self.flags[v]['RoC'][np.nonzero(x <= cfg['RoC'])] = 1 self.flags[v]['RoC'][np.nonzero(x > cfg['RoC'])] = 4 self.flags[v]['RoC'][ma.getmaskarray(self.input[v])] = 9 if 'anomaly_detection' in cfg: features = {} for f in cfg['anomaly_detection']['features']: if f == 'spike': features['spike'] = spike(self.input[v]) elif f == 'gradient': features['gradient'] = gradient(self.input[v]) elif f == 'tukey53H_norm': features['tukey53H_norm'] = tukey53H_norm(self.input[v], k=1.5) else: logging.error("Sorry, I can't evaluate anomaly_detection with: %s" % f) self.flags[v]['anomaly_detection'] = \ anomaly_detection(features, cfg['anomaly_detection']) if 'fuzzylogic' in cfg: self.flags[v]['fuzzylogic'] = fuzzylogic( self.auxiliary[v], v, cfg['fuzzylogic'])
def log_linear_vinterp(T, P, levs): ''' # Author Charles Doutriaux # Version 1.1 # Expect 2D field here so there''s no reorder which I suspect to do a memory leak # email: [email protected] # Converts a field from sigma levels to pressure levels # Log linear interpolation # Input # T : temperature on sigma levels # P : pressure field from TOP (level 0) to BOTTOM (last level) # levs : pressure levels to interplate to (same units as P) # Output # t : temperature on pressure levels (levs) # External: Numeric''' import numpy.ma as MA ## from numpy.oldnumeric.ma import ones,Float,greater,less,logical_and,where,equal,log,asarray,Float16 sh = P.shape nsigma = sh[0] # Number of sigma levels try: nlev = len(levs) # Number of pressure levels except: nlev = 1 # if only one level len(levs) would breaks t = [] for ilv in range(nlev): # loop through pressure levels try: lev = levs[ilv] # get value for the level except: lev = levs # only 1 level passed # print ' ......... level:',lev Pabv = MA.ones(P[0].shape, Numeric.Float) Tabv = -Pabv # Temperature on sigma level Above Tbel = -Pabv # Temperature on sigma level Below Pbel = -Pabv # Pressure on sigma level Below Pabv = -Pabv # Pressure on sigma level Above for isg in range(1, nsigma): # loop from second sigma level to last one ## print 'Sigma level #',isg a = MA.greater(P[isg], lev) # Where is the pressure greater than lev b = MA.less(P[isg - 1], lev) # Where is the pressure less than lev # Now looks if the pressure level is in between the 2 sigma levels # If yes, sets Pabv, Pbel and Tabv, Tbel Pabv = MA.where(MA.logical_and(a, b), P[isg], Pabv) # Pressure on sigma level Above Tabv = MA.where(MA.logical_and(a, b), T[isg], Tabv) # Temperature on sigma level Above Pbel = MA.where(MA.logical_and(a, b), P[isg - 1], Pbel) # Pressure on sigma level Below Tbel = MA.where(MA.logical_and(a, b), T[isg - 1], Tbel) # Temperature on sigma level Below # end of for isg in range(1,nsigma) # val=where(equal(Pbel,-1.),Pbel.missing_value,lev) # set to missing value if no data below lev if there is tl = MA.masked_where( MA.equal(Pbel, -1.), MA.log(lev / MA.absolute(Pbel)) / MA.log(Pabv / Pbel) * (Tabv - Tbel) + Tbel) # Interpolation t.append(tl) # add a level to the output # end of for ilv in range(nlev) return asMA(t).astype(Numeric.Float32) # convert t to an array