def test_detrend_linear(self): # 0D. assert mlab.detrend_linear(0.) == 0. assert mlab.detrend_linear(5.5) == 0. assert mlab.detrend(5.5, key="linear") == 0. assert mlab.detrend(5.5, key=mlab.detrend_linear) == 0. for sig in [ # 1D. self.sig_off, self.sig_slope, self.sig_slope + self.sig_off, ]: self.allclose(mlab.detrend_linear(sig), self.sig_zeros)
def test_detrend_linear_2d(self): input = np.vstack( [self.sig_off, self.sig_slope, self.sig_slope + self.sig_off]) target = np.vstack([self.sig_zeros, self.sig_zeros, self.sig_zeros]) self.allclose(mlab.detrend(input.T, key="linear", axis=0), target.T) self.allclose(mlab.detrend(input.T, key=mlab.detrend_linear, axis=0), target.T) self.allclose(mlab.detrend(input, key="linear", axis=1), target) self.allclose(mlab.detrend(input, key=mlab.detrend_linear, axis=1), target) with pytest.raises(ValueError): mlab.detrend_linear(self.sig_slope[np.newaxis])
def _dominant_freq(arr): ma = np.mean(arr,1) if np.max(ma) < 1e-7: print "mean wavelet power %e too low"%np.mean(ma) return np.nan i = np.argmax(mlab.detrend_linear(ma)) return freqs[i]
def LombScargle(self): '''find significance of highest peak in COS event list This routine bins an event list in 0.032s indervals (the clock time) and runs a Lomb-Scargle periodogram. It return the false alarm probability (FAP) of the highest peak. ''' # LombScargle expects even number of entries start = self.hist.shape[0] % 2 lc = pyPeriod.TimeSeries(self.bins[start + 1:], mlab.detrend_linear(self.hist[start:])) self.ls = pyPeriod.LombScargle(lc, ofac=1, hifac=1) # ignore the lowest freq (= the length of the dataset) return self.ls.FAP(max(self.ls.power[3:])), max(self.ls.power[3:])
def timeseries_boundary(x, opt_b, bdetrend): """ TIMESERIES_BOUNDARY applies periodic, zero-padded, or mirror boundary conditions to a time series. """ M = len(x) if bool(M % 2): raise AssertionError("Even number of samples") if bdetrend: x = mlab.detrend_linear(x) # Allocate space for solution if opt_b == "zer": y = np.hstack([np.zeros(M / 2), x, np.zeros(M / 2)]) if opt_b == "con": y = np.hstack([x[0] * np.ones(M / 2), x, x[-1] * np.ones(M / 2)]) elif opt_b == "mir": y = np.hstack([x[::-1][M / 2:], x, x[::-1][:M / 2]]) elif opt_b == "per": y = x elif opt_b == "exp_sin": # Attempt to fit an exponential periodic function to each end of # the signal, such that the total length is 2*M # Declare a temporary variable t, all my functions operate on an # x and y pair rather than dimensionless frequency t = np.linspace(0, 100, len(x)) period = estimate_period(t, x) # Get last two periods of x and y ind_end = np.abs(t - (t[-1] - 2 * period)).argmin() ind_start = np.abs(t - (2 * period)).argmin() t_end = t[ind_end:] x_end = x[ind_end:] t_start = t[:ind_start] x_start = x[:ind_start] x_mid = x[ind_start:ind_end] t_end_ext, x_end_ext = extend(t_end, x_end, M / 2) t_start_ext, x_start_ext = extend(t_start, x_start[::-1], M / 2) y = np.hstack([x_start_ext[::-1], x_mid, x_end_ext]) # tt = np.linspace(t[0] - t.mean(), t[-1] + t.mean(), num=len(yt)) if opt_b is not 'per': index = np.arange(M / 2, M + M / 2) else: index = np.arange(M) return y, index
def update(self): pb = pbar.ProgressBar() winlen_samples = np.int(self.win_len_/self.spacing_) half_winlen = np.int(winlen_samples / 2) print('each window have size: ' + str(half_winlen * 2)) ffts = [] ns = [] for n in pb(np.arange(0 + half_winlen, len(self.x_) - half_winlen)[::self.sub_fact_]): ns.append(n) #get a piece of the signal small_signal = self.y_[n-half_winlen:n+half_winlen] #perform detrending if self.detrend_method_ == 'linear': small_signal = mlab.detrend_linear(small_signal) elif self.detrend_method_ == 'mean': small_signal = small_signal - np.mean(small_signal) elif self.detrend_method_ == 'none': #simply go ahead pass #do padding if self.N_zeros_ > len(small_signal): small_signal = np.concatenate((small_signal, np.zeros(self.N_zeros_-len(small_signal)))) #get a spectral estimation using some method if self.method_ == 'mtspec': ps, f = mtspec.mtspec(small_signal, self.spacing_, self.pi_) #normalize the single spectrum if self.max_normalization_: ps = ps / np.max(ps) #append to the ffts ffts.append(ps[f<=self.highfreq_]) self.ffts_ = np.array(ffts) self.eval_pos_ = self.x_[ns] #the real max frequency? self.real_maxfreq_ = np.max(f[f<=self.highfreq_])
def timeseries_boundary_old(x, opt_b, bdetrend): """ TIMESERIES_BOUNDARY applies periodic, zero-padded, or mirror boundary conditions to a time series. """ M = len(x) if bdetrend: x = mlab.detrend_linear(x) # Allocate space for solution if opt_b == "zer": y = np.hstack([np.zeros(M), x, np.zeros(M)]) elif opt_b == "mir": y = np.hstack([x[::-1], x, x[::-1]]) elif opt_b == "per": y = x if opt_b is not 'per': index = np.arange(M, 2 * M) else: index = np.arange(M) return y, index
def plot_periodogram(hist, bins): def lspow(sig): return scipy.optimize.fmin(lambda x : np.abs(ls.FAP(x)-sig),[10.], disp = False) # LombScargle expects even number of entries start = hist.shape[0]%2 lc = pyPeriod.TimeSeries(bins[start+1:], mlab.detrend_linear(hist[start:])) ls = pyPeriod.LombScargle(lc, ofac=1, hifac=1) #fig = plt.figure() #ax = fig.add_subplot(111) #ax.plot(ls.freq, ls.power, lw = 2.) #ax.plot(ax.get_xlim(), lspow(0.1) * np.array(1., 1., dtype = np.float), 'k:' ) #ax.plot(ax.get_xlim(), lspow(0.01) * np.array(1., 1., dtype = np.float), 'k:') fig, ax = ls.plot(lw = 2., FAPlevels = [.1, 0.01]) #ax.set_ylabel('Scargle Power') ax.set_xlabel('Frequency [Hz]') pd.plotfile(fig, 'LSperiodogram') ax.set_yscale('log') ax.set_xscale('log') pd.plotfile(fig, 'LSloglog') return fig, ax
def test_detrend_str_linear_1d(self): input = self.sig_slope + self.sig_off target = self.sig_zeros self.allclose(mlab.detrend(input, key="linear"), target) self.allclose(mlab.detrend(input, key=mlab.detrend_linear), target) self.allclose(mlab.detrend_linear(input.tolist()), target)
def calculate_season(self): """ calculates the season """ seasons_params = {} seasons_params['DJF'] = (3,2) seasons_params['JFM'] = (3,3) seasons_params['FMA'] = (3,4) seasons_params['MAM'] = (3,5) seasons_params['AMJ'] = (3,6) seasons_params['MJJ'] = (3,7) seasons_params['JJA'] = (3,8) seasons_params['JAS'] = (3,9) seasons_params['ASO'] = (3,10) seasons_params['SON'] = (3,11) seasons_params['OND'] = (3,12) seasons_params['NDJ'] = (3,1) seasons_params['Warm Season (Dec. - May)'] = (6, 5) seasons_params['Cold Season (Jun. - Nov.)'] = (6, 11) seasons_params['Year (Jan. - Dec.)'] = (12, 12) seasons_params['Hydro. year (Jul. - Jun.)'] = (12, 6) self.seasons_params = seasons_params if not(hasattr(self, 'dset_dict')): self._read_dset_params() # get the name of the file to open fname = self.dset_dict['path'] # `dset` is now an attribute of the ensemble object self.dset = xray.open_dataset(fname) # get the variable and its index m_var = self.dset[self.variable].data index = self.dset['time'].to_index() # if the variable is rainfall, we calculate the running SUM if self.dset_dict['units'] in ['mm']: seas_field = bn.move_sum(m_var, self.seasons_params[self.season][0], \ min_count=self.seasons_params[self.season][0], axis=0) # if not, then we calculate the running MEAN (average) else: seas_field = bn.move_mean(m_var, self.seasons_params[self.season][0], \ min_count=self.seasons_params[self.season][0], axis=0) # get rid of the first nans in the time-series / fields after move_mean or move_sum seas_field = seas_field[(self.seasons_params[self.season][0]-1)::,:,:] index = index[(self.seasons_params[self.season][0]-1)::] # now selects the SEASON of interest iseas = np.where(index.month == self.seasons_params[self.season][1])[0] dates = index[iseas] seas_field = np.take(seas_field, iseas, axis=0) # if detrend is set to `True`, we detrend # detrend_linear from matplotlib.mlab is faster than detrend from scipy.signal if self.detrend: dseas_field = np.ones(seas_field.shape) * np.nan # if there is a mask, we have to test each variable if 'mask' in self.dset.data_vars: for ilat in range(dseas_field.shape[1]): for ilon in range(dseas_field.shape[2]): if np.logical_not(np.all(np.isnan(seas_field[:,ilat, ilon]))): dseas_field[:,ilat, ilon] = detrend_linear(seas_field[:,ilat,ilon]) \ + seas_field[:,ilat,ilon].mean() # if not, we can proceed over the whole dataset else: for ilat in range(dseas_field.shape[1]): for ilon in range(dseas_field.shape[2]): dseas_field[:,ilat, ilon] = detrend_linear(seas_field[:,ilat,ilon]) \ + seas_field[:,ilat,ilon].mean() self.dset['dates'] = (('dates',), dates) self.dset['seas_var'] = (('dates', 'latitudes', 'longitudes'), dseas_field) # if detrend is False, then just add the seaosnal values else: self.dset['dates'] = (('dates',), dates) self.dset['seas_var'] = (('dates', 'latitudes', 'longitudes'), seas_field)
a = np.where( (x.time[:,0] >= 1972) )[0][0] z = np.where( (x.time[:,0] <= 2010) )[0][-1] + 1 x.time = x.time[a:z,...] x.data = x.data[a:z,...] from matplotlib.mlab import detrend_linear ### ============================================================================================================== ### detrend the data itself ? datad = np.empty(x.data.shape) for i in xrange(x.data.shape[1]): datad[:,i] = detrend_linear(x.data[:,i]) x.data = datad clus_eof_file = loadmat(os.path.join(dpath, "KidsonTypes", "clus_eof.mat"), struct_as_record=False) clus_eof = clus_eof_file['clus_eof'][0,0] ### normalize za = x.data - np.tile(clus_eof.mean.T, (x.data.shape[0],1)) ### multiply by the EOFs to get the Principal components pc = np.dot(za,clus_eof.vect) pc_mean = clus_eof_file['pc_mean'] ### normalize by the mean of the original PCs
def calculate_season(self): """ calculates the season """ self.seasons_params = seasons_params() if not (hasattr(self, 'dset_dict')): self._read_dset_params() # get the name of the file to open fname = self.dset_dict['path'] # `dset` is now an attribute of the ensemble object self.dset = xray.open_dataset(fname) # get the variable and its index m_var = self.dset[self.variable].data index = self.dset['time'].to_index() # if the variable is rainfall, we calculate the running SUM if self.dset_dict['units'] in ['mm']: seas_field = bn.move_sum(m_var, self.seasons_params[self.season][0], \ min_count=self.seasons_params[self.season][0], axis=0) # if not, then we calculate the running MEAN (average) else: seas_field = bn.move_mean(m_var, self.seasons_params[self.season][0], \ min_count=self.seasons_params[self.season][0], axis=0) # get rid of the first nans in the time-series / fields after move_mean or move_sum seas_field = seas_field[(self.seasons_params[self.season][0] - 1)::, :, :] index = index[(self.seasons_params[self.season][0] - 1)::] # now selects the SEASON of interest iseas = np.where(index.month == self.seasons_params[self.season][1])[0] dates = index[iseas] seas_field = np.take(seas_field, iseas, axis=0) # if detrend is set to `True`, we detrend # detrend_linear from matplotlib.mlab is faster than detrend from scipy.signal if self.detrend: dseas_field = np.ones(seas_field.shape) * np.nan # if there is a mask, we have to test each variable if 'mask' in self.dset.data_vars: for ilat in range(dseas_field.shape[1]): for ilon in range(dseas_field.shape[2]): if np.logical_not( np.all(np.isnan(seas_field[:, ilat, ilon]))): dseas_field[:,ilat, ilon] = detrend_linear(seas_field[:,ilat,ilon]) \ + seas_field[:,ilat,ilon].mean() # if not, we can proceed over the whole dataset else: for ilat in range(dseas_field.shape[1]): for ilon in range(dseas_field.shape[2]): dseas_field[:,ilat, ilon] = detrend_linear(seas_field[:,ilat,ilon]) \ + seas_field[:,ilat,ilon].mean() self.dset['dates'] = (('dates', ), dates) self.dset['seas_var'] = (('dates', 'latitudes', 'longitudes'), dseas_field) # if detrend is False, then just add the seaosnal values else: self.dset['dates'] = (('dates', ), dates) self.dset['seas_var'] = (('dates', 'latitudes', 'longitudes'), seas_field)
# ================================================================ # Read AMO index # ================================================================ year_st = 1948 year_ed = 2013 year_ref = 1856 t0 = (year_st - year_ref) t1 = (year_ed - year_ref) + 1 dirname = '/stormtrack/data4/yliang/Indices/' filename = 'amon.us.long.data' f = open(dirname + filename, "r") data_read = genfromtxt(dirname + filename, dtype='float', skip_header=0)[t0:t1, 1:] f.close() amo_index = np.nanmean(data_read, axis=1) amo_index = mlab.detrend_linear(amo_index) N = 11 amo_rmean = amo_index.copy() * np.nan amo_rmean[int((N - 1) / 2):-int((N - 1) / 2)] = np.convolve(amo_index, np.ones( (N, )) / N, mode='valid') # ================================================================ # Read PDO index # ================================================================ year_st = 1948 year_ed = 2013 year_ref = 1900 t0 = (year_st - year_ref) t1 = (year_ed - year_ref) + 1
def detrend_func(data): data = mlab.detrend_mean(data) data = mlab.detrend_linear(data) return data