Exemplo n.º 1
0
 def test_detrend_linear(self):
     # 0D.
     assert mlab.detrend_linear(0.) == 0.
     assert mlab.detrend_linear(5.5) == 0.
     assert mlab.detrend(5.5, key="linear") == 0.
     assert mlab.detrend(5.5, key=mlab.detrend_linear) == 0.
     for sig in [  # 1D.
             self.sig_off,
             self.sig_slope,
             self.sig_slope + self.sig_off,
     ]:
         self.allclose(mlab.detrend_linear(sig), self.sig_zeros)
Exemplo n.º 2
0
    def test_detrend_linear_2d(self):
        input = np.vstack(
            [self.sig_off, self.sig_slope, self.sig_slope + self.sig_off])
        target = np.vstack([self.sig_zeros, self.sig_zeros, self.sig_zeros])
        self.allclose(mlab.detrend(input.T, key="linear", axis=0), target.T)
        self.allclose(mlab.detrend(input.T, key=mlab.detrend_linear, axis=0),
                      target.T)
        self.allclose(mlab.detrend(input, key="linear", axis=1), target)
        self.allclose(mlab.detrend(input, key=mlab.detrend_linear, axis=1),
                      target)

        with pytest.raises(ValueError):
            mlab.detrend_linear(self.sig_slope[np.newaxis])
Exemplo n.º 3
0
    def _dominant_freq(arr):
        ma = np.mean(arr,1) 
        if np.max(ma) < 1e-7:
            print "mean wavelet power %e too low"%np.mean(ma)
            return np.nan
	i = np.argmax(mlab.detrend_linear(ma))
	return freqs[i]
Exemplo n.º 4
0
    def LombScargle(self):
        '''find significance of highest peak in COS event list

        This routine bins an event list in 0.032s indervals (the clock time)
        and runs a Lomb-Scargle periodogram.
        It return the false alarm probability (FAP) of the highest peak.
        '''
        # LombScargle expects even number of entries
        start = self.hist.shape[0] % 2
        lc = pyPeriod.TimeSeries(self.bins[start + 1:],
                                 mlab.detrend_linear(self.hist[start:]))
        self.ls = pyPeriod.LombScargle(lc, ofac=1, hifac=1)
        # ignore the lowest freq (= the length of the dataset)
        return self.ls.FAP(max(self.ls.power[3:])), max(self.ls.power[3:])
Exemplo n.º 5
0
def timeseries_boundary(x, opt_b, bdetrend):
    """ TIMESERIES_BOUNDARY applies periodic, zero-padded, or mirror
    boundary conditions to a time series. """

    M = len(x)

    if bool(M % 2): raise AssertionError("Even number of samples")

    if bdetrend: x = mlab.detrend_linear(x)

    # Allocate space for solution
    if opt_b == "zer":
        y = np.hstack([np.zeros(M / 2), x, np.zeros(M / 2)])
    if opt_b == "con":
        y = np.hstack([x[0] * np.ones(M / 2), x, x[-1] * np.ones(M / 2)])
    elif opt_b == "mir":
        y = np.hstack([x[::-1][M / 2:], x, x[::-1][:M / 2]])
    elif opt_b == "per":
        y = x

    elif opt_b == "exp_sin":
        # Attempt to fit an exponential periodic function to each end of
        # the signal, such that the total length is 2*M

        # Declare a temporary variable t, all my functions operate on an
        # x and y pair rather than dimensionless frequency
        t = np.linspace(0, 100, len(x))
        period = estimate_period(t, x)

        # Get last two periods of x and y
        ind_end = np.abs(t - (t[-1] - 2 * period)).argmin()
        ind_start = np.abs(t - (2 * period)).argmin()
        t_end = t[ind_end:]
        x_end = x[ind_end:]
        t_start = t[:ind_start]
        x_start = x[:ind_start]
        x_mid = x[ind_start:ind_end]

        t_end_ext, x_end_ext = extend(t_end, x_end, M / 2)
        t_start_ext, x_start_ext = extend(t_start, x_start[::-1], M / 2)

        y = np.hstack([x_start_ext[::-1], x_mid, x_end_ext])
        # tt = np.linspace(t[0] - t.mean(), t[-1] + t.mean(), num=len(yt))

    if opt_b is not 'per': index = np.arange(M / 2, M + M / 2)
    else: index = np.arange(M)

    return y, index
Exemplo n.º 6
0
    def update(self):
        pb = pbar.ProgressBar()
        winlen_samples = np.int(self.win_len_/self.spacing_)
        half_winlen = np.int(winlen_samples / 2)
        print('each window have size: ' + str(half_winlen * 2))
        ffts = []
        ns = []
        
        for n in pb(np.arange(0 + half_winlen, len(self.x_) - half_winlen)[::self.sub_fact_]):
            ns.append(n)
            
            #get a piece of the signal
            small_signal = self.y_[n-half_winlen:n+half_winlen]
                        
            #perform detrending
            if self.detrend_method_ == 'linear':
                small_signal = mlab.detrend_linear(small_signal)
                
            elif self.detrend_method_ == 'mean':
                small_signal = small_signal - np.mean(small_signal)
            
            elif self.detrend_method_ == 'none': #simply go ahead
                pass
            
                           
            #do padding
            if self.N_zeros_ > len(small_signal):
                small_signal = np.concatenate((small_signal, np.zeros(self.N_zeros_-len(small_signal))))
                

            #get a spectral estimation using some method                 
            if self.method_ == 'mtspec':
                ps, f = mtspec.mtspec(small_signal, self.spacing_, self.pi_)
                
            #normalize the single spectrum
            if self.max_normalization_:
                ps = ps / np.max(ps)
    		
            #append to the ffts
            ffts.append(ps[f<=self.highfreq_])
            
            
        self.ffts_ = np.array(ffts)
        self.eval_pos_ = self.x_[ns]
        
        #the real max frequency?
        self.real_maxfreq_ = np.max(f[f<=self.highfreq_])
Exemplo n.º 7
0
def timeseries_boundary_old(x, opt_b, bdetrend):
    """ TIMESERIES_BOUNDARY applies periodic, zero-padded, or mirror
    boundary conditions to a time series. """

    M = len(x)

    if bdetrend: x = mlab.detrend_linear(x)

    # Allocate space for solution
    if opt_b == "zer":
        y = np.hstack([np.zeros(M), x, np.zeros(M)])
    elif opt_b == "mir":
        y = np.hstack([x[::-1], x, x[::-1]])
    elif opt_b == "per":
        y = x

    if opt_b is not 'per': index = np.arange(M, 2 * M)
    else: index = np.arange(M)

    return y, index
Exemplo n.º 8
0
def plot_periodogram(hist, bins):
    def lspow(sig):
        return scipy.optimize.fmin(lambda x : np.abs(ls.FAP(x)-sig),[10.], disp = False)
    # LombScargle expects even number of entries
    start = hist.shape[0]%2
    lc = pyPeriod.TimeSeries(bins[start+1:], mlab.detrend_linear(hist[start:]))
    ls = pyPeriod.LombScargle(lc, ofac=1, hifac=1)
    #fig = plt.figure()
    #ax = fig.add_subplot(111)
    #ax.plot(ls.freq, ls.power, lw = 2.)
    #ax.plot(ax.get_xlim(), lspow(0.1) * np.array(1., 1., dtype = np.float), 'k:' )
    #ax.plot(ax.get_xlim(), lspow(0.01) * np.array(1., 1., dtype = np.float), 'k:')
    fig, ax = ls.plot(lw = 2., FAPlevels = [.1, 0.01])
    #ax.set_ylabel('Scargle Power')
    ax.set_xlabel('Frequency [Hz]')
    pd.plotfile(fig, 'LSperiodogram')
    ax.set_yscale('log')
    ax.set_xscale('log')
    pd.plotfile(fig, 'LSloglog')
    return fig, ax
Exemplo n.º 9
0
 def test_detrend_str_linear_1d(self):
     input = self.sig_slope + self.sig_off
     target = self.sig_zeros
     self.allclose(mlab.detrend(input, key="linear"), target)
     self.allclose(mlab.detrend(input, key=mlab.detrend_linear), target)
     self.allclose(mlab.detrend_linear(input.tolist()), target)
Exemplo n.º 10
0
    def calculate_season(self):
        """
        calculates the season
        """
        seasons_params = {}
        seasons_params['DJF'] = (3,2)
        seasons_params['JFM'] = (3,3)
        seasons_params['FMA'] = (3,4)
        seasons_params['MAM'] = (3,5)
        seasons_params['AMJ'] = (3,6)
        seasons_params['MJJ'] = (3,7)
        seasons_params['JJA'] = (3,8)
        seasons_params['JAS'] = (3,9)
        seasons_params['ASO'] = (3,10)
        seasons_params['SON'] = (3,11)
        seasons_params['OND'] = (3,12)
        seasons_params['NDJ'] = (3,1)
        seasons_params['Warm Season (Dec. - May)'] = (6, 5)
        seasons_params['Cold Season (Jun. - Nov.)'] = (6, 11)
        seasons_params['Year (Jan. - Dec.)'] = (12, 12)
        seasons_params['Hydro. year (Jul. - Jun.)'] = (12, 6)
        self.seasons_params = seasons_params

        if not(hasattr(self, 'dset_dict')):
            self._read_dset_params()

        # get the name of the file to open
        fname = self.dset_dict['path']

        # `dset` is now an attribute of the ensemble object
        self.dset = xray.open_dataset(fname)

        # get the variable and its index
        m_var = self.dset[self.variable].data
        index = self.dset['time'].to_index()

        # if the variable is rainfall, we calculate the running SUM
        if self.dset_dict['units'] in ['mm']:
            seas_field = bn.move_sum(m_var, self.seasons_params[self.season][0], \
                                          min_count=self.seasons_params[self.season][0], axis=0)
        # if not, then we calculate the running MEAN (average)
        else:
            seas_field = bn.move_mean(m_var, self.seasons_params[self.season][0], \
                                          min_count=self.seasons_params[self.season][0], axis=0)

        # get rid of the first nans in the time-series / fields after move_mean or move_sum
        seas_field = seas_field[(self.seasons_params[self.season][0]-1)::,:,:]
        index = index[(self.seasons_params[self.season][0]-1)::]

        # now selects the SEASON of interest
        iseas = np.where(index.month == self.seasons_params[self.season][1])[0]
        dates = index[iseas]
        seas_field = np.take(seas_field, iseas, axis=0)

        # if detrend is set to `True`, we detrend
        # detrend_linear from matplotlib.mlab is faster than detrend from scipy.signal
        if self.detrend:
            dseas_field = np.ones(seas_field.shape) * np.nan
            # if there is a mask, we have to test each variable
            if 'mask' in self.dset.data_vars:
                for ilat in range(dseas_field.shape[1]):
                    for ilon in range(dseas_field.shape[2]):
                        if np.logical_not(np.all(np.isnan(seas_field[:,ilat, ilon]))):
                            dseas_field[:,ilat, ilon] = detrend_linear(seas_field[:,ilat,ilon]) \
                            + seas_field[:,ilat,ilon].mean()

            # if not, we can proceed over the whole dataset
            else:
                for ilat in range(dseas_field.shape[1]):
                    for ilon in range(dseas_field.shape[2]):
                        dseas_field[:,ilat, ilon] = detrend_linear(seas_field[:,ilat,ilon]) \
                        + seas_field[:,ilat,ilon].mean()

            self.dset['dates'] = (('dates',), dates)
            self.dset['seas_var'] = (('dates', 'latitudes', 'longitudes'), dseas_field)

        # if detrend is False, then just add the seaosnal values
        else:
            self.dset['dates'] = (('dates',), dates)
            self.dset['seas_var'] = (('dates', 'latitudes', 'longitudes'), seas_field)
Exemplo n.º 11
0
a = np.where( (x.time[:,0] >= 1972) )[0][0]
z = np.where( (x.time[:,0] <= 2010) )[0][-1] + 1

x.time = x.time[a:z,...]
x.data = x.data[a:z,...]

from matplotlib.mlab import detrend_linear

### ==============================================================================================================
### detrend the data itself ?

datad = np.empty(x.data.shape)

for i in xrange(x.data.shape[1]):
    datad[:,i] = detrend_linear(x.data[:,i])

x.data = datad

clus_eof_file = loadmat(os.path.join(dpath, "KidsonTypes", "clus_eof.mat"), struct_as_record=False)
clus_eof = clus_eof_file['clus_eof'][0,0]

### normalize 
za = x.data -  np.tile(clus_eof.mean.T, (x.data.shape[0],1))

### multiply by the EOFs to get the Principal components 
pc = np.dot(za,clus_eof.vect)

pc_mean = clus_eof_file['pc_mean']

### normalize by the mean of the original PCs 
Exemplo n.º 12
0
    def calculate_season(self):
        """
        calculates the season
        """
        self.seasons_params = seasons_params()

        if not (hasattr(self, 'dset_dict')):
            self._read_dset_params()

        # get the name of the file to open
        fname = self.dset_dict['path']

        # `dset` is now an attribute of the ensemble object
        self.dset = xray.open_dataset(fname)

        # get the variable and its index
        m_var = self.dset[self.variable].data
        index = self.dset['time'].to_index()

        # if the variable is rainfall, we calculate the running SUM
        if self.dset_dict['units'] in ['mm']:
            seas_field = bn.move_sum(m_var, self.seasons_params[self.season][0], \
                                          min_count=self.seasons_params[self.season][0], axis=0)
        # if not, then we calculate the running MEAN (average)
        else:
            seas_field = bn.move_mean(m_var, self.seasons_params[self.season][0], \
                                          min_count=self.seasons_params[self.season][0], axis=0)

        # get rid of the first nans in the time-series / fields after move_mean or move_sum
        seas_field = seas_field[(self.seasons_params[self.season][0] -
                                 1)::, :, :]
        index = index[(self.seasons_params[self.season][0] - 1)::]

        # now selects the SEASON of interest
        iseas = np.where(index.month == self.seasons_params[self.season][1])[0]
        dates = index[iseas]
        seas_field = np.take(seas_field, iseas, axis=0)

        # if detrend is set to `True`, we detrend
        # detrend_linear from matplotlib.mlab is faster than detrend from scipy.signal
        if self.detrend:
            dseas_field = np.ones(seas_field.shape) * np.nan
            # if there is a mask, we have to test each variable
            if 'mask' in self.dset.data_vars:
                for ilat in range(dseas_field.shape[1]):
                    for ilon in range(dseas_field.shape[2]):
                        if np.logical_not(
                                np.all(np.isnan(seas_field[:, ilat, ilon]))):
                            dseas_field[:,ilat, ilon] = detrend_linear(seas_field[:,ilat,ilon]) \
                            + seas_field[:,ilat,ilon].mean()

            # if not, we can proceed over the whole dataset
            else:
                for ilat in range(dseas_field.shape[1]):
                    for ilon in range(dseas_field.shape[2]):
                        dseas_field[:,ilat, ilon] = detrend_linear(seas_field[:,ilat,ilon]) \
                        + seas_field[:,ilat,ilon].mean()

            self.dset['dates'] = (('dates', ), dates)
            self.dset['seas_var'] = (('dates', 'latitudes', 'longitudes'),
                                     dseas_field)

        # if detrend is False, then just add the seaosnal values
        else:
            self.dset['dates'] = (('dates', ), dates)
            self.dset['seas_var'] = (('dates', 'latitudes', 'longitudes'),
                                     seas_field)
Exemplo n.º 13
0
    # ================================================================
    # Read AMO index
    # ================================================================
    year_st = 1948
    year_ed = 2013
    year_ref = 1856
    t0 = (year_st - year_ref)
    t1 = (year_ed - year_ref) + 1
    dirname = '/stormtrack/data4/yliang/Indices/'
    filename = 'amon.us.long.data'
    f = open(dirname + filename, "r")
    data_read = genfromtxt(dirname + filename, dtype='float',
                           skip_header=0)[t0:t1, 1:]
    f.close()
    amo_index = np.nanmean(data_read, axis=1)
    amo_index = mlab.detrend_linear(amo_index)
    N = 11
    amo_rmean = amo_index.copy() * np.nan
    amo_rmean[int((N - 1) / 2):-int((N - 1) / 2)] = np.convolve(amo_index,
                                                                np.ones(
                                                                    (N, )) / N,
                                                                mode='valid')

    # ================================================================
    # Read PDO index
    # ================================================================
    year_st = 1948
    year_ed = 2013
    year_ref = 1900
    t0 = (year_st - year_ref)
    t1 = (year_ed - year_ref) + 1
Exemplo n.º 14
0
def detrend_func(data):
    data = mlab.detrend_mean(data)
    data = mlab.detrend_linear(data)
    return data