def butter(self, time, data, **kwargs): """ Source: https://azitech.wordpress.com/2011/03/15/ designing-a-butterworth-low-pass-filter-with-scipy/ """ sample_rate = kwargs.get("sample_rate", None) if not sample_rate: sample_rate = calc_sample_rate(time) # The cutoff frequency of the filter. cutoff_hz = kwargs.get("cutoff_hz", 1.0) # design filter norm_pass = cutoff_hz / (sample_rate / 2.0) norm_stop = 1.5 * norm_pass (N, Wn) = sp.signal.buttord(wp=norm_pass, ws=norm_stop, gpass=2, gstop=30, analog=0) (b, a) = sp.signal.butter(N, Wn, btype="low", analog=0, output="ba") # filtered output # zi = signal.lfiltic(b, a, x[0:5], x[0:5]) # (y, zi) = signal.lfilter(b, a, x, zi=zi) data_filt = sp.signal.lfilter(b, a, data) return data_filt
def __init__(self, time, freq_ds=10): """ Parameters ---------- time : ndarray(n) freq_ds : float, default=0.1 Frequency in Hz of the downsampled signal. Downsampling is applied after the low pass filter. outputs : ndarray(m,n) Signals corresponding to the output of the system inputs : ndarray(m,n), default=None Signals that corresponds to the inputs of the system. When None, inputs are ignored and now additional overlaying is done in order to achieve both steady curves on the in- and outputs. """ self.sps = calc_sample_rate(time) self.t_ds = np.arange(time[0], time[-1], 1.0/freq_ds) self.time = time self.freq_ds = freq_ds
def setup_filter(self, time, data, **kwargs): """ Load the callibration runs and convert voltage signal to yaw angles Parameters ---------- time : ndarray(k) data : ndarray(k) Returns ------- time_stair : ndarray(n) Average time stamp over the stair step data_stair : ndarray(n) Average value of the selected stair step """ # time and data should both be 1D and have the same shape! assert time.shape == data.shape runid = kwargs.get('runid', self.runid) # smoothen method: spline or moving average smoothen = kwargs.get('smoothen', 'spline') # what is the window of the moving average in seconds smooth_window = kwargs.get('smooth_window', 2) # specify the window of the staircase #start, end = 30100, -30001 start = kwargs.get('start', 0) end = kwargs.get('end', len(time)) dt = kwargs.get('dt', 1) cutoff_hz = kwargs.get('cutoff_hz', None) self.points_per_stair = kwargs.get('points_per_stair', 20) # at what is the minimum required value on dt or dt2 for a new stair self.stair_step_tresh = kwargs.get('stair_step_tresh', 1) # plot_data = kwargs.get('plot_data', False) # respath = kwargs.get('respath', None) # run = kwargs.get('run', None) # sample rate of the signal sample_rate = calc_sample_rate(time) # prepare the data time = time[start:end] # the actual raw signal data = data[start:end] # ------------------------------------------------- # Progress plotting # ---------------------------------------------- if self.plt_progress: plt.figure() Pxx, freqs = plt.psd(data, Fs=sample_rate, label='data') plt.show() plt.figure() plt.plot(time, data, label='raw data') # ------------------------------------------------- # setup plot # ------------------------------------------------- # labels = np.ndarray(3, dtype='<U100') # labels[0] = label # labels[1] = 'yawchan derivative' # labels[2] = 'psd' # remove any underscores for latex printing grandtitle = self.figfile.replace('_', '\_') plot = plotting.A4Tuned(scale=1.5) plot.setup(self.figpath+self.figfile+'_filter', nr_plots=3, grandtitle=grandtitle, wsleft_cm=1.5, wsright_cm=1.8, hspace_cm=1.2, size_x_perfig=10, size_y_perfig=5, wsbottom_cm=1.0, wstop_cm=1.5) # ------------------------------------------------- # plotting original and smoothend signal # ------------------------------------------------- ax1 = plot.fig.add_subplot(plot.nr_rows, plot.nr_cols, 1) ax1.plot(time, data, 'b', label='raw data', alpha=0.6) data_raw = data.copy() # ------------------------------------------------- # signal frequency filtering, if applicable # ------------------------------------------------- # filter the local derivatives if applicable if cutoff_hz: filt = Filters() data_filt, N, delay = filt.fir(time, data, ripple_db=20, freq_trans_width=0.5, cutoff_hz=cutoff_hz, figpath=self.figpath, figfile=self.figfile + 'filter_design', sample_rate=sample_rate, plot=False,) if self.plt_progress: # add the results of the filtering technique plt.plot(time[N-1:], data_filt[N-1:], 'r', label='freq filt') data = data_filt time = time[N-1:]#-delay else: N = 1 # ------------------------------------------------------- # smoothen the signal with some splines or moving average # ------------------------------------------------------- # NOTE: the smoothing will make the transitions also smoother. This # is not good. The edges of the stair need to be steep! # for the binary data this is actually a good thing, since the dt's # are almost always the same between time steps. We would otherwise # need a dt based on several time steps if smoothen == 'splines': print 'start applying spline ...', uni_spline = UnivariateSpline(time, data) data = uni_spline(time) print 'done!' NN = 0 # no time shift due to filtering? if self.plt_progress: plt.plot(time, data, label='spline data') elif smoothen == 'moving': print 'start calculating movering average ...', filt = Filters() # take av2s window, calculate the number of samples per window ws = int(smooth_window*sample_rate) data = filt.smooth(data, window_len=ws, window='hanning') NN = len(data) - len(time) data = data[NN:] print 'done!' if self.plt_progress: plt.plot(time, data, label='moving average') else: raise ValueError, 'smoothen method should be moving or splines' # ------------------------------------------------- # additional smoothening: downsampling # ------------------------------------------------- # and up again in order not to brake the plotting further down time_down = np.arange(time[0], time[-1], 0.1) data_down = sp.interpolate.griddata(time, data, time_down) # and upsampling again data = sp.interpolate.griddata(time_down, data_down, time) # ------------------------------------------------- # plotting original and smoothend signal # ------------------------------------------------- ax1.plot(time, data, 'r', label='data smooth') ax1.grid(True) leg1 = ax1.legend(loc='best') leg1.get_frame().set_alpha(0.5) ax1.set_title('smoothing method: ' + smoothen) # ------------------------------------------------- # local derivatives of the signal and filtering # ------------------------------------------------- data_dt = np.ndarray(data.shape) data_dt[1:] = data[1:] - data[0:-1] data_dt[0] = np.nan data_dt = np.abs(data_dt) # frequency filter was applied here originally data_filt_dt = data_dt # if no threshold is given, just take the 20% of the max value dt_max = np.nanmax(np.abs(data_filt_dt))*0.2 dt_treshold = kwargs.get('dt_treshold', dt_max) # ------------------------------------------------- # filter dt or dt2 above certain treshold? # ----------------------------------------------- # only keep values which are steady, meaning dt signal is low! if dt == 2: tmp = np.ndarray(data_filt_dt.shape) tmp[1:] = data_filt_dt[1:] - data_filt_dt[0:-1] tmp[0] = np.nan data_filt_dt = tmp # based upon the filtering, only select data points for which the # filtered derivative is between a certain treshold staircase_i = np.abs(data_filt_dt).__ge__(dt_treshold) # reduce to 1D staircase_arg=np.argwhere(np.abs(data_filt_dt)<=dt_treshold).flatten() # ------------------------------------------------- # replace values for too high dt with Nan # ------------------------------------------------ # --------------------------------- # METHOD version2, slower because of staircase_arg computation above data_masked = data.copy() data_masked[staircase_i] = np.nan data_masked_dt = data_filt_dt.copy() data_masked_dt[staircase_i] = np.nan data_trim = data[staircase_arg] time_trim = time[staircase_arg] print 'max in data_masked_dt:', np.nanmax(data_masked_dt) # --------------------------------- # METHOD version2, faster if staircase_arg is not required! ## make a copy of the original signal and fill in Nans on the selected ## values #data_masked = data.copy() #data_masked[staircase_i] = np.nan # #data_masked_dt = data_filt_dt.copy() #data_masked_dt[staircase_i] = np.nan # ## remove all the nan values #data_trim = data_masked[np.isnan(data_masked).__invert__()] #time_trim = time[np.isnan(data_masked).__invert__()] # #dt_noise_treshold = np.nanmax(data_masked_dt) #print 'max in data_masked_dt', dt_noise_treshold # --------------------------------- # # figure out which dt's are above the treshold # data_trim2 = data_trim.copy() # data_trim2.sort() # data_trim2. # # where the dt of the masked format is above the noise treshold, # # we have a stair # data_trim_dt = np.abs(data_trim[1:] - data_trim[:-1]) # argstairs = data_trim_dt.__gt__(dt_noise_treshold) # data_trim2 = data_trim_dt.copy() # data_trim_dt.sort() # data_trim_dt.__gt__(dt_noise_treshold) # ------------------------------------------------- # intermediate checking of the signal # ------------------------------------------------- if self.plt_progress: # add the results of the filtering technique plt.plot(time[N-1:], data_masked[N-1:], 'rs', label='data red') plt.legend(loc='best') plt.grid(True) plt.twinx() # plt.plot(time, data_filt_dt, label='data_filt_dt') plt.plot(time, data_masked_dt, 'm', label='data\_masked\_dt', alpha=0.4) plt.legend(loc='best') plt.show() print 'saving plt_progress:', print self.figpath+'filter_design_progress.png' plt.savefig(self.figpath+'filter_design_progress.png') # ------------------------------------------------- # check if we have had sane filtering # ------------------------------------------------- print 'data :', data.shape print 'data_trim :', data_trim.shape print 'trim ratio:', len(data)/len(data_trim) # there should be at least one True value assert staircase_i.any() # they can't all be True, than filtering is too heavy if len(data_trim) < len(data)*0.01: msg = 'dt_treshold is too low, not enough data left' raise ValueError, msg # if no data is filtered at all, filtering is too conservative elif len(data_trim) > len(data)*0.95: msg = 'dt_treshold is too high, too much data left' raise ValueError, msg # if the data array is too big, abort on memory concerns if len(data_trim) > 200000: msg = 'too much data points for stair case analysis (cfr memory)' raise ValueError, msg # ------------------------------------------------- # read the average value over each stair (time and data) # ------------------------------------------------ #try: ##np.save('time_trim', time_trim) ##np.save('data_trim', data_trim) ##np.save('staircase_arg', staircase_arg) ##tmp = np.array([self.points_per_stair, self.stair_step_tresh]) ##np.save('tmp', tmp) #data_ordered, time_stair, data_stair, arg_stair \ #= cython_func.order_staircase(time_trim, data_trim, #staircase_arg, self.points_per_stair, self.stair_step_tresh) #except ImportError: data_ordered, time_stair, data_stair, arg_stair \ = self.order_staircase(time_trim, data_trim, staircase_arg) # convert the arg_stair to a flat set and replace start/stop pairs # with all indices in between. Now we can select all stair values # in the raw dataset arg_st_fl = np.empty(data_raw.shape, dtype=np.int) i = 0 for k in range(arg_stair.shape[1]): #print '%6i %6i' % (arg_stair[0,k],arg_stair[1,k]) tmp = np.arange(arg_stair[0,k], arg_stair[1,k]+1, 1, dtype=np.int) #print tmp, '->', i, ':', i+len(tmp) arg_st_fl[i:i+len(tmp)] = tmp i += len(tmp) # remove the unused elements from the array arg_st_fl = arg_st_fl[:i] # ------------------------------------------------- # plotting of smoothen signal and stairs # ------------------------------------------------- ax1 = plot.fig.add_subplot(plot.nr_rows, plot.nr_cols, 2) ax1.plot(time, data, label='data smooth', alpha=0.6) # add the results of the filtering technique ax1.plot(time[N-1:], data_masked[N-1:], 'r', label='data masked') # ax1.plot(time[N-1:], data_filt[N-1:], 'g', label='data_filt') # also include the selected chair data figlabel = '%i stairs' % data_stair.shape[0] ax1.plot(time_stair, data_stair, 'ko', label=figlabel, alpha=0.4) ax1.grid(True) # the legend, on or off? #leg1 = ax1.legend(loc='upper left') #leg1.get_frame().set_alpha(0.5) # ------------------------------------------------- # plotting derivatives on right axis # ------------------------------------------------- ax1b = ax1.twinx() # ax1b.plot(time[N:]-delay,data_s_dt[N:],alpha=0.2,label='data_s_dt') ax1b.plot(time[N:], data_filt_dt[N:], 'r', alpha=0.35, label='data\_filt\_dt') majorFormatter = FormatStrFormatter('%8.1e') ax1b.yaxis.set_major_formatter(majorFormatter) # ax1b.plot(time[N:], data_masked_dt[N:], 'b', alpha=0.2, # label='data_masked_dt') # ax1b.plot(time[N-1:]-delay, filtered_x_dt[N-1:], alpha=0.2) # leg1b = ax1b.legend(loc='best') # leg1b.get_frame().set_alpha(0.5) # ax1b.grid(True) # ------------------------------------------------- # 3th plot to check if the raw chair signal is ok # ------------------------------------------------- ax1 = plot.fig.add_subplot(plot.nr_rows, plot.nr_cols, 3) ax1.plot(time[arg_st_fl], data_raw[arg_st_fl], 'k+', label='rawstair', alpha=0.1) ax1.plot(time[N-1:], data_masked[N-1:], 'r', label='data masked') ax1.set_xlabel('time [s]') # ------------------------------------------------- # the power spectral density # ------------------------------------------------- # ax3 = plot.fig.add_subplot(plot.nr_rows, plot.nr_cols, 3) # Pxx, freqs = ax3.psd(data, Fs=sample_rate, label='data smooth') ## Pxx, freqs = ax3.psd(data_dt, Fs=sample_rate, label='data_dt') ## Pxx, freqs = ax3.psd(data_filt_dt[N-1:], Fs=sample_rate, ## label='data_filt_dt') # ax3.legend() ## print Pxx.shape, freqs.shape plot.save_fig() # ------------------------------------------------- # get amplitudes of the stair edges # ------------------------------------------------- # # max step # data_trim_dt_sort = data_trim_dt.sort()[0] # # estimate at what kind of a delta we are looking for when changing # # stairs # data_dt_std = data_trim_dt.std() # data_dt_mean = (np.abs(data_trim_dt)).mean() # # time_data_dt = np.transpose(np.array([time, data_filt_dt])) # data_filt_dt_amps = HawcPy.dynprop().amplitudes(time_data_dt, h=1e-3) # # print '=== nr amplitudes' # print len(data_filt_dt_amps) # print data_filt_dt_amps # ------------------------------------------------- # save the data # ------------------------------------------------- filename = runid + '-time_stair' np.savetxt(self.pprpath + filename, time_stair) filename = runid + '-data_stair' np.savetxt(self.pprpath + filename, data_stair) # in order to maintain backwards compatibility, save the arguments # of the stair to self self.arg_st_fl = arg_st_fl # flat, contains all indices on the stairs # start/stop indeces for stair k = arg_stair[0,k], arg_stair[1,k] self.arg_stair = arg_stair return time_stair, data_stair
def scipy_example(self, time, data, sample_rate=None): """ Example from the SciPy Cookboock, see http://www.scipy.org/Cookbook/FIRFilter """ chk.array_1d(time) chk.array_1d(data) if not sample_rate: sample_rate = calc_sample_rate(time) # ------------------------------------------------ # Create a FIR filter and apply it to data[:,channel] # ------------------------------------------------ # The Nyquist rate of the signal. nyq_rate = sample_rate / 2.0 # The desired width of the transition from pass to stop, # relative to the Nyquist rate. We'll design the filter # with a 5 Hz transition width. width = 5.0 / nyq_rate # The desired attenuation in the stop band, in dB. ripple_db = 60.0 # Compute the order and Kaiser parameter for the FIR filter. N, beta = sp.signal.kaiserord(ripple_db, width) # The cutoff frequency of the filter. cutoff_hz = 10.0 # Use firwin with a Kaiser window to create a lowpass FIR filter. taps = sp.signal.firwin(N, cutoff_hz / nyq_rate, window=("kaiser", beta)) # Use lfilter to filter x with the FIR filter. filtered_x = sp.signal.lfilter(taps, 1.0, data) # ------------------------------------------------ # Setup the figure parameters # ------------------------------------------------ figpath = "processing/" figfile = "filterdesign" plot = plotting.A4Tuned() plot.setup(figpath + figfile, nr_plots=3, grandtitle=figfile, figsize_y=20, wsleft_cm=2.0) # ------------------------------------------------ # Plot the FIR filter coefficients. # ------------------------------------------------ plot_nr = 1 ax1 = plot.fig.add_subplot(plot.nr_rows, plot.nr_cols, plot_nr) ax1.plot(taps, "bo-", linewidth=2) ax1.set_title("Filter Coefficients (%d taps)" % N) ax1.grid(True) # ------------------------------------------------ # Plot the magnitude response of the filter. # ------------------------------------------------ plot_nr += 1 ax2 = plot.fig.add_subplot(plot.nr_rows, plot.nr_cols, plot_nr) w, h = sp.signal.freqz(taps, worN=8000) ax2.plot((w / np.pi) * nyq_rate, np.absolute(h), linewidth=2) ax2.set_xlabel("Frequency (Hz)") ax2.set_ylabel("Gain") ax2.set_title("Frequency Response") ax2.set_ylim(-0.05, 1.05) # ax2.grid(True) # in order to place the nex axes inside following figure, first # determine the ax2 bounding box # points: a 2x2 numpy array of the form [[x0, y0], [x1, y1]] ax2box = ax2.get_window_extent().get_points() # seems to be expressed in pixels so convert to relative coordinates # print ax2box # figure size in pixels figsize_x_pix = plot.figsize_x * plot.dpi figsize_y_pix = plot.figsize_y * plot.dpi # ax2 box in relative coordinates ax2box[:, 0] = ax2box[:, 0] / figsize_x_pix ax2box[:, 1] = ax2box[:, 1] / figsize_y_pix # print ax2box[0,0], ax2box[1,0], ax2box[0,1], ax2box[1,1] # left position new box at 10% of x1 left = ax2box[0, 0] + ((ax2box[1, 0] - ax2box[0, 0]) * 0.15) bottom = ax2box[0, 1] + ((ax2box[1, 1] - ax2box[0, 1]) * 0.30) # x2 width = (ax2box[1, 0] - ax2box[0, 0]) * 0.35 height = (ax2box[1, 1] - ax2box[0, 1]) * 0.6 # print [left, bottom, width, height] # left inset plot. # [left, bottom, width, height] # ax2a = plot.fig.add_axes([0.42, 0.6, .45, .25]) ax2a = plot.fig.add_axes([left, bottom, width, height]) ax2a.plot((w / np.pi) * nyq_rate, np.absolute(h), linewidth=2) ax2a.set_xlim(0, 8.0) ax2a.set_ylim(0.9985, 1.001) ax2a.grid(True) # right inset plot left = ax2box[0, 0] + ((ax2box[1, 0] - ax2box[0, 0]) * 0.62) bottom = ax2box[0, 1] + ((ax2box[1, 1] - ax2box[0, 1]) * 0.30) # x2 width = (ax2box[1, 0] - ax2box[0, 0]) * 0.35 height = (ax2box[1, 1] - ax2box[0, 1]) * 0.6 # Lower inset plot # ax2b = plot.fig.add_axes([0.42, 0.25, .45, .25]) ax2b = plot.fig.add_axes([left, bottom, width, height]) ax2b.plot((w / np.pi) * nyq_rate, np.absolute(h), linewidth=2) ax2b.set_xlim(12.0, 20.0) ax2b.set_ylim(0.0, 0.0025) ax2b.grid(True) # ------------------------------------------------ # Plot the original and filtered signals. # ------------------------------------------------ # The phase delay of the filtered signal. delay = 0.5 * (N - 1) / sample_rate plot_nr += 1 ax3 = plot.fig.add_subplot(plot.nr_rows, plot.nr_cols, plot_nr) # Plot the original signal. ax3.plot(time, data, label="original signal") # Plot the filtered signal, shifted to compensate for the phase delay. ax3.plot(time - delay, filtered_x, "r-", label="filtered signal") # Plot just the "good" part of the filtered signal. The first N-1 # samples are "corrupted" by the initial conditions. ax3.plot(time[N - 1 :] - delay, filtered_x[N - 1 :], "g", linewidth=4) ax3.set_xlabel("t") ax3.grid(True) plot.save_fig()
def fir(self, time, data, **kwargs): """ Based on the xxample from the SciPy cook boock, see http://www.scipy.org/Cookbook/FIRFilter Parameters ---------- time : ndarray(n) data : ndarray(n) plot : boolean, default=False figpath : str, default=False figfile : str, default=False sample_rate : int, default=None If None, sample rate will be calculated from the given signal freq_trans_width : float, default=1 The desired width of the transition from pass to stop, relative to the Nyquist rate. ripple_db : float, default=10 The desired attenuation in the stop band, in dB. cutoff_hz : float, default=10 Frequencies above cutoff_hz are filtered out Returns ------- filtered_x : ndarray(n - (N-1)) filtered signal N : float order of the firwin filter delay : float phase delay due to the filtering process """ plot = kwargs.get("plot", False) figpath = kwargs.get("figpath", False) figfile = kwargs.get("figfile", False) sample_rate = kwargs.get("sample_rate", None) # The desired width of the transition from pass to stop, # relative to the Nyquist rate. We'll design the filter # with a 5 Hz transition width. freq_trans_width = kwargs.get("freq_trans_width", 1) # The desired attenuation in the stop band, in dB. ripple_db = kwargs.get("ripple_db", 10) # The cutoff frequency of the filter. cutoff_hz = kwargs.get("cutoff_hz", 10) chk.array_1d(time) chk.array_1d(data) if not sample_rate: sample_rate = calc_sample_rate(time) # ------------------------------------------------ # Create a FIR filter and apply it to data[:,channel] # ------------------------------------------------ # The Nyquist rate of the signal. nyq_rate = sample_rate / 2.0 # The desired width of the transition from pass to stop, # relative to the Nyquist rate. We'll design the filter # with a 5 Hz transition width. width = freq_trans_width / nyq_rate # Compute the order and Kaiser parameter for the FIR filter. N, beta = sp.signal.kaiserord(ripple_db, width) # Use firwin with a Kaiser window to create a lowpass FIR filter. taps = sp.signal.firwin(N, cutoff_hz / nyq_rate, window=("kaiser", beta)) # Use lfilter to filter x with the FIR filter. filtered_x = sp.signal.lfilter(taps, 1.0, data) # The phase delay of the filtered signal. delay = 0.5 * (N - 1) / sample_rate # # the filtered signal, shifted to compensate for the phase delay. # time_shifted = time-delay # # the "good" part of the filtered signal. The first N-1 # # samples are "corrupted" by the initial conditions. # time_good = time[N-1:] - delay if plot: self.plot_fir(figpath, figfile, time, data, filtered_x, N, delay, sample_rate, taps, nyq_rate) return filtered_x, N, delay