def test_stair_freeyaw(self): """ """ fname = '/home/dave/Repositories/public/0_davidovitch/' fname += 'freeyaw-ojf-wt-tests/data/calibrated/DataFrame/' fname += '0212_run_064_9.0ms_dc1_freeyawplaying_stiffblades' fname += '_coning_pwm1000_highrpm.h5' res = pd.read_hdf(fname, 'table') time = res.time.values sps = 1.0 / np.diff(time).mean() ff = Filters() cutoff_hz = 1.0 order = 2 Wn = cutoff_hz*2.0/sps B, A = sp.signal.butter(order, Wn, output='ba') yawf = sp.signal.filtfilt(B, A, res.yaw_angle.values) # YAW plt.figure('yaw') plt.plot(res.time, res.yaw_angle, 'r-') plt.plot(res.time, yawf, 'b-') B, A = sp.signal.butter(order, 1.0*2.0/sps, output='ba') yawf2 = sp.signal.filtfilt(B, A, res.yaw_angle.values) plt.plot(res.time, yawf2, 'k--') # RPM data = res.rpm.values data_f = ff.butter_lowpass(sps, data, order=2, cutoff_hz=1.0) plt.figure('rpm') plt.plot(res.time, data, 'r-') plt.plot(res.time, data_f, 'b-') # filtered_x, N, delay = ff.fir(time, res.rpm, cutoff_hz=1.0, # freq_trans_width=1.0, ripple_db=50.0) # plt.plot(res.time, filtered_x, 'k--') smooth_window = 2.0 ws = int(smooth_window*sps) data_s = ff.smooth(res.rpm, window_len=ws, window='hanning') NN = len(data_s) - len(time) data_s = data_s[NN:] # time_s = time[NN:] plt.plot(time+(smooth_window/2.0), data_s, 'k--') # and up again in order not to brake the plotting further down time_down = np.arange(time[0], time[-1], 0.1) data_f_down = sp.interpolate.griddata(time, data_f, time_down) plt.plot(time_down, data_f_down, 'm-', alpha=0.7) # # and upsampling again # data = sp.interpolate.griddata(time_down, data_down, time) slope, intercept, r_value, p_value, std_err \ = sp.stats.linregress(data_f_down, y=time_down)
def setup_filter(self, time, data, **kwargs): """ Load the callibration runs and convert voltage signal to yaw angles Parameters ---------- time : ndarray(k) data : ndarray(k) Returns ------- time_stair : ndarray(n) Average time stamp over the stair step data_stair : ndarray(n) Average value of the selected stair step """ # time and data should both be 1D and have the same shape! assert time.shape == data.shape runid = kwargs.get('runid', self.runid) # smoothen method: spline or moving average smoothen = kwargs.get('smoothen', 'spline') # what is the window of the moving average in seconds smooth_window = kwargs.get('smooth_window', 2) # specify the window of the staircase #start, end = 30100, -30001 start = kwargs.get('start', 0) end = kwargs.get('end', len(time)) dt = kwargs.get('dt', 1) cutoff_hz = kwargs.get('cutoff_hz', None) self.points_per_stair = kwargs.get('points_per_stair', 20) # at what is the minimum required value on dt or dt2 for a new stair self.stair_step_tresh = kwargs.get('stair_step_tresh', 1) # plot_data = kwargs.get('plot_data', False) # respath = kwargs.get('respath', None) # run = kwargs.get('run', None) # sample rate of the signal sample_rate = calc_sample_rate(time) # prepare the data time = time[start:end] # the actual raw signal data = data[start:end] # ------------------------------------------------- # Progress plotting # ---------------------------------------------- if self.plt_progress: plt.figure() Pxx, freqs = plt.psd(data, Fs=sample_rate, label='data') plt.show() plt.figure() plt.plot(time, data, label='raw data') # ------------------------------------------------- # setup plot # ------------------------------------------------- # labels = np.ndarray(3, dtype='<U100') # labels[0] = label # labels[1] = 'yawchan derivative' # labels[2] = 'psd' # remove any underscores for latex printing grandtitle = self.figfile.replace('_', '\_') plot = plotting.A4Tuned(scale=1.5) plot.setup(self.figpath+self.figfile+'_filter', nr_plots=3, grandtitle=grandtitle, wsleft_cm=1.5, wsright_cm=1.8, hspace_cm=1.2, size_x_perfig=10, size_y_perfig=5, wsbottom_cm=1.0, wstop_cm=1.5) # ------------------------------------------------- # plotting original and smoothend signal # ------------------------------------------------- ax1 = plot.fig.add_subplot(plot.nr_rows, plot.nr_cols, 1) ax1.plot(time, data, 'b', label='raw data', alpha=0.6) data_raw = data.copy() # ------------------------------------------------- # signal frequency filtering, if applicable # ------------------------------------------------- # filter the local derivatives if applicable if cutoff_hz: filt = Filters() data_filt, N, delay = filt.fir(time, data, ripple_db=20, freq_trans_width=0.5, cutoff_hz=cutoff_hz, figpath=self.figpath, figfile=self.figfile + 'filter_design', sample_rate=sample_rate, plot=False,) if self.plt_progress: # add the results of the filtering technique plt.plot(time[N-1:], data_filt[N-1:], 'r', label='freq filt') data = data_filt time = time[N-1:]#-delay else: N = 1 # ------------------------------------------------------- # smoothen the signal with some splines or moving average # ------------------------------------------------------- # NOTE: the smoothing will make the transitions also smoother. This # is not good. The edges of the stair need to be steep! # for the binary data this is actually a good thing, since the dt's # are almost always the same between time steps. We would otherwise # need a dt based on several time steps if smoothen == 'splines': print 'start applying spline ...', uni_spline = UnivariateSpline(time, data) data = uni_spline(time) print 'done!' NN = 0 # no time shift due to filtering? if self.plt_progress: plt.plot(time, data, label='spline data') elif smoothen == 'moving': print 'start calculating movering average ...', filt = Filters() # take av2s window, calculate the number of samples per window ws = int(smooth_window*sample_rate) data = filt.smooth(data, window_len=ws, window='hanning') NN = len(data) - len(time) data = data[NN:] print 'done!' if self.plt_progress: plt.plot(time, data, label='moving average') else: raise ValueError, 'smoothen method should be moving or splines' # ------------------------------------------------- # additional smoothening: downsampling # ------------------------------------------------- # and up again in order not to brake the plotting further down time_down = np.arange(time[0], time[-1], 0.1) data_down = sp.interpolate.griddata(time, data, time_down) # and upsampling again data = sp.interpolate.griddata(time_down, data_down, time) # ------------------------------------------------- # plotting original and smoothend signal # ------------------------------------------------- ax1.plot(time, data, 'r', label='data smooth') ax1.grid(True) leg1 = ax1.legend(loc='best') leg1.get_frame().set_alpha(0.5) ax1.set_title('smoothing method: ' + smoothen) # ------------------------------------------------- # local derivatives of the signal and filtering # ------------------------------------------------- data_dt = np.ndarray(data.shape) data_dt[1:] = data[1:] - data[0:-1] data_dt[0] = np.nan data_dt = np.abs(data_dt) # frequency filter was applied here originally data_filt_dt = data_dt # if no threshold is given, just take the 20% of the max value dt_max = np.nanmax(np.abs(data_filt_dt))*0.2 dt_treshold = kwargs.get('dt_treshold', dt_max) # ------------------------------------------------- # filter dt or dt2 above certain treshold? # ----------------------------------------------- # only keep values which are steady, meaning dt signal is low! if dt == 2: tmp = np.ndarray(data_filt_dt.shape) tmp[1:] = data_filt_dt[1:] - data_filt_dt[0:-1] tmp[0] = np.nan data_filt_dt = tmp # based upon the filtering, only select data points for which the # filtered derivative is between a certain treshold staircase_i = np.abs(data_filt_dt).__ge__(dt_treshold) # reduce to 1D staircase_arg=np.argwhere(np.abs(data_filt_dt)<=dt_treshold).flatten() # ------------------------------------------------- # replace values for too high dt with Nan # ------------------------------------------------ # --------------------------------- # METHOD version2, slower because of staircase_arg computation above data_masked = data.copy() data_masked[staircase_i] = np.nan data_masked_dt = data_filt_dt.copy() data_masked_dt[staircase_i] = np.nan data_trim = data[staircase_arg] time_trim = time[staircase_arg] print 'max in data_masked_dt:', np.nanmax(data_masked_dt) # --------------------------------- # METHOD version2, faster if staircase_arg is not required! ## make a copy of the original signal and fill in Nans on the selected ## values #data_masked = data.copy() #data_masked[staircase_i] = np.nan # #data_masked_dt = data_filt_dt.copy() #data_masked_dt[staircase_i] = np.nan # ## remove all the nan values #data_trim = data_masked[np.isnan(data_masked).__invert__()] #time_trim = time[np.isnan(data_masked).__invert__()] # #dt_noise_treshold = np.nanmax(data_masked_dt) #print 'max in data_masked_dt', dt_noise_treshold # --------------------------------- # # figure out which dt's are above the treshold # data_trim2 = data_trim.copy() # data_trim2.sort() # data_trim2. # # where the dt of the masked format is above the noise treshold, # # we have a stair # data_trim_dt = np.abs(data_trim[1:] - data_trim[:-1]) # argstairs = data_trim_dt.__gt__(dt_noise_treshold) # data_trim2 = data_trim_dt.copy() # data_trim_dt.sort() # data_trim_dt.__gt__(dt_noise_treshold) # ------------------------------------------------- # intermediate checking of the signal # ------------------------------------------------- if self.plt_progress: # add the results of the filtering technique plt.plot(time[N-1:], data_masked[N-1:], 'rs', label='data red') plt.legend(loc='best') plt.grid(True) plt.twinx() # plt.plot(time, data_filt_dt, label='data_filt_dt') plt.plot(time, data_masked_dt, 'm', label='data\_masked\_dt', alpha=0.4) plt.legend(loc='best') plt.show() print 'saving plt_progress:', print self.figpath+'filter_design_progress.png' plt.savefig(self.figpath+'filter_design_progress.png') # ------------------------------------------------- # check if we have had sane filtering # ------------------------------------------------- print 'data :', data.shape print 'data_trim :', data_trim.shape print 'trim ratio:', len(data)/len(data_trim) # there should be at least one True value assert staircase_i.any() # they can't all be True, than filtering is too heavy if len(data_trim) < len(data)*0.01: msg = 'dt_treshold is too low, not enough data left' raise ValueError, msg # if no data is filtered at all, filtering is too conservative elif len(data_trim) > len(data)*0.95: msg = 'dt_treshold is too high, too much data left' raise ValueError, msg # if the data array is too big, abort on memory concerns if len(data_trim) > 200000: msg = 'too much data points for stair case analysis (cfr memory)' raise ValueError, msg # ------------------------------------------------- # read the average value over each stair (time and data) # ------------------------------------------------ #try: ##np.save('time_trim', time_trim) ##np.save('data_trim', data_trim) ##np.save('staircase_arg', staircase_arg) ##tmp = np.array([self.points_per_stair, self.stair_step_tresh]) ##np.save('tmp', tmp) #data_ordered, time_stair, data_stair, arg_stair \ #= cython_func.order_staircase(time_trim, data_trim, #staircase_arg, self.points_per_stair, self.stair_step_tresh) #except ImportError: data_ordered, time_stair, data_stair, arg_stair \ = self.order_staircase(time_trim, data_trim, staircase_arg) # convert the arg_stair to a flat set and replace start/stop pairs # with all indices in between. Now we can select all stair values # in the raw dataset arg_st_fl = np.empty(data_raw.shape, dtype=np.int) i = 0 for k in range(arg_stair.shape[1]): #print '%6i %6i' % (arg_stair[0,k],arg_stair[1,k]) tmp = np.arange(arg_stair[0,k], arg_stair[1,k]+1, 1, dtype=np.int) #print tmp, '->', i, ':', i+len(tmp) arg_st_fl[i:i+len(tmp)] = tmp i += len(tmp) # remove the unused elements from the array arg_st_fl = arg_st_fl[:i] # ------------------------------------------------- # plotting of smoothen signal and stairs # ------------------------------------------------- ax1 = plot.fig.add_subplot(plot.nr_rows, plot.nr_cols, 2) ax1.plot(time, data, label='data smooth', alpha=0.6) # add the results of the filtering technique ax1.plot(time[N-1:], data_masked[N-1:], 'r', label='data masked') # ax1.plot(time[N-1:], data_filt[N-1:], 'g', label='data_filt') # also include the selected chair data figlabel = '%i stairs' % data_stair.shape[0] ax1.plot(time_stair, data_stair, 'ko', label=figlabel, alpha=0.4) ax1.grid(True) # the legend, on or off? #leg1 = ax1.legend(loc='upper left') #leg1.get_frame().set_alpha(0.5) # ------------------------------------------------- # plotting derivatives on right axis # ------------------------------------------------- ax1b = ax1.twinx() # ax1b.plot(time[N:]-delay,data_s_dt[N:],alpha=0.2,label='data_s_dt') ax1b.plot(time[N:], data_filt_dt[N:], 'r', alpha=0.35, label='data\_filt\_dt') majorFormatter = FormatStrFormatter('%8.1e') ax1b.yaxis.set_major_formatter(majorFormatter) # ax1b.plot(time[N:], data_masked_dt[N:], 'b', alpha=0.2, # label='data_masked_dt') # ax1b.plot(time[N-1:]-delay, filtered_x_dt[N-1:], alpha=0.2) # leg1b = ax1b.legend(loc='best') # leg1b.get_frame().set_alpha(0.5) # ax1b.grid(True) # ------------------------------------------------- # 3th plot to check if the raw chair signal is ok # ------------------------------------------------- ax1 = plot.fig.add_subplot(plot.nr_rows, plot.nr_cols, 3) ax1.plot(time[arg_st_fl], data_raw[arg_st_fl], 'k+', label='rawstair', alpha=0.1) ax1.plot(time[N-1:], data_masked[N-1:], 'r', label='data masked') ax1.set_xlabel('time [s]') # ------------------------------------------------- # the power spectral density # ------------------------------------------------- # ax3 = plot.fig.add_subplot(plot.nr_rows, plot.nr_cols, 3) # Pxx, freqs = ax3.psd(data, Fs=sample_rate, label='data smooth') ## Pxx, freqs = ax3.psd(data_dt, Fs=sample_rate, label='data_dt') ## Pxx, freqs = ax3.psd(data_filt_dt[N-1:], Fs=sample_rate, ## label='data_filt_dt') # ax3.legend() ## print Pxx.shape, freqs.shape plot.save_fig() # ------------------------------------------------- # get amplitudes of the stair edges # ------------------------------------------------- # # max step # data_trim_dt_sort = data_trim_dt.sort()[0] # # estimate at what kind of a delta we are looking for when changing # # stairs # data_dt_std = data_trim_dt.std() # data_dt_mean = (np.abs(data_trim_dt)).mean() # # time_data_dt = np.transpose(np.array([time, data_filt_dt])) # data_filt_dt_amps = HawcPy.dynprop().amplitudes(time_data_dt, h=1e-3) # # print '=== nr amplitudes' # print len(data_filt_dt_amps) # print data_filt_dt_amps # ------------------------------------------------- # save the data # ------------------------------------------------- filename = runid + '-time_stair' np.savetxt(self.pprpath + filename, time_stair) filename = runid + '-data_stair' np.savetxt(self.pprpath + filename, data_stair) # in order to maintain backwards compatibility, save the arguments # of the stair to self self.arg_st_fl = arg_st_fl # flat, contains all indices on the stairs # start/stop indeces for stair k = arg_stair[0,k], arg_stair[1,k] self.arg_stair = arg_stair return time_stair, data_stair